Initial support for optional navigation translation into LEFT OUTER J…

…OIN.

Currently we translate all navigations into INNER JOINs, which may produce invalid results for some queries.

Fix is to translate optional navigations into LEFT OUTER JOIN. In order to do that we convert optional navigation into SelectMany-GroupJoin-DefaultIfEmpty expression, that translates to LOJ.
We also need to add null checks in various places because now the intermittent results can be null.

e.g.
from c in ctx.Customers
where c.Detail.Name == "Foo"
select c

will get translated into:

from c in ctx.Customers
join d in ctx.Detail on c.Id equals d.CustomerId into grouping
from d in grouping.DefaultIfEmpty()
where (d != null ? d.Name : null) == "Foo"
select c;

Also fixed a number of bugs uncovered by this change, due to generating significantly more complex queries in some cases.

Known issues:
- DefaultIfEmpty() cannot be translated into SQL so everything happening after it is computed on the client (filters, projections, nested navigations) (#4539),
- Optional navigations don't work with Include (#4589),
- Optional navigations don't work with some queries involving SelectMany operator (#4539),
- Optional navigations don't work for soem complex queries involving subqueries and/or navigation inside inner key selector of a Join statement (#4547)

CR: Andrew

src/Microsoft.EntityFrameworkCore.Relational/Query/ExpressionVisitors/Internal/BufferedEntityShaper`.cs

@@ -55,5 +55,10 @@ public override EntityShaper WithOffset(int offset)
                 Key,
                 Materializer)
                 .SetOffset(offset);
+
+        public override string ToString()
+        {
+            return "BufferedEntityShaper<" + typeof(TEntity).Name + ">";
+        }
     }
 }

src/Microsoft.EntityFrameworkCore.Relational/Query/ExpressionVisitors/Internal/BufferedOffsetEntityShaper.cs

@@ -24,5 +24,10 @@ public class BufferedOffsetEntityShaper<TEntity> : BufferedEntityShaper<TEntity>
 
         public override TEntity Shape(QueryContext queryContext, ValueBuffer valueBuffer)
             => base.Shape(queryContext, valueBuffer.WithOffset(ValueBufferOffset));
+
+        public override string ToString()
+        {
+            return "BufferedOffsetEntityShaper<" + typeof(TEntity).Name + ">";
+        }
     }
 }

src/Microsoft.EntityFrameworkCore.Relational/Query/ExpressionVisitors/Internal/IsNullExpressionBuildingVisitor.cs

@@ -9,13 +9,16 @@ namespace Microsoft.EntityFrameworkCore.Query.ExpressionVisitors.Internal
 {
     public class IsNullExpressionBuildingVisitor : RelinqExpressionVisitor
     {
+        private bool _nullConstantAdded = false;
+
         public virtual Expression ResultExpression { get; private set; }
 
         protected override Expression VisitConstant(ConstantExpression node)
         {
-            if (node.Value == null)
+            if (node.Value == null && !_nullConstantAdded)
             {
-                AddToResult(node);
+                AddToResult(new IsNullExpression(node));
+                _nullConstantAdded = true;
             }
 
             return node;

src/Microsoft.EntityFrameworkCore.Relational/Query/ExpressionVisitors/Internal/QueryFlattener.cs

@@ -42,6 +42,7 @@ public virtual Expression Flatten([NotNull] MethodCallExpression methodCallExpre
             if (methodCallExpression.Method.MethodIsClosedFormOf(_operatorToFlatten))
             {
                 var outerShapedQuery = (MethodCallExpression)methodCallExpression.Arguments[0];
+
                 var outerShaper = (Shaper)((ConstantExpression)outerShapedQuery.Arguments[2]).Value;
 
                 var innerLambda

src/Microsoft.EntityFrameworkCore.Relational/Query/ExpressionVisitors/Internal/UnbufferedEntityShaper.cs

@@ -55,5 +55,10 @@ public override EntityShaper WithOffset(int offset)
                 Key,
                 Materializer)
                 .SetOffset(offset);
+
+        public override string ToString()
+        {
+            return "UnbufferedEntityShaper<" + typeof(TEntity).Name + ">";
+        }
     }
 }

src/Microsoft.EntityFrameworkCore.Relational/Query/ExpressionVisitors/Internal/UnbufferedOffsetEntityShaper.cs

@@ -24,5 +24,10 @@ public class UnbufferedOffsetEntityShaper<TEntity> : UnbufferedEntityShaper<TEnt
 
         public override TEntity Shape(QueryContext queryContext, ValueBuffer valueBuffer) 
             => base.Shape(queryContext, valueBuffer.WithOffset(ValueBufferOffset));
+
+        public override string ToString()
+        {
+            return "UnbufferedOffsetEntityShaper<" + typeof(TEntity).Name + ">";
+        }
     }
 }

src/Microsoft.EntityFrameworkCore.Relational/Query/ExpressionVisitors/SqlTranslatingExpressionVisitor.cs

@@ -172,6 +172,12 @@ protected override Expression VisitConditional(ConditionalExpression expression)
         {
             Check.NotNull(expression, nameof(expression));
 
+            var nullCheckRemoved = TryRemoveNullCheck(expression);
+            if (nullCheckRemoved != null)
+            {
+                return Visit(nullCheckRemoved);
+            }
+
             var test = Visit(expression.Test);
             var ifTrue = Visit(expression.IfTrue);
             var ifFalse = Visit(expression.IfFalse);
@@ -186,6 +192,38 @@ protected override Expression VisitConditional(ConditionalExpression expression)
             return null;
         }
 
+        private Expression TryRemoveNullCheck(ConditionalExpression node)
+        {
+            var binaryTest = node.Test as BinaryExpression;
+            if (binaryTest == null || binaryTest.NodeType != ExpressionType.NotEqual)
+            {
+                return null;
+            }
+
+            var rightConstant = binaryTest.Right as ConstantExpression;
+            if (rightConstant == null || rightConstant.Value != null)
+            {
+                return null;
+            }
+
+            var ifFalseConstant = node.IfFalse as ConstantExpression;
+            if (ifFalseConstant == null || ifFalseConstant.Value != null)
+            {
+                return null;
+            }
+
+            var ifTrueMemberExpression = node.IfTrue.RemoveConvert() as MemberExpression;
+            var correctMemberExpression = ifTrueMemberExpression != null
+                 && ifTrueMemberExpression.Expression == binaryTest.Left;
+
+            var ifTruePropertyMethodCallExpression = node.IfTrue.RemoveConvert() as MethodCallExpression;
+            var correctPropertyMethodCallExpression = ifTruePropertyMethodCallExpression != null
+                 && EntityQueryModelVisitor.IsPropertyMethod(ifTruePropertyMethodCallExpression.Method)
+                 && ifTruePropertyMethodCallExpression.Arguments[0] == binaryTest.Left;
+
+            return correctMemberExpression || correctPropertyMethodCallExpression ? node.IfTrue : null;
+        }
+
         private static Expression UnfoldStructuralComparison(ExpressionType expressionType, Expression expression)
         {
             var binaryExpression = expression as BinaryExpression;
@@ -242,7 +280,7 @@ private Expression ProcessComparisonExpression(BinaryExpression binaryExpression
                 return null;
             }
 
-            var nullExpression
+            var nullExpression 
                 = TransformNullComparison(leftExpression, rightExpression, binaryExpression.NodeType);
 
             return nullExpression
@@ -256,8 +294,8 @@ var nullExpression
                 || expressionType == ExpressionType.NotEqual)
             {
                 var constantExpression
-                    = right as ConstantExpression
-                      ?? left as ConstantExpression;
+                    = right.RemoveConvert() as ConstantExpression
+                      ?? left.RemoveConvert() as ConstantExpression;
 
                 if (constantExpression != null
                     && constantExpression.Value == null)

src/Microsoft.EntityFrameworkCore.SqlServer/Query/Sql/Internal/SqlServerQuerySqlGenerator.cs

@@ -116,6 +116,24 @@ protected override Expression VisitUnary(UnaryExpression node)
                         Expression.Constant(false, typeof(bool)));
                 }
 
+                if (!_insideConditionalTest
+                    && node.NodeType == ExpressionType.Not
+                    && node.Operand is IsNullExpression)
+                {
+                    return Expression.Condition(
+                        node.Operand,
+                        Expression.Constant(false, typeof(bool)),
+                        Expression.Constant(true, typeof(bool)));
+                }
+
+                if (!_insideConditionalTest && node.Operand is IsNullExpression)
+                {
+                    return Expression.Condition(
+                        node,
+                        Expression.Constant(true, typeof(bool)),
+                        Expression.Constant(false, typeof(bool)));
+                }
+
                 return base.VisitUnary(node);
             }