Work on set operations and inheritance

Added support for set operations over different entity types.

When performing a set operation, if two properties in the entity
inheritance hierarchy were mapped to the same database column, that
column was projected twice.

Closes dotnet#16298
Fixes dotnet#18832

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -491,10 +491,7 @@
   <data name="PendingAmbientTransaction" xml:space="preserve">
     <value>This connection was used with an ambient transaction. The original ambient transaction needs to be completed before this connection can be used outside of it.</value>
   </data>
-  <data name="SetOperationNotWithinEntityTypeHierarchy" xml:space="preserve">
-    <value>Set operations (Union, Concat, Intersect, Except) are only supported over entity types within the same type hierarchy.</value>
-  </data>
   <data name="FromSqlNonComposable" xml:space="preserve">
     <value>FromSqlRaw or FromSqlInterpolated was called with non-composable SQL and with a query composing over it. Consider calling `AsEnumerable` after the FromSqlRaw or FromSqlInterpolated method to perform the composition on the client side.</value>
   </data>
-</root>
+</root>

src/EFCore.Relational/Query/EntityProjectionExpression.cs

@@ -125,6 +125,12 @@ public virtual ColumnExpression BindProperty([NotNull] IProperty property)
 
             if (!_propertyExpressionsCache.TryGetValue(property, out var expression))
             {
+                if (_innerTable == null)
+                {
+                    throw new InvalidOperationException(
+                        $"Could not bind property '{property.Name}' on entity type '{EntityType.DisplayName()}': "
+                        + "projection was initialized with an expression cache but the property isn't in it.");
+                }
                 expression = new ColumnExpression(property, _innerTable, _nullable);
                 _propertyExpressionsCache[property] = expression;
             }

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -201,7 +201,7 @@ protected override ShapedQueryExpression TranslateConcat(ShapedQueryExpression s
             Check.NotNull(source2, nameof(source2));
 
             ((SelectExpression)source1.QueryExpression).ApplyUnion((SelectExpression)source2.QueryExpression, distinct: false);
-
+            ModifyShaperForSetOperation(source1, source2);
             return source1;
         }
 
@@ -301,6 +301,7 @@ protected override ShapedQueryExpression TranslateExcept(ShapedQueryExpression s
             Check.NotNull(source2, nameof(source2));
 
             ((SelectExpression)source1.QueryExpression).ApplyExcept((SelectExpression)source2.QueryExpression, distinct: true);
+            ModifyShaperForSetOperation(source1, source2);
             return source1;
         }
 
@@ -456,7 +457,7 @@ protected override ShapedQueryExpression TranslateIntersect(ShapedQueryExpressio
             Check.NotNull(source2, nameof(source2));
 
             ((SelectExpression)source1.QueryExpression).ApplyIntersect((SelectExpression)source2.QueryExpression, distinct: true);
-
+            ModifyShaperForSetOperation(source1, source2);
             return source1;
         }
 
@@ -1015,6 +1016,7 @@ protected override ShapedQueryExpression TranslateUnion(ShapedQueryExpression so
             Check.NotNull(source2, nameof(source2));
 
             ((SelectExpression)source1.QueryExpression).ApplyUnion((SelectExpression)source2.QueryExpression, distinct: true);
+            ModifyShaperForSetOperation(source1, source2);
             return source1;
         }
 
@@ -1276,5 +1278,51 @@ private Expression TryExpand(Expression source, MemberIdentity member)
 
             return source;
         }
+
+        /// <summary>
+        ///     If a set operation is between different entity types, the query will return their closest common ancestor.
+        ///     Modify the shaper accordingly.
+        /// </summary>
+        private void ModifyShaperForSetOperation(ShapedQueryExpression source1, ShapedQueryExpression source2)
+        {
+            if (RemoveConvert(source1.ShaperExpression) is EntityShaperExpression shaper1
+                && RemoveConvert(source2.ShaperExpression) is EntityShaperExpression shaper2
+                && shaper1.EntityType != shaper2.EntityType)
+            {
+                var closestCommonParent = shaper1.EntityType.GetClosestCommonParent(shaper2.EntityType);
+
+                source1.ShaperExpression = new EntityShaperExpression(
+                    closestCommonParent,
+                    shaper1.ValueBufferExpression,
+                    shaper1.IsNullable);
+
+                // If there's a convert node on either side (set operation over different entity type) and it's
+                // converting to a higher type in the hierarchy, add back a convert node to that type.
+                var convertType =
+                    source1.ShaperExpression is UnaryExpression unary1
+                    && unary1.NodeType == ExpressionType.Convert
+                        ? unary1.Type
+                        : source2.ShaperExpression is UnaryExpression unary2
+                        && unary2.NodeType == ExpressionType.Convert
+                            ? unary2.Type
+                            : null;
+
+                if (convertType != null && convertType != closestCommonParent.ClrType)
+                {
+                    source1.ShaperExpression = Expression.Convert(source1.ShaperExpression, convertType);
+                }
+            }
+
+            static Expression RemoveConvert(Expression expression)
+            {
+                if (expression is UnaryExpression unaryExpression
+                    && expression.NodeType == ExpressionType.Convert)
+                {
+                    return RemoveConvert(unaryExpression.Operand);
+                }
+
+                return expression;
+            }
+        }
     }
 }

src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs

@@ -7,6 +7,7 @@
 using System.Linq.Expressions;
 using System.Reflection;
 using JetBrains.Annotations;
+using Microsoft.EntityFrameworkCore.Diagnostics;
 using Microsoft.EntityFrameworkCore.Metadata;
 using Microsoft.EntityFrameworkCore.Metadata.Internal;
 using Microsoft.EntityFrameworkCore.Utilities;
@@ -567,7 +568,7 @@ private void ApplySetOperation(SetOperationType setOperationType, SelectExpressi
                 if (joinedMapping.Value1 is EntityProjectionExpression entityProjection1
                     && joinedMapping.Value2 is EntityProjectionExpression entityProjection2)
                 {
-                    HandleEntityMapping(joinedMapping.Key, select1, entityProjection1, select2, entityProjection2);
+                    _projectionMapping[joinedMapping.Key] = LiftEntityProjectionFromSetOperands(entityProjection1, entityProjection2);
                     continue;
                 }
 
@@ -616,50 +617,183 @@ private void ApplySetOperation(SetOperationType setOperationType, SelectExpressi
             _tables.Clear();
             _tables.Add(setExpression);
 
-            void HandleEntityMapping(
-                ProjectionMember projectionMember,
-                SelectExpression select1, EntityProjectionExpression projection1,
-                SelectExpression select2, EntityProjectionExpression projection2)
+            EntityProjectionExpression LiftEntityProjectionFromSetOperands(EntityProjectionExpression projection1, EntityProjectionExpression projection2)
             {
-                if (projection1.EntityType != projection2.EntityType)
+                var (entityType1, entityType2) = (projection1.EntityType, projection2.EntityType);
+
+                var propertyExpressions = new Dictionary<IProperty, ColumnExpression>();
+                var expressionsByColumnName = new Dictionary<string, ColumnExpression>();
+
+                if (entityType1 == entityType2)
                 {
-                    throw new InvalidOperationException(
-                        "Set operations over different entity types are currently unsupported (see Issue#16298)");
+                    foreach (var property in GetAllPropertiesInHierarchy(entityType1))
+                    {
+                        propertyExpressions[property] = GenerateOuterSetOperationColumn(
+                            property.GetColumnName(),
+                            projection1.BindProperty(property),
+                            projection2.BindProperty(property));
+                    }
+
+                    return new EntityProjectionExpression(entityType1, propertyExpressions);
                 }
 
-                var propertyExpressions = new Dictionary<IProperty, ColumnExpression>();
-                foreach (var property in GetAllPropertiesInHierarchy(projection1.EntityType))
+                // We're doing a set operation over two different entity types (within the same hierarchy).
+                // Since both sides of the set operations must produce the same result shape, find the
+                // closest common ancestor and load all the columns for that, adding null projections where
+                // necessary. Note this means we add null projections for properties which neither sibling
+                // actually needs, since the shaper doesn't know that only those sibling types will be coming
+                // back.
+                var commonParentEntityType = entityType1.GetClosestCommonParent(entityType2);
+
+                if (commonParentEntityType == null)
                 {
-                    propertyExpressions[property] = AddSetOperationColumnProjections(
-                        select1, projection1.BindProperty(property),
-                        select2, projection2.BindProperty(property));
+                    throw new InvalidOperationException("No common parent in set operation over different types!");
                 }
 
-                _projectionMapping[projectionMember] = new EntityProjectionExpression(projection1.EntityType, propertyExpressions);
-            }
+                var allProperties1 = GetAllPropertiesInHierarchy(entityType1).ToList();
+                var allProperties2 = GetAllPropertiesInHierarchy(entityType2).ToList();
+                var properties1 = allProperties1.ToList();
+                var properties2 = allProperties2.ToList();
 
-            ColumnExpression AddSetOperationColumnProjections(
-                SelectExpression select1, ColumnExpression column1,
-                SelectExpression select2, ColumnExpression column2)
-            {
-                var alias = GenerateUniqueAlias(column1.Name);
-                var innerProjection1 = new ProjectionExpression(column1, alias);
-                var innerProjection2 = new ProjectionExpression(column2, alias);
-                select1._projection.Add(innerProjection1);
-                select2._projection.Add(innerProjection2);
-                var outerProjection = new ColumnExpression(innerProjection1, setExpression);
-                if (IsNullableProjection(innerProjection1)
-                    || IsNullableProjection(innerProjection2))
+                // First handle shared properties that come from common base entity types
+                foreach (var property in properties1.Intersect(properties2).ToArray())
+                {
+                    propertyExpressions[property] = GenerateOuterSetOperationColumn(
+                        property.GetColumnName(),
+                        projection1.BindProperty(property),
+                        projection2.BindProperty(property));
+
+                    properties1.Remove(property);
+                    properties2.Remove(property);
+                }
+
+                // Next, find different model property pairs which are mapped to the same database column
+                foreach (var (group1, group2) in properties1
+                    .GroupBy(p => p.GetColumnName())
+                    .Join(
+                        properties2.GroupBy(p => p.GetColumnName()),
+                        g => g.Key,
+                        g => g.Key,
+                        (p1, p2) => (p1, p2))
+                    .ToArray())
+                {
+                    var outerProjection = GenerateOuterSetOperationColumn(
+                        group1.Key,
+                        projection1.BindProperty(group1.First()),
+                        projection2.BindProperty(group2.First()));
+
+                    foreach (var property in group1)
+                    {
+                        propertyExpressions[property] = outerProjection;
+                        properties1.Remove(property);
+                    }
+
+                    foreach (var property in group2)
+                    {
+                        propertyExpressions[property] = outerProjection;
+                        properties2.Remove(property);
+                    }
+                }
+
+                // Remaining properties exist only on one side, so inject a null constant projection on the other side.
+                foreach (var property in properties1)
                 {
-                    outerProjection = outerProjection.MakeNullable();
+                    propertyExpressions[property] = GenerateOuterSetOperationColumn(
+                        property.GetColumnName(),
+                        projection1.BindProperty(property),
+                        new SqlConstantExpression(
+                            Constant(null, property.ClrType.MakeNullable()),
+                            property.GetRelationalTypeMapping()));
                 }
 
-                if (select1._identifier.Contains(column1))
+                foreach (var property in properties2)
+                {
+                    propertyExpressions[property] = GenerateOuterSetOperationColumn(
+                        property.GetColumnName(),
+                        new SqlConstantExpression(
+                            Constant(null, property.ClrType.MakeNullable()),
+                            property.GetRelationalTypeMapping()),
+                        projection2.BindProperty(property));
+                }
+
+                // Finally, the shaper will expect to read properties from unrelated siblings, since the set operations
+                // return type is the common ancestor. Add appropriate null constant projections for both sides.
+                // See #16215 for a possible optimization.
+                var unrelatedSiblingProperties = GetAllPropertiesInHierarchy(commonParentEntityType).ToList()
+                    .Except(allProperties1)
+                    .Except(allProperties2);
+                foreach (var property in unrelatedSiblingProperties)
+                {
+                    propertyExpressions[property] = GenerateOuterSetOperationColumn(
+                        property.GetColumnName(),
+                        new SqlConstantExpression(
+                            Constant(null, property.ClrType.MakeNullable()),
+                            property.GetRelationalTypeMapping()),
+                        new SqlConstantExpression(
+                            Constant(null, property.ClrType.MakeNullable()),
+                            property.GetRelationalTypeMapping()));
+                }
+
+                var newEntityProjection = new EntityProjectionExpression(commonParentEntityType, propertyExpressions);
+
+                // Also lift nested entity projections
+                foreach (var navigation in projection1.EntityType.GetTypesInHierarchy()
+                    .SelectMany(EntityTypeExtensions.GetDeclaredNavigations))
                 {
-                    _identifier.Add(outerProjection);
+                    var boundEntityShaperExpression1 = projection1.BindNavigation(navigation);
+                    var boundEntityShaperExpression2 = projection2.BindNavigation(navigation);
+
+                    if (boundEntityShaperExpression1 == null
+                        && boundEntityShaperExpression2 == null)
+                    {
+                        continue;
+                    }
+
+                    if (boundEntityShaperExpression1 == null
+                        && boundEntityShaperExpression2 != null
+                        || boundEntityShaperExpression2 == null
+                        && boundEntityShaperExpression1 != null)
+                    {
+                        throw new InvalidOperationException(CoreStrings.SetOperationWithDifferentIncludesInOperands);
+                    }
+
+                    var newInnerEntityProjection = LiftEntityProjectionFromSetOperands(
+                        (EntityProjectionExpression)boundEntityShaperExpression1.ValueBufferExpression,
+                        (EntityProjectionExpression)boundEntityShaperExpression2.ValueBufferExpression);
+                    boundEntityShaperExpression1 = boundEntityShaperExpression1.Update(newInnerEntityProjection);
+                    newEntityProjection.AddNavigationBinding(navigation, boundEntityShaperExpression1);
                 }
 
-                return outerProjection;
+                return newEntityProjection;
+
+                ColumnExpression GenerateOuterSetOperationColumn(string columnName, SqlExpression innerExpression1, SqlExpression innerExpression2)
+                {
+                    if (expressionsByColumnName.TryGetValue(columnName, out var outerProjection))
+                    {
+                        return outerProjection;
+                    }
+
+                    var alias = GenerateUniqueAlias(columnName);
+                    var innerProjection1 = new ProjectionExpression(innerExpression1, alias);
+                    var innerProjection2 = new ProjectionExpression(innerExpression2, alias);
+                    select1._projection.Add(innerProjection1);
+                    select2._projection.Add(innerProjection2);
+
+                    outerProjection = new ColumnExpression(innerProjection1, setExpression);
+
+                    if (IsNullableProjection(innerProjection1)
+                        || IsNullableProjection(innerProjection2))
+                    {
+                        outerProjection = outerProjection.MakeNullable();
+                    }
+
+                    if (select1._identifier.Contains(innerExpression1))
+                    {
+                        _identifier.Add(outerProjection);
+                    }
+
+                    return expressionsByColumnName[columnName] = outerProjection;
+                }
             }
 
             string GenerateUniqueAlias(string baseAlias)