[Bugfix] [SQL-Search] Fix SQL code generation when :not operator is included.

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/FlatteningRewriterTests.cs

@@ -5,10 +5,13 @@
 
 using Microsoft.Health.Fhir.Core.Features.Search.Expressions;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class FlatteningRewriterTests
     {
         [Fact]

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/IncludeRewriterTests.cs

@@ -17,11 +17,13 @@
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors.QueryGenerators;
 using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using NSubstitute;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class IncludeRewriterTests : IClassFixture<IncludeRewriterTests.IncludeRewriterFixture>, IAsyncLifetime
     {
         private readonly IncludeRewriterFixture _fixture;

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/LastUpdatedToResourceSurrogateIdRewriterTests.cs

@@ -8,10 +8,13 @@
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
 using Microsoft.Health.Fhir.SqlServer.Features.Storage;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class LastUpdatedToResourceSurrogateIdRewriterTests
     {
         [InlineData(BinaryOperator.GreaterThan, "2020-09-24T12:00:00.500Z", BinaryOperator.GreaterThanOrEqual, "2020-09-24T12:00:00.501Z")]

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/MissingSearchParamVisitorTests.cs

@@ -8,10 +8,13 @@
 using Microsoft.Health.Fhir.Core.Models;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class MissingSearchParamVisitorTests
     {
         [Fact]

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/NotExpressionRewriterTests.cs

@@ -8,10 +8,13 @@
 using Microsoft.Health.Fhir.Core.Models;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class NotExpressionRewriterTests
     {
         [Fact]
@@ -27,7 +30,7 @@ public void GivenExpressionWithNotExpression_WhenVisited_AllExpressionPrependedT
             var visitedExpression = (SqlRootExpression)inputExpression.AcceptVisitor(NotExpressionRewriter.Instance);
             Assert.Collection(
                 visitedExpression.SearchParamTableExpressions,
-                e => { Assert.Equal(SearchParamTableExpressionKind.All, e.Kind); },
+                e => { Assert.Equal(SearchParamTableExpressionKind.Normal, e.Kind); },
                 e => { ValidateNotExpression(subExpression, e); });
             Assert.Equal(searchParamTableExpressions.Count + 1, visitedExpression.SearchParamTableExpressions.Count);
         }

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/RemoveIncludesRewriterTests.cs

@@ -6,10 +6,13 @@
 using Microsoft.Health.Fhir.Core.Features.Search.Expressions;
 using Microsoft.Health.Fhir.Core.Models;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class RemoveIncludesRewriterTests
     {
         [Fact]

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ResourceColumnPredicatePushdownRewriterTests.cs

@@ -11,10 +11,13 @@
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors.QueryGenerators;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class ResourceColumnPredicatePushdownRewriterTests
     {
         [Fact]

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/SearchParamTableExpressionReordererTests.cs

@@ -17,6 +17,7 @@
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class SearchParamTableExpressionReordererTests
     {
         private static readonly SearchParameterExpression NormalExpression = new SearchParameterExpression(new SearchParameterInfo("TestParam", "TestParam"), Expression.Equals(FieldName.TokenCode, null, "TestValue"));

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/SqlServerSortingValidatorTests.cs

@@ -9,12 +9,15 @@
 using Microsoft.Health.Fhir.Core.Models;
 using Microsoft.Health.Fhir.SqlServer.Features.Schema;
 using Microsoft.Health.Fhir.SqlServer.Features.Search;
+using Microsoft.Health.Fhir.Tests.Common;
 using Microsoft.Health.Fhir.ValueSets;
 using Microsoft.Health.SqlServer.Features.Schema;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class SqlServerSortingValidatorTests
     {
         private SqlServerSortingValidator _sqlServerSortingValidator;

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/TypeConstraintVisitorTests.cs

@@ -13,12 +13,15 @@
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
 using Microsoft.Health.Fhir.SqlServer.Features.Storage;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using NSubstitute;
 using Xunit;
 using static Microsoft.Health.Fhir.Core.Features.Search.Expressions.Expression;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class TypeConstraintVisitorTests
     {
         private const short AllergyIntolerance = 1;

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/UntypedReferenceRewriterTests.cs

@@ -6,11 +6,14 @@
 using Microsoft.Health.Fhir.Core.Features.Search.Expressions;
 using Microsoft.Health.Fhir.Core.Models;
 using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors;
+using Microsoft.Health.Fhir.Tests.Common;
 using Microsoft.Health.Fhir.ValueSets;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class UntypedReferenceRewriterTests
     {
         private static readonly SearchParameterInfo ReferenceSearchParameterWithOneTargetType = new SearchParameterInfo("p", "p", SearchParamType.Reference, targetResourceTypes: new[] { "Organization" });

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/HashingSqlQueryParameterManagerTests.cs

@@ -8,12 +8,15 @@
 using Microsoft.Data.SqlClient;
 using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model;
 using Microsoft.Health.Fhir.SqlServer.Features.Search;
+using Microsoft.Health.Fhir.Tests.Common;
 using Microsoft.Health.SqlServer;
 using Microsoft.Health.SqlServer.Features.Storage;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class HashingSqlQueryParameterManagerTests
     {
         public static readonly TheoryData<object> Data = new()

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs

@@ -7,12 +7,15 @@
 using Microsoft.Data.SqlClient;
 using Microsoft.Extensions.Logging;
 using Microsoft.Health.Fhir.SqlServer.Features.Search;
+using Microsoft.Health.Fhir.Tests.Common;
 using Microsoft.Health.SqlServer;
+using Microsoft.Health.Test.Utilities;
 using NSubstitute;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class SqlCommandSimplifierTests
     {
         private ILogger _logger;

src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlSesrverSearchParameterValidatorTests.cs

@@ -7,10 +7,13 @@
 using Microsoft.Health.Fhir.Core.Models;
 using Microsoft.Health.Fhir.SqlServer.Features.Search;
 using Microsoft.Health.Fhir.SqlServer.Features.Storage;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.SqlServer.UnitTests.Features.Search.Expressions
 {
+    [Trait(Traits.Category, Categories.Search)]
     public class SqlSesrverSearchParameterValidatorTests
     {
         private SearchParameterToSearchValueTypeMap _parameterToSearchValueTypeMap;

src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/NotExpressionRewriter.cs

@@ -41,7 +41,7 @@ public override Expression VisitSqlRoot(SqlRootExpression expression, object con
                             new SearchParamTableExpression(
                                 tableExpression.QueryGenerator,
                                 null,
-                                SearchParamTableExpressionKind.All));
+                                SearchParamTableExpressionKind.Normal));
                     }
 
                     newTableExpressions.Add((SearchParamTableExpression)tableExpression.AcceptVisitor(this, context));

src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj

@@ -94,7 +94,7 @@
     <None Remove="TestFiles\Patient-f001.json" />
     <None Remove="TestFiles\Patient.json" />
     <None Remove="TestFiles\Patient.xml" />
-    <None Remove="TestFiles\R4\Bundle-ChainingAndSortSearchValidation.json" />
+    <None Remove="TestFiles\R4\Bundle-ChainingSortAndSearchValidation.json" />
     <None Remove="TestFiles\R4\conversion_templates.tar.gz" />
     <None Remove="TestFiles\R4\DocumentReference-example-relatesTo-code-appends.indexes.json" />
     <None Remove="TestFiles\R4\DocumentReference-example-relatesTo-code-appends.json" />
@@ -149,7 +149,7 @@
     <None Remove="TestFiles\Normative\PatientWithMinimalData.json" />
     <None Remove="TestFiles\Normative\Weight.json" />
     <None Remove="TestFiles\Normative\WeightInGrams.json" />
-    <None Remove="TestFiles\R5\Bundle-ChainingAndSortSearchValidation.json" />
+    <None Remove="TestFiles\R5\Bundle-ChainingSortAndSearchValidation.json" />
     <None Remove="TestFiles\R5\CarePlan.json" />
     <None Remove="TestFiles\R5\Condition-For-Patient-f001.json" />
     <None Remove="TestFiles\R5\Device-d1.json" />
@@ -187,7 +187,7 @@
     <None Remove="TestFiles\R4\ValidCompartmentDefinition.json" />
     <None Remove="TestFiles\ResourceWrapperNoVersion.json" />
     <None Remove="TestFiles\Stu3\Bundle-Batch.json" />
-    <None Remove="TestFiles\Stu3\Bundle-ChainingAndSortSearchValidation.json" />
+    <None Remove="TestFiles\Stu3\Bundle-ChainingSortAndSearchValidation.json" />
     <None Remove="TestFiles\Stu3\Bundle-Transaction.json" />
     <None Remove="TestFiles\Stu3\Bundle-TransactionForRollBack.json" />
     <None Remove="TestFiles\Stu3\Bundle-TransactionWithAllValidRoutes.json" />
@@ -279,7 +279,7 @@
     <EmbeddedResource Include="TestFiles\Normative\SearchParameterMissingExpression.json" />
     <EmbeddedResource Include="TestFiles\Normative\SearchParameterMissingBase.json" />
     <EmbeddedResource Include="TestFiles\Normative\Specimen.json" />
-    <EmbeddedResource Include="TestFiles\R4\Bundle-ChainingAndSortSearchValidation.json" />
+    <EmbeddedResource Include="TestFiles\R4\Bundle-ChainingSortAndSearchValidation.json" />
     <EmbeddedResource Include="TestFiles\R4\CarePlan.json" />
     <EmbeddedResource Include="TestFiles\Normative\CodeSystem-careplan-category.json" />
     <EmbeddedResource Include="TestFiles\Normative\list-example-long.json" />
@@ -354,7 +354,7 @@
     <EmbeddedResource Include="TestFiles\R5\Appointment.json" />
     <EmbeddedResource Include="TestFiles\R5\BasicExampleNarrative.json" />
     <EmbeddedResource Include="TestFiles\R5\BasicExampleNarrative.xml" />
-    <EmbeddedResource Include="TestFiles\R5\Bundle-ChainingAndSortSearchValidation.json" />
+    <EmbeddedResource Include="TestFiles\R5\Bundle-ChainingSortAndSearchValidation.json" />
     <EmbeddedResource Include="TestFiles\R5\Bundle-TransactionWithReferenceInResourceBody.json" />
     <EmbeddedResource Include="TestFiles\R5\CarePlan.json" />
     <EmbeddedResource Include="TestFiles\R5\Condition-For-Patient-f001.json" />
@@ -391,7 +391,7 @@
     <EmbeddedResource Include="TestFiles\Stu3\BloodGlucose.json" />
     <EmbeddedResource Include="TestFiles\Stu3\BloodPressure.json" />
     <EmbeddedResource Include="TestFiles\Stu3\Bundle-Batch.json" />
-    <EmbeddedResource Include="TestFiles\Stu3\Bundle-ChainingAndSortSearchValidation.json" />
+    <EmbeddedResource Include="TestFiles\Stu3\Bundle-ChainingSortAndSearchValidation.json" />
     <EmbeddedResource Include="TestFiles\Stu3\Bundle-TransactionForRollBack.json" />
     <EmbeddedResource Include="TestFiles\Stu3\Bundle-TransactionWithForeignReferenceInResourceBody.json" />
     <EmbeddedResource Include="TestFiles\Stu3\Bundle-TransactionWithInvalidProcessingRoutes.json" />

test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems

@@ -90,6 +90,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)Rest\Search\DateSearchTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Rest\Search\IncludeSearchTestFixture.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Rest\Search\IncludeSearchTests.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Rest\Search\NotExpressionTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Rest\Search\NumberSearchTestFixture.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Rest\Search\NumberSearchTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Rest\Search\QuantitySearchTestFixture.cs" />

test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingAndSortTests.cs

@@ -4,8 +4,11 @@
 // -------------------------------------------------------------------------------------------------
 
 using System;
+using System.Collections.Generic;
 using System.Net;
+using System.Threading;
 using Hl7.Fhir.Model;
+using Hl7.Fhir.Serialization;
 using Microsoft.Health.Fhir.Client;
 using Microsoft.Health.Fhir.Tests.Common;
 using Microsoft.Health.Fhir.Tests.Common.FixtureParameters;
@@ -31,24 +34,8 @@ public ChainingAndSortTests(HttpIntegrationTestFixture fixture)
         [Trait(Traits.Priority, Priority.One)]
         public async Task GivenAChainedSearchPattern_WhenSearched_ThenCompareTheResultsWithDifferentVariationsOfSortingExpressions()
         {
-            string requestBundleAsString = Samples.GetJson("Bundle-ChainingAndSortSearchValidation");
-            var parser = new Hl7.Fhir.Serialization.FhirJsonParser();
-            var requestBundle = parser.Parse<Bundle>(requestBundleAsString);
-
-            using FhirResponse<Bundle> fhirResponse = await _client.PostBundleAsync(requestBundle);
-            Assert.NotNull(fhirResponse);
-            Assert.Equal(HttpStatusCode.OK, fhirResponse.StatusCode);
-
-            // Ensure all records were ingested.
-            Assert.Equal(requestBundle.Entry.Count, fhirResponse.Resource.Entry.Count);
-            foreach (Bundle.EntryComponent component in fhirResponse.Resource.Entry)
-            {
-                Assert.NotNull(component.Response.Status);
-                HttpStatusCode httpStatusCode = (HttpStatusCode)Convert.ToInt32(component.Response.Status);
-                Assert.True(httpStatusCode == HttpStatusCode.OK || httpStatusCode == HttpStatusCode.Created);
-            }
+            await IngestBundleWithTestDataAsync(CancellationToken.None);
 
-            const int totalNumberOfHealthcareServices = 15;             // Total number of healthcare services ingested.
             const int expectedNumberOfEntriesInFirstPage = 10;          // Max number of entries in the first page.
             const int expectedNumberOfLinks = 2;                        // Expected number of pages/links.
             const int totalNumberOfFilteredHealthcareServices = 13;     // Expected number of health care services when filters are applied.
@@ -57,7 +44,6 @@ public async Task GivenAChainedSearchPattern_WhenSearched_ThenCompareTheResultsW
             Bundle bundleAllHealthcaseServices = await _client.SearchAsync(ResourceType.HealthcareService, "_total=accurate");
             Assert.Equal(expectedNumberOfEntriesInFirstPage, bundleAllHealthcaseServices.Entry.Count);
             Assert.Equal(expectedNumberOfLinks, bundleAllHealthcaseServices.Link.Count);
-            Assert.Equal(totalNumberOfHealthcareServices, bundleAllHealthcaseServices.Total.Value);
 
             string commonQuery = "name:missing=false&_has:PractitionerRole:service:practitioner=2ec3586b-9454-4c7f-8eaf-7a0e64cecf17&active:not=false&location:missing=false&_has:PractitionerRole:service:active=true";
 
@@ -95,5 +81,36 @@ public async Task GivenAChainedSearchPattern_WhenSearched_ThenCompareTheResultsW
             Assert.Equal(expectedNumberOfLinks, bundleWithSortAndTotalAccurate.Link.Count);
             Assert.Equal(totalNumberOfFilteredHealthcareServices, bundleWithSortAndTotalAccurate.Total.Value);
         }
+
+        private async Task IngestBundleWithTestDataAsync(CancellationToken cancellationToken)
+        {
+            string requestBundleAsString = Samples.GetJson("Bundle-ChainingSortAndSearchValidation");
+            var parser = new FhirJsonParser();
+            var requestBundle = parser.Parse<Bundle>(requestBundleAsString);
+
+            using FhirResponse<Bundle> fhirResponse = await _client.PostBundleAsync(requestBundle, cancellationToken);
+            Assert.NotNull(fhirResponse);
+            Assert.Equal(HttpStatusCode.OK, fhirResponse.StatusCode);
+
+            // Set of Healthcare Services created along the initialization.
+            var healthcareServiceIds = new List<string>();
+
+            // Ensure all records were ingested.
+            Assert.Equal(requestBundle.Entry.Count, fhirResponse.Resource.Entry.Count);
+            foreach (Bundle.EntryComponent component in fhirResponse.Resource.Entry)
+            {
+                Assert.NotNull(component.Response.Status);
+                HttpStatusCode httpStatusCode = (HttpStatusCode)Convert.ToInt32(component.Response.Status);
+                Assert.True(httpStatusCode == HttpStatusCode.OK || httpStatusCode == HttpStatusCode.Created);
+
+                if (component.Resource.TypeName == ResourceType.HealthcareService.ToString())
+                {
+                    healthcareServiceIds.Add(component.Resource.Id);
+                }
+            }
+
+            const int totalNumberOfHealthcareServices = 15; // Total number of healthcare services ingested.
+            Assert.Equal(totalNumberOfHealthcareServices, healthcareServiceIds.Count);
+        }
     }
 }