Skip to content

[Do not review] Experimental sourcenode parser #5013

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

[Do not review] Experimental sourcenode parser #5013

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c55846e
Experimental sourcenode parser
brendankowitz Jun 2, 2025
b0bdc96
Extended functionality / fixes
brendankowitz Jun 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions Microsoft.Health.Fhir.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Cosmo
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Api.OpenIddict", "src\Microsoft.Health.Fhir.Api.OpenIddict\Microsoft.Health.Fhir.Api.OpenIddict.csproj", "{869F0CD6-280E-403E-8034-185A8B63D03E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.SourceNodeSerialization", "src\Microsoft.Health.Fhir.SourceNodeSerialization\Microsoft.Health.Fhir.SourceNodeSerialization.csproj", "{7F7FCE18-C2EC-436C-9D89-B47F9E07EB46}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.SourceNodeSerialization.UnitTests", "src\Microsoft.Health.Fhir.SourceNodeSerialization.UnitTests\Microsoft.Health.Fhir.SourceNodeSerialization.UnitTests.csproj", "{AD19560A-ED76-4A4E-B809-E2224ADDC092}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -487,6 +491,14 @@ Global
{869F0CD6-280E-403E-8034-185A8B63D03E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{869F0CD6-280E-403E-8034-185A8B63D03E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{869F0CD6-280E-403E-8034-185A8B63D03E}.Release|Any CPU.Build.0 = Release|Any CPU
{7F7FCE18-C2EC-436C-9D89-B47F9E07EB46}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7F7FCE18-C2EC-436C-9D89-B47F9E07EB46}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7F7FCE18-C2EC-436C-9D89-B47F9E07EB46}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7F7FCE18-C2EC-436C-9D89-B47F9E07EB46}.Release|Any CPU.Build.0 = Release|Any CPU
{AD19560A-ED76-4A4E-B809-E2224ADDC092}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{AD19560A-ED76-4A4E-B809-E2224ADDC092}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AD19560A-ED76-4A4E-B809-E2224ADDC092}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AD19560A-ED76-4A4E-B809-E2224ADDC092}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -582,6 +594,8 @@ Global
{B9AAA11D-8C8C-44C3-AADE-801376EF82F0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69}
{1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69}
{869F0CD6-280E-403E-8034-185A8B63D03E} = {1295CCC3-73FB-4376-AE95-F6F31A37B152}
{7F7FCE18-C2EC-436C-9D89-B47F9E07EB46} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4}
{AD19560A-ED76-4A4E-B809-E2224ADDC092} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8}
Expand Down
8 changes: 4 additions & 4 deletions R4.slnf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
"projects": [
"samples\\apps\\SmartLauncher\\SmartLauncher.csproj",
"src\\Microsoft.Health.Extensions.Xunit\\Microsoft.Health.Extensions.Xunit.csproj",
"src\\Microsoft.Health.Fhir.Api.OpenIddict\\Microsoft.Health.Fhir.Api.OpenIddict.csproj",
"src\\Microsoft.Health.Fhir.Api.UnitTests\\Microsoft.Health.Fhir.R4.Api.UnitTests.csproj",
"src\\Microsoft.Health.Fhir.Api\\Microsoft.Health.Fhir.Api.csproj",
"src\\Microsoft.Health.Fhir.Api.OpenIddict\\Microsoft.Health.Fhir.Api.OpenIddict.csproj",
"src\\Microsoft.Health.Fhir.Azure.UnitTests\\Microsoft.Health.Fhir.Azure.UnitTests.csproj",
"src\\Microsoft.Health.Fhir.Azure\\Microsoft.Health.Fhir.Azure.csproj",
"src\\Microsoft.Health.Fhir.Core.UnitTests\\Microsoft.Health.Fhir.Core.UnitTests.csproj",
Expand All @@ -28,11 +28,11 @@
"src\\Microsoft.Health.Fhir.Shared.Core\\Microsoft.Health.Fhir.Shared.Core.shproj",
"src\\Microsoft.Health.Fhir.Shared.Tests\\Microsoft.Health.Fhir.Shared.Tests.Common.shproj",
"src\\Microsoft.Health.Fhir.Shared.Web\\Microsoft.Health.Fhir.Shared.Web.shproj",
"src\\Microsoft.Health.Fhir.SourceNodeSerialization.UnitTests\\Microsoft.Health.Fhir.SourceNodeSerialization.UnitTests.csproj",
"src\\Microsoft.Health.Fhir.SourceNodeSerialization\\Microsoft.Health.Fhir.SourceNodeSerialization.csproj",
"src\\Microsoft.Health.Fhir.SqlServer.UnitTests\\Microsoft.Health.Fhir.SqlServer.UnitTests.csproj",
"src\\Microsoft.Health.Fhir.SqlServer\\Microsoft.Health.Fhir.SqlServer.csproj",
"src\\Microsoft.Health.Fhir.Tests.Common\\Microsoft.Health.Fhir.Tests.Common.csproj",
"src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj",
"src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj",
"src\\Microsoft.Health.Fhir.ValueSets\\Microsoft.Health.Fhir.ValueSets.csproj",
"src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj",
"src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj",
Expand All @@ -44,4 +44,4 @@
"test\\Microsoft.Health.Fhir.Shared.Tests.Integration\\Microsoft.Health.Fhir.Shared.Tests.Integration.shproj"
]
}
}
}
3 changes: 3 additions & 0 deletions build/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ COPY .editorconfig \
COPY ./src/Microsoft.Health.Fhir.ValueSets/Microsoft.Health.Fhir.ValueSets.csproj \
./src/Microsoft.Health.Fhir.ValueSets/Microsoft.Health.Fhir.ValueSets.csproj

COPY ./src/Microsoft.Health.Fhir.SourceNodeSerialization/Microsoft.Health.Fhir.SourceNodeSerialization.csproj \
./src/Microsoft.Health.Fhir.SourceNodeSerialization/Microsoft.Health.Fhir.SourceNodeSerialization.csproj

COPY ./src/Microsoft.Health.Fhir.Core/Microsoft.Health.Fhir.Core.csproj \
./src/Microsoft.Health.Fhir.Core/Microsoft.Health.Fhir.Core.csproj

Expand Down
7 changes: 7 additions & 0 deletions src/Microsoft.Health.Fhir.Core/Extensions/ResourceTypeExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
using System;
using EnsureThat;
using Hl7.Fhir.ElementModel;
using Hl7.Fhir.Specification;
using Microsoft.Health.Fhir.Core.Models;
using Microsoft.Health.Fhir.SourceNodeSerialization.SourceNodes.Models;

namespace Microsoft.Health.Fhir.Core.Extensions
{
Expand All @@ -22,5 +24,10 @@ public static ResourceElement ToResourceElement(this ITypedElement typedElement)
{
return new ResourceElement(typedElement);
}

public static ResourceElement ToResourceElement(this ResourceJsonNode typedElement, IStructureDefinitionSummaryProvider structureDefinitionSummaryProvider)
{
return new ResourceElement(typedElement.ToSourceNode().ToTypedElement(structureDefinitionSummaryProvider), typedElement);
}
}
}
1 change: 1 addition & 0 deletions src/Microsoft.Health.Fhir.Core/Microsoft.Health.Fhir.Core.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
<PackageReference Include="System.Drawing.Common" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Microsoft.Health.Fhir.SourceNodeSerialization\Microsoft.Health.Fhir.SourceNodeSerialization.csproj" />
<ProjectReference Include="..\Microsoft.Health.TaskManagement\Microsoft.Health.TaskManagement.csproj" />
<ProjectReference Include="..\Microsoft.Health.Fhir.ValueSets\Microsoft.Health.Fhir.ValueSets.csproj" />
</ItemGroup>
Expand Down
3 changes: 3 additions & 0 deletions src/Microsoft.Health.Fhir.Core/Models/IModelInfoProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System;
using System.Collections.Generic;
using Hl7.Fhir.ElementModel;
using Hl7.Fhir.Introspection;
using Hl7.Fhir.Specification;
using Hl7.FhirPath;
using Microsoft.Health.Fhir.Core.Features.Persistence;
Expand All @@ -20,6 +21,8 @@ public interface IModelInfoProvider

IStructureDefinitionSummaryProvider StructureDefinitionSummaryProvider { get; }

ModelInspector ModelInspector { get; }

string GetFhirTypeNameForType(Type type);

bool IsKnownResource(string name, bool ignoreCase = true);
Expand Down
20 changes: 20 additions & 0 deletions src/Microsoft.Health.Fhir.Core/Models/ResourceElement.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@
using System.Linq;
using EnsureThat;
using Hl7.Fhir.ElementModel;
using Hl7.Fhir.Model;
using Hl7.Fhir.Serialization;
using Hl7.FhirPath;
using Microsoft.Health.Fhir.SourceNodeSerialization.SourceNodes;

namespace Microsoft.Health.Fhir.Core.Models
{
Expand All @@ -33,6 +35,17 @@ public ResourceElement(ITypedElement instance)
Instance = instance;
_context = new Lazy<EvaluationContext>(() =>
new EvaluationContext().WithResourceOverrides(instance));

Poco = new Lazy<Base>(() =>
{
if (ResourceInstance is Base resourcePoco)
{
return resourcePoco;
}

// Perform conversion from ITypedElement to Base (more expensive)
return instance.ToPoco(ModelInfoProvider.Instance.ModelInspector);
});
}

internal ResourceElement(ITypedElement instance, object resourceInstance)
Expand All @@ -46,8 +59,15 @@ internal ResourceElement(ITypedElement instance, object resourceInstance)

internal object ResourceInstance { get; }

internal bool IsJsonNode => ResourceInstance is IResourceNode;

public ITypedElement Instance { get; }

/// <summary>
/// Provides a lazy-loaded POCO representation of the resource (this might perform an initial conversion, so is more expensive)
/// </summary>
public Lazy<Base> Poco { get; }

public string Id => Scalar<string>("Resource.id");

public string VersionId => Scalar<string>("Resource.meta.versionId");
Expand Down
10 changes: 9 additions & 1 deletion src/Microsoft.Health.Fhir.Shared.Core/Extensions/ModelExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using Microsoft.Health.Fhir.Core.Features.Definition;
using Microsoft.Health.Fhir.Core.Features.Persistence;
using Microsoft.Health.Fhir.Core.Models;
using Microsoft.Health.Fhir.SourceNodeSerialization.SourceNodes.Models;

namespace Microsoft.Health.Fhir.Core.Extensions
{
Expand Down Expand Up @@ -43,7 +44,14 @@ public static T ToPoco<T>(this ResourceElement resource)
{
EnsureArg.IsNotNull(resource, nameof(resource));

return (T)resource.ResourceInstance ?? resource.Instance.ToPoco<T>();
return (T)resource.Poco.Value;
}

public static ResourceJsonNode ToJsonNode(this ResourceElement resource)
{
EnsureArg.IsNotNull(resource, nameof(resource));

return (ResourceJsonNode)resource.ResourceInstance ?? throw new NotSupportedException();
}

public static Resource ToPoco(this ResourceElement resource)
Expand Down
40 changes: 23 additions & 17 deletions src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,18 @@
using System.Linq;
using System.Text.RegularExpressions;
using EnsureThat;
using Hl7.Fhir.ElementModel;
using Hl7.Fhir.Model;
using Hl7.Fhir.Serialization;
using Hl7.Fhir.Specification;
using Microsoft.Health.Core.Extensions;
using Microsoft.Health.Fhir.Core.Extensions;
using Microsoft.Health.Fhir.Core.Features.Persistence;
using Microsoft.Health.Fhir.Core.Features.Resources;
using Microsoft.Health.Fhir.Core.Models;
using Microsoft.Health.Fhir.SourceNodeSerialization;
using Microsoft.Health.Fhir.SourceNodeSerialization.Extensions;
using Microsoft.Health.Fhir.SourceNodeSerialization.SourceNodes.Models;

namespace Microsoft.Health.Fhir.Core.Features.Operations.Import
{
Expand All @@ -35,19 +40,21 @@

public ImportResource Parse(long index, long offset, int length, string rawResource, ImportMode importMode)
{
var resource = _parser.Parse<Resource>(rawResource);
var resource = ResourceJsonNode.Parse(rawResource);

ValidateResourceId(resource?.Id);
CheckConditionalReferenceInResource(resource, importMode);

if (resource.Meta == null)

Check warning

Code scanning / CodeQL

Dereferenced variable may be null Warning

Variable
resource
Loading
may be null at this access as suggested by
this
Loading
null check.
{
resource.Meta = new Meta();
resource.Meta = new MetaJsonNode();
}

var lastUpdatedIsNull = importMode == ImportMode.InitialLoad || resource.Meta.LastUpdated == null;
var lastUpdated = lastUpdatedIsNull ? Clock.UtcNow : resource.Meta.LastUpdated.Value;
resource.Meta.LastUpdated = new DateTimeOffset(lastUpdated.DateTime.TruncateToMillisecond(), lastUpdated.Offset);
if (!lastUpdatedIsNull && resource.Meta.LastUpdated.Value > Clock.UtcNow.AddSeconds(10)) // 5 sec is the max for the computers in the domain
var lastUpdated = lastUpdatedIsNull ? Clock.UtcNow : resource.Meta.LastUpdated;
var updatedDateTime = new DateTimeOffset(lastUpdated.Value.DateTime.TruncateToMillisecond(), lastUpdated.Value.Offset);

Check warning

Code scanning / CodeQL

Dereferenced variable may be null Warning

Variable
lastUpdated
Loading
may be null at this access because it has a nullable type.
Variable
lastUpdated
Loading
may be null at this access because it has a nullable type.

Copilot Autofix

AI about 1 month ago

To fix the issue, we need to ensure that lastUpdated is never dereferenced when it is null. This can be achieved by adding a null check before accessing lastUpdated.Value. If lastUpdated is null, an appropriate fallback value (e.g., Clock.UtcNow) should be used.

The fix involves:

  1. Adding a null check for lastUpdated before accessing lastUpdated.Value.
  2. Ensuring that the fallback value is used if lastUpdated is null.
Suggested changeset 1
src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs

Autofix patch

Autofix patch
Run the following command in your local git repository to apply this patch
cat << 'EOF' | git apply
diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs
--- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs
+++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs
@@ -53,2 +53,6 @@
             var lastUpdated = lastUpdatedIsNull ? Clock.UtcNow : resource.Meta.LastUpdated;
+            if (lastUpdated == null)
+            {
+                lastUpdated = Clock.UtcNow;
+            }
             var updatedDateTime = new DateTimeOffset(lastUpdated.Value.DateTime.TruncateToMillisecond(), lastUpdated.Value.Offset);
EOF
@@ -53,2 +53,6 @@
var lastUpdated = lastUpdatedIsNull ? Clock.UtcNow : resource.Meta.LastUpdated;
if (lastUpdated == null)
{
lastUpdated = Clock.UtcNow;
}
var updatedDateTime = new DateTimeOffset(lastUpdated.Value.DateTime.TruncateToMillisecond(), lastUpdated.Value.Offset);
Copilot is powered by AI and may make mistakes. Always verify output.
resource.Meta.LastUpdated = updatedDateTime;

if (!lastUpdatedIsNull && updatedDateTime > Clock.UtcNow.AddSeconds(10)) // 5 sec is the max for the computers in the domain
{
throw new NotSupportedException("LastUpdated in the resource cannot be in the future.");
}
Expand All @@ -59,36 +66,35 @@
keepVersion = false;
}

var resourceElement = resource.ToResourceElement();
// Returns true if the extension was removed, false if it was not present.
var isDeleted = resource.Meta.RemoveExtension(KnownFhirPaths.AzureSoftDeletedExtensionUrl);

var isDeleted = resourceElement.IsSoftDeleted();
var resourceElement = resource
.ToResourceElement(ModelInfoProvider.StructureDefinitionSummaryProvider);

if (isDeleted)
{
resource.Meta.RemoveExtension(KnownFhirPaths.AzureSoftDeletedExtensionUrl);
}
CheckConditionalReferenceInResourceJsonNode(resourceElement, importMode);

var resourceWapper = _resourceFactory.Create(resourceElement, isDeleted, true, keepVersion);

return new ImportResource(index, offset, length, !lastUpdatedIsNull, keepVersion, isDeleted, resourceWapper);
}

private static void CheckConditionalReferenceInResource(Resource resource, ImportMode importMode)
private static void CheckConditionalReferenceInResourceJsonNode(ResourceElement resource, ImportMode importMode)
{
if (importMode == ImportMode.IncrementalLoad)
{
return;
}

IEnumerable<ResourceReference> references = resource.GetAllChildren<ResourceReference>();
foreach (ResourceReference reference in references)
IEnumerable<(string Path, string ReferenceValue)> references = resource.Instance.GetReferenceValues();
foreach (var reference in references)
{
if (string.IsNullOrWhiteSpace(reference.Reference))
if (string.IsNullOrWhiteSpace(reference.ReferenceValue))
{
continue;
}

if (reference.Reference.Contains('?', StringComparison.Ordinal))
if (reference.ReferenceValue.Contains('?', StringComparison.Ordinal))
{
throw new NotSupportedException($"Conditional reference is not supported for $import in {ImportMode.InitialLoad}.");
}
Expand Down
36 changes: 36 additions & 0 deletions src/Microsoft.Health.Fhir.Shared.Core/Features/Persistence/RawResourceFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Hl7.Fhir.Serialization;
using Microsoft.Health.Fhir.Core.Extensions;
using Microsoft.Health.Fhir.Core.Models;
using Microsoft.Health.Fhir.SourceNodeSerialization;

namespace Microsoft.Health.Fhir.Core.Features.Persistence
{
Expand All @@ -35,6 +36,11 @@ public RawResource Create(ResourceElement resource, bool keepMeta, bool keepVers
{
EnsureArg.IsNotNull(resource, nameof(resource));

if (resource.IsJsonNode)
{
return CreateJsonNode(resource, keepMeta, keepVersion);
}

var poco = resource.ToPoco<Resource>();

poco.Meta = poco.Meta ?? new Meta();
Expand Down Expand Up @@ -63,5 +69,35 @@ public RawResource Create(ResourceElement resource, bool keepMeta, bool keepVers
}
}
}

private static RawResource CreateJsonNode(ResourceElement resource, bool keepMeta, bool keepVersion)
{
var poco = resource.ToJsonNode();

var versionId = poco.Meta.VersionId;

try
{
// Clear meta version if keepMeta is false since this is set based on generated values when saving the resource
if (!keepMeta)
{
poco.Meta.VersionId = null;
}
else if (!keepVersion)
{
// Assume it's 1, though it may get changed by the database.
poco.Meta.VersionId = "1";
}

return new RawResource(poco.SerializeToString(), FhirResourceFormat.Json, keepMeta);
}
finally
{
if (!keepMeta)
{
poco.Meta.VersionId = versionId;
}
}
}
}
}
3 changes: 3 additions & 0 deletions src/Microsoft.Health.Fhir.Shared.Core/VersionSpecificModelInfoProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using EnsureThat;
using Hl7.Fhir.ElementModel;
using Hl7.Fhir.FhirPath;
using Hl7.Fhir.Introspection;
using Hl7.Fhir.Model;
using Hl7.Fhir.Serialization;
using Hl7.Fhir.Specification;
Expand All @@ -30,6 +31,8 @@ public partial class VersionSpecificModelInfoProvider : IModelInfoProvider

public IStructureDefinitionSummaryProvider StructureDefinitionSummaryProvider { get; } = new PocoStructureDefinitionSummaryProvider();

public ModelInspector ModelInspector => ModelInfo.ModelInspector;

public string GetFhirTypeNameForType(Type type)
{
return ModelInfo.GetFhirTypeNameForType(type);
Expand Down
Loading
Loading