feat: Add Document Processing packages (MEDI)

.github/workflows/medi.yml

@@ -0,0 +1,64 @@
+name: MEDI
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'MEDI/**'
+      - 'Shared/**'
+      - 'eng/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/medi.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'MEDI/**'
+      - 'Shared/**'
+      - 'eng/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/medi.yml'
+
+env:
+  DOTNET_NOLOGO: true
+  DOTNET_CLI_TELEMETRY_OPTOUT: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Build
+        run: dotnet build MEDI/MEDI.slnf
+
+  unit-tests:
+    name: 'Unit: ${{ matrix.package }} (${{ matrix.os }})'
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        package:
+          - PdfPig.OnnxLayoutAnalysis
+          - PdfPig.DataIngestion
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Test
+        run: dotnet test MEDI/test/${{ matrix.package }}.UnitTests

CommunityToolkit.AI.slnx

@@ -6,6 +6,15 @@
     <File Path="Directory.Build.props" />
     <File Path="Directory.Packages.props" />
   </Folder>
+  <Folder Name="/MEDI/src/">
+    <File Path="MEDI/src/Directory.Build.props" />
+    <Project Path="MEDI/src/PdfPig.OnnxLayoutAnalysis/PdfPig.OnnxLayoutAnalysis.csproj" />
+    <Project Path="MEDI/src/PdfPig.DataIngestion/PdfPig.DataIngestion.csproj" />
+  </Folder>
+  <Folder Name="/MEDI/test/">
+    <Project Path="MEDI/test/PdfPig.OnnxLayoutAnalysis.UnitTests/PdfPig.OnnxLayoutAnalysis.UnitTests.csproj" />
+    <Project Path="MEDI/test/PdfPig.DataIngestion.UnitTests/PdfPig.DataIngestion.UnitTests.csproj" />
+  </Folder>
   <Folder Name="/MEVD/src/">
     <File Path="MEVD/src/Directory.Build.props" />
     <Project Path="MEVD/src/AzureAISearch/AzureAISearch.csproj" />

Directory.Packages.props

@@ -3,8 +3,8 @@
     <PackageVersion Include="System.Linq.AsyncEnumerable" Version="10.0.6" />
     <PackageVersion Include="System.Numerics.Tensors" Version="10.0.6" />
     <PackageVersion Include="System.Text.Json" Version="10.0.6" />
-    <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.5.2" />
-    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.6" />
+    <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.7.0" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.9" />
   </ItemGroup>
 
   <ItemGroup Label="MEVD">
@@ -22,6 +22,15 @@
     <PackageVersion Include="sqlite-vec" Version="0.1.7-alpha.2.1" />
   </ItemGroup>
 
+  <ItemGroup Label="MEDI">
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion" Version="10.7.0-preview.1.26309.5" />
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion.Abstractions" Version="10.7.0-preview.1.26309.5" />
+    <PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.6" />
+    <PackageVersion Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.22.0" />
+    <PackageVersion Include="SkiaSharp" Version="3.119.0" />
+    <PackageVersion Include="PdfPig" Version="0.1.14" />
+  </ItemGroup>
+
   <ItemGroup Label="Build">
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.102" />
   </ItemGroup>
@@ -40,6 +49,8 @@
     <PackageVersion Include="coverlet.collector" Version="8.0.1" />
     <PackageVersion Include="Humanizer" Version="3.0.10" />
     <PackageVersion Include="Moq" Version="4.20.72" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="1.0.1" />
+    <PackageVersion Include="Xunit.SkippableFact" Version="1.4.13" />
     <PackageVersion Include="Testcontainers" Version="4.11.0" />
     <PackageVersion Include="Testcontainers.PostgreSql" Version="4.11.0" />
     <PackageVersion Include="Testcontainers.Qdrant" Version="4.11.0" />

MEDI/MEDI.slnf

@@ -0,0 +1,13 @@
+{
+  "solution": {
+    "path": "../CommunityToolkit.AI.slnx",
+    "projects":
+    [
+      "MEDI/src/PdfPig.OnnxLayoutAnalysis/PdfPig.OnnxLayoutAnalysis.csproj",
+      "MEDI/src/PdfPig.DataIngestion/PdfPig.DataIngestion.csproj",
+
+      "MEDI/test/PdfPig.OnnxLayoutAnalysis.UnitTests/PdfPig.OnnxLayoutAnalysis.UnitTests.csproj",
+      "MEDI/test/PdfPig.DataIngestion.UnitTests/PdfPig.DataIngestion.UnitTests.csproj"
+    ]
+  }
+}

MEDI/src/Directory.Build.props

@@ -0,0 +1,13 @@
+<Project>
+
+  <Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
+
+  <PropertyGroup>
+    <InjectIsExternalInitOnLegacy>true</InjectIsExternalInitOnLegacy>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="$(MSBuildThisFileDirectory)../../Shared/Throw/Throw.cs" LinkBase="Shared" />
+  </ItemGroup>
+
+</Project>

MEDI/src/PdfPig.DataIngestion/MetadataAwareSectionChunker.cs

@@ -0,0 +1,120 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DataIngestion;
+using Microsoft.Extensions.DataIngestion.Chunkers;
+
+namespace CommunityToolkit.DocumentProcessing.PdfPig.DataIngestion;
+
+/// <summary>
+/// A chunker that wraps <see cref="SectionChunker"/> and propagates element metadata
+/// (such as "element_type") to the resulting chunks. This works around the fact that
+/// the built-in MEDI chunkers do not copy element metadata to chunks.
+/// </summary>
+/// <remarks>
+/// After the inner chunker produces chunks, this class matches each chunk's content
+/// back to its source elements via substring containment and copies metadata from
+/// the matching elements to the chunk. For keys with conflicting values across
+/// multiple matching elements, the value from the first match wins.
+/// </remarks>
+public sealed class MetadataAwareSectionChunker : IngestionChunker<string>
+{
+    private readonly SectionChunker _inner;
+
+    /// <summary>
+    /// Creates a new <see cref="MetadataAwareSectionChunker"/>.
+    /// </summary>
+    /// <param name="options">The chunker options (tokenizer, max tokens, overlap).</param>
+    public MetadataAwareSectionChunker(IngestionChunkerOptions options)
+    {
+        _inner = new SectionChunker(options);
+    }
+
+    /// <inheritdoc/>
+    public override async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(
+        IngestionDocument document,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        var elementIndex = BuildElementIndex(document);
+
+        await foreach (var chunk in _inner.ProcessAsync(document, cancellationToken).ConfigureAwait(false))
+        {
+            ResolveMetadata(chunk, elementIndex);
+            yield return chunk;
+        }
+    }
+
+    /// <summary>
+    /// Builds an index of element text snippets to their metadata.
+    /// Only includes elements that actually have metadata set.
+    /// </summary>
+    private static List<(string Text, IDictionary<string, object?> Metadata)> BuildElementIndex(
+        IngestionDocument document)
+    {
+        var index = new List<(string Text, IDictionary<string, object?> Metadata)>();
+
+        foreach (var section in document.Sections)
+        {
+            foreach (var element in section.Elements)
+            {
+                if (!element.HasMetadata)
+                {
+                    continue;
+                }
+
+                // Use the element's text content for matching.
+                // SectionChunker concatenates element GetMarkdown() values with newlines,
+                // so we match against GetMarkdown() for reliable substring containment.
+                var text = element.GetMarkdown();
+                if (string.IsNullOrWhiteSpace(text))
+                {
+                    continue;
+                }
+
+                index.Add((text, element.Metadata));
+            }
+        }
+
+        return index;
+    }
+
+    /// <summary>
+    /// Matches a chunk's content back to source elements and copies their metadata.
+    /// For keys that appear in multiple matching elements, the first match's value wins.
+    /// </summary>
+    private static void ResolveMetadata(
+        IngestionChunk<string> chunk,
+        List<(string Text, IDictionary<string, object?> Metadata)> elementIndex)
+    {
+        var content = chunk.Content;
+        if (string.IsNullOrEmpty(content))
+        {
+            return;
+        }
+
+        foreach (var (text, metadata) in elementIndex)
+        {
+            if (!content.Contains(text, StringComparison.Ordinal))
+            {
+                continue;
+            }
+
+            foreach (var kvp in metadata)
+            {
+                // Skip bounding box metadata — it's per-element, not meaningful at chunk level
+                if (kvp.Key.StartsWith("BoundingBox.", StringComparison.Ordinal))
+                {
+                    continue;
+                }
+
+                // First match wins for each key
+                if (kvp.Value is not null && !chunk.Metadata.ContainsKey(kvp.Key))
+                {
+                    chunk.Metadata[kvp.Key] = kvp.Value;
+                }
+            }
+        }
+    }
+}