Add CommunityToolkit.DataIngestion package with LLM chunk processors

.github/workflows/medi.yml

@@ -0,0 +1,64 @@
+name: MEDI
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'MEDI/**'
+      - 'Shared/**'
+      - 'eng/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/medi.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'MEDI/**'
+      - 'Shared/**'
+      - 'eng/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/medi.yml'
+
+env:
+  DOTNET_NOLOGO: true
+  DOTNET_CLI_TELEMETRY_OPTOUT: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Build
+        run: dotnet build MEDI/MEDI.slnf
+
+  unit-tests:
+    name: 'Unit: ${{ matrix.package }} (${{ matrix.os }})'
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        package:
+          - PdfPig.OnnxLayoutAnalysis
+          - PdfPig.DataIngestion
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Test
+        run: dotnet test MEDI/test/${{ matrix.package }}.UnitTests

CommunityToolkit.AI.slnx

@@ -6,6 +6,15 @@
     <File Path="Directory.Build.props" />
     <File Path="Directory.Packages.props" />
   </Folder>
+  <Folder Name="/MEDI/src/">
+    <File Path="MEDI/src/Directory.Build.props" />
+    <Project Path="MEDI/src/PdfPig.OnnxLayoutAnalysis/PdfPig.OnnxLayoutAnalysis.csproj" />
+    <Project Path="MEDI/src/PdfPig.DataIngestion/PdfPig.DataIngestion.csproj" />
+  </Folder>
+  <Folder Name="/MEDI/test/">
+    <Project Path="MEDI/test/PdfPig.OnnxLayoutAnalysis.UnitTests/PdfPig.OnnxLayoutAnalysis.UnitTests.csproj" />
+    <Project Path="MEDI/test/PdfPig.DataIngestion.UnitTests/PdfPig.DataIngestion.UnitTests.csproj" />
+  </Folder>
   <Folder Name="/MEVD/src/">
     <File Path="MEVD/src/Directory.Build.props" />
     <Project Path="MEVD/src/AzureAISearch/AzureAISearch.csproj" />

Directory.Packages.props

@@ -22,6 +22,21 @@
     <PackageVersion Include="sqlite-vec" Version="0.1.7-alpha.2.1" />
   </ItemGroup>
 
+  <ItemGroup Label="MEDI">
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion" Version="10.5.0-preview.1.26213.4" />
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion.Abstractions" Version="10.5.0-preview.1.26213.4" />
+    <PackageVersion Include="Microsoft.Extensions.DataRetrieval" Version="10.5.0-dev" />
+    <PackageVersion Include="Microsoft.Extensions.DataRetrieval.Abstractions" Version="10.5.0-dev" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.6" />
+    <PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.6" />
+    <PackageVersion Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.22.0" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
+    <PackageVersion Include="SkiaSharp" Version="3.119.0" />
+    <PackageVersion Include="UglyToad.PdfPig" Version="0.1.15" />
+    <PackageVersion Include="UglyToad.PdfPig.Core" Version="0.1.15" />
+    <PackageVersion Include="UglyToad.PdfPig.DocumentLayoutAnalysis" Version="0.1.15" />
+  </ItemGroup>
+
   <ItemGroup Label="Test">
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.4.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.6" />
@@ -36,6 +51,8 @@
     <PackageVersion Include="coverlet.collector" Version="8.0.1" />
     <PackageVersion Include="Humanizer" Version="3.0.10" />
     <PackageVersion Include="Moq" Version="4.20.72" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="1.0.1" />
+    <PackageVersion Include="Xunit.SkippableFact" Version="1.4.13" />
     <PackageVersion Include="Testcontainers" Version="4.11.0" />
     <PackageVersion Include="Testcontainers.PostgreSql" Version="4.11.0" />
     <PackageVersion Include="Testcontainers.Qdrant" Version="4.11.0" />

MEDI/MEDI.slnf

@@ -0,0 +1,15 @@
+{
+  "solution": {
+    "path": "../CommunityToolkit.AI.slnx",
+    "projects":
+    [
+      "MEDI/src/PdfPig.OnnxLayoutAnalysis/PdfPig.OnnxLayoutAnalysis.csproj",
+      "MEDI/src/PdfPig.DataIngestion/PdfPig.DataIngestion.csproj",
+      "MEDI/src/DataRetrieval/DataRetrieval.csproj",
+      "MEDI/src/DataRetrieval.OnnxReranker/DataRetrieval.OnnxReranker.csproj",
+
+      "MEDI/test/PdfPig.OnnxLayoutAnalysis.UnitTests/PdfPig.OnnxLayoutAnalysis.UnitTests.csproj",
+      "MEDI/test/PdfPig.DataIngestion.UnitTests/PdfPig.DataIngestion.UnitTests.csproj"
+    ]
+  }
+}

MEDI/src/DataIngestion/ContextualChunkEnricher.cs

@@ -0,0 +1,52 @@
+using System.Runtime.CompilerServices;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.DataIngestion;
+
+namespace CommunityToolkit.DataIngestion;
+
+/// <summary>
+/// An <see cref="IngestionChunkProcessor{T}"/> that enriches text chunks with contextual
+/// summaries for improved RAG retrieval.
+/// </summary>
+public sealed class ContextualChunkEnricher : IngestionChunkProcessor<string>
+{
+    private readonly IChatClient _chatClient;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ContextualChunkEnricher"/> class.
+    /// </summary>
+    /// <param name="chatClient">The chat client used to generate contextual summaries.</param>
+    public ContextualChunkEnricher(IChatClient chatClient)
+    {
+        _chatClient = chatClient ?? throw new ArgumentNullException(nameof(chatClient));
+    }
+
+    /// <inheritdoc/>
+    public override async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(
+        IAsyncEnumerable<IngestionChunk<string>> chunks,
+        [EnumeratorCancellation] CancellationToken ct = default)
+    {
+        await foreach (var chunk in chunks.WithCancellation(ct))
+        {
+            var prompt = GetPromptForChunk(chunk);
+            var messages = new[] { new ChatMessage(ChatRole.User, prompt) };
+            var response = await _chatClient.GetResponseAsync(messages, cancellationToken: ct);
+
+            chunk.Metadata[MetadataKeys.ContextualSummary] = response.Text;
+            yield return chunk;
+        }
+    }
+
+    private static string GetPromptForChunk(IngestionChunk<string> chunk)
+    {
+        var elementType = chunk.Metadata.TryGetValue("element_type", out var t) ? t as string : null;
+        var instruction = elementType switch
+        {
+            "table" => "Summarize this table for search retrieval. Describe what data it contains, including key metrics, column headers, and notable values. Output only the summary sentence, nothing else.",
+            "picture" or "caption" => "Summarize what this figure or image refers to for search retrieval. Output only the summary sentence, nothing else.",
+            _ => "Provide a single concise sentence summarizing the following text for use in search retrieval. Output only the summary sentence, nothing else."
+        };
+
+        return instruction + "\n\n" + chunk.Content;
+    }
+}

MEDI/src/DataIngestion/DataIngestion.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Version>1.0.0-preview.1</Version>
+    <AssemblyName>CommunityToolkit.DataIngestion</AssemblyName>
+    <RootNamespace>$(AssemblyName)</RootNamespace>
+    <TargetFrameworks>net10.0;net8.0</TargetFrameworks>
+
+    <Title>LLM-powered chunk processors for Microsoft.Extensions.DataIngestion pipelines</Title>
+    <Description>LLM-powered ingestion chunk processors including contextual chunk enrichment, entity extraction, topic classification, hypothetical query generation (reverse HyDE), and RAPTOR-style tree indexing for advanced RAG applications.</Description>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.DataIngestion" />
+    <PackageReference Include="Microsoft.Extensions.DataIngestion.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <InternalsVisibleTo Include="CommunityToolkit.DataIngestion.UnitTests" />
+    <InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
+  </ItemGroup>
+</Project>

MEDI/src/DataIngestion/DataIngestionServiceCollectionExtensions.cs

@@ -0,0 +1,39 @@
+using Microsoft.Extensions.DataIngestion;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace CommunityToolkit.DataIngestion;
+
+/// <summary>
+/// Extension methods for registering ingestion pipeline builders with dependency injection.
+/// </summary>
+public static class DataIngestionServiceCollectionExtensions
+{
+    /// <summary>
+    /// Registers an <see cref="IngestionPipelineBuilder{T}"/> singleton for composing
+    /// ingestion pipelines with LLM-powered chunk processors.
+    /// </summary>
+    /// <example>
+    /// <code>
+    /// builder.Services.AddIngestionPipeline&lt;string&gt;()
+    ///     .UseEntityExtraction()
+    ///     .UseTopicClassification(o => o.Taxonomy = ["web", "data", "security"])
+    ///     .UseHypotheticalQueries(o => o.QuestionsPerChunk = 5)
+    ///     .UseContextualChunkEnrichment()
+    ///     .UseTreeIndex();
+    /// </code>
+    /// </example>
+    public static IngestionPipelineBuilder<T> AddIngestionPipeline<T>(
+        this IServiceCollection services)
+    {
+        var pipelineBuilder = new IngestionPipelineBuilder<T>();
+        services.AddSingleton(pipelineBuilder);
+        return pipelineBuilder;
+    }
+
+    /// <summary>
+    /// Registers an <see cref="IngestionPipelineBuilder{T}"/> with <c>string</c> content type (the common case).
+    /// </summary>
+    public static IngestionPipelineBuilder<string> AddIngestionPipeline(
+        this IServiceCollection services)
+        => services.AddIngestionPipeline<string>();
+}

MEDI/src/DataIngestion/EntityExtractionProcessor.cs

@@ -0,0 +1,105 @@
+using System.Runtime.CompilerServices;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.DataIngestion;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace CommunityToolkit.DataIngestion;
+
+/// <summary>
+/// An <see cref="IngestionChunkProcessor{T}"/> that uses an LLM to extract named entities
+/// (people, organizations, technologies, versions) from each chunk and stores them as
+/// metadata for downstream filtered vector search.
+/// </summary>
+public sealed class EntityExtractionProcessor : IngestionChunkProcessor<string>
+{
+    private readonly IChatClient _chatClient;
+    private readonly ILogger _logger;
+
+    private static readonly string[] EntityKeys =
+        [MetadataKeys.EntitiesPeople, MetadataKeys.EntitiesOrganizations, MetadataKeys.EntitiesTechnologies, MetadataKeys.EntitiesVersions];
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="EntityExtractionProcessor"/> class.
+    /// </summary>
+    /// <param name="chatClient">The chat client used for entity extraction.</param>
+    /// <param name="logger">Optional logger for diagnostics.</param>
+    public EntityExtractionProcessor(IChatClient chatClient, ILogger<EntityExtractionProcessor>? logger = null)
+    {
+        _chatClient = chatClient ?? throw new ArgumentNullException(nameof(chatClient));
+        _logger = logger ?? NullLogger<EntityExtractionProcessor>.Instance;
+    }
+
+    /// <inheritdoc/>
+    public override async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(
+        IAsyncEnumerable<IngestionChunk<string>> chunks,
+        [EnumeratorCancellation] CancellationToken ct = default)
+    {
+        await foreach (var chunk in chunks.WithCancellation(ct))
+        {
+            try
+            {
+                var prompt = $$"""
+                    Extract named entities from the following text.
+                    Return ONLY valid JSON matching: {"people": ["name"], "organizations": ["org"], "technologies": ["tech"], "versions": ["v1"]}
+                    Use empty arrays [] when none found for a category.
+
+                    Text:
+                    {{chunk.Content}}
+                    """;
+
+                var options = new ChatOptions
+                {
+                    MaxOutputTokens = 300,
+                    ResponseFormat = ChatResponseFormat.Json
+                };
+
+                var response = await _chatClient.GetResponseAsync(prompt, options, ct);
+                var entities = JsonSerializer.Deserialize<EntityResponse>(
+                    response.Text ?? "{}", JsonDefaults.Options);
+
+                if (entities is not null)
+                {
+                    chunk.Metadata[MetadataKeys.EntitiesPeople] = string.Join(", ", entities.People ?? []);
+                    chunk.Metadata[MetadataKeys.EntitiesOrganizations] = string.Join(", ", entities.Organizations ?? []);
+                    chunk.Metadata[MetadataKeys.EntitiesTechnologies] = string.Join(", ", entities.Technologies ?? []);
+                    chunk.Metadata[MetadataKeys.EntitiesVersions] = string.Join(", ", entities.Versions ?? []);
+                }
+                else
+                {
+                    SetEmptyEntities(chunk);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Entity extraction failed for chunk; using empty defaults.");
+                SetEmptyEntities(chunk);
+            }
+
+            yield return chunk;
+        }
+    }
+
+    private static void SetEmptyEntities(IngestionChunk<string> chunk)
+    {
+        foreach (var key in EntityKeys)
+            chunk.Metadata[key] = "";
+    }
+
+    private sealed class EntityResponse
+    {
+        [JsonPropertyName("people")]
+        public string[]? People { get; set; }
+
+        [JsonPropertyName("organizations")]
+        public string[]? Organizations { get; set; }
+
+        [JsonPropertyName("technologies")]
+        public string[]? Technologies { get; set; }
+
+        [JsonPropertyName("versions")]
+        public string[]? Versions { get; set; }
+    }
+}