Add CommunityToolkit.DataIngestion.VisionLMOcr (IOcrClient over vision IChatClient) (draft)

.github/workflows/medi.yml

@@ -0,0 +1,64 @@
+name: MEDI
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'MEDI/**'
+      - 'Shared/**'
+      - 'eng/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/medi.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'MEDI/**'
+      - 'Shared/**'
+      - 'eng/**'
+      - 'Directory.Build.props'
+      - 'Directory.Packages.props'
+      - 'global.json'
+      - '.github/workflows/medi.yml'
+
+env:
+  DOTNET_NOLOGO: true
+  DOTNET_CLI_TELEMETRY_OPTOUT: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Build
+        run: dotnet build MEDI/MEDI.slnf
+
+  unit-tests:
+    name: 'Unit: ${{ matrix.package }} (${{ matrix.os }})'
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        package:
+          - PdfPig.OnnxLayoutAnalysis
+          - PdfPig.DataIngestion
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      - name: Test
+        run: dotnet test MEDI/test/${{ matrix.package }}.UnitTests

CommunityToolkit.AI.slnx

@@ -6,6 +6,17 @@
     <File Path="Directory.Build.props" />
     <File Path="Directory.Packages.props" />
   </Folder>
+  <Folder Name="/MEDI/src/">
+    <File Path="MEDI/src/Directory.Build.props" />
+    <Project Path="MEDI/src/PdfPig.OnnxLayoutAnalysis/PdfPig.OnnxLayoutAnalysis.csproj" />
+    <Project Path="MEDI/src/PdfPig.DataIngestion/PdfPig.DataIngestion.csproj" />
+    <Project Path="MEDI/src/DataIngestion.VisionLMOcr/DataIngestion.VisionLMOcr.csproj" />
+  </Folder>
+  <Folder Name="/MEDI/test/">
+    <Project Path="MEDI/test/PdfPig.OnnxLayoutAnalysis.UnitTests/PdfPig.OnnxLayoutAnalysis.UnitTests.csproj" />
+    <Project Path="MEDI/test/PdfPig.DataIngestion.UnitTests/PdfPig.DataIngestion.UnitTests.csproj" />
+    <Project Path="MEDI/test/DataIngestion.VisionLMOcr.UnitTests/DataIngestion.VisionLMOcr.UnitTests.csproj" />
+  </Folder>
   <Folder Name="/MEVD/src/">
     <File Path="MEVD/src/Directory.Build.props" />
     <Project Path="MEVD/src/AzureAISearch/AzureAISearch.csproj" />

Directory.Packages.props

@@ -22,6 +22,21 @@
     <PackageVersion Include="sqlite-vec" Version="0.1.7-alpha.2.1" />
   </ItemGroup>
 
+  <ItemGroup Label="MEDI">
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion" Version="10.4.0-preview.1.26160.2" />
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion.Abstractions" Version="10.4.0-preview.1.26160.2" />
+    <PackageVersion Include="Microsoft.Extensions.DataRetrieval" Version="10.5.0-dev" />
+    <PackageVersion Include="Microsoft.Extensions.DataRetrieval.Abstractions" Version="10.5.0-dev" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.6" />
+    <PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.6" />
+    <PackageVersion Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.22.0" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
+    <PackageVersion Include="SkiaSharp" Version="3.119.0" />
+    <PackageVersion Include="UglyToad.PdfPig" Version="0.1.15" />
+    <PackageVersion Include="UglyToad.PdfPig.Core" Version="0.1.15" />
+    <PackageVersion Include="UglyToad.PdfPig.DocumentLayoutAnalysis" Version="0.1.15" />
+  </ItemGroup>
+
   <ItemGroup Label="Test">
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.4.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.6" />
@@ -36,6 +51,8 @@
     <PackageVersion Include="coverlet.collector" Version="8.0.1" />
     <PackageVersion Include="Humanizer" Version="3.0.10" />
     <PackageVersion Include="Moq" Version="4.20.72" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="1.0.1" />
+    <PackageVersion Include="Xunit.SkippableFact" Version="1.4.13" />
     <PackageVersion Include="Testcontainers" Version="4.11.0" />
     <PackageVersion Include="Testcontainers.PostgreSql" Version="4.11.0" />
     <PackageVersion Include="Testcontainers.Qdrant" Version="4.11.0" />

MEDI/MEDI.slnf

@@ -0,0 +1,15 @@
+{
+  "solution": {
+    "path": "../CommunityToolkit.AI.slnx",
+    "projects":
+    [
+      "MEDI/src/PdfPig.OnnxLayoutAnalysis/PdfPig.OnnxLayoutAnalysis.csproj",
+      "MEDI/src/PdfPig.DataIngestion/PdfPig.DataIngestion.csproj",
+      "MEDI/src/DataRetrieval/DataRetrieval.csproj",
+      "MEDI/src/DataRetrieval.OnnxReranker/DataRetrieval.OnnxReranker.csproj",
+
+      "MEDI/test/PdfPig.OnnxLayoutAnalysis.UnitTests/PdfPig.OnnxLayoutAnalysis.UnitTests.csproj",
+      "MEDI/test/PdfPig.DataIngestion.UnitTests/PdfPig.DataIngestion.UnitTests.csproj"
+    ]
+  }
+}

MEDI/src/DataIngestion.VisionLMOcr/DataIngestion.VisionLMOcr.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Version>1.0.0-preview.1</Version>
+    <AssemblyName>CommunityToolkit.DataIngestion.VisionLMOcr</AssemblyName>
+    <RootNamespace>$(AssemblyName)</RootNamespace>
+    <TargetFrameworks>net10.0;net8.0</TargetFrameworks>
+
+    <Title>Vision LLM OCR client for DataIngestion pipelines</Title>
+    <Description>Provider-neutral IOcrClient implementation backed by a vision-capable Microsoft.Extensions.AI.IChatClient.</Description>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <InternalsVisibleTo Include="CommunityToolkit.DataIngestion.VisionLMOcr.UnitTests" />
+    <InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
+  </ItemGroup>
+</Project>

MEDI/src/DataIngestion.VisionLMOcr/Ocr/OcrAbstractions.cs

@@ -0,0 +1,140 @@
+#pragma warning disable CS1591
+// VENDORED copy of the unpublished dotnet/extensions #7588 IOcrClient shape; delete and reference Microsoft.Extensions.DataIngestion.Abstractions when #7588 ships.
+using Microsoft.Extensions.AI;
+
+namespace CommunityToolkit.DataIngestion.VisionLMOcr;
+
+/// <summary>Provider-neutral OCR / document-extraction capability.</summary>
+public interface IOcrClient
+{
+    Task<OcrResult> GetTextAsync(
+        Stream document,
+        string mediaType,
+        OcrOptions? options = null,
+        IProgress<OcrProgress>? progress = null,
+        CancellationToken cancellationToken = default);
+
+    object? GetService(Type serviceType, object? serviceKey = null);
+}
+
+public sealed class OcrResult
+{
+    public OcrResult(IReadOnlyList<OcrPage> pages) => Pages = pages;
+
+    public IReadOnlyList<OcrPage> Pages { get; }
+    public string Markdown => string.Join("\n\n", Pages.Select(p => p.Markdown));
+    public string? OcrSource { get; init; }
+    public string? ModelId { get; init; }
+    public OcrUsage? Usage { get; init; }
+    public object? RawRepresentation { get; init; }
+    public AdditionalPropertiesDictionary? AdditionalProperties { get; init; }
+}
+
+public sealed class OcrPage
+{
+    public OcrPage(int index, string markdown)
+    {
+        Index = index;
+        Markdown = markdown;
+    }
+
+    public int Index { get; }
+    public string Markdown { get; }
+    public IReadOnlyList<OcrBlock> Blocks { get; init; } = [];
+    public IReadOnlyList<OcrTable> Tables { get; init; } = [];
+    public double? Confidence { get; init; }
+    public AdditionalPropertiesDictionary? AdditionalProperties { get; init; }
+}
+
+public sealed class OcrBlock
+{
+    public OcrBlock(string text) => Text = text;
+    public string Text { get; }
+    public string? Kind { get; init; }
+    public OcrBoundingRegion? BoundingRegion { get; init; }
+    public double? Confidence { get; init; }
+}
+
+public sealed class OcrTable
+{
+    public OcrTable(int rowCount, int columnCount, OcrTableCell[,]? cells, string markdownRepresentation)
+    {
+        RowCount = rowCount;
+        ColumnCount = columnCount;
+        Cells = cells;
+        MarkdownRepresentation = markdownRepresentation;
+    }
+
+    public int RowCount { get; }
+    public int ColumnCount { get; }
+    public OcrTableCell[,]? Cells { get; }
+    public string MarkdownRepresentation { get; }
+    public OcrBoundingRegion? BoundingRegion { get; init; }
+    public double? Confidence { get; init; }
+
+    public string ToMarkdown() => MarkdownRepresentation;
+}
+
+public sealed class OcrTableCell
+{
+    public OcrTableCell(string text) => Text = text;
+    public string Text { get; }
+    public int RowIndex { get; init; }
+    public int ColumnIndex { get; init; }
+}
+
+/// <summary>Shared, provider-neutral geometry primitive (clockwise polygon).</summary>
+public sealed class OcrBoundingRegion
+{
+    public OcrBoundingRegion(int pageNumber, IReadOnlyList<float> polygon)
+    {
+        PageNumber = pageNumber;
+        Polygon = polygon;
+    }
+
+    public int PageNumber { get; }
+    public IReadOnlyList<float> Polygon { get; }
+
+    public static OcrBoundingRegion FromRectangle(int pageNumber, double left, double top, double right, double bottom)
+        => new(pageNumber,
+        [
+            (float)left, (float)top,
+            (float)right, (float)top,
+            (float)right, (float)bottom,
+            (float)left, (float)bottom,
+        ]);
+
+    /// <summary>Axis-aligned bounds.</summary>
+    public (float Left, float Top, float Right, float Bottom) GetBounds()
+    {
+        float minX = float.MaxValue, minY = float.MaxValue, maxX = float.MinValue, maxY = float.MinValue;
+        for (int i = 0; i + 1 < Polygon.Count; i += 2)
+        {
+            minX = Math.Min(minX, Polygon[i]);
+            maxX = Math.Max(maxX, Polygon[i]);
+            minY = Math.Min(minY, Polygon[i + 1]);
+            maxY = Math.Max(maxY, Polygon[i + 1]);
+        }
+        return (minX, minY, maxX, maxY);
+    }
+}
+
+public sealed class OcrProgress
+{
+    public int? PagesProcessed { get; init; }
+    public int? TotalPages { get; init; }
+    public string? Status { get; init; }
+}
+
+public sealed class OcrUsage
+{
+    public int? PagesProcessed { get; init; }
+    public AdditionalPropertiesDictionary? AdditionalProperties { get; init; }
+}
+
+public sealed class OcrOptions
+{
+    public string? ModelId { get; set; }
+    public bool IncludeImages { get; set; }
+    public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
+}

MEDI/src/DataIngestion.VisionLMOcr/VisionLMOcrClient.cs

@@ -0,0 +1,86 @@
+using Microsoft.Extensions.AI;
+
+namespace CommunityToolkit.DataIngestion.VisionLMOcr;
+
+/// <summary>
+/// Implements <see cref="IOcrClient"/> by sending image bytes to a vision-capable <see cref="IChatClient"/>.
+/// </summary>
+public sealed class VisionLMOcrClient : IOcrClient
+{
+    private readonly IChatClient chatClient;
+    private readonly VisionLMOcrOptions options;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="VisionLMOcrClient"/> class.
+    /// </summary>
+    /// <param name="chatClient">The vision-capable chat client used for OCR.</param>
+    /// <param name="options">The default OCR options.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="chatClient"/> is <see langword="null"/>.</exception>
+    public VisionLMOcrClient(IChatClient chatClient, VisionLMOcrOptions? options = null)
+    {
+        this.chatClient = chatClient ?? throw new ArgumentNullException(nameof(chatClient));
+        this.options = options ?? new VisionLMOcrOptions();
+    }
+
+    /// <inheritdoc />
+    public async Task<OcrResult> GetTextAsync(
+        Stream document,
+        string mediaType,
+        OcrOptions? options = null,
+        IProgress<OcrProgress>? progress = null,
+        CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(document);
+        ArgumentException.ThrowIfNullOrWhiteSpace(mediaType);
+
+        using var buffer = new MemoryStream();
+        await document.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
+        var imageBytes = buffer.ToArray();
+
+        var systemPrompt = this.options.SystemPrompt ?? VisionLMOcrPrompts.DefaultSystemPrompt;
+        var userPrompt = this.options.UserPrompt ?? VisionLMOcrPrompts.DefaultUserPrompt;
+
+        ChatMessage[] messages =
+        [
+            new(ChatRole.System, systemPrompt),
+            new(ChatRole.User, (IList<AIContent>)
+            [
+                new DataContent(imageBytes, mediaType),
+                new TextContent(userPrompt),
+            ]),
+        ];
+
+        var response = await chatClient.GetResponseAsync(
+            messages,
+            cancellationToken: cancellationToken).ConfigureAwait(false);
+
+        var text = response.Text ?? string.Empty;
+        progress?.Report(new OcrProgress
+        {
+            PagesProcessed = 1,
+            TotalPages = 1,
+            Status = "Completed",
+        });
+
+        return new OcrResult([new OcrPage(0, text)])
+        {
+            OcrSource = "vision_lm",
+            ModelId = options?.ModelId ?? this.options.ModelId,
+            RawRepresentation = response,
+            Usage = new OcrUsage { PagesProcessed = 1 },
+        };
+    }
+
+    /// <inheritdoc />
+    public object? GetService(Type serviceType, object? serviceKey = null)
+    {
+        ArgumentNullException.ThrowIfNull(serviceType);
+
+        if (serviceType == typeof(IOcrClient) || serviceType == typeof(VisionLMOcrClient))
+        {
+            return this;
+        }
+
+        return chatClient.GetService(serviceType, serviceKey);
+    }
+}