Skip to content

Commit

Permalink
.Net: Added support for BinaryData for ImageContent. (#5008)
Browse files Browse the repository at this point in the history
Reopend PR. See #4919
for previous PR.
Reopening this PR from another feature branch makes rebasing easier.

### Motivation and Context

As Described in #4781 right now there is no possibility in SK to add
Images as DataUris to ChatCompletion APIs, although the Azure OpenAI API
and the Open AI API both support this.

Fixes #4781

### Description

As per Discussion added overload to the ImageContent ctor that takes
BinaryData.
For backward Compat we kept the ctor that takes an URI.

Also the new ctor throws, if the BinaryData is null, empty or if there
is not MediaType provided.

I thought about allowing plain, non base64 encoded DataUris with
BinaryData.
The Idea was to not encode to base64, if the MediaType is set to
"text/plain", but then I decided, that this is not needed, since `Uri`
in general allows for DataUris like `new
Uri("data:text/plain;http://exmpaledomain.com")` just not for DataUris
that are longer than 65520 bytes. I feel like that is ok, for plain
DataUris.

We can still add this if needed.

Also as per discussion in the issue, I did not add additional overloads
for direct Streams support.

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

---------

Co-authored-by: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com>
  • Loading branch information
dersia and RogerBarreto committed Feb 20, 2024
1 parent abc8e5d commit 08569f5
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 4 deletions.
57 changes: 55 additions & 2 deletions dotnet/src/SemanticKernel.Abstractions/Contents/ImageContent.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ public sealed class ImageContent : KernelContent
/// </summary>
public Uri? Uri { get; set; }

/// <summary>
/// The image binary data.
/// </summary>
public BinaryData? Data { get; }

/// <summary>
/// Initializes a new instance of the <see cref="ImageContent"/> class.
/// </summary>
Expand All @@ -35,9 +40,57 @@ public sealed class ImageContent : KernelContent
this.Uri = uri;
}

/// <inheritdoc/>
/// <summary>
/// Initializes a new instance of the <see cref="ImageContent"/> class.
/// </summary>
/// <param name="data">The Data used as DataUri for the image.</param>
/// <param name="modelId">The model ID used to generate the content</param>
/// <param name="innerContent">Inner content</param>
/// <param name="encoding">Encoding of the text</param>
/// <param name="metadata">Additional metadata</param>
public ImageContent(
BinaryData data,
string? modelId = null,
object? innerContent = null,
Encoding? encoding = null,
IReadOnlyDictionary<string, object?>? metadata = null)
: base(innerContent, modelId, metadata)
{
Verify.NotNull(data, nameof(data));

if (data!.IsEmpty)
{
throw new ArgumentException("Data cannot be empty", nameof(data));
}

if (string.IsNullOrWhiteSpace(data!.MediaType))
{
throw new ArgumentException("MediaType is needed for DataUri Images", nameof(data));
}

this.Data = data;
}

/// <summary>
/// Returns the string representation of the image.
/// BinaryData images will be represented as DataUri
/// Remote Uri images will be represented as is
/// </summary>
/// <remarks>
/// When Data is provided it takes precedence over URI
/// </remarks>
public override string ToString()
{
return this.Uri?.ToString() ?? string.Empty;
return this.BuildDataUri() ?? this.Uri?.ToString() ?? string.Empty;
}

private string? BuildDataUri()
{
if (this.Data is null)
{
return null;
}

return $"data:{this.Data.MediaType};base64,{Convert.ToBase64String(this.Data.ToArray())}";
}
}
100 changes: 98 additions & 2 deletions dotnet/src/SemanticKernel.UnitTests/Contents/ImageContentTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ namespace SemanticKernel.UnitTests.Contents;
public sealed class ImageContentTests
{
[Fact]
public void ToStringReturnsString()
public void ToStringForUriReturnsString()
{
// Arrange
var content1 = new ImageContent(null!);
var content1 = new ImageContent((Uri)null!);
var content2 = new ImageContent(new Uri("https://endpoint/"));

// Act
Expand All @@ -26,4 +26,100 @@ public void ToStringReturnsString()
Assert.Empty(result1);
Assert.Equal("https://endpoint/", result2);
}

[Fact]
public void ToStringForDataUriReturnsDataUriString()
{
// Arrange
var data = BinaryData.FromString("this is a test", "text/plain");
var content1 = new ImageContent(data);

// Act
var result1 = content1.ToString();
var dataUriToExpect = $"data:{data.MediaType};base64,{Convert.ToBase64String(data.ToArray())}";

// Assert
Assert.Equal(dataUriToExpect, result1);
}

[Fact]
public void ToStringForUriAndDataUriReturnsDataUriString()
{
// Arrange
var data = BinaryData.FromString("this is a test", "text/plain");
var content1 = new ImageContent(data);
content1.Uri = new Uri("https://endpoint/");

// Act
var result1 = content1.ToString();
var dataUriToExpect = $"data:{data.MediaType};base64,{Convert.ToBase64String(data.ToArray())}";

// Assert
Assert.Equal(dataUriToExpect, result1);
}

[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public void CreateForWithoutMediaTypeThrows(string? mediaType)
{
// Arrange
var data = BinaryData.FromString("this is a test", mediaType);

// Assert
Assert.Throws<ArgumentException>(() => new ImageContent(data!));
}

[Fact]
public void CreateForNullDataUriThrows()
{
// Assert
Assert.Throws<ArgumentNullException>(() => new ImageContent((BinaryData)null!));
}

[Fact]
public void CreateForEmptyDataUriThrows()
{
// Arrange
var data = BinaryData.Empty;

// Assert
Assert.Throws<ArgumentException>(() => new ImageContent(data));
}

[Fact]
public void ToStringForDataUriFromBytesReturnsDataUriString()
{
// Arrange
var bytes = System.Text.Encoding.UTF8.GetBytes("this is a test");
var data = BinaryData.FromBytes(bytes, "text/plain");
var content1 = new ImageContent(data);

// Act
var result1 = content1.ToString();
var dataUriToExpect = $"data:{data.MediaType};base64,{Convert.ToBase64String(data.ToArray())}";

// Assert
Assert.Equal(dataUriToExpect, result1);
}

[Fact]
public void ToStringForDataUriFromStreamReturnsDataUriString()
{
// Arrange
using var ms = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes("this is a test"));
var data = BinaryData.FromStream(ms, "text/plain");
var content1 = new ImageContent(data);

// Act
var result1 = content1.ToString();
var dataUriToExpect = $"data:{data.MediaType};base64,{Convert.ToBase64String(data.ToArray())}";

// Assert
Assert.Equal(dataUriToExpect, result1);

// Assert throws if mediatype is null
Assert.Throws<ArgumentException>(() => new ImageContent(BinaryData.FromStream(ms, null)));
}
}

0 comments on commit 08569f5

Please sign in to comment.