From 201423b0c1298ffec9aec62c54f0e56568539471 Mon Sep 17 00:00:00 2001
From: aghimir3 <22482815+aghimir3@users.noreply.github.com>
Date: Tue, 13 Aug 2024 20:49:34 -0700
Subject: [PATCH 1/3] Add quality and style to TextToImageRequest
---
.../TextToImage/TextToImageRequest.cs | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs
index 70b5ac5418ee..704f973c29b5 100644
--- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs
+++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs
@@ -39,4 +39,16 @@ internal sealed class TextToImageRequest
///
[JsonPropertyName("response_format")]
public string Format { get; set; } = "url";
+
+ ///
+ /// Image quality, "standard" or "hd"
+ ///
+ [JsonPropertyName("quality")]
+ public string Quality { get; set; } = "standard";
+
+ ///
+ /// Image style, "vivid" or "natural"
+ ///
+ [JsonPropertyName("style")]
+ public string Style { get; set; } = "vivid";
}
From 6895c051be0fc91616d3acdc7a3b5598e563f6e6 Mon Sep 17 00:00:00 2001
From: aghimir3 <22482815+aghimir3@users.noreply.github.com>
Date: Tue, 13 Aug 2024 21:46:23 -0700
Subject: [PATCH 2/3] Implement GetImageContentsAsync in
OpenAITextToImageService
- Added the GetImageContentsAsync method to the OpenAITextToImageService class.
- Implemented validation for input, including width, height, quality, and style settings.
- Supported image sizes include 256x256, 512x512, 1024x1024, 1792x1024, and 1024x1792.
- Added checks for supported qualities ('standard', 'hd') and styles ('vivid', 'natural').
- Constructed the request body for image generation and processed the response to handle both URLs and base64-encoded images.
- Converted image strings into ImageContent objects, ensuring proper handling of data URIs and HTTP URLs.
---
.../TextToImage/OpenAITextToImageService.cs | 68 ++++++++++++++++++-
1 file changed, 66 insertions(+), 2 deletions(-)
diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs
index ea6420fcfccb..6e2be2425abf 100644
--- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs
+++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs
@@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
+using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
@@ -94,9 +95,72 @@ public Task GenerateImageAsync(string description, int width, int height
}
///
- public Task> GetImageContentsAsync(TextContent input, PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, CancellationToken cancellationToken = default)
+ public async Task> GetImageContentsAsync(
+ TextContent input,
+ PromptExecutionSettings? executionSettings = null,
+ Kernel? kernel = null,
+ CancellationToken cancellationToken = default)
{
- throw new NotImplementedException();
+ // Ensure the input is valid
+ Verify.NotNull(input);
+
+ // Convert the generic execution settings to OpenAI-specific settings
+ var imageSettings = OpenAITextToImageExecutionSettings.FromExecutionSettings(executionSettings);
+
+ // Determine the size of the image based on the width and height settings
+ var size = (imageSettings.Width, imageSettings.Height) switch
+ {
+ (256, 256) => "256x256",
+ (512, 512) => "512x512",
+ (1024, 1024) => "1024x1024",
+ (1792, 1024) => "1792x1024",
+ (1024, 1792) => "1024x1792",
+ _ => throw new NotSupportedException($"The provided size is not supported: {imageSettings.Width}x{imageSettings.Height}")
+ };
+
+ // Validate quality and style
+ var supportedQualities = new[] { "standard", "hd" };
+ var supportedStyles = new[] { "vivid", "natural" };
+
+ if (!string.IsNullOrEmpty(imageSettings.Quality) && !supportedQualities.Contains(imageSettings.Quality))
+ {
+ throw new NotSupportedException($"The provided quality '{imageSettings.Quality}' is not supported.");
+ }
+
+ if (!string.IsNullOrEmpty(imageSettings.Style) && !supportedStyles.Contains(imageSettings.Style))
+ {
+ throw new NotSupportedException($"The provided style '{imageSettings.Style}' is not supported.");
+ }
+
+ // Create the request body for the image generation
+ var requestBody = JsonSerializer.Serialize(new TextToImageRequest
+ {
+ Model = imageSettings.ModelId ?? this._modelId,
+ Prompt = input.Text ?? string.Empty,
+ Size = size,
+ Count = imageSettings.ImageCount ?? 1,
+ Quality = imageSettings.Quality ?? "standard",
+ Style = imageSettings.Style ?? "vivid"
+ });
+
+ // Execute the request using the core client and return Image objects
+ var imageStrings = await this._core.ExecuteImageGenerationRequestAsync(OpenAIEndpoint, requestBody, x => x.Url ?? x.AsBase64, cancellationToken).ConfigureAwait(false);
+
+ // Convert the strings to ImageContent objects
+ var images = new List();
+ foreach (var imageString in imageStrings)
+ {
+ if (Uri.TryCreate(imageString, UriKind.Absolute, out var uriResult) && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps))
+ {
+ images.Add(new ImageContent(uriResult));
+ }
+ else
+ {
+ images.Add(new ImageContent($"data:;base64,{imageString}"));
+ }
+ }
+
+ return images.AsReadOnly();
}
private async Task GenerateImageAsync(
From ef12678b49a71006fc773f56fd0168ed93ccbd08 Mon Sep 17 00:00:00 2001
From: aghimir3 <22482815+aghimir3@users.noreply.github.com>
Date: Tue, 13 Aug 2024 21:48:02 -0700
Subject: [PATCH 3/3] Add unit tests for GetImageContentsAsync method
- Implemented unit tests for the GetImageContentsAsync method in OpenAITextToImageService.
- Added a test to verify that the method returns expected ImageContent when provided with valid input.
- Added parameterized tests using [Theory] and [InlineData] to cover a variety of scenarios:
- Valid URL and base64 image data inputs.
- Validation of input sizes, quality, and style parameters.
- Ensured NotSupportedException is thrown for unsupported sizes, quality, and style.
- Tests ensure that both HTTP URLs and base64-encoded images are handled correctly, with proper assertions on the returned ImageContent objects.
---
.../OpenAITextToImageServiceTests.cs | 118 ++++++++++++++++++
1 file changed, 118 insertions(+)
diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs
index 1f31ec076edd..8855a233b27f 100644
--- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs
+++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs
@@ -1,10 +1,12 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
+using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Moq;
using Xunit;
@@ -81,6 +83,122 @@ public async Task GenerateImageWorksCorrectlyAsync(int width, int height, bool e
}
}
+ [Fact]
+ public async Task GetImageContentsAsyncWithValidInputReturnsImageContentsAsync()
+ {
+ // Arrange
+ var service = new OpenAITextToImageService("api-key", "organization", "dall-e-3", this._httpClient);
+ Assert.Equal("dall-e-3", service.Attributes["ModelId"]);
+
+ this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
+ {
+ Content = new StringContent("""
+ {
+ "created": 1702575371,
+ "data": [
+ {
+ "url": "https://image-url"
+ }
+ ]
+ }
+ """, Encoding.UTF8, "application/json")
+ };
+
+ var input = new TextContent("A cute baby sea otter");
+ var executionSettings = new OpenAITextToImageExecutionSettings
+ {
+ Width = 1024,
+ Height = 1024,
+ Quality = "hd",
+ Style = "natural",
+ ImageCount = 1
+ };
+
+ // Act
+ var result = await service.GetImageContentsAsync(input, executionSettings);
+
+ // Assert
+ Assert.NotNull(result);
+ Assert.Single(result);
+ Assert.Equal(new Uri("https://image-url"), result[0].Uri);
+ }
+
+ [Theory]
+ [InlineData(1024, 1024, "hd", "natural", 1, "https://image-url", false)]
+ [InlineData(123, 456, "hd", "natural", 1, "", true)]
+ [InlineData(1024, 1024, "hd", "natural", 2, "https://image-url1|https://image-url2", false)]
+ [InlineData(1024, 1024, "ultra", "natural", 1, "", true)]
+ [InlineData(1024, 1024, "hd", "artistic", 1, "", true)]
+ public async Task GetImageContentsReturnsExpectedResultsAsync(
+ int width,
+ int height,
+ string quality,
+ string style,
+ int imageCount,
+ string expectedUrls,
+ bool expectException)
+ {
+ // Arrange
+ var service = new OpenAITextToImageService("api-key", "organization", "dall-e-3", this._httpClient);
+
+ if (!expectException)
+ {
+ var urls = expectedUrls.Split('|').Select(url =>
+ {
+ return url.StartsWith("http", StringComparison.OrdinalIgnoreCase) ?
+ $"{{ \"url\": \"{url}\" }}" :
+ $"{{ \"b64_json\": \"{url}\" }}";
+ });
+ var jsonResponse = $"{{ \"created\": 1702575371, \"data\": [ {string.Join(",", urls)} ] }}";
+
+ this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
+ {
+ Content = new StringContent(jsonResponse, Encoding.UTF8, "application/json")
+ };
+ }
+
+ var input = new TextContent("A picturesque landscape");
+ var executionSettings = new OpenAITextToImageExecutionSettings
+ {
+ Width = width,
+ Height = height,
+ Quality = quality,
+ Style = style,
+ ImageCount = imageCount
+ };
+
+ // Act & Assert
+ if (expectException)
+ {
+ await Assert.ThrowsAsync(async () =>
+ {
+ await service.GetImageContentsAsync(input, executionSettings);
+ });
+ }
+ else
+ {
+ var result = await service.GetImageContentsAsync(input, executionSettings);
+
+ Assert.NotNull(result);
+ Assert.Equal(imageCount, result.Count);
+
+ var expectedUrlList = expectedUrls.Split('|').ToList();
+ for (int i = 0; i < result.Count; i++)
+ {
+ if (Uri.TryCreate(expectedUrlList[i], UriKind.Absolute, out var uriResult) &&
+ (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps))
+ {
+ Assert.Equal(uriResult, result[i].Uri);
+ }
+ else
+ {
+ Assert.StartsWith("data:;base64,", result[i].DataUri);
+ Assert.Contains(expectedUrlList[i], result[i].DataUri);
+ }
+ }
+ }
+ }
+
public void Dispose()
{
this._httpClient.Dispose();