From 201423b0c1298ffec9aec62c54f0e56568539471 Mon Sep 17 00:00:00 2001 From: aghimir3 <22482815+aghimir3@users.noreply.github.com> Date: Tue, 13 Aug 2024 20:49:34 -0700 Subject: [PATCH 1/3] Add quality and style to TextToImageRequest --- .../TextToImage/TextToImageRequest.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs index 70b5ac5418ee..704f973c29b5 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/TextToImageRequest.cs @@ -39,4 +39,16 @@ internal sealed class TextToImageRequest /// [JsonPropertyName("response_format")] public string Format { get; set; } = "url"; + + /// + /// Image quality, "standard" or "hd" + /// + [JsonPropertyName("quality")] + public string Quality { get; set; } = "standard"; + + /// + /// Image style, "vivid" or "natural" + /// + [JsonPropertyName("style")] + public string Style { get; set; } = "vivid"; } From 6895c051be0fc91616d3acdc7a3b5598e563f6e6 Mon Sep 17 00:00:00 2001 From: aghimir3 <22482815+aghimir3@users.noreply.github.com> Date: Tue, 13 Aug 2024 21:46:23 -0700 Subject: [PATCH 2/3] Implement GetImageContentsAsync in OpenAITextToImageService - Added the GetImageContentsAsync method to the OpenAITextToImageService class. - Implemented validation for input, including width, height, quality, and style settings. - Supported image sizes include 256x256, 512x512, 1024x1024, 1792x1024, and 1024x1792. - Added checks for supported qualities ('standard', 'hd') and styles ('vivid', 'natural'). - Constructed the request body for image generation and processed the response to handle both URLs and base64-encoded images. - Converted image strings into ImageContent objects, ensuring proper handling of data URIs and HTTP URLs. --- .../TextToImage/OpenAITextToImageService.cs | 68 ++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs index ea6420fcfccb..6e2be2425abf 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/TextToImage/OpenAITextToImageService.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Net.Http; using System.Text.Json; using System.Threading; @@ -94,9 +95,72 @@ public Task GenerateImageAsync(string description, int width, int height } /// - public Task> GetImageContentsAsync(TextContent input, PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, CancellationToken cancellationToken = default) + public async Task> GetImageContentsAsync( + TextContent input, + PromptExecutionSettings? executionSettings = null, + Kernel? kernel = null, + CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + // Ensure the input is valid + Verify.NotNull(input); + + // Convert the generic execution settings to OpenAI-specific settings + var imageSettings = OpenAITextToImageExecutionSettings.FromExecutionSettings(executionSettings); + + // Determine the size of the image based on the width and height settings + var size = (imageSettings.Width, imageSettings.Height) switch + { + (256, 256) => "256x256", + (512, 512) => "512x512", + (1024, 1024) => "1024x1024", + (1792, 1024) => "1792x1024", + (1024, 1792) => "1024x1792", + _ => throw new NotSupportedException($"The provided size is not supported: {imageSettings.Width}x{imageSettings.Height}") + }; + + // Validate quality and style + var supportedQualities = new[] { "standard", "hd" }; + var supportedStyles = new[] { "vivid", "natural" }; + + if (!string.IsNullOrEmpty(imageSettings.Quality) && !supportedQualities.Contains(imageSettings.Quality)) + { + throw new NotSupportedException($"The provided quality '{imageSettings.Quality}' is not supported."); + } + + if (!string.IsNullOrEmpty(imageSettings.Style) && !supportedStyles.Contains(imageSettings.Style)) + { + throw new NotSupportedException($"The provided style '{imageSettings.Style}' is not supported."); + } + + // Create the request body for the image generation + var requestBody = JsonSerializer.Serialize(new TextToImageRequest + { + Model = imageSettings.ModelId ?? this._modelId, + Prompt = input.Text ?? string.Empty, + Size = size, + Count = imageSettings.ImageCount ?? 1, + Quality = imageSettings.Quality ?? "standard", + Style = imageSettings.Style ?? "vivid" + }); + + // Execute the request using the core client and return Image objects + var imageStrings = await this._core.ExecuteImageGenerationRequestAsync(OpenAIEndpoint, requestBody, x => x.Url ?? x.AsBase64, cancellationToken).ConfigureAwait(false); + + // Convert the strings to ImageContent objects + var images = new List(); + foreach (var imageString in imageStrings) + { + if (Uri.TryCreate(imageString, UriKind.Absolute, out var uriResult) && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps)) + { + images.Add(new ImageContent(uriResult)); + } + else + { + images.Add(new ImageContent($"data:;base64,{imageString}")); + } + } + + return images.AsReadOnly(); } private async Task GenerateImageAsync( From ef12678b49a71006fc773f56fd0168ed93ccbd08 Mon Sep 17 00:00:00 2001 From: aghimir3 <22482815+aghimir3@users.noreply.github.com> Date: Tue, 13 Aug 2024 21:48:02 -0700 Subject: [PATCH 3/3] Add unit tests for GetImageContentsAsync method - Implemented unit tests for the GetImageContentsAsync method in OpenAITextToImageService. - Added a test to verify that the method returns expected ImageContent when provided with valid input. - Added parameterized tests using [Theory] and [InlineData] to cover a variety of scenarios: - Valid URL and base64 image data inputs. - Validation of input sizes, quality, and style parameters. - Ensured NotSupportedException is thrown for unsupported sizes, quality, and style. - Tests ensure that both HTTP URLs and base64-encoded images are handled correctly, with proper assertions on the returned ImageContent objects. --- .../OpenAITextToImageServiceTests.cs | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs index 1f31ec076edd..8855a233b27f 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToImage/OpenAITextToImageServiceTests.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Linq; using System.Net.Http; using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.OpenAI; using Moq; using Xunit; @@ -81,6 +83,122 @@ public async Task GenerateImageWorksCorrectlyAsync(int width, int height, bool e } } + [Fact] + public async Task GetImageContentsAsyncWithValidInputReturnsImageContentsAsync() + { + // Arrange + var service = new OpenAITextToImageService("api-key", "organization", "dall-e-3", this._httpClient); + Assert.Equal("dall-e-3", service.Attributes["ModelId"]); + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "created": 1702575371, + "data": [ + { + "url": "https://image-url" + } + ] + } + """, Encoding.UTF8, "application/json") + }; + + var input = new TextContent("A cute baby sea otter"); + var executionSettings = new OpenAITextToImageExecutionSettings + { + Width = 1024, + Height = 1024, + Quality = "hd", + Style = "natural", + ImageCount = 1 + }; + + // Act + var result = await service.GetImageContentsAsync(input, executionSettings); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + Assert.Equal(new Uri("https://image-url"), result[0].Uri); + } + + [Theory] + [InlineData(1024, 1024, "hd", "natural", 1, "https://image-url", false)] + [InlineData(123, 456, "hd", "natural", 1, "", true)] + [InlineData(1024, 1024, "hd", "natural", 2, "https://image-url1|https://image-url2", false)] + [InlineData(1024, 1024, "ultra", "natural", 1, "", true)] + [InlineData(1024, 1024, "hd", "artistic", 1, "", true)] + public async Task GetImageContentsReturnsExpectedResultsAsync( + int width, + int height, + string quality, + string style, + int imageCount, + string expectedUrls, + bool expectException) + { + // Arrange + var service = new OpenAITextToImageService("api-key", "organization", "dall-e-3", this._httpClient); + + if (!expectException) + { + var urls = expectedUrls.Split('|').Select(url => + { + return url.StartsWith("http", StringComparison.OrdinalIgnoreCase) ? + $"{{ \"url\": \"{url}\" }}" : + $"{{ \"b64_json\": \"{url}\" }}"; + }); + var jsonResponse = $"{{ \"created\": 1702575371, \"data\": [ {string.Join(",", urls)} ] }}"; + + this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(jsonResponse, Encoding.UTF8, "application/json") + }; + } + + var input = new TextContent("A picturesque landscape"); + var executionSettings = new OpenAITextToImageExecutionSettings + { + Width = width, + Height = height, + Quality = quality, + Style = style, + ImageCount = imageCount + }; + + // Act & Assert + if (expectException) + { + await Assert.ThrowsAsync(async () => + { + await service.GetImageContentsAsync(input, executionSettings); + }); + } + else + { + var result = await service.GetImageContentsAsync(input, executionSettings); + + Assert.NotNull(result); + Assert.Equal(imageCount, result.Count); + + var expectedUrlList = expectedUrls.Split('|').ToList(); + for (int i = 0; i < result.Count; i++) + { + if (Uri.TryCreate(expectedUrlList[i], UriKind.Absolute, out var uriResult) && + (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps)) + { + Assert.Equal(uriResult, result[i].Uri); + } + else + { + Assert.StartsWith("data:;base64,", result[i].DataUri); + Assert.Contains(expectedUrlList[i], result[i].DataUri); + } + } + } + } + public void Dispose() { this._httpClient.Dispose();