Support complex payload for OpenAPI skills (microsoft#680)

### Motivation and Context

This PR changes the way payload/body for POST and PUT operations is
resolved.

This change is required to support the following scenarios:
- A POST/PUT operation payload has more than one property with the same
name - { 'sender': { '**name**': 'val1'}, 'receiver': {'**name**':
'val2'} }
- A POST/PUT operation payload has a property of array of objects with
the same name - { 'receivers': [ {'**name**':'val1'},
{'**name**':'val2'} ] }
- A POST/PUT operation payload metadata is missing.

### Description
Curent RestApi functionality builds operation payload dynamically and
relies on:
- Payload metadata that describes the payload/body schema/shape and
usually obtained from OpenAPI documents.
- Arguments resolution mechanism that resolves payload parameters in SK
context.

As practice shows, in some cases (malformed OpenAPI document), the
payload metadata is missing which means that it's not possible to create
operation payload dynamically and as a result it's not possible to run
the operation. In other cases, if a payload has two of more parameters
with the same name or parameters of array type with object items it is
impossible to resolve them from SK context.

To support these scenarios, the RestApi functionality does not build the
payload dynamically anymore and instead relies on either AI or a caller
of the APi to create/supply it. To help AI generate the payload, a new
schema property will be added to the ParameterView class to describe the
payload structure/shape to AI.

dotnet/src/SemanticKernel.Abstractions/Connectors/WebApi/Rest/Model/RestApiOperation.cs

@@ -20,9 +20,14 @@ internal sealed class RestApiOperation
     internal const string ServerUrlArgumentName = "server-url";
 
     /// <summary>
-    /// An artificial parameter to be advertised and used for operation having "text/plain" payload media type.
+    /// An artificial parameter to be used for operation having "text/plain" payload media type.
     /// </summary>
-    internal const string InputArgumentName = "input";
+    internal const string PayloadArgumentName = "payload";
+
+    /// <summary>
+    /// An artificial parameter to be used for indicate payload media-type if it's missing in payload metadata.
+    /// </summary>
+    internal const string ContentTypeArgumentName = "content-type";
 
     /// <summary>
     /// The operation identifier.

dotnet/src/SemanticKernel.Abstractions/Connectors/WebApi/Rest/RestApiOperationRunner.cs

@@ -56,7 +56,7 @@ public RestApiOperationRunner(HttpClient httpClient, AuthenticateRequestAsyncCal
 
         var headers = operation.RenderHeaders(arguments);
 
-        var payload = BuildOperationPayload(operation.Payload, arguments);
+        var payload = BuildOperationPayload(operation, arguments);
 
         return this.SendAsync(url, operation.Method, headers, payload, cancellationToken);
     }
@@ -109,78 +109,60 @@ public RestApiOperationRunner(HttpClient httpClient, AuthenticateRequestAsyncCal
     /// <summary>
     /// Builds operation payload.
     /// </summary>
-    /// <param name="payloadMetadata">The payload meta-data.</param>
+    /// <param name="operation">The operation.</param>
     /// <param name="arguments">The payload arguments.</param>
     /// <returns>The HttpContent representing the payload.</returns>
-    private static HttpContent? BuildOperationPayload(RestApiOperationPayload? payloadMetadata, IDictionary<string, string> arguments)
+    private static HttpContent? BuildOperationPayload(RestApiOperation operation, IDictionary<string, string> arguments)
     {
-        if (payloadMetadata == null)
+        if (operation?.Method != HttpMethod.Put && operation?.Method != HttpMethod.Post)
         {
             return null;
         }
 
-        if (!s_payloadFactoryByMediaType.TryGetValue(payloadMetadata.MediaType, out var payloadFactory))
+        var mediaType = operation.Payload?.MediaType;
+
+        //A try to resolve payload content type from the operation arguments if it's missing in the payload metadata.
+        if (string.IsNullOrEmpty(mediaType))
         {
-            throw new RestApiOperationException($"The media type {payloadMetadata.MediaType} is not supported by {nameof(RestApiOperationRunner)}.");
+            if (!arguments.TryGetValue(RestApiOperation.ContentTypeArgumentName, out mediaType))
+            {
+                throw new RestApiOperationException($"No content type is provided for the {operation.Id} operation.");
+            }
         }
 
-        return payloadFactory.Invoke(payloadMetadata, arguments);
+        if (!s_payloadFactoryByMediaType.TryGetValue(mediaType!, out var payloadFactory))
+        {
+            throw new RestApiOperationException($"The media type {mediaType} of the {operation.Id} operation is not supported by {nameof(RestApiOperationRunner)}.");
+        }
+
+        return payloadFactory.Invoke(arguments);
     }
 
     /// <summary>
     /// Builds "application/json" payload.
     /// </summary>
-    /// <param name="payloadMetadata">The payload meta-data.</param>
-    /// <param name="arguments">The payload arguments.</param>    /// <returns></returns>
+    /// <param name="arguments">The payload arguments.</param>
     /// <returns>The HttpContent representing the payload.</returns>
-    private static HttpContent BuildAppJsonPayload(RestApiOperationPayload payloadMetadata, IDictionary<string, string> arguments)
+    private static HttpContent BuildAppJsonPayload(IDictionary<string, string> arguments)
     {
-        JsonNode BuildPayload(IList<RestApiOperationPayloadProperty> properties)
+        if (!arguments.TryGetValue(RestApiOperation.PayloadArgumentName, out var content))
         {
-            var result = new JsonObject();
-
-            foreach (var propertyMetadata in properties)
-            {
-                switch (propertyMetadata.Type)
-                {
-                    case "object":
-                    {
-                        var propertyValue = BuildPayload(propertyMetadata.Properties);
-                        result.Add(propertyMetadata.Name, propertyValue);
-                        break;
-                    }
-                    default: //TODO: Use the default case for unsupported types.
-                    {
-                        if (!arguments.TryGetValue(propertyMetadata.Name, out var propertyValue))
-                        {
-                            throw new RestApiOperationException($"No argument is found for the '{propertyMetadata.Name}' payload property.");
-                        }
-
-                        result.Add(propertyMetadata.Name, propertyValue);
-                        break;
-                    }
-                }
-            }
-
-            return result;
+            throw new RestApiOperationException($"No argument is found for the '{RestApiOperation.PayloadArgumentName}' payload content.");
         }
 
-        var payload = BuildPayload(payloadMetadata.Properties);
-
-        return new StringContent(payload.ToJsonString(), Encoding.UTF8, MediaTypeApplicationJson);
+        return new StringContent(content, Encoding.UTF8, MediaTypeApplicationJson);
     }
 
     /// <summary>
     /// Builds "text/plain" payload.
     /// </summary>
-    /// <param name="payloadMetadata">The payload meta-data.</param>
-    /// <param name="arguments">The payload arguments.</param>    /// <returns></returns>
+    /// <param name="arguments">The payload arguments.</param>
     /// <returns>The HttpContent representing the payload.</returns>
-    private static HttpContent BuildPlainTextPayload(RestApiOperationPayload payloadMetadata, IDictionary<string, string> arguments)
+    private static HttpContent BuildPlainTextPayload(IDictionary<string, string> arguments)
     {
-        if (!arguments.TryGetValue(RestApiOperation.InputArgumentName, out var propertyValue))
+        if (!arguments.TryGetValue(RestApiOperation.PayloadArgumentName, out var propertyValue))
         {
-            throw new RestApiOperationException($"No argument is found for the '{RestApiOperation.InputArgumentName}' payload content.");
+            throw new RestApiOperationException($"No argument is found for the '{RestApiOperation.PayloadArgumentName}' payload content.");
         }
 
         return new StringContent(propertyValue, Encoding.UTF8, MediaTypeTextPlain);
@@ -189,8 +171,8 @@ private static HttpContent BuildPlainTextPayload(RestApiOperationPayload payload
     /// <summary>
     /// List of payload builders/factories.
     /// </summary>
-    private static readonly Dictionary<string, Func<RestApiOperationPayload, IDictionary<string, string>, HttpContent>> s_payloadFactoryByMediaType =
-        new Dictionary<string, Func<RestApiOperationPayload, IDictionary<string, string>, HttpContent>>()
+    private static readonly Dictionary<string, Func<IDictionary<string, string>, HttpContent>> s_payloadFactoryByMediaType =
+        new Dictionary<string, Func<IDictionary<string, string>, HttpContent>>()
         {
             { MediaTypeApplicationJson, BuildAppJsonPayload },
             { MediaTypeTextPlain, BuildPlainTextPayload }

dotnet/src/SemanticKernel.UnitTests/Connectors/WebApi/Rest/RestApiOperationRunnerTests.cs

@@ -48,21 +48,12 @@ public RestApiOperationRunnerTests()
     }
 
     [Fact]
-    public async Task ItCanRunCrudOperationWithJsonPayloadSuccessfullyAsync()
+    public async Task ItCanRunCreateAndUpdateOperationsWithJsonPayloadSuccessfullyAsync()
     {
         // Arrange
         this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json);
 
-        List<RestApiOperationPayloadProperty> payloadProperties = new()
-        {
-            new("value", "string", true, new List<RestApiOperationPayloadProperty>(), "fake-value-description"),
-            new("attributes", "object", false, new List<RestApiOperationPayloadProperty>()
-            {
-                new("enabled", "boolean", false, new List<RestApiOperationPayloadProperty>(), "fake-enabled-description"),
-            })
-        };
-
-        var payload = new RestApiOperationPayload(MediaTypeNames.Application.Json, payloadProperties);
+        var payloadMetadata = new RestApiOperationPayload(MediaTypeNames.Application.Json, new List<RestApiOperationPayloadProperty>());
 
         var operation = new RestApiOperation(
             "fake-id",
@@ -72,12 +63,20 @@ public async Task ItCanRunCrudOperationWithJsonPayloadSuccessfullyAsync()
             "fake-description",
             new List<RestApiOperationParameter>(),
             new Dictionary<string, string>(),
-            payload
+            payloadMetadata
         );
 
+        var payload = new
+        {
+            value = "fake-value",
+            attributes = new
+            {
+                enabled = true
+            }
+        };
+
         var arguments = new Dictionary<string, string>();
-        arguments.Add("value", "fake-value");
-        arguments.Add("enabled", "true");
+        arguments.Add("payload", System.Text.Json.JsonSerializer.Serialize(payload));
 
         var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object);
 
@@ -122,7 +121,7 @@ public async Task ItCanRunCrudOperationWithJsonPayloadSuccessfullyAsync()
     }
 
     [Fact]
-    public async Task ItCanRunCrudOperationWithPlainTextPayloadSuccessfullyAsync()
+    public async Task ItCanRunCreateAndUpdateOperationsWithPlainTextPayloadSuccessfullyAsync()
     {
         // Arrange
         this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Text.Plain);
@@ -141,7 +140,7 @@ public async Task ItCanRunCrudOperationWithPlainTextPayloadSuccessfullyAsync()
         );
 
         var arguments = new Dictionary<string, string>();
-        arguments.Add("input", "fake-input-value");
+        arguments.Add("payload", "fake-input-value");
 
         var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object);
 
@@ -186,7 +185,7 @@ public async Task ItShouldAddHeadersToHttpRequestAsync()
             "fake-id",
             "https://fake-random-test-host",
             "fake-path",
-            HttpMethod.Post,
+            HttpMethod.Get,
             "fake-description",
             new List<RestApiOperationParameter>(),
             headers
@@ -207,6 +206,62 @@ public async Task ItShouldAddHeadersToHttpRequestAsync()
         Assert.Contains(this._httpMessageHandlerStub.RequestHeaders, h => h.Key == "fake-header" && h.Value.Contains("fake-header-value"));
     }
 
+    [Fact]
+    public async Task ItShouldUsePayloadAndContentTypeArgumentsIfPayloadMetadataIsMissingAsync()
+    {
+        // Arrange
+        this._httpMessageHandlerStub.ResponseToReturn.Content = new StringContent("fake-content", Encoding.UTF8, MediaTypeNames.Application.Json);
+
+        var operation = new RestApiOperation(
+            "fake-id",
+            "https://fake-random-test-host",
+            "fake-path",
+            HttpMethod.Post,
+            "fake-description",
+            new List<RestApiOperationParameter>(),
+            new Dictionary<string, string>()
+        );
+
+        var payload = new
+        {
+            value = "fake-value",
+            attributes = new
+            {
+                enabled = true
+            }
+        };
+
+        var arguments = new Dictionary<string, string>();
+        arguments.Add("payload", System.Text.Json.JsonSerializer.Serialize(payload));
+        arguments.Add("content-type", "application/json");
+
+        var sut = new RestApiOperationRunner(this._httpClient, this._authenticationHandlerMock.Object);
+
+        // Act
+        var result = await sut.RunAsync(operation, arguments);
+
+        // Assert
+        Assert.NotNull(this._httpMessageHandlerStub.ContentHeaders);
+        Assert.Contains(this._httpMessageHandlerStub.ContentHeaders, h => h.Key == "Content-Type" && h.Value.Contains("application/json; charset=utf-8"));
+
+        var messageContent = this._httpMessageHandlerStub.RequestContent;
+        Assert.NotNull(messageContent);
+        Assert.True(messageContent.Length != 0);
+
+        var deserializedPayload = JsonNode.Parse(new MemoryStream(messageContent));
+        Assert.NotNull(deserializedPayload);
+
+        var valueProperty = deserializedPayload["value"]?.ToString();
+        Assert.Equal("fake-value", valueProperty);
+
+        var attributesProperty = deserializedPayload["attributes"];
+        Assert.NotNull(attributesProperty);
+
+        var enabledProperty = attributesProperty["enabled"]?.AsValue();
+        Assert.NotNull(enabledProperty);
+        Assert.Equal("true", enabledProperty.ToString());
+    }
+
     /// <summary>
     /// Disposes resources used by this class.
     /// </summary>

dotnet/src/Skills/Skills.OpenAPI/Extensions/RestApiOperationExtensions.cs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System.Collections.Generic;
-using System.Linq;
+using System.Net.Http;
 using System.Text.RegularExpressions;
 using Microsoft.SemanticKernel.Connectors.WebApi.Rest.Model;
 
@@ -32,20 +32,27 @@ public static IReadOnlyList<RestApiOperationParameter> GetParameters(this RestAp
             RestApiOperationParameterStyle.Simple,
             defaultValue: operation.ServerUrl));
 
-        //Register the "input" parameter to be advertised and used for "text/plain" requests.
-        if (operation.Payload?.MediaType == MediaTypeTextPlain)
+        //Register the "payload" parameter to be advertised for Put and Post operations.
+        if (operation.Method == HttpMethod.Put || operation.Method == HttpMethod.Post)
         {
+            var type = operation.Payload?.MediaType == MediaTypeTextPlain ? "string" : "object";
+
             parameters.Add(new RestApiOperationParameter(
-                RestApiOperation.InputArgumentName,
-                "string",
+                RestApiOperation.PayloadArgumentName,
+                type,
                 true,
                 RestApiOperationParameterLocation.Body,
                 RestApiOperationParameterStyle.Simple,
-                description: operation.Payload.Description));
-        }
+                description: operation.Payload?.Description ?? "REST API request body."));
 
-        //Add Payload properties.
-        parameters.AddRange(CreateParametersFromPayloadProperties(operation.Payload));
+            parameters.Add(new RestApiOperationParameter(
+                RestApiOperation.ContentTypeArgumentName,
+                "string",
+                false,
+                RestApiOperationParameterLocation.Body,
+                RestApiOperationParameterStyle.Simple,
+                description: "Content type of REST API request body."));
+        }
 
         //Create a property alternative name without special symbols that are not supported by SK template language.
         foreach (var parameter in parameters)
@@ -56,50 +63,5 @@ public static IReadOnlyList<RestApiOperationParameter> GetParameters(this RestAp
         return parameters;
     }
 
-    /// <summary>
-    /// Creates parameters from REST API operation payload properties.
-    /// </summary>
-    /// <param name="payload">REST API operation payload.</param>
-    /// <returns>The list of parameters.</returns>
-    private static IEnumerable<RestApiOperationParameter> CreateParametersFromPayloadProperties(RestApiOperationPayload? payload)
-    {
-        if (payload == null)
-        {
-            return Enumerable.Empty<RestApiOperationParameter>();
-        }
-
-        IList<RestApiOperationParameter> ConvertLeafProperties(RestApiOperationPayloadProperty property)
-        {
-            var parameters = new List<RestApiOperationParameter>();
-
-            if (!property.Properties.Any()) //It's a leaf property
-            {
-                parameters.Add(new RestApiOperationParameter(
-                    property.Name,
-                    property.Type,
-                    property.IsRequired,
-                    RestApiOperationParameterLocation.Body,
-                    RestApiOperationParameterStyle.Simple,
-                    description: property.Description));
-            }
-
-            foreach (var childProperty in property.Properties)
-            {
-                parameters.AddRange(ConvertLeafProperties(childProperty));
-            }
-
-            return parameters;
-        }
-
-        var result = new List<RestApiOperationParameter>();
-
-        foreach (var property in payload.Properties)
-        {
-            result.AddRange(ConvertLeafProperties(property));
-        }
-
-        return result;
-    }
-
     private const string MediaTypeTextPlain = "text/plain";
 }