Pure python request pipeline

CHANGES.md

@@ -2,4 +2,13 @@
 
 ## Unreleased
 
-* Description of change. (Issue Number)
+### Breaking Changes
+
+* Removed the `http_client` config option in favor of the generic `transport`.
+
+### Features
+
+* Removed code-generated protocol implementations in favor of hand-written
+  implementations based on schemas.
+* Moved documentation for structure members into doc strings after the member's
+  dataclass field declaration.

codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java

@@ -124,13 +124,6 @@ public void generateService(GenerateServiceDirective<GenerationContext, PythonSe
         if (protocolGenerator == null) {
             return;
         }
-
-        protocolGenerator.generateSharedSerializerComponents(directive.context());
-        protocolGenerator.generateRequestSerializers(directive.context());
-
-        protocolGenerator.generateSharedDeserializerComponents(directive.context());
-        protocolGenerator.generateResponseDeserializers(directive.context());
-
         protocolGenerator.generateProtocolTests(directive.context());
     }
 

codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java

@@ -186,7 +186,7 @@ private void generateRequestTest(OperationShape operation, HttpRequestTestCase t
                         writer.write("""
                                 config = $T(
                                     endpoint_uri="https://$L/$L",
-                                    http_client = $T(),
+                                    transport = $T(),
                                     retry_strategy=SimpleRetryStrategy(max_attempts=1),
                                 )
                                 """,
@@ -432,7 +432,7 @@ private void generateResponseTest(OperationShape operation, HttpResponseTestCase
                         writer.write("""
                                 config = $T(
                                     endpoint_uri="https://example.com",
-                                    http_client = $T(
+                                    transport = $T(
                                         status=$L,
                                         headers=$J,
                                         body=b$S,
@@ -485,7 +485,7 @@ private void generateErrorResponseTest(
                         writer.write("""
                                 config = $T(
                                     endpoint_uri="https://example.com",
-                                    http_client = $T(
+                                    transport = $T(
                                         status=$L,
                                         headers=$J,
                                         body=b$S,

codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java

@@ -73,6 +73,15 @@ public final class SmithyPythonDependency {
             Type.DEPENDENCY,
             false);
 
+    /**
+     * Smithy core functionality for AWS.
+     */
+    public static final PythonDependency SMITHY_AWS_CORE = new PythonDependency(
+            "smithy_aws_core",
+            "<0.1.0",
+            Type.DEPENDENCY,
+            false);
+
     /**
      * testing framework used in generated functional tests.
      */

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java

@@ -117,38 +117,58 @@ public ConfigGenerator(PythonSettings settings, GenerationContext context) {
         this.settings = settings;
     }
 
-    private static List<ConfigProperty> getHttpProperties(GenerationContext context) {
-        var properties = new ArrayList<ConfigProperty>(HTTP_PROPERTIES.size() + 2);
-        var clientBuilder = ConfigProperty.builder()
-                .name("http_client")
+    private static List<ConfigProperty> getProtocolProperties(GenerationContext context) {
+        var properties = new ArrayList<ConfigProperty>();
+        var protocolBuilder = ConfigProperty.builder()
+                .name("protocol")
                 .type(Symbol.builder()
-                        .name("HTTPClient")
-                        .namespace("smithy_http.aio.interfaces", ".")
-                        .addDependency(SmithyPythonDependency.SMITHY_HTTP)
+                        .name("ClientProtocol[Any, Any]")
+                        .addReference(Symbol.builder()
+                                .name("ClientProtocol")
+                                .namespace("smithy_core.aio.interfaces", ".")
+                                .build())
+                        .build())
+                .documentation("The protocol to serialize and deserialize requests with.")
+                .initialize(w -> {
+                    w.write("self.protocol = protocol or ${C|}",
+                            w.consumer(writer -> context.protocolGenerator().initializeProtocol(context, writer)));
+                });
+
+        var transportBuilder = ConfigProperty.builder()
+                .name("transport")
+                .type(Symbol.builder()
+                        .name("ClientTransport[Any, Any]")
+                        .addReference(Symbol.builder()
+                                .name("ClientTransport")
+                                .namespace("smithy_core.aio.interfaces", ".")
+                                .build())
                         .build())
-                .documentation("The HTTP client used to make requests.")
-                .nullable(false);
+                .documentation("The transport to use to send requests (e.g. an HTTP client).");
 
-        if (usesHttp2(context)) {
-            clientBuilder
-                    .initialize(writer -> {
-                        writer.addDependency(SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("awscrt"));
-                        writer.addImport("smithy_http.aio.crt", "AWSCRTHTTPClient");
-                        writer.write("self.http_client = http_client or AWSCRTHTTPClient()");
-                    });
+        if (context.applicationProtocol().isHttpProtocol()) {
+            properties.addAll(HTTP_PROPERTIES);
+            if (usesHttp2(context)) {
+                transportBuilder
+                        .initialize(writer -> {
+                            writer.addDependency(SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("awscrt"));
+                            writer.addImport("smithy_http.aio.crt", "AWSCRTHTTPClient");
+                            writer.write("self.transport = transport or AWSCRTHTTPClient()");
+                        });
 
-        } else {
-            clientBuilder
-                    .initialize(writer -> {
-                        writer.addDependency(SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("aiohttp"));
-                        writer.addImport("smithy_http.aio.aiohttp", "AIOHTTPClient");
-                        writer.write("self.http_client = http_client or AIOHTTPClient()");
-                    });
+            } else {
+                transportBuilder
+                        .initialize(writer -> {
+                            writer.addDependency(
+                                    SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("aiohttp"));
+                            writer.addImport("smithy_http.aio.aiohttp", "AIOHTTPClient");
+                            writer.write("self.transport = transport or AIOHTTPClient()");
+                        });
+            }
         }
-        properties.add(clientBuilder.build());
 
-        properties.addAll(HTTP_PROPERTIES);
-        return List.copyOf(properties);
+        properties.add(protocolBuilder.build());
+        properties.add(transportBuilder.build());
+        return properties;
     }
 
     private static boolean usesHttp2(GenerationContext context) {
@@ -289,6 +309,7 @@ private void generateConfig(GenerationContext context, PythonWriter writer) {
         // Initialize a set of config properties with our base properties.
         var properties = new TreeSet<>(Comparator.comparing(ConfigProperty::name));
         properties.addAll(BASE_PROPERTIES);
+        properties.addAll(getProtocolProperties(context));
 
         // Add in auth configuration if the service supports auth.
         var serviceIndex = ServiceIndex.of(context.model());
@@ -297,13 +318,6 @@ private void generateConfig(GenerationContext context, PythonWriter writer) {
             writer.onSection(new AddAuthHelper());
         }
 
-        // Smithy is transport agnostic, so we don't add http-related properties by default.
-        // Nevertheless, HTTP is the most common use case so we standardize those settings
-        // and add them in if the protocol is going to need them.
-        if (context.applicationProtocol().isHttpProtocol()) {
-            properties.addAll(getHttpProperties(context));
-        }
-
         var model = context.model();
         var service = context.settings().service(model);
 

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberDeserializerGenerator.java

@@ -28,6 +28,7 @@
 import software.amazon.smithy.model.shapes.StructureShape;
 import software.amazon.smithy.model.shapes.TimestampShape;
 import software.amazon.smithy.model.shapes.UnionShape;
+import software.amazon.smithy.model.traits.StreamingTrait;
 import software.amazon.smithy.python.codegen.GenerationContext;
 import software.amazon.smithy.python.codegen.SymbolProperties;
 import software.amazon.smithy.python.codegen.writer.PythonWriter;
@@ -100,7 +101,11 @@ private void writeSchema() {
 
     @Override
     public Void blobShape(BlobShape shape) {
-        writeDeserializer(shape);
+        if (shape.hasTrait(StreamingTrait.class)) {
+            writeDeserializer("data_stream");
+        } else {
+            writeDeserializer(shape);
+        }
         return null;
     }
 

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberSerializerGenerator.java

@@ -29,6 +29,7 @@
 import software.amazon.smithy.model.shapes.StructureShape;
 import software.amazon.smithy.model.shapes.TimestampShape;
 import software.amazon.smithy.model.shapes.UnionShape;
+import software.amazon.smithy.model.traits.StreamingTrait;
 import software.amazon.smithy.python.codegen.GenerationContext;
 import software.amazon.smithy.python.codegen.SymbolProperties;
 import software.amazon.smithy.python.codegen.writer.PythonWriter;
@@ -107,7 +108,11 @@ private void writeSchema() {
 
     @Override
     public Void blobShape(BlobShape shape) {
-        writeSerializer(shape);
+        if (shape.hasTrait(StreamingTrait.class)) {
+            writeSerializer("data_stream");
+        } else {
+            writeSerializer(shape);
+        }
         return null;
     }
 

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ProtocolGenerator.java

@@ -4,15 +4,10 @@
  */
 package software.amazon.smithy.python.codegen.generators;
 
-import static java.lang.String.format;
-
-import software.amazon.smithy.codegen.core.Symbol;
 import software.amazon.smithy.model.shapes.ShapeId;
-import software.amazon.smithy.model.shapes.ToShapeId;
 import software.amazon.smithy.python.codegen.ApplicationProtocol;
 import software.amazon.smithy.python.codegen.GenerationContext;
 import software.amazon.smithy.python.codegen.writer.PythonWriter;
-import software.amazon.smithy.utils.CaseUtils;
 import software.amazon.smithy.utils.SmithyUnstableApi;
 
 /**
@@ -46,115 +41,12 @@ default String getName() {
      */
     ApplicationProtocol getApplicationProtocol(GenerationContext context);
 
-    /**
-     * Generates the name of a serializer function for shapes of a service that is not protocol-specific.
-     *
-     * @param context The code generation context.
-     * @param shapeId The shape the serializer function is being generated for.
-     * @return Returns the generated function name.
-     */
-    default String getSerializationFunctionName(GenerationContext context, ToShapeId shapeId) {
-        var name = context.settings().service(context.model()).getContextualName(shapeId);
-        return "_serialize_" + CaseUtils.toSnakeCase(name);
-    }
-
-    /**
-     * Generates the symbol for a serializer function for shapes of a service that is not protocol-specific.
-     *
-     * @param context The code generation context.
-     * @param shapeId The shape the serializer function is being generated for.
-     * @return Returns the generated symbol.
-     */
-    default Symbol getSerializationFunction(GenerationContext context, ToShapeId shapeId) {
-        return Symbol.builder()
-                .name(getSerializationFunctionName(context, shapeId))
-                .namespace(format("%s.serialize", context.settings().moduleName()), "")
-                .definitionFile(format("./src/%s/serialize.py", context.settings().moduleName()))
-                .build();
-    }
-
-    /**
-     * Generates the name of a deserializer function for shapes of a service that is not protocol-specific.
-     *
-     * @param context The code generation context.
-     * @param shapeId The shape the deserializer function is being generated for.
-     * @return Returns the generated function name.
-     */
-    default String getDeserializationFunctionName(GenerationContext context, ToShapeId shapeId) {
-        var name = context.settings().service(context.model()).getContextualName(shapeId);
-        return "_deserialize_" + CaseUtils.toSnakeCase(name);
-    }
-
-    /**
-     * Generates the symbol for a deserializer function for shapes of a service that is not protocol-specific.
-     *
-     * @param context The code generation context.
-     * @param shapeId The shape the deserializer function is being generated for.
-     * @return Returns the generated symbol.
-     */
-    default Symbol getDeserializationFunction(GenerationContext context, ToShapeId shapeId) {
-        return Symbol.builder()
-                .name(getDeserializationFunctionName(context, shapeId))
-                .namespace(format("%s.deserialize", context.settings().moduleName()), "")
-                .definitionFile(format("./src/%s/deserialize.py", context.settings().moduleName()))
-                .build();
-    }
-
-    /**
-     * Generates the symbol for the error deserializer function for an shape of a service that is not
-     * protocol-specific.
-     *
-     * @param context The code generation context.
-     * @param shapeId The shape id of the shape the error deserializer function is being generated for.
-     * @return Returns the generated symbol.
-     */
-    default Symbol getErrorDeserializationFunction(GenerationContext context, ToShapeId shapeId) {
-        var name = context.settings().service(context.model()).getContextualName(shapeId);
-        return Symbol.builder()
-                .name("_deserialize_error_" + CaseUtils.toSnakeCase(name))
-                .namespace(format("%s.deserialize", context.settings().moduleName()), "")
-                .definitionFile(format("./src/%s/deserialize.py", context.settings().moduleName()))
-                .build();
-    }
-
-    /**
-     * Generates any standard code for service request/response serde.
-     *
-     * @param context Serde context.
-     */
-    default void generateSharedSerializerComponents(GenerationContext context) {}
-
-    /**
-     * Generates the code used to serialize the shapes of a service
-     * for requests.
-     *
-     * @param context Serialization context.
-     */
-    void generateRequestSerializers(GenerationContext context);
-
-    /**
-     * Generates any standard code for service response deserialization.
-     *
-     * @param context Serde context.
-     */
-    default void generateSharedDeserializerComponents(GenerationContext context) {}
-
-    /**
-     * Generates the code used to deserialize the shapes of a service
-     * for responses.
-     *
-     * @param context Deserialization context.
-     */
-    void generateResponseDeserializers(GenerationContext context);
+    void initializeProtocol(GenerationContext context, PythonWriter writer);
 
     /**
      * Generates the code for validating the generated protocol's serializers and deserializers.
      *
      * @param context Generation context
      */
     default void generateProtocolTests(GenerationContext context) {}
-
-    default void wrapInputStream(GenerationContext context, PythonWriter writer) {}
-
-    default void wrapOutputStream(GenerationContext context, PythonWriter writer) {}
 }