Add Gzip request compression feature #467
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| { | ||
| "id": "80ed2832-7bcd-4301-a264-f318efaf8216", | ||
| "type": "feature", | ||
| "description": "Support modeled request compression.", | ||
| "modules": [ | ||
| "." | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,160 @@ | ||
| /* | ||
| * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"). | ||
| * You may not use this file except in compliance with the License. | ||
| * A copy of the License is located at | ||
| * | ||
| * http://aws.amazon.com/apache2.0 | ||
| * | ||
| * or in the "license" file accompanying this file. This file is distributed | ||
| * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either | ||
| * express or implied. See the License for the specific language governing | ||
| * permissions and limitations under the License. | ||
| */ | ||
|
|
||
| package software.amazon.smithy.go.codegen.requestcompression; | ||
|
|
||
| import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import software.amazon.smithy.codegen.core.SymbolProvider; | ||
| import software.amazon.smithy.go.codegen.GoCodegenPlugin; | ||
| import software.amazon.smithy.go.codegen.GoDelegator; | ||
| import software.amazon.smithy.go.codegen.GoSettings; | ||
| import software.amazon.smithy.go.codegen.GoUniverseTypes; | ||
| import software.amazon.smithy.go.codegen.GoWriter; | ||
| import software.amazon.smithy.go.codegen.SmithyGoTypes; | ||
| import software.amazon.smithy.go.codegen.SymbolUtils; | ||
| import software.amazon.smithy.go.codegen.integration.ConfigField; | ||
| import software.amazon.smithy.go.codegen.integration.GoIntegration; | ||
| import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; | ||
| import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; | ||
| import software.amazon.smithy.model.Model; | ||
| import software.amazon.smithy.model.knowledge.TopDownIndex; | ||
| import software.amazon.smithy.model.shapes.OperationShape; | ||
| import software.amazon.smithy.model.shapes.ServiceShape; | ||
| import software.amazon.smithy.model.shapes.ShapeId; | ||
| import software.amazon.smithy.model.traits.RequestCompressionTrait; | ||
| import software.amazon.smithy.utils.ListUtils; | ||
| import software.amazon.smithy.utils.MapUtils; | ||
|
|
||
|
|
||
| public final class RequestCompression implements GoIntegration { | ||
| private static final String DISABLE_REQUEST_COMPRESSION = "DisableRequestCompression"; | ||
|
|
||
| private static final String REQUEST_MIN_COMPRESSION_SIZE_BYTES = "RequestMinCompressSizeBytes"; | ||
|
|
||
| private final List<RuntimeClientPlugin> runtimeClientPlugins = new ArrayList<>(); | ||
|
There was a problem hiding this comment. Keeping state in an integration is generally not best practice. |
||
|
|
||
| // Write operation plugin for request compression middleware | ||
| @Override | ||
| public void processFinalizedModel(GoSettings settings, Model model) { | ||
| ServiceShape service = settings.getService(model); | ||
| TopDownIndex.of(model) | ||
| .getContainedOperations(service).forEach(operation -> { | ||
| if (!operation.hasTrait(RequestCompressionTrait.class)) { | ||
| return; | ||
| } | ||
| SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings); | ||
| String funcName = getAddRequestCompressionMiddlewareFuncName( | ||
| symbolProvider.toSymbol(operation).getName() | ||
| ); | ||
| runtimeClientPlugins.add(RuntimeClientPlugin.builder().operationPredicate((m, s, o) -> { | ||
| if (!o.hasTrait(RequestCompressionTrait.class)) { | ||
| return false; | ||
| } | ||
| return o.equals(operation); | ||
syall marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| }).registerMiddleware(MiddlewareRegistrar.builder() | ||
| .resolvedFunction(SymbolUtils.buildPackageSymbol(funcName)) | ||
| .useClientOptions().build()) | ||
| .build()); | ||
| }); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeAdditionalFiles( | ||
| GoSettings settings, | ||
| Model model, | ||
| SymbolProvider symbolProvider, | ||
| GoDelegator goDelegator | ||
| ) { | ||
| ServiceShape service = settings.getService(model); | ||
| for (ShapeId operationID : service.getAllOperations()) { | ||
| OperationShape operation = model.expectShape(operationID, OperationShape.class); | ||
| if (!operation.hasTrait(RequestCompressionTrait.class)) { | ||
| continue; | ||
| } | ||
| goDelegator.useShapeWriter(operation, writeMiddlewareHelper(symbolProvider, operation)); | ||
| } | ||
| } | ||
|
|
||
|
|
||
| public static boolean isRequestCompressionService(Model model, ServiceShape service) { | ||
| return TopDownIndex.of(model) | ||
| .getContainedOperations(service).stream() | ||
| .anyMatch(it -> it.hasTrait(RequestCompressionTrait.class)); | ||
| } | ||
|
|
||
| @Override | ||
| public List<RuntimeClientPlugin> getClientPlugins() { | ||
| runtimeClientPlugins.add( | ||
|
There was a problem hiding this comment. Use There was a problem hiding this comment. FYI he's doing this because of having to add per-operation middlewares elsewhere in There was a problem hiding this comment.
There was a problem hiding this comment. we have confirmed that in resolved comments, thx for clarification |
||
| RuntimeClientPlugin.builder() | ||
| .servicePredicate(RequestCompression::isRequestCompressionService) | ||
| .configFields(ListUtils.of( | ||
| ConfigField.builder() | ||
| .name(DISABLE_REQUEST_COMPRESSION) | ||
| .type(GoUniverseTypes.Bool) | ||
| .documentation( | ||
| "Whether to disable automatic request compression for supported operations.") | ||
| .build(), | ||
| ConfigField.builder() | ||
| .name(REQUEST_MIN_COMPRESSION_SIZE_BYTES) | ||
| .type(GoUniverseTypes.Int64) | ||
| .documentation("The minimum request body size, in bytes, at which compression " | ||
| + "should occur. The default value is 10 KiB. Values must fall within " | ||
| + "[0, 1MiB].") | ||
| .build() | ||
| )) | ||
| .build() | ||
| ); | ||
|
|
||
| return runtimeClientPlugins; | ||
| } | ||
|
|
||
| private GoWriter.Writable generateAlgorithmList(List<String> algorithms) { | ||
| return goTemplate(""" | ||
| []string{ | ||
| $W | ||
| } | ||
| """, | ||
| GoWriter.ChainWritable.of( | ||
| algorithms.stream() | ||
| .map(it -> goTemplate("$S,", it)) | ||
| .toList() | ||
| ).compose(false)); | ||
| } | ||
|
|
||
| private static String getAddRequestCompressionMiddlewareFuncName(String operationName) { | ||
| return String.format("addOperation%sRequestCompressionMiddleware", operationName); | ||
| } | ||
|
|
||
| private GoWriter.Writable writeMiddlewareHelper(SymbolProvider symbolProvider, OperationShape operation) { | ||
| String operationName = symbolProvider.toSymbol(operation).getName(); | ||
| RequestCompressionTrait trait = operation.expectTrait(RequestCompressionTrait.class); | ||
|
|
||
| return goTemplate(""" | ||
| func $add:L(stack $stack:P, options Options) error { | ||
| return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, | ||
| $algorithms:W) | ||
| } | ||
| """, | ||
| MapUtils.of( | ||
| "add", getAddRequestCompressionMiddlewareFuncName(operationName), | ||
| "stack", SmithyGoTypes.Middleware.Stack, | ||
| "addInternal", SmithyGoTypes.Private.RequestCompression.AddRequestCompression, | ||
| "algorithms", generateAlgorithmList(trait.getEncodings()) | ||
| )); | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package requestcompression | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "compress/gzip" | ||
| "fmt" | ||
| "io" | ||
| ) | ||
|
|
||
| func gzipCompress(input io.Reader) ([]byte, error) { | ||
| var b bytes.Buffer | ||
| w, err := gzip.NewWriterLevel(&b, gzip.DefaultCompression) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create gzip writer, %v", err) | ||
| } | ||
|
|
||
| inBytes, err := io.ReadAll(input) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed read payload to compress, %v", err) | ||
| } | ||
|
|
||
| if _, err = w.Write(inBytes); err != nil { | ||
| return nil, fmt.Errorf("failed to write payload to be compressed, %v", err) | ||
| } | ||
| if err = w.Close(); err != nil { | ||
| return nil, fmt.Errorf("failed to flush payload being compressed, %v", err) | ||
| } | ||
|
|
||
| return b.Bytes(), nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| package requestcompression | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "github.com/aws/smithy-go/middleware" | ||
| smithyhttp "github.com/aws/smithy-go/transport/http" | ||
| "io" | ||
| "net/http" | ||
| ) | ||
|
|
||
| const captureUncompressedRequestID = "CaptureUncompressedRequest" | ||
|
|
||
| // AddCaptureUncompressedRequestMiddleware captures http request before compress encoding for check | ||
| func AddCaptureUncompressedRequestMiddleware(stack *middleware.Stack, buf *bytes.Buffer) error { | ||
| return stack.Serialize.Insert(&captureUncompressedRequestMiddleware{ | ||
| buf: buf, | ||
| }, "RequestCompression", middleware.Before) | ||
| } | ||
|
|
||
| type captureUncompressedRequestMiddleware struct { | ||
| req *http.Request | ||
| buf *bytes.Buffer | ||
| bytes []byte | ||
| } | ||
|
|
||
| // ID returns id of the captureUncompressedRequestMiddleware | ||
| func (*captureUncompressedRequestMiddleware) ID() string { | ||
| return captureUncompressedRequestID | ||
| } | ||
|
|
||
| // HandleSerialize captures request payload before it is compressed by request compression middleware | ||
| func (m *captureUncompressedRequestMiddleware) HandleSerialize(ctx context.Context, input middleware.SerializeInput, next middleware.SerializeHandler, | ||
| ) ( | ||
| output middleware.SerializeOutput, metadata middleware.Metadata, err error, | ||
| ) { | ||
| request, ok := input.Request.(*smithyhttp.Request) | ||
| if !ok { | ||
| return output, metadata, fmt.Errorf("error when retrieving http request") | ||
| } | ||
|
|
||
| _, err = io.Copy(m.buf, request.GetStream()) | ||
| if err != nil { | ||
| return output, metadata, fmt.Errorf("error when copying http request stream: %q", err) | ||
| } | ||
| if err = request.RewindStream(); err != nil { | ||
| return output, metadata, fmt.Errorf("error when rewinding request stream: %q", err) | ||
| } | ||
|
|
||
| return next.HandleSerialize(ctx, input) | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit overall: private fields should go below public fields, and private methods should go towards the bottom of the class below all the public methods.