From ec0d124ae1a11db2c11474cbaa44356a586a1661 Mon Sep 17 00:00:00 2001 From: dprotaso Date: Thu, 12 Oct 2023 12:56:41 -0400 Subject: [PATCH] GEP-1911 - h2c backend protocol --- conformance/base/manifests.yaml | 4 + .../tests/httproute-backend-protocol-h2c.go | 79 +++++++++++ .../tests/httproute-backend-protocol-h2c.yaml | 17 +++ .../utils/roundtripper/roundtripper.go | 124 ++++++++++++++++-- conformance/utils/suite/features.go | 4 + 5 files changed, 214 insertions(+), 14 deletions(-) create mode 100644 conformance/tests/httproute-backend-protocol-h2c.go create mode 100644 conformance/tests/httproute-backend-protocol-h2c.yaml diff --git a/conformance/base/manifests.yaml b/conformance/base/manifests.yaml index a2afefd810..8046e378b5 100644 --- a/conformance/base/manifests.yaml +++ b/conformance/base/manifests.yaml @@ -94,6 +94,10 @@ spec: - protocol: TCP port: 8080 targetPort: 3000 + - protocol: TCP + appProtocol: kubernetes.io/h2c + port: 8081 + targetPort: 3001 --- apiVersion: apps/v1 kind: Deployment diff --git a/conformance/tests/httproute-backend-protocol-h2c.go b/conformance/tests/httproute-backend-protocol-h2c.go new file mode 100644 index 0000000000..8961e6d4fe --- /dev/null +++ b/conformance/tests/httproute-backend-protocol-h2c.go @@ -0,0 +1,79 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/roundtripper" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +func init() { + ConformanceTests = append(ConformanceTests, HTTPRouteBackendProtocolH2C) +} + +var HTTPRouteBackendProtocolH2C = suite.ConformanceTest{ + ShortName: "HTTPRouteBackendProtocolH2C", + Description: "A HTTPRoute with a BackendRef that has an appProtocol kubernetes.io/h2c should be functional", + Features: []suite.SupportedFeature{ + suite.SupportGateway, + suite.SupportHTTPRoute, + suite.SupportHTTPRouteBackendProtocolH2C, + }, + Manifests: []string{ + "tests/httproute-backend-protocol-h2c.yaml", + }, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "backend-protocol-h2c", Namespace: ns} + gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + + t.Run("http2 prior knowledge request should reach backend", func(t *testing.T) { + expected := http.ExpectedResponse{ + Response: http.Response{StatusCode: 200}, + Backend: "infra-backend-v1", + Namespace: "gateway-conformance-infra", + } + + req := http.MakeRequest(t, &expected, gwAddr, roundtripper.H2CPriorKnowledgeProtocol, "http") + + http.WaitForConsistentResponse(t, suite.RoundTripper, req, expected, + suite.TimeoutConfig.RequiredConsecutiveSuccesses, + suite.TimeoutConfig.MaxTimeToConsistency) + }) + + t.Run("h2c upgrade request should reach backend", func(t *testing.T) { + expected := http.ExpectedResponse{ + Response: http.Response{StatusCode: 200}, + Backend: "infra-backend-v1", + Namespace: "gateway-conformance-infra", + } + + req := http.MakeRequest(t, &expected, gwAddr, roundtripper.H2CUpgradeProtocol, "http") + http.WaitForConsistentResponse(t, suite.RoundTripper, req, expected, + suite.TimeoutConfig.RequiredConsecutiveSuccesses, + suite.TimeoutConfig.MaxTimeToConsistency) + + }) + }, +} diff --git a/conformance/tests/httproute-backend-protocol-h2c.yaml b/conformance/tests/httproute-backend-protocol-h2c.yaml new file mode 100644 index 0000000000..12106973f7 --- /dev/null +++ b/conformance/tests/httproute-backend-protocol-h2c.yaml @@ -0,0 +1,17 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: backend-protocol-h2c + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + rules: + - backendRefs: + # This points to a Service with the following ServicePort + # - protocol: TCP + # appProtocol: kubernetes.io/h2c + # port: 8081 + # targetPort: 3001 + - name: infra-backend-v1 + port: 8081 diff --git a/conformance/utils/roundtripper/roundtripper.go b/conformance/utils/roundtripper/roundtripper.go index efeeab3e2b..08df4df62d 100644 --- a/conformance/utils/roundtripper/roundtripper.go +++ b/conformance/utils/roundtripper/roundtripper.go @@ -17,10 +17,12 @@ limitations under the License. package roundtripper import ( + "bufio" "context" "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "io" "net" @@ -29,9 +31,15 @@ import ( "net/url" "regexp" + "golang.org/x/net/http2" "sigs.k8s.io/gateway-api/conformance/utils/config" ) +const ( + H2CUpgradeProtocol = "H2C_UPGRADE" + H2CPriorKnowledgeProtocol = "H2C_PRIOR_KNOWLEDGE" +) + // RoundTripper is an interface used to make requests within conformance tests. // This can be overridden with custom implementations whenever necessary. type RoundTripper interface { @@ -104,19 +112,7 @@ type DefaultRoundTripper struct { CustomDialContext func(context.Context, string, string) (net.Conn, error) } -// CaptureRoundTrip makes a request with the provided parameters and returns the -// captured request and response from echoserver. An error will be returned if -// there is an error running the function but not if an HTTP error status code -// is received. -func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedRequest, *CapturedResponse, error) { - client := &http.Client{} - - if request.UnfollowRedirect { - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - } - } - +func (d *DefaultRoundTripper) httpTransport(request Request) (http.RoundTripper, error) { transport := &http.Transport{ DialContext: d.CustomDialContext, // We disable keep-alives so that we don't leak established TCP connections. @@ -131,10 +127,110 @@ func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedReques if request.Server != "" && len(request.CertPem) != 0 && len(request.KeyPem) != 0 { tlsConfig, err := tlsClientConfig(request.Server, request.CertPem, request.KeyPem) if err != nil { - return nil, nil, err + return nil, err } transport.TLSClientConfig = tlsConfig } + + return transport, nil +} + +func (d *DefaultRoundTripper) h2cPriorKnowledgeTransport(request Request) (http.RoundTripper, error) { + if request.Server != "" && len(request.CertPem) != 0 && len(request.KeyPem) != 0 { + return nil, errors.New("request has configured cert and key but h2 prior knowledge is not encrypted") + } + + transport := &http2.Transport{ + AllowHTTP: true, + DialTLSContext: func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, network, addr) + }, + } + + return transport, nil +} + +// This actually makes a H2C upgrade request when dialing to establish +// an HTTP/2 connection. Then the transport will re-send the same request +// +// This seemed like the easiest way to accomplish testing h2c upgrade flow +// with golang. Open issue is here https://github.com/golang/go/issues/46249 +// +// The alternative would be to re-implement parts of http2.Transport +func (d *DefaultRoundTripper) h2cUpgradeTransport(request Request) (http.RoundTripper, error) { + if request.Server != "" && len(request.CertPem) != 0 && len(request.KeyPem) != 0 { + return nil, errors.New("request has configured cert and key but h2c is not encrypted") + } + return &http2.Transport{ + AllowHTTP: true, + DialTLSContext: func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) { + dialer := net.Dialer{} + conn, err := dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + bw := bufio.NewWriter(conn) + br := bufio.NewReader(conn) + + req, _ := http.NewRequestWithContext(ctx, request.Method, "http://"+addr, nil) + req.Header.Set("Connection", "Upgrade, HTTP2-Settings") + req.Header.Set("Upgrade", "h2c") + req.Header.Set("HTTP2-Settings", "") + + if err = req.Write(bw); err != nil { + return nil, err + } + if err = bw.Flush(); err != nil { + return nil, err + } + + resp, err := http.ReadResponse(br, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusSwitchingProtocols { + return nil, errors.New("switching protocols failed") + } + return conn, nil + }, + }, nil +} + +// CaptureRoundTrip makes a request with the provided parameters and returns the +// captured request and response from echoserver. An error will be returned if +// there is an error running the function but not if an HTTP error status code +// is received. +func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedRequest, *CapturedResponse, error) { + var transport http.RoundTripper + var err error + + switch request.Protocol { + case H2CUpgradeProtocol: + transport, err = d.h2cUpgradeTransport(request) + case H2CPriorKnowledgeProtocol: + transport, err = d.h2cPriorKnowledgeTransport(request) + default: + transport, err = d.httpTransport(request) + } + + if err != nil { + return nil, nil, err + } + + return d.defaultRoundTrip(request, transport) +} + +func (d *DefaultRoundTripper) defaultRoundTrip(request Request, transport http.RoundTripper) (*CapturedRequest, *CapturedResponse, error) { + client := &http.Client{} + + if request.UnfollowRedirect { + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + } + client.Transport = transport method := "GET" diff --git a/conformance/utils/suite/features.go b/conformance/utils/suite/features.go index cf687c33ec..62a7bca734 100644 --- a/conformance/utils/suite/features.go +++ b/conformance/utils/suite/features.go @@ -133,6 +133,9 @@ const ( // This option indicates support for HTTPRoute backendRequest timeouts (extended conformance). SupportHTTPRouteBackendTimeout SupportedFeature = "HTTPRouteBackendTimeout" + + // This option indicates support for HTTPRoute with a backendref with an appProtoocol 'kubernetes.io/h2c' + SupportHTTPRouteBackendProtocolH2C SupportedFeature = "HTTPRouteBackendProtocolH2C" ) // HTTPRouteExtendedFeatures includes all the supported features for HTTPRoute @@ -167,6 +170,7 @@ const ( // Implementations have the flexibility to opt-in for either specific features or the entire set. var HTTPRouteExperimentalFeatures = sets.New( SupportHTTPRouteDestinationPortMatching, + SupportHTTPRouteBackendProtocolH2C, ) // -----------------------------------------------------------------------------