upstream adding QUIC-to-TCP failover (envoyproxy#15894)

Risk Level: n/a (hidden by default)
Testing: e2e tests
Docs Changes: n/a
Release Notes: n/a
Part of envoyproxy#14829

Signed-off-by: Gokul Nair <gnair@twitter.com>

api/envoy/extensions/upstreams/http/v3/http_protocol_options.proto

@@ -97,6 +97,13 @@ message HttpProtocolOptions {
     config.core.v3.Http1ProtocolOptions http_protocol_options = 1;
 
     config.core.v3.Http2ProtocolOptions http2_protocol_options = 2;
+
+    // [#not-implemented-hide:]
+    // Unlike HTTP/1 and HTTP/2, HTTP/3 will not be configured unless it is
+    // present. If HTTP/3 is present, attempts to connect will first be made
+    // via HTTP/3, and if the HTTP/3 connection fails, TCP (HTTP/1 or HTTP/2
+    // based on ALPN) will be used instead.
+    config.core.v3.Http3ProtocolOptions http3_protocol_options = 3;
   }
 
   // This contains options common across HTTP/1 and HTTP/2

source/common/http/conn_pool_grid.cc

@@ -6,18 +6,6 @@
 namespace Envoy {
 namespace Http {
 
-// Helper function to make sure each protocol in expected_protocols is present
-// in protocols (only used for an ASSERT in debug builds)
-bool contains(const std::vector<Http::Protocol>& protocols,
-              const std::vector<Http::Protocol>& expected_protocols) {
-  for (auto protocol : expected_protocols) {
-    if (std::find(protocols.begin(), protocols.end(), protocol) == protocols.end()) {
-      return false;
-    }
-  }
-  return true;
-}
-
 absl::string_view describePool(const ConnectionPool::Instance& pool) {
   return pool.protocolDescription();
 }
@@ -157,10 +145,10 @@ ConnectivityGrid::ConnectivityGrid(
     : dispatcher_(dispatcher), random_generator_(random_generator), host_(host),
       priority_(priority), options_(options), transport_socket_options_(transport_socket_options),
       state_(state), next_attempt_duration_(next_attempt_duration), time_source_(time_source) {
+  // ProdClusterManagerFactory::allocateConnPool verifies the protocols are HTTP/1, HTTP/2 and
+  // HTTP/3.
   // TODO(#15649) support v6/v4, WiFi/cellular.
   ASSERT(connectivity_options.protocols_.size() == 3);
-  ASSERT(contains(connectivity_options.protocols_,
-                  {Http::Protocol::Http11, Http::Protocol::Http2, Http::Protocol::Http3}));
 }
 
 ConnectivityGrid::~ConnectivityGrid() {

source/common/quic/BUILD

@@ -371,6 +371,7 @@ envoy_cc_library(
         "//source/common/common:assert_lib",
         "//source/extensions/transport_sockets:well_known_names",
         "//source/extensions/transport_sockets/tls:context_config_lib",
+        "//source/extensions/transport_sockets/tls:ssl_socket_lib",
         "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto",
     ],
 )

source/common/quic/quic_transport_socket_factory.cc

@@ -1,8 +1,7 @@
 #include "common/quic/quic_transport_socket_factory.h"
 
-// #include "envoy/extensions/transport_sockets/tls/v3/tls.pb.validate.h"
-#include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h"
 #include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.validate.h"
+
 #include "extensions/transport_sockets/tls/context_config_impl.h"
 
 namespace Envoy {
@@ -34,9 +33,15 @@ QuicClientTransportSocketConfigFactory::createTransportSocketFactory(
       config, context.messageValidationVisitor());
   auto client_config = std::make_unique<Extensions::TransportSockets::Tls::ClientContextConfigImpl>(
       quic_transport.upstream_tls_context(), context);
-  return std::make_unique<QuicClientTransportSocketFactory>(std::move(client_config));
+  return std::make_unique<QuicClientTransportSocketFactory>(std::move(client_config), context);
 }
 
+QuicClientTransportSocketFactory::QuicClientTransportSocketFactory(
+    Ssl::ClientContextConfigPtr config,
+    Server::Configuration::TransportSocketFactoryContext& factory_context)
+    : fallback_factory_(std::make_unique<Extensions::TransportSockets::Tls::ClientSslSocketFactory>(
+          std::move(config), factory_context.sslContextManager(), factory_context.scope())) {}
+
 ProtobufTypes::MessagePtr QuicClientTransportSocketConfigFactory::createEmptyConfigProto() {
   return std::make_unique<envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport>();
 }

source/common/quic/quic_transport_socket_factory.h

@@ -1,11 +1,13 @@
 #pragma once
 
+#include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h"
 #include "envoy/network/transport_socket.h"
 #include "envoy/server/transport_socket_config.h"
 #include "envoy/ssl/context_config.h"
 
 #include "common/common/assert.h"
 
+#include "extensions/transport_sockets/tls/ssl_socket.h"
 #include "extensions/transport_sockets/well_known_names.h"
 
 namespace Envoy {
@@ -25,6 +27,7 @@ class QuicTransportSocketFactoryBase : public Network::TransportSocketFactory {
   }
   bool implementsSecureTransport() const override { return true; }
   bool usesProxyProtocolOptions() const override { return false; }
+  bool supportsAlpn() const override { return true; }
 };
 
 // TODO(danzh): when implement ProofSource, examine of it's necessary to
@@ -42,13 +45,29 @@ class QuicServerTransportSocketFactory : public QuicTransportSocketFactoryBase {
 
 class QuicClientTransportSocketFactory : public QuicTransportSocketFactoryBase {
 public:
-  QuicClientTransportSocketFactory(Envoy::Ssl::ClientContextConfigPtr config)
-      : config_(std::move(config)) {}
+  QuicClientTransportSocketFactory(
+      Ssl::ClientContextConfigPtr config,
+      Server::Configuration::TransportSocketFactoryContext& factory_context);
+
+  // As documented above for QuicTransportSocketFactoryBase, the actual HTTP/3
+  // code does not create transport sockets.
+  // QuicClientTransportSocketFactory::createTransportSocket is called by the
+  // connection grid when upstream HTTP/3 fails over to TCP, and a raw SSL socket
+  // is needed. In this case the QuicClientTransportSocketFactory falls over to
+  // using the fallback factory.
+  Network::TransportSocketPtr
+  createTransportSocket(Network::TransportSocketOptionsSharedPtr options) const override {
+    return fallback_factory_->createTransportSocket(options);
+  }
 
-  const Ssl::ClientContextConfig& clientContextConfig() const { return *config_; }
+  // TODO(14829) make sure that clientContextConfig() is safe when secrets are updated.
+  const Ssl::ClientContextConfig& clientContextConfig() const {
+    return fallback_factory_->config();
+  }
 
 private:
-  std::unique_ptr<const Ssl::ClientContextConfig> config_;
+  // The QUIC client transport socket can create TLS sockets for fallback to TCP.
+  std::unique_ptr<Extensions::TransportSockets::Tls::ClientSslSocketFactory> fallback_factory_;
 };
 
 // Base class to create above QuicTransportSocketFactory for server and client

source/common/upstream/BUILD

@@ -92,6 +92,7 @@ envoy_cc_library(
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
     ] + envoy_select_enable_http3([
         "//source/common/http/http3:conn_pool_lib",
+        "//source/common/http:conn_pool_grid",
     ]),
 )
 

source/common/upstream/cluster_manager_impl.cc

@@ -45,6 +45,7 @@
 #include "common/upstream/subset_lb.h"
 
 #ifdef ENVOY_ENABLE_QUIC
+#include "common/http/conn_pool_grid.h"
 #include "common/http/http3/conn_pool.h"
 #endif
 
@@ -59,6 +60,18 @@ void addOptionsIfNotNull(Network::Socket::OptionsSharedPtr& options,
   }
 }
 
+// Helper function to make sure each protocol in expected_protocols is present
+// in protocols (only used for an ASSERT in debug builds)
+bool contains(const std::vector<Http::Protocol>& protocols,
+              const std::vector<Http::Protocol>& expected_protocols) {
+  for (auto protocol : expected_protocols) {
+    if (std::find(protocols.begin(), protocols.end(), protocol) == protocols.end()) {
+      return false;
+    }
+  }
+  return true;
+}
+
 } // namespace
 
 void ClusterManagerInitHelper::addCluster(ClusterManagerCluster& cm_cluster) {
@@ -1505,14 +1518,25 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool(
     const Network::ConnectionSocket::OptionsSharedPtr& options,
     const Network::TransportSocketOptionsSharedPtr& transport_socket_options, TimeSource& source,
     ClusterConnectivityState& state) {
-  if (protocols.size() == 2) {
-    ASSERT((protocols[0] == Http::Protocol::Http2 && protocols[1] == Http::Protocol::Http11) ||
-           (protocols[1] == Http::Protocol::Http2 && protocols[0] == Http::Protocol::Http11));
+  if (protocols.size() == 3 && runtime_.snapshot().featureEnabled("upstream.use_http3", 100)) {
+    ASSERT(contains(protocols,
+                    {Http::Protocol::Http11, Http::Protocol::Http2, Http::Protocol::Http3}));
+#ifdef ENVOY_ENABLE_QUIC
+    Envoy::Http::ConnectivityGrid::ConnectivityOptions coptions{protocols};
+    return std::make_unique<Http::ConnectivityGrid>(
+        dispatcher, api_.randomGenerator(), host, priority, options, transport_socket_options,
+        state, source, std::chrono::milliseconds(300), coptions);
+#else
+    // Should be blocked by configuration checking at an earlier point.
+    NOT_REACHED_GCOVR_EXCL_LINE;
+#endif
+  }
+  if (protocols.size() >= 2) {
+    ASSERT(contains(protocols, {Http::Protocol::Http11, Http::Protocol::Http2}));
     return std::make_unique<Http::HttpConnPoolImplMixed>(dispatcher, api_.randomGenerator(), host,
                                                          priority, options,
                                                          transport_socket_options, state);
   }
-
   if (protocols.size() == 1 && protocols[0] == Http::Protocol::Http2 &&
       runtime_.snapshot().featureEnabled("upstream.use_http2", 100)) {
     return Http::Http2::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority,

source/common/upstream/upstream_impl.cc

@@ -925,16 +925,21 @@ ClusterInfoImpl::upstreamHttpProtocol(absl::optional<Http::Protocol> downstream_
   if (downstream_protocol.has_value() &&
       features_ & Upstream::ClusterInfo::Features::USE_DOWNSTREAM_PROTOCOL) {
     return {downstream_protocol.value()};
-  } else if (features_ & Upstream::ClusterInfo::Features::USE_ALPN) {
-    ASSERT(!(features_ & Upstream::ClusterInfo::Features::HTTP3));
-    return {Http::Protocol::Http2, Http::Protocol::Http11};
-  } else {
-    if (features_ & Upstream::ClusterInfo::Features::HTTP3) {
-      return {Http::Protocol::Http3};
+  }
+
+  if (features_ & Upstream::ClusterInfo::Features::USE_ALPN) {
+    if (!(features_ & Upstream::ClusterInfo::Features::HTTP3)) {
+      return {Http::Protocol::Http2, Http::Protocol::Http11};
     }
-    return {(features_ & Upstream::ClusterInfo::Features::HTTP2) ? Http::Protocol::Http2
-                                                                 : Http::Protocol::Http11};
+    return {Http::Protocol::Http3, Http::Protocol::Http2, Http::Protocol::Http11};
   }
+
+  if (features_ & Upstream::ClusterInfo::Features::HTTP3) {
+    return {Http::Protocol::Http3};
+  }
+
+  return {(features_ & Upstream::ClusterInfo::Features::HTTP2) ? Http::Protocol::Http2
+                                                               : Http::Protocol::Http11};
 }
 
 ClusterImplBase::ClusterImplBase(

source/extensions/transport_sockets/tls/ssl_socket.h

@@ -116,6 +116,8 @@ class ClientSslSocketFactory : public Network::TransportSocketFactory,
   // Secret::SecretCallbacks
   void onAddOrUpdateSecret() override;
 
+  const Ssl::ClientContextConfig& config() const { return *config_; }
+
 private:
   Envoy::Ssl::ContextManager& manager_;
   Stats::Scope& stats_scope_;

source/extensions/upstreams/http/config.cc

@@ -48,6 +48,9 @@ getHttp3Options(const envoy::extensions::upstreams::http::v3::HttpProtocolOption
       options.use_downstream_protocol_config().has_http3_protocol_options()) {
     return options.use_downstream_protocol_config().http3_protocol_options();
   }
+  if (options.has_auto_config()) {
+    return options.auto_config().http3_protocol_options();
+  }
   return options.explicit_http_config().http3_protocol_options();
 }
 
@@ -107,6 +110,7 @@ ProtocolOptionsConfigImpl::ProtocolOptionsConfigImpl(
   if (options.has_auto_config()) {
     use_http2_ = true;
     use_alpn_ = true;
+    use_http3_ = options.auto_config().has_http3_protocol_options();
   }
 }
 

test/common/upstream/BUILD

@@ -608,6 +608,7 @@ envoy_cc_test(
         "//source/common/upstream:static_cluster_lib",
         "//source/common/upstream:strict_dns_cluster_lib",
         "//source/extensions/transport_sockets/raw_buffer:config",
+        "//source/extensions/transport_sockets/tls:config",
         "//source/extensions/upstreams/http:config",
         "//source/server:transport_socket_config_lib",
         "//test/common/stats:stat_test_utility_lib",

test/common/upstream/upstream_impl_test.cc

@@ -62,7 +62,7 @@ class UpstreamImplTestBase {
   UpstreamImplTestBase() : api_(Api::createApiForTest(stats_, random_)) {}
 
   NiceMock<Server::MockAdmin> admin_;
-  Ssl::MockContextManager ssl_context_manager_;
+  NiceMock<Ssl::MockContextManager> ssl_context_manager_;
   NiceMock<MockClusterManager> cm_;
   NiceMock<LocalInfo::MockLocalInfo> local_info_;
   NiceMock<Event::MockDispatcher> dispatcher_;
@@ -2218,7 +2218,7 @@ class ClusterInfoImplTest : public testing::Test {
   };
 
   Stats::TestUtil::TestStore stats_;
-  Ssl::MockContextManager ssl_context_manager_;
+  NiceMock<Ssl::MockContextManager> ssl_context_manager_;
   std::shared_ptr<Network::MockDnsResolver> dns_resolver_{new NiceMock<Network::MockDnsResolver>()};
   NiceMock<Event::MockDispatcher> dispatcher_;
   NiceMock<Runtime::MockLoader> runtime_;
@@ -3165,7 +3165,6 @@ TEST_F(ClusterInfoImplTest, Http3) {
               - exact: 127.0.0.1
   )EOF",
                                                        Network::Address::IpVersion::v4);
-
   auto cluster1 = makeCluster(yaml);
   ASSERT_TRUE(cluster1->info()->idleTimeout().has_value());
   EXPECT_EQ(std::chrono::hours(1), cluster1->info()->idleTimeout().value());
@@ -3250,6 +3249,64 @@ TEST_F(ClusterInfoImplTest, Http3BadConfig) {
   EXPECT_THROW_WITH_REGEX(makeCluster(yaml), EnvoyException,
                           "HTTP3 requires a QuicUpstreamTransport transport socket: name.*");
 }
+
+TEST_F(ClusterInfoImplTest, Http3Auto) {
+  const std::string yaml = TestEnvironment::substitute(R"EOF(
+    name: name
+    connect_timeout: 0.25s
+    type: STRICT_DNS
+    lb_policy: MAGLEV
+    load_assignment:
+        endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: foo.bar.com
+                    port_value: 443
+    transport_socket:
+      name: envoy.transport_sockets.quic
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicUpstreamTransport
+        upstream_tls_context:
+          common_tls_context:
+            tls_certificates:
+            - certificate_chain:
+                filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem"
+              private_key:
+                filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_key.pem"
+            validation_context:
+              trusted_ca:
+                filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem"
+              match_subject_alt_names:
+              - exact: localhost
+              - exact: 127.0.0.1
+  )EOF",
+                                                       Network::Address::IpVersion::v4);
+
+  auto cluster1 = makeCluster(yaml);
+  ASSERT_TRUE(cluster1->info()->idleTimeout().has_value());
+  EXPECT_EQ(std::chrono::hours(1), cluster1->info()->idleTimeout().value());
+
+  const std::string auto_http3 = R"EOF(
+    typed_extension_protocol_options:
+      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+        auto_config:
+          http3_protocol_options:
+            quic_protocol_options:
+              max_concurrent_streams: 2
+        common_http_protocol_options:
+          idle_timeout: 1s
+  )EOF";
+
+  auto auto_h3 = makeCluster(yaml + auto_http3);
+  EXPECT_EQ(Http::Protocol::Http3,
+            auto_h3->info()->upstreamHttpProtocol({Http::Protocol::Http10})[0]);
+  EXPECT_EQ(
+      auto_h3->info()->http3Options().quic_protocol_options().max_concurrent_streams().value(), 2);
+}
+
 #else
 TEST_F(ClusterInfoImplTest, Http3BadConfig) {
   const std::string yaml = TestEnvironment::substitute(R"EOF(