Skip to content

Commit

Permalink
quic: HTTP/3 upstream initial checkin (#15027)
Browse files Browse the repository at this point in the history
This is working end to end for a number of the protocol integration tests, but still needs a bunch of work before it's production ready (I triaged some of the excludes but not all, watermarks not working etc).
I think given this works for the happy path it's worth checking in, and I can iterate on getting individual tests working and doing the TODOs.

Risk Level: low (minor core refactors, major changes are all behind hidden config)
Testing: integration tests, some unit tests
Docs Changes: n/a
Release Notes: n/a
#14829

Signed-off-by: Alyssa Wilk <alyssar@chromium.org>
  • Loading branch information
alyssawilk authored Mar 3, 2021
1 parent 50e8127 commit 0a58f84
Show file tree
Hide file tree
Showing 34 changed files with 592 additions and 93 deletions.
2 changes: 1 addition & 1 deletion include/envoy/upstream/cluster_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ class ClusterManagerFactory {
ResourcePriority priority, std::vector<Http::Protocol>& protocol,
const Network::ConnectionSocket::OptionsSharedPtr& options,
const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
ClusterConnectivityState& state) PURE;
TimeSource& time_source, ClusterConnectivityState& state) PURE;

/**
* Allocate a TCP connection pool for the host. Pools are separated by 'priority' and
Expand Down
1 change: 1 addition & 0 deletions include/envoy/upstream/upstream.h
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,7 @@ class PrioritySet {
COUNTER(upstream_cx_destroy_with_active_rq) \
COUNTER(upstream_cx_http1_total) \
COUNTER(upstream_cx_http2_total) \
COUNTER(upstream_cx_http3_total) \
COUNTER(upstream_cx_idle_timeout) \
COUNTER(upstream_cx_max_requests) \
COUNTER(upstream_cx_none_healthy) \
Expand Down
3 changes: 1 addition & 2 deletions source/common/http/conn_pool_base.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ wrapTransportSocketOptions(Network::TransportSocketOptionsSharedPtr transport_so
fallbacks.push_back(Http::Utility::AlpnNames::get().Http2);
break;
case Http::Protocol::Http3:
// TODO(snowp): Add once HTTP/3 upstream support is added.
NOT_IMPLEMENTED_GCOVR_EXCL_LINE;
// TODO(#14829) hard-code H3 ALPN, consider failing if other things are negotiated.
break;
}
}
Expand Down
46 changes: 35 additions & 11 deletions source/common/http/http2/conn_pool.cc
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,12 @@

namespace Envoy {
namespace Http {
namespace Http2 {

// All streams are 2^31. Client streams are half that, minus stream 0. Just to be on the safe
// side we do 2^29.
static const uint64_t DEFAULT_MAX_STREAMS = (1 << 29);

void ActiveClient::onGoAway(Http::GoAwayErrorCode) {
void MultiplexedActiveClientBase::onGoAway(Http::GoAwayErrorCode) {
ENVOY_CONN_LOG(debug, "remote goaway", *codec_client_);
parent_.host()->cluster().stats().upstream_cx_close_notify_.inc();
if (state_ != ActiveClient::State::DRAINING) {
Expand All @@ -28,7 +27,7 @@ void ActiveClient::onGoAway(Http::GoAwayErrorCode) {
}
}

void ActiveClient::onStreamDestroy() {
void MultiplexedActiveClientBase::onStreamDestroy() {
parent().onStreamClosed(*this, false);

// If we are destroying this stream because of a disconnect, do not check for drain here. We will
Expand All @@ -39,7 +38,7 @@ void ActiveClient::onStreamDestroy() {
}
}

void ActiveClient::onStreamReset(Http::StreamResetReason reason) {
void MultiplexedActiveClientBase::onStreamReset(Http::StreamResetReason reason) {
if (reason == StreamResetReason::ConnectionTermination ||
reason == StreamResetReason::ConnectionFailure) {
parent_.host()->cluster().stats().upstream_rq_pending_failure_eject_.inc();
Expand All @@ -55,31 +54,56 @@ uint64_t maxStreamsPerConnection(uint64_t max_streams_config) {
return (max_streams_config != 0) ? max_streams_config : DEFAULT_MAX_STREAMS;
}

ActiveClient::ActiveClient(HttpConnPoolImplBase& parent)
MultiplexedActiveClientBase::MultiplexedActiveClientBase(HttpConnPoolImplBase& parent,
Stats::Counter& cx_total)
: Envoy::Http::ActiveClient(
parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()),
parent.host()->cluster().http2Options().max_concurrent_streams().value()) {
codec_client_->setCodecClientCallbacks(*this);
codec_client_->setCodecConnectionCallbacks(*this);
parent.host()->cluster().stats().upstream_cx_http2_total_.inc();
cx_total.inc();
}

ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
Upstream::Host::CreateConnectionData& data)
MultiplexedActiveClientBase::MultiplexedActiveClientBase(HttpConnPoolImplBase& parent,
Stats::Counter& cx_total,
Upstream::Host::CreateConnectionData& data)
: Envoy::Http::ActiveClient(
parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()),
parent.host()->cluster().http2Options().max_concurrent_streams().value(), data) {
codec_client_->setCodecClientCallbacks(*this);
codec_client_->setCodecConnectionCallbacks(*this);
cx_total.inc();
}

MultiplexedActiveClientBase::MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent,
Upstream::Host::CreateConnectionData& data,
Stats::Counter& cx_total)
: Envoy::Http::ActiveClient(
parent, maxStreamsPerConnection(parent.host()->cluster().maxRequestsPerConnection()),
parent.host()->cluster().http2Options().max_concurrent_streams().value(), data) {
codec_client_->setCodecClientCallbacks(*this);
codec_client_->setCodecConnectionCallbacks(*this);
parent.host()->cluster().stats().upstream_cx_http2_total_.inc();
cx_total.inc();
}

bool ActiveClient::closingWithIncompleteStream() const { return closed_with_active_rq_; }
bool MultiplexedActiveClientBase::closingWithIncompleteStream() const {
return closed_with_active_rq_;
}

RequestEncoder& ActiveClient::newStreamEncoder(ResponseDecoder& response_decoder) {
RequestEncoder& MultiplexedActiveClientBase::newStreamEncoder(ResponseDecoder& response_decoder) {
return codec_client_->newStream(response_decoder);
}

namespace Http2 {
ActiveClient::ActiveClient(HttpConnPoolImplBase& parent)
: MultiplexedActiveClientBase(parent,
parent.host()->cluster().stats().upstream_cx_http2_total_) {}

ActiveClient::ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
Upstream::Host::CreateConnectionData& data)
: MultiplexedActiveClientBase(parent, data,
parent.host()->cluster().stats().upstream_cx_http2_total_) {}

ConnectionPool::InstancePtr
allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
Expand Down
35 changes: 26 additions & 9 deletions source/common/http/http2/conn_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,19 @@

namespace Envoy {
namespace Http {
namespace Http2 {

/**
* Implementation of an active client for HTTP/2
* Active client base for HTTP/2 and HTTP/3
*/
class ActiveClient : public CodecClientCallbacks,
public Http::ConnectionCallbacks,
public Envoy::Http::ActiveClient {
// TODO(#14829) move to source/common/http/conn_pool_base.h
class MultiplexedActiveClientBase : public CodecClientCallbacks,
public Http::ConnectionCallbacks,
public Envoy::Http::ActiveClient {
public:
ActiveClient(HttpConnPoolImplBase& parent);
ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
Upstream::Host::CreateConnectionData& data);
~ActiveClient() override = default;
MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, Stats::Counter& cx_total);
MultiplexedActiveClientBase(HttpConnPoolImplBase& parent, Stats::Counter& cx_total,
Upstream::Host::CreateConnectionData& data);
~MultiplexedActiveClientBase() override = default;

// ConnPoolImpl::ActiveClient
bool closingWithIncompleteStream() const override;
Expand All @@ -34,9 +34,26 @@ class ActiveClient : public CodecClientCallbacks,
// Http::ConnectionCallbacks
void onGoAway(Http::GoAwayErrorCode error_code) override;

protected:
MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent,
Upstream::Host::CreateConnectionData& data, Stats::Counter& cx_total);

private:
bool closed_with_active_rq_{};
};

namespace Http2 {

/**
* Implementation of an active client for HTTP/2
*/
class ActiveClient : public MultiplexedActiveClientBase {
public:
ActiveClient(HttpConnPoolImplBase& parent);
ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
Upstream::Host::CreateConnectionData& data);
};

ConnectionPool::InstancePtr
allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
Expand Down
25 changes: 25 additions & 0 deletions source/common/http/http3/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,20 @@ licenses(["notice"]) # Apache 2

envoy_package()

envoy_cc_library(
name = "conn_pool_lib",
srcs = ["conn_pool.cc"],
hdrs = ["conn_pool.h"],
deps = [
":quic_client_connection_factory_lib",
"//include/envoy/event:dispatcher_interface",
"//include/envoy/upstream:upstream_interface",
"//source/common/http:codec_client_lib",
"//source/common/http:conn_pool_base_lib",
"//source/common/http/http2:conn_pool_lib",
],
)

envoy_cc_library(
name = "quic_codec_factory_lib",
hdrs = ["quic_codec_factory.h"],
Expand All @@ -18,6 +32,17 @@ envoy_cc_library(
],
)

envoy_cc_library(
name = "quic_client_connection_factory_lib",
hdrs = ["quic_client_connection_factory.h"],
deps = [
"//include/envoy/config:typed_config_interface",
"//include/envoy/network:connection_interface",
"//include/envoy/ssl:context_config_interface",
"@envoy_api//envoy/config/listener/v3:pkg_cc_proto",
],
)

envoy_cc_library(
name = "well_known_names",
hdrs = ["well_known_names.h"],
Expand Down
57 changes: 57 additions & 0 deletions source/common/http/http3/conn_pool.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#include "common/http/http3/conn_pool.h"

#include <cstdint>

#include "envoy/event/dispatcher.h"
#include "envoy/upstream/upstream.h"

#include "common/config/utility.h"
#include "common/http/http3/quic_client_connection_factory.h"
#include "common/http/http3/well_known_names.h"
#include "common/http/utility.h"
#include "common/network/address_impl.h"
#include "common/network/utility.h"
#include "common/runtime/runtime_features.h"

namespace Envoy {
namespace Http {
namespace Http3 {

ConnectionPool::InstancePtr
allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
const Network::ConnectionSocket::OptionsSharedPtr& options,
const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
Upstream::ClusterConnectivityState& state, TimeSource& time_source) {
return std::make_unique<FixedHttpConnPoolImpl>(
host, priority, dispatcher, options, transport_socket_options, random_generator, state,
[&dispatcher, &time_source](HttpConnPoolImplBase* pool) {
Upstream::Host::CreateConnectionData data{};
data.host_description_ = pool->host();
auto host_address = data.host_description_->address();
auto source_address = data.host_description_->cluster().sourceAddress();
if (!source_address.get()) {
source_address = Network::Utility::getLocalAddress(host_address->ip()->version());
}
Network::TransportSocketFactory& transport_socket_factory =
data.host_description_->transportSocketFactory();
data.connection_ =
Config::Utility::getAndCheckFactoryByName<Http::QuicClientConnectionFactory>(
Http::QuicCodecNames::get().Quiche)
.createQuicNetworkConnection(host_address, source_address, transport_socket_factory,
data.host_description_->cluster().statsScope(),
dispatcher, time_source);
return std::make_unique<ActiveClient>(*pool, data);
},
[](Upstream::Host::CreateConnectionData& data, HttpConnPoolImplBase* pool) {
CodecClientPtr codec{new CodecClientProd(
CodecClient::Type::HTTP3, std::move(data.connection_), data.host_description_,
pool->dispatcher(), pool->randomGenerator())};
return codec;
},
std::vector<Protocol>{Protocol::Http3});
}

} // namespace Http3
} // namespace Http
} // namespace Envoy
33 changes: 33 additions & 0 deletions source/common/http/http3/conn_pool.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#pragma once

#include <cstdint>

#include "envoy/upstream/upstream.h"

#include "common/http/codec_client.h"
#include "common/http/http2/conn_pool.h"

namespace Envoy {
namespace Http {
namespace Http3 {

// TODO(#14829) the constructor of Http2::ActiveClient sets max requests per
// connection based on HTTP/2 config. Sort out the HTTP/3 config story.
class ActiveClient : public MultiplexedActiveClientBase {
public:
ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
Upstream::Host::CreateConnectionData& data)
: MultiplexedActiveClientBase(
parent, parent.host()->cluster().stats().upstream_cx_http3_total_, data) {}
};

ConnectionPool::InstancePtr
allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
const Network::ConnectionSocket::OptionsSharedPtr& options,
const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
Upstream::ClusterConnectivityState& state, TimeSource& time_source);

} // namespace Http3
} // namespace Http
} // namespace Envoy
29 changes: 29 additions & 0 deletions source/common/http/http3/quic_client_connection_factory.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#pragma once

#include <string>

#include "envoy/config/listener/v3/quic_config.pb.h"
#include "envoy/config/typed_config.h"
#include "envoy/network/connection.h"
#include "envoy/ssl/context_config.h"

namespace Envoy {
namespace Http {

// A factory to create EnvoyQuicClientSession and EnvoyQuicClientConnection for QUIC
class QuicClientConnectionFactory : public Config::UntypedFactory {
public:
~QuicClientConnectionFactory() override = default;

virtual std::unique_ptr<Network::ClientConnection>
createQuicNetworkConnection(Network::Address::InstanceConstSharedPtr server_addr,
Network::Address::InstanceConstSharedPtr local_addr,
Network::TransportSocketFactory& transport_socket_factory,
Stats::Scope& stats_scope, Event::Dispatcher& dispatcher,
TimeSource& time_source) PURE;

std::string category() const override { return "envoy.quic_connection"; }
};

} // namespace Http
} // namespace Envoy
1 change: 1 addition & 0 deletions source/common/upstream/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ envoy_cc_library(
"//source/common/http:mixed_conn_pool",
"//source/common/http/http1:conn_pool_lib",
"//source/common/http/http2:conn_pool_lib",
"//source/common/http/http3:conn_pool_lib",
"//source/common/network:resolver_lib",
"//source/common/network:utility_lib",
"//source/common/protobuf:utility_lib",
Expand Down
10 changes: 8 additions & 2 deletions source/common/upstream/cluster_manager_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "common/http/async_client_impl.h"
#include "common/http/http1/conn_pool.h"
#include "common/http/http2/conn_pool.h"
#include "common/http/http3/conn_pool.h"
#include "common/http/mixed_conn_pool.h"
#include "common/network/resolver_impl.h"
#include "common/network/utility.h"
Expand Down Expand Up @@ -1415,7 +1416,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool(
parent_.thread_local_dispatcher_, host, priority, upstream_protocols,
!upstream_options->empty() ? upstream_options : nullptr,
have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr,
parent_.cluster_manager_state_);
parent_.parent_.time_source_, parent_.cluster_manager_state_);
});

if (pool.has_value()) {
Expand Down Expand Up @@ -1484,7 +1485,7 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool(
Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority,
std::vector<Http::Protocol>& protocols,
const Network::ConnectionSocket::OptionsSharedPtr& options,
const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
const Network::TransportSocketOptionsSharedPtr& transport_socket_options, TimeSource& source,
ClusterConnectivityState& state) {
if (protocols.size() == 2) {
ASSERT((protocols[0] == Http::Protocol::Http2 && protocols[1] == Http::Protocol::Http11) ||
Expand All @@ -1499,6 +1500,11 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool(
return Http::Http2::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority,
options, transport_socket_options, state);
}
if (protocols.size() == 1 && protocols[0] == Http::Protocol::Http3 &&
runtime_.snapshot().featureEnabled("upstream.use_http3", 100)) {
return Http::Http3::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority,
options, transport_socket_options, state, source);
}
ASSERT(protocols.size() == 1 && protocols[0] == Http::Protocol::Http11);
return Http::Http1::allocateConnPool(dispatcher, api_.randomGenerator(), host, priority, options,
transport_socket_options, state);
Expand Down
2 changes: 1 addition & 1 deletion source/common/upstream/cluster_manager_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory {
ResourcePriority priority, std::vector<Http::Protocol>& protocol,
const Network::ConnectionSocket::OptionsSharedPtr& options,
const Network::TransportSocketOptionsSharedPtr& transport_socket_options,
ClusterConnectivityState& state) override;
TimeSource& time_source, ClusterConnectivityState& state) override;
Tcp::ConnectionPool::InstancePtr
allocateTcpConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
ResourcePriority priority,
Expand Down
Loading

0 comments on commit 0a58f84

Please sign in to comment.