Skip to content

Commit

Permalink
Move cluster dns config and validate in strict dns clusters
Browse files Browse the repository at this point in the history
Signed-off-by: Steven Jin Xuan <sjinxuan@microsoft.com>
  • Loading branch information
Stevenjin8 committed Sep 26, 2024
1 parent b203e3e commit e1a1e5f
Show file tree
Hide file tree
Showing 11 changed files with 207 additions and 37 deletions.
1 change: 1 addition & 0 deletions api/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ proto_library(
"//envoy/extensions/access_loggers/wasm/v3:pkg",
"//envoy/extensions/bootstrap/internal_listener/v3:pkg",
"//envoy/extensions/clusters/aggregate/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
"//envoy/extensions/clusters/redis/v3:pkg",
"//envoy/extensions/common/async_files/v3:pkg",
Expand Down
1 change: 1 addition & 0 deletions api/envoy/config/cluster/v3/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ api_proto_package(
"//envoy/annotations:pkg",
"//envoy/config/core/v3:pkg",
"//envoy/config/endpoint/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"//envoy/type/metadata/v3:pkg",
"//envoy/type/v3:pkg",
"@com_github_cncf_xds//udpa/annotations:pkg",
Expand Down
27 changes: 7 additions & 20 deletions api/envoy/config/cluster/v3/cluster.proto
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import "envoy/config/core/v3/health_check.proto";
import "envoy/config/core/v3/protocol.proto";
import "envoy/config/core/v3/resolver.proto";
import "envoy/config/endpoint/v3/endpoint.proto";
import "envoy/extensions/clusters/dns/v3/cluster.proto";
import "envoy/type/metadata/v3/metadata.proto";
import "envoy/type/v3/percent.proto";

Expand Down Expand Up @@ -45,7 +46,7 @@ message ClusterCollection {
}

// Configuration for a single upstream cluster.
// [#next-free-field: 59]
// [#next-free-field: 60]
message Cluster {
option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster";

Expand Down Expand Up @@ -678,24 +679,6 @@ message Cluster {
core.v3.HealthStatusSet override_host_status = 8;
}

message RefreshRate {
option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster.RefreshRate";

// Specifies the base interval between refreshes. This parameter is required and must be greater
// than zero and less than
// :ref:`max_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.max_interval>`.
google.protobuf.Duration base_interval = 1 [(validate.rules).duration = {
required: true
gt {nanos: 1000000}
}];

// Specifies the maximum interval between refreshes. This parameter is optional, but must be
// greater than or equal to the
// :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>` if set. The default
// is 10 times the :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>`.
google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
}

message PreconnectPolicy {
// Indicates how many streams (rounded up) can be anticipated per-upstream for each
// incoming stream. This is useful for high-QPS or latency-sensitive services. Preconnecting
Expand Down Expand Up @@ -944,6 +927,9 @@ message Cluster {
// [#next-major-version: make this a list of typed extensions.]
map<string, google.protobuf.Any> typed_extension_protocol_options = 36;

// [#extension-category: envoy.clusters.dns]
envoy.extensions.clusters.dns.v3.DnsConfig dns_config = 59;

// If the DNS refresh rate is specified and the cluster type is either
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
Expand Down Expand Up @@ -975,7 +961,8 @@ message Cluster {
// other than :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>` and
// :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>` this setting is
// ignored.
RefreshRate dns_failure_refresh_rate = 44;

envoy.extensions.clusters.dns.v3.DnsConfig.RefreshRate dns_failure_refresh_rate = 44;

// Optional configuration for setting cluster's DNS refresh rate. If the value is set to true,
// cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS
Expand Down
9 changes: 9 additions & 0 deletions api/envoy/extensions/clusters/dns/v3/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.

load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")

licenses(["notice"]) # Apache 2

api_proto_package(
deps = ["@com_github_cncf_xds//udpa/annotations:pkg"],
)
130 changes: 130 additions & 0 deletions api/envoy/extensions/clusters/dns/v3/cluster.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
syntax = "proto3";

package envoy.extensions.clusters.dns.v3;

import "google/protobuf/any.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/wrappers.proto";

import "udpa/annotations/migrate.proto";
import "udpa/annotations/security.proto";
import "udpa/annotations/status.proto";
import "udpa/annotations/versioning.proto";
import "validate/validate.proto";

option java_package = "io.envoyproxy.envoy.extensions.clusters.dns.v3";
option java_outer_classname = "ClusterProto";
option java_multiple_files = true;
option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dns/v3;dnsv3";
option (udpa.annotations.file_status).package_version_status = ACTIVE;

// import "envoy/config/core/v3/extension.proto";

// [#protodoc-title: DNS cluster configuration]

// Configuration for the dynamic forward proxy cluster. See the :ref:`architecture overview
// <arch_overview_http_dynamic_forward_proxy>` for more information.
// [#extension: envoy.clusters.dns]

// [#next-free-field: 55]
message DnsConfig {
// When V4_ONLY is selected, the DNS resolver will only perform a lookup for
// addresses in the IPv4 family. If V6_ONLY is selected, the DNS resolver will
// only perform a lookup for addresses in the IPv6 family. If AUTO is
// specified, the DNS resolver will first perform a lookup for addresses in
// the IPv6 family and fallback to a lookup for addresses in the IPv4 family.
// This is semantically equivalent to a non-existent V6_PREFERRED option.
// AUTO is a legacy name that is more opaque than
// necessary and will be deprecated in favor of V6_PREFERRED in a future major version of the API.
// If V4_PREFERRED is specified, the DNS resolver will first perform a lookup for addresses in the
// IPv4 family and fallback to a lookup for addresses in the IPv6 family. i.e., the callback
// target will only get v6 addresses if there were NO v4 addresses to return.
// If ALL is specified, the DNS resolver will perform a lookup for both IPv4 and IPv6 families,
// and return all resolved addresses. When this is used, Happy Eyeballs will be enabled for
// upstream connections. Refer to :ref:`Happy Eyeballs Support <arch_overview_happy_eyeballs>`
// for more information.
// For cluster types other than
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>` and
// :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// this setting is
// ignored.
// [#next-major-version: deprecate AUTO in favor of a V6_PREFERRED option.]
enum LookupFamily {
AUTO = 0;
V4_ONLY = 1;
V6_ONLY = 2;
V4_PREFERRED = 3;
ALL = 4;
}

message RefreshRate {
option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster.RefreshRate";

// Specifies the base interval between refreshes. This parameter is required and must be greater
// than zero and less than
// :ref:`max_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.max_interval>`.
google.protobuf.Duration base_interval = 1 [(validate.rules).duration = {
required: true
gt {nanos: 1000000}
}];

// Specifies the maximum interval between refreshes. This parameter is optional, but must be
// greater than or equal to the
// :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>` if set. The default
// is 10 times the :ref:`base_interval <envoy_v3_api_field_config.cluster.v3.Cluster.RefreshRate.base_interval>`.
google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
}

// If the DNS refresh rate is specified and the cluster type is either
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// this value is used as the cluster’s DNS refresh
// rate. The value configured must be at least 1ms. If this setting is not specified, the
// value defaults to 5000ms. For cluster types other than
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`
// and :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`
// this setting is ignored.
google.protobuf.Duration refresh_rate = 16 [(validate.rules).duration = {gt {nanos: 1000000}}];

// If the DNS failure refresh rate is specified and the cluster type is either
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// this is used as the cluster’s DNS refresh rate when requests are failing. If this setting is
// not specified, the failure refresh rate defaults to the DNS refresh rate. For cluster types
// other than :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>` and
// :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>` this setting is
// ignored.
RefreshRate dns_failure_refresh_rate = 44;

// Optional configuration for setting cluster's DNS refresh rate. If the value is set to true,
// cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS
// resolution.
bool respect_ttl = 39;

// The DNS IP address resolution policy. If this setting is not specified, the
// value defaults to
// :ref:`AUTO<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DnsLookupFamily.AUTO>`.
LookupFamily lookup_family = 17 [(validate.rules).enum = {defined_only: true}];

// DNS resolver type configuration extension. This extension can be used to configure c-ares, apple,
// or any other DNS resolver types and the related parameters.
// For example, an object of
// :ref:`CaresDnsResolverConfig <envoy_v3_api_msg_extensions.network.dns_resolver.cares.v3.CaresDnsResolverConfig>`
// can be packed into this ``typed_dns_resolver_config``. This configuration replaces the
// :ref:`dns_resolution_config <envoy_v3_api_field_config.cluster.v3.Cluster.dns_resolution_config>`
// configuration.
// During the transition period when both ``dns_resolution_config`` and ``typed_dns_resolver_config`` exists,
// when ``typed_dns_resolver_config`` is in place, Envoy will use it and ignore ``dns_resolution_config``.
// When ``typed_dns_resolver_config`` is missing, the default behavior is in place.
// [#extension-category: envoy.network.dns_resolver]
// FIXMEcore.v3.TypedExtensionConfig typed_resolver_config = 55;

// Optional configuration for having cluster readiness block on warm-up. Currently, only applicable for
// :ref:`STRICT_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS>`,
// or :ref:`LOGICAL_DNS<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.LOGICAL_DNS>`,
// or :ref:`Redis Cluster<arch_overview_redis>`.
// If true, cluster readiness blocks on warm-up. If false, the cluster will complete
// initialization whether or not warm-up has completed. Defaults to true.
google.protobuf.BoolValue wait_for_warm_on_init = 54;
}
1 change: 1 addition & 0 deletions api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ api_proto_package(
"//envoy/config/cluster/v3:pkg",
"//envoy/config/common/key_value/v3:pkg",
"//envoy/config/core/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"@com_github_cncf_xds//udpa/annotations:pkg",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import "envoy/config/common/key_value/v3/config.proto";
import "envoy/config/core/v3/address.proto";
import "envoy/config/core/v3/extension.proto";
import "envoy/config/core/v3/resolver.proto";
import "envoy/extensions/clusters/dns/v3/cluster.proto";

import "google/protobuf/duration.proto";
import "google/protobuf/wrappers.proto";
Expand Down Expand Up @@ -97,7 +98,7 @@ message DnsCacheConfig {
// If the DNS failure refresh rate is specified,
// this is used as the cache's DNS refresh rate when DNS requests are failing. If this setting is
// not specified, the failure refresh rate defaults to the dns_refresh_rate.
config.cluster.v3.Cluster.RefreshRate dns_failure_refresh_rate = 6;
clusters.dns.v3.DnsConfig.RefreshRate dns_failure_refresh_rate = 6;

// The config of circuit breakers for resolver. It provides a configurable threshold.
// Envoy will use dns cache circuit breakers with default settings even if this value is not set.
Expand Down
1 change: 1 addition & 0 deletions api/versioning/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ proto_library(
"//envoy/extensions/access_loggers/wasm/v3:pkg",
"//envoy/extensions/bootstrap/internal_listener/v3:pkg",
"//envoy/extensions/clusters/aggregate/v3:pkg",
"//envoy/extensions/clusters/dns/v3:pkg",
"//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
"//envoy/extensions/clusters/redis/v3:pkg",
"//envoy/extensions/common/async_files/v3:pkg",
Expand Down
59 changes: 46 additions & 13 deletions source/extensions/clusters/strict_dns/strict_dns_cluster.cc
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,44 @@ StrictDnsClusterImpl::StrictDnsClusterImpl(const envoy::config::cluster::v3::Clu
absl::Status& creation_status)
: BaseDynamicClusterImpl(cluster, context, creation_status),
load_assignment_(cluster.load_assignment()),
local_info_(context.serverFactoryContext().localInfo()), dns_resolver_(dns_resolver),
dns_refresh_rate_ms_(
std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))),
dns_jitter_ms_(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_jitter, 0)),
respect_dns_ttl_(cluster.respect_dns_ttl()) {
local_info_(context.serverFactoryContext().localInfo()), dns_resolver_(dns_resolver) {
// dns_refresh_rate_ms_(
// std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate,
// 5000))),
// dns_jitter_ms_(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_jitter, 0)),
// respect_dns_ttl_(cluster.respect_dns_ttl())

if (cluster.has_dns_config()) {
if (cluster.has_dns_refresh_rate() /* deprecated */) {
throw EnvoyException("Only one of dns_refresh_rate or dns_config can be specified.");
}
if (cluster.has_dns_jitter() /* deprecated */) {
throw EnvoyException("Only one of dns_jitter or dns_config can be specified.");
}
if (cluster.has_typed_dns_resolver_config() /* deprecated */) {
throw EnvoyException(
"Only one of typed_dns_resolution_config or dns_config can be specified.");
}
if (cluster.dns_lookup_family() !=
envoy::config::cluster::v3::Cluster_DnsLookupFamily::Cluster_DnsLookupFamily_AUTO &&
cluster.dns_config().lookup_family() !=
envoy::extensions::clusters::dns::v3::DnsConfig::LookupFamily::
DnsConfig_LookupFamily_AUTO /* deprecated */) {
throw EnvoyException("Only one of dns_lookup_family or dns_config can be specified.");
}
if (cluster.has_dns_refresh_rate() /* deprecated */) {
throw EnvoyException("Only one of dns_refresh_rate or dns_config can be specified.");
}
if (cluster.has_dns_failure_refresh_rate() /* deprecated */) {
throw EnvoyException("Only one of dns_failure_refresh_rate or dns_config can be specified.");
}

if (cluster.has_wait_for_warm_on_init() /* deprecated */) {
throw EnvoyException("Only one of wait_for_warm_on_init or dns_config can be specified.");
}
// no good way of checking respect_ttl
}

failure_backoff_strategy_ =
Config::Utility::prepareDnsRefreshStrategy<envoy::config::cluster::v3::Cluster>(
cluster, dns_refresh_rate_ms_.count(),
Expand Down Expand Up @@ -66,8 +99,8 @@ void StrictDnsClusterImpl::startPreInit() {
for (const ResolveTargetPtr& target : resolve_targets_) {
target->startResolve();
}
// If the config provides no endpoints, the cluster is initialized immediately as if all hosts are
// resolved in failure.
// If the config provides no endpoints, the cluster is initialized immediately as if all hosts
// are resolved in failure.
if (resolve_targets_.empty() || !wait_for_warm_on_init_) {
onPreInitComplete();
}
Expand All @@ -79,10 +112,10 @@ void StrictDnsClusterImpl::updateAllHosts(const HostVector& hosts_added,
PriorityStateManager priority_state_manager(*this, local_info_, nullptr, random_);
// At this point we know that we are different so make a new host list and notify.
//
// TODO(dio): The uniqueness of a host address resolved in STRICT_DNS cluster per priority is not
// guaranteed. Need a clear agreement on the behavior here, whether it is allowable to have
// duplicated hosts inside a priority. And if we want to enforce this behavior, it should be done
// inside the priority state manager.
// TODO(dio): The uniqueness of a host address resolved in STRICT_DNS cluster per priority is
// not guaranteed. Need a clear agreement on the behavior here, whether it is allowable to have
// duplicated hosts inside a priority. And if we want to enforce this behavior, it should be
// done inside the priority state manager.
for (const ResolveTargetPtr& target : resolve_targets_) {
priority_state_manager.initializePriorityFor(target->locality_lb_endpoints_);
for (const HostSharedPtr& host : target->hosts_) {
Expand Down Expand Up @@ -140,8 +173,8 @@ void StrictDnsClusterImpl::ResolveTarget::startResolve() {
const auto& addrinfo = resp.addrInfo();
// TODO(mattklein123): Currently the DNS interface does not consider port. We need to
// make a new address that has port in it. We need to both support IPv6 as well as
// potentially move port handling into the DNS interface itself, which would work better
// for SRV.
// potentially move port handling into the DNS interface itself, which would work
// better for SRV.
ASSERT(addrinfo.address_ != nullptr);
auto address = Network::Utility::getAddressWithPort(*(addrinfo.address_), port_);
if (all_new_hosts.count(address->asString()) > 0) {
Expand Down
6 changes: 3 additions & 3 deletions source/extensions/clusters/strict_dns/strict_dns_cluster.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ class StrictDnsClusterImpl : public BaseDynamicClusterImpl {
const LocalInfo::LocalInfo& local_info_;
Network::DnsResolverSharedPtr dns_resolver_;
std::list<ResolveTargetPtr> resolve_targets_;
const std::chrono::milliseconds dns_refresh_rate_ms_;
const std::chrono::milliseconds dns_jitter_ms_;
std::chrono::milliseconds dns_refresh_rate_ms_;
std::chrono::milliseconds dns_jitter_ms_;
BackOffStrategyPtr failure_backoff_strategy_;
const bool respect_dns_ttl_;
bool respect_dns_ttl_;
Network::DnsLookupFamily dns_lookup_family_;
uint32_t overprovisioning_factor_;
bool weighted_priority_health_;
Expand Down
6 changes: 6 additions & 0 deletions source/extensions/extensions_metadata.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,12 @@ envoy.clusters.aggregate:
- envoy.clusters
security_posture: requires_trusted_downstream_and_upstream
status: stable
# FIXME
envoy.clusters.dns:
categories:
- envoy.clusters
security_posture: unknown
status: wip
envoy.clusters.dynamic_forward_proxy:
categories:
- envoy.clusters
Expand Down

0 comments on commit e1a1e5f

Please sign in to comment.