Enable simple remote connection strategy (#49561)

This commit back ports three commits related to enabling the simple
connection strategy.

Allow simple connection strategy to be configured (#49066)

Currently the simple connection strategy only exists in the code. It
cannot be configured. This commit moves in the direction of allowing it
to be configured. It introduces settings for the addresses and socket
count. Additionally it introduces new settings for the sniff strategy
so that the more generic number of connections and seed node settings
can be deprecated.

The simple settings are not yet registered as the registration is
dependent on follow-up work to validate the settings.

Ensure at least 1 seed configured in remote test (#49389)

This fixes #49384. Currently when we select a random subset of seed
nodes from a list, it is possible for 0 seeds to be selected. This test
depends on at least 1 seed being selected.

Add the simple strategy to cluster settings (#49414)

This is related to #49067. This commit adds the simple connection
strategy settings and strategy mode setting to the cluster settings
registry. With these changes, the simple connection mode can be used.
Additionally, it adds validation to ensure that settings cannot be
misconfigured.

client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java

@@ -83,7 +83,7 @@ public void setupRemoteClusterConfig() throws Exception {
         String transportAddress = (String) nodesResponse.get("transport_address");
 
         ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
-        updateSettingsRequest.transientSettings(Collections.singletonMap("cluster.remote.local_cluster.seeds", transportAddress));
+        updateSettingsRequest.transientSettings(Collections.singletonMap("cluster.remote.local_cluster.sniff.seeds", transportAddress));
         ClusterUpdateSettingsResponse updateSettingsResponse =
             highLevelClient().cluster().putSettings(updateSettingsRequest, RequestOptions.DEFAULT);
         assertThat(updateSettingsResponse.isAcknowledged(), is(true));

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java

@@ -87,7 +87,7 @@ public void setupRemoteClusterConfig() throws IOException {
         String transportAddress = (String) nodesResponse.get("transport_address");
 
         ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
-        updateSettingsRequest.transientSettings(Collections.singletonMap("cluster.remote.local.seeds", transportAddress));
+        updateSettingsRequest.transientSettings(Collections.singletonMap("cluster.remote.local.sniff.seeds", transportAddress));
         ClusterUpdateSettingsResponse updateSettingsResponse =
             client.cluster().putSettings(updateSettingsRequest, RequestOptions.DEFAULT);
         assertThat(updateSettingsResponse.isAcknowledged(), is(true));

qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java

@@ -144,7 +144,7 @@ public void testSearchSkipUnavailable() throws IOException {
         try (MockTransportService remoteTransport = startTransport("node0", new CopyOnWriteArrayList<>(), Version.CURRENT, threadPool)) {
             DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode();
 
-            updateRemoteClusterSettings(Collections.singletonMap("seeds", remoteNode.getAddress().toString()));
+            updateRemoteClusterSettings(Collections.singletonMap("sniff.seeds", remoteNode.getAddress().toString()));
 
             for (int i = 0; i < 10; i++) {
                 restHighLevelClient.index(
@@ -229,7 +229,7 @@ public void testSearchSkipUnavailable() throws IOException {
             assertSearchConnectFailure();
 
             Map<String, Object> map = new HashMap<>();
-            map.put("seeds", null);
+            map.put("sniff.seeds", null);
             map.put("skip_unavailable", null);
             updateRemoteClusterSettings(map);
         }
@@ -248,32 +248,32 @@ public void testSkipUnavailableDependsOnSeeds() throws IOException {
                         () -> client().performRequest(request));
                 assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
                 assertThat(responseException.getMessage(),
-                        containsString("missing required setting [cluster.remote.remote1.seeds] " +
+                        containsString("missing required setting [cluster.remote.remote1.sniff.seeds] " +
                                 "for setting [cluster.remote.remote1.skip_unavailable]"));
             }
 
             Map<String, Object> settingsMap = new HashMap<>();
-            settingsMap.put("seeds", remoteNode.getAddress().toString());
+            settingsMap.put("sniff.seeds", remoteNode.getAddress().toString());
             settingsMap.put("skip_unavailable", randomBoolean());
             updateRemoteClusterSettings(settingsMap);
 
             {
                 //check that seeds cannot be reset alone if skip_unavailable is set
                 Request request = new Request("PUT", "/_cluster/settings");
-                request.setEntity(buildUpdateSettingsRequestBody(Collections.singletonMap("seeds", null)));
+                request.setEntity(buildUpdateSettingsRequestBody(Collections.singletonMap("sniff.seeds", null)));
                 ResponseException responseException = expectThrows(ResponseException.class,
                         () -> client().performRequest(request));
                 assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode());
-                assertThat(responseException.getMessage(), containsString("missing required setting [cluster.remote.remote1.seeds] " +
+                assertThat(responseException.getMessage(), containsString("missing required setting [cluster.remote.remote1.sniff.seeds] " +
                         "for setting [cluster.remote.remote1.skip_unavailable]"));
             }
 
             if (randomBoolean()) {
                 updateRemoteClusterSettings(Collections.singletonMap("skip_unavailable", null));
-                updateRemoteClusterSettings(Collections.singletonMap("seeds", null));
+                updateRemoteClusterSettings(Collections.singletonMap("sniff.seeds", null));
             } else {
                 Map<String, Object> nullMap = new HashMap<>();
-                nullMap.put("seeds", null);
+                nullMap.put("sniff.seeds", null);
                 nullMap.put("skip_unavailable", null);
                 updateRemoteClusterSettings(nullMap);
             }

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.java

@@ -30,12 +30,13 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.SniffConnectionStrategy;
 
 import java.io.IOException;
 import java.util.Collections;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.transport.RemoteClusterAware.SEARCH_REMOTE_CLUSTERS_SEEDS;
+import static org.elasticsearch.transport.SniffConnectionStrategy.SEARCH_REMOTE_CLUSTERS_SEEDS;
 import static org.elasticsearch.transport.RemoteClusterService.SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -91,9 +92,9 @@ public void testRemoteClusterSettingsUpgraded() throws IOException {
                         RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo").exists(settings));
                 assertTrue(RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace("foo").get(settings));
                 assertFalse(SEARCH_REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").exists(settings));
-                assertTrue(RemoteClusterService.REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").exists(settings));
+                assertTrue(SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS.getConcreteSettingForNamespace("foo").exists(settings));
                 assertThat(
-                        RemoteClusterService.REMOTE_CLUSTERS_SEEDS.getConcreteSettingForNamespace("foo").get(settings),
+                        SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS.getConcreteSettingForNamespace("foo").get(settings),
                         equalTo(Collections.singletonList("localhost:9200")));
             }
         }

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_connection_mode_configuration.yml

@@ -0,0 +1,212 @@
+---
+"Add transient remote cluster in simple mode with invalid sniff settings":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.cluster.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      catch: bad_request
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.mode: "simple"
+            cluster.remote.test_remote_cluster.sniff.node_connections: "5"
+            cluster.remote.test_remote_cluster.simple.addresses: $remote_ip
+
+  - match: { status: 400 }
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Setting \"cluster.remote.test_remote_cluster.sniff.node_connections\" cannot be
+  used with the configured \"cluster.remote.test_remote_cluster.mode\" [required=SNIFF, configured=SIMPLE]" }
+
+  - do:
+      catch: bad_request
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.mode: "simple"
+            cluster.remote.test_remote_cluster.sniff.seeds: $remote_ip
+            cluster.remote.test_remote_cluster.simple.addresses: $remote_ip
+
+  - match: { status: 400 }
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Setting \"cluster.remote.test_remote_cluster.sniff.seeds\" cannot be
+   used with the configured \"cluster.remote.test_remote_cluster.mode\" [required=SNIFF, configured=SIMPLE]" }
+
+---
+"Add transient remote cluster in sniff mode with invalid simple settings":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.cluster.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      catch: bad_request
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.simple.socket_connections: "20"
+            cluster.remote.test_remote_cluster.sniff.seeds: $remote_ip
+
+  - match: { status: 400 }
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Setting \"cluster.remote.test_remote_cluster.simple.socket_connections\" cannot be
+  used with the configured \"cluster.remote.test_remote_cluster.mode\" [required=SIMPLE, configured=SNIFF]" }
+
+  - do:
+      catch: bad_request
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.simple.addresses: $remote_ip
+            cluster.remote.test_remote_cluster.sniff.seeds: $remote_ip
+
+  - match: { status: 400 }
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Setting \"cluster.remote.test_remote_cluster.simple.addresses\" cannot be
+   used with the configured \"cluster.remote.test_remote_cluster.mode\" [required=SIMPLE, configured=SNIFF]" }
+
+---
+"Add transient remote cluster using simple connection mode using valid settings":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.cluster.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.mode: "simple"
+            cluster.remote.test_remote_cluster.simple.socket_connections: "3"
+            cluster.remote.test_remote_cluster.simple.addresses: $remote_ip
+
+  - match: {transient.cluster\.remote\.test_remote_cluster\.mode: "simple"}
+  - match: {transient.cluster\.remote\.test_remote_cluster\.simple\.socket_connections: "3"}
+  - match: {transient.cluster\.remote\.test_remote_cluster\.simple\.addresses: $remote_ip}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_remote_cluster:test_index
+
+  - is_false: num_reduce_phases
+  - match: {_clusters.total: 1}
+  - match: {_clusters.successful: 1}
+  - match: {_clusters.skipped: 0}
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - match: { hits.hits.0._index: "test_remote_cluster:test_index" }
+
+---
+"Add transient remote cluster using sniff connection mode using valid settings":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.cluster.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.mode: "sniff"
+            cluster.remote.test_remote_cluster.sniff.node_connections: "3"
+            cluster.remote.test_remote_cluster.sniff.seeds: $remote_ip
+
+  - match: {transient.cluster\.remote\.test_remote_cluster\.mode: "sniff"}
+  - match: {transient.cluster\.remote\.test_remote_cluster\.sniff\.node_connections: "3"}
+  - match: {transient.cluster\.remote\.test_remote_cluster\.sniff\.seeds: $remote_ip}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_remote_cluster:test_index
+
+  - is_false: num_reduce_phases
+  - match: {_clusters.total: 1}
+  - match: {_clusters.successful: 1}
+  - match: {_clusters.skipped: 0}
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - match: { hits.hits.0._index: "test_remote_cluster:test_index" }
+
+---
+"Switch connection mode for configured cluster":
+  - do:
+      cluster.get_settings:
+        include_defaults: true
+
+  - set: { defaults.cluster.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.mode: "sniff"
+            cluster.remote.test_remote_cluster.sniff.seeds: $remote_ip
+
+  - match: {transient.cluster\.remote\.test_remote_cluster\.mode: "sniff"}
+  - match: {transient.cluster\.remote\.test_remote_cluster\.sniff\.seeds: $remote_ip}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_remote_cluster:test_index
+
+  - is_false: num_reduce_phases
+  - match: {_clusters.total: 1}
+  - match: {_clusters.successful: 1}
+  - match: {_clusters.skipped: 0}
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - match: { hits.hits.0._index: "test_remote_cluster:test_index" }
+
+  - do:
+      catch: bad_request
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.mode: "simple"
+            cluster.remote.test_remote_cluster.simple.addresses: $remote_ip
+
+  - match: { status: 400 }
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Setting \"cluster.remote.test_remote_cluster.sniff.seeds\" cannot be
+  used with the configured \"cluster.remote.test_remote_cluster.mode\" [required=SNIFF, configured=SIMPLE]" }
+
+  - do:
+      cluster.put_settings:
+        flat_settings: true
+        body:
+          transient:
+            cluster.remote.test_remote_cluster.mode: "simple"
+            cluster.remote.test_remote_cluster.sniff.seeds: null
+            cluster.remote.test_remote_cluster.simple.addresses: $remote_ip
+
+  - match: {transient.cluster\.remote\.test_remote_cluster\.mode: "simple"}
+  - match: {transient.cluster\.remote\.test_remote_cluster\.simple\.addresses: $remote_ip}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_remote_cluster:test_index
+
+  - is_false: num_reduce_phases
+  - match: {_clusters.total: 1}
+  - match: {_clusters.successful: 1}
+  - match: {_clusters.skipped: 0}
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - match: { hits.hits.0._index: "test_remote_cluster:test_index" }

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -108,8 +108,10 @@
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.RemoteClusterService;
+import org.elasticsearch.transport.RemoteConnectionStrategy;
+import org.elasticsearch.transport.SimpleConnectionStrategy;
+import org.elasticsearch.transport.SniffConnectionStrategy;
 import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
@@ -303,14 +305,8 @@ public void apply(Settings value, Settings current, Settings previous) {
             SearchService.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS,
             ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
             TransportSearchAction.SHARD_COUNT_LIMIT_SETTING,
-            RemoteClusterAware.REMOTE_CLUSTERS_SEEDS,
-            RemoteClusterAware.SEARCH_REMOTE_CLUSTERS_SEEDS,
-            RemoteClusterAware.REMOTE_CLUSTERS_PROXY,
-            RemoteClusterAware.SEARCH_REMOTE_CLUSTERS_PROXY,
             RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE,
             RemoteClusterService.SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE,
-            RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER,
-            RemoteClusterService.SEARCH_REMOTE_CONNECTIONS_PER_CLUSTER,
             RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
             RemoteClusterService.SEARCH_REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
             RemoteClusterService.REMOTE_NODE_ATTRIBUTE,
@@ -319,6 +315,17 @@ public void apply(Settings value, Settings current, Settings previous) {
             RemoteClusterService.SEARCH_ENABLE_REMOTE_CLUSTERS,
             RemoteClusterService.REMOTE_CLUSTER_PING_SCHEDULE,
             RemoteClusterService.REMOTE_CLUSTER_COMPRESS,
+            RemoteConnectionStrategy.REMOTE_CONNECTION_MODE,
+            SimpleConnectionStrategy.REMOTE_CLUSTER_ADDRESSES,
+            SimpleConnectionStrategy.REMOTE_SOCKET_CONNECTIONS,
+            SniffConnectionStrategy.SEARCH_REMOTE_CLUSTERS_SEEDS,
+            SniffConnectionStrategy.SEARCH_REMOTE_CLUSTERS_PROXY,
+            SniffConnectionStrategy.SEARCH_REMOTE_CONNECTIONS_PER_CLUSTER,
+            SniffConnectionStrategy.REMOTE_CONNECTIONS_PER_CLUSTER,
+            SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS_OLD,
+            SniffConnectionStrategy.REMOTE_CLUSTERS_PROXY,
+            SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS,
+            SniffConnectionStrategy.REMOTE_NODE_CONNECTIONS,
             TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING,
             ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING,
             NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING,
@@ -520,8 +527,8 @@ public void apply(Settings value, Settings current, Settings previous) {
             DiscoveryUpgradeService.ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING)));
 
     public static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.unmodifiableList(Arrays.asList(
-            RemoteClusterAware.SEARCH_REMOTE_CLUSTER_SEEDS_UPGRADER,
-            RemoteClusterAware.SEARCH_REMOTE_CLUSTERS_PROXY_UPGRADER,
+            SniffConnectionStrategy.SEARCH_REMOTE_CLUSTER_SEEDS_UPGRADER,
+            SniffConnectionStrategy.SEARCH_REMOTE_CLUSTERS_PROXY_UPGRADER,
             RemoteClusterService.SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE_UPGRADER));
 
 }