Enforce node level limits if node is started in production env

This commit tries to 'guess' if a user starts a node in production by
checking if any network host is configured. If that is the case soft-limits
that are only logged otherwise are enforced like number of open file descriptors.

Closes #16727

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.logging.log4j.LogConfigurator;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -41,12 +42,17 @@
 import org.elasticsearch.monitor.process.ProcessProbe;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.transport.TransportSettings;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.Locale;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
 import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
@@ -57,7 +63,6 @@
 final class Bootstrap {
 
     private static volatile Bootstrap INSTANCE;
-
     private volatile Node node;
     private final CountDownLatch keepAliveLatch = new CountDownLatch(1);
     private final Thread keepAliveThread;
@@ -184,12 +189,13 @@ public void run() {
                 .put(settings)
                 .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
                 .build();
+        enforceOrLogLimits(nodeSettings);
 
         node = new Node(nodeSettings);
     }
 
     @SuppressForbidden(reason = "Exception#printStackTrace()")
-    private static void setupLogging(Settings settings, Environment environment) {
+    private static void setupLogging(Settings settings) {
         try {
             Class.forName("org.apache.log4j.Logger");
             LogConfigurator.configure(settings, true);
@@ -249,7 +255,7 @@ static void init(String[] args) throws Throwable {
 
         Environment environment = initialSettings(foreground);
         Settings settings = environment.settings();
-        setupLogging(settings, environment);
+        setupLogging(settings);
         checkForCustomConfFile();
 
         if (environment.pidFile() != null) {
@@ -362,4 +368,48 @@ private static void checkLucene() {
                 + Version.CURRENT.luceneVersion + "]  but the current lucene version is [" + org.apache.lucene.util.Version.LATEST + "]");
         }
     }
+
+    static final Set<Setting> ENFORCE_SETTINGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+        TransportSettings.BIND_HOST,
+        TransportSettings.HOST,
+        TransportSettings.PUBLISH_HOST,
+        NetworkService.GLOBAL_NETWORK_HOST_SETTING,
+        NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING,
+        NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING
+    )));
+
+    private static boolean enforceLimits(Settings settings) {
+        for (Setting setting : ENFORCE_SETTINGS) {
+            if (setting.exists(settings)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    static void enforceOrLogLimits(Settings settings) { // pkg private for testing
+        /* We enforce limits once any network host is configured. In this case we assume the node is running in production
+         * and all production limit checks must pass. This should be extended as we go to settings like:
+         *   - discovery.zen.minimum_master_nodes
+         *   - discovery.zen.ping.unicast.hosts is set if we use zen disco
+         *   - ensure we can write in all data directories
+         *   - fail if mlockall failed and was configured
+         *   - fail if vm.max_map_count is under a certain limit (not sure if this works cross platform)
+         *   - fail if the default cluster.name is used, if this is setup on network a real clustername should be used?*/
+        final boolean enforceLimits = enforceLimits(settings);
+        final ESLogger logger = Loggers.getLogger(Bootstrap.class);
+        final long maxFileDescriptorCount = ProcessProbe.getInstance().getMaxFileDescriptorCount();
+        if (maxFileDescriptorCount != -1) {
+            final int fileDescriptorCountThreshold = (1 << 16);
+            if (maxFileDescriptorCount < fileDescriptorCountThreshold) {
+                if (enforceLimits){
+                    throw new IllegalStateException("max file descriptors [" + maxFileDescriptorCount
+                        + "] for elasticsearch process likely too low, increase it to at least [" + fileDescriptorCountThreshold +"]");
+                }
+                logger.warn(
+                    "max file descriptors [{}] for elasticsearch process likely too low, consider increasing to at least [{}]",
+                    maxFileDescriptorCount, fileDescriptorCountThreshold);
+            }
+        }
+    }
 }

core/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -222,7 +222,6 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
 
         maybeLogPathDetails();
         maybeLogHeapDetails();
-        maybeWarnFileDescriptors();
 
         applySegmentInfosTrace(settings);
     }
@@ -315,19 +314,6 @@ private void maybeLogHeapDetails() {
         logger.info("heap size [{}], compressed ordinary object pointers [{}]", maxHeapSize, useCompressedOops);
     }
 
-    private void maybeWarnFileDescriptors() {
-        long maxFileDescriptorCount = ProcessProbe.getInstance().getMaxFileDescriptorCount();
-        if (maxFileDescriptorCount == -1) {
-            return;
-        }
-        int fileDescriptorCountThreshold = (1 << 16);
-        if (maxFileDescriptorCount < fileDescriptorCountThreshold) {
-            logger.warn(
-                    "max file descriptors [{}] for elasticsearch process likely too low, consider increasing to at least [{}]",
-                    maxFileDescriptorCount,
-                    fileDescriptorCountThreshold);
-        }
-    }
 
     @SuppressForbidden(reason = "System.out.*")
     static void applySegmentInfosTrace(Settings settings) {

core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java

@@ -19,7 +19,10 @@
 
 package org.elasticsearch.bootstrap;
 
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.monitor.os.OsProbe;
+import org.elasticsearch.monitor.process.ProcessProbe;
 import org.elasticsearch.test.ESTestCase;
 
 public class BootstrapSettingsTests extends ESTestCase {
@@ -31,4 +34,21 @@ public void testDefaultSettings() {
         assertTrue(BootstrapSettings.CTRLHANDLER_SETTING.get(Settings.EMPTY));
     }
 
+    public void testEnforceMaxFileDescriptorLimits() {
+        // nothing should happen since we are in OOB mode
+        Bootstrap.enforceOrLogLimits(Settings.EMPTY);
+
+        Settings build = Settings.builder().put(randomFrom(Bootstrap.ENFORCE_SETTINGS.toArray(new Setting[0])).getKey(),
+            "127.0.0.1").build();
+        long maxFileDescriptorCount = ProcessProbe.getInstance().getMaxFileDescriptorCount();
+        try {
+            Bootstrap.enforceOrLogLimits(build);
+            if (maxFileDescriptorCount != -1 || maxFileDescriptorCount < (1 << 16)) {
+                fail("must have enforced limits: " + maxFileDescriptorCount);
+            }
+        } catch (IllegalStateException ex) {
+            assertTrue(ex.getMessage(), ex.getMessage().startsWith("max file descriptors"));
+        }
+    }
+
 }