[master] Merge branch 'trac5649'

doc/examples/kea4/ha-load-balancing-primary.json

@@ -114,6 +114,11 @@
                     // This specifies the number of clients which send messages to
                     // the partner but appear to not receive any response.
                     "max-unacked-clients": 5,
+                    // This specifies the maximum timeout (in milliseconds) for the server
+                    // to complete sync. If you have a large deployment (high tens or
+                    // hundreds of thausands of clients), you may need to increase it
+                    // further. The default value is 60000ms (60 seconds).
+                    "sync-timeout": 60000,
                     "peers": [
                          // This is the configuration of this server instance.
                          {

doc/guide/hooks-ha.xml

@@ -977,6 +977,76 @@
 
       </section>
 
+      <section xml:id="ha-syncing-timeouts">
+        <title>Discussion about Timeouts</title>
+        <para>In deployments with large number of clients connected to the
+        network, lease database synchronization after the server failure
+        may be a time consuming operation. The synchronizing server needs
+        to gather all leases from the partner which yields a large response
+        over the RESTful interface. The time required for generating the
+        response and sending it to the synchronizing server may take from
+        several seconds to tens of seconds. The default timeout value for
+        such communication is set to 60 seconds. However, the server
+        administrator may need to extend this timeout if necessary. The
+        following configuration snippet demonstrates how to extend this
+        timeout to 90 seconds with the <command>sync-timeout</command>
+        parameter:
+<screen>
+{
+"Dhcp4": {
+
+    ...
+
+    "hooks-libraries": [
+        {
+            "library": "/usr/lib/hooks/libdhcp_lease_cmds.so",
+            "parameters": { }
+        },
+        {
+            "library": "/usr/lib/hooks/libdhcp_ha.so",
+            "parameters": {
+                "high-availability": [ {
+                    "this-server-name": "server1",
+                    "mode": "load-balancing",
+                    "sync-timeout": 90000,
+                    "peers": [
+                        {
+                            "name": "server1",
+                            "url": "http://192.168.56.33:8080/",
+                            "role": "primary"
+                        },
+                        {
+                            "name": "server2",
+                            "url": "http://192.168.56.66:8080/",
+                            "role": "secondary"
+                        }
+                    ]
+                } ]
+            }
+        }
+    ],
+
+    ...
+
+}
+</screen>
+        </para>
+
+        <para>
+          It is important to note that extending this value may sometimes
+          be insufficient to prevent issues with timeouts during
+          lease database synchronization. The control commands travel via
+          Control Agent, which also monitors incoming (with synchronizing
+          server) and outgoing (with DHCP server) connections for timeouts.
+          The DHCP server also monitors the connection from the Control
+          Agent for timeouts. Those timeouts can't be currently modified
+          via configuration. Extending these timeouts is only possible by
+          modifing them in the Kea code and recompiling the server. The
+          relevant constants are located in the Kea sources:
+          <filename>src/lib/config/timeouts.h</filename>.
+        </para>
+      </section>
+
       <section xml:id="ha-ctrl-agent-config">
         <title>Control Agent Configuration</title>
         <para>The <xref linkend="kea-ctrl-agent"/> describes in detail the

src/bin/agent/ca_command_mgr.cc

@@ -18,6 +18,7 @@
 #include <cc/data.h>
 #include <cc/json_feed.h>
 #include <config/client_connection.h>
+#include <config/timeouts.h>
 #include <boost/pointer_cast.hpp>
 #include <iterator>
 #include <sstream>
@@ -30,14 +31,6 @@ using namespace isc::data;
 using namespace isc::hooks;
 using namespace isc::process;
 
-namespace {
-
-/// @brief Client side connection timeout.
-/// @todo Make it configurable.
-const long CONNECTION_TIMEOUT = 5000;
-
-}
-
 namespace isc {
 namespace agent {
 
@@ -241,7 +234,7 @@ CtrlAgentCommandMgr::forwardCommand(const std::string& service,
                    // Got the IO service so stop IO service. This causes to
                    // stop IO service when all handlers have been invoked.
                    io_service->stopWork();
-               }, ClientConnection::Timeout(CONNECTION_TIMEOUT));
+               }, ClientConnection::Timeout(TIMEOUT_AGENT_FORWARD_COMMAND));
     io_service->run();
 
     if (received_ec) {

src/bin/agent/ca_process.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2016-2017 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2016-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -13,22 +13,15 @@
 #include <asiolink/io_address.h>
 #include <asiolink/io_error.h>
 #include <cc/command_interpreter.h>
+#include <config/timeouts.h>
 #include <boost/pointer_cast.hpp>
 
 using namespace isc::asiolink;
+using namespace isc::config;
 using namespace isc::data;
 using namespace isc::http;
 using namespace isc::process;
 
-// Temporarily hardcoded configuration.
-/// @todo: remove once 5134 is merged.
-namespace {
-
-const long REQUEST_TIMEOUT = 10000;
-
-const long IDLE_TIMEOUT = 30000;
-
-}
 
 namespace isc {
 namespace agent {
@@ -153,11 +146,11 @@ CtrlAgentProcess::configure(isc::data::ConstElementPtr config_set,
 
             // Create http listener. It will open up a TCP socket and be
             // prepared to accept incoming connection.
-            HttpListenerPtr
-                http_listener(new HttpListener(*getIoService(), server_address,
-                                               server_port, rcf,
-                                               HttpListener::RequestTimeout(REQUEST_TIMEOUT),
-                                               HttpListener::IdleTimeout(IDLE_TIMEOUT)));
+            HttpListenerPtr http_listener
+                (new HttpListener(*getIoService(), server_address,
+                                  server_port, rcf,
+                                  HttpListener::RequestTimeout(TIMEOUT_AGENT_RECEIVE_COMMAND),
+                                  HttpListener::IdleTimeout(TIMEOUT_AGENT_IDLE_CONNECTION_TIMEOUT)));
 
             // Instruct the http listener to actually open socket, install
             // callback and start listening.

src/bin/dhcp4/tests/ctrl_dhcp4_srv_unittest.cc

@@ -10,6 +10,7 @@
 #include <asiolink/io_service.h>
 #include <cc/command_interpreter.h>
 #include <config/command_mgr.h>
+#include <config/timeouts.h>
 #include <dhcp/dhcp4.h>
 #include <dhcp4/ctrl_dhcp4_srv.h>
 #include <dhcp4/tests/dhcp4_test_utils.h>
@@ -89,9 +90,6 @@ class NakedControlledDhcpv4Srv: public ControlledDhcpv4Srv {
     using Dhcpv4Srv::network_state_;
 };
 
-/// @brief Default control connection timeout.
-const size_t DEFAULT_CONNECTION_TIMEOUT = 10;
-
 /// @brief Fixture class intended for testin control channel in the DHCPv4Srv
 class CtrlChannelDhcpv4SrvTest : public ::testing::Test {
 public:
@@ -122,7 +120,7 @@ class CtrlChannelDhcpv4SrvTest : public ::testing::Test {
 
         CommandMgr::instance().closeCommandSocket();
         CommandMgr::instance().deregisterAll();
-        CommandMgr::instance().setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
+        CommandMgr::instance().setConnectionTimeout(TIMEOUT_DHCP_SERVER_RECEIVE_COMMAND);
 
         server_.reset();
     };
@@ -1420,7 +1418,7 @@ TEST_F(CtrlChannelDhcpv4SrvTest, connectionTimeoutPartialCommand) {
 
     // Set connection timeout to 2s to prevent long waiting time for the
     // timeout during this test.
-    const unsigned short timeout = 2;
+    const unsigned short timeout = 2000;
     CommandMgr::instance().setConnectionTimeout(timeout);
 
     // Server's response will be assigned to this variable.
@@ -1474,7 +1472,7 @@ TEST_F(CtrlChannelDhcpv4SrvTest, connectionTimeoutNoData) {
 
     // Set connection timeout to 2s to prevent long waiting time for the
     // timeout during this test.
-    const unsigned short timeout = 2;
+    const unsigned short timeout = 2000;
     CommandMgr::instance().setConnectionTimeout(timeout);
 
     // Server's response will be assigned to this variable.

src/bin/dhcp6/tests/ctrl_dhcp6_srv_unittest.cc

@@ -87,7 +87,7 @@ class NakedControlledDhcpv6Srv: public ControlledDhcpv6Srv {
 };
 
 /// @brief Default control connection timeout.
-const size_t DEFAULT_CONNECTION_TIMEOUT = 10;
+const size_t DEFAULT_CONNECTION_TIMEOUT = 10000;
 
 class CtrlDhcpv6SrvTest : public BaseServerTest {
 public:
@@ -1441,7 +1441,7 @@ TEST_F(CtrlChannelDhcpv6SrvTest, connectionTimeoutPartialCommand) {
 
     // Set connection timeout to 2s to prevent long waiting time for the
     // timeout during this test.
-    const unsigned short timeout = 2;
+    const unsigned short timeout = 2000;
     CommandMgr::instance().setConnectionTimeout(timeout);
 
     // Server's response will be assigned to this variable.
@@ -1495,7 +1495,7 @@ TEST_F(CtrlChannelDhcpv6SrvTest, connectionTimeoutNoData) {
 
     // Set connection timeout to 2s to prevent long waiting time for the
     // timeout during this test.
-    const unsigned short timeout = 2;
+    const unsigned short timeout = 2000;
     CommandMgr::instance().setConnectionTimeout(timeout);
 
     // Server's response will be assigned to this variable.

src/hooks/dhcp/high_availability/ha_config.cc

@@ -78,8 +78,9 @@ HAConfig::PeerConfig::roleToString(const HAConfig::PeerConfig::Role& role) {
 
 HAConfig::HAConfig()
     : this_server_name_(), ha_mode_(HOT_STANDBY), send_lease_updates_(true),
-      sync_leases_(true), heartbeat_delay_(10), max_response_delay_(60),
-      max_ack_delay_(10), max_unacked_clients_(10), peers_() {
+      sync_leases_(true), sync_timeout_(60000), heartbeat_delay_(10000),
+      max_response_delay_(60000), max_ack_delay_(10000), max_unacked_clients_(10),
+      peers_() {
 }
 
 HAConfig::PeerConfigPtr

src/hooks/dhcp/high_availability/ha_config.h

@@ -252,6 +252,20 @@ class HAConfig {
         sync_leases_ = sync_leases;
     }
 
+    /// @brief Returns timeout for lease database synchronization.
+    ///
+    /// @return Timeout in milliseconds.
+    long getSyncTimeout() const {
+        return (sync_timeout_);
+    }
+
+    /// @brief Sets new lease database syncing timeout in milliseconds.
+    ///
+    /// @param sync_timeout new timeout for lease database synchornization.
+    void setSyncTimeout(const long sync_timeout) {
+        sync_timeout_ = sync_timeout;
+    }
+
     /// @brief Returns heartbeat delay in milliseconds.
     ///
     /// This value indicates the delay in sending a heartbeat command after
@@ -377,6 +391,7 @@ class HAConfig {
     HAMode ha_mode_;               ///< Mode of operation.
     bool send_lease_updates_;      ///< Send lease updates to partner?
     bool sync_leases_;             ///< Synchronize databases on startup?
+    long sync_timeout_;            ///< Timeout for syncing lease database (ms)
     uint32_t heartbeat_delay_;     ///< Heartbeat delay in milliseconds.
     uint32_t max_response_delay_;  ///< Max delay in response to heartbeats.
     uint32_t max_ack_delay_;       ///< Maximum DHCP message ack delay.

src/hooks/dhcp/high_availability/ha_config_parser.cc

@@ -20,6 +20,7 @@ namespace {
 const SimpleDefaults HA_CONFIG_DEFAULTS = {
     { "send-lease-updates", Element::boolean, "true" },
     { "sync-leases", Element::boolean, "true" },
+    { "sync-timeout", Element::integer, "60000" },
     { "heartbeat-delay", Element::integer, "10000" },
     { "max-response-delay", Element::integer, "60000" },
     { "max-ack-delay", Element::integer, "10000" },
@@ -109,6 +110,10 @@ HAConfigParser::parseInternal(const HAConfigPtr& config_storage,
     // Get 'sync-leases'.
     config_storage->setSyncLeases(getBoolean(c, "sync-leases"));
 
+    // Get 'sync-timeout'.
+    uint16_t sync_timeout = getAndValidateInteger<uint16_t>(c, "sync-timeout");
+    config_storage->setSyncTimeout(sync_timeout);
+
     // Get 'heartbeat-delay'.
     uint16_t heartbeat_delay = getAndValidateInteger<uint16_t>(c, "heartbeat-delay");
     config_storage->setHeartbeatDelay(heartbeat_delay);

src/hooks/dhcp/high_availability/ha_service.cc

@@ -1215,7 +1215,7 @@ HAService::asyncSyncLeases(http::HttpClient& http_client,
                 post_sync_action(error_message.empty(),
                                  error_message);
             }
-    });
+    }, HttpClient::RequestTimeout(config_->getSyncTimeout()));
 }
 
 ConstElementPtr

src/hooks/dhcp/high_availability/tests/ha_config_unittest.cc

@@ -63,6 +63,7 @@ TEST_F(HAConfigTest, configureLoadBalancing) {
         "        \"mode\": \"load-balancing\","
         "        \"send-lease-updates\": false,"
         "        \"sync-leases\": false,"
+        "        \"sync-timeout\": 20000,"
         "        \"heartbeat-delay\": 8,"
         "        \"max-response-delay\": 11,"
         "        \"max-ack-delay\": 5,"
@@ -95,6 +96,7 @@ TEST_F(HAConfigTest, configureLoadBalancing) {
     EXPECT_EQ(HAConfig::LOAD_BALANCING, impl->getConfig()->getHAMode());
     EXPECT_FALSE(impl->getConfig()->amSendingLeaseUpdates());
     EXPECT_FALSE(impl->getConfig()->amSyncingLeases());
+    EXPECT_EQ(20000, impl->getConfig()->getSyncTimeout());
     EXPECT_EQ(8, impl->getConfig()->getHeartbeatDelay());
     EXPECT_EQ(11, impl->getConfig()->getMaxResponseDelay());
     EXPECT_EQ(5, impl->getConfig()->getMaxAckDelay());
@@ -161,6 +163,7 @@ TEST_F(HAConfigTest, configureHotStandby) {
     EXPECT_EQ(HAConfig::HOT_STANDBY, impl->getConfig()->getHAMode());
     EXPECT_TRUE(impl->getConfig()->amSendingLeaseUpdates());
     EXPECT_TRUE(impl->getConfig()->amSyncingLeases());
+    EXPECT_EQ(60000, impl->getConfig()->getSyncTimeout());
     EXPECT_EQ(10000, impl->getConfig()->getHeartbeatDelay());
     EXPECT_EQ(10000, impl->getConfig()->getMaxAckDelay());
     EXPECT_EQ(10, impl->getConfig()->getMaxUnackedClients());

src/lib/asiolink/Makefile.am

@@ -15,7 +15,7 @@ CLEANFILES = *.gcno *.gcda
 
 lib_LTLIBRARIES = libkea-asiolink.la
 
-libkea_asiolink_la_LDFLAGS = -no-undefined -version-info 6:0:0
+libkea_asiolink_la_LDFLAGS = -no-undefined -version-info 6:1:0
 
 libkea_asiolink_la_SOURCES  = asiolink.h
 libkea_asiolink_la_SOURCES += asio_wrapper.h

src/lib/cc/Makefile.am

@@ -17,7 +17,7 @@ libkea_cc_la_LIBADD += $(top_builddir)/src/lib/asiolink/libkea-asiolink.la
 libkea_cc_la_LIBADD += $(top_builddir)/src/lib/exceptions/libkea-exceptions.la
 libkea_cc_la_LIBADD += $(BOOST_LIBS)
 
-libkea_cc_la_LDFLAGS = -no-undefined -version-info 4:0:0
+libkea_cc_la_LDFLAGS = -no-undefined -version-info 5:0:0
 
 # Specify the headers for copying into the installation directory tree.
 libkea_cc_includedir = $(pkgincludedir)/cc

src/lib/config/Makefile.am

@@ -21,6 +21,7 @@ libkea_cfgclient_la_SOURCES += client_connection.cc client_connection.h
 libkea_cfgclient_la_SOURCES += command_mgr.cc command_mgr.h
 libkea_cfgclient_la_SOURCES += config_log.h config_log.cc
 libkea_cfgclient_la_SOURCES += hooked_command_mgr.cc hooked_command_mgr.h
+libkea_cfgclient_la_SOURCES += timeouts.h
 
 libkea_cfgclient_la_LIBADD = $(top_builddir)/src/lib/dhcp/libkea-dhcp++.la
 libkea_cfgclient_la_LIBADD += $(top_builddir)/src/lib/asiolink/libkea-asiolink.la
@@ -34,7 +35,7 @@ libkea_cfgclient_la_LIBADD += $(top_builddir)/src/lib/util/libkea-util.la
 libkea_cfgclient_la_LIBADD += $(top_builddir)/src/lib/exceptions/libkea-exceptions.la
 libkea_cfgclient_la_LIBADD += $(LOG4CPLUS_LIBS) $(CRYPTO_LIBS) $(BOOST_LIBS)
 
-libkea_cfgclient_la_LDFLAGS = -no-undefined -version-info 5:0:0
+libkea_cfgclient_la_LDFLAGS = -no-undefined -version-info 5:1:0
 libkea_cfgclient_la_LDFLAGS += $(CRYPTO_LDFLAGS)
 
 nodist_libkea_cfgclient_la_SOURCES  = config_messages.h config_messages.cc
@@ -52,6 +53,6 @@ libkea_cfgclient_include_HEADERS = \
 	cmds_impl.h \
 	command_mgr.h \
 	config_log.h \
-	hooked_command_mgr.h
-
+	hooked_command_mgr.h \
+	timeouts.h