[Win32 Signals] Add term and ctrl-c signal handlers (#13954)

Part 1 of #13188. Adds support for ctrl+c and ctrl+break for Envoy on Windows.

The implementation for this following:

* On platform_impl we register a CtrlHandler which runs on a separate thread.
* On signal_impl we register a read event reader.
Thread (1) and (2) communicate via a socket pair and the event is handled on Windows the same way as it is handled on POSIX

Signed-off-by: Sotiris Nanopoulos <sonanopo@microsoft.com>

.bazelrc

@@ -298,6 +298,7 @@ build:windows --action_env=TMPDIR
 build:windows --define signal_trace=disabled
 build:windows --define hot_restart=disabled
 build:windows --define tcmalloc=disabled
+build:windows --define wasm=disabled
 build:windows --define manual_stamp=manual_stamp
 build:windows --cxxopt="/std:c++17"
 

ci/run_clang_tidy.sh

@@ -31,7 +31,7 @@ echo "Generating compilation database..."
 # Do not run clang-tidy against win32 impl
 # TODO(scw00): We should run clang-tidy against win32 impl once we have clang-cl support for Windows
 function exclude_win32_impl() {
-  grep -v source/common/filesystem/win32/ | grep -v source/common/common/win32 | grep -v source/exe/win32 | grep -v source/common/api/win32
+  grep -v source/common/filesystem/win32/ | grep -v source/common/common/win32 | grep -v source/exe/win32 | grep -v source/common/api/win32 | grep -v source/common/event/win32
 }
 
 # Do not run clang-tidy against macOS impl

ci/windows_ci_steps.sh

@@ -49,7 +49,6 @@ BAZEL_BUILD_OPTIONS=(
     -c opt
     --show_task_finish
     --verbose_failures
-    --define "wasm=disabled"
     "--test_output=errors"
     "${BAZEL_BUILD_EXTRA_OPTIONS[@]}"
     "${BAZEL_EXTRA_TEST_OPTIONS[@]}")

include/envoy/common/platform.h

@@ -65,6 +65,7 @@ typedef uint32_t mode_t;
 
 typedef SOCKET os_fd_t;
 typedef HANDLE filesystem_os_id_t; // NOLINT(modernize-use-using)
+typedef DWORD signal_t;            // NOLINT(modernize-use-using)
 
 typedef unsigned int sa_family_t;
 
@@ -151,6 +152,9 @@ struct msghdr {
 #define HANDLE_ERROR_PERM ERROR_ACCESS_DENIED
 #define HANDLE_ERROR_INVALID ERROR_INVALID_HANDLE
 
+#define ENVOY_WIN32_SIGNAL_COUNT 1
+#define ENVOY_SIGTERM 0
+
 namespace Platform {
 constexpr absl::string_view null_device_path{"NUL"};
 }
@@ -215,6 +219,7 @@ constexpr absl::string_view null_device_path{"NUL"};
 
 typedef int os_fd_t;
 typedef int filesystem_os_id_t; // NOLINT(modernize-use-using)
+typedef int signal_t;           // NOLINT(modernize-use-using)
 
 #define INVALID_HANDLE -1
 #define INVALID_SOCKET -1
@@ -245,6 +250,8 @@ typedef int filesystem_os_id_t; // NOLINT(modernize-use-using)
 #define HANDLE_ERROR_PERM EACCES
 #define HANDLE_ERROR_INVALID EBADF
 
+#define ENVOY_SIGTERM SIGTERM
+
 namespace Platform {
 constexpr absl::string_view null_device_path{"/dev/null"};
 }

include/envoy/event/dispatcher.h

@@ -205,7 +205,7 @@ class Dispatcher {
    * @param cb supplies the callback to invoke when the signal fires.
    * @return SignalEventPtr a signal event that is owned by the caller.
    */
-  virtual SignalEventPtr listenForSignal(int signal_num, SignalCb cb) PURE;
+  virtual SignalEventPtr listenForSignal(signal_t signal_num, SignalCb cb) PURE;
 
   /**
    * Posts a functor to the dispatcher. This is safe cross thread. The functor runs in the context

source/common/event/BUILD

@@ -1,6 +1,9 @@
 load(
     "//bazel:envoy_build_system.bzl",
     "envoy_cc_library",
+    "envoy_cc_platform_dep",
+    "envoy_cc_posix_library",
+    "envoy_cc_win32_library",
     "envoy_package",
 )
 
@@ -13,15 +16,24 @@ envoy_cc_library(
     srcs = [
         "dispatcher_impl.cc",
         "file_event_impl.cc",
-        "signal_impl.cc",
-    ],
-    hdrs = [
-        "signal_impl.h",
     ],
+    hdrs = select({
+        "//bazel:windows_x86_64": [
+            "win32/signal_impl.h",
+        ],
+        "//conditions:default": [
+            "posix/signal_impl.h",
+        ],
+    }),
+    strip_include_prefix = select({
+        "//bazel:windows_x86_64": "win32",
+        "//conditions:default": "posix",
+    }),
     deps = [
         ":dispatcher_includes",
         ":libevent_scheduler_lib",
         ":real_time_system_lib",
+        ":signal_lib",
         "//include/envoy/common:scope_tracker_interface",
         "//include/envoy/common:time_interface",
         "//include/envoy/event:signal_interface",
@@ -39,6 +51,37 @@ envoy_cc_library(
     }),
 )
 
+envoy_cc_library(
+    name = "signal_lib",
+    deps = envoy_cc_platform_dep("signal_impl_lib"),
+)
+
+envoy_cc_posix_library(
+    name = "signal_impl_lib",
+    srcs = ["posix/signal_impl.cc"],
+    hdrs = ["posix/signal_impl.h"],
+    strip_include_prefix = "posix",
+    deps = [
+        ":dispatcher_includes",
+        "//include/envoy/event:signal_interface",
+        "//source/common/common:thread_lib",
+    ],
+)
+
+envoy_cc_win32_library(
+    name = "signal_impl_lib",
+    srcs = ["win32/signal_impl.cc"],
+    hdrs = ["win32/signal_impl.h"],
+    strip_include_prefix = "win32",
+    deps = [
+        ":dispatcher_includes",
+        "//include/envoy/event:signal_interface",
+        "//source/common/api:os_sys_calls_lib",
+        "//source/common/common:thread_lib",
+        "//source/common/network:default_socket_interface_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "event_impl_base_lib",
     srcs = ["event_impl_base.cc"],

source/common/event/dispatcher_impl.cc

@@ -218,7 +218,7 @@ void DispatcherImpl::deferredDelete(DeferredDeletablePtr&& to_delete) {
 
 void DispatcherImpl::exit() { base_scheduler_.loopExit(); }
 
-SignalEventPtr DispatcherImpl::listenForSignal(int signal_num, SignalCb cb) {
+SignalEventPtr DispatcherImpl::listenForSignal(signal_t signal_num, SignalCb cb) {
   ASSERT(isThreadSafe());
   return SignalEventPtr{new SignalEventImpl(*this, signal_num, cb)};
 }

source/common/event/dispatcher_impl.h

@@ -71,7 +71,7 @@ class DispatcherImpl : Logger::Loggable<Logger::Id::main>,
   Event::SchedulableCallbackPtr createSchedulableCallback(std::function<void()> cb) override;
   void deferredDelete(DeferredDeletablePtr&& to_delete) override;
   void exit() override;
-  SignalEventPtr listenForSignal(int signal_num, SignalCb cb) override;
+  SignalEventPtr listenForSignal(signal_t signal_num, SignalCb cb) override;
   void post(std::function<void()> callback) override;
   void run(RunType type) override;
   Buffer::WatermarkFactory& getWatermarkFactory() override { return *buffer_factory_; }

source/common/event/signal_impl.cc → source/common/event/posix/signal_impl.cc

@@ -1,13 +1,12 @@
-#include "common/event/signal_impl.h"
-
 #include "common/event/dispatcher_impl.h"
+#include "common/event/signal_impl.h"
 
 #include "event2/event.h"
 
 namespace Envoy {
 namespace Event {
 
-SignalEventImpl::SignalEventImpl(DispatcherImpl& dispatcher, int signal_num, SignalCb cb)
+SignalEventImpl::SignalEventImpl(DispatcherImpl& dispatcher, signal_t signal_num, SignalCb cb)
     : cb_(cb) {
   evsignal_assign(
       &raw_event_, &dispatcher.base(), signal_num,

source/common/event/signal_impl.h → source/common/event/posix/signal_impl.h

@@ -13,11 +13,10 @@ namespace Event {
  */
 class SignalEventImpl : public SignalEvent, ImplBase {
 public:
-  SignalEventImpl(DispatcherImpl& dispatcher, int signal_num, SignalCb cb);
+  SignalEventImpl(DispatcherImpl& dispatcher, signal_t signal_num, SignalCb cb);
 
 private:
   SignalCb cb_;
 };
-
 } // namespace Event
 } // namespace Envoy

source/common/event/win32/signal_impl.cc

@@ -0,0 +1,44 @@
+#include "common/api/os_sys_calls_impl.h"
+#include "common/event/dispatcher_impl.h"
+#include "common/event/signal_impl.h"
+
+#include "event2/event.h"
+
+namespace Envoy {
+namespace Event {
+
+SignalEventImpl::SignalEventImpl(DispatcherImpl& dispatcher, signal_t signal_num, SignalCb cb)
+    : cb_(cb) {
+
+  if (signal_num > eventBridgeHandlersSingleton::get().size()) {
+    PANIC("Attempting to create SignalEventImpl with a signal id that exceeds the number of "
+          "supported signals.");
+  }
+
+  if (eventBridgeHandlersSingleton::get()[signal_num]) {
+    return;
+  }
+  os_fd_t socks[2];
+  Api::SysCallIntResult result =
+      Api::OsSysCallsSingleton::get().socketpair(AF_INET, SOCK_STREAM, IPPROTO_TCP, socks);
+  ASSERT(result.rc_ == 0);
+
+  read_handle_ = std::make_unique<Network::IoSocketHandleImpl>(socks[0], false, AF_INET);
+  result = read_handle_->setBlocking(false);
+  ASSERT(result.rc_ == 0);
+  auto write_handle = std::make_shared<Network::IoSocketHandleImpl>(socks[1], false, AF_INET);
+  result = write_handle->setBlocking(false);
+  ASSERT(result.rc_ == 0);
+
+  read_handle_->initializeFileEvent(
+      dispatcher,
+      [this](uint32_t events) -> void {
+        ASSERT(events == Event::FileReadyType::Read);
+        cb_();
+      },
+      Event::FileTriggerType::Level, Event::FileReadyType::Read);
+  eventBridgeHandlersSingleton::get()[signal_num] = write_handle;
+}
+
+} // namespace Event
+} // namespace Envoy

source/common/event/win32/signal_impl.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "envoy/event/signal.h"
+#include "envoy/network/io_handle.h"
+
+#include "common/event/dispatcher_impl.h"
+#include "common/event/event_impl_base.h"
+#include "common/network/io_socket_handle_impl.h"
+#include "common/singleton/threadsafe_singleton.h"
+
+#include "absl/container/flat_hash_map.h"
+
+namespace Envoy {
+namespace Event {
+
+/**
+ * libevent implementation of Event::SignalEvent.
+ */
+class SignalEventImpl : public SignalEvent {
+public:
+  SignalEventImpl(DispatcherImpl& dispatcher, signal_t signal_num, SignalCb cb);
+
+private:
+  SignalCb cb_;
+  Network::IoHandlePtr read_handle_;
+};
+
+// Windows ConsoleControlHandler does not allow for a context. As a result the thread
+// spawned to handle the console events communicates with the main program with this socketpair.
+// Here we have a map from signal types to IoHandle. When we write to this handle we trigger an
+// event that notifies Envoy to act on the signal.
+using eventBridgeHandlersSingleton =
+    ThreadSafeSingleton<std::array<std::shared_ptr<Network::IoHandle>, ENVOY_WIN32_SIGNAL_COUNT>>;
+} // namespace Event
+} // namespace Envoy

source/exe/BUILD

@@ -168,8 +168,10 @@ envoy_cc_win32_library(
     srcs = ["win32/platform_impl.cc"],
     deps = [
         ":platform_header_lib",
+        "//source/common/buffer:buffer_lib",
         "//source/common/common:assert_lib",
         "//source/common/common:thread_lib",
+        "//source/common/event:signal_lib",
         "//source/common/filesystem:filesystem_lib",
     ],
 )

source/exe/win32/platform_impl.cc

@@ -1,17 +1,59 @@
+#include <chrono>
+#include <thread>
+
+#include "common/buffer/buffer_impl.h"
 #include "common/common/assert.h"
 #include "common/common/thread_impl.h"
+#include "common/event/signal_impl.h"
 #include "common/filesystem/filesystem_impl.h"
 
 #include "exe/platform_impl.h"
 
 namespace Envoy {
 
+static std::atomic<bool> shutdown_pending = false;
+
+BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) {
+  if (shutdown_pending) {
+    return 0;
+  }
+  shutdown_pending = true;
+
+  auto handler = Event::eventBridgeHandlersSingleton::get()[ENVOY_SIGTERM];
+  if (!handler) {
+    return 0;
+  }
+
+  // This code is executed as part of a thread running under a thread owned and
+  // managed by Windows console host. For that reason we want to avoid allocating
+  // substantial amount of memory or taking locks.
+  // This is why we write to a socket to wake up the signal handler.
+  char data[] = {'a'};
+  Buffer::RawSlice buffer{data, 1};
+  auto result = handler->writev(&buffer, 1);
+  RELEASE_ASSERT(result.rc_ == 1,
+                 fmt::format("failed to write 1 byte: {}", result.err_->getErrorDetails()));
+
+  if (fdwCtrlType == CTRL_LOGOFF_EVENT || fdwCtrlType == CTRL_SHUTDOWN_EVENT) {
+    // These events terminate the process immediately so we want to give a couple of seconds
+    // to the dispatcher to shutdown the server.
+    constexpr size_t delay = 3;
+    absl::SleepFor(absl::Seconds(delay));
+  }
+  return 1;
+}
+
 PlatformImpl::PlatformImpl()
     : thread_factory_(std::make_unique<Thread::ThreadFactoryImplWin32>()),
       file_system_(std::make_unique<Filesystem::InstanceImplWin32>()) {
   WSADATA wsa_data;
   const WORD version_requested = MAKEWORD(2, 2);
   RELEASE_ASSERT(WSAStartup(version_requested, &wsa_data) == 0, "WSAStartup failed with error");
+
+  if (!SetConsoleCtrlHandler(CtrlHandler, 1)) {
+    // The Control Handler is executing in a different thread.
+    ENVOY_LOG_MISC(warn, "Could not set Windows Control Handlers. Continuing without them.");
+  }
 }
 
 PlatformImpl::~PlatformImpl() { ::WSACleanup(); }

source/server/server.cc

@@ -648,15 +648,16 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch
         }
       }) {
   // Setup signals.
+  // Since signals are not supported on Windows we have an internal definition for `SIGTERM`
+  // On POSIX it resolves as expected to SIGTERM
+  // On Windows we use it internally for all the console events that indicate that we should
+  // terminate the process.
   if (options.signalHandlingEnabled()) {
-// TODO(Pivotal): Figure out solution to graceful shutdown on Windows. None of these signals exist
-// on Windows.
-#ifndef WIN32
-    sigterm_ = dispatcher.listenForSignal(SIGTERM, [&instance]() {
-      ENVOY_LOG(warn, "caught SIGTERM");
+    sigterm_ = dispatcher.listenForSignal(ENVOY_SIGTERM, [&instance]() {
+      ENVOY_LOG(warn, "caught ENVOY_SIGTERM");
       instance.shutdown();
     });
-
+#ifndef WIN32
     sigint_ = dispatcher.listenForSignal(SIGINT, [&instance]() {
       ENVOY_LOG(warn, "caught SIGINT");
       instance.shutdown();

test/exe/BUILD

@@ -84,3 +84,25 @@ envoy_cc_test(
         "//test/test_common:utility_lib",
     ],
 )
+
+# Due to the limitiations of how Windows signals work
+# this test cannot be executed through bazel and it must
+# be executed as a standalone executable.
+# e.g.: `./bazel-bin/test/exe/win32_outofproc_main_test.exe`
+envoy_cc_test(
+    name = "win32_outofproc_main_test",
+    srcs = ["win32_outofproc_main_test.cc"],
+    data = [
+        "//source/exe:envoy-static",
+        "//test/config/integration:google_com_proxy_port_0",
+    ],
+    # TODO(envoyproxy/windows-dev): Disable the manual tag.
+    tags = ["manual"],
+    deps = [
+        "//source/common/api:api_lib",
+        "//source/exe:main_common_lib",
+        "//test/mocks/runtime:runtime_mocks",
+        "//test/test_common:contention_lib",
+        "//test/test_common:environment_lib",
+    ],
+)