diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 5e5196894b8..3b952ce8da5 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -202,6 +202,8 @@ const ( FrontendThrottledLogRPS = "frontend.throttledLogRPS" // FrontendShutdownDrainDuration is the duration of traffic drain during shutdown FrontendShutdownDrainDuration = "frontend.shutdownDrainDuration" + // FrontendShutdownFailHealthcheckDuration is the duration of shutdown failure detection + FrontendShutdownFailHealthcheckDuration = "frontend.shutdownFailHealthcheckDuration" // FrontendMaxBadBinaries is the max number of bad binaries in namespace config FrontendMaxBadBinaries = "frontend.maxBadBinaries" // SendRawWorkflowHistory is whether to enable raw history retrieving diff --git a/service/frontend/service.go b/service/frontend/service.go index 63d6b4dd8c7..42d4ddbd12e 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -85,6 +85,7 @@ type Config struct { WorkerBuildIdSizeLimit dynamicconfig.IntPropertyFn DisallowQuery dynamicconfig.BoolPropertyFnWithNamespaceFilter ShutdownDrainDuration dynamicconfig.DurationPropertyFn + ShutdownFailHealthcheckDuration dynamicconfig.DurationPropertyFn MaxBadBinaries dynamicconfig.IntPropertyFnWithNamespaceFilter @@ -207,6 +208,7 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int32, enableReadF BlobSizeLimitWarn: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.BlobSizeLimitWarn, 256*1024), ThrottledLogRPS: dc.GetIntProperty(dynamicconfig.FrontendThrottledLogRPS, 20), ShutdownDrainDuration: dc.GetDurationProperty(dynamicconfig.FrontendShutdownDrainDuration, 0*time.Second), + ShutdownFailHealthcheckDuration: dc.GetDurationProperty(dynamicconfig.FrontendShutdownFailHealthcheckDuration, 10*time.Second), EnableNamespaceNotActiveAutoForwarding: dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.EnableNamespaceNotActiveAutoForwarding, true), SearchAttributesNumberOfKeysLimit: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.SearchAttributesNumberOfKeysLimit, 100), SearchAttributesSizeOfValueLimit: dc.GetIntPropertyFilteredByNamespace(dynamicconfig.SearchAttributesSizeOfValueLimit, 2*1024), @@ -336,11 +338,11 @@ func (s *Service) Stop() { // 1. Fail rpc health check, this will cause client side load balancer to stop forwarding requests to this node // 2. wait for failure detection time // 3. stop taking new requests by returning InternalServiceError - // 4. Wait for a second + // 4. Wait for X second // 5. Stop everything forcefully and return - requestDrainTime := util.Min(time.Second, s.config.ShutdownDrainDuration()) - failureDetectionTime := util.Max(0, s.config.ShutdownDrainDuration()-requestDrainTime) + requestDrainTime := util.Max(time.Second, s.config.ShutdownDrainDuration()) + failureDetectionTime := util.Max(0, s.config.ShutdownFailHealthcheckDuration()) logger.Info("ShutdownHandler: Updating gRPC health status to ShuttingDown") s.healthServer.Shutdown()