Set defaults and expose configuration of tchannel timeouts.

src/cmd/services/m3dbnode/config/config.go

@@ -157,6 +157,9 @@ type DBConfiguration struct {
 	// Limits contains configuration for limits that can be applied to M3DB for the purposes
 	// of applying back-pressure or protecting the db nodes.
 	Limits Limits `yaml:"limits"`
+
+	// TChannel exposes TChannel config options.
+	TChannel *TChannelConfiguration `yaml:"tchannel"`
 }
 
 // InitDefaultsAndValidate initializes all default values and validates the Configuration.
@@ -573,3 +576,9 @@ func IsSeedNode(initialCluster []environment.SeedNode, hostID string) bool {
 
 	return false
 }
+
+// TChannelConfiguration holds TChannel config options.
+type TChannelConfiguration struct {
+	MaxIdleTime       time.Duration `yaml:"maxIdleTime"`
+	IdleCheckInterval time.Duration `yaml:"idleCheckInterval"`
+}

src/cmd/services/m3dbnode/config/config_test.go

@@ -714,6 +714,7 @@ func TestConfiguration(t *testing.T) {
     maxOutstandingWriteRequests: 0
     maxOutstandingReadRequests: 0
     maxOutstandingRepairedBytes: 0
+  tchannel: null
 coordinator: null
 `
 

src/dbnode/client/config.go

@@ -31,7 +31,6 @@ import (
 	"github.com/m3db/m3/src/dbnode/environment"
 	"github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/dbnode/topology"
-	xtchannel "github.com/m3db/m3/src/dbnode/x/tchannel"
 	xerrors "github.com/m3db/m3/src/x/errors"
 	"github.com/m3db/m3/src/x/ident"
 	"github.com/m3db/m3/src/x/instrument"
@@ -297,7 +296,6 @@ func (c Configuration) NewAdminClient(
 	v := NewAdminOptions().
 		SetTopologyInitializer(syncTopoInit).
 		SetAsyncTopologyInitializers(asyncTopoInits).
-		SetChannelOptions(xtchannel.NewDefaultChannelOptions()).
 		SetInstrumentOptions(iopts)
 
 	if c.UseV2BatchAPIs != nil {

src/dbnode/client/options.go

@@ -199,6 +199,12 @@ var (
 			SetJitter(true),
 	)
 
+	// defaultChannelOptions are default tchannel channel options.
+	defaultChannelOptions = &tchannel.ChannelOptions{
+		MaxIdleTime:       5 * time.Minute,
+		IdleCheckInterval: 5 * time.Minute,
+	}
+
 	errNoTopologyInitializerSet    = errors.New("no topology initializer set")
 	errNoReaderIteratorAllocateSet = errors.New("no reader iterator allocator set, encoding not set")
 )
@@ -312,6 +318,7 @@ func newOptions() *options {
 	opts := &options{
 		clockOpts:                               clock.NewOptions(),
 		instrumentOpts:                          instrument.NewOptions(),
+		channelOptions:                          defaultChannelOptions,
 		writeConsistencyLevel:                   defaultWriteConsistencyLevel,
 		readConsistencyLevel:                    defaultReadConsistencyLevel,
 		bootstrapConsistencyLevel:               defaultBootstrapConsistencyLevel,

src/dbnode/server/server.go

@@ -90,6 +90,7 @@ import (
 	apachethrift "github.com/apache/thrift/lib/go/thrift"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/uber-go/tally"
+	"github.com/uber/tchannel-go"
 	"go.etcd.io/etcd/embed"
 	"go.uber.org/zap"
 )
@@ -599,6 +600,10 @@ func Run(runOpts RunOptions) {
 		// SetDatabase() once we've initialized it.
 		service = ttnode.NewService(nil, ttopts)
 	)
+	if cfg.TChannel != nil {
+		tchannelOpts.MaxIdleTime = cfg.TChannel.MaxIdleTime
+		tchannelOpts.IdleCheckInterval = cfg.TChannel.IdleCheckInterval
+	}
 	tchannelthriftNodeClose, err := ttnode.NewServer(service,
 		cfg.ListenAddress, contextPool, tchannelOpts).ListenAndServe()
 	if err != nil {
@@ -694,7 +699,7 @@ func Run(runOpts RunOptions) {
 
 	origin := topology.NewHost(hostID, "")
 	m3dbClient, err := newAdminClient(
-		cfg.Client, iopts, syncCfg.TopologyInitializer, runtimeOptsMgr,
+		cfg.Client, iopts, tchannelOpts, syncCfg.TopologyInitializer, runtimeOptsMgr,
 		origin, protoEnabled, schemaRegistry, syncCfg.KVStore, logger)
 	if err != nil {
 		logger.Fatal("could not create m3db client", zap.Error(err))
@@ -729,7 +734,7 @@ func Run(runOpts RunOptions) {
 			// Guaranteed to not be nil if repair is enabled by config validation.
 			clientCfg := *cluster.Client
 			clusterClient, err := newAdminClient(
-				clientCfg, iopts, topologyInitializer, runtimeOptsMgr,
+				clientCfg, iopts, tchannelOpts, topologyInitializer, runtimeOptsMgr,
 				origin, protoEnabled, schemaRegistry, syncCfg.KVStore, logger)
 			if err != nil {
 				logger.Fatal(
@@ -1544,6 +1549,7 @@ func withEncodingAndPoolingOptions(
 func newAdminClient(
 	config client.Configuration,
 	iopts instrument.Options,
+	tchannelOpts *tchannel.ChannelOptions,
 	topologyInitializer topology.Initializer,
 	runtimeOptsMgr m3dbruntime.OptionsManager,
 	origin topology.Host,
@@ -1564,6 +1570,9 @@ func newAdminClient(
 				SetMetricsScope(iopts.MetricsScope().SubScope("m3dbclient")),
 			TopologyInitializer: topologyInitializer,
 		},
+		func(opts client.AdminOptions) client.AdminOptions {
+			return opts.SetChannelOptions(tchannelOpts).(client.AdminOptions)
+		},
 		func(opts client.AdminOptions) client.AdminOptions {
 			return opts.SetRuntimeOptionsManager(runtimeOptsMgr).(client.AdminOptions)
 		},

src/dbnode/x/tchannel/options.go

@@ -20,11 +20,22 @@
 
 package xtchannel
 
-import tchannel "github.com/uber/tchannel-go"
+import (
+	"time"
+
+	tchannel "github.com/uber/tchannel-go"
+)
+
+const (
+	defaultIdleCheckInterval = 5 * time.Minute
+	defaultMaxIdleTime       = 5 * time.Minute
+)
 
 // NewDefaultChannelOptions returns the default tchannel options used.
 func NewDefaultChannelOptions() *tchannel.ChannelOptions {
 	return &tchannel.ChannelOptions{
-		Logger: NewNoopLogger(),
+		Logger:            NewNoopLogger(),
+		MaxIdleTime:       defaultMaxIdleTime,
+		IdleCheckInterval: defaultIdleCheckInterval,
 	}
 }

src/query/storage/m3/config.go

@@ -31,13 +31,21 @@ import (
 	"github.com/m3db/m3/src/query/stores/m3db"
 	"github.com/m3db/m3/src/x/ident"
 	"github.com/m3db/m3/src/x/instrument"
+
+	"github.com/uber/tchannel-go"
 )
 
 var (
 	errNotAggregatedClusterNamespace              = goerrors.New("not an aggregated cluster namespace")
 	errBothNamespaceTypeNewAndDeprecatedFieldsSet = goerrors.New("cannot specify both deprecated and non-deprecated fields for namespace type")
 )
 
+// TODO(bodu): Could make these configurable at some point.
+const (
+	idleCheckInterval = 5 * time.Minute
+	maxIdleTime       = 5 * time.Minute
+)
+
 // ClustersStaticConfiguration is a set of static cluster configurations.
 type ClustersStaticConfiguration []ClusterStaticConfiguration
 
@@ -187,9 +195,16 @@ func (c ClustersStaticConfiguration) NewClusters(
 
 		if opts.ProvidedSession == nil {
 			// NB(r): Only create client session if not already provided.
-			result, err = clusterCfg.newClient(client.ConfigurationParameters{
-				InstrumentOptions: instrumentOpts,
-			})
+			result, err = clusterCfg.newClient(
+				client.ConfigurationParameters{
+					InstrumentOptions: instrumentOpts,
+				},
+				func(opts client.Options) client.Options {
+					return opts.SetChannelOptions(&tchannel.ChannelOptions{
+						IdleCheckInterval: idleCheckInterval,
+						MaxIdleTime:       maxIdleTime,
+					})
+				})
 			if err != nil {
 				return nil, err
 			}