[dbnode] Fix integration code under race with TChannel channel option…

…s mutation issue (#2958)

src/dbnode/client/options.go

@@ -318,9 +318,16 @@ func NewOptionsForAsyncClusters(opts Options, topoInits []topology.Initializer,
 }
 
 func defaultNewConnectionFn(
-	channelName string, address string, opts Options,
+	channelName string, address string, clientOpts Options,
 ) (xresource.SimpleCloser, rpc.TChanNode, error) {
-	channel, err := tchannel.NewChannel(channelName, opts.ChannelOptions())
+	// NB(r): Keep ref to a local channel options since it's actually modified
+	// by TChannel itself to set defaults.
+	var opts *tchannel.ChannelOptions
+	if chanOpts := clientOpts.ChannelOptions(); chanOpts != nil {
+		immutableOpts := *chanOpts
+		opts = &immutableOpts
+	}
+	channel, err := tchannel.NewChannel(channelName, opts)
 	if err != nil {
 		return nil, nil, err
 	}

src/dbnode/network/server/tchannelthrift/cluster/server.go

@@ -50,11 +50,6 @@ func NewServer(
 	contextPool context.Pool,
 	opts *tchannel.ChannelOptions,
 ) ns.NetworkService {
-	// Make the opts immutable on the way in
-	if opts != nil {
-		immutableOpts := *opts
-		opts = &immutableOpts
-	}
 	return &server{
 		address:     address,
 		client:      client,
@@ -64,7 +59,14 @@ func NewServer(
 }
 
 func (s *server) ListenAndServe() (ns.Close, error) {
-	channel, err := tchannel.NewChannel(ChannelName, s.opts)
+	// NB(r): Keep ref to a local channel options since it's actually modified
+	// by TChannel itself to set defaults.
+	var opts *tchannel.ChannelOptions
+	if s.opts != nil {
+		immutableOpts := *s.opts
+		opts = &immutableOpts
+	}
+	channel, err := tchannel.NewChannel(ChannelName, opts)
 	if err != nil {
 		return nil, err
 	}

src/dbnode/network/server/tchannelthrift/node/server.go

@@ -52,8 +52,14 @@ func NewServer(
 }
 
 func (s *server) ListenAndServe() (ns.Close, error) {
-	chanOpts := s.opts.ChannelOptions()
-	channel, err := tchannel.NewChannel(channel.ChannelName, chanOpts)
+	// NB(r): Keep ref to a local channel options since it's actually modified
+	// by TChannel itself to set defaults.
+	var opts *tchannel.ChannelOptions
+	if chanOpts := s.opts.ChannelOptions(); chanOpts != nil {
+		immutableOpts := *chanOpts
+		opts = &immutableOpts
+	}
+	channel, err := tchannel.NewChannel(channel.ChannelName, opts)
 	if err != nil {
 		return nil, err
 	}