kv: log if request appears stuck

See cockroachdb#8975. Print either when there are many retries for the current chunk,
or a high sequence number (which could be caused by retries higher up the
stack).

kv/dist_sender.go

@@ -680,7 +680,22 @@ func (ds *DistSender) sendChunk(ctx context.Context, ba roachpb.BatchRequest) (*
 		var needAnother bool
 		var pErr *roachpb.Error
 		var finished bool
+		var numAttempts int
 		for r := retry.StartWithCtx(ctx, ds.rpcRetryOptions); r.Next(); {
+			numAttempts++
+			const magicLogCurAttempt = 20
+			if (numAttempts%magicLogCurAttempt == 0) || (ba.Txn != nil && (ba.Txn.Sequence%magicLogCurAttempt == 0)) {
+				// Log a message if a request appears to get stuck for a long
+				// time or, potentially, forever. See #8975.
+				// The local counter captures this loop here; the Sequence number
+				// should capture anything higher up (as it needs to be
+				// incremented every time this method is called).
+				log.Warningf(
+					ctx,
+					"%d retries for an RPC at sequence %d, last error was: %s, remaining key ranges %s: %s",
+					numAttempts, ba.Txn.Sequence, pErr, rs, ba,
+				)
+			}
 			// Get range descriptor (or, when spanning range, descriptors). Our
 			// error handling below may clear them on certain errors, so we
 			// refresh (likely from the cache) on every retry.