I170105 16:17:11.471570 301 server/status/runtime.go:228 runtime stats: 8.3 GiB RSS, 469 goroutines, 793 MiB/230 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 23429.32cgo/sec, 2.45/0.47 %(u/s)time, 0.00 %gc (2x) I170105 16:17:21.342134 301 server/status/runtime.go:228 runtime stats: 8.3 GiB RSS, 363 goroutines, 542 MiB/356 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.9 GiB CGO alloc/total, 30909.95cgo/sec, 2.88/0.53 %(u/s)time, 0.00 %gc (2x) I170105 16:17:31.354507 301 server/status/runtime.go:228 runtime stats: 8.3 GiB RSS, 363 goroutines, 772 MiB/260 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.9 GiB CGO alloc/total, 55498.70cgo/sec, 3.19/0.59 %(u/s)time, 0.00 %gc (2x) W170105 16:17:31.970487 220 storage/replica.go:581 [n10,s19,r21092/1:/Table/51/1/1091{3736…-6845…},@c420ea8d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 546.509335ms (>500ms): goroutine 220 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x20931217) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420ea8df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420ea8d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x5264) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219768a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:31.971072 220 storage/store.go:3358 [n10,s19,r21092/1:/Table/51/1/1091{3736…-6845…},@c420ea8d80] handle raft ready: 0.5s [processed=1] W170105 16:17:31.973193 226 storage/replica.go:581 [n10,s19,r3883/2:/Table/51/1/8716{5198…-9820…},@c421e5f680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 547.988037ms (>500ms): goroutine 226 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x20a9a245) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc421e5f6f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421e5f680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0xf2b) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc421976a20) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:31.973483 226 storage/store.go:3358 [n10,s19,r3883/2:/Table/51/1/8716{5198…-9820…},@c421e5f680] handle raft ready: 0.5s [processed=1] W170105 16:17:31.974519 79520957 storage/replica.go:1518 [n10,s19,r11031/1:/Table/51/1/4625{4722…-7721…},@c421affb00] context canceled before command queue: [txn: f65d1886], BeginTransaction [/Table/51/1/4625625523160184062/"ef2583ef-e8de-4fbb-a733-84246f818362"/15968806/0,/Min), ConditionalPut [/Table/51/1/4625625523160184062/"ef2583ef-e8de-4fbb-a733-84246f818362"/15968806/0,/Min), EndTransaction [/Table/51/1/4625625523160184062/"ef2583ef-e8de-4fbb-a733-84246f818362"/15968806/0,/Min) W170105 16:17:31.977762 323 storage/replica.go:581 [n10,s20,r5560/1:/Table/51/1/2626{4635…-7746…},@c420f0a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 552.847759ms (>500ms): goroutine 323 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x20f3c98f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420f0a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420f0a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x15b8) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a00c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:31.977828 323 storage/store.go:3358 [n10,s20,r5560/1:/Table/51/1/2626{4635…-7746…},@c420f0a480] handle raft ready: 0.6s [processed=0] W170105 16:17:31.980959 212 storage/replica.go:581 [n10,s19,r7917/1:/Table/51/1/114{08651…-12808…},@c4214cbb00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 557.056776ms (>500ms): goroutine 212 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x21340308) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4214cbb78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4214cbb00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x1eed) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219767a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:31.980996 212 storage/store.go:3358 [n10,s19,r7917/1:/Table/51/1/114{08651…-12808…},@c4214cbb00] handle raft ready: 0.6s [processed=1] W170105 16:17:31.982490 324 storage/replica.go:581 [n10,s20,r24303/3:/Table/51/1/1554{3639…-6352…},@c4228b8000] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 557.736625ms (>500ms): goroutine 324 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x213e62b1) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4228b8078) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4228b8000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x5eef) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a00e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:31.982783 324 storage/store.go:3358 [n10,s20,r24303/3:/Table/51/1/1554{3639…-6352…},@c4228b8000] handle raft ready: 0.6s [processed=1] W170105 16:17:31.983650 222 storage/replica.go:581 [n10,s19,r17154/3:/Table/51/1/4432{5045…-8824…},@c421a6e900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 558.63073ms (>500ms): goroutine 222 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x214c074a) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc421a6e978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421a6e900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x4302) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219768e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:31.984000 222 storage/store.go:3358 [n10,s19,r17154/3:/Table/51/1/4432{5045…-8824…},@c421a6e900] handle raft ready: 0.6s [processed=0] W170105 16:17:31.985250 79520987 storage/replica.go:1518 [n10,s19,r6397/2:/Table/51/1/8713{5777…-8521…},@c421e7e480] context canceled before command queue: [txn: 885c0cfd], BeginTransaction [/Table/51/1/8713836967577799312/"2da89b47-4e48-42ee-b595-8b895fd79628"/16671902/0,/Min), ConditionalPut [/Table/51/1/8713836967577799312/"2da89b47-4e48-42ee-b595-8b895fd79628"/16671902/0,/Min), EndTransaction [/Table/51/1/8713836967577799312/"2da89b47-4e48-42ee-b595-8b895fd79628"/16671902/0,/Min) W170105 16:17:31.999711 223 storage/replica.go:581 [n10,s19,r12169/2:/Table/51/1/8630{6491…-9250…},@c42232a000] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 574.82339ms (>500ms): goroutine 223 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x22431bde) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42232a078) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42232a000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x2f89) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc421976900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:31.999759 223 storage/store.go:3358 [n10,s19,r12169/2:/Table/51/1/8630{6491…-9250…},@c42232a000] handle raft ready: 0.6s [processed=1] W170105 16:17:32.002959 217 storage/replica.go:581 [n10,s19,r16203/1:/Table/51/1/1150{4146…-8291…},@c42153ad80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 585.397006ms (>500ms): goroutine 217 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x22e4730e) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42153adf8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42153ad80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x3f4b) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc421976840) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:32.003000 217 storage/store.go:3358 [n10,s19,r16203/1:/Table/51/1/1150{4146…-8291…},@c42153ad80] handle raft ready: 0.6s [processed=1] W170105 16:17:32.006191 241 storage/replica.go:581 [n10,s20,r16460/3:/Table/51/1/691{68217…-72188…},@c42340f680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 581.042592ms (>500ms): goroutine 241 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x22a201a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42340f6f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42340f680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x404c) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0080) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:32.006237 241 storage/store.go:3358 [n10,s20,r16460/3:/Table/51/1/691{68217…-72188…},@c42340f680] handle raft ready: 0.6s [processed=0] W170105 16:17:32.019612 325 storage/replica.go:581 [n10,s20,r15177/3:/Table/51/1/714{24015…-68456…},@c421df8d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 595.739561ms (>500ms): goroutine 325 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x238243a9) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc421df8df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421df8d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x3b49) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0100) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:17:32.019645 325 storage/store.go:3358 [n10,s20,r15177/3:/Table/51/1/714{24015…-68456…},@c421df8d80] handle raft ready: 0.6s [processed=1] I170105 16:17:41.337846 301 server/status/runtime.go:228 runtime stats: 8.3 GiB RSS, 363 goroutines, 728 MiB/255 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.9 GiB CGO alloc/total, 23351.79cgo/sec, 2.40/0.45 %(u/s)time, 0.00 %gc (2x) I170105 16:17:51.343515 301 server/status/runtime.go:228 runtime stats: 8.3 GiB RSS, 367 goroutines, 709 MiB/282 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.9 GiB CGO alloc/total, 41923.63cgo/sec, 3.60/0.66 %(u/s)time, 0.00 %gc (3x) W170105 16:17:54.485202 79539108 storage/replica.go:581 [n10,s20,r4852/3:/Table/51/1/1574{1251…-4826…},@c422b97b00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose for 507.950103ms (>500ms): goroutine 79539108 [running]: runtime/debug.Stack(0x1f4a8b8, 0x1f4a8f8, 0x3f) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e46b417) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422b97b78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc422b97b00, 0x7fe50b4c4880, 0xc531cd5da0, 0xc54d1fbd10, 0x1496ec6b8f524a6e, 0x0, 0x140000000a, 0x3, 0x12f4, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2317 +0x78c github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc422b97b00, 0x7fe50b4c4880, 0xc531cd5da0, 0x1496ec6b8f524a6e, 0x0, 0x140000000a, 0x3, 0x12f4, 0x0, 0xc438c64000, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2036 +0x1fe github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc422b97b00, 0x7fe50b4c4880, 0xc531cd5da0, 0x1496ec6b8f524a6e, 0x0, 0x140000000a, 0x3, 0x12f4, 0x0, 0xc438c64000, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1872 +0x90 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc422b97b00, 0x7fe50b4c4880, 0xc531cd5da0, 0x1496ec6b8f524a6e, 0x0, 0x140000000a, 0x3, 0x12f4, 0x0, 0xc438c64000, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1324 +0x24d github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420358a80, 0x7fe50b4c4880, 0xc531cd5ce0, 0x1496ec6b8f524a6e, 0x0, 0x140000000a, 0x3, 0x12f4, 0x0, 0xc438c64000, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2485 +0x6c5 github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc420174f60, 0x7fe50b4c4880, 0xc531cd5b90, 0x0, 0x0, 0x140000000a, 0x3, 0x12f4, 0x0, 0xc438c64000, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:825 +0x2a1 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc420304090, 0xc445cdd9f8, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0x100 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4205fab00, 0x7fe50b4c4880, 0xc531cd57a0, 0xc4fe3e1ec8, 0xc531cd57a0, 0xc531cd5320, 0x18370e0) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:836 +0x1ed github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4205fab00, 0x7fe50b4c4880, 0xc531cd57a0, 0xc4fe3e1ec8, 0x1f, 0x0, 0x1496ec6b8f524a6e) /go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:858 +0xa0 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc454c46f00, 0xc4fe3e1ea0, 0x7fe50b4c4880, 0xc531cd5620, 0xc4209d0768, 0xd86435, 0xc4209d0760) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:189 +0x49d github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0x263a660, 0xc4bd490140, 0xc454c46f00, 0xc4fe3e1ea0, 0x7fe50b4c4880, 0xc531cd5620, 0xc4520f0600) /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:203 +0xbd created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:206 +0x142 W170105 16:17:57.356251 79542256 storage/replica.go:1518 [n10,s20,r16307/3:/Table/51/1/28{57093…-98416…},@c421171680] context canceled before command queue: [txn: e1212c4c], BeginTransaction [/Table/51/1/28979529935415129/"caa98563-e7df-4c14-a435-0ee049b0b8ae"/15610068/0,/Min), ConditionalPut [/Table/51/1/28979529935415129/"caa98563-e7df-4c14-a435-0ee049b0b8ae"/15610068/0,/Min), EndTransaction [/Table/51/1/28979529935415129/"caa98563-e7df-4c14-a435-0ee049b0b8ae"/15610068/0,/Min) W170105 16:18:00.589025 212 storage/replica.go:581 [n10,s19,r8317/3:/Table/51/1/21{52304…-93681…},@c420a9db00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tick for 585.818612ms (>500ms): goroutine 212 [running]: runtime/debug.Stack(0x1f4b430, 0x1f4b470, 0x3c) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x22eae1f4) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420a9db78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tick(0xc420a9db00, 0xc4202ca500, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2706 +0xa3 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processTick(0xc420435180, 0x207d, 0xc4866b9f00) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3388 +0xfb github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:225 +0x2b6 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219767a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 I170105 16:18:01.345770 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 368 goroutines, 507 MiB/400 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.9 GiB CGO alloc/total, 86495.72cgo/sec, 4.22/0.63 %(u/s)time, 0.00 %gc (4x) I170105 16:18:11.326884 299 gossip/gossip.go:437 [n10] gossip status (ok, 10 nodes) gossip client (2/3 cur/max conns) 7: cockroach-delta-07:26257 (13h28m39s: infos 113898/227894 sent/received, bytes 26963562B/50915933B sent/received) 3: cockroach-delta-03:26257 (3h4m13s: infos 33103/25833 sent/received, bytes 7925642B/9462062B sent/received) gossip server (3/3 cur/max conns, infos 857781/1442218 sent/received, bytes 208604661B/282161350B sent/received) 5: cockroach-delta-05:26257 (4h48m0s) 6: cockroach-delta-06:26257 (3h43m43s) 9: cockroach-delta-09:26257 (3h45m47s) I170105 16:18:11.343193 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 364 goroutines, 681 MiB/322 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 49281.16cgo/sec, 4.12/0.73 %(u/s)time, 0.00 %gc (3x) W170105 16:18:15.340411 326 storage/replica.go:593 [n10,s20,r20559/1:/Table/51/1/6047{1204…-4463…},@c422800480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).withRaftGroup for 660.894327ms (>500ms): goroutine 326 [running]: runtime/debug.Stack(0x1f472d8, 0x1f47318, 0x45) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x27647277) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422800568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).withRaftGroup(0xc422800480, 0xc43cd9b858, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:551 +0xa7 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420358a80, 0x7fe50b4c4880, 0xc519625da0, 0xc544923380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3082 +0xbe5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc420358a80, 0x504f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3324 +0x174 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:241 +0x267 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:15.348694 326 storage/replica.go:581 [n10,s20,r20559/1:/Table/51/1/6047{1204…-4463…},@c422800480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest for 669.212835ms (>500ms): goroutine 326 [running]: runtime/debug.Stack(0x1f5ed60, 0x1f5eda0, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x27e360a3) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4228004f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420358a80, 0x7fe50b4c4880, 0xc519625da0, 0xc544923380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3099 +0xc80 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc420358a80, 0x504f) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3324 +0x174 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:241 +0x267 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:15.348725 331 storage/replica.go:593 [n10,s20,r13634/1:/Table/51/1/6058{1619…-6570…},@c422fa1680] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 666.04124ms (>500ms): goroutine 331 [running]: runtime/debug.Stack(0x1f4b0d8, 0x1f4b118, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x27b2fb98) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422fa1768) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc422fa1680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2505 +0x22e github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422fa1680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x3542) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a01c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:15.354426 331 storage/replica.go:581 [n10,s20,r13634/1:/Table/51/1/6058{1619…-6570…},@c422fa1680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 674.650802ms (>500ms): goroutine 331 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x28365ab2) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422fa16f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422fa1680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x3542) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a01c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:15.354470 331 storage/store.go:3358 [n10,s20,r13634/1:/Table/51/1/6058{1619…-6570…},@c422fa1680] handle raft ready: 0.7s [processed=1] W170105 16:18:18.726656 79564634 storage/replica.go:1518 [n10,s20,r18032/3:/System/tsd/cr.store.raft.rcv…,@c4210b0000] context canceled before command queue: Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, Noop, ... 514 skipped ..., Noop, Noop, Noop, Noop, Noop I170105 16:18:21.345646 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 369 goroutines, 740 MiB/267 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 45056.54cgo/sec, 3.79/0.64 %(u/s)time, 0.00 %gc (3x) W170105 16:18:24.531627 239 storage/replica.go:581 [n10,s20,r10247/2:/Table/51/1/303{01877…-35128…},@c42216ed80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 587.238738ms (>500ms): goroutine 239 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x23008d52) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42216edf8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42216ed80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x2807) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc422795fe0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:24.531671 239 storage/store.go:3358 [n10,s20,r10247/2:/Table/51/1/303{01877…-35128…},@c42216ed80] handle raft ready: 0.6s [processed=1] I170105 16:18:31.352270 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 513 goroutines, 762 MiB/265 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 46330.94cgo/sec, 4.48/0.73 %(u/s)time, 0.00 %gc (4x) W170105 16:18:40.560365 322 storage/replica.go:581 [n10,s20,r23303/3:/Table/51/1/1468{5614…-8396…},@c4214a7b00] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 535.841543ms (>500ms): goroutine 322 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1ff04b07) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4214a7b78) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4214a7b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x5b07) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a00a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:40.563130 322 storage/store.go:3358 [n10,s20,r23303/3:/Table/51/1/1468{5614…-8396…},@c4214a7b00] handle raft ready: 0.5s [processed=1] W170105 16:18:41.139056 213 storage/engine/rocksdb.go:1036 batch [3/237/3] commit took 1.195576127s (>500ms): goroutine 213 [running]: runtime/debug.Stack(0x18def40, 0x262f9a0, 0xc42000e2f8) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/storage/engine.(*rocksDBBatch).Commit(0xc4fe936c40, 0xc433d8b3b0, 0x2639aa0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1036 +0x169 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommand(0xc421f5c480, 0x7fe50b4c4880, 0xc433d8b3b0, 0xc53ccda1b0, 0x8, 0x0, 0x403b7, 0x3fb8c, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3683 +0x4c2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc421f5c480, 0x7fe50b4c4880, 0xc433d8b3b0, 0xc53ccda1b0, 0x8, 0x403b7, 0x600000003, 0x1, 0xc42c944550, 0x3fb8c, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3468 +0x59b github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc421f5c480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2652 +0x845 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421f5c480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x2d89) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219767c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:41.141699 213 storage/replica.go:581 [n10,s19,r11657/2:/Table/51/1/8702{0710…-3469…},@c421f5c480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.202613175s (>500ms): goroutine 213 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x47ae6bb7) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc421f5c4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421f5c480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x2d89) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219767c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:41.141917 213 storage/store.go:3358 [n10,s19,r11657/2:/Table/51/1/8702{0710…-3469…},@c421f5c480] handle raft ready: 1.2s [processed=1] W170105 16:18:41.836861 220 storage/replica.go:581 [n10,s19,r13736/1:/Table/51/1/49{46351…-51263…},@c4205c2d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.903800941s (>500ms): goroutine 220 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7179b26d) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4205c2df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4205c2d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x35a8) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219768a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:41.837116 220 storage/store.go:3358 [n10,s19,r13736/1:/Table/51/1/49{46351…-51263…},@c4205c2d80] handle raft ready: 1.9s [processed=1] W170105 16:18:42.155902 331 storage/replica.go:593 [n10,s20,r15345/1:/Table/51/1/911{29861…-34124…},@c422b7a900] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 538.48002ms (>500ms): goroutine 331 [running]: runtime/debug.Stack(0x1f4b0d8, 0x1f4b118, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x20188d94) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422b7a9e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc422b7a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2505 +0x22e github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422b7a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x3bf1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a01c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:42.169094 331 storage/replica.go:581 [n10,s20,r15345/1:/Table/51/1/911{29861…-34124…},@c422b7a900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 547.091197ms (>500ms): goroutine 331 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x209bf2fd) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422b7a978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422b7a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x3bf1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a01c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:42.171347 331 storage/store.go:3358 [n10,s20,r15345/1:/Table/51/1/911{29861…-34124…},@c422b7a900] handle raft ready: 0.6s [processed=1] I170105 16:18:43.134365 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 411 goroutines, 856 MiB/164 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 19060.34cgo/sec, 4.80/0.61 %(u/s)time, 0.00 %gc (2x) W170105 16:18:48.285191 322 storage/replica.go:581 [n10,s20,r3520/1:/Table/51/1/2038{2485…-7506…},@c421fca900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 813.244641ms (>500ms): goroutine 322 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x307920e1) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc421fca978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421fca900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0xdc0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a00a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:48.320179 322 storage/store.go:3358 [n10,s20,r3520/1:/Table/51/1/2038{2485…-7506…},@c421fca900] handle raft ready: 0.8s [processed=2] W170105 16:18:48.910983 1418 storage/store.go:867 [n10] storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).HandleRaftUncoalescedRequest for 114.513801ms (>100ms): goroutine 1418 [running]: runtime/debug.Stack(0x1f5ec50, 0x1f5ec90, 0x52) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6d35789) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420435460) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).HandleRaftUncoalescedRequest(0xc420435180, 0x7fe50b4c4880, 0xc423181c20, 0xc44b8a1770, 0x7fe50b4b3760, 0xc423345b20, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2837 +0x205 github.com/cockroachdb/cockroach/pkg/storage.(*Store).uncoalesceBeats(0xc420435180, 0x7fe50b4c4880, 0xc423181c20, 0xc43229bcc0, 0x6, 0x8, 0xa00000005, 0xa00000000, 0x13, 0x7fe500000008, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2784 +0x331 github.com/cockroachdb/cockroach/pkg/storage.(*Store).HandleRaftRequest(0xc420435180, 0x7fe50b4c4880, 0xc423181c20, 0xc431c43650, 0x7fe50b4b3760, 0xc423345b20, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2799 +0xee github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).handleRaftRequest(0xc420469b60, 0x7fe50b4c4880, 0xc423181c20, 0xc431c43650, 0x7fe50b4b3760, 0xc423345b20, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:262 +0xfb github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x26380e0, 0xc423345b20, 0xc420469b60, 0xc4209cd788, 0xc4209cd790) /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:316 +0x173 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1() /go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc423181c50) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.182806 223 storage/replica.go:581 [n10,s19,r17690/1:/Table/51/1/1320{2394…-6086…},@c42131ad80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 571.913889ms (>500ms): goroutine 223 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x2216b6a1) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42131adf8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42131ad80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x451a) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc421976900) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.185287 223 storage/store.go:3358 [n10,s19,r17690/1:/Table/51/1/1320{2394…-6086…},@c42131ad80] handle raft ready: 0.6s [processed=1] W170105 16:18:49.549713 331 storage/replica.go:581 [n10,s20,r2691/1:/Table/51/1/805{09744…-13064…},@c42390cd80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.041701054s (>500ms): goroutine 331 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x79b1e2be) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42390cdf8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42390cd80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0xa83) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a01c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.549893 331 storage/store.go:3358 [n10,s20,r2691/1:/Table/51/1/805{09744…-13064…},@c42390cd80] handle raft ready: 2.1s [processed=1] W170105 16:18:49.550152 326 storage/replica.go:581 [n10,s20,r26263/2:/Table/51/1/1905{4066…-6605…},@c422b32d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.063579255s (>500ms): goroutine 326 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7affb877) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422b32df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422b32d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x6697) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.550291 326 storage/store.go:3358 [n10,s20,r26263/2:/Table/51/1/1905{4066…-6605…},@c422b32d80] handle raft ready: 2.1s [processed=1] W170105 16:18:49.550364 327 storage/replica.go:581 [n10,s20,r925/4:/Table/51/1/608{48895…-81603…},@c42204f680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.050439212s (>500ms): goroutine 327 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7a37382c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42204f6f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42204f680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x39d) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0140) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.550467 239 storage/replica.go:581 [n10,s20,r19428/2:/Table/51/1/4862{1055…-4498…},@c422f8c480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.053385386s (>500ms): goroutine 239 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7a642caa) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422f8c4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422f8c480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x4be4) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc422795fe0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.550597 239 storage/store.go:3358 [n10,s20,r19428/2:/Table/51/1/4862{1055…-4498…},@c422f8c480] handle raft ready: 2.1s [processed=1] W170105 16:18:49.550652 327 storage/store.go:3358 [n10,s20,r925/4:/Table/51/1/608{48895…-81603…},@c42204f680] handle raft ready: 2.1s [processed=1] W170105 16:18:49.553615 240 storage/replica.go:581 [n10,s20,r15281/3:/Table/51/1/1517{1112…-5503…},@c4229cf200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.060001669s (>500ms): goroutine 240 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7ac92185) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4229cf278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4229cf200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x3bb1) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0060) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.576991 226 storage/replica.go:581 [n10,s19,r26022/2:/System/tsd/cr.store.capacity…,@c423373680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 583.177231ms (>500ms): goroutine 226 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x22c2940f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4233736f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc423373680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x65a6) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc421976a20) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.578171 226 storage/store.go:3358 [n10,s19,r26022/2:/System/tsd/cr.store.capacity…,@c423373680] handle raft ready: 0.6s [processed=1] W170105 16:18:49.580321 330 storage/replica.go:581 [n10,s20,r18032/3:/System/tsd/cr.store.raft.rcv…,@c4210b0000] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 665.06786ms (>500ms): goroutine 330 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x27a42154) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4210b0078) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4210b0000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x4670) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a01a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.582076 330 storage/store.go:3358 [n10,s20,r18032/3:/System/tsd/cr.store.raft.rcv…,@c4210b0000] handle raft ready: 0.7s [processed=2] W170105 16:18:49.585456 329 storage/replica.go:581 [n10,s20,r7025/1:/Table/51/1/2035{2425…-7436…},@c42249c480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.075394496s (>500ms): goroutine 329 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7bb401c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42249c4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42249c480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x1b71) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0180) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.587139 329 storage/store.go:3358 [n10,s20,r7025/1:/Table/51/1/2035{2425…-7436…},@c42249c480] handle raft ready: 2.1s [processed=1] W170105 16:18:49.590327 328 storage/replica.go:581 [n10,s20,r8452/3:/Table/51/1/28{15904…-57093…},@c421170480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.044615462s (>500ms): goroutine 328 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x79de5b26) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4211704f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421170480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x2104) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0160) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.591836 328 storage/store.go:3358 [n10,s20,r8452/3:/Table/51/1/28{15904…-57093…},@c421170480] handle raft ready: 2.1s [processed=1] W170105 16:18:49.596367 236 storage/replica.go:581 [n10,s20,r12348/1:/Table/51/1/9031{3864…-9374…},@c423570000] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.127363089s (>500ms): goroutine 236 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7eccfc11) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc423570078) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc423570000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x303c) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc422795e00) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.597290 236 storage/store.go:3358 [n10,s20,r12348/1:/Table/51/1/9031{3864…-9374…},@c423570000] handle raft ready: 2.1s [processed=2] W170105 16:18:49.599845 241 storage/replica.go:581 [n10,s20,r4556/2:/Table/51/1/192{69966…-73661…},@c422ba6000] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.072659393s (>500ms): goroutine 241 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7b8a45c1) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422ba6078) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422ba6000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x11cc) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0080) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.600580 241 storage/store.go:3358 [n10,s20,r4556/2:/Table/51/1/192{69966…-73661…},@c422ba6000] handle raft ready: 2.1s [processed=1] W170105 16:18:49.601450 213 storage/replica.go:581 [n10,s19,r2414/2:/System/tsd/cr.node.txn.durat…,@c420a84480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 618.701795ms (>500ms): goroutine 213 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x24e0a3e3) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420a844f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420a84480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x96e) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219767c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.602297 213 storage/store.go:3358 [n10,s19,r2414/2:/System/tsd/cr.node.txn.durat…,@c420a84480] handle raft ready: 0.6s [processed=1] W170105 16:18:49.602947 216 storage/replica.go:581 [n10,s19,r20355/2:/System/tsd/cr.node.txn.resta…,@c420a84d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.648927051s (>500ms): goroutine 216 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6248a14b) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420a84df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420a84d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x4f83) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc421976820) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.604008 216 storage/store.go:3358 [n10,s19,r20355/2:/System/tsd/cr.node.txn.resta…,@c420a84d80] handle raft ready: 1.7s [processed=3] W170105 16:18:49.605068 240 storage/store.go:3358 [n10,s20,r15281/3:/Table/51/1/1517{1112…-5503…},@c4229cf200] handle raft ready: 2.1s [processed=1] W170105 16:18:49.608902 222 storage/replica.go:581 [n10,s19,r26227/2:/System/tsd/cr.node.txn.…,@c422b7b200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.009918262s (>500ms): goroutine 222 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3c322136) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422b7b278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422b7b200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x6673) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219768e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.610352 222 storage/store.go:3358 [n10,s19,r26227/2:/System/tsd/cr.node.txn.…,@c422b7b200] handle raft ready: 1.0s [processed=3] W170105 16:18:49.611852 238 storage/replica.go:581 [n10,s20,r3776/1:/Table/51/1/5873{4692…-9287…},@c421ad1200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 2.066272262s (>500ms): goroutine 238 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x7b28d006) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc421ad1278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc421ad1200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0xec0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc422795f00) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.612039 238 storage/store.go:3358 [n10,s20,r3776/1:/Table/51/1/5873{4692…-9287…},@c421ad1200] handle raft ready: 2.1s [processed=1] W170105 16:18:49.612315 221 storage/replica.go:581 [n10,s19,r1917/2:/System/tsd/cr.{node.…-store…},@c420a8c900] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 979.224369ms (>500ms): goroutine 221 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3a5dc731) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420a8c978) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc420a8c900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x77d) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219768c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.612392 221 storage/store.go:3358 [n10,s19,r1917/2:/System/tsd/cr.{node.…-store…},@c420a8c900] handle raft ready: 1.0s [processed=3] W170105 16:18:49.616293 322 storage/replica.go:581 [n10,s20,r3639/3:/System/tsd/cr.node.sql.mon.c…,@c4213db200] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 668.788419ms (>500ms): goroutine 322 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x27dce6c3) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4213db278) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4213db200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0xe37) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a00a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.617499 322 storage/store.go:3358 [n10,s20,r3639/3:/System/tsd/cr.node.sql.mon.c…,@c4213db200] handle raft ready: 0.7s [processed=3] W170105 16:18:49.618420 212 storage/replica.go:581 [n10,s19,r25942/2:/System/tsd/cr.node.txn.…,@c423114480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 1.628090519s (>500ms): goroutine 212 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x610ab097) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4231144f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc423114480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x6556) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219767a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:49.618519 212 storage/store.go:3358 [n10,s19,r25942/2:/System/tsd/cr.node.txn.…,@c423114480] handle raft ready: 1.7s [processed=3] I170105 16:18:51.718310 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 929 goroutines, 698 MiB/258 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 18153.25cgo/sec, 6.40/0.76 %(u/s)time, 0.00 %gc (1x) W170105 16:18:54.273218 326 storage/replica.go:593 [n10,s20,r843/1:/Table/51/1/8992{6841…-9702…},@c42394ad80] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).checkIfTxnAborted for 866.202171ms (>500ms): goroutine 326 [running]: runtime/debug.Stack(0x1f4cb00, 0x1f4cb40, 0x49) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x33a1323b) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42394ae68) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).checkIfTxnAborted(0xc42394ad80, 0x7fe50b4c4880, 0xc44a96ec90, 0x2635a40, 0xc4204539a0, 0xc4291dd0b0, 0x0, 0xc481176e40, 0x38, 0x40, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3814 +0x41a github.com/cockroachdb/cockroach/pkg/storage.(*Replica).applyRaftCommandInBatch(0xc42394ad80, 0x7fe50b4c4880, 0xc44a96ec90, 0xc4291dd090, 0x8, 0x1496ec78bfbd423e, 0x0, 0x1200000009, 0x2, 0x34b, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3716 +0x5e2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).evaluateProposal(0xc42394ad80, 0x7fe50b4c4880, 0xc44a96ec90, 0xc4291dd090, 0x8, 0x1496ec78bfbd423e, 0x0, 0x1200000009, 0x2, 0x34b, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2158 +0x10c github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc42394ad80, 0x7fe50b4c4880, 0xc44a96ec90, 0xc4291dd090, 0x8, 0x12e5c3, 0x1200000009, 0x2, 0xc57e6fb360, 0x12a58d, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3429 +0xb26 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc42394ad80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2652 +0x845 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42394ad80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x34b) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:54.275673 326 storage/replica.go:581 [n10,s20,r843/1:/Table/51/1/8992{6841…-9702…},@c42394ad80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 868.690035ms (>500ms): goroutine 326 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x33c72873) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42394adf8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc42394ad80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x34b) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:54.275760 326 storage/store.go:3358 [n10,s20,r843/1:/Table/51/1/8992{6841…-9702…},@c42394ad80] handle raft ready: 0.9s [processed=1] W170105 16:18:54.275790 330 storage/replica.go:581 [n10,s20,r10018/3:/Table/51/1/7149{4260…-7692…},@c42342a480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest for 840.231883ms (>500ms): goroutine 330 [running]: runtime/debug.Stack(0x1f5ed60, 0x1f5eda0, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x3214ebcb) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42342a4f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420358a80, 0x7fe50b4c4880, 0xc42840a060, 0xc502005f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2863 +0xfa4 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc420358a80, 0x2722) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3324 +0x174 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:241 +0x267 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a01a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:54.304075 329 storage/replica.go:581 [n10,s20,r14221/3:/System/tsd/cr.node.sql.mon.c…,@c4231e6d80] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 937.538103ms (>500ms): goroutine 329 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x37e1b237) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4231e6df8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4231e6d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420358a80, 0x378d) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4227a0180) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:18:54.304566 329 storage/store.go:3358 [n10,s20,r14221/3:/System/tsd/cr.node.sql.mon.c…,@c4231e6d80] handle raft ready: 0.9s [processed=4] I170105 16:19:01.342987 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 366 goroutines, 447 MiB/408 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 61896.52cgo/sec, 4.70/0.69 %(u/s)time, 0.00 %gc (3x) I170105 16:19:11.326895 299 gossip/gossip.go:437 [n10] gossip status (ok, 10 nodes) gossip client (2/3 cur/max conns) 7: cockroach-delta-07:26257 (13h29m39s: infos 114078/228096 sent/received, bytes 27004251B/50974730B sent/received) 3: cockroach-delta-03:26257 (3h5m13s: infos 33293/25977 sent/received, bytes 7971407B/9516831B sent/received) gossip server (3/3 cur/max conns, infos 858825/1443022 sent/received, bytes 208856428B/282383864B sent/received) 5: cockroach-delta-05:26257 (4h49m0s) 6: cockroach-delta-06:26257 (3h44m43s) 9: cockroach-delta-09:26257 (3h46m47s) I170105 16:19:11.346939 301 server/status/runtime.go:228 runtime stats: 8.2 GiB RSS, 370 goroutines, 468 MiB/402 MiB/1.6 GiB GO alloc/idle/total, 5.3 GiB/6.8 GiB CGO alloc/total, 25607.70cgo/sec, 2.49/0.47 %(u/s)time, 0.00 %gc (2x) W170105 16:19:13.731249 221 storage/replica.go:593 [n10,s19,r3888/2:/Table/51/1/8643{3099…-7685…},@c422368480] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked for 502.894623ms (>500ms): goroutine 221 [running]: runtime/debug.Stack(0x1f4b0d8, 0x1f4b118, 0x53) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1df9901f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc422368568) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc422368480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2505 +0x22e github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422368480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xc5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0xf30) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219768c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:19:13.735892 239 storage/replica.go:593 [n10,s20,r15667/1:/Table/51/1/9116{3906…-8166…},@c42236e000] replicaMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).withRaftGroup for 503.160104ms (>500ms): goroutine 239 [running]: runtime/debug.Stack(0x1f472d8, 0x1f47318, 0x45) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1dfd9d28) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42236e0e8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).withRaftGroup(0xc42236e000, 0xc4657a9858, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:551 +0xa7 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420358a80, 0x7fe50b4c4880, 0xc4525e3b60, 0xc4373a6180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3082 +0xbe5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc420358a80, 0x3d33) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3324 +0x174 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:241 +0x267 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc422795fe0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:19:13.736353 239 storage/replica.go:581 [n10,s20,r15667/1:/Table/51/1/9116{3906…-8166…},@c42236e000] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest for 508.012812ms (>500ms): goroutine 239 [running]: runtime/debug.Stack(0x1f5ed60, 0x1f5eda0, 0x48) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e47a90c) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc42236e078) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420358a80, 0x7fe50b4c4880, 0xc4525e3b60, 0xc4373a6180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3099 +0xc80 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc420358a80, 0x3d33) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3324 +0x174 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4225482c0, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:241 +0x267 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc422795fe0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:19:13.736667 216 storage/replica.go:581 [n10,s19,r7435/1:/Table/51/1/1404{4255…-8499…},@c4218e1680] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 504.014888ms (>500ms): goroutine 216 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e0aa828) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4218e16f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc4218e1680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0x1d0b) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc421976820) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:19:13.737000 216 storage/store.go:3358 [n10,s19,r7435/1:/Table/51/1/1404{4255…-8499…},@c4218e1680] handle raft ready: 0.5s [processed=0] W170105 16:19:13.762820 221 storage/replica.go:581 [n10,s19,r3888/2:/Table/51/1/8643{3099…-7685…},@c422368480] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady for 534.471535ms (>500ms): goroutine 221 [running]: runtime/debug.Stack(0x1f4b020, 0x1f4b060, 0x47) /usr/local/go/src/runtime/debug/stack.go:24 +0x79 github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1fdb636f) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:67 +0xe1 github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4223684f8) /go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:97 +0xb6 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReady(0xc422368480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2481 +0xed github.com/cockroachdb/cockroach/pkg/storage.(*Store).processReady(0xc420435180, 0xf30) /go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3345 +0x137 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4200ae630, 0xc420304090) /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:230 +0x28c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2() /go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420304090, 0xc4219768c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66 W170105 16:19:13.763249 221 storage/store.go:3358 [n10,s19,r3888/2:/Table/51/1/8643{3099…-7685…},@c422368480] handle raft ready: 0.5s [processed=2]