-
Notifications
You must be signed in to change notification settings - Fork 666
/
alerting-rules.yaml
171 lines (157 loc) · 6.39 KB
/
alerting-rules.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
groups:
- name: substrate.rules
rules:
##############################################################################
# Block production
##############################################################################
- alert: BlockProductionSlow
annotations:
message: 'Best block on instance {{ $labels.instance }} increases by
less than 1 per minute for more than 3 minutes.'
expr: increase(substrate_block_height{status="best"}[1m]) < 1
for: 3m
labels:
severity: warning
- alert: BlockProductionSlow
annotations:
message: 'Best block on instance {{ $labels.instance }} increases by
less than 1 per minute for more than 10 minutes.'
expr: increase(substrate_block_height{status="best"}[1m]) < 1
for: 10m
labels:
severity: critical
##############################################################################
# Block finalization
##############################################################################
- alert: BlockFinalizationSlow
expr: increase(substrate_block_height{status="finalized"}[1m]) < 1
for: 3m
labels:
severity: warning
annotations:
message: 'Finalized block on instance {{ $labels.instance }} increases by
less than 1 per minute for more than 3 minutes.'
- alert: BlockFinalizationSlow
expr: increase(substrate_block_height{status="finalized"}[1m]) < 1
for: 10m
labels:
severity: critical
annotations:
message: 'Finalized block on instance {{ $labels.instance }} increases by
less than 1 per minute for more than 10 minutes.'
- alert: BlockFinalizationLaggingBehind
# Under the assumption of an average block production of 6 seconds,
# "best" and "finalized" being more than 10 blocks apart would imply
# more than a 1 minute delay between block production and finalization.
expr: '(substrate_block_height{status="best"} - ignoring(status)
substrate_block_height{status="finalized"}) > 10'
for: 8m
labels:
severity: critical
annotations:
message: "Block finalization on instance {{ $labels.instance }} is behind
block production by {{ $value }} for more than 8 minutes."
##############################################################################
# Transaction queue
##############################################################################
- alert: TransactionQueueSizeIncreasing
expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) -
increase(substrate_sub_txpool_validations_finished[5m]) > 0'
for: 10m
labels:
severity: warning
annotations:
message: 'The transaction pool size on node {{ $labels.instance }} has
been monotonically increasing for more than 10 minutes.'
- alert: TransactionQueueSizeIncreasing
expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) -
increase(substrate_sub_txpool_validations_finished[5m]) > 0'
for: 30m
labels:
severity: warning
annotations:
message: 'The transaction pool size on node {{ $labels.instance }} has
been monotonically increasing for more than 30 minutes.'
- alert: TransactionQueueSizeHigh
expr: 'substrate_sub_txpool_validations_scheduled -
substrate_sub_txpool_validations_finished > 10000'
for: 5m
labels:
severity: warning
annotations:
message: 'The transaction pool size on node {{ $labels.instance }} has
been above 10_000 for more than 5 minutes.'
##############################################################################
# Networking
##############################################################################
- alert: NumberOfPeersLow
expr: substrate_sub_libp2p_peers_count < 3
for: 3m
labels:
severity: warning
annotations:
message: 'The node {{ $labels.instance }} has less than 3 peers for more
than 3 minutes'
- alert: NumberOfPeersLow
expr: substrate_sub_libp2p_peers_count < 3
for: 15m
labels:
severity: critical
annotations:
message: 'The node {{ $labels.instance }} has less than 3 peers for more
than 15 minutes'
- alert: NoIncomingConnection
expr: increase(substrate_sub_libp2p_incoming_connections_total[20m]) == 0
labels:
severity: warning
annotations:
message: 'The node {{ $labels.instance }} has not received any new incoming
TCP connection in the past 20 minutes. Is it connected to the Internet?'
##############################################################################
# System
##############################################################################
- alert: NumberOfFileDescriptorsHigh
expr: 'node_filefd_allocated{chain!=""} > 10000'
for: 3m
labels:
severity: warning
annotations:
message: 'The node {{ $labels.instance }} has more than 10_000 file
descriptors allocated for more than 3 minutes'
##############################################################################
# Others
##############################################################################
- alert: AuthorityDiscoveryDiscoveryFailureHigh
expr: 'substrate_authority_discovery_handle_value_found_event_failure /
ignoring(name)
substrate_authority_discovery_dht_event_received{name="value_found"} > 0.5'
for: 2h
labels:
severity: warning
annotations:
message: 'Authority discovery on node {{ $labels.instance }} fails to
process more than 50 % of the values found on the DHT for more than 2
hours.'
- alert: UnboundedChannelPersistentlyLarge
expr: '(
(substrate_unbounded_channel_len{action = "send"} -
ignoring(action) substrate_unbounded_channel_len{action = "received"})
or on(instance) substrate_unbounded_channel_len{action = "send"}
) >= 200'
for: 5m
labels:
severity: warning
annotations:
message: 'Channel {{ $labels.entity }} on node {{ $labels.instance }} contains
more than 200 items for more than 5 minutes. Node might be frozen.'
- alert: UnboundedChannelVeryLarge
expr: '(
(substrate_unbounded_channel_len{action = "send"} -
ignoring(action) substrate_unbounded_channel_len{action = "received"})
or on(instance) substrate_unbounded_channel_len{action = "send"}
) > 15000'
labels:
severity: warning
annotations:
message: 'Channel {{ $labels.entity }} on node {{ $labels.instance }} contains more than
15000 items.'