feat: initialize Kurdistan SDK - independent fork of Polkadot SDK
This commit is contained in:
@@ -0,0 +1,239 @@
|
||||
rule_files:
|
||||
- /dev/stdin
|
||||
|
||||
evaluation_interval: 1m
|
||||
|
||||
tests:
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'substrate_sub_libp2p_peers_count{
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '3 2+0x4 1+0x9' # 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
|
||||
|
||||
- series: 'substrate_sub_txpool_validations_scheduled{
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '11+1x10 22+2x30 10043x5'
|
||||
|
||||
- series: 'substrate_sub_txpool_validations_finished{
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '0+1x42 42x5'
|
||||
|
||||
- series: 'substrate_block_height{
|
||||
status="best", job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ...
|
||||
|
||||
- series: 'substrate_block_height{
|
||||
status="finalized",
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ...
|
||||
|
||||
alert_rule_test:
|
||||
|
||||
######################################################################
|
||||
# Block production
|
||||
######################################################################
|
||||
|
||||
- eval_time: 6m
|
||||
alertname: BlockProductionSlow
|
||||
exp_alerts:
|
||||
- eval_time: 7m
|
||||
alertname: BlockProductionSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: best
|
||||
exp_annotations:
|
||||
message: "Best block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
|
||||
- eval_time: 14m
|
||||
alertname: BlockProductionSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: best
|
||||
exp_annotations:
|
||||
message: "Best block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: best
|
||||
exp_annotations:
|
||||
message: "Best block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 10 minutes."
|
||||
|
||||
######################################################################
|
||||
# Block finalization
|
||||
######################################################################
|
||||
|
||||
- eval_time: 6m
|
||||
alertname: BlockFinalizationSlow
|
||||
exp_alerts:
|
||||
- eval_time: 7m
|
||||
alertname: BlockFinalizationSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: finalized
|
||||
exp_annotations:
|
||||
message: "Finalized block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
|
||||
- eval_time: 14m
|
||||
alertname: BlockFinalizationSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: finalized
|
||||
exp_annotations:
|
||||
message: "Finalized block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: finalized
|
||||
exp_annotations:
|
||||
message: "Finalized block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 10 minutes."
|
||||
|
||||
######################################################################
|
||||
# Transaction queue
|
||||
######################################################################
|
||||
|
||||
- eval_time: 11m
|
||||
alertname: TransactionQueueSizeIncreasing
|
||||
# Number of validations scheduled and finished both grow at a rate
|
||||
# of 1 in the first 10 minutes, thereby the queue is not increasing
|
||||
# in size, thus don't expect an alert.
|
||||
exp_alerts:
|
||||
- eval_time: 22m
|
||||
alertname: TransactionQueueSizeIncreasing
|
||||
# Number of validations scheduled is growing twice as fast as the
|
||||
# number of validations finished after minute 10. Thus expect
|
||||
# warning alert after 20 minutes.
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been monotonically
|
||||
increasing for more than 10 minutes."
|
||||
- eval_time: 43m
|
||||
alertname: TransactionQueueSizeIncreasing
|
||||
# Number of validations scheduled is growing twice as fast as the
|
||||
# number of validations finished after minute 10. Thus expect
|
||||
# both warning and critical alert after 40 minutes.
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been monotonically
|
||||
increasing for more than 10 minutes."
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been monotonically
|
||||
increasing for more than 30 minutes."
|
||||
- eval_time: 49m
|
||||
alertname: TransactionQueueSizeHigh
|
||||
# After minute 43 the number of validations scheduled jumps up
|
||||
# drastically while the number of validations finished stays the
|
||||
# same. Thus expect an alert.
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been above 10_000 for more
|
||||
than 5 minutes."
|
||||
|
||||
######################################################################
|
||||
# Networking
|
||||
######################################################################
|
||||
|
||||
- eval_time: 3m # Values: 3 2 2
|
||||
alertname: NumberOfPeersLow
|
||||
exp_alerts:
|
||||
- eval_time: 4m # Values: 2 2 2
|
||||
alertname: NumberOfPeersLow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The node substrate-abcdef01234-abcdef has less
|
||||
than 3 peers for more than 3 minutes"
|
||||
|
||||
- eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1
|
||||
alertname: NumberOfPeersLow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The node substrate-abcdef01234-abcdef has less
|
||||
than 3 peers for more than 3 minutes"
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The node substrate-abcdef01234-abcdef has less
|
||||
than 3 peers for more than 15 minutes"
|
||||
@@ -0,0 +1,167 @@
|
||||
groups:
|
||||
- name: substrate.rules
|
||||
rules:
|
||||
|
||||
##############################################################################
|
||||
# Block production
|
||||
##############################################################################
|
||||
|
||||
- alert: BlockProductionSlow
|
||||
annotations:
|
||||
message: 'Best block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 3 minutes.'
|
||||
expr: increase(substrate_block_height{status="best"}[1m]) < 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: BlockProductionSlow
|
||||
annotations:
|
||||
message: 'Best block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 10 minutes.'
|
||||
expr: increase(substrate_block_height{status="best"}[1m]) < 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
|
||||
##############################################################################
|
||||
# Block finalization
|
||||
##############################################################################
|
||||
|
||||
- alert: BlockFinalizationSlow
|
||||
expr: increase(substrate_block_height{status="finalized"}[1m]) < 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Finalized block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 3 minutes.'
|
||||
- alert: BlockFinalizationSlow
|
||||
expr: increase(substrate_block_height{status="finalized"}[1m]) < 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: 'Finalized block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 10 minutes.'
|
||||
- alert: BlockFinalizationLaggingBehind
|
||||
# Under the assumption of an average block production of 6 seconds,
|
||||
# "best" and "finalized" being more than 10 blocks apart would imply
|
||||
# more than a 1 minute delay between block production and finalization.
|
||||
expr: '(substrate_block_height{status="best"} - ignoring(status)
|
||||
substrate_block_height{status="finalized"}) > 10'
|
||||
for: 8m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: "Block finalization on instance {{ $labels.instance }} is behind
|
||||
block production by {{ $value }} for more than 8 minutes."
|
||||
|
||||
##############################################################################
|
||||
# Transaction queue
|
||||
##############################################################################
|
||||
|
||||
- alert: TransactionQueueSizeIncreasing
|
||||
expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) -
|
||||
increase(substrate_sub_txpool_validations_finished[5m]) > 0'
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The transaction pool size on node {{ $labels.instance }} has
|
||||
been monotonically increasing for more than 10 minutes.'
|
||||
- alert: TransactionQueueSizeIncreasing
|
||||
expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) -
|
||||
increase(substrate_sub_txpool_validations_finished[5m]) > 0'
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The transaction pool size on node {{ $labels.instance }} has
|
||||
been monotonically increasing for more than 30 minutes.'
|
||||
- alert: TransactionQueueSizeHigh
|
||||
expr: 'substrate_sub_txpool_validations_scheduled -
|
||||
substrate_sub_txpool_validations_finished > 10000'
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The transaction pool size on node {{ $labels.instance }} has
|
||||
been above 10_000 for more than 5 minutes.'
|
||||
|
||||
##############################################################################
|
||||
# Networking
|
||||
##############################################################################
|
||||
|
||||
- alert: NumberOfPeersLow
|
||||
expr: substrate_sub_libp2p_peers_count < 3
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has less than 3 peers for more
|
||||
than 3 minutes'
|
||||
- alert: NumberOfPeersLow
|
||||
expr: substrate_sub_libp2p_peers_count < 3
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has less than 3 peers for more
|
||||
than 15 minutes'
|
||||
- alert: NoIncomingConnection
|
||||
expr: increase(substrate_sub_libp2p_incoming_connections_total[20m]) == 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has not received any new incoming
|
||||
TCP connection in the past 20 minutes. Is it connected to the Internet?'
|
||||
|
||||
##############################################################################
|
||||
# System
|
||||
##############################################################################
|
||||
|
||||
- alert: NumberOfFileDescriptorsHigh
|
||||
expr: 'node_filefd_allocated{chain!=""} > 10000'
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has more than 10_000 file
|
||||
descriptors allocated for more than 3 minutes'
|
||||
|
||||
##############################################################################
|
||||
# Others
|
||||
##############################################################################
|
||||
|
||||
- alert: AuthorityDiscoveryDiscoveryFailureHigh
|
||||
expr: 'substrate_authority_discovery_handle_value_found_event_failure /
|
||||
ignoring(name)
|
||||
substrate_authority_discovery_dht_event_received{name="value_found"} > 0.5'
|
||||
for: 2h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Authority discovery on node {{ $labels.instance }} fails to
|
||||
process more than 50 % of the values found on the DHT for more than 2
|
||||
hours.'
|
||||
|
||||
- alert: UnboundedChannelPersistentlyLarge
|
||||
expr: 'substrate_unbounded_channel_size >= 200'
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Channel {{ $labels.entity }} on node {{ $labels.instance }} contains
|
||||
more than 200 items for more than 5 minutes. Node might be frozen.'
|
||||
|
||||
- alert: UnboundedChannelVeryLarge
|
||||
expr: '(
|
||||
(substrate_unbounded_channel_len{action = "send"} -
|
||||
ignoring(action) substrate_unbounded_channel_len{action = "received"})
|
||||
or on(instance) substrate_unbounded_channel_len{action = "send"}
|
||||
) > 15000'
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Channel {{ $labels.entity }} on node {{ $labels.instance }} contains more than
|
||||
15000 items.'
|
||||
Reference in New Issue
Block a user