mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-05-30 05:51:02 +00:00
Max Inden
585ea531a3
The transaction queue size alert has been firing with a constant 10 transactions in the queue. While maybe problematic those 10 transactions don't need to be the same across scrape intervals. Instead of alerting with a size above 10, alert based on two things: 1. Monotonically increasing queue size 2. Upper limit queue size reached
263 lines
11 KiB
YAML
263 lines
11 KiB
YAML
rule_files:
|
|
- /dev/stdin
|
|
|
|
evaluation_interval: 1m
|
|
|
|
tests:
|
|
- interval: 1m
|
|
input_series:
|
|
- series: 'polkadot_sub_libp2p_peers_count{
|
|
job="polkadot",
|
|
pod="polkadot-abcdef01234-abcdef",
|
|
instance="polkadot-abcdef01234-abcdef",
|
|
}'
|
|
values: '3 2+0x4 1+0x9' # 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
|
|
|
|
- series: 'polkadot_sub_txpool_validations_scheduled{
|
|
job="polkadot",
|
|
pod="polkadot-abcdef01234-abcdef",
|
|
instance="polkadot-abcdef01234-abcdef",
|
|
}'
|
|
values: '11+1x10 22+2x30 10043x5'
|
|
|
|
- series: 'polkadot_sub_txpool_validations_finished{
|
|
job="polkadot",
|
|
pod="polkadot-abcdef01234-abcdef",
|
|
instance="polkadot-abcdef01234-abcdef",
|
|
}'
|
|
values: '0+1x42 42x5'
|
|
|
|
- series: 'polkadot_block_height{
|
|
status="best", job="polkadot",
|
|
pod="polkadot-abcdef01234-abcdef",
|
|
instance="polkadot-abcdef01234-abcdef",
|
|
}'
|
|
values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ...
|
|
|
|
- series: 'polkadot_block_height{
|
|
status="finalized",
|
|
job="polkadot",
|
|
pod="polkadot-abcdef01234-abcdef",
|
|
instance="polkadot-abcdef01234-abcdef",
|
|
}'
|
|
values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ...
|
|
|
|
- series: 'polkadot_cpu_usage_percentage{
|
|
job="polkadot",
|
|
pod="polkadot-abcdef01234-abcdef",
|
|
instance="polkadot-abcdef01234-abcdef",
|
|
}'
|
|
values: '0+20x5 100+0x5' # 0 20 40 60 80 100 100 100 100 100 100
|
|
|
|
alert_rule_test:
|
|
|
|
######################################################################
|
|
# Resource usage
|
|
######################################################################
|
|
|
|
- eval_time: 9m
|
|
alertname: HighCPUUsage
|
|
exp_alerts:
|
|
- eval_time: 10m
|
|
alertname: HighCPUUsage
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The node polkadot-abcdef01234-abcdef has a CPU
|
|
usage higher than 100% for more than 5 minutes"
|
|
|
|
######################################################################
|
|
# Block production
|
|
######################################################################
|
|
|
|
- eval_time: 6m
|
|
alertname: LowNumberOfNewBlocks
|
|
exp_alerts:
|
|
- eval_time: 7m
|
|
alertname: LowNumberOfNewBlocks
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
status: best
|
|
exp_annotations:
|
|
message: "Less than one new block per minute on instance
|
|
polkadot-abcdef01234-abcdef."
|
|
|
|
- eval_time: 14m
|
|
alertname: LowNumberOfNewBlocks
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
status: best
|
|
exp_annotations:
|
|
message: "Less than one new block per minute on instance
|
|
polkadot-abcdef01234-abcdef."
|
|
- exp_labels:
|
|
severity: critical
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
status: best
|
|
exp_annotations:
|
|
message: "Less than one new block per minute on instance
|
|
polkadot-abcdef01234-abcdef."
|
|
|
|
######################################################################
|
|
# Block finalization
|
|
######################################################################
|
|
|
|
- eval_time: 6m
|
|
alertname: BlockFinalizationSlow
|
|
exp_alerts:
|
|
- eval_time: 7m
|
|
alertname: BlockFinalizationSlow
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
status: finalized
|
|
exp_annotations:
|
|
message: "Finalized block on instance
|
|
polkadot-abcdef01234-abcdef increases by less than 1 per
|
|
minute."
|
|
|
|
- eval_time: 14m
|
|
alertname: BlockFinalizationSlow
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
status: finalized
|
|
exp_annotations:
|
|
message: "Finalized block on instance
|
|
polkadot-abcdef01234-abcdef increases by less than 1 per
|
|
minute."
|
|
- exp_labels:
|
|
severity: critical
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
status: finalized
|
|
exp_annotations:
|
|
message: "Finalized block on instance
|
|
polkadot-abcdef01234-abcdef increases by less than 1 per
|
|
minute."
|
|
|
|
######################################################################
|
|
# Transaction queue
|
|
######################################################################
|
|
|
|
- eval_time: 11m
|
|
alertname: TransactionQueueSizeIncreasing
|
|
# Number of validations scheduled and finished both grow at a rate
|
|
# of 1 in the first 10 minutes, thereby the queue is not increasing
|
|
# in size, thus don't expect an alert.
|
|
exp_alerts:
|
|
- eval_time: 22m
|
|
alertname: TransactionQueueSizeIncreasing
|
|
# Number of validations scheduled is growing twice as fast as the
|
|
# number of validations finished after minute 10. Thus expect
|
|
# warning alert after 20 minutes.
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The transaction pool size on node
|
|
polkadot-abcdef01234-abcdef has been monotonically
|
|
increasing for the last 10 minutes."
|
|
- eval_time: 43m
|
|
alertname: TransactionQueueSizeIncreasing
|
|
# Number of validations scheduled is growing twice as fast as the
|
|
# number of validations finished after minute 10. Thus expect
|
|
# both warning and critical alert after 40 minutes.
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The transaction pool size on node
|
|
polkadot-abcdef01234-abcdef has been monotonically
|
|
increasing for the last 10 minutes."
|
|
- exp_labels:
|
|
severity: critical
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The transaction pool size on node
|
|
polkadot-abcdef01234-abcdef has been monotonically
|
|
increasing for the last 30 minutes."
|
|
- eval_time: 49m
|
|
alertname: TransactionQueueSizeHigh
|
|
# After minute 43 the number of validations scheduled jumps up
|
|
# drastically while the number of validations finished stays the
|
|
# same. Thus expect an alert.
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: critical
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The transaction pool size on node
|
|
polkadot-abcdef01234-abcdef has been above 10_000 for the
|
|
last 5 minutes."
|
|
|
|
######################################################################
|
|
# Networking
|
|
######################################################################
|
|
|
|
- eval_time: 3m # Values: 3 2 2
|
|
alertname: LowNumberOfPeers
|
|
exp_alerts:
|
|
- eval_time: 4m # Values: 2 2 2
|
|
alertname: LowNumberOfPeers
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The node polkadot-abcdef01234-abcdef has less
|
|
than 3 peers for more than 3 minutes"
|
|
|
|
- eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1
|
|
alertname: LowNumberOfPeers
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The node polkadot-abcdef01234-abcdef has less
|
|
than 3 peers for more than 3 minutes"
|
|
- exp_labels:
|
|
severity: critical
|
|
pod: polkadot-abcdef01234-abcdef
|
|
instance: polkadot-abcdef01234-abcdef
|
|
job: polkadot
|
|
exp_annotations:
|
|
message: "The node polkadot-abcdef01234-abcdef has less
|
|
than 3 peers for more than 15 minutes"
|