pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-07-19 23:55:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

.maintain/monitoring: Normalize alerting rules (#7232)

Browse Source 
* .maintain/monitoring: Normalize alerting rules

- Start alert names with their component and end with the describing
adjective.

- Describe alert duration in `message` with `for more than` across all
alerts.

* .maintain/monitoring: Fix alert tests
This commit is contained in:
Max Inden
2020-09-30 10:48:48 +02:00
committed by GitHub
parent 59afb1081c
commit 51c0d27aa1
2 changed files with 42 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/.maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
+23 -20
View File
@@ -49,10 +49,10 @@ tests:
###################################################################### ######################################################################
- eval_time: 6m - eval_time: 6m
alertname: LowNumberOfNewBlocks alertname: BlockProductionSlow
exp_alerts: exp_alerts:
- eval_time: 7m - eval_time: 7m
alertname: LowNumberOfNewBlocks alertname: BlockProductionSlow
exp_alerts: exp_alerts:
- exp_labels: - exp_labels:
severity: warning severity: warning
@@ -61,11 +61,12 @@ tests:
job: polkadot job: polkadot
status: best status: best
exp_annotations: exp_annotations:
message: "Less than one new block per minute on instance message: "Best block on instance
polkadot-abcdef01234-abcdef." polkadot-abcdef01234-abcdef increases by less than 1 per
minute for more than 3 minutes."
- eval_time: 14m - eval_time: 14m
alertname: LowNumberOfNewBlocks alertname: BlockProductionSlow
exp_alerts: exp_alerts:
- exp_labels: - exp_labels:
severity: warning severity: warning
@@ -74,8 +75,9 @@ tests:
job: polkadot job: polkadot
status: best status: best
exp_annotations: exp_annotations:
message: "Less than one new block per minute on instance message: "Best block on instance
polkadot-abcdef01234-abcdef." polkadot-abcdef01234-abcdef increases by less than 1 per
minute for more than 3 minutes."
- exp_labels: - exp_labels:
severity: critical severity: critical
pod: polkadot-abcdef01234-abcdef pod: polkadot-abcdef01234-abcdef
@@ -83,8 +85,9 @@ tests:
job: polkadot job: polkadot
status: best status: best
exp_annotations: exp_annotations:
message: "Less than one new block per minute on instance message: "Best block on instance
polkadot-abcdef01234-abcdef." polkadot-abcdef01234-abcdef increases by less than 1 per
minute for more than 10 minutes."
###################################################################### ######################################################################
# Block finalization # Block finalization
@@ -105,7 +108,7 @@ tests:
exp_annotations: exp_annotations:
message: "Finalized block on instance message: "Finalized block on instance
polkadot-abcdef01234-abcdef increases by less than 1 per polkadot-abcdef01234-abcdef increases by less than 1 per
minute." minute for more than 3 minutes."
- eval_time: 14m - eval_time: 14m
alertname: BlockFinalizationSlow alertname: BlockFinalizationSlow
@@ -119,7 +122,7 @@ tests:
exp_annotations: exp_annotations:
message: "Finalized block on instance message: "Finalized block on instance
polkadot-abcdef01234-abcdef increases by less than 1 per polkadot-abcdef01234-abcdef increases by less than 1 per
minute." minute for more than 3 minutes."
- exp_labels: - exp_labels:
severity: critical severity: critical
pod: polkadot-abcdef01234-abcdef pod: polkadot-abcdef01234-abcdef
@@ -129,7 +132,7 @@ tests:
exp_annotations: exp_annotations:
message: "Finalized block on instance message: "Finalized block on instance
polkadot-abcdef01234-abcdef increases by less than 1 per polkadot-abcdef01234-abcdef increases by less than 1 per
minute." minute for more than 10 minutes."
###################################################################### ######################################################################
# Transaction queue # Transaction queue
@@ -155,7 +158,7 @@ tests:
exp_annotations: exp_annotations:
message: "The transaction pool size on node message: "The transaction pool size on node
polkadot-abcdef01234-abcdef has been monotonically polkadot-abcdef01234-abcdef has been monotonically
increasing for the last 10 minutes." increasing for more than 10 minutes."
- eval_time: 43m - eval_time: 43m
alertname: TransactionQueueSizeIncreasing alertname: TransactionQueueSizeIncreasing
# Number of validations scheduled is growing twice as fast as the # Number of validations scheduled is growing twice as fast as the
@@ -170,7 +173,7 @@ tests:
exp_annotations: exp_annotations:
message: "The transaction pool size on node message: "The transaction pool size on node
polkadot-abcdef01234-abcdef has been monotonically polkadot-abcdef01234-abcdef has been monotonically
increasing for the last 10 minutes." increasing for more than 10 minutes."
- exp_labels: - exp_labels:
severity: critical severity: critical
pod: polkadot-abcdef01234-abcdef pod: polkadot-abcdef01234-abcdef
@@ -179,7 +182,7 @@ tests:
exp_annotations: exp_annotations:
message: "The transaction pool size on node message: "The transaction pool size on node
polkadot-abcdef01234-abcdef has been monotonically polkadot-abcdef01234-abcdef has been monotonically
increasing for the last 30 minutes." increasing for more than 30 minutes."
- eval_time: 49m - eval_time: 49m
alertname: TransactionQueueSizeHigh alertname: TransactionQueueSizeHigh
# After minute 43 the number of validations scheduled jumps up # After minute 43 the number of validations scheduled jumps up
@@ -193,18 +196,18 @@ tests:
job: polkadot job: polkadot
exp_annotations: exp_annotations:
message: "The transaction pool size on node message: "The transaction pool size on node
polkadot-abcdef01234-abcdef has been above 10_000 for the polkadot-abcdef01234-abcdef has been above 10_000 for more
last 5 minutes." than 5 minutes."
###################################################################### ######################################################################
# Networking # Networking
###################################################################### ######################################################################
- eval_time: 3m # Values: 3 2 2 - eval_time: 3m # Values: 3 2 2
alertname: LowNumberOfPeers alertname: NumberOfPeersLow
exp_alerts: exp_alerts:
- eval_time: 4m # Values: 2 2 2 - eval_time: 4m # Values: 2 2 2
alertname: LowNumberOfPeers alertname: NumberOfPeersLow
exp_alerts: exp_alerts:
- exp_labels: - exp_labels:
severity: warning severity: warning
@@ -216,7 +219,7 @@ tests:
than 3 peers for more than 3 minutes" than 3 peers for more than 3 minutes"
- eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 - eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1
alertname: LowNumberOfPeers alertname: NumberOfPeersLow
exp_alerts: exp_alerts:
- exp_labels: - exp_labels:
severity: warning severity: warning
substrate/.maintain/monitoring/alerting-rules/alerting-rules.yaml
+19 -18
View File
@@ -6,18 +6,18 @@ groups:
# Block production # Block production
############################################################################## ##############################################################################
- alert: LowNumberOfNewBlocks - alert: BlockProductionSlow
annotations: annotations:
message: 'Less than one new block per minute on instance {{ message: 'Best block on instance {{ $labels.instance }} increases by
$labels.instance }}.' less than 1 per minute for more than 3 minutes.'
expr: increase(polkadot_block_height{status="best"}[1m]) < 1 expr: increase(polkadot_block_height{status="best"}[1m]) < 1
for: 3m for: 3m
labels: labels:
severity: warning severity: warning
- alert: LowNumberOfNewBlocks - alert: BlockProductionSlow
annotations: annotations:
message: 'Less than one new block per minute on instance {{ message: 'Best block on instance {{ $labels.instance }} increases by
$labels.instance }}.' less than 1 per minute for more than 10 minutes.'
expr: increase(polkadot_block_height{status="best"}[1m]) < 1 expr: increase(polkadot_block_height{status="best"}[1m]) < 1
for: 10m for: 10m
labels: labels:
@@ -34,7 +34,7 @@ groups:
severity: warning severity: warning
annotations: annotations:
message: 'Finalized block on instance {{ $labels.instance }} increases by message: 'Finalized block on instance {{ $labels.instance }} increases by
less than 1 per minute.' less than 1 per minute for more than 3 minutes.'
- alert: BlockFinalizationSlow - alert: BlockFinalizationSlow
expr: increase(polkadot_block_height{status="finalized"}[1m]) < 1 expr: increase(polkadot_block_height{status="finalized"}[1m]) < 1
for: 10m for: 10m
@@ -42,7 +42,7 @@ groups:
severity: critical severity: critical
annotations: annotations:
message: 'Finalized block on instance {{ $labels.instance }} increases by message: 'Finalized block on instance {{ $labels.instance }} increases by
less than 1 per minute.' less than 1 per minute for more than 10 minutes.'
- alert: BlockFinalizationLaggingBehind - alert: BlockFinalizationLaggingBehind
# Under the assumption of an average block production of 6 seconds, # Under the assumption of an average block production of 6 seconds,
# "best" and "finalized" being more than 10 blocks apart would imply # "best" and "finalized" being more than 10 blocks apart would imply
@@ -54,7 +54,7 @@ groups:
severity: critical severity: critical
annotations: annotations:
message: "Block finalization on instance {{ $labels.instance }} is behind message: "Block finalization on instance {{ $labels.instance }} is behind
block production by {{ $value }} for more than 8m" block production by {{ $value }} for more than 8 minutes."
############################################################################## ##############################################################################
# Transaction queue # Transaction queue
@@ -68,7 +68,7 @@ groups:
severity: warning severity: warning
annotations: annotations:
message: 'The transaction pool size on node {{ $labels.instance }} has message: 'The transaction pool size on node {{ $labels.instance }} has
been monotonically increasing for the last 10 minutes.' been monotonically increasing for more than 10 minutes.'
- alert: TransactionQueueSizeIncreasing - alert: TransactionQueueSizeIncreasing
expr: 'increase(polkadot_sub_txpool_validations_scheduled[5m]) - expr: 'increase(polkadot_sub_txpool_validations_scheduled[5m]) -
increase(polkadot_sub_txpool_validations_finished[5m]) > 0' increase(polkadot_sub_txpool_validations_finished[5m]) > 0'
@@ -77,7 +77,7 @@ groups:
severity: critical severity: critical
annotations: annotations:
message: 'The transaction pool size on node {{ $labels.instance }} has message: 'The transaction pool size on node {{ $labels.instance }} has
been monotonically increasing for the last 30 minutes.' been monotonically increasing for more than 30 minutes.'
- alert: TransactionQueueSizeHigh - alert: TransactionQueueSizeHigh
expr: 'polkadot_sub_txpool_validations_scheduled - expr: 'polkadot_sub_txpool_validations_scheduled -
polkadot_sub_txpool_validations_finished > 10000' polkadot_sub_txpool_validations_finished > 10000'
@@ -86,13 +86,13 @@ groups:
severity: critical severity: critical
annotations: annotations:
message: 'The transaction pool size on node {{ $labels.instance }} has message: 'The transaction pool size on node {{ $labels.instance }} has
been above 10_000 for the last 5 minutes.' been above 10_000 for more than 5 minutes.'
############################################################################## ##############################################################################
# Networking # Networking
############################################################################## ##############################################################################
- alert: LowNumberOfPeers - alert: NumberOfPeersLow
expr: polkadot_sub_libp2p_peers_count < 3 expr: polkadot_sub_libp2p_peers_count < 3
for: 3m for: 3m
labels: labels:
@@ -100,7 +100,7 @@ groups:
annotations: annotations:
message: 'The node {{ $labels.instance }} has less than 3 peers for more message: 'The node {{ $labels.instance }} has less than 3 peers for more
than 3 minutes' than 3 minutes'
- alert: LowNumberOfPeers - alert: NumberOfPeersLow
expr: polkadot_sub_libp2p_peers_count < 3 expr: polkadot_sub_libp2p_peers_count < 3
for: 15m for: 15m
labels: labels:
@@ -113,7 +113,7 @@ groups:
# System # System
############################################################################## ##############################################################################
- alert: HighNumberOfFileDescriptors - alert: NumberOfFileDescriptorsHigh
expr: 'node_filefd_allocated{domain=~"kusama|polkadot"} > 10000' expr: 'node_filefd_allocated{domain=~"kusama|polkadot"} > 10000'
for: 3m for: 3m
labels: labels:
@@ -126,7 +126,7 @@ groups:
# Others # Others
############################################################################## ##############################################################################
- alert: AuthorityDiscoveryHighDiscoveryFailure - alert: AuthorityDiscoveryDiscoveryFailureHigh
expr: 'polkadot_authority_discovery_handle_value_found_event_failure / expr: 'polkadot_authority_discovery_handle_value_found_event_failure /
ignoring(name) ignoring(name)
polkadot_authority_discovery_dht_event_received{name="value_found"} > 0.5' polkadot_authority_discovery_dht_event_received{name="value_found"} > 0.5'
@@ -134,5 +134,6 @@ groups:
labels: labels:
severity: warning severity: warning
annotations: annotations:
message: "Authority discovery on node {{ $labels.instance }} fails to message: 'Authority discovery on node {{ $labels.instance }} fails to
process more than 50 % of the values found on the DHT." process more than 50 % of the values found on the DHT for more than 2
hours.'