Alert on frequent network errors (#7410)

* Introduce is_frequent util

* Add dirty warn_if_frequent! implementation

* Add freq

* Fix order in condition

* Update

* Update docs

* Fix

* Remove old impl

* Fix errors

* Add wif to av-distr

* Add wif to col prot

* Rename

* Add wif to state-distr

* Address review comments

* Change Freq implementation

* Remove the zero division check

* Make rate explicit

* Fix typo

* Update rate constant

* Introduce explicit rates

* Update docs

* Split errors freq

* Downgrade coarsetime

polkadot/node/network/statement-distribution/src/error.rs

@@ -82,13 +82,18 @@ pub enum Error {
 ///
 /// We basically always want to try and continue on error. This utility function is meant to
 /// consume top-level errors by simply logging them.
-pub fn log_error(result: Result<()>, ctx: &'static str) -> std::result::Result<(), FatalError> {
+pub fn log_error(
+	result: Result<()>,
+	ctx: &'static str,
+	warn_freq: &mut gum::Freq,
+) -> std::result::Result<(), FatalError> {
 	match result.into_nested()? {
 		Err(jfyi) => {
 			match jfyi {
 				JfyiError::RequestedUnannouncedCandidate(_, _) =>
 					gum::warn!(target: LOG_TARGET, error = %jfyi, ctx),
-				_ => gum::debug!(target: LOG_TARGET, error = %jfyi, ctx),
+				_ =>
+					gum::warn_if_frequent!(freq: warn_freq, max_rate: gum::Times::PerHour(100), target: LOG_TARGET, error = %jfyi, ctx),
 			}
 			Ok(())
 		},

polkadot/node/network/statement-distribution/src/lib.rs

@@ -1810,6 +1810,8 @@ impl<R: rand::Rng> StatementDistributionSubsystem<R> {
 		)
 		.map_err(FatalError::SpawnTask)?;
 
+		let mut warn_freq = gum::Freq::new();
+
 		loop {
 			select! {
 				_ = reputation_delay => {
@@ -1851,7 +1853,7 @@ impl<R: rand::Rng> StatementDistributionSubsystem<R> {
 									result.ok_or(FatalError::RequesterReceiverFinished)?,
 								)
 								.await;
-							log_error(result.map_err(From::from), "handle_requester_message")?;
+							log_error(result.map_err(From::from), "handle_requester_message", &mut warn_freq)?;
 						},
 						MuxedMessage::Responder(result) => {
 							let result = self
@@ -1861,7 +1863,7 @@ impl<R: rand::Rng> StatementDistributionSubsystem<R> {
 									result.ok_or(FatalError::ResponderReceiverFinished)?,
 								)
 								.await;
-							log_error(result.map_err(From::from), "handle_responder_message")?;
+							log_error(result.map_err(From::from), "handle_responder_message", &mut warn_freq)?;
 						},
 					};
 				}