Add retry mechanism for pov-recovery, fix full-node pov-recovery (#2164)

* Increase delay for pov-recovery

* Update client/service/src/lib.rs

Co-authored-by: Bastian Köcher <git@kchr.de>

* Comment

* FMT

* Clear waiting_recovery when block is recovered or recovery failed

* Introduce recovery queue that preserved insertion order

* Better error logs

* Decrease slot duration

* Style improvements

* Add option to use unordered queue

* Maintain cache of finalized blocks

* Wait for one relay chain slot before recovery

* Make retries testable

* fmt

* Improve docs

* Improve docs

* Simplify RecoveryQueue

* Remove unwanted changes

* Adjust to comments

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <git@kchr.de>

* Move recovery delay into the queue

* Check for finalized number

* Clean up

* Use timer

Co-authored-by: Bastian Köcher <git@kchr.de>

* Simplify implementation

* Revert "Use timer"

This reverts commit 3809eed840d3a09d54212f99486782ff80cdc1c9.

* Properly clear `to_recover` flag

---------

Co-authored-by: Bastian Köcher <git@kchr.de>

test/service/src/main.rs

@@ -123,6 +123,9 @@ fn main() -> Result<(), sc_cli::Error> {
 				"Is collating: {}",
 				if config.role.is_authority() { "yes" } else { "no" }
 			);
+			if cli.fail_pov_recovery {
+				tracing::info!("PoV recovery failure enabled");
+			}
 
 			let collator_key = config.role.is_authority().then(|| CollatorPair::generate().0);
 
@@ -141,6 +144,7 @@ fn main() -> Result<(), sc_cli::Error> {
 					polkadot_config,
 					parachain_id,
 					cli.disable_block_announcements.then(wrap_announce_block),
+					cli.fail_pov_recovery,
 					|_| Ok(jsonrpsee::RpcModule::new(())),
 					consensus,
 					collator_options,