chainHead based backend implementation (#1161)

* add follow_stream impl

* follow_stream_unpin first draft

* add tests for follow_stream_unpin

* more tests and fixes for follow_stream_unpin

* first pass follow_stream_driver

* follow_stream_driver: add tests, fix things, buffer events from last finalized

* First pass finishing Backend impl

* Fix test compile issues

* clippy fixes

* clippy fix and consistify light_client

* revert lightclient tweak

* revert other lightclient thing

* cargo fmt

* start testing unstable backend behind feature flag

* more test fixes and move test-runtime metadata path just incase

* fix compile error

* ensure transaction progress stream actually used and fix another test

* cargo fmt

* CI tweak

* improve some comments and address some feedback bits

* update CI to use our own nightly binary

* wait for finalized block perhaps

subxt/src/tx/tx_client.rs

@@ -169,7 +169,7 @@ where
 {
     /// Get the account nonce for a given account ID.
     pub async fn account_nonce(&self, account_id: &T::AccountId) -> Result<u64, Error> {
-        let block_ref = self.client.backend().latest_best_block_ref().await?;
+        let block_ref = self.client.backend().latest_finalized_block_ref().await?;
         let account_nonce_bytes = self
             .client
             .backend()
@@ -486,6 +486,7 @@ where
                 TransactionStatus::Validated
                 | TransactionStatus::Broadcasted { .. }
                 | TransactionStatus::InBestBlock { .. }
+                | TransactionStatus::NoLongerInBestBlock
                 | TransactionStatus::InFinalizedBlock { .. } => Ok(ext_hash),
                 TransactionStatus::Error { message } => {
                     Err(Error::Other(format!("Transaction error: {message}")))
@@ -509,7 +510,7 @@ where
     ///
     /// Returns `Ok` with a [`ValidationResult`], which is the result of attempting to dry run the extrinsic.
     pub async fn validate(&self) -> Result<ValidationResult, Error> {
-        let latest_block_ref = self.client.backend().latest_best_block_ref().await?;
+        let latest_block_ref = self.client.backend().latest_finalized_block_ref().await?;
         self.validate_at(latest_block_ref).await
     }
 
@@ -547,7 +548,7 @@ where
     pub async fn partial_fee_estimate(&self) -> Result<u128, Error> {
         let mut params = self.encoded().to_vec();
         (self.encoded().len() as u32).encode_to(&mut params);
-        let latest_block_ref = self.client.backend().latest_best_block_ref().await?;
+        let latest_block_ref = self.client.backend().latest_finalized_block_ref().await?;
 
         // destructuring RuntimeDispatchInfo, see type information <https://paritytech.github.io/substrate/master/pallet_transaction_payment_rpc_runtime_api/struct.RuntimeDispatchInfo.html>
         // data layout: {weight_ref_time: Compact<u64>, weight_proof_size: Compact<u64>, class: u8, partial_fee: u128}

subxt/src/tx/tx_progress.rs

@@ -8,7 +8,7 @@ use std::task::Poll;
 
 use crate::utils::strip_compact_prefix;
 use crate::{
-    backend::{StreamOfResults, TransactionStatus as BackendTxStatus},
+    backend::{BlockRef, StreamOfResults, TransactionStatus as BackendTxStatus},
     client::OnlineClientT,
     error::{DispatchError, Error, RpcError, TransactionError},
     events::EventsClient,
@@ -167,6 +167,7 @@ impl<T: Config, C: Clone> Stream for TxProgress<T, C> {
             match status {
                 BackendTxStatus::Validated => TxStatus::Validated,
                 BackendTxStatus::Broadcasted { num_peers } => TxStatus::Broadcasted { num_peers },
+                BackendTxStatus::NoLongerInBestBlock => TxStatus::NoLongerInBestBlock,
                 BackendTxStatus::InBestBlock { hash } => {
                     TxStatus::InBestBlock(TxInBlock::new(hash, self.ext_hash, self.client.clone()))
                 }
@@ -207,6 +208,8 @@ pub enum TxStatus<T: Config, C> {
         /// Number of peers it's been broadcast to.
         num_peers: u32,
     },
+    /// Transaction is no longer in a best block.
+    NoLongerInBestBlock,
     /// Transaction has been included in block with given hash.
     InBestBlock(TxInBlock<T, C>),
     /// Transaction has been finalized by a finality-gadget, e.g GRANDPA
@@ -252,15 +255,15 @@ impl<T: Config, C> TxStatus<T, C> {
 #[derive(Derivative)]
 #[derivative(Debug(bound = "C: std::fmt::Debug"))]
 pub struct TxInBlock<T: Config, C> {
-    block_hash: T::Hash,
+    block_ref: BlockRef<T::Hash>,
     ext_hash: T::Hash,
     client: C,
 }
 
 impl<T: Config, C> TxInBlock<T, C> {
-    pub(crate) fn new(block_hash: T::Hash, ext_hash: T::Hash, client: C) -> Self {
+    pub(crate) fn new(block_ref: BlockRef<T::Hash>, ext_hash: T::Hash, client: C) -> Self {
         Self {
-            block_hash,
+            block_ref,
             ext_hash,
             client,
         }
@@ -268,7 +271,7 @@ impl<T: Config, C> TxInBlock<T, C> {
 
     /// Return the hash of the block that the transaction has made it into.
     pub fn block_hash(&self) -> T::Hash {
-        self.block_hash
+        self.block_ref.hash()
     }
 
     /// Return the hash of the extrinsic that was submitted.
@@ -317,7 +320,7 @@ impl<T: Config, C: OnlineClientT<T>> TxInBlock<T, C> {
         let block_body = self
             .client
             .backend()
-            .block_body(self.block_hash)
+            .block_body(self.block_ref.hash())
             .await?
             .ok_or(Error::Transaction(TransactionError::BlockNotFound))?;
 
@@ -336,7 +339,7 @@ impl<T: Config, C: OnlineClientT<T>> TxInBlock<T, C> {
             .ok_or(Error::Transaction(TransactionError::BlockNotFound))?;
 
         let events = EventsClient::new(self.client.clone())
-            .at(self.block_hash)
+            .at(self.block_ref.clone())
             .await?;
 
         Ok(crate::blocks::ExtrinsicEvents::new(