chainHead based backend implementation (#1161)

* add follow_stream impl

* follow_stream_unpin first draft

* add tests for follow_stream_unpin

* more tests and fixes for follow_stream_unpin

* first pass follow_stream_driver

* follow_stream_driver: add tests, fix things, buffer events from last finalized

* First pass finishing Backend impl

* Fix test compile issues

* clippy fixes

* clippy fix and consistify light_client

* revert lightclient tweak

* revert other lightclient thing

* cargo fmt

* start testing unstable backend behind feature flag

* more test fixes and move test-runtime metadata path just incase

* fix compile error

* ensure transaction progress stream actually used and fix another test

* cargo fmt

* CI tweak

* improve some comments and address some feedback bits

* update CI to use our own nightly binary

* wait for finalized block perhaps

testing/integration-tests/src/full_client/client/mod.rs

@@ -72,7 +72,7 @@ async fn transaction_validation() {
 
     let tx = node_runtime::tx()
         .balances()
-        .transfer(bob.public_key().into(), 10_000);
+        .transfer_allow_death(bob.public_key().into(), 10_000);
 
     let signed_extrinsic = api
         .tx()
@@ -110,7 +110,7 @@ async fn validation_fails() {
     // The actual TX is not important; the account has no funds to pay for it.
     let tx = node_runtime::tx()
         .balances()
-        .transfer(to.public_key().into(), 1);
+        .transfer_allow_death(to.public_key().into(), 1);
 
     let signed_extrinsic = api
         .tx()
@@ -232,7 +232,7 @@ async fn unsigned_extrinsic_is_same_shape_as_polkadotjs() {
 
     let tx = node_runtime::tx()
         .balances()
-        .transfer(dev::alice().public_key().into(), 12345000000000000);
+        .transfer_allow_death(dev::alice().public_key().into(), 12345000000000000);
 
     let actual_tx = api.tx().create_unsigned(&tx).unwrap();
 
@@ -242,10 +242,10 @@ async fn unsigned_extrinsic_is_same_shape_as_polkadotjs() {
     // - start local substrate node.
     // - open polkadot.js UI in browser and point at local node.
     // - open dev console (may need to refresh page now) and find the WS connection.
-    // - create a balances.transfer to ALICE with 12345 and "submit unsigned".
+    // - create a balances.transferAllowDeath to ALICE (doesn't matter who from) with 12345 and "submit unsigned".
     // - find the submitAndWatchExtrinsic call in the WS connection to get these bytes:
     let expected_tx_bytes = hex::decode(
-        "b004060700d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d0f0090c04bb6db2b"
+        "b004060000d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d0f0090c04bb6db2b"
     )
         .unwrap();
 
@@ -261,7 +261,7 @@ async fn extrinsic_hash_is_same_as_returned() {
 
     let payload = node_runtime::tx()
         .balances()
-        .transfer(dev::alice().public_key().into(), 12345000000000000);
+        .transfer_allow_death(dev::alice().public_key().into(), 12345000000000000);
 
     let tx = api
         .tx()
@@ -314,7 +314,7 @@ async fn partial_fee_estimate_correct() {
     let bob = dev::bob();
     let tx = node_runtime::tx()
         .balances()
-        .transfer(bob.public_key().into(), 1_000_000_000_000);
+        .transfer_allow_death(bob.public_key().into(), 1_000_000_000_000);
 
     let signed_extrinsic = api
         .tx()
@@ -326,7 +326,7 @@ async fn partial_fee_estimate_correct() {
     let partial_fee_1 = signed_extrinsic.partial_fee_estimate().await.unwrap();
 
     // Method II: TransactionPaymentApi_query_fee_details + calculations
-    let latest_block_ref = api.backend().latest_best_block_ref().await.unwrap();
+    let latest_block_ref = api.backend().latest_finalized_block_ref().await.unwrap();
     let len_bytes: [u8; 4] = (signed_extrinsic.encoded().len() as u32).to_le_bytes();
     let encoded_with_len = [signed_extrinsic.encoded(), &len_bytes[..]].concat();
     let InclusionFee {

testing/integration-tests/src/full_client/client/unstable_rpcs.rs

@@ -8,8 +8,8 @@
 use crate::{test_context, utils::node_runtime};
 use assert_matches::assert_matches;
 use codec::Encode;
+use futures::Stream;
 use subxt::{
-    backend::rpc::RpcSubscription,
     backend::unstable::rpc_methods::{
         FollowEvent, Initialized, MethodResponse, RuntimeEvent, RuntimeVersionEvent, StorageQuery,
         StorageQueryType,
@@ -153,7 +153,7 @@ async fn chainhead_unstable_storage() {
         event,
         FollowEvent::OperationStorageItems(res) if res.operation_id == operation_id &&
             res.items.len() == 1 &&
-            res.items[0].key == format!("0x{}", hex::encode(addr_bytes))
+            res.items[0].key.0 == addr_bytes
     );
 
     let event = next_operation_event(&mut blocks).await;
@@ -218,8 +218,6 @@ async fn chainhead_unstable_unpin() {
     assert!(rpc.chainhead_unstable_unpin(sub_id, hash).await.is_err());
 }
 
-// Ignored until this is implemented in Substrate
-#[ignore]
 #[tokio::test]
 async fn chainspec_v1_genesishash() {
     let ctx = test_context().await;
@@ -232,22 +230,18 @@ async fn chainspec_v1_genesishash() {
     assert_eq!(a, b);
 }
 
-// Ignored until this is implemented in Substrate
-#[ignore]
 #[tokio::test]
 async fn chainspec_v1_chainname() {
     let ctx = test_context().await;
     let old_rpc = ctx.legacy_rpc_methods().await;
     let rpc = ctx.unstable_rpc_methods().await;
 
-    let a = old_rpc.system_name().await.unwrap();
+    let a = old_rpc.system_chain().await.unwrap();
     let b = rpc.chainspec_v1_chain_name().await.unwrap();
 
     assert_eq!(a, b);
 }
 
-// Ignored until this is implemented in Substrate
-#[ignore]
 #[tokio::test]
 async fn chainspec_v1_properties() {
     let ctx = test_context().await;
@@ -290,9 +284,14 @@ async fn transaction_unstable_submit_and_watch() {
 }
 
 /// Ignore block related events and obtain the next event related to an operation.
-async fn next_operation_event<T: serde::de::DeserializeOwned>(
-    sub: &mut RpcSubscription<FollowEvent<T>>,
+async fn next_operation_event<
+    T: serde::de::DeserializeOwned,
+    S: Unpin + Stream<Item = Result<FollowEvent<T>, subxt::Error>>,
+>(
+    sub: &mut S,
 ) -> FollowEvent<T> {
+    use futures::StreamExt;
+
     // Number of events to wait for the next operation event.
     const NUM_EVENTS: usize = 10;