chainHead based backend implementation (#1161)

* add follow_stream impl

* follow_stream_unpin first draft

* add tests for follow_stream_unpin

* more tests and fixes for follow_stream_unpin

* first pass follow_stream_driver

* follow_stream_driver: add tests, fix things, buffer events from last finalized

* First pass finishing Backend impl

* Fix test compile issues

* clippy fixes

* clippy fix and consistify light_client

* revert lightclient tweak

* revert other lightclient thing

* cargo fmt

* start testing unstable backend behind feature flag

* more test fixes and move test-runtime metadata path just incase

* fix compile error

* ensure transaction progress stream actually used and fix another test

* cargo fmt

* CI tweak

* improve some comments and address some feedback bits

* update CI to use our own nightly binary

* wait for finalized block perhaps

.github/workflows/nightly.yml

@@ -2,14 +2,12 @@ name: Daily compatibility check against latest substrate
 
 on:
   schedule:
-    # Run at 8am every day
+    # Run at 8am every day, well after the new binary is built
     - cron: "0 8 * * *"
 
 
 env:
   CARGO_TERM_COLOR: always
-  # Use latest substrate for nightly runs:
-  SUBSTRATE_URL: https://releases.parity.io/substrate/x86_64-debian:bullseye/latest/substrate/substrate
 
 jobs:
   tests:
@@ -19,13 +17,19 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v4
 
-      - name: Download Substrate
+      - name: Download substrate-node binary
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e # v2.28.0
+        with:
+          workflow: build-substrate.yml
+          name: nightly-substrate-binary
+
+      - name: Prepare substrate-node binary
         run: |
-            curl $SUBSTRATE_URL --output substrate-node --location
-            chmod +x substrate-node
-            ./substrate-node --version
-            mkdir -p ~/.local/bin
-            mv substrate-node ~/.local/bin
+          chmod u+x ./substrate-node
+          ./substrate-node --version
+          mkdir -p ~/.local/bin
+          cp ./substrate-node ~/.local/bin
 
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1
@@ -43,8 +47,7 @@ jobs:
           command: test
           args: --all-targets --workspace
 
-      # If the previous step fails, create a new Github issue
-      # to nofity us about it.
+      # If any previous step fails, create a new Github issue to notify us about it.
       - if: ${{ failure() }}
         uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
         env: