Add runtime_metadata_url to pull metadata directly from a node (#689)

* Allow metadata to be pulled directly from a node with runtime_metadata_url

* Update docs

* https too, and abstract out block_on fn

* tweak a comment in the example

cli/src/commands/codegen.rs

@@ -4,12 +4,7 @@
 
 use clap::Parser as ClapParser;
 use color_eyre::eyre;
-use frame_metadata::RuntimeMetadataPrefixed;
 use jsonrpsee::client_transport::ws::Uri;
-use scale::{
-    Decode,
-    Input,
-};
 use std::{
     fs,
     io::Read,
@@ -48,7 +43,7 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
         let mut file = fs::File::open(file)?;
         let mut bytes = Vec::new();
         file.read_to_end(&mut bytes)?;
-        codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
+        codegen(&bytes, opts.derives, opts.crate_path)?;
         return Ok(())
     }
 
@@ -57,18 +52,16 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
             .parse::<Uri>()
             .expect("default url is valid")
     });
-    let (_, bytes) = super::metadata::fetch_metadata(&url).await?;
-    codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
+    let bytes = subxt_codegen::utils::fetch_metadata_bytes(&url).await?;
+    codegen(&bytes, opts.derives, opts.crate_path)?;
     Ok(())
 }
 
-fn codegen<I: Input>(
-    encoded: &mut I,
+fn codegen(
+    metadata_bytes: &[u8],
     raw_derives: Vec<String>,
     crate_path: Option<String>,
 ) -> color_eyre::Result<()> {
-    let metadata = <RuntimeMetadataPrefixed as Decode>::decode(encoded)?;
-    let generator = subxt_codegen::RuntimeGenerator::new(metadata);
     let item_mod = syn::parse_quote!(
         pub mod api {}
     );
@@ -82,7 +75,12 @@ fn codegen<I: Input>(
     let mut derives = DerivesRegistry::new(&crate_path);
     derives.extend_for_all(p.into_iter());
 
-    let runtime_api = generator.generate_runtime(item_mod, derives, crate_path);
+    let runtime_api = subxt_codegen::generate_runtime_api_from_bytes(
+        item_mod,
+        metadata_bytes,
+        derives,
+        crate_path,
+    );
     println!("{}", runtime_api);
     Ok(())
 }

cli/src/commands/compatibility.rs

@@ -111,7 +111,7 @@ async fn handle_full_metadata(nodes: &[Uri]) -> color_eyre::Result<()> {
 }
 
 async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result<RuntimeMetadataV14> {
-    let (_, bytes) = super::metadata::fetch_metadata(url).await?;
+    let bytes = subxt_codegen::utils::fetch_metadata_bytes(url).await?;
 
     let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
     if metadata.0 != META_RESERVED {

cli/src/commands/metadata.rs

@@ -5,24 +5,13 @@
 use clap::Parser as ClapParser;
 use color_eyre::eyre;
 use frame_metadata::RuntimeMetadataPrefixed;
-use jsonrpsee::{
-    async_client::ClientBuilder,
-    client_transport::ws::{
-        Uri,
-        WsTransportClientBuilder,
-    },
-    core::{
-        client::ClientT,
-        Error,
-    },
-    http_client::HttpClientBuilder,
-    rpc_params,
-};
+use jsonrpsee::client_transport::ws::Uri;
 use scale::Decode;
 use std::io::{
     self,
     Write,
 };
+use subxt_codegen::utils::fetch_metadata_hex;
 
 /// Download metadata from a substrate node, for use with `subxt` codegen.
 #[derive(Debug, ClapParser)]
@@ -41,10 +30,11 @@ pub struct Opts {
 }
 
 pub async fn run(opts: Opts) -> color_eyre::Result<()> {
-    let (hex_data, bytes) = fetch_metadata(&opts.url).await?;
+    let hex_data = fetch_metadata_hex(&opts.url).await?;
 
     match opts.format.as_str() {
         "json" => {
+            let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
             let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
             let json = serde_json::to_string_pretty(&metadata)?;
             println!("{}", json);
@@ -54,7 +44,10 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
             println!("{}", hex_data);
             Ok(())
         }
-        "bytes" => Ok(io::stdout().write_all(&bytes)?),
+        "bytes" => {
+            let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
+            Ok(io::stdout().write_all(&bytes)?)
+        }
         _ => {
             Err(eyre::eyre!(
                 "Unsupported format `{}`, expected `json`, `hex` or `bytes`",
@@ -63,40 +56,3 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
         }
     }
 }
-
-pub async fn fetch_metadata(url: &Uri) -> color_eyre::Result<(String, Vec<u8>)> {
-    let hex_data = match url.scheme_str() {
-        Some("http") => fetch_metadata_http(url).await,
-        Some("ws") | Some("wss") => fetch_metadata_ws(url).await,
-        invalid_scheme => {
-            let scheme = invalid_scheme.unwrap_or("no scheme");
-            Err(eyre::eyre!(format!(
-                "`{}` not supported, expects 'http', 'ws', or 'wss'",
-                scheme
-            )))
-        }
-    }?;
-
-    let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
-
-    Ok((hex_data, bytes))
-}
-
-async fn fetch_metadata_ws(url: &Uri) -> color_eyre::Result<String> {
-    let (sender, receiver) = WsTransportClientBuilder::default()
-        .build(url.to_string().parse::<Uri>().unwrap())
-        .await
-        .map_err(|e| Error::Transport(e.into()))?;
-
-    let client = ClientBuilder::default()
-        .max_notifs_per_subscription(4096)
-        .build_with_tokio(sender, receiver);
-
-    Ok(client.request("state_getMetadata", rpc_params![]).await?)
-}
-
-async fn fetch_metadata_http(url: &Uri) -> color_eyre::Result<String> {
-    let client = HttpClientBuilder::default().build(url.to_string())?;
-
-    Ok(client.request::<String>("state_getMetadata", None).await?)
-}

codegen/Cargo.toml

@@ -21,6 +21,9 @@ quote = "1.0.8"
 syn = "1.0.58"
 scale-info = { version = "2.0.0", features = ["bit-vec"] }
 subxt-metadata = { version = "0.24.0", path = "../metadata" }
+jsonrpsee = { version = "0.15.1", features = ["async-client", "client-ws-transport", "http-client"] }
+hex = "0.4.3"
+tokio = { version = "1.8", features = ["macros", "rt-multi-thread"] }
 
 [dev-dependencies]
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }

codegen/src/api/mod.rs

@@ -19,6 +19,10 @@ use crate::{
         CompositeDefFields,
         TypeGenerator,
     },
+    utils::{
+        fetch_metadata_bytes_blocking,
+        Uri,
+    },
     CratePath,
 };
 use codec::Decode;
@@ -50,9 +54,10 @@ use syn::parse_quote;
 /// * `item_mod` - The module declaration for which the API is implemented.
 /// * `path` - The path to the scale encoded metadata of the runtime node.
 /// * `derives` - Provide custom derives for the generated types.
+/// * `crate_path` - Path to the `subxt` crate.
 ///
 /// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
-pub fn generate_runtime_api<P>(
+pub fn generate_runtime_api_from_path<P>(
     item_mod: syn::ItemMod,
     path: P,
     derives: DerivesRegistry,
@@ -69,6 +74,49 @@ where
     file.read_to_end(&mut bytes)
         .unwrap_or_else(|e| abort_call_site!("Failed to read metadata file: {}", e));
 
+    generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path)
+}
+
+/// Generates the API for interacting with a substrate runtime, using metadata
+/// that can be downloaded from a node at the provided URL. This function blocks
+/// while retrieving the metadata.
+///
+/// # Arguments
+///
+/// * `item_mod` - The module declaration for which the API is implemented.
+/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from.
+/// * `derives` - Provide custom derives for the generated types.
+/// * `crate_path` - Path to the `subxt` crate.
+///
+/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
+pub fn generate_runtime_api_from_url(
+    item_mod: syn::ItemMod,
+    url: &Uri,
+    derives: DerivesRegistry,
+    crate_path: CratePath,
+) -> TokenStream2 {
+    let bytes = fetch_metadata_bytes_blocking(url)
+        .unwrap_or_else(|e| abort_call_site!("Failed to obtain metadata: {}", e));
+
+    generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path)
+}
+
+/// Generates the API for interacting with a substrate runtime, using metadata bytes.
+///
+/// # Arguments
+///
+/// * `item_mod` - The module declaration for which the API is implemented.
+/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from.
+/// * `derives` - Provide custom derives for the generated types.
+/// * `crate_path` - Path to the `subxt` crate.
+///
+/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
+pub fn generate_runtime_api_from_bytes(
+    item_mod: syn::ItemMod,
+    bytes: &[u8],
+    derives: DerivesRegistry,
+    crate_path: CratePath,
+) -> TokenStream2 {
     let metadata = frame_metadata::RuntimeMetadataPrefixed::decode(&mut &bytes[..])
         .unwrap_or_else(|e| abort_call_site!("Failed to decode metadata: {}", e));
 
@@ -84,8 +132,9 @@ pub struct RuntimeGenerator {
 impl RuntimeGenerator {
     /// Create a new runtime generator from the provided metadata.
     ///
-    /// **Note:** If you have a path to the metadata, prefer to use [generate_runtime_api]
-    /// for generating the runtime API.
+    /// **Note:** If you have the metadata path, URL or bytes to hand, prefer to use
+    /// one of the `generate_runtime_api_from_*` functions for generating the runtime API
+    /// from that.
     pub fn new(metadata: RuntimeMetadataPrefixed) -> Self {
         match metadata.1 {
             RuntimeMetadata::V14(v14) => Self { metadata: v14 },

codegen/src/lib.rs

@@ -45,9 +45,13 @@ mod api;
 mod ir;
 mod types;
 
+pub mod utils;
+
 pub use self::{
     api::{
-        generate_runtime_api,
+        generate_runtime_api_from_bytes,
+        generate_runtime_api_from_path,
+        generate_runtime_api_from_url,
         RuntimeGenerator,
     },
     types::{

codegen/src/utils/fetch_metadata.rs

@@ -0,0 +1,108 @@
+use jsonrpsee::{
+    async_client::ClientBuilder,
+    client_transport::ws::{
+        Uri,
+        WsTransportClientBuilder,
+    },
+    core::{
+        client::ClientT,
+        Error,
+    },
+    http_client::HttpClientBuilder,
+    rpc_params,
+};
+
+/// Returns the metadata bytes from the provided URL, blocking the current thread.
+pub fn fetch_metadata_bytes_blocking(url: &Uri) -> Result<Vec<u8>, FetchMetadataError> {
+    tokio_block_on(fetch_metadata_bytes(url))
+}
+
+/// Returns the raw, 0x prefixed metadata hex from the provided URL, blocking the current thread.
+pub fn fetch_metadata_hex_blocking(url: &Uri) -> Result<String, FetchMetadataError> {
+    tokio_block_on(fetch_metadata_hex(url))
+}
+
+// Block on some tokio runtime for sync contexts
+fn tokio_block_on<T, Fut: std::future::Future<Output = T>>(fut: Fut) -> T {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(fut)
+}
+
+/// Returns the metadata bytes from the provided URL.
+pub async fn fetch_metadata_bytes(url: &Uri) -> Result<Vec<u8>, FetchMetadataError> {
+    let hex = fetch_metadata_hex(url).await?;
+    let bytes = hex::decode(hex.trim_start_matches("0x"))?;
+    Ok(bytes)
+}
+
+/// Returns the raw, 0x prefixed metadata hex from the provided URL.
+pub async fn fetch_metadata_hex(url: &Uri) -> Result<String, FetchMetadataError> {
+    let hex_data = match url.scheme_str() {
+        Some("http") | Some("https") => fetch_metadata_http(url).await,
+        Some("ws") | Some("wss") => fetch_metadata_ws(url).await,
+        invalid_scheme => {
+            let scheme = invalid_scheme.unwrap_or("no scheme");
+            Err(FetchMetadataError::InvalidScheme(scheme.to_owned()))
+        }
+    }?;
+    Ok(hex_data)
+}
+
+async fn fetch_metadata_ws(url: &Uri) -> Result<String, FetchMetadataError> {
+    let (sender, receiver) = WsTransportClientBuilder::default()
+        .build(url.to_string().parse::<Uri>().unwrap())
+        .await
+        .map_err(|e| Error::Transport(e.into()))?;
+
+    let client = ClientBuilder::default()
+        .max_notifs_per_subscription(4096)
+        .build_with_tokio(sender, receiver);
+
+    Ok(client.request("state_getMetadata", rpc_params![]).await?)
+}
+
+async fn fetch_metadata_http(url: &Uri) -> Result<String, FetchMetadataError> {
+    let client = HttpClientBuilder::default().build(url.to_string())?;
+
+    Ok(client.request::<String>("state_getMetadata", None).await?)
+}
+
+#[derive(Debug)]
+pub enum FetchMetadataError {
+    DecodeError(hex::FromHexError),
+    RequestError(jsonrpsee::core::Error),
+    InvalidScheme(String),
+}
+
+impl std::fmt::Display for FetchMetadataError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FetchMetadataError::DecodeError(e) => {
+                write!(f, "Cannot decode hex value: {e}")
+            }
+            FetchMetadataError::RequestError(e) => write!(f, "Request error: {e}"),
+            FetchMetadataError::InvalidScheme(s) => {
+                write!(
+                    f,
+                    "'{s}' not supported, supported URI schemes are http, https, ws or wss."
+                )
+            }
+        }
+    }
+}
+
+impl std::error::Error for FetchMetadataError {}
+
+impl From<hex::FromHexError> for FetchMetadataError {
+    fn from(e: hex::FromHexError) -> Self {
+        FetchMetadataError::DecodeError(e)
+    }
+}
+impl From<jsonrpsee::core::Error> for FetchMetadataError {
+    fn from(e: jsonrpsee::core::Error) -> Self {
+        FetchMetadataError::RequestError(e)
+    }
+}

codegen/src/utils/mod.rs

@@ -0,0 +1,12 @@
+mod fetch_metadata;
+
+// easy access to this type needed for fetching metadata:
+pub use jsonrpsee::client_transport::ws::Uri;
+
+pub use fetch_metadata::{
+    fetch_metadata_bytes,
+    fetch_metadata_bytes_blocking,
+    fetch_metadata_hex,
+    fetch_metadata_hex_blocking,
+    FetchMetadataError,
+};

examples/examples/custom_metadata_url.rs

@@ -0,0 +1,12 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+// If you'd like to use metadata directly from a running node, you
+// can provide a URL to that node here. HTTP or WebSocket URLs can be
+// provided. Note that if the metadata cannot be retrieved from this
+// node URL at compile time, compilation will fail.
+#[subxt::subxt(runtime_metadata_url = "wss://rpc.polkadot.io:443")]
+pub mod polkadot {}
+
+fn main() {}

macro/src/lib.rs

@@ -89,10 +89,18 @@
 
 extern crate proc_macro;
 
+use std::str::FromStr;
+
 use darling::FromMeta;
 use proc_macro::TokenStream;
-use proc_macro_error::proc_macro_error;
-use subxt_codegen::DerivesRegistry;
+use proc_macro_error::{
+    abort_call_site,
+    proc_macro_error,
+};
+use subxt_codegen::{
+    utils::Uri,
+    DerivesRegistry,
+};
 use syn::{
     parse_macro_input,
     punctuated::Punctuated,
@@ -100,7 +108,10 @@ use syn::{
 
 #[derive(Debug, FromMeta)]
 struct RuntimeMetadataArgs {
-    runtime_metadata_path: String,
+    #[darling(default)]
+    runtime_metadata_path: Option<String>,
+    #[darling(default)]
+    runtime_metadata_url: Option<String>,
     #[darling(default)]
     derive_for_all_types: Option<Punctuated<syn::Path, syn::Token![,]>>,
     #[darling(multiple)]
@@ -126,10 +137,6 @@ pub fn subxt(args: TokenStream, input: TokenStream) -> TokenStream {
         Err(e) => return TokenStream::from(e.write_errors()),
     };
 
-    let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
-    let root_path = std::path::Path::new(&root);
-    let path = root_path.join(args.runtime_metadata_path);
-
     let crate_path = match args.crate_path {
         Some(crate_path) => crate_path.into(),
         None => subxt_codegen::CratePath::default(),
@@ -146,6 +153,36 @@ pub fn subxt(args: TokenStream, input: TokenStream) -> TokenStream {
         )
     }
 
-    subxt_codegen::generate_runtime_api(item_mod, &path, derives_registry, crate_path)
-        .into()
+    match (args.runtime_metadata_path, args.runtime_metadata_url) {
+        (Some(rest_of_path), None) => {
+            let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
+            let root_path = std::path::Path::new(&root);
+            let path = root_path.join(rest_of_path);
+            subxt_codegen::generate_runtime_api_from_path(
+                item_mod,
+                &path,
+                derives_registry,
+                crate_path,
+            )
+            .into()
+        }
+        (None, Some(url_string)) => {
+            let url = Uri::from_str(&url_string).unwrap_or_else(|_| {
+                abort_call_site!("Cannot download metadata; invalid url: {}", url_string)
+            });
+            subxt_codegen::generate_runtime_api_from_url(
+                item_mod,
+                &url,
+                derives_registry,
+                crate_path,
+            )
+            .into()
+        }
+        (None, None) => {
+            abort_call_site!("One of 'runtime_metadata_path' or 'runtime_metadata_url' must be provided")
+        }
+        (Some(_), Some(_)) => {
+            abort_call_site!("Only one of 'runtime_metadata_path' or 'runtime_metadata_url' can be provided")
+        }
+    }
 }

testing/ui-tests/src/incorrect/need_url_or_path.rs

@@ -0,0 +1,4 @@
+#[subxt::subxt()]
+pub mod node_runtime {}
+
+fn main() {}

testing/ui-tests/src/incorrect/need_url_or_path.stderr

@@ -0,0 +1,7 @@
+error: One of 'runtime_metadata_path' or 'runtime_metadata_url' must be provided
+ --> src/incorrect/need_url_or_path.rs:1:1
+  |
+1 | #[subxt::subxt()]
+  | ^^^^^^^^^^^^^^^^^
+  |
+  = note: this error originates in the attribute macro `subxt::subxt` (in Nightly builds, run with -Z macro-backtrace for more info)

testing/ui-tests/src/incorrect/url_and_path_provided.rs

@@ -0,0 +1,7 @@
+#[subxt::subxt(
+    runtime_metadata_path = "../../../artifacts/polkadot_metadata.scale",
+    runtime_metadata_url = "wss://rpc.polkadot.io:443"
+)]
+pub mod node_runtime {}
+
+fn main() {}

testing/ui-tests/src/incorrect/url_and_path_provided.stderr

@@ -0,0 +1,10 @@
+error: Only one of 'runtime_metadata_path' or 'runtime_metadata_url' can be provided
+ --> src/incorrect/url_and_path_provided.rs:1:1
+  |
+1 | / #[subxt::subxt(
+2 | |     runtime_metadata_path = "../../../artifacts/polkadot_metadata.scale",
+3 | |     runtime_metadata_url = "wss://rpc.polkadot.io:443"
+4 | | )]
+  | |__^
+  |
+  = note: this error originates in the attribute macro `subxt::subxt` (in Nightly builds, run with -Z macro-backtrace for more info)