runtime API: Substitute UncheckedExtrinsic with custom encoding (#1076)

* codegen: Add uncheckedExtrinsic substitute

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Add uncheckedExtrinsic replacement

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* testing: Test uncheckedExtrinsic encoding

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* testing: Apply clippy

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Implement encode_to instead of encode for uncheckedExtrinsic

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Remove encode_as_fields from uncheckedExtrinsic

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: Extend the UncheckedExtrinsic interface

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: Use Static<Encoded> for uncheckedExtrinsic

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: Remove extra impl on the uncheckedExtrinsic

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: Add back the EncodeAsType

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: Simplify the decode_as_type

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: Use encode_as_type

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: impl Decode for UncheckedExtrinsic

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Update subxt/src/utils/unchecked_extrinsic.rs

Co-authored-by: James Wilson <james@jsdw.me>

* utils: Apply cargo fmt

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils: Check encoding / decoding of uncheckedExtrinsic

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* utils/tests: Use an already encoded tx bytes to start with

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
Co-authored-by: James Wilson <james@jsdw.me>

codegen/src/types/substitutes.rs

@@ -99,6 +99,14 @@ impl TypeSubstitutes {
                 parse_quote!(#crate_path::utils::KeyedVec),
             ),
             (path_segments!(BTreeSet), parse_quote!(::std::vec::Vec)),
+            // The `UncheckedExtrinsic(pub Vec<u8>)` is part of the runtime API calls.
+            // The inner bytes represent the encoded extrinsic, however when deriving the
+            // `EncodeAsType` the bytes would be re-encoded. This leads to the bytes
+            // being altered by adding the length prefix in front of them.
+            (
+                path_segments!(sp_runtime::generic::unchecked_extrinsic::UncheckedExtrinsic),
+                parse_quote!(#crate_path::utils::UncheckedExtrinsic),
+            ),
         ];
 
         let default_substitutes = defaults
@@ -339,7 +347,7 @@ impl<T: scale_info::form::Form> From<&scale_info::Path<T>> for PathSegments {
 /// to = ::subxt::utils::Static<::sp_runtime::MultiAddress<A, B>>
 /// ```
 ///
-/// And we encounter a `sp_runtime::MultiAddress<Foo, bar>`, then we will pass the `::sp_runtime::MultiAddress<A, B>`
+/// And we encounter a `sp_runtime::MultiAddress<Foo, Bar>`, then we will pass the `::sp_runtime::MultiAddress<A, B>`
 /// type param value into this call to turn it into `::sp_runtime::MultiAddress<Foo, Bar>`.
 fn replace_path_params_recursively<I: Borrow<syn::Ident>, P: Borrow<TypePath>>(
     path: &mut syn::Path,

subxt/src/utils/mod.rs

@@ -9,6 +9,7 @@ pub mod bits;
 mod multi_address;
 mod multi_signature;
 mod static_type;
+mod unchecked_extrinsic;
 mod wrapper_opaque;
 
 use codec::{Compact, Decode, Encode};
@@ -18,6 +19,7 @@ pub use account_id::AccountId32;
 pub use multi_address::MultiAddress;
 pub use multi_signature::MultiSignature;
 pub use static_type::Static;
+pub use unchecked_extrinsic::UncheckedExtrinsic;
 pub use wrapper_opaque::WrapperKeepOpaque;
 
 // Used in codegen
@@ -26,7 +28,7 @@ pub use primitive_types::{H160, H256, H512};
 
 /// Wraps an already encoded byte vector, prevents being encoded as a raw byte vector as part of
 /// the transaction payload
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, Decode)]
 pub struct Encoded(pub Vec<u8>);
 
 impl codec::Encode for Encoded {

subxt/src/utils/unchecked_extrinsic.rs

@@ -0,0 +1,136 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+//! The "default" Substrate/Polkadot UncheckedExtrinsic.
+//! This is used in codegen for runtime API calls.
+//!
+//! The inner bytes represent the encoded extrinsic expected by the
+//! runtime APIs. Deriving `EncodeAsType` would lead to the inner
+//! bytes to be re-encoded (length prefixed).
+
+use std::marker::PhantomData;
+
+use codec::{Decode, Encode};
+use scale_decode::{visitor::DecodeAsTypeResult, DecodeAsType, IntoVisitor, Visitor};
+
+use super::{Encoded, Static};
+
+/// The unchecked extrinsic from substrate.
+#[derive(Clone, Debug, Eq, PartialEq, Encode)]
+pub struct UncheckedExtrinsic<Address, Call, Signature, Extra>(
+    Static<Encoded>,
+    #[codec(skip)] PhantomData<(Address, Call, Signature, Extra)>,
+);
+
+impl<Address, Call, Signature, Extra> UncheckedExtrinsic<Address, Call, Signature, Extra> {
+    /// Construct a new [`UncheckedExtrinsic`].
+    pub fn new(bytes: Vec<u8>) -> Self {
+        Self(Static(Encoded(bytes)), PhantomData)
+    }
+
+    /// Get the bytes of the encoded extrinsic.
+    pub fn bytes(&self) -> &[u8] {
+        self.0 .0 .0.as_slice()
+    }
+}
+
+impl<Address, Call, Signature, Extra> Decode
+    for UncheckedExtrinsic<Address, Call, Signature, Extra>
+{
+    fn decode<I: codec::Input>(input: &mut I) -> Result<Self, codec::Error> {
+        // The bytes for an UncheckedExtrinsic are first a compact
+        // encoded length, and then the bytes following. This is the
+        // same encoding as a Vec, so easiest ATM is just to decode
+        // into that, and then encode the vec bytes to get our extrinsic
+        // bytes, which we save into an `Encoded` to preserve as-is.
+        let xt_vec: Vec<u8> = Decode::decode(input)?;
+        Ok(UncheckedExtrinsic::new(xt_vec))
+    }
+}
+
+impl<Address, Call, Signature, Extra> scale_encode::EncodeAsType
+    for UncheckedExtrinsic<Address, Call, Signature, Extra>
+{
+    fn encode_as_type_to(
+        &self,
+        type_id: u32,
+        types: &scale_info::PortableRegistry,
+        out: &mut Vec<u8>,
+    ) -> Result<(), scale_encode::Error> {
+        self.0.encode_as_type_to(type_id, types, out)
+    }
+}
+
+impl<Address, Call, Signature, Extra> From<Vec<u8>>
+    for UncheckedExtrinsic<Address, Call, Signature, Extra>
+{
+    fn from(bytes: Vec<u8>) -> Self {
+        UncheckedExtrinsic::new(bytes)
+    }
+}
+
+impl<Address, Call, Signature, Extra> From<UncheckedExtrinsic<Address, Call, Signature, Extra>>
+    for Vec<u8>
+{
+    fn from(bytes: UncheckedExtrinsic<Address, Call, Signature, Extra>) -> Self {
+        bytes.0 .0 .0
+    }
+}
+
+pub struct UncheckedExtrinsicDecodeAsTypeVisitor<Address, Call, Signature, Extra>(
+    PhantomData<(Address, Call, Signature, Extra)>,
+);
+
+impl<Address, Call, Signature, Extra> Visitor
+    for UncheckedExtrinsicDecodeAsTypeVisitor<Address, Call, Signature, Extra>
+{
+    type Value<'scale, 'info> = UncheckedExtrinsic<Address, Call, Signature, Extra>;
+    type Error = scale_decode::Error;
+
+    fn unchecked_decode_as_type<'scale, 'info>(
+        self,
+        input: &mut &'scale [u8],
+        type_id: scale_decode::visitor::TypeId,
+        types: &'info scale_info::PortableRegistry,
+    ) -> DecodeAsTypeResult<Self, Result<Self::Value<'scale, 'info>, Self::Error>> {
+        DecodeAsTypeResult::Decoded(Self::Value::decode_as_type(input, type_id.0, types))
+    }
+}
+
+impl<Address, Call, Signature, Extra> IntoVisitor
+    for UncheckedExtrinsic<Address, Call, Signature, Extra>
+{
+    type Visitor = UncheckedExtrinsicDecodeAsTypeVisitor<Address, Call, Signature, Extra>;
+
+    fn into_visitor() -> Self::Visitor {
+        UncheckedExtrinsicDecodeAsTypeVisitor(PhantomData)
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+
+    #[test]
+    fn unchecked_extrinsic_encoding() {
+        // A tx is basically some bytes with a compact length prefix; ie an encoded vec:
+        let tx_bytes = vec![1u8, 2, 3].encode();
+
+        let unchecked_extrinsic = UncheckedExtrinsic::<(), (), (), ()>::new(tx_bytes.clone());
+        let encoded_tx_bytes = unchecked_extrinsic.encode();
+
+        // The encoded representation must not alter the provided bytes.
+        assert_eq!(tx_bytes, encoded_tx_bytes);
+
+        // However, for decoding we expect to be able to read the extrinsic from the wire
+        // which would be length prefixed.
+        let decoded_tx = UncheckedExtrinsic::<(), (), (), ()>::decode(&mut &tx_bytes[..]).unwrap();
+        let decoded_tx_bytes = decoded_tx.bytes();
+        let encoded_tx_bytes = decoded_tx.encode();
+
+        assert_eq!(decoded_tx_bytes, encoded_tx_bytes);
+        // Ensure we can decode the tx and fetch only the tx bytes.
+        assert_eq!(vec![1, 2, 3], encoded_tx_bytes);
+    }
+}

testing/integration-tests/src/full_client/runtime_api/mod.rs

@@ -3,6 +3,7 @@
 // see LICENSE for license details.
 
 use crate::{node_runtime, test_context};
+use codec::Encode;
 use subxt::utils::AccountId32;
 use subxt_signer::sr25519::dev;
 
@@ -47,3 +48,57 @@ async fn account_nonce() -> Result<(), subxt::Error> {
 
     Ok(())
 }
+
+#[tokio::test]
+async fn unchecked_extrinsic_encoding() -> Result<(), subxt::Error> {
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let alice = dev::alice();
+    let bob = dev::bob();
+    let bob_address = bob.public_key().to_address();
+
+    // Construct a tx from Alice to Bob.
+    let tx = node_runtime::tx().balances().transfer(bob_address, 10_000);
+
+    let signed_extrinsic = api
+        .tx()
+        .create_signed(&tx, &alice, Default::default())
+        .await
+        .unwrap();
+
+    let tx_bytes = signed_extrinsic.into_encoded();
+    let len = tx_bytes.len() as u32;
+
+    // Manually encode the runtime API call arguments to make a raw call.
+    let mut encoded = tx_bytes.clone();
+    encoded.extend(len.encode());
+
+    let expected_result: node_runtime::runtime_types::pallet_transaction_payment::types::FeeDetails<
+        ::core::primitive::u128,
+    > = api
+        .runtime_api()
+        .at_latest()
+        .await?
+        .call_raw(
+            "TransactionPaymentApi_query_fee_details",
+            Some(encoded.as_ref()),
+        )
+        .await?;
+
+    // Use the generated API to confirm the result with the raw call.
+    let runtime_api_call = node_runtime::apis()
+        .transaction_payment_api()
+        .query_fee_details(tx_bytes.into(), len);
+
+    let result = api
+        .runtime_api()
+        .at_latest()
+        .await?
+        .call(runtime_api_call)
+        .await?;
+
+    assert_eq!(expected_result, result);
+
+    Ok(())
+}