Reference key storage api (#447)

* codegen: Update polkadot.rs

polkadot commit-hash: d96d3bea85
polkadot tag: v0.9.16-rc2

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Reference key storage api

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Regenerate polkadot.rs with reference api

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* tests: Update tests with reference interface

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* cli: Fix polkadot.rs license check

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Update polkadot.rs with copyright

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Revert "codegen: Update polkadot.rs with copyright"

This reverts commit 2970d0573dc0b11d01072b270a525ad497992ddf.

Revert "cli: Fix polkadot.rs license check"

This reverts commit 6fe8818582ae39669c059c1ed0424b6606620295.

* codegen: Implement AccountData trait in the expected order

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Store implementation of StorageEntry

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Generate AccountDefaultData wrapper struct

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Allow `Account` references

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Update polkadot.rs

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Utilize AccountDefaultData instead of Account

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Update polkadot.rs

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* tests: Update tests to utilize `Account` reference

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Rename AccountDefaultData to AccountOwned

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Add comments for wrapper account

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Obtain vector type parameter for TypePath::Type

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Use slices instead of `& std::vec` in storage API

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Update polkadot.rs

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Fix documentation

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* tests: Remove extra reference

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* examples: Add staking example to exercise storage API

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Update polkadot.rs

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* tests: Update storage tests

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Fix cargo clippy

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Simplify vec_type_param

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* examples: Rename staking_details.rs to fetch_staking_details.rs

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* tests: Remove dummy variable

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* examples: Update polkadot version

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Apply rust-fmt

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* codegen: Regenerate polkadot.rs

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* examples: Remove comment

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

codegen/src/api/mod.rs

@@ -443,7 +443,8 @@ fn generate_default_account_data_impl(
         };
 
     // this path to the storage entry depends on storage codegen.
-    let storage_entry_path = quote!(self::system::storage::Account);
+    // AccountOwned contains the same data as Account does, but without references.
+    let storage_entry_path = quote!(self::system::storage::AccountOwned);
 
     Some(quote! {
         /// The default storage entry from which to fetch an account nonce, required for
@@ -455,12 +456,12 @@ fn generate_default_account_data_impl(
             type AccountId = #account_id_ty;
             type Index = #account_nonce_ty;
 
-            fn nonce(result: &<Self::StorageEntry as ::subxt::StorageEntry>::Value) -> Self::Index {
-                result.nonce
-            }
             fn storage_entry(account_id: Self::AccountId) -> Self::StorageEntry {
                 #storage_entry_path(account_id)
             }
+            fn nonce(result: &<Self::StorageEntry as ::subxt::StorageEntry>::Value) -> Self::Index {
+                result.nonce
+            }
         }
     })
 }

codegen/src/api/storage.rs

@@ -74,12 +74,15 @@ fn generate_storage_entry_fns(
     storage_entry: &StorageEntryMetadata<PortableForm>,
 ) -> (TokenStream2, TokenStream2) {
     let entry_struct_ident = format_ident!("{}", storage_entry.name);
-    let (fields, entry_struct, constructor, key_impl) = match storage_entry.ty {
+    let is_account_wrapper = pallet.name == "System" && storage_entry.name == "Account";
+    let wrapper_struct_ident = format_ident!("{}Owned", storage_entry.name);
+    let (fields, entry_struct, constructor, key_impl, should_ref) = match storage_entry.ty
+    {
         StorageEntryType::Plain(_) => {
             let entry_struct = quote!( pub struct #entry_struct_ident; );
             let constructor = quote!( #entry_struct_ident );
             let key_impl = quote!(::subxt::StorageEntryKey::Plain);
-            (vec![], entry_struct, constructor, key_impl)
+            (vec![], entry_struct, constructor, key_impl, false)
         }
         StorageEntryType::Map {
             ref key,
@@ -117,10 +120,18 @@ fn generate_storage_entry_fns(
                         .collect::<Vec<_>>();
 
                     let field_names = fields.iter().map(|(n, _)| n);
-                    let field_types = fields.iter().map(|(_, t)| t);
+                    let field_types = fields.iter().map(|(_, t)| {
+                        // If the field type is `::std::vec::Vec<T>` obtain the type parameter and
+                        // surround with slice brackets. Otherwise, utilize the field_type as is.
+                        match t.vec_type_param() {
+                            Some(ty) => quote!([#ty]),
+                            None => quote!(#t),
+                        }
+                    });
 
                     let entry_struct = quote! {
-                        pub struct #entry_struct_ident( #( pub #field_types ),* );
+                        pub struct #entry_struct_ident <'a>( #( pub &'a #field_types ),* );
+
                     };
                     let constructor =
                         quote!( #entry_struct_ident( #( #field_names ),* ) );
@@ -165,13 +176,35 @@ fn generate_storage_entry_fns(
                         )
                     };
 
-                    (fields, entry_struct, constructor, key_impl)
+                    (fields, entry_struct, constructor, key_impl, true)
                 }
                 _ => {
+                    let (lifetime_param, lifetime_ref) = (quote!(<'a>), quote!(&'a));
+
                     let ty_path = type_gen.resolve_type_path(key.id(), &[]);
                     let fields = vec![(format_ident!("_0"), ty_path.clone())];
+                    // `::system::storage::Account` was utilized as associated type `StorageEntry`
+                    // for `::subxt::AccountData` implementation by the generated `DefaultAccountData`.
+                    // Due to changes in the storage API, `::system::storage::Account` cannot be
+                    // used without specifying a lifetime. To satisfy `::subxt::AccountData`
+                    // implementation, a non-reference wrapper `AccountOwned` is generated.
+                    let wrapper_struct = if is_account_wrapper {
+                        quote!(
+                            pub struct #wrapper_struct_ident ( pub #ty_path );
+                        )
+                    } else {
+                        quote!()
+                    };
+
+                    // `ty_path` can be `std::vec::Vec<T>`. In such cases, the entry struct
+                    // should contain a slice reference.
+                    let ty_slice = match ty_path.vec_type_param() {
+                        Some(ty) => quote!([#ty]),
+                        None => quote!(#ty_path),
+                    };
                     let entry_struct = quote! {
-                        pub struct #entry_struct_ident( pub #ty_path );
+                        pub struct #entry_struct_ident #lifetime_param( pub #lifetime_ref #ty_slice );
+                        #wrapper_struct
                     };
                     let constructor = quote!( #entry_struct_ident(_0) );
                     let hasher = hashers.get(0).unwrap_or_else(|| {
@@ -182,7 +215,7 @@ fn generate_storage_entry_fns(
                             vec![ ::subxt::StorageMapKey::new(&self.0, #hasher) ]
                         )
                     };
-                    (fields, entry_struct, constructor, key_impl)
+                    (fields, entry_struct, constructor, key_impl, true)
                 }
             }
         }
@@ -208,17 +241,42 @@ fn generate_storage_entry_fns(
         }
     };
 
+    let (lifetime_param, reference, anon_lifetime) = if should_ref {
+        (quote!(<'a>), quote!(&), quote!(<'_>))
+    } else {
+        (quote!(), quote!(), quote!())
+    };
+
+    let storage_entry_impl = quote! (
+        const PALLET: &'static str = #pallet_name;
+        const STORAGE: &'static str = #storage_name;
+        type Value = #storage_entry_value_ty;
+        fn key(&self) -> ::subxt::StorageEntryKey {
+            #key_impl
+        }
+    );
+
+    // The wrapper account must implement the same trait as the counter-part Account,
+    // with the same pallet and storage. This continues the implementation of the wrapper
+    // generated with the Account.
+    let wrapper_entry_impl = if is_account_wrapper {
+        quote!(
+            impl ::subxt::StorageEntry for #wrapper_struct_ident {
+                #storage_entry_impl
+            }
+        )
+    } else {
+        quote!()
+    };
+
     let storage_entry_type = quote! {
         #entry_struct
 
-        impl ::subxt::StorageEntry for #entry_struct_ident {
+        impl ::subxt::StorageEntry for #entry_struct_ident #anon_lifetime {
-            const PALLET: &'static str = #pallet_name;
+            #storage_entry_impl
-            const STORAGE: &'static str = #storage_name;
-            type Value = #storage_entry_value_ty;
-            fn key(&self) -> ::subxt::StorageEntryKey {
-                #key_impl
-            }
         }
+
+        #wrapper_entry_impl
     };
 
     let client_iter_fn = if matches!(storage_entry.ty, StorageEntryType::Map { .. }) {
@@ -226,7 +284,7 @@ fn generate_storage_entry_fns(
             pub async fn #fn_name_iter(
                 &self,
                 hash: ::core::option::Option<T::Hash>,
-            ) -> ::core::result::Result<::subxt::KeyIter<'a, T, #entry_struct_ident>, ::subxt::BasicError> {
+            ) -> ::core::result::Result<::subxt::KeyIter<'a, T, #entry_struct_ident #lifetime_param>, ::subxt::BasicError> {
                 self.client.storage().iter(hash).await
             }
         )
@@ -234,9 +292,16 @@ fn generate_storage_entry_fns(
         quote!()
     };
 
-    let key_args = fields
+    let key_args = fields.iter().map(|(field_name, field_type)| {
-        .iter()
+        // The field type is translated from `std::vec::Vec<T>` to `[T]`, if the
-        .map(|(field_name, field_type)| quote!( #field_name: #field_type ));
+        // interface should generate a reference. In such cases, the vector ultimately is
+        // a slice.
+        let field_ty = match field_type.vec_type_param() {
+            Some(ty) if should_ref => quote!([#ty]),
+            _ => quote!(#field_type),
+        };
+        quote!( #field_name: #reference #field_ty )
+    });
     let client_fns = quote! {
         pub async fn #fn_name(
             &self,

codegen/src/types/type_path.rs

@@ -76,6 +76,21 @@ impl TypePath {
             Self::Substitute(sub) => sub.parent_type_params(acc),
         }
     }
+
+    /// Gets the vector type parameter if the data is represented as `TypeDef::Sequence`.
+    ///
+    /// **Note:** Utilized for transforming `std::vec::Vec<T>` into slices `&[T]` for the storage API.
+    pub fn vec_type_param(&self) -> Option<&TypePath> {
+        let ty = match self {
+            TypePath::Type(ty) => ty,
+            _ => return None,
+        };
+
+        match ty.ty.type_def() {
+            TypeDef::Sequence(_) => Some(&ty.params[0]),
+            _ => None,
+        }
+    }
 }
 
 #[derive(Clone, Debug)]

examples/examples/fetch_staking_details.rs

@@ -0,0 +1,82 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is part of subxt.
+//
+// subxt is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// subxt is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with subxt.  If not, see <http://www.gnu.org/licenses/>.
+
+//! To run this example, a local polkadot node should be running. Example verified against polkadot 0.9.13-d96d3bea85-aarch64-macos.
+//!
+//! E.g.
+//! ```bash
+//! curl "https://github.com/paritytech/polkadot/releases/download/v0.9.13/polkadot" --output /usr/local/bin/polkadot --location
+//! polkadot --dev --tmp
+//! ```
+
+use sp_keyring::AccountKeyring;
+use subxt::{
+    sp_core::{
+        sr25519,
+        Pair,
+    },
+    sp_runtime::AccountId32,
+    ClientBuilder,
+    DefaultConfig,
+    DefaultExtra,
+};
+
+#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+pub mod polkadot {}
+
+#[async_std::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    env_logger::init();
+
+    let api = ClientBuilder::new()
+        .build()
+        .await?
+        .to_runtime_api::<polkadot::RuntimeApi<DefaultConfig, DefaultExtra<DefaultConfig>>>();
+
+    let era = api.storage().staking().active_era(None).await?.unwrap();
+    println!(
+        "Staking active era: index: {:?}, start: {:?}",
+        era.index, era.start
+    );
+
+    let alice_id = AccountKeyring::Alice.to_account_id();
+    println!("  Alice account id:        {:?}", alice_id);
+
+    // Get Alice' Stash account ID
+    let alice_stash_id: AccountId32 = sr25519::Pair::from_string("//Alice//stash", None)
+        .expect("Could not obtain stash signer pair")
+        .public()
+        .into();
+    println!("  Alice//stash account id: {:?}", alice_stash_id);
+
+    // Map from all locked "stash" accounts to the controller account.
+    let controller_acc = api
+        .storage()
+        .staking()
+        .bonded(&alice_stash_id, None)
+        .await?
+        .unwrap();
+    println!("    account controlled by: {:?}", controller_acc);
+
+    let era_result = api
+        .storage()
+        .staking()
+        .eras_reward_points(&era.index, None)
+        .await?;
+    println!("Era reward points: {:?}", era_result);
+
+    Ok(())
+}

subxt/tests/integration/frame/balances.rs

@@ -46,12 +46,12 @@ async fn tx_basic_transfer() -> Result<(), subxt::Error<DispatchError>> {
     let alice_pre = api
         .storage()
         .system()
-        .account(alice.account_id().clone(), None)
+        .account(alice.account_id(), None)
         .await?;
     let bob_pre = api
         .storage()
         .system()
-        .account(bob.account_id().clone(), None)
+        .account(bob.account_id(), None)
         .await?;
 
     let events = api
@@ -81,12 +81,12 @@ async fn tx_basic_transfer() -> Result<(), subxt::Error<DispatchError>> {
     let alice_post = api
         .storage()
         .system()
-        .account(alice.account_id().clone(), None)
+        .account(alice.account_id(), None)
         .await?;
     let bob_post = api
         .storage()
         .system()
-        .account(bob.account_id().clone(), None)
+        .account(bob.account_id(), None)
         .await?;
 
     assert!(alice_pre.data.free - 10_000 >= alice_post.data.free);
@@ -128,11 +128,12 @@ async fn storage_balance_lock() -> Result<(), subxt::Error<DispatchError>> {
         .find_first::<system::events::ExtrinsicSuccess>()?
         .expect("No ExtrinsicSuccess Event found");
 
+    let locked_account = AccountKeyring::Bob.to_account_id();
     let locks = cxt
         .api
         .storage()
         .balances()
-        .locks(AccountKeyring::Bob.to_account_id(), None)
+        .locks(&locked_account, None)
         .await?;
 
     assert_eq!(

subxt/tests/integration/frame/contracts.rs

@@ -209,7 +209,7 @@ async fn tx_call() {
         .api
         .storage()
         .contracts()
-        .contract_info_of(contract.clone(), None)
+        .contract_info_of(&contract, None)
         .await;
     assert!(contract_info.is_ok());
 

subxt/tests/integration/frame/staking.rs

@@ -148,7 +148,7 @@ async fn chill_works_for_controller_only() -> Result<(), Error<DispatchError>> {
         .api
         .storage()
         .staking()
-        .ledger(alice.account_id().clone(), None)
+        .ledger(alice.account_id(), None)
         .await?
         .unwrap();
     assert_eq!(alice_stash.account_id(), &ledger.stash);
@@ -248,12 +248,12 @@ async fn storage_current_era() -> Result<(), Error<DispatchError>> {
 
 #[async_std::test]
 async fn storage_era_reward_points() -> Result<(), Error<DispatchError>> {
-    let ctx = test_context().await;
+    let cxt = test_context().await;
-    let current_era_result = ctx
+    let current_era_result = cxt
         .api
         .storage()
         .staking()
-        .eras_reward_points(0, None)
+        .eras_reward_points(&0, None)
         .await;
     assert!(current_era_result.is_ok());
 

subxt/tests/integration/frame/system.rs

@@ -35,7 +35,7 @@ async fn storage_account() -> Result<(), subxt::Error<DispatchError>> {
         .api
         .storage()
         .system()
-        .account(alice.account_id().clone(), None)
+        .account(alice.account_id(), None)
         .await;
 
     assert_matches!(account_info, Ok(_));

subxt/tests/integration/storage.rs

@@ -55,7 +55,7 @@ async fn storage_map_lookup() -> Result<(), subxt::Error<DispatchError>> {
         .await?;
 
     // Look up the nonce for the user (we expect it to be 1).
-    let entry = ctx.api.storage().system().account(alice, None).await?;
+    let entry = ctx.api.storage().system().account(&alice, None).await?;
     assert_eq!(entry.nonce, 1);
 
     Ok(())
@@ -79,7 +79,7 @@ async fn storage_n_mapish_key_is_properly_created(
     };
 
     // This is what the generated code hashes a `session().key_owner(..)` key into:
-    let actual_key_bytes = KeyOwner(KeyTypeId([1, 2, 3, 4]), vec![5u8, 6, 7, 8])
+    let actual_key_bytes = KeyOwner(&KeyTypeId([1, 2, 3, 4]), &[5u8, 6, 7, 8])
         .key()
         .final_key(StorageKeyPrefix::new::<KeyOwner>())
         .0;
@@ -132,7 +132,7 @@ async fn storage_n_map_storage_lookup() -> Result<(), subxt::Error<DispatchError
         .api
         .storage()
         .assets()
-        .approvals(99, alice, bob, None)
+        .approvals(&99, &alice, &bob, None)
         .await?;
     assert_eq!(entry.map(|a| a.amount), Some(123));
     Ok(())