Codegen for custom values in metadata (#1117)

* work in progress

* add custom types access

* nit

* custom values client

* adjust light client

* adjust doc comments

* adjust book for custom values in code gen

* format and check docs

* work in progress

* add custom types access

* nit

* custom values client

* adjust light client

* codegen and validation

* adjust docs

* use ignore in docs in book

* change iter implementation

* use validation hash and other codegen changes

* add ui test for custom values codegen

* allow 'latest' metadata to be returned from the fallback code (#1127)

* nits

* fix validation check

* fix comments

* nits

---------

Co-authored-by: James Wilson <james@jsdw.me>
This commit is contained in:
Tadeo Hepperle
2023-08-24 09:50:44 +02:00
committed by GitHub
parent a8dbd9d6d5
commit b413e5e84e
12 changed files with 354 additions and 43 deletions
+61
View File
@@ -0,0 +1,61 @@
// Copyright 2019-2023 Parity Technologies (UK) Ltd.
// This file is dual-licensed as Apache-2.0 or GPL-3.0.
// see LICENSE for license details.
use std::collections::HashSet;
use crate::{types::TypeGenerator, CratePath};
use heck::ToSnakeCase as _;
use subxt_metadata::{CustomValueMetadata, Metadata};
use proc_macro2::TokenStream as TokenStream2;
use quote::{format_ident, quote};
/// Generate the custom values mod, if there are any custom values in the metadata. Else returns None.
pub fn generate_custom_values<'a>(
metadata: &'a Metadata,
type_gen: &'a TypeGenerator,
crate_path: &'a CratePath,
) -> TokenStream2 {
let mut fn_names_taken = HashSet::new();
let custom = metadata.custom();
let custom_values_fns = custom.iter().filter_map(|custom_value| {
generate_custom_value_fn(custom_value, type_gen, crate_path, &mut fn_names_taken)
});
quote! {
pub struct CustomValuesApi;
impl CustomValuesApi {
#(#custom_values_fns)*
}
}
}
/// Generates runtime functions for the given API metadata.
/// Returns None, if the name would not make for a valid identifier.
fn generate_custom_value_fn(
custom_value: CustomValueMetadata,
type_gen: &TypeGenerator,
crate_path: &CratePath,
fn_names_taken: &mut HashSet<String>,
) -> Option<TokenStream2> {
// names are transformed to snake case to make for good function identifiers.
let name = custom_value.name();
let fn_name = name.to_snake_case();
// Skip elements where the fn name is already occupied. E.g. if you have custom values with names "Foo" and "foo" in the metadata.
if fn_names_taken.contains(&fn_name) {
return None;
}
let fn_name_ident = format_ident!("{fn_name}");
fn_names_taken.insert(fn_name);
let custom_value_hash = custom_value.hash();
let return_ty = type_gen.resolve_type_path(custom_value.type_id());
Some(quote!(
pub fn #fn_name_ident() -> #crate_path::custom_values::StaticAddress<#return_ty> {
#crate_path::custom_values::StaticAddress::new_static(#name, [#(#custom_value_hash,)*])
}
))
}
+10
View File
@@ -6,6 +6,7 @@
mod calls;
mod constants;
mod custom_values;
mod errors;
mod events;
mod runtime_apis;
@@ -14,6 +15,7 @@ mod storage;
use subxt_metadata::Metadata;
use super::DerivesRegistry;
use crate::api::custom_values::generate_custom_values;
use crate::error::CodegenError;
use crate::{
ir,
@@ -469,6 +471,8 @@ impl RuntimeGenerator {
let event_path = type_gen.resolve_type_path(self.metadata.outer_enums().event_enum_ty());
let error_path = type_gen.resolve_type_path(self.metadata.outer_enums().error_enum_ty());
let custom_values = generate_custom_values(&self.metadata, &type_gen, &crate_path);
Ok(quote! {
#( #item_mod_attrs )*
#[allow(dead_code, unused_imports, non_camel_case_types)]
@@ -521,6 +525,12 @@ impl RuntimeGenerator {
#apis_mod
pub fn custom() -> CustomValuesApi {
CustomValuesApi
}
#custom_values
pub struct ConstantsApi;
impl ConstantsApi {
#(
+2 -2
View File
@@ -146,8 +146,8 @@ impl Derives {
/// Extend this set of `Derives` from another.
pub fn extend_from(&mut self, other: Derives) {
self.derives.extend(other.derives.into_iter());
self.attributes.extend(other.attributes.into_iter());
self.derives.extend(other.derives);
self.attributes.extend(other.attributes);
}
/// Add `#crate_path::ext::codec::CompactAs` to the derives.
+5 -5
View File
@@ -5,10 +5,10 @@
use super::TryFromError;
use crate::utils::variant_index::VariantIndex;
use crate::{
utils::ordered_map::OrderedMap, ArcStr, ConstantMetadata, CustomMetadata, ExtrinsicMetadata,
Metadata, OuterEnumsMetadata, PalletMetadataInner, RuntimeApiMetadataInner,
RuntimeApiMethodMetadata, RuntimeApiMethodParamMetadata, SignedExtensionMetadata,
StorageEntryMetadata, StorageEntryModifier, StorageEntryType, StorageHasher, StorageMetadata,
utils::ordered_map::OrderedMap, ArcStr, ConstantMetadata, ExtrinsicMetadata, Metadata,
OuterEnumsMetadata, PalletMetadataInner, RuntimeApiMetadataInner, RuntimeApiMethodMetadata,
RuntimeApiMethodParamMetadata, SignedExtensionMetadata, StorageEntryMetadata,
StorageEntryModifier, StorageEntryType, StorageHasher, StorageMetadata,
};
use frame_metadata::v15;
use scale_info::form::PortableForm;
@@ -93,7 +93,7 @@ mod from_v15 {
event_enum_ty: m.outer_enums.event_enum_ty.id,
error_enum_ty: m.outer_enums.error_enum_ty.id,
},
custom: CustomMetadata { map: m.custom.map },
custom: m.custom,
})
}
}
+63 -21
View File
@@ -20,13 +20,14 @@ mod from_into;
mod utils;
use scale_info::{form::PortableForm, PortableRegistry, Variant};
use std::collections::{BTreeMap, HashMap};
use std::collections::HashMap;
use std::sync::Arc;
use utils::ordered_map::OrderedMap;
use utils::variant_index::VariantIndex;
type ArcStr = Arc<str>;
use crate::utils::validation::{get_custom_value_hash, HASH_LEN};
pub use from_into::TryFromError;
pub use utils::validation::MetadataHasher;
@@ -52,7 +53,7 @@ pub struct Metadata {
/// Details about each of the runtime API traits.
apis: OrderedMap<ArcStr, RuntimeApiMetadataInner>,
/// Allows users to add custom types to the metadata. A map that associates a string key to a `CustomValueMetadata`.
custom: CustomMetadata,
custom: frame_metadata::v15::CustomMetadata<PortableForm>,
}
impl Metadata {
@@ -135,8 +136,11 @@ impl Metadata {
}
/// Returns custom user defined types
pub fn custom(&self) -> &CustomMetadata {
&self.custom
pub fn custom(&self) -> CustomMetadata<'_> {
CustomMetadata {
types: self.types(),
inner: &self.custom,
}
}
/// Obtain a unique hash representing this metadata or specific parts of it.
@@ -154,7 +158,7 @@ impl Metadata {
}
/// Get type hash for a type in the registry
pub fn type_hash(&self, id: u32) -> Option<[u8; 32]> {
pub fn type_hash(&self, id: u32) -> Option<[u8; HASH_LEN]> {
self.types.resolve(id)?;
Some(crate::utils::validation::get_type_hash(
&self.types,
@@ -265,22 +269,22 @@ impl<'a> PalletMetadata<'a> {
}
/// Return a hash for the storage entry, or None if it was not found.
pub fn storage_hash(&self, entry_name: &str) -> Option<[u8; 32]> {
pub fn storage_hash(&self, entry_name: &str) -> Option<[u8; HASH_LEN]> {
crate::utils::validation::get_storage_hash(self, entry_name)
}
/// Return a hash for the constant, or None if it was not found.
pub fn constant_hash(&self, constant_name: &str) -> Option<[u8; 32]> {
pub fn constant_hash(&self, constant_name: &str) -> Option<[u8; HASH_LEN]> {
crate::utils::validation::get_constant_hash(self, constant_name)
}
/// Return a hash for the call, or None if it was not found.
pub fn call_hash(&self, call_name: &str) -> Option<[u8; 32]> {
pub fn call_hash(&self, call_name: &str) -> Option<[u8; HASH_LEN]> {
crate::utils::validation::get_call_hash(self, call_name)
}
/// Return a hash for the entire pallet.
pub fn hash(&self) -> [u8; 32] {
pub fn hash(&self) -> [u8; HASH_LEN] {
crate::utils::validation::get_pallet_hash(*self)
}
}
@@ -577,12 +581,12 @@ impl<'a> RuntimeApiMetadata<'a> {
self.inner.methods.get_by_key(name)
}
/// Return a hash for the constant, or None if it was not found.
pub fn method_hash(&self, method_name: &str) -> Option<[u8; 32]> {
pub fn method_hash(&self, method_name: &str) -> Option<[u8; HASH_LEN]> {
crate::utils::validation::get_runtime_api_hash(self, method_name)
}
/// Return a hash for the runtime API trait.
pub fn hash(&self) -> [u8; 32] {
pub fn hash(&self) -> [u8; HASH_LEN] {
crate::utils::validation::get_runtime_trait_hash(*self)
}
}
@@ -640,36 +644,74 @@ pub struct RuntimeApiMethodParamMetadata {
/// Metadata of custom types with custom values, basically the same as `frame_metadata::v15::CustomMetadata<PortableForm>>`.
#[derive(Debug, Clone)]
pub struct CustomMetadata {
map: BTreeMap<String, frame_metadata::v15::CustomValueMetadata<PortableForm>>,
pub struct CustomMetadata<'a> {
types: &'a PortableRegistry,
inner: &'a frame_metadata::v15::CustomMetadata<PortableForm>,
}
impl CustomMetadata {
/// Get a certain [CustomMetadataValue] by its name.
pub fn get(&self, name: &str) -> Option<CustomMetadataValue<'_>> {
self.map.get(name).map(|e| CustomMetadataValue {
impl<'a> CustomMetadata<'a> {
/// Get a certain [CustomValueMetadata] by its name.
pub fn get(&self, name: &str) -> Option<CustomValueMetadata<'a>> {
self.inner
.map
.get_key_value(name)
.map(|(name, e)| CustomValueMetadata {
types: self.types,
type_id: e.ty.id,
data: &e.value,
name,
})
}
/// Iterates over names (keys) and associated custom values
pub fn iter(&self) -> impl Iterator<Item = CustomValueMetadata> {
self.inner.map.iter().map(|(name, e)| CustomValueMetadata {
types: self.types,
type_id: e.ty.id,
data: &e.value,
name: name.as_ref(),
})
}
/// Access the underlying type registry.
pub fn types(&self) -> &PortableRegistry {
self.types
}
}
/// Basically the same as `frame_metadata::v15::CustomValueMetadata<PortableForm>>`, but borrowed.
pub struct CustomMetadataValue<'a> {
pub struct CustomValueMetadata<'a> {
types: &'a PortableRegistry,
type_id: u32,
data: &'a [u8],
name: &'a str,
}
impl<'a> CustomMetadataValue<'a> {
/// the scale encoded value
impl<'a> CustomValueMetadata<'a> {
/// The scale encoded value
pub fn bytes(&self) -> &'a [u8] {
self.data
}
/// the type id in the TypeRegistry
/// The type id in the TypeRegistry
pub fn type_id(&self) -> u32 {
self.type_id
}
/// The name under which the custom value is registered.
pub fn name(&self) -> &str {
self.name
}
/// Calculates the hash for the CustomValueMetadata.
///
/// # Panics
///
/// Panics if `self.type_id` is not registered in the provided type registry
pub fn hash(&self) -> [u8; HASH_LEN] {
let mut cache = HashMap::new();
get_custom_value_hash(self, &mut cache)
}
}
// Support decoding metadata from the "wire" format directly into this.
+48 -7
View File
@@ -5,14 +5,15 @@
//! Utility functions for metadata validation.
use crate::{
ExtrinsicMetadata, Metadata, OuterEnumsMetadata, PalletMetadata, RuntimeApiMetadata,
RuntimeApiMethodMetadata, StorageEntryMetadata, StorageEntryType,
CustomMetadata, CustomValueMetadata, ExtrinsicMetadata, Metadata, OuterEnumsMetadata,
PalletMetadata, RuntimeApiMetadata, RuntimeApiMethodMetadata, StorageEntryMetadata,
StorageEntryType,
};
use scale_info::{form::PortableForm, Field, PortableRegistry, TypeDef, TypeDefVariant, Variant};
use std::collections::HashMap;
// The number of bytes our `hash` function produces.
const HASH_LEN: usize = 32;
pub(crate) const HASH_LEN: usize = 32;
/// Internal byte representation for various metadata types utilized for
/// generating deterministic hashes between different rust versions.
@@ -67,6 +68,7 @@ concat_and_hash_n!(concat_and_hash2(a b));
concat_and_hash_n!(concat_and_hash3(a b c));
concat_and_hash_n!(concat_and_hash4(a b c d));
concat_and_hash_n!(concat_and_hash5(a b c d e));
concat_and_hash_n!(concat_and_hash6(a b c d e f));
/// Obtain the hash representation of a `scale_info::Field`.
fn get_field_hash(
@@ -393,6 +395,27 @@ pub fn get_runtime_trait_hash(trait_metadata: RuntimeApiMetadata) -> [u8; HASH_L
concat_and_hash2(&hash(trait_name.as_bytes()), &method_bytes)
}
pub fn get_custom_metadata_hash(custom_metadata: &CustomMetadata) -> [u8; HASH_LEN] {
let mut cache = HashMap::new();
custom_metadata
.iter()
.fold([0u8; HASH_LEN], |bytes, custom_value| {
xor(bytes, get_custom_value_hash(&custom_value, &mut cache))
})
}
/// Obtain the hash of some custom value in the metadata including it's name/key.
pub fn get_custom_value_hash(
custom_value: &CustomValueMetadata,
cache: &mut HashMap<u32, CachedHash>,
) -> [u8; HASH_LEN] {
concat_and_hash3(
&hash(custom_value.name.as_bytes()),
&get_type_hash(custom_value.types, custom_value.type_id(), cache),
&hash(custom_value.bytes()),
)
}
/// Obtain the hash for a specific storage item, or an error if it's not found.
pub fn get_storage_hash(pallet: &PalletMetadata, entry_name: &str) -> Option<[u8; HASH_LEN]> {
let storage = pallet.storage()?;
@@ -494,6 +517,7 @@ pub struct MetadataHasher<'a> {
metadata: &'a Metadata,
specific_pallets: Option<Vec<&'a str>>,
specific_runtime_apis: Option<Vec<&'a str>>,
include_custom_values: bool,
}
impl<'a> MetadataHasher<'a> {
@@ -503,6 +527,7 @@ impl<'a> MetadataHasher<'a> {
metadata,
specific_pallets: None,
specific_runtime_apis: None,
include_custom_values: true,
}
}
@@ -522,6 +547,12 @@ impl<'a> MetadataHasher<'a> {
self
}
/// Do not hash the custom values
pub fn ignore_custom_values(&mut self) -> &mut Self {
self.include_custom_values = false;
self
}
/// Hash the given metadata.
pub fn hash(&self) -> [u8; HASH_LEN] {
let metadata = self.metadata;
@@ -554,7 +585,7 @@ impl<'a> MetadataHasher<'a> {
// We don't care what order the runtime APIs are seen in, so XOR their
// hashes together to be order independent.
if should_hash {
xor(bytes, xor(bytes, get_runtime_trait_hash(api)))
xor(bytes, get_runtime_trait_hash(api))
} else {
bytes
}
@@ -569,12 +600,18 @@ impl<'a> MetadataHasher<'a> {
self.specific_pallets.as_deref(),
);
concat_and_hash5(
let custom_values_hash = self
.include_custom_values
.then(|| get_custom_metadata_hash(&metadata.custom()))
.unwrap_or_default();
concat_and_hash6(
&pallet_hash,
&apis_hash,
&extrinsic_hash,
&runtime_hash,
&outer_enums_hash,
&custom_values_hash,
)
}
}
@@ -811,8 +848,12 @@ mod tests {
let a_hash2 = get_type_hash(&registry, a_type_id, &mut cache);
let b_hash = get_type_hash(&registry, b_type_id, &mut cache);
let CachedHash::Hash(a_cache_hash) = cache[&a_type_id] else { panic!() };
let CachedHash::Hash(b_cache_hash) = cache[&b_type_id] else { panic!() };
let CachedHash::Hash(a_cache_hash) = cache[&a_type_id] else {
panic!()
};
let CachedHash::Hash(b_cache_hash) = cache[&b_type_id] else {
panic!()
};
assert_eq!(a_hash, a_cache_hash);
assert_eq!(b_hash, b_cache_hash);
+3 -5
View File
@@ -21,7 +21,7 @@ impl VariantIndex {
/// Build indexes from the optional variant ID.
pub fn build(variant_id: Option<u32>, types: &PortableRegistry) -> Self {
let Some(variants) = Self::get(variant_id, types) else {
return Self::empty()
return Self::empty();
};
let mut by_name = HashMap::new();
@@ -47,11 +47,9 @@ impl VariantIndex {
variant_id: Option<u32>,
types: &PortableRegistry,
) -> Option<&[Variant<PortableForm>]> {
let Some(variant_id) = variant_id else {
return None
};
let variant_id = variant_id?;
let TypeDef::Variant(v) = &types.resolve(variant_id)?.type_def else {
return None
return None;
};
Some(&v.variants)
}
@@ -1,15 +1,23 @@
use std::marker::PhantomData;
use crate::dynamic::DecodedValueThunk;
use crate::metadata::DecodeWithMetadata;
/// This represents the address of a custom value in in the metadata.
/// Anything, that implements the [CustomValueAddress] trait can be used, to fetch
/// custom values from the metadata.
/// The trait is implemented by [str] for dynamic loopup and [StaticAddress] for static queries.
pub trait CustomValueAddress {
/// The type of the custom value.
type Target: DecodeWithMetadata;
/// the name (key) by which the custom value can be accessed in the metadata.
fn name(&self) -> &str;
/// An optional hash which, if present, can be checked against node metadata.
fn validation_hash(&self) -> Option<[u8; 32]> {
None
}
}
impl CustomValueAddress for str {
@@ -19,3 +27,43 @@ impl CustomValueAddress for str {
self
}
}
/// A static address to a custom value.
pub struct StaticAddress<R> {
name: &'static str,
hash: Option<[u8; 32]>,
phantom: PhantomData<R>,
}
impl<R> StaticAddress<R> {
#[doc(hidden)]
/// Creates a new StaticAddress.
pub fn new_static(name: &'static str, hash: [u8; 32]) -> Self {
StaticAddress {
name,
hash: Some(hash),
phantom: PhantomData,
}
}
/// Do not validate this custom value prior to accessing it.
pub fn unvalidated(self) -> Self {
Self {
name: self.name,
hash: None,
phantom: self.phantom,
}
}
}
impl<R: DecodeWithMetadata> CustomValueAddress for StaticAddress<R> {
type Target = R;
fn name(&self) -> &str {
self.name
}
fn validation_hash(&self) -> Option<[u8; 32]> {
self.hash
}
}
@@ -30,9 +30,13 @@ impl<T: Config, Client: OfflineClientT<T>> CustomValuesClient<T, Client> {
&self,
address: &Address,
) -> Result<Address::Target, Error> {
// 1. Validate custom value shape if hash given:
self.validate(address)?;
// 2. Attempt to decode custom value:
let metadata = self.client.metadata();
let custom_value = metadata
.custom()
let custom = metadata.custom();
let custom_value = custom
.get(address.name())
.ok_or_else(|| MetadataError::CustomValueNameNotFound(address.name().to_string()))?;
@@ -43,6 +47,30 @@ impl<T: Config, Client: OfflineClientT<T>> CustomValuesClient<T, Client> {
)?;
Ok(value)
}
/// Run the validation logic against some custom value address you'd like to access. Returns `Ok(())`
/// if the address is valid (or if it's not possible to check since the address has no validation hash).
/// Returns an error if the address was not valid (wrong name, type or raw bytes)
pub fn validate<Address: CustomValueAddress + ?Sized>(
&self,
address: &Address,
) -> Result<(), Error> {
let metadata = self.client.metadata();
if let Some(actual_hash) = address.validation_hash() {
let custom = metadata.custom();
let custom_value = custom
.get(address.name())
.ok_or_else(|| MetadataError::CustomValueNameNotFound(address.name().into()))?;
let expected_hash = custom_value.hash();
if actual_hash != expected_hash {
return Err(MetadataError::IncompatibleCodegen.into());
}
}
if metadata.custom().get(address.name()).is_none() {
return Err(MetadataError::IncompatibleCodegen.into());
}
Ok(())
}
}
#[cfg(test)]
+1 -1
View File
@@ -7,5 +7,5 @@
mod custom_value_address;
mod custom_values_client;
pub use custom_value_address::CustomValueAddress;
pub use custom_value_address::{CustomValueAddress, StaticAddress};
pub use custom_values_client::CustomValuesClient;
+76
View File
@@ -0,0 +1,76 @@
// Copyright 2019-2023 Parity Technologies (UK) Ltd.
// This file is dual-licensed as Apache-2.0 or GPL-3.0.
// see LICENSE for license details.
use codec::Encode;
use frame_metadata::v15::{CustomMetadata, ExtrinsicMetadata, OuterEnums, RuntimeMetadataV15};
use frame_metadata::RuntimeMetadataPrefixed;
use scale_info::form::PortableForm;
use scale_info::TypeInfo;
use scale_info::{meta_type, IntoPortable};
use std::collections::BTreeMap;
/// Generate metadata which contains a `Foo { a: u8, b: &str }` custom value.
pub fn metadata_custom_values_foo() -> RuntimeMetadataPrefixed {
let mut registry = scale_info::Registry::new();
// create foo value and type:
#[derive(TypeInfo, Encode)]
struct Foo {
a: u8,
b: &'static str,
}
let foo_value_metadata: frame_metadata::v15::CustomValueMetadata<PortableForm> = {
let value = Foo { a: 0, b: "Hello" };
let foo_ty = scale_info::MetaType::new::<Foo>();
let foo_ty_id = registry.register_type(&foo_ty);
frame_metadata::v15::CustomValueMetadata {
ty: foo_ty_id,
value: value.encode(),
}
};
// We don't care about the extrinsic type.
let extrinsic = ExtrinsicMetadata {
version: 0,
signed_extensions: vec![],
address_ty: meta_type::<()>(),
call_ty: meta_type::<()>(),
signature_ty: meta_type::<()>(),
extra_ty: meta_type::<()>(),
};
let pallets = vec![];
let extrinsic = extrinsic.into_portable(&mut registry);
let unit_ty = registry.register_type(&meta_type::<()>());
// Metadata needs to contain this DispatchError, since codegen looks for it.
registry.register_type(&meta_type::<crate::utils::dispatch_error::ArrayDispatchError>());
let metadata = RuntimeMetadataV15 {
types: registry.into(),
pallets,
extrinsic,
ty: unit_ty,
apis: vec![],
outer_enums: OuterEnums {
call_enum_ty: unit_ty,
event_enum_ty: unit_ty,
error_enum_ty: unit_ty,
},
custom: CustomMetadata {
// provide foo twice, to make sure nothing breaks in these cases:
map: BTreeMap::from_iter([
("Foo".into(), foo_value_metadata.clone()),
("foo".into(), foo_value_metadata.clone()),
("12".into(), foo_value_metadata.clone()),
("&Hello".into(), foo_value_metadata.clone()),
]),
},
};
RuntimeMetadataPrefixed::from(metadata)
}
+7
View File
@@ -11,6 +11,7 @@
//! Use with `TRYBUILD=overwrite` after updating codebase (see `trybuild` docs for more details on that)
//! to automatically regenerate `stderr` files, but don't forget to check that new files make sense.
mod custom_values;
mod dispatch_errors;
mod storage;
mod utils;
@@ -51,6 +52,12 @@ fn ui_tests() {
.build(dispatch_errors::metadata_array_dispatch_error()),
);
t.pass(
m.new_test_case()
.name("custom_values_foo")
.build(custom_values::metadata_custom_values_foo()),
);
// Test retaining only specific pallets and ensure that works.
for pallet in ["Babe", "Claims", "Grandpa", "Balances"] {
let mut metadata = MetadataTestRunner::load_metadata();