Codegen for custom values in metadata (#1117)

* work in progress

* add custom types access

* nit

* custom values client

* adjust light client

* adjust doc comments

* adjust book for custom values in code gen

* format and check docs

* work in progress

* add custom types access

* nit

* custom values client

* adjust light client

* codegen and validation

* adjust docs

* use ignore in docs in book

* change iter implementation

* use validation hash and other codegen changes

* add ui test for custom values codegen

* allow 'latest' metadata to be returned from the fallback code (#1127)

* nits

* fix validation check

* fix comments

* nits

---------

Co-authored-by: James Wilson <james@jsdw.me>

metadata/src/utils/validation.rs

@@ -5,14 +5,15 @@
 //! Utility functions for metadata validation.
 
 use crate::{
-    ExtrinsicMetadata, Metadata, OuterEnumsMetadata, PalletMetadata, RuntimeApiMetadata,
-    RuntimeApiMethodMetadata, StorageEntryMetadata, StorageEntryType,
+    CustomMetadata, CustomValueMetadata, ExtrinsicMetadata, Metadata, OuterEnumsMetadata,
+    PalletMetadata, RuntimeApiMetadata, RuntimeApiMethodMetadata, StorageEntryMetadata,
+    StorageEntryType,
 };
 use scale_info::{form::PortableForm, Field, PortableRegistry, TypeDef, TypeDefVariant, Variant};
 use std::collections::HashMap;
 
 // The number of bytes our `hash` function produces.
-const HASH_LEN: usize = 32;
+pub(crate) const HASH_LEN: usize = 32;
 
 /// Internal byte representation for various metadata types utilized for
 /// generating deterministic hashes between different rust versions.
@@ -67,6 +68,7 @@ concat_and_hash_n!(concat_and_hash2(a b));
 concat_and_hash_n!(concat_and_hash3(a b c));
 concat_and_hash_n!(concat_and_hash4(a b c d));
 concat_and_hash_n!(concat_and_hash5(a b c d e));
+concat_and_hash_n!(concat_and_hash6(a b c d e f));
 
 /// Obtain the hash representation of a `scale_info::Field`.
 fn get_field_hash(
@@ -393,6 +395,27 @@ pub fn get_runtime_trait_hash(trait_metadata: RuntimeApiMetadata) -> [u8; HASH_L
     concat_and_hash2(&hash(trait_name.as_bytes()), &method_bytes)
 }
 
+pub fn get_custom_metadata_hash(custom_metadata: &CustomMetadata) -> [u8; HASH_LEN] {
+    let mut cache = HashMap::new();
+    custom_metadata
+        .iter()
+        .fold([0u8; HASH_LEN], |bytes, custom_value| {
+            xor(bytes, get_custom_value_hash(&custom_value, &mut cache))
+        })
+}
+
+/// Obtain the hash of some custom value in the metadata including it's name/key.
+pub fn get_custom_value_hash(
+    custom_value: &CustomValueMetadata,
+    cache: &mut HashMap<u32, CachedHash>,
+) -> [u8; HASH_LEN] {
+    concat_and_hash3(
+        &hash(custom_value.name.as_bytes()),
+        &get_type_hash(custom_value.types, custom_value.type_id(), cache),
+        &hash(custom_value.bytes()),
+    )
+}
+
 /// Obtain the hash for a specific storage item, or an error if it's not found.
 pub fn get_storage_hash(pallet: &PalletMetadata, entry_name: &str) -> Option<[u8; HASH_LEN]> {
     let storage = pallet.storage()?;
@@ -494,6 +517,7 @@ pub struct MetadataHasher<'a> {
     metadata: &'a Metadata,
     specific_pallets: Option<Vec<&'a str>>,
     specific_runtime_apis: Option<Vec<&'a str>>,
+    include_custom_values: bool,
 }
 
 impl<'a> MetadataHasher<'a> {
@@ -503,6 +527,7 @@ impl<'a> MetadataHasher<'a> {
             metadata,
             specific_pallets: None,
             specific_runtime_apis: None,
+            include_custom_values: true,
         }
     }
 
@@ -522,6 +547,12 @@ impl<'a> MetadataHasher<'a> {
         self
     }
 
+    /// Do not hash the custom values
+    pub fn ignore_custom_values(&mut self) -> &mut Self {
+        self.include_custom_values = false;
+        self
+    }
+
     /// Hash the given metadata.
     pub fn hash(&self) -> [u8; HASH_LEN] {
         let metadata = self.metadata;
@@ -554,7 +585,7 @@ impl<'a> MetadataHasher<'a> {
                 // We don't care what order the runtime APIs are seen in, so XOR their
                 // hashes together to be order independent.
                 if should_hash {
-                    xor(bytes, xor(bytes, get_runtime_trait_hash(api)))
+                    xor(bytes, get_runtime_trait_hash(api))
                 } else {
                     bytes
                 }
@@ -569,12 +600,18 @@ impl<'a> MetadataHasher<'a> {
             self.specific_pallets.as_deref(),
         );
 
-        concat_and_hash5(
+        let custom_values_hash = self
+            .include_custom_values
+            .then(|| get_custom_metadata_hash(&metadata.custom()))
+            .unwrap_or_default();
+
+        concat_and_hash6(
             &pallet_hash,
             &apis_hash,
             &extrinsic_hash,
             &runtime_hash,
             &outer_enums_hash,
+            &custom_values_hash,
         )
     }
 }
@@ -811,8 +848,12 @@ mod tests {
         let a_hash2 = get_type_hash(&registry, a_type_id, &mut cache);
         let b_hash = get_type_hash(&registry, b_type_id, &mut cache);
 
-        let CachedHash::Hash(a_cache_hash) = cache[&a_type_id] else { panic!() };
-        let CachedHash::Hash(b_cache_hash) = cache[&b_type_id] else { panic!() };
+        let CachedHash::Hash(a_cache_hash) = cache[&a_type_id] else {
+            panic!()
+        };
+        let CachedHash::Hash(b_cache_hash) = cache[&b_type_id] else {
+            panic!()
+        };
 
         assert_eq!(a_hash, a_cache_hash);
         assert_eq!(b_hash, b_cache_hash);

metadata/src/utils/variant_index.rs

@@ -21,7 +21,7 @@ impl VariantIndex {
     /// Build indexes from the optional variant ID.
     pub fn build(variant_id: Option<u32>, types: &PortableRegistry) -> Self {
         let Some(variants) = Self::get(variant_id, types) else {
-            return Self::empty()
+            return Self::empty();
         };
 
         let mut by_name = HashMap::new();
@@ -47,11 +47,9 @@ impl VariantIndex {
         variant_id: Option<u32>,
         types: &PortableRegistry,
     ) -> Option<&[Variant<PortableForm>]> {
-        let Some(variant_id) = variant_id else {
-            return None
-        };
+        let variant_id = variant_id?;
         let TypeDef::Variant(v) = &types.resolve(variant_id)?.type_def else {
-            return None
+            return None;
         };
         Some(&v.variants)
     }