From ceb9e2161c160afac1a0188daf15da4c66ffb98b Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Tue, 5 Jan 2021 14:45:16 +0100 Subject: [PATCH] Parachains well known keys and abridged primitives (#2194) * Add well_known_keys * Reorder HrmpChannel and HostConfiguration members * abridged versions and well known keys tests * Add some comments * Add a note on generation of the prefixes and other magic values * Recommend accessing the well known values through abridged structs --- polkadot/Cargo.lock | 2 + polkadot/primitives/Cargo.toml | 3 + polkadot/primitives/src/v1.rs | 138 ++++++++++++++ .../runtime/parachains/src/configuration.rs | 177 ++++++++++++------ polkadot/runtime/parachains/src/hrmp.rs | 76 +++++++- polkadot/runtime/parachains/src/ump.rs | 27 +++ 6 files changed, 360 insertions(+), 63 deletions(-) diff --git a/polkadot/Cargo.lock b/polkadot/Cargo.lock index 2bb7c933b6..2cebe26d7d 100644 --- a/polkadot/Cargo.lock +++ b/polkadot/Cargo.lock @@ -5382,6 +5382,7 @@ version = "0.8.27" dependencies = [ "bitvec", "frame-system", + "hex-literal", "parity-scale-codec", "polkadot-core-primitives", "polkadot-parachain", @@ -5393,6 +5394,7 @@ dependencies = [ "sp-authority-discovery", "sp-core", "sp-inherents", + "sp-io", "sp-keystore", "sp-runtime", "sp-serializer", diff --git a/polkadot/primitives/Cargo.toml b/polkadot/primitives/Cargo.toml index efe65aec09..08fbfd999d 100644 --- a/polkadot/primitives/Cargo.toml +++ b/polkadot/primitives/Cargo.toml @@ -14,6 +14,7 @@ sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste sp-api = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-version = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-std = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +sp-io = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" } sp-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-arithmetic = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } @@ -23,6 +24,7 @@ polkadot-core-primitives = { path = "../core-primitives", default-features = fal trie = { package = "sp-trie", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } bitvec = { version = "0.17.4", default-features = false, features = ["alloc"] } frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +hex-literal = "0.3.1" [dev-dependencies] sp-serializer = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -40,6 +42,7 @@ std = [ "sp-authority-discovery/std", "sp-keystore", "sp-std/std", + "sp-io/std", "sp-version/std", "sp-staking/std", "sp-arithmetic/std", diff --git a/polkadot/primitives/src/v1.rs b/polkadot/primitives/src/v1.rs index 6104d6aae2..5a3d25efee 100644 --- a/polkadot/primitives/src/v1.rs +++ b/polkadot/primitives/src/v1.rs @@ -55,6 +55,84 @@ pub use crate::v0::{ValidatorPair, CollatorPair}; pub use sp_staking::SessionIndex; pub use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; +/// A declarations of storage keys where an external observer can find some interesting data. +pub mod well_known_keys { + use super::{Id, HrmpChannelId}; + use hex_literal::hex; + use sp_io::hashing::twox_64; + use sp_std::prelude::*; + use parity_scale_codec::Encode as _; + + // A note on generating these magic values below: + // + // The `StorageValue`, such as `ACTIVE_CONFIG` was obtained by calling: + // + // ::ActiveConfig::hashed_key() + // + // The `StorageMap` values require `prefix`, and for example for `hrmp_egress_channel_index`, + // it could be obtained like: + // + // ::HrmpEgressChannelsIndex::prefix_hash(); + // + + /// The currently active host configuration. + /// + /// The storage entry should be accessed as an `AbridgedHostConfiguration` encoded value. + pub const ACTIVE_CONFIG: &[u8] = + &hex!["06de3d8a54d27e44a9d5ce189618f22db4b49d95320d9021994c850f25b8e385"]; + + /// The upward message dispatch queue for the given para id. + /// + /// The storage entry stores a tuple of two values: + /// + /// - `count: u32`, the number of messages currently in the queue for given para, + /// - `total_size: u32`, the total size of all messages in the queue. + pub fn relay_dispatch_queue_size(para_id: Id) -> Vec { + let prefix = hex!["f5207f03cfdce586301014700e2c2593fad157e461d71fd4c1f936839a5f1f3e"]; + + para_id.using_encoded(|para_id: &[u8]| { + prefix.as_ref() + .iter() + .chain(twox_64(para_id).iter()) + .chain(para_id.iter()) + .cloned() + .collect() + }) + } + + /// The hrmp channel for the given identifier. + /// + /// The storage entry should be accessed as an `AbridgedHrmpChannel` encoded value. + pub fn hrmp_channels(channel: HrmpChannelId) -> Vec { + let prefix = hex!["6a0da05ca59913bc38a8630590f2627cb6604cff828a6e3f579ca6c59ace013d"]; + + channel.using_encoded(|channel: &[u8]| { + prefix.as_ref() + .iter() + .chain(twox_64(channel).iter()) + .chain(channel.iter()) + .cloned() + .collect() + }) + } + + /// The list of outbound channels for the given para. + /// + /// The storage entry stores a `Vec` + pub fn hrmp_egress_channel_index(para_id: Id) -> Vec { + let prefix = hex!["6a0da05ca59913bc38a8630590f2627cf12b746dcf32e843354583c9702cc020"]; + + para_id.using_encoded(|para_id: &[u8]| { + prefix.as_ref() + .iter() + .chain(twox_64(para_id).iter()) + .chain(para_id.iter()) + .cloned() + .collect() + }) + } +} + /// Unique identifier for the Inclusion Inherent pub const INCLUSION_INHERENT_IDENTIFIER: InherentIdentifier = *b"inclusn0"; @@ -844,6 +922,66 @@ impl From for u8 { } } +/// Abridged version of `HostConfiguration` (from the `Configuration` parachains host runtime module) +/// meant to be used by a parachain or PDK such as cumulus. +#[derive(Clone, Encode, Decode, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(PartialEq))] +pub struct AbridgedHostConfiguration { + /// The maximum validation code size, in bytes. + pub max_code_size: u32, + /// The maximum head-data size, in bytes. + pub max_head_data_size: u32, + /// Total number of individual messages allowed in the parachain -> relay-chain message queue. + pub max_upward_queue_count: u32, + /// Total size of messages allowed in the parachain -> relay-chain message queue before which + /// no further messages may be added to it. If it exceeds this then the queue may contain only + /// a single message. + pub max_upward_queue_size: u32, + /// The maximum size of an upward message that can be sent by a candidate. + /// + /// This parameter affects the size upper bound of the `CandidateCommitments`. + pub max_upward_message_size: u32, + /// The maximum number of messages that a candidate can contain. + /// + /// This parameter affects the size upper bound of the `CandidateCommitments`. + pub max_upward_message_num_per_candidate: u32, + /// The maximum number of outbound HRMP messages can be sent by a candidate. + /// + /// This parameter affects the upper bound of size of `CandidateCommitments`. + pub hrmp_max_message_num_per_candidate: u32, + /// The minimum frequency at which parachains can update their validation code. + pub validation_upgrade_frequency: BlockNumber, + /// The delay, in blocks, before a validation upgrade is applied. + pub validation_upgrade_delay: BlockNumber, +} + +/// Abridged version of `HrmpChannel` (from the `Hrmp` parachains host runtime module) meant to be +/// used by a parachain or PDK such as cumulus. +#[derive(Clone, Encode, Decode, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(PartialEq))] +pub struct AbridgedHrmpChannel { + /// The maximum number of messages that can be pending in the channel at once. + pub max_capacity: u32, + /// The maximum total size of the messages that can be pending in the channel at once. + pub max_total_size: u32, + /// The maximum message size that could be put into the channel. + pub max_message_size: u32, + /// The current number of messages pending in the channel. + /// Invariant: should be less or equal to `max_capacity`.s`. + pub msg_count: u32, + /// The total size in bytes of all message payloads in the channel. + /// Invariant: should be less or equal to `max_total_size`. + pub total_size: u32, + /// A head of the Message Queue Chain for this channel. Each link in this chain has a form: + /// `(prev_head, B, H(M))`, where + /// - `prev_head`: is the previous value of `mqc_head` or zero if none. + /// - `B`: is the [relay-chain] block number in which a message was appended + /// - `H(M)`: is the hash of the message being appended. + /// This value is initialized to a special value that consists of all zeroes which indicates + /// that no messages were previously added. + pub mqc_head: Option, +} + #[cfg(test)] mod tests { use super::*; diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index af693ed676..847a16b11c 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -34,19 +34,92 @@ use sp_runtime::traits::Zero; #[derive(Clone, Encode, Decode, PartialEq, sp_core::RuntimeDebug)] #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub struct HostConfiguration { - /// The minimum frequency at which parachains can update their validation code. - pub validation_upgrade_frequency: BlockNumber, - /// The delay, in blocks, before a validation upgrade is applied. - pub validation_upgrade_delay: BlockNumber, - /// The acceptance period, in blocks. This is the amount of blocks after availability that validators - /// and fishermen have to perform secondary checks or issue reports. - pub acceptance_period: BlockNumber, + // NOTE: This structure is used by parachains via merkle proofs. Therefore, this struct requires + // special treatment. + // + // A parachain requested this struct can only depend on the subset of this struct. Specifically, + // only a first few fields can be depended upon. These fields cannot be changed without + // corresponding migration of the parachains. + + /** + * The parameters that are required for the parachains. + */ + /// The maximum validation code size, in bytes. pub max_code_size: u32, /// The maximum head-data size, in bytes. pub max_head_data_size: u32, + /// Total number of individual messages allowed in the parachain -> relay-chain message queue. + pub max_upward_queue_count: u32, + /// Total size of messages allowed in the parachain -> relay-chain message queue before which + /// no further messages may be added to it. If it exceeds this then the queue may contain only + /// a single message. + pub max_upward_queue_size: u32, + /// The maximum size of an upward message that can be sent by a candidate. + /// + /// This parameter affects the size upper bound of the `CandidateCommitments`. + pub max_upward_message_size: u32, + /// The maximum number of messages that a candidate can contain. + /// + /// This parameter affects the size upper bound of the `CandidateCommitments`. + pub max_upward_message_num_per_candidate: u32, + /// The maximum number of outbound HRMP messages can be sent by a candidate. + /// + /// This parameter affects the upper bound of size of `CandidateCommitments`. + pub hrmp_max_message_num_per_candidate: u32, + /// The minimum frequency at which parachains can update their validation code. + pub validation_upgrade_frequency: BlockNumber, + /// The delay, in blocks, before a validation upgrade is applied. + pub validation_upgrade_delay: BlockNumber, + + /** + * The parameters that are not essential, but still may be of interest for parachains. + */ + /// The maximum POV block size, in bytes. pub max_pov_size: u32, + /// The maximum size of a message that can be put in a downward message queue. + /// + /// Since we require receiving at least one DMP message the obvious upper bound of the size is + /// the PoV size. Of course, there is a lot of other different things that a parachain may + /// decide to do with its PoV so this value in practice will be picked as a fraction of the PoV + /// size. + pub max_downward_message_size: u32, + /// The amount of weight we wish to devote to the processing the dispatchable upward messages + /// stage. + /// + /// NOTE that this is a soft limit and could be exceeded. + pub preferred_dispatchable_upward_messages_step_weight: Weight, + /// The maximum number of outbound HRMP channels a parachain is allowed to open. + pub hrmp_max_parachain_outbound_channels: u32, + /// The maximum number of outbound HRMP channels a parathread is allowed to open. + pub hrmp_max_parathread_outbound_channels: u32, + /// Number of sessions after which an HRMP open channel request expires. + pub hrmp_open_request_ttl: u32, + /// The deposit that the sender should provide for opening an HRMP channel. + pub hrmp_sender_deposit: Balance, + /// The deposit that the recipient should provide for accepting opening an HRMP channel. + pub hrmp_recipient_deposit: Balance, + /// The maximum number of messages allowed in an HRMP channel at once. + pub hrmp_channel_max_capacity: u32, + /// The maximum total size of messages in bytes allowed in an HRMP channel at once. + pub hrmp_channel_max_total_size: u32, + /// The maximum number of inbound HRMP channels a parachain is allowed to accept. + pub hrmp_max_parachain_inbound_channels: u32, + /// The maximum number of inbound HRMP channels a parathread is allowed to accept. + pub hrmp_max_parathread_inbound_channels: u32, + /// The maximum size of a message that could ever be put into an HRMP channel. + /// + /// This parameter affects the upper bound of size of `CandidateCommitments`. + pub hrmp_channel_max_message_size: u32, + + /** + * Parameters that will unlikely be needed by parachains. + */ + + /// The acceptance period, in blocks. This is the amount of blocks after availability that validators + /// and fishermen have to perform secondary checks or issue reports. + pub acceptance_period: BlockNumber, /// The amount of execution cores to dedicate to parathread execution. pub parathread_cores: u32, /// The number of retries that a parathread author has to submit their block. @@ -88,58 +161,6 @@ pub struct HostConfiguration { pub needed_approvals: u32, /// The number of samples to do of the RelayVRFModulo approval assignment criterion. pub relay_vrf_modulo_samples: u32, - /// Total number of individual messages allowed in the parachain -> relay-chain message queue. - pub max_upward_queue_count: u32, - /// Total size of messages allowed in the parachain -> relay-chain message queue before which - /// no further messages may be added to it. If it exceeds this then the queue may contain only - /// a single message. - pub max_upward_queue_size: u32, - /// The maximum size of a message that can be put in a downward message queue. - /// - /// Since we require receiving at least one DMP message the obvious upper bound of the size is - /// the PoV size. Of course, there is a lot of other different things that a parachain may - /// decide to do with its PoV so this value in practice will be picked as a fraction of the PoV - /// size. - pub max_downward_message_size: u32, - /// The amount of weight we wish to devote to the processing the dispatchable upward messages - /// stage. - /// - /// NOTE that this is a soft limit and could be exceeded. - pub preferred_dispatchable_upward_messages_step_weight: Weight, - /// The maximum size of an upward message that can be sent by a candidate. - /// - /// This parameter affects the size upper bound of the `CandidateCommitments`. - pub max_upward_message_size: u32, - /// The maximum number of messages that a candidate can contain. - /// - /// This parameter affects the size upper bound of the `CandidateCommitments`. - pub max_upward_message_num_per_candidate: u32, - /// Number of sessions after which an HRMP open channel request expires. - pub hrmp_open_request_ttl: u32, - /// The deposit that the sender should provide for opening an HRMP channel. - pub hrmp_sender_deposit: Balance, - /// The deposit that the recipient should provide for accepting opening an HRMP channel. - pub hrmp_recipient_deposit: Balance, - /// The maximum number of messages allowed in an HRMP channel at once. - pub hrmp_channel_max_capacity: u32, - /// The maximum total size of messages in bytes allowed in an HRMP channel at once. - pub hrmp_channel_max_total_size: u32, - /// The maximum number of inbound HRMP channels a parachain is allowed to accept. - pub hrmp_max_parachain_inbound_channels: u32, - /// The maximum number of inbound HRMP channels a parathread is allowed to accept. - pub hrmp_max_parathread_inbound_channels: u32, - /// The maximum size of a message that could ever be put into an HRMP channel. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub hrmp_channel_max_message_size: u32, - /// The maximum number of outbound HRMP channels a parachain is allowed to open. - pub hrmp_max_parachain_outbound_channels: u32, - /// The maximum number of outbound HRMP channels a parathread is allowed to open. - pub hrmp_max_parathread_outbound_channels: u32, - /// The maximum number of outbound HRMP messages can be sent by a candidate. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub hrmp_max_message_num_per_candidate: u32, } impl> Default for HostConfiguration { @@ -857,4 +878,42 @@ mod tests { assert!(::PendingConfig::get().is_none()) }); } + + #[test] + fn verify_externally_accessible() { + // This test verifies that the value can be accessed through the well known keys and the + // host configuration decodes into the abridged version. + + use primitives::v1::{well_known_keys, AbridgedHostConfiguration}; + + new_test_ext(Default::default()).execute_with(|| { + let ground_truth = HostConfiguration::default(); + + // Make sure that the configuration is stored in the storage. + ::ActiveConfig::put(ground_truth.clone()); + + // Extract the active config via the well known key. + let raw_active_config = sp_io::storage::get(well_known_keys::ACTIVE_CONFIG) + .expect("config must be present in storage under ACTIVE_CONFIG"); + let abridged_config = AbridgedHostConfiguration::decode(&mut &raw_active_config[..]) + .expect("HostConfiguration must be decodable into AbridgedHostConfiguration"); + + assert_eq!( + abridged_config, + AbridgedHostConfiguration { + max_code_size: ground_truth.max_code_size, + max_head_data_size: ground_truth.max_head_data_size, + max_upward_queue_count: ground_truth.max_upward_queue_count, + max_upward_queue_size: ground_truth.max_upward_queue_size, + max_upward_message_size: ground_truth.max_upward_message_size, + max_upward_message_num_per_candidate: ground_truth + .max_upward_message_num_per_candidate, + hrmp_max_message_num_per_candidate: ground_truth + .hrmp_max_message_num_per_candidate, + validation_upgrade_frequency: ground_truth.validation_upgrade_frequency, + validation_upgrade_delay: ground_truth.validation_upgrade_delay, + }, + ); + }); + } } diff --git a/polkadot/runtime/parachains/src/hrmp.rs b/polkadot/runtime/parachains/src/hrmp.rs index 8021ebf2e3..eb0e9aa603 100644 --- a/polkadot/runtime/parachains/src/hrmp.rs +++ b/polkadot/runtime/parachains/src/hrmp.rs @@ -56,10 +56,13 @@ pub struct HrmpOpenChannelRequest { #[derive(Encode, Decode)] #[cfg_attr(test, derive(Debug))] pub struct HrmpChannel { - /// The amount that the sender supplied as a deposit when opening this channel. - pub sender_deposit: Balance, - /// The amount that the recipient supplied as a deposit when accepting opening this channel. - pub recipient_deposit: Balance, + // NOTE: This structure is used by parachains via merkle proofs. Therefore, this struct requires + // special treatment. + // + // A parachain requested this struct can only depend on the subset of this struct. Specifically, + // only a first few fields can be depended upon (See `AbridgedHrmpChannel`). These fields cannot + // be changed without corresponding migration of parachains. + /// The maximum number of messages that can be pending in the channel at once. pub max_capacity: u32, /// The maximum total size of the messages that can be pending in the channel at once. @@ -80,6 +83,10 @@ pub struct HrmpChannel { /// This value is initialized to a special value that consists of all zeroes which indicates /// that no messages were previously added. pub mqc_head: Option, + /// The amount that the sender supplied as a deposit when opening this channel. + pub sender_deposit: Balance, + /// The amount that the recipient supplied as a deposit when accepting opening this channel. + pub recipient_deposit: Balance, } /// An error returned by [`check_hrmp_watermark`] that indicates an acceptance criteria check @@ -270,7 +277,10 @@ decl_storage! { /// - there should be no other dangling channels in `HrmpChannels`. /// - the vectors are sorted. HrmpIngressChannelsIndex: map hasher(twox_64_concat) ParaId => Vec; + // NOTE that this field is used by parachains via merkle storage proofs, therefore changing + // the format will require migration of parachains. HrmpEgressChannelsIndex: map hasher(twox_64_concat) ParaId => Vec; + /// Storage for the messages for each channel. /// Invariant: cannot be non-empty if the corresponding channel in `HrmpChannels` is `None`. HrmpChannelContents: map hasher(twox_64_concat) HrmpChannelId => Vec>; @@ -1554,4 +1564,62 @@ mod tests { assert_storage_consistency_exhaustive(); }); } + + #[test] + fn verify_externally_accessible() { + use primitives::v1::{well_known_keys, AbridgedHrmpChannel}; + + let para_a = 20.into(); + let para_b = 21.into(); + + new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { + // Register two parachains, wait until a session change, then initiate channel open + // request and accept that, and finally wait until the next session. + register_parachain(para_a); + register_parachain(para_b); + run_to_block(5, Some(vec![5])); + Hrmp::init_open_channel(para_a, para_b, 2, 8).unwrap(); + Hrmp::accept_open_channel(para_b, para_a).unwrap(); + run_to_block(8, Some(vec![8])); + + // Here we have a channel a->b opened. + // + // Try to obtain this channel from the storage and + // decode it into the abridged version. + assert!(channel_exists(para_a, para_b)); + let raw_hrmp_channel = + sp_io::storage::get(&well_known_keys::hrmp_channels(HrmpChannelId { + sender: para_a, + recipient: para_b, + })) + .expect("the channel exists and we must be able to get it through well known keys"); + let abridged_hrmp_channel = AbridgedHrmpChannel::decode(&mut &raw_hrmp_channel[..]) + .expect("HrmpChannel should be decodable as AbridgedHrmpChannel"); + + assert_eq!( + abridged_hrmp_channel, + AbridgedHrmpChannel { + max_capacity: 2, + max_total_size: 16, + max_message_size: 8, + msg_count: 0, + total_size: 0, + mqc_head: None, + }, + ); + + // Now, verify that we can access and decode the egress index. + let raw_egress_index = + sp_io::storage::get( + &well_known_keys::hrmp_egress_channel_index(para_a) + ) + .expect("the egress index must be present for para_a"); + let egress_index = >::decode(&mut &raw_egress_index[..]) + .expect("egress index should be decodable as a list of para ids"); + assert_eq!( + egress_index, + vec![para_b], + ); + }); + } } diff --git a/polkadot/runtime/parachains/src/ump.rs b/polkadot/runtime/parachains/src/ump.rs index 8974f9c9ff..35c0188c5b 100644 --- a/polkadot/runtime/parachains/src/ump.rs +++ b/polkadot/runtime/parachains/src/ump.rs @@ -170,6 +170,8 @@ decl_storage! { /// /// Invariant: /// - The set of keys should exactly match the set of keys of `RelayDispatchQueues`. + // NOTE that this field is used by parachains via merkle storage proofs, therefore changing + // the format will require migration of parachains. RelayDispatchQueueSize: map hasher(twox_64_concat) ParaId => (u32, u32); /// The ordered list of `ParaId`s that have a `RelayDispatchQueue` entry. /// @@ -907,4 +909,29 @@ mod tests { } }); } + + #[test] + fn verify_relay_dispatch_queue_size_is_externally_accessible() { + // Make sure that the relay dispatch queue size storage entry is accessible via well known + // keys and is decodable into a (u32, u32). + + use primitives::v1::well_known_keys; + use parity_scale_codec::Decode as _; + + let a = ParaId::from(228); + let msg = vec![1, 2, 3]; + + new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { + queue_upward_msg(a, msg); + + let raw_queue_size = sp_io::storage::get(&well_known_keys::relay_dispatch_queue_size(a)) + .expect("enqueing a message should create the dispatch queue\ + and it should be accessible via the well known keys"); + let (cnt, size) = <(u32, u32)>::decode(&mut &raw_queue_size[..]) + .expect("the dispatch queue size should be decodable into (u32, u32)"); + + assert_eq!(cnt, 1); + assert_eq!(size, 3); + }); + } }