From 33b435894ae3ae206e1418030e91d00144557f60 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Fri, 26 Feb 2021 15:41:23 +0000 Subject: [PATCH] Better identifier and logging for runtime upgrades (#8123) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * A clean new attempt * Checkpoint to move remote. * A lot of dependency wiring to make it feature gated. * bad macro, bad macro. * Undo the DB mess. * Update frame/support/src/traits.rs Co-authored-by: Alexander Popiak * Apply suggestions from code review Co-authored-by: Alexander Popiak * unbreak the build * Better logging and ids for migrations * Fix doc. * Test * Update frame/try-runtime/src/lib.rs Co-authored-by: Bastian Köcher * Update utils/frame/try-runtime/cli/Cargo.toml Co-authored-by: Shawn Tabrizi * Update frame/try-runtime/Cargo.toml Co-authored-by: Shawn Tabrizi * Address most review grumbles. * Fix build * Add some comments * Remove allowing one pallet at a time. * Rework the PR * nit * Slightly better error handling. * Remove files * Update utils/frame/remote-externalities/src/lib.rs Co-authored-by: Bastian Köcher * Update frame/support/src/dispatch.rs * Update frame/support/src/dispatch.rs * Fix test * Make extension trait. * Bring back try-runtime/std * remove bincode * Remove warning * Change test features Co-authored-by: Alexander Popiak Co-authored-by: Bastian Köcher Co-authored-by: Shawn Tabrizi --- substrate/Cargo.lock | 4 +- substrate/bin/node/cli/src/command.rs | 2 +- substrate/frame/support/procedural/src/lib.rs | 2 +- substrate/frame/support/src/dispatch.rs | 110 ++++++++++------ substrate/frame/support/src/lib.rs | 3 + substrate/frame/support/src/traits.rs | 48 +++++++ substrate/primitives/storage/src/lib.rs | 4 +- .../frame/remote-externalities/Cargo.toml | 4 +- .../frame/remote-externalities/proxy_test | Bin 26476 -> 0 bytes .../frame/remote-externalities/src/lib.rs | 123 +++++++++--------- .../utils/frame/try-runtime/cli/src/lib.rs | 2 +- 11 files changed, 194 insertions(+), 108 deletions(-) delete mode 100644 substrate/utils/frame/remote-externalities/proxy_test diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 5a1f7bf670..7cb69617b1 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -6436,12 +6436,12 @@ name = "remote-externalities" version = "0.9.0" dependencies = [ "async-std", - "bincode", "env_logger 0.8.2", - "futures 0.1.30", + "futures 0.3.12", "hex-literal", "jsonrpc-core-client", "log", + "parity-scale-codec", "sc-rpc", "sc-rpc-api", "sp-core", diff --git a/substrate/bin/node/cli/src/command.rs b/substrate/bin/node/cli/src/command.rs index d3689bdcd6..ece97436bf 100644 --- a/substrate/bin/node/cli/src/command.rs +++ b/substrate/bin/node/cli/src/command.rs @@ -159,7 +159,7 @@ pub fn run() -> Result<()> { let task_manager = sc_service::TaskManager::new( config.task_executor.clone(), registry, - ).unwrap(); + ).map_err(|e| sc_cli::Error::Service(sc_service::Error::Prometheus(e)))?; Ok((cmd.run::(config), task_manager)) }) diff --git a/substrate/frame/support/procedural/src/lib.rs b/substrate/frame/support/procedural/src/lib.rs index 2c2cdf00a0..e64a364d29 100644 --- a/substrate/frame/support/procedural/src/lib.rs +++ b/substrate/frame/support/procedural/src/lib.rs @@ -17,7 +17,7 @@ //! Proc macro of Support code for the runtime. -#![recursion_limit="512"] +#![recursion_limit = "512"] mod storage; mod construct_runtime; diff --git a/substrate/frame/support/src/dispatch.rs b/substrate/frame/support/src/dispatch.rs index f5c8f017e3..c2315e66e3 100644 --- a/substrate/frame/support/src/dispatch.rs +++ b/substrate/frame/support/src/dispatch.rs @@ -80,18 +80,18 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// // FRAME pallets. /// #[weight = 0] /// fn my_function(origin, var: u64) -> dispatch::DispatchResult { -/// // Your implementation -/// Ok(()) +/// // Your implementation +/// Ok(()) /// } /// -/// // Public functions are both dispatchable and available to other +/// // Public functions are both dispatchable and available to other /// // FRAME pallets. /// #[weight = 0] -/// pub fn my_public_function(origin) -> dispatch::DispatchResult { +/// pub fn my_public_function(origin) -> dispatch::DispatchResult { /// // Your implementation -/// Ok(()) +/// Ok(()) +/// } /// } -/// } /// } /// # fn main() {} /// ``` @@ -99,8 +99,10 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// The declaration is set with the header where: /// /// * `Module`: The struct generated by the macro, with type `Config`. -/// * `Call`: The enum generated for every pallet, which implements [`Callable`](./dispatch/trait.Callable.html). -/// * `origin`: Alias of `T::Origin`, declared by the [`impl_outer_origin!`](./macro.impl_outer_origin.html) macro. +/// * `Call`: The enum generated for every pallet, which implements +/// [`Callable`](./dispatch/trait.Callable.html). +/// * `origin`: Alias of `T::Origin`, declared by the +/// [`impl_outer_origin!`](./macro.impl_outer_origin.html) macro. /// * `Result`: The expected return type from pallet functions. /// /// The first parameter of dispatchable functions must always be `origin`. @@ -119,15 +121,15 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// pub struct Module for enum Call where origin: T::Origin { /// #[weight = 0] /// fn my_long_function(origin) -> dispatch::DispatchResult { -/// // Your implementation +/// // Your implementation /// Ok(()) /// } /// /// #[weight = 0] /// fn my_short_function(origin) { -/// // Your implementation +/// // Your implementation +/// } /// } -/// } /// } /// # fn main() {} /// ``` @@ -184,7 +186,7 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// #[weight = 0] /// #[transactional] /// fn my_short_function(origin) { -/// // Your implementation +/// // Your implementation /// } /// } /// } @@ -203,12 +205,12 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { /// #[weight = 0] -/// fn my_privileged_function(origin) -> dispatch::DispatchResult { +/// fn my_privileged_function(origin) -> dispatch::DispatchResult { /// ensure_root(origin)?; -/// // Your implementation +/// // Your implementation /// Ok(()) /// } -/// } +/// } /// } /// # fn main() {} /// ``` @@ -218,15 +220,17 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// Attributes on functions are supported, but must be in the order of: /// 1. Optional #\[doc\] attribute. /// 2. #\[weight\] attribute. -/// 3. Optional function attributes, for instance #\[transactional\]. Those function attributes will be written -/// only on the dispatchable functions implemented on `Module`, not on the `Call` enum variant. +/// 3. Optional function attributes, for instance #\[transactional\]. Those function attributes will +/// be written only on the dispatchable functions implemented on `Module`, not on the `Call` enum +/// variant. /// /// ## Multiple Module Instances Example /// -/// A Substrate module can be built such that multiple instances of the same module can be used within a single -/// runtime. For example, the [Balances module](../pallet_balances/index.html) can be added multiple times to your -/// runtime in order to support multiple, independent currencies for your blockchain. Here is an example of how -/// you would declare such a module using the `decl_module!` macro: +/// A Substrate module can be built such that multiple instances of the same module can be used +/// within a single runtime. For example, the [Balances module](../pallet_balances/index.html) can +/// be added multiple times to your runtime in order to support multiple, independent currencies for +/// your blockchain. Here is an example of how you would declare such a module using the +/// `decl_module!` macro: /// /// ``` /// # #[macro_use] @@ -251,10 +255,10 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// /// ## Where clause /// -/// Besides the default `origin: T::Origin`, you can also pass other bounds to the module declaration. -/// This where bound will be replicated to all types generated by this macro. The chaining of multiple -/// trait bounds with `+` is not supported. If multiple bounds for one type are required, it needs to -/// be split up into multiple bounds. +/// Besides the default `origin: T::Origin`, you can also pass other bounds to the module +/// declaration. This where bound will be replicated to all types generated by this macro. The +/// chaining of multiple trait bounds with `+` is not supported. If multiple bounds for one type are +/// required, it needs to be split up into multiple bounds. /// /// ``` /// # #[macro_use] @@ -276,16 +280,18 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// The following are reserved function signatures: /// /// * `deposit_event`: Helper function for depositing an [event](https://docs.substrate.dev/docs/event-enum). -/// The default behavior is to call `deposit_event` from the [System module](../frame_system/index.html). -/// However, you can write your own implementation for events in your runtime. To use the default behavior, -/// add `fn deposit_event() = default;` to your `Module`. +/// The default behavior is to call `deposit_event` from the [System +/// module](../frame_system/index.html). However, you can write your own implementation for events +/// in your runtime. To use the default behavior, add `fn deposit_event() = default;` to your +/// `Module`. /// -/// The following reserved functions also take the block number (with type `T::BlockNumber`) as an optional input: +/// The following reserved functions also take the block number (with type `T::BlockNumber`) as an +/// optional input: /// /// * `on_runtime_upgrade`: Executes at the beginning of a block prior to on_initialize when there -/// is a runtime upgrade. This allows each module to upgrade its storage before the storage items are used. -/// As such, **calling other modules must be avoided**!! Using this function will implement the -/// [`OnRuntimeUpgrade`](../sp_runtime/traits/trait.OnRuntimeUpgrade.html) trait. +/// is a runtime upgrade. This allows each module to upgrade its storage before the storage items +/// are used. As such, **calling other modules must be avoided**!! Using this function will +/// implement the [`OnRuntimeUpgrade`](../sp_runtime/traits/trait.OnRuntimeUpgrade.html) trait. /// Function signature must be `fn on_runtime_upgrade() -> frame_support::weights::Weight`. /// /// * `on_initialize`: Executes at the beginning of a block. Using this function will @@ -300,11 +306,11 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// * `fn on_finalize(n: BlockNumber) -> frame_support::weights::Weight` or /// * `fn on_finalize() -> frame_support::weights::Weight` /// -/// * `offchain_worker`: Executes at the beginning of a block and produces extrinsics for a future block -/// upon completion. Using this function will implement the +/// * `offchain_worker`: Executes at the beginning of a block and produces extrinsics for a future +/// block upon completion. Using this function will implement the /// [`OffchainWorker`](./traits/trait.OffchainWorker.html) trait. -/// * `integrity_test`: Executes in a test generated by `construct_runtime`, note it doesn't -/// execute in an externalities-provided environment. Implement +/// * `integrity_test`: Executes in a test generated by `construct_runtime`, note it doesn't execute +/// in an externalities-provided environment. Implement /// [`IntegrityTest`](./trait.IntegrityTest.html) trait. #[macro_export] macro_rules! decl_module { @@ -1325,13 +1331,27 @@ macro_rules! decl_module { $crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!("on_runtime_upgrade")); let result: $return = (|| { $( $impl )* })(); - $crate::crate_to_pallet_version!() + let new_storage_version = $crate::crate_to_pallet_version!(); + new_storage_version .put_into_storage::<<$trait_instance as $system::Config>::PalletInfo, Self>(); let additional_write = < <$trait_instance as $system::Config>::DbWeight as $crate::traits::Get<_> >::get().writes(1); + let pallet_name = << + $trait_instance + as + $system::Config + >::PalletInfo as $crate::traits::PalletInfo>::name::().expect("pallet will have name in the runtime; qed"); + + $crate::debug::info!( + target: $crate::LOG_TARGET, + "⚠️ running migration for {} and setting new storage version to {:?}", + pallet_name, + new_storage_version, + ); + result.saturating_add(additional_write) } @@ -1359,9 +1379,23 @@ macro_rules! decl_module { fn on_runtime_upgrade() -> $crate::dispatch::Weight { $crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!("on_runtime_upgrade")); - $crate::crate_to_pallet_version!() + let new_storage_version = $crate::crate_to_pallet_version!(); + new_storage_version .put_into_storage::<<$trait_instance as $system::Config>::PalletInfo, Self>(); + let pallet_name = << + $trait_instance + as + $system::Config + >::PalletInfo as $crate::traits::PalletInfo>::name::().expect("pallet will have name in the runtime; qed"); + + $crate::debug::info!( + target: $crate::LOG_TARGET, + "✅ no migration for '{}' and setting new storage version to {:?}", + pallet_name, + new_storage_version, + ); + < <$trait_instance as $system::Config>::DbWeight as $crate::traits::Get<_> >::get().writes(1) diff --git a/substrate/frame/support/src/lib.rs b/substrate/frame/support/src/lib.rs index e7af1ccab6..e16200ef0b 100644 --- a/substrate/frame/support/src/lib.rs +++ b/substrate/frame/support/src/lib.rs @@ -80,6 +80,9 @@ pub use self::storage::{ pub use self::dispatch::{Parameter, Callable}; pub use sp_runtime::{self, ConsensusEngineId, print, traits::Printable}; +/// A unified log target for support operations. +pub const LOG_TARGET: &'static str = "runtime::frame-support"; + /// A type that cannot be instantiated. #[derive(Debug, PartialEq, Eq, Clone)] pub enum Never {} diff --git a/substrate/frame/support/src/traits.rs b/substrate/frame/support/src/traits.rs index f0ab3be642..395a23d581 100644 --- a/substrate/frame/support/src/traits.rs +++ b/substrate/frame/support/src/traits.rs @@ -1553,6 +1553,54 @@ pub trait OnGenesis { fn on_genesis() {} } +/// Prefix to be used (optionally) for implementing [`OnRuntimeUpgrade::storage_key`]. +#[cfg(feature = "try-runtime")] +pub const ON_RUNTIME_UPGRADE_PREFIX: &[u8] = b"__ON_RUNTIME_UPGRADE__"; + +/// Some helper functions for [`OnRuntimeUpgrade`] during `try-runtime` testing. +#[cfg(feature = "try-runtime")] +pub trait OnRuntimeUpgradeHelpersExt { + /// Generate a storage key unique to this runtime upgrade. + /// + /// This can be used to communicate data from pre-upgrade to post-upgrade state and check + /// them. See [`set_temp_storage`] and [`get_temp_storage`]. + #[cfg(feature = "try-runtime")] + fn storage_key(ident: &str) -> [u8; 32] { + let prefix = sp_io::hashing::twox_128(ON_RUNTIME_UPGRADE_PREFIX); + let ident = sp_io::hashing::twox_128(ident.as_bytes()); + + let mut final_key = [0u8; 32]; + final_key[..16].copy_from_slice(&prefix); + final_key[16..].copy_from_slice(&ident); + + final_key + } + + /// Get temporary storage data written by [`set_temp_storage`]. + /// + /// Returns `None` if either the data is unavailable or un-decodable. + /// + /// A `at` storage identifier must be provided to indicate where the storage is being read from. + #[cfg(feature = "try-runtime")] + fn get_temp_storage(at: &str) -> Option { + sp_io::storage::get(&Self::storage_key(at)) + .and_then(|bytes| Decode::decode(&mut &*bytes).ok()) + } + + /// Write some temporary data to a specific storage that can be read (potentially in + /// post-upgrade hook) via [`get_temp_storage`]. + /// + /// A `at` storage identifier must be provided to indicate where the storage is being written + /// to. + #[cfg(feature = "try-runtime")] + fn set_temp_storage(data: T, at: &str) { + sp_io::storage::set(&Self::storage_key(at), &data.encode()); + } +} + +#[cfg(feature = "try-runtime")] +impl OnRuntimeUpgradeHelpersExt for U {} + /// The runtime upgrade trait. /// /// Implementing this lets you express what should happen when the runtime upgrades, diff --git a/substrate/primitives/storage/src/lib.rs b/substrate/primitives/storage/src/lib.rs index 1016b73eb1..ced8d8c02a 100644 --- a/substrate/primitives/storage/src/lib.rs +++ b/substrate/primitives/storage/src/lib.rs @@ -29,7 +29,7 @@ use codec::{Encode, Decode}; /// Storage key. #[derive(PartialEq, Eq, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode))] pub struct StorageKey( #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, ); @@ -107,7 +107,7 @@ impl PrefixedStorageKey { /// Storage data associated to a [`StorageKey`]. #[derive(PartialEq, Eq, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode))] pub struct StorageData( #[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))] pub Vec, diff --git a/substrate/utils/frame/remote-externalities/Cargo.toml b/substrate/utils/frame/remote-externalities/Cargo.toml index 41a3b26217..d4825211d8 100644 --- a/substrate/utils/frame/remote-externalities/Cargo.toml +++ b/substrate/utils/frame/remote-externalities/Cargo.toml @@ -16,12 +16,12 @@ targets = ["x86_64-unknown-linux-gnu"] jsonrpc-core-client = { version = "15.1.0", features = ["http"] } sc-rpc-api = { version = "0.9.0", path = "../../../client/rpc-api" } sc-rpc = { version = "3.0.0", path = "../../../client/rpc" } -futures = "0.1.29" +futures = "0.3" hex-literal = "0.3.1" env_logger = "0.8.2" log = "0.4.11" -bincode = "1.3.1" +codec = { package = "parity-scale-codec", version = "2.0.0" } tokio = "0.1.22" sp-io = { version = "3.0.0", path = "../../../primitives/io" } diff --git a/substrate/utils/frame/remote-externalities/proxy_test b/substrate/utils/frame/remote-externalities/proxy_test deleted file mode 100644 index adb93f5ba270c3e3740394f183d96bc54a3211f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26476 zcmd_zJC^Q9k{n=?;%M9h7k~)F15ihx#7zVu0Fm;ngR9~4_<45aOBwPov;V3(lQZ2_ zeQ#AJ{(y(OxtY1&|Nfu;@%6v{`0Ms>>DbQIkG-^>``GTM9&>xfwqN5Jd*0JJj=JV^ z<=&rk)F-dwZh82dbDKxq`gWC^(>(kuTiV9BeX~^4IBT6fUt?|i>iNmX)9#~etIhMM zXB$25eXKj3dwupgw$^K_?SFqf*>;qBUw51Jx{veP+B4eI%gFnE9HU;TweqBPUUxfd zyZy`Qe$K6zQt!3Zr`}uM{`jid??1)!ls%O*G@mEm z`OJISSDyLa_Uc#fR<3<1R^=!^EU!LoqejaDt_Gi1VanF^m$8(;?x>nmp zulMtKuygtCyz+C*EAKfuul3yPIlOJ3ZyU#RU452Oa!Fs$;}|I=f!~|ku;o10b&OK> zy{`K_i=VMp8mpdnD_bhZd7tGv_w&d#J!kQ2f5Le@dEU*t*=8%H?=3w~OG(6B=Uuk_ z+#cbbZb4LXzw0^gv7Kk%k37bC=H#GUmeO2R7x4Oe^RAIj!ElW0sQH-ZQ+JUssLIt+ zEop7ue^Dwlzgw*`?WXsy^iOU%I4X`L_Y3%)Y6_>q@=p9L)z=l zm1-|Xc}5jtDK$6mS}u*l-SgK-+e>luZ5``8gm1g|*{?Pn#-5In_pR-z_2wUY9+{~f zGMDI-i_POe_rJS}V+mGI=Ozy+?>vUZ6m=fdT|E}C_ zzw81%eI~hlS3CBRwy)>$)3g1IwAY=7va(-KIp)3Ac`ffR^&=-)rXK21>-{KdpaY30 zI@WWp{ke;ZGxix(XUf@qKhvd<-no3edB0clB8JvhRL(rkJ-03Gm%=HGxB1j`s2JtA z?`=rw+r9RyUqu*l>t?>{*Ze)ZF&gPld>n-yZXTQ}%s(j^S=JefBr|&+x~5vzjv~#Yd)W6Ynw_SFO=zpDgv|3*4!`jU5Bxd(`IOC|Ty&)&1R zntR_?5p<>8^F0@A+uRi4Gk)?vyAEfUe;jJ)waW1M>UjxnZ$^ZTTS0lwS#y*Rus^>;q!)b~s+(2U0HnH}gYPNt~wbLP6P-U53k1X)^b(JLg`zVs!T-14@giKEpfz{=zP7HL!W&;}ql zCh#2S=$3h+qbmV$%PtD_m~*?+@eEIJ$e$7yjB!Y(@23%<>Q%kg4yNn*<(*x7cu3c` zl*5u^HFY|={{4_$pd+`pq#0jQ~o zUzc|DoKPAD=)CYA`CCpa*t$#7%h-~Eo;tBFBD~9Tb#Jcr-L~tJJC&YGRc(sujd+2% z9M^`(c|VXB#_Vx6@cd|q&FwzZPnn6YIqj$1%V$oDVw;`|fBAW@-_I)k`g{ChId^GC zE-uGD)%546NVDX21|>0Xx%S?pZBh&6WR(qLs1^pd(e)Zzy7FlGK~W%YQvY^`tM32M zrs#X_XTm8-7IQPUOU2yLu-h#VBuv+4t5eOpP(K?wB*@Ay==-nOwY~5TkKi|D8-!WL zQ#UyDgvQGHM?FHxB8g6U>8yJf)3C4rKNnUQ-vA&_7ZjVNXn`2V*O)z`z0h@@K<=TU zMM{&MQ&Dj@EgBwR)8dX>Yjhr{W!X%0=>5`ErEwtdarwGCEHE&T<%BB%130sxpla`R z*YocM3yiMz$PyY_a-UQRy+c`7-kx)`mViycyl2?8UHu#LLXnWJ5n7-ya(5ungZe~p z9S0I;caS^675m>WMStKa^=i4tA>WX4a1t4j4&FU8=s#iy!ljWvrrsZop0F`428`4t zP~`dG~mdq3A#a-KEUsWRw!po|-|Afg=0=X#zx^-5ZgNI~evP!y6`&QpD;Ywdwzfjq@XC+_8Fjojr>sug)D{T? z3I-gd9x}|MyZJni7e<=r>Mj$qTiM5~yL{rv%fY`Cc^e#49fRw- zfrP{gw=}QO`u^=aTvLaL@j-Jtm_pERB(sX=+Y0e-^T{DWN=i5+H!aQ`r|*kcqGHu^GJub{;CQp zzIkdEwE8$_5O34mHsy2(LMK3dqTjf;TAqxbma!zjKnQ<)z3@ExBb98PMjzSy*Q!7a zSb$Hs9F11FOfjO+6pVOZyR>}5vdYQ1AHlvy{sMl6Cbf>QnMfDMl5hrFg^Mp+PA58b zNhKIXsr^P-tc-Lc6C*}MWxWWP#K08L=*xLH#(j;8`Wh%1mPiOSgpMcnPm(<~{k}Cs zC9AGrED!mURGTXPDX5e8uh(TFk4EFJI(P_uAPGgkt9WN6>B=I==UoYXui6cH6g{Jt z1Q?vsTl?SZ)cnQ24Bi~u*N)()c;d4|`aQ*P>OqRGMt0?bXfkR?RE9tvB1&38J}ckk zfdgi$%p@oLUXay;BZTd*mFTqIURJ7tk z#jDzfm}#mrLhwEIUx|Rj#e(*P$onL80}G*hOZ^Z(brI6jctH}kHi@fZJ-imR9~iIW zcy4OVP0KyV5Z+CJ_rK;@e8W1mVp!wJSQG!0I2w0P3J_OH4m%6&Rbn`Csa~HG(l@j% z?%Mp44PKXLJ8IYKn>i#X?`nH)#5WugtWH1>-4rrU52FFaTMn-qt8_a;G@s5O(Zp$K zVH4q$NMEzTFGkw0X|amPV7|yOhAa;X?9bUl3)cWLxzBWO3dYh+wKt5*8;|vPV z7{aHOuQB^~RLHt&Pg`MD*#?P>>x5S7*uujfR>&YI?o`Qqu|Jcn=YnHT^6iE z(xRex2AKP^dcH2F9ebXIF7fn?lfIUZyDLpB3tUYXOWUSEg?sMd=OA5hw0ZDdGd2Mo zNUWnkHS9xGz%qT)ae@?5j?r@!lcC&c%Z}n0H1?z5of#lPT1C1s5rU6_1T~= zH9iEIPF8`#BtQCCsyZr!5o+Z|0+59Y`k1qYNOUGy3S&TFBnf(==*mCP)eGZ;6LLQq zz6UquatDJ!vI9$4kNBba+bEKcCh>7?+5&PBc&sJ|QjnGf+TcVKaNyD9`o{V=GePgt zQS*4hG*(M_-A35ZO=Ru|g(m)nXHeb?D zeW73DQ(6NX+Bv?hm7@ZsUU1B~=Y1ln@D1K0?;WZeU0 zy!c#sn740ENI(O%Be&=`5f1?=UA@${?1m`U;j!S9m0kTZ*5dKKm=D5IfT<84A1dSP z;)4%b$hLr~$a54gJ4%cI%*;CLAe>P^LU@=fpQDu|%0Ut)HFgsDfQ5P^468G!Ecx|o ze4ykix?RsbBu~_*OU5cOM{2w$T8ZHF)LJ|(U>b@Q{2Y9Sp@1S*{|uJ&L^MoIyiW_Z zld^W6UYf8jjl~khUE0pxOp!%Z?0z{ENn#q}N|C7CB?Jm4LY{qn-ZlT~m>Ju|<9IX# zz`65CpaupT1fC?~{l?xdui?92dWFZkKmFb@P{WK9W)8Efe?gBd?pc* z*%4K2Tm$DQx`Lvfg5sh zItAx1Tf86b>o`xt8sfTKovy-3PF6wo0d&YTqMr+)+Ouw>*S3yHC)6T{58TEARCQHM z*<2HbIR@bDsK1`Ujk&i3S=EaB^)kk{icsEVu-u4L(fztGHfF8nkwA3Z2G0%glSlh~Ag> z?}c7|DmH{FD#C>m&AKi_{X)=+u1e2<;NJLGv_R(B4vY2-2oi84CvGXFaZO4MV7Scwj(w zQhsW_!mr5q$DQP#;2+>X*A*eiv`J-Yr-5U5ao_`tHn52@goa<8EhGS09y11oV+aVG zXP-uGQe9vYgRR)vzkW6d%?LTr5MUHDdbauL2R-yQ*BI*LeH(oCJNT1E%QoXjk;K$d z`?;?^GxqCH275lQ0ah?8C&pYA{2oqlswk?XFa!n(!b_onp$(@KgjFx~06q+ZSqvG3 z$p^`XhR)@Gd`w?y#(r(S;K$$T17-Z)q{5HC)5kM~pBhbwmi_EsKV#uH{h!9gkrpu9 zrBq?|F7sqZOqa_z^eQ#m-1HnV@rgL5B*;dI8H$-le+n&Db8kd;O%3XE?tk_AYn!C4K*bIDG-3XvKp4D6Wb0hFbiy1OO7#ae#c87V2OsCxDq^O`tcu|a5|A9k zQEIvfAzmnzn|=W&-D7kd=+!MFv+$o`qRRd*2A}^7ewGv((j0}@Fep_4_%J8w!y=(B z(*`Reo~l!M$o8xZY>8-EqqyM_%2iXdog|#gFe2?Xe>;ypeJWCkBuL}L(wSroD`ZTZ zJ!-UQkO$Gfi>O3!XXZpzgWXtBm&hQ`dw{%-e+vRyQ$Sb`uM%S{T}p={BElt0n1SxX z^p7)S0Wjf4v>7pcuZo)Vq5>il5pyC;7uLSj&I34%~0D7z|Doty1y$0YvmpYFZ6G!6+p?gcUV`EE03RfL1 z17&j^hBsJ=!~Mf8!2SPilfi#%_QO`25YeYI3XPpavl3fYGlpUv2p0pVM0uJIpxliFK}!0Y){Ku8WLYD-Wg2XMpnT5yM1; z^s#ir@iFWO5Hj(PzJLARMv&L_76t}jTJkhP(i`Af`Po*JZ+Z5vmZTkd3$KgJ8DnhB zF3^kNSv`%+tgYeGW}&uWO0uP?OUMRlC=u@9$JYz$db42 zII~c3%bzjj3S5T~y{|F*ciaJYV^WRD z7&I^(#-wCF1Q;4>q2W(uA7O`S)`3v$82G6v7!MFsAFaLD!|Uyh^>K)T@q+s6d2lHe zi8z3LEJ{q+0WidN0&XNrJ@k%}JW>**GkjKKpw0-qK3*&JmZ7XMiYjDXl#t2D4S*j3{pL@B7qru|L!b zWo?=W511t;rUq(tOLr~-qPhij{HF#Aa80JS1gj-zaf@|3VwA(S#`s&LCvfeDGvJaL zl!CI6L-hiTdEsd_^PoLsEz?8sGz1wK?92f!#e>HFTRD*ZIW+7)p5tfT1NH|K&ArZV zO%9HsYls28Yv3;C%WM%u7HLOfZYMTuFlktq=b3H#z&lOw^zA%;_+ISai?F2Fqgn0B zwS$M*k(h-#hk10`I%1uYac?rQbpZ^r=HyrqEdPv6gT$=gcrR~O^w|dG$$Td=%|xsw zHr7hS@)c7lTstvia)yF{_wpp^Z=XI0<|#J7+Zs}<4O=wtzPk? zn&&_!BAjRiXaSZ3_lQ+%Xtl!K1nA?ae|lexPb-e|7ISYFu~;n;8yeY@jCl(Ic1C^x zT&k_Gu_nQ!7w43CM#b|}F?am&96x(6+>^ZN3tgq5G@Of#bcd}Nj38*%DHJ>ZVvj|Y zOpI6>-p7gVM2K>LgB7+fU*w%PqZ|iRSKG}*r*1*0b z{2R{!;TqM;rVGphxgTIpEDMpP{Kqx5`Z~ml(98-&2jeH7y>5FND98yV?9>R6w%2QmFWda`5O^oZJNB%GHRb2$A!kMAieY1GvivA3 z5Ex~x(n2gf=jpv9F46L1jxW`yC#AKAyt|KZd zI|eIx2(Ai^H5}Y@0=Jt0Vt=*%5i$SV9i=Bcz?q81pk zG6@Dm2Qpe;*)`R%ffwEP8_!|wOSCOg0I(G%+?&C`#G+YDn|r7zNe|OodBi5;e_%Bx~Tbv-SqY(xe*;rq=Bq~P*5%p~e6I(L86fsI#wQPk=(6U0~ zgg<{Tx+de*Q7LR9W!V*~(wJeAx9nw78)xjuC3V#lIqk%Bj|Pr-Q3TdA=h)Dcufpcw)gL_hA&E%=4=2=OcS z0C*V|7LZ;glQ$CxW9{3j*SB3=G;Y#A3g@=f)56lQPduRu#v*^C=s$lis#Q$$$liuL zgkmF*K9X^-~NQGwJ0ez3@{3SK?VQ51R;Zod5~e>a@CbvVJM|mSVXZWdo2`iD(QKc|KQ3CH z9A%c<4}=ctolV1J0o_B@!@{dYTALU##rwua#$PxO=o1mDoVpKh`w4&^Ym;p*pj*XG zG(U^rLEMB-PWJb(TWa=VqCBKvT$(?j!Fv%S(7^waTdVZMD zt_m~8RLtAxf=q5+&XuO$#(*dTbqvmIv7_wg@d3){qW9SRwCvWAu}R37Kl$5ew2fUt p#^SFqX&9c6*Bw`EWr32VKMjsu$nhNTtv|y?__%zZ{QT$N{U5`qT;l)$ diff --git a/substrate/utils/frame/remote-externalities/src/lib.rs b/substrate/utils/frame/remote-externalities/src/lib.rs index 6c8b49c7c8..ab26226253 100644 --- a/substrate/utils/frame/remote-externalities/src/lib.rs +++ b/substrate/utils/frame/remote-externalities/src/lib.rs @@ -112,7 +112,11 @@ use sp_core::{ hexdisplay::HexDisplay, storage::{StorageKey, StorageData}, }; -use futures::future::Future; +use futures::{ + compat::Future01CompatExt, + TryFutureExt, +}; +use codec::{Encode, Decode}; type KeyPair = (StorageKey, StorageData); type Number = u32; @@ -192,7 +196,6 @@ impl CacheConfig { pub struct Builder { inject: Vec, mode: Mode, - chain: String, } impl Default for Builder { @@ -205,7 +208,6 @@ impl Default for Builder { cache: None, modules: Default::default(), }), - chain: "UNSET".into(), } } } @@ -229,74 +231,71 @@ impl Builder { // RPC methods impl Builder { - async fn rpc_get_head(&self) -> Hash { - let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + async fn rpc_get_head(&self) -> Result { let uri = self.as_online().uri.clone(); - rt.block_on::<_, _, ()>(futures::lazy(move || { - trace!(target: LOG_TARGET, "rpc: finalized_head"); - let client: sc_rpc_api::chain::ChainClient = - jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); - Ok(client.finalized_head().wait().unwrap()) - })) - .unwrap() + trace!(target: LOG_TARGET, "rpc: finalized_head"); + let client: sc_rpc_api::chain::ChainClient = + jsonrpc_core_client::transports::http::connect(&uri) + .compat() + .map_err(|_| "client initialization failed") + .await?; + client.finalized_head().compat().map_err(|_| "rpc finalized_head failed.").await } /// Relay the request to `state_getPairs` rpc endpoint. /// /// Note that this is an unsafe RPC. - async fn rpc_get_pairs(&self, prefix: StorageKey, at: Hash) -> Vec { - let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + async fn rpc_get_pairs( + &self, + prefix: StorageKey, + at: Hash, + ) -> Result, &'static str> { let uri = self.as_online().uri.clone(); - rt.block_on::<_, _, ()>(futures::lazy(move || { - trace!(target: LOG_TARGET, "rpc: storage_pairs: {:?} / {:?}", prefix, at); - let client: sc_rpc_api::state::StateClient = - jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); - Ok(client.storage_pairs(prefix, Some(at)).wait().unwrap()) - })) - .unwrap() - } - - /// Get the chain name. - async fn chain_name(&self) -> String { - let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); - let uri = self.as_online().uri.clone(); - rt.block_on::<_, _, ()>(futures::lazy(move || { - trace!(target: LOG_TARGET, "rpc: system_chain"); - let client: sc_rpc_api::system::SystemClient<(), ()> = - jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); - Ok(client.system_chain().wait().unwrap()) - })) - .unwrap() + trace!(target: LOG_TARGET, "rpc: storage_pairs: {:?} / {:?}", prefix, at); + let client: sc_rpc_api::state::StateClient = + jsonrpc_core_client::transports::http::connect(&uri) + .compat() + .map_err(|_| "client initialization failed") + .await?; + client + .storage_pairs(prefix, Some(at)) + .compat() + .map_err(|_| "rpc finalized_head failed.") + .await } } // Internal methods impl Builder { /// Save the given data as cache. - fn save_cache(&self, data: &[KeyPair], path: &Path) { - let bdata = bincode::serialize(data).unwrap(); + fn save_cache(&self, data: &[KeyPair], path: &Path) -> Result<(), &'static str> { info!(target: LOG_TARGET, "writing to cache file {:?}", path); - fs::write(path, bdata).unwrap(); + fs::write(path, data.encode()).map_err(|_| "fs::write failed.")?; + Ok(()) } /// initialize `Self` from cache. Panics if the file does not exist. - fn load_cache(&self, path: &Path) -> Vec { + fn load_cache(&self, path: &Path) -> Result, &'static str> { info!(target: LOG_TARGET, "scraping keypairs from cache {:?}", path,); - let bytes = fs::read(path).unwrap(); - bincode::deserialize(&bytes[..]).unwrap() + let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; + Decode::decode(&mut &*bytes).map_err(|_| "decode failed") } /// Build `Self` from a network node denoted by `uri`. - async fn load_remote(&self) -> Vec { + async fn load_remote(&self) -> Result, &'static str> { let config = self.as_online(); - let at = self.as_online().at.unwrap().clone(); + let at = self + .as_online() + .at + .expect("online config must be initialized by this point; qed.") + .clone(); info!(target: LOG_TARGET, "scraping keypairs from remote node {} @ {:?}", config.uri, at); let keys_and_values = if config.modules.len() > 0 { let mut filtered_kv = vec![]; for f in config.modules.iter() { let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec()); - let module_kv = self.rpc_get_pairs(hashed_prefix.clone(), at).await; + let module_kv = self.rpc_get_pairs(hashed_prefix.clone(), at).await?; info!( target: LOG_TARGET, "downloaded data for module {} (count: {} / prefix: {:?}).", @@ -309,25 +308,26 @@ impl Builder { filtered_kv } else { info!(target: LOG_TARGET, "downloading data for all modules."); - self.rpc_get_pairs(StorageKey(vec![]), at).await.into_iter().collect::>() + self.rpc_get_pairs(StorageKey(vec![]), at).await?.into_iter().collect::>() }; - keys_and_values + Ok(keys_and_values) } - async fn init_remote_client(&mut self) { - self.as_online_mut().at = Some(self.rpc_get_head().await); - self.chain = self.chain_name().await; + async fn init_remote_client(&mut self) -> Result<(), &'static str> { + let at = self.rpc_get_head().await?; + self.as_online_mut().at = Some(at); + Ok(()) } - async fn pre_build(mut self) -> Vec { + async fn pre_build(mut self) -> Result, &'static str> { let mut base_kv = match self.mode.clone() { - Mode::Offline(config) => self.load_cache(&config.cache.path()), + Mode::Offline(config) => self.load_cache(&config.cache.path())?, Mode::Online(config) => { - self.init_remote_client().await; - let kp = self.load_remote().await; + self.init_remote_client().await?; + let kp = self.load_remote().await?; if let Some(c) = config.cache { - self.save_cache(&kp, &c.path()); + self.save_cache(&kp, &c.path())?; } kp } @@ -339,7 +339,7 @@ impl Builder { self.inject.len() ); base_kv.extend(self.inject.clone()); - base_kv + Ok(base_kv) } } @@ -365,8 +365,8 @@ impl Builder { } /// Build the test externalities. - pub async fn build(self) -> TestExternalities { - let kv = self.pre_build().await; + pub async fn build(self) -> Result { + let kv = self.pre_build().await?; let mut ext = TestExternalities::new_empty(); info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); @@ -374,10 +374,11 @@ impl Builder { let (k, v) = (k.0, v.0); ext.insert(k, v); } - ext + Ok(ext) } } +#[cfg(feature = "remote-test")] #[cfg(test)] mod tests { use super::*; @@ -390,7 +391,6 @@ mod tests { } #[async_std::test] - #[cfg(feature = "remote-test")] async fn can_build_one_pallet() { init_logger(); Builder::new() @@ -400,6 +400,7 @@ mod tests { })) .build() .await + .unwrap() .execute_with(|| {}); } @@ -412,11 +413,11 @@ mod tests { })) .build() .await + .unwrap() .execute_with(|| {}); } #[async_std::test] - #[cfg(feature = "remote-test")] async fn can_create_cache() { init_logger(); Builder::new() @@ -429,6 +430,7 @@ mod tests { })) .build() .await + .unwrap() .execute_with(|| {}); let to_delete = std::fs::read_dir(CacheConfig::default().directory) @@ -446,9 +448,8 @@ mod tests { } #[async_std::test] - #[cfg(feature = "remote-test")] async fn can_build_all() { init_logger(); - Builder::new().build().await.execute_with(|| {}); + Builder::new().build().await.unwrap().execute_with(|| {}); } } diff --git a/substrate/utils/frame/try-runtime/cli/src/lib.rs b/substrate/utils/frame/try-runtime/cli/src/lib.rs index 92526379f4..4ab38692a5 100644 --- a/substrate/utils/frame/try-runtime/cli/src/lib.rs +++ b/substrate/utils/frame/try-runtime/cli/src/lib.rs @@ -133,7 +133,7 @@ impl TryRuntimeCmd { }; // inject the code into this ext. - builder.inject(&[(code_key, code)]).build().await + builder.inject(&[(code_key, code)]).build().await? }; let encoded_result = StateMachine::<_, _, NumberFor, _>::new(