contracts: Multi block migrations (#14045)

* Frame Add translate_next This works similarly to to `translate` but only translate a single entry. This function will be useful in the context of multi-block migration. * Move to lazy migration * Updates * simplify MockMigration * wip * wip * add bench * add bench * fmt * fix bench * add . * ".git/.scripts/commands/bench/bench.sh" pallet dev pallet_contracts * Apply suggestions from code review Co-authored-by: Alexander Theißen <alex.theissen@me.com> * Scalfold v10 / v11 fix tests * PR comment * tweak pub use * wip * wip * wip * misc merge master * misc merge master * wip * rm tmp stuff * wip * wip * wip * wip * wip * fixes * add state * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * fix * fixed compilation * clean up logs * wip * Revert "Frame Add translate_next" This reverts commit 10318fc95c42b1f7f25efeb35e6d947ea02bed88. * Fix v10 logic * Apply suggestions from code review Co-authored-by: Alexander Theißen <alex.theissen@me.com> * wip * fixes * exercise del_queue * bump sample size * fmt * wip * blank line * fix lint * fix rustdoc job lint * PR comment do not use dangerous into() * Ad macros for updating mod visibility * Add doc * Add max_weight to integrity_test * fix compilation * Add no migration tests * ".git/.scripts/commands/bench/bench.sh" pallet dev pallet_contracts * fix clippy * PR review * Update frame/contracts/src/lib.rs Co-authored-by: Sasha Gryaznov <hi@agryaznov.com> * Fix master merge * fix merge 2 * fix tryruntime * fix lint --------- Co-authored-by: Alexander Theißen <alex.theissen@me.com> Co-authored-by: command-bot <> Co-authored-by: Sasha Gryaznov <hi@agryaznov.com>
2026-04-30 00:17:56 +00:00 · 2023-05-31 16:19:31 +02:00
parent b43a1b0b55
commit c7c5fc709c
10 changed files with 2841 additions and 1482 deletions
@@ -0,0 +1,272 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Don't rely on reserved balances keeping an account alive
+//! See <https://github.com/paritytech/substrate/pull/13370>.
+
+use crate::{
+	address::AddressGenerator,
+	exec::AccountIdOf,
+	migration::{IsFinished, Migrate},
+	weights::WeightInfo,
+	BalanceOf, CodeHash, Config, Pallet, TrieId, Weight, LOG_TARGET,
+};
+use codec::{Decode, Encode};
+use core::cmp::{max, min};
+use frame_support::{
+	codec,
+	pallet_prelude::*,
+	storage_alias,
+	traits::{
+		fungible::Inspect,
+		tokens::{Fortitude::Polite, Preservation::Preserve},
+		Currency, ExistenceRequirement, ReservableCurrency,
+	},
+	DefaultNoBound,
+};
+use sp_core::hexdisplay::HexDisplay;
+#[cfg(feature = "try-runtime")]
+use sp_runtime::TryRuntimeError;
+use sp_runtime::{traits::Zero, Perbill, Saturating};
+use sp_std::{marker::PhantomData, ops::Deref, prelude::*};
+
+mod old {
+	use super::*;
+
+	#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)]
+	#[scale_info(skip_type_params(T))]
+	pub struct ContractInfo<T: Config> {
+		pub trie_id: TrieId,
+		pub code_hash: CodeHash<T>,
+		pub storage_bytes: u32,
+		pub storage_items: u32,
+		pub storage_byte_deposit: BalanceOf<T>,
+		pub storage_item_deposit: BalanceOf<T>,
+		pub storage_base_deposit: BalanceOf<T>,
+	}
+
+	#[storage_alias]
+	pub type ContractInfoOf<T: Config> = StorageMap<
+		Pallet<T>,
+		Twox64Concat,
+		<T as frame_system::Config>::AccountId,
+		ContractInfo<T>,
+	>;
+}
+
+#[cfg(feature = "runtime-benchmarks")]
+pub fn store_old_contrat_info<T: Config>(account: T::AccountId, info: crate::ContractInfo<T>) {
+	let info = old::ContractInfo {
+		trie_id: info.trie_id,
+		code_hash: info.code_hash,
+		storage_bytes: Default::default(),
+		storage_items: Default::default(),
+		storage_byte_deposit: Default::default(),
+		storage_item_deposit: Default::default(),
+		storage_base_deposit: Default::default(),
+	};
+	old::ContractInfoOf::<T>::insert(account, info);
+}
+
+#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebugNoBound, TypeInfo, MaxEncodedLen)]
+#[scale_info(skip_type_params(T))]
+pub struct DepositAccount<T: Config>(AccountIdOf<T>);
+
+impl<T: Config> Deref for DepositAccount<T> {
+	type Target = AccountIdOf<T>;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)]
+#[scale_info(skip_type_params(T))]
+pub struct ContractInfo<T: Config> {
+	pub trie_id: TrieId,
+	deposit_account: DepositAccount<T>,
+	pub code_hash: CodeHash<T>,
+	storage_bytes: u32,
+	storage_items: u32,
+	pub storage_byte_deposit: BalanceOf<T>,
+	storage_item_deposit: BalanceOf<T>,
+	storage_base_deposit: BalanceOf<T>,
+}
+
+#[derive(Encode, Decode, MaxEncodedLen, DefaultNoBound)]
+pub struct Migration<T: Config> {
+	last_key: Option<BoundedVec<u8, ConstU32<256>>>,
+	_phantom: PhantomData<T>,
+}
+
+#[storage_alias]
+type ContractInfoOf<T: Config> =
+	StorageMap<Pallet<T>, Twox64Concat, <T as frame_system::Config>::AccountId, ContractInfo<T>>;
+
+impl<T: Config> Migrate for Migration<T> {
+	const VERSION: u16 = 10;
+
+	fn max_step_weight() -> Weight {
+		T::WeightInfo::v10_migration_step()
+	}
+
+	fn step(&mut self) -> (IsFinished, Weight) {
+		let mut iter = if let Some(last_key) = self.last_key.take() {
+			old::ContractInfoOf::<T>::iter_from(last_key.to_vec())
+		} else {
+			old::ContractInfoOf::<T>::iter()
+		};
+
+		if let Some((account, contract)) = iter.next() {
+			let min_balance = Pallet::<T>::min_balance();
+			log::debug!(target: LOG_TARGET, "Account: 0x{} ", HexDisplay::from(&account.encode()));
+
+			// Store last key for next migration step
+			self.last_key = Some(iter.last_raw_key().to_vec().try_into().unwrap());
+
+			// Get the new deposit account address
+			let deposit_account: DepositAccount<T> =
+				DepositAccount(T::AddressGenerator::deposit_address(&account));
+
+			// Calculate the existing deposit, that should be reserved on the contract account
+			let old_deposit = contract
+				.storage_base_deposit
+				.saturating_add(contract.storage_item_deposit)
+				.saturating_add(contract.storage_byte_deposit);
+
+			// Unreserve the existing deposit
+			// Note we can't use repatriate_reserve, because it only works with existing accounts
+			let remaining = T::Currency::unreserve(&account, old_deposit);
+			if !remaining.is_zero() {
+				log::warn!(
+					target: LOG_TARGET,
+					"Partially unreserved. Remaining {:?} out of {:?} asked",
+					remaining,
+					old_deposit
+				);
+			}
+
+			// Attempt to transfer the old deposit to the deposit account.
+			let amount = old_deposit
+				.saturating_sub(min_balance)
+				.min(T::Currency::reducible_balance(&account, Preserve, Polite));
+
+			let new_deposit = T::Currency::transfer(
+				&account,
+				&deposit_account,
+				amount,
+				ExistenceRequirement::KeepAlive,
+			)
+			.map(|_| {
+				log::debug!(
+					target: LOG_TARGET,
+					"Transferred deposit ({:?}) to deposit account",
+					amount
+				);
+				amount
+			})
+			// If it fails we fallback to minting the ED.
+			.unwrap_or_else(|err| {
+				log::error!(target: LOG_TARGET, "Failed to transfer ED, reason: {:?}", err);
+				T::Currency::deposit_creating(&deposit_account, min_balance);
+				min_balance
+			});
+
+			// Calculate the new base_deposit to store in the contract:
+			// Ideally: it should be the same as the old one
+			// Ideally, it should be at least 2xED (for the contract and deposit account).
+			// It can't be more than the `new_deposit`.
+			let new_base_deposit = min(
+				max(contract.storage_base_deposit, min_balance.saturating_add(min_balance)),
+				new_deposit,
+			);
+
+			// Calculate the ratio to adjust storage_byte and storage_item deposits.
+			let new_deposit_without_base = new_deposit.saturating_sub(new_base_deposit);
+			let old_deposit_without_base =
+				old_deposit.saturating_sub(contract.storage_base_deposit);
+			let ratio = Perbill::from_rational(new_deposit_without_base, old_deposit_without_base);
+
+			// Calculate the new storage deposits based on the ratio
+			let storage_byte_deposit = ratio.mul_ceil(contract.storage_byte_deposit);
+			let storage_item_deposit = ratio.mul_ceil(contract.storage_item_deposit);
+
+			// Recalculate the new base deposit, instead of using new_base_deposit to avoid rounding
+			// errors
+			let storage_base_deposit = new_deposit
+				.saturating_sub(storage_byte_deposit)
+				.saturating_sub(storage_item_deposit);
+
+			let new_contract_info = ContractInfo {
+				trie_id: contract.trie_id,
+				deposit_account,
+				code_hash: contract.code_hash,
+				storage_bytes: contract.storage_bytes,
+				storage_items: contract.storage_items,
+				storage_byte_deposit,
+				storage_item_deposit,
+				storage_base_deposit,
+			};
+
+			ContractInfoOf::<T>::insert(&account, new_contract_info);
+			(IsFinished::No, T::WeightInfo::v10_migration_step())
+		} else {
+			log::debug!(target: LOG_TARGET, "Done Migrating contract info");
+			(IsFinished::Yes, T::WeightInfo::v10_migration_step())
+		}
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn pre_upgrade_step() -> Result<Vec<u8>, TryRuntimeError> {
+		let sample: Vec<_> = old::ContractInfoOf::<T>::iter().take(10).collect();
+
+		log::debug!(target: LOG_TARGET, "Taking sample of {} contracts", sample.len());
+		Ok(sample.encode())
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn post_upgrade_step(state: Vec<u8>) -> Result<(), TryRuntimeError> {
+		let sample =
+			<Vec<(T::AccountId, old::ContractInfo<T>)> as Decode>::decode(&mut &state[..]).unwrap();
+
+		log::debug!(target: LOG_TARGET, "Validating sample of {} contracts", sample.len());
+		for (account, old_contract) in sample {
+			log::debug!(target: LOG_TARGET, "===");
+			log::debug!(target: LOG_TARGET, "Account: 0x{} ", HexDisplay::from(&account.encode()));
+			let contract = ContractInfoOf::<T>::get(&account).unwrap();
+			ensure!(old_contract.trie_id == contract.trie_id, "invalid trie_id");
+			ensure!(old_contract.code_hash == contract.code_hash, "invalid code_hash");
+			ensure!(old_contract.storage_bytes == contract.storage_bytes, "invalid storage_bytes");
+			ensure!(old_contract.storage_items == contract.storage_items, "invalid storage_items");
+
+			let deposit =
+				<<T as Config>::Currency as frame_support::traits::Currency<_>>::total_balance(
+					&contract.deposit_account,
+				);
+			ensure!(
+				deposit ==
+					contract
+						.storage_base_deposit
+						.saturating_add(contract.storage_item_deposit)
+						.saturating_add(contract.storage_byte_deposit),
+				"deposit mismatch"
+			);
+		}
+
+		Ok(())
+	}
+}
@@ -0,0 +1,130 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Overflowing bounded DeletionQueue.
+//! See <https://github.com/paritytech/substrate/pull/13702>.
+
+use crate::{
+	migration::{IsFinished, Migrate},
+	weights::WeightInfo,
+	Config, Pallet, TrieId, Weight, LOG_TARGET,
+};
+#[cfg(feature = "try-runtime")]
+use sp_runtime::TryRuntimeError;
+
+use codec::{Decode, Encode};
+use frame_support::{codec, pallet_prelude::*, storage_alias, DefaultNoBound};
+use sp_std::{marker::PhantomData, prelude::*};
+mod old {
+	use super::*;
+
+	#[derive(Encode, Decode, TypeInfo, MaxEncodedLen)]
+	pub struct DeletedContract {
+		pub(crate) trie_id: TrieId,
+	}
+
+	#[storage_alias]
+	pub type DeletionQueue<T: Config> = StorageValue<Pallet<T>, Vec<DeletedContract>>;
+}
+
+#[derive(Encode, Decode, TypeInfo, MaxEncodedLen, DefaultNoBound, Clone)]
+#[scale_info(skip_type_params(T))]
+pub struct DeletionQueueManager<T: Config> {
+	insert_counter: u32,
+	delete_counter: u32,
+	_phantom: PhantomData<T>,
+}
+
+#[cfg(any(feature = "runtime-benchmarks", feature = "try-runtime"))]
+pub fn fill_old_queue<T: Config>(len: usize) {
+	let queue: Vec<old::DeletedContract> =
+		core::iter::repeat_with(|| old::DeletedContract { trie_id: Default::default() })
+			.take(len)
+			.collect();
+	old::DeletionQueue::<T>::set(Some(queue));
+}
+
+#[storage_alias]
+type DeletionQueue<T: Config> = StorageMap<Pallet<T>, Twox64Concat, u32, TrieId>;
+
+#[storage_alias]
+type DeletionQueueCounter<T: Config> = StorageValue<Pallet<T>, DeletionQueueManager<T>, ValueQuery>;
+
+#[derive(Encode, Decode, MaxEncodedLen, DefaultNoBound)]
+pub struct Migration<T: Config> {
+	_phantom: PhantomData<T>,
+}
+
+impl<T: Config> Migrate for Migration<T> {
+	const VERSION: u16 = 11;
+
+	// It would be more correct to make our use the now removed [DeletionQueueDepth](https://github.com/paritytech/substrate/pull/13702/files#diff-70e9723e9db62816e35f6f885b6770a8449c75a6c2733e9fa7a245fe52c4656c)
+	// but in practice the queue is always empty, so 128 is a good enough approximation for not
+	// underestimating the weight of our migration.
+	fn max_step_weight() -> Weight {
+		T::WeightInfo::v11_migration_step(128)
+	}
+
+	fn step(&mut self) -> (IsFinished, Weight) {
+		let Some(old_queue) = old::DeletionQueue::<T>::take() else { return (IsFinished::Yes, Weight::zero()) };
+		let len = old_queue.len();
+
+		log::debug!(
+			target: LOG_TARGET,
+			"Migrating deletion queue with {} deleted contracts",
+			old_queue.len()
+		);
+
+		if !old_queue.is_empty() {
+			let mut queue = DeletionQueueManager::<T>::default();
+			for contract in old_queue {
+				<DeletionQueue<T>>::insert(queue.insert_counter, contract.trie_id);
+				queue.insert_counter += 1;
+			}
+
+			<DeletionQueueCounter<T>>::set(queue);
+		}
+
+		(IsFinished::Yes, T::WeightInfo::v11_migration_step(len as u32))
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn pre_upgrade_step() -> Result<Vec<u8>, TryRuntimeError> {
+		let old_queue = old::DeletionQueue::<T>::take().unwrap_or_default();
+
+		if old_queue.is_empty() {
+			let len = 10u32;
+			log::debug!(
+				target: LOG_TARGET,
+				"Injecting {len} entries to deletion queue to test migration"
+			);
+			fill_old_queue::<T>(len as usize);
+			return Ok(len.encode())
+		}
+
+		Ok((old_queue.len() as u32).encode())
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn post_upgrade_step(state: Vec<u8>) -> Result<(), TryRuntimeError> {
+		let len = <u32 as Decode>::decode(&mut &state[..]).unwrap();
+		let counter = <DeletionQueueCounter<T>>::get();
+		ensure!(counter.insert_counter == len, "invalid insert counter");
+		ensure!(counter.delete_counter == 0, "invalid delete counter");
+		Ok(())
+	}
+}
@@ -0,0 +1,149 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Update `CodeStorage` with the new `determinism` field.
+
+use crate::{
+	migration::{IsFinished, Migrate},
+	weights::WeightInfo,
+	CodeHash, Config, Determinism, Pallet, Weight, LOG_TARGET,
+};
+use codec::{Decode, Encode};
+use frame_support::{
+	codec, pallet_prelude::*, storage_alias, BoundedVec, DefaultNoBound, Identity,
+};
+#[cfg(feature = "try-runtime")]
+use sp_runtime::TryRuntimeError;
+use sp_std::{marker::PhantomData, prelude::*};
+
+mod old {
+	use super::*;
+
+	#[derive(Encode, Decode)]
+	pub struct PrefabWasmModule {
+		#[codec(compact)]
+		pub instruction_weights_version: u32,
+		#[codec(compact)]
+		pub initial: u32,
+		#[codec(compact)]
+		pub maximum: u32,
+		pub code: Vec<u8>,
+	}
+
+	#[storage_alias]
+	pub type CodeStorage<T: Config> =
+		StorageMap<Pallet<T>, Identity, CodeHash<T>, PrefabWasmModule>;
+}
+
+#[cfg(feature = "runtime-benchmarks")]
+pub fn store_old_dummy_code<T: Config>(len: usize) {
+	use sp_runtime::traits::Hash;
+	let module = old::PrefabWasmModule {
+		instruction_weights_version: 0,
+		initial: 0,
+		maximum: 0,
+		code: vec![42u8; len],
+	};
+	let hash = T::Hashing::hash(&module.code);
+	old::CodeStorage::<T>::insert(hash, module);
+}
+
+#[derive(Encode, Decode)]
+struct PrefabWasmModule {
+	#[codec(compact)]
+	pub instruction_weights_version: u32,
+	#[codec(compact)]
+	pub initial: u32,
+	#[codec(compact)]
+	pub maximum: u32,
+	pub code: Vec<u8>,
+	pub determinism: Determinism,
+}
+
+#[storage_alias]
+type CodeStorage<T: Config> = StorageMap<Pallet<T>, Identity, CodeHash<T>, PrefabWasmModule>;
+
+#[derive(Encode, Decode, MaxEncodedLen, DefaultNoBound)]
+pub struct Migration<T: Config> {
+	last_key: Option<BoundedVec<u8, ConstU32<256>>>,
+	_phantom: PhantomData<T>,
+}
+
+impl<T: Config> Migrate for Migration<T> {
+	const VERSION: u16 = 9;
+
+	fn max_step_weight() -> Weight {
+		T::WeightInfo::v9_migration_step(T::MaxCodeLen::get())
+	}
+
+	fn step(&mut self) -> (IsFinished, Weight) {
+		let mut iter = if let Some(last_key) = self.last_key.take() {
+			old::CodeStorage::<T>::iter_from(last_key.to_vec())
+		} else {
+			old::CodeStorage::<T>::iter()
+		};
+
+		if let Some((key, old)) = iter.next() {
+			log::debug!(target: LOG_TARGET, "Migrating contract code {:?}", key);
+			let len = old.code.len() as u32;
+			let module = PrefabWasmModule {
+				instruction_weights_version: old.instruction_weights_version,
+				initial: old.initial,
+				maximum: old.maximum,
+				code: old.code,
+				determinism: Determinism::Enforced,
+			};
+			CodeStorage::<T>::insert(key, module);
+			self.last_key = Some(iter.last_raw_key().to_vec().try_into().unwrap());
+			(IsFinished::No, T::WeightInfo::v9_migration_step(len))
+		} else {
+			log::debug!(target: LOG_TARGET, "No more contracts code to migrate");
+			(IsFinished::Yes, T::WeightInfo::v9_migration_step(0))
+		}
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn pre_upgrade_step() -> Result<Vec<u8>, TryRuntimeError> {
+		let sample: Vec<_> = old::CodeStorage::<T>::iter().take(100).collect();
+
+		log::debug!(target: LOG_TARGET, "Taking sample of {} contract codes", sample.len());
+		Ok(sample.encode())
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn post_upgrade_step(state: Vec<u8>) -> Result<(), TryRuntimeError> {
+		let sample =
+			<Vec<(CodeHash<T>, old::PrefabWasmModule)> as Decode>::decode(&mut &state[..]).unwrap();
+
+		log::debug!(target: LOG_TARGET, "Validating sample of {} contract codes", sample.len());
+		for (code_hash, old) in sample {
+			let module = CodeStorage::<T>::get(&code_hash).unwrap();
+			ensure!(
+				module.instruction_weights_version == old.instruction_weights_version,
+				"invalid isntruction weights version"
+			);
+			ensure!(module.determinism == Determinism::Enforced, "invalid determinism");
+			ensure!(module.initial == old.initial, "invalid initial");
+			ensure!(module.maximum == old.maximum, "invalid maximum");
+			ensure!(module.code == old.code, "invalid code");
+			ensure!(module.maximum == old.maximum, "invalid maximum");
+			ensure!(module.code == old.code, "invalid code");
+		}
+
+		Ok(())
+	}
+}