contracts: Lazy storage removal (#7740)

* Do not evict a contract from within a call stack

We don't want to trigger contract eviction automatically when
a contract is called. This is because those changes can be
reverted due to how storage transactions are used at the moment.
More Information:
https://github.com/paritytech/substrate/issues/6439#issuecomment-648754324

It can be re-introduced once the linked issue is resolved. In the meantime
`claim_surcharge` must be called to evict a contract.

* Lazily delete storage in on_initialize instead of when removing the contract

* Add missing documentation of new error

* Make Module::claim_surcharge public

It being the only dispatchable that is private is an oversight.

* review: Add final newline

* review: Simplify assert statement

* Add test that checks that partial remove of a contract works

* Premote warning to error

* Added missing docs for seal_terminate

* Lazy deletion should only take AVERAGE_ON_INITIALIZE_RATIO of the block

* Added informational about the lazy deletion throughput

* Avoid lazy deletion in case the block is already full

* Prevent queue decoding in case of an already full block

* Add test that checks that on_initialize honors block limits

substrate/bin/node/runtime/src/lib.rs

@@ -71,6 +71,7 @@ pub use pallet_transaction_payment::{Multiplier, TargetedFeeAdjustment, Currency
 use pallet_session::{historical as pallet_session_historical};
 use sp_inherents::{InherentData, CheckInherentsResult};
 use static_assertions::const_assert;
+use pallet_contracts::WeightInfo;
 
 #[cfg(any(feature = "std", test))]
 pub use sp_runtime::BuildStorage;
@@ -716,6 +717,15 @@ parameter_types! {
 	pub const MaxDepth: u32 = 32;
 	pub const StorageSizeOffset: u32 = 8;
 	pub const MaxValueSize: u32 = 16 * 1024;
+	// The lazy deletion runs inside on_initialize.
+	pub DeletionWeightLimit: Weight = AVERAGE_ON_INITIALIZE_RATIO *
+		RuntimeBlockWeights::get().max_block;
+	// The weight needed for decoding the queue should be less or equal than a fifth
+	// of the overall weight dedicated to the lazy deletion.
+	pub DeletionQueueDepth: u32 = ((DeletionWeightLimit::get() / (
+			<Runtime as pallet_contracts::Config>::WeightInfo::on_initialize_per_queue_item(1) -
+			<Runtime as pallet_contracts::Config>::WeightInfo::on_initialize_per_queue_item(0)
+		)) / 5) as u32;
 }
 
 impl pallet_contracts::Config for Runtime {
@@ -735,6 +745,8 @@ impl pallet_contracts::Config for Runtime {
 	type WeightPrice = pallet_transaction_payment::Module<Self>;
 	type WeightInfo = pallet_contracts::weights::SubstrateWeight<Self>;
 	type ChainExtension = ();
+	type DeletionQueueDepth = DeletionQueueDepth;
+	type DeletionWeightLimit = DeletionWeightLimit;
 }
 
 impl pallet_sudo::Config for Runtime {

substrate/frame/contracts/src/benchmarking/mod.rs

@@ -39,7 +39,7 @@ use self::{
 use frame_benchmarking::{benchmarks, account, whitelisted_caller};
 use frame_system::{Module as System, RawOrigin};
 use parity_wasm::elements::{Instruction, ValueType, BlockType};
-use sp_runtime::traits::{Hash, Bounded};
+use sp_runtime::traits::{Hash, Bounded, Zero};
 use sp_std::{default::Default, convert::{TryInto}, vec::Vec, vec};
 use pallet_contracts_primitives::RentProjection;
 
@@ -209,37 +209,52 @@ where
 	}
 }
 
-/// A `Contract` that was evicted after accumulating some storage.
+/// A `Contract` that contains some storage items.
 ///
-/// This is used to benchmark contract resurrection.
-struct Tombstone<T: Config> {
+/// This is used to benchmark contract destruction and resurection. Those operations'
+/// weight depend on the amount of storage accumulated.
+struct ContractWithStorage<T: Config> {
 	/// The contract that was evicted.
 	contract: Contract<T>,
 	/// The storage the contract held when it was avicted.
 	storage: Vec<(StorageKey, Vec<u8>)>,
 }
 
-impl<T: Config> Tombstone<T>
+impl<T: Config> ContractWithStorage<T>
 where
 	T: Config,
 	T::AccountId: UncheckedFrom<T::Hash> + AsRef<[u8]>,
 {
-	/// Create and evict a new contract with the supplied storage item count and size each.
+	/// Same as [`Self::with_code`] but with dummy contract code.
 	fn new(stor_num: u32, stor_size: u32) -> Result<Self, &'static str> {
-		let contract = Contract::<T>::new(WasmModule::dummy(), vec![], Endow::CollectRent)?;
+		Self::with_code(WasmModule::dummy(), stor_num, stor_size)
+	}
+
+	/// Create and evict a new contract with the supplied storage item count and size each.
+	fn with_code(code: WasmModule<T>, stor_num: u32, stor_size: u32) -> Result<Self, &'static str> {
+		let contract = Contract::<T>::new(code, vec![], Endow::CollectRent)?;
 		let storage_items = create_storage::<T>(stor_num, stor_size)?;
 		contract.store(&storage_items)?;
-		System::<T>::set_block_number(
-			contract.eviction_at()? + T::SignedClaimHandicap::get() + 5u32.into()
-		);
-		Rent::<T>::collect(&contract.account_id);
-		contract.ensure_tombstone()?;
-
-		Ok(Tombstone {
+		Ok(Self {
 			contract,
 			storage: storage_items,
 		})
 	}
+
+	/// Increase the system block number so that this contract is eligible for eviction.
+	fn set_block_num_for_eviction(&self) -> Result<(), &'static str>  {
+		System::<T>::set_block_number(
+			self.contract.eviction_at()? + T::SignedClaimHandicap::get() + 5u32.into()
+		);
+		Ok(())
+	}
+
+	/// Evict this contract.
+	fn evict(&mut self) -> Result<(), &'static str> {
+		self.set_block_num_for_eviction()?;
+		Rent::<T>::snitch_contract_should_be_evicted(&self.contract.account_id, Zero::zero())?;
+		self.contract.ensure_tombstone()
+	}
 }
 
 /// Generate `stor_num` storage items. Each has the size `stor_size`.
@@ -270,6 +285,30 @@ benchmarks! {
 	_ {
 	}
 
+	// The base weight without any actual work performed apart from the setup costs.
+	on_initialize {}: {
+		Storage::<T>::process_deletion_queue_batch(Weight::max_value())
+	}
+
+	on_initialize_per_trie_key {
+		let k in 0..1024;
+		let instance = ContractWithStorage::<T>::new(k, T::MaxValueSize::get())?;
+		Storage::<T>::queue_trie_for_deletion(&instance.contract.alive_info()?)?;
+	}: {
+		Storage::<T>::process_deletion_queue_batch(Weight::max_value())
+	}
+
+	on_initialize_per_queue_item {
+		let q in 0..1024.min(T::DeletionQueueDepth::get());
+		for i in 0 .. q {
+			let instance = Contract::<T>::with_index(i, WasmModule::dummy(), vec![], Endow::Max)?;
+			Storage::<T>::queue_trie_for_deletion(&instance.alive_info()?)?;
+			ContractInfoOf::<T>::remove(instance.account_id);
+		}
+	}: {
+		Storage::<T>::process_deletion_queue_batch(Weight::max_value())
+	}
+
 	// This extrinsic is pretty much constant as it is only a simple setter.
 	update_schedule {
 		let schedule = Schedule {
@@ -650,7 +689,8 @@ benchmarks! {
 		// Restore just moves the trie id from origin to destination and therefore
 		// does not depend on the size of the destination contract. However, to not
 		// trigger any edge case we won't use an empty contract as destination.
-		let tombstone = Tombstone::<T>::new(10, T::MaxValueSize::get())?;
+		let mut tombstone = ContractWithStorage::<T>::new(10, T::MaxValueSize::get())?;
+		tombstone.evict()?;
 
 		let dest = tombstone.contract.account_id.encode();
 		let dest_len = dest.len();
@@ -723,7 +763,8 @@ benchmarks! {
 
 	seal_restore_to_per_delta {
 		let d in 0 .. API_BENCHMARK_BATCHES;
-		let tombstone = Tombstone::<T>::new(0, 0)?;
+		let mut tombstone = ContractWithStorage::<T>::new(0, 0)?;
+		tombstone.evict()?;
 		let delta = create_storage::<T>(d * API_BENCHMARK_BATCH_SIZE, T::MaxValueSize::get())?;
 
 		let dest = tombstone.contract.account_id.encode();
@@ -2368,7 +2409,20 @@ benchmarks! {
 	#[extra]
 	print_schedule {
 		#[cfg(feature = "std")]
-		println!("{:#?}", Schedule::<T>::default());
+		{
+			let weight_per_key = T::WeightInfo::on_initialize_per_trie_key(1) -
+				T::WeightInfo::on_initialize_per_trie_key(0);
+			let weight_per_queue_item = T::WeightInfo::on_initialize_per_queue_item(1) -
+				T::WeightInfo::on_initialize_per_queue_item(0);
+			let weight_limit = T::DeletionWeightLimit::get();
+			let queue_depth: u64 = T::DeletionQueueDepth::get().into();
+			println!("{:#?}", Schedule::<T>::default());
+			println!("###############################################");
+			println!("Lazy deletion throughput per block (empty queue, full queue): {}, {}",
+				weight_limit / weight_per_key,
+				(weight_limit - weight_per_queue_item * queue_depth) / weight_per_key,
+			);
+		}
 		#[cfg(not(feature = "std"))]
 		return Err("Run this bench with a native runtime in order to see the schedule.");
 	}: {}
@@ -2394,6 +2448,10 @@ mod tests {
 		}
 	}
 
+	create_test!(on_initialize);
+	create_test!(on_initialize_per_trie_key);
+	create_test!(on_initialize_per_queue_item);
+
 	create_test!(update_schedule);
 	create_test!(put_code);
 	create_test!(instantiate);

substrate/frame/contracts/src/exec.rs

@@ -268,12 +268,12 @@ where
 			Err(Error::<T>::MaxCallDepthReached)?
 		}
 
-		// Assumption: `collect` doesn't collide with overlay because
-		// `collect` will be done on first call and destination contract and balance
-		// cannot be changed before the first call
-		// We do not allow 'calling' plain accounts. For transfering value
-		// `seal_transfer` must be used.
-		let contract = if let Some(ContractInfo::Alive(info)) = Rent::<T>::collect(&dest) {
+		// This charges the rent and denies access to a contract that is in need of
+		// eviction by returning `None`. We cannot evict eagerly here because those
+		// changes would be rolled back in case this contract is called by another
+		// contract.
+		// See: https://github.com/paritytech/substrate/issues/6439#issuecomment-648754324
+		let contract = if let Ok(Some(ContractInfo::Alive(info))) = Rent::<T>::charge(&dest) {
 			info
 		} else {
 			Err(Error::<T>::NotCallable)?
@@ -575,13 +575,16 @@ where
 			value,
 			self.ctx,
 		)?;
-		let self_trie_id = self.ctx.self_trie_id.as_ref().expect(
-			"this function is only invoked by in the context of a contract;\
-				a contract has a trie id;\
-				this can't be None; qed",
-		);
-		Storage::<T>::destroy_contract(&self_id, self_trie_id);
-		Ok(())
+		if let Some(ContractInfo::Alive(info)) = ContractInfoOf::<T>::take(&self_id) {
+			Storage::<T>::queue_trie_for_deletion(&info)?;
+			Ok(())
+		} else {
+			panic!(
+				"this function is only invoked by in the context of a contract;\
+				this contract is therefore alive;\
+				qed"
+			);
+		}
 	}
 
 	fn call(

substrate/frame/contracts/src/lib.rs

@@ -123,7 +123,7 @@ use frame_support::{
 	dispatch::{DispatchResult, DispatchResultWithPostInfo},
 	traits::{OnUnbalanced, Currency, Get, Time, Randomness},
 };
-use frame_system::{ensure_signed, ensure_root};
+use frame_system::{ensure_signed, ensure_root, Module as System};
 use pallet_contracts_primitives::{
 	RentProjectionResult, GetStorageResult, ContractAccessError, ContractExecResult, ExecResult,
 };
@@ -325,6 +325,12 @@ pub trait Config: frame_system::Config {
 
 	/// Type that allows the runtime authors to add new host functions for a contract to call.
 	type ChainExtension: chain_extension::ChainExtension;
+
+	/// The maximum number of tries that can be queued for deletion.
+	type DeletionQueueDepth: Get<u32>;
+
+	/// The maximum amount of weight that can be consumed per block for lazy trie removal.
+	type DeletionWeightLimit: Get<Weight>;
 }
 
 decl_error! {
@@ -396,6 +402,17 @@ decl_error! {
 		/// in this error. Note that this usually  shouldn't happen as deploying such contracts
 		/// is rejected.
 		NoChainExtension,
+		/// Removal of a contract failed because the deletion queue is full.
+		///
+		/// This can happen when either calling [`Module::claim_surcharge`] or `seal_terminate`.
+		/// The queue is filled by deleting contracts and emptied by a fixed amount each block.
+		/// Trying again during another block is the only way to resolve this issue.
+		DeletionQueueFull,
+		/// A contract could not be evicted because it has enough balance to pay rent.
+		///
+		/// This can be returned from [`Module::claim_surcharge`] because the target
+		/// contract has enough balance to pay for its rent.
+		ContractNotEvictable,
 	}
 }
 
@@ -449,8 +466,24 @@ decl_module! {
 		/// The maximum size of a storage value in bytes. A reasonable default is 16 KiB.
 		const MaxValueSize: u32 = T::MaxValueSize::get();
 
+		/// The maximum number of tries that can be queued for deletion.
+		const DeletionQueueDepth: u32 = T::DeletionQueueDepth::get();
+
+		/// The maximum amount of weight that can be consumed per block for lazy trie removal.
+		const DeletionWeightLimit: Weight = T::DeletionWeightLimit::get();
+
 		fn deposit_event() = default;
 
+		fn on_initialize() -> Weight {
+			// We do not want to go above the block limit and rather avoid lazy deletion
+			// in that case. This should only happen on runtime upgrades.
+			let weight_limit = T::BlockWeights::get().max_block
+				.saturating_sub(System::<T>::block_weight().total())
+				.min(T::DeletionWeightLimit::get());
+			Storage::<T>::process_deletion_queue_batch(weight_limit)
+				.saturating_add(T::WeightInfo::on_initialize())
+		}
+
 		/// Updates the schedule for metering contracts.
 		///
 		/// The schedule must have a greater version than the stored schedule.
@@ -549,10 +582,14 @@ decl_module! {
 		/// Allows block producers to claim a small reward for evicting a contract. If a block producer
 		/// fails to do so, a regular users will be allowed to claim the reward.
 		///
-		/// If contract is not evicted as a result of this call, no actions are taken and
-		/// the sender is not eligible for the reward.
+		/// If contract is not evicted as a result of this call, [`Error::ContractNotEvictable`]
+		/// is returned and the sender is not eligible for the reward.
 		#[weight = T::WeightInfo::claim_surcharge()]
-		fn claim_surcharge(origin, dest: T::AccountId, aux_sender: Option<T::AccountId>) {
+		pub fn claim_surcharge(
+			origin,
+			dest: T::AccountId,
+			aux_sender: Option<T::AccountId>
+		) -> DispatchResult {
 			let origin = origin.into();
 			let (signed, rewarded) = match (origin, aux_sender) {
 				(Ok(frame_system::RawOrigin::Signed(account)), None) => {
@@ -574,8 +611,10 @@ decl_module! {
 			};
 
 			// If poking the contract has lead to eviction of the contract, give out the rewards.
-			if Rent::<T>::snitch_contract_should_be_evicted(&dest, handicap) {
-				T::Currency::deposit_into_existing(&rewarded, T::SurchargeReward::get())?;
+			if Rent::<T>::snitch_contract_should_be_evicted(&dest, handicap)? {
+				T::Currency::deposit_into_existing(&rewarded, T::SurchargeReward::get()).map(|_| ())
+			} else {
+				Err(Error::<T>::ContractNotEvictable.into())
 			}
 		}
 	}
@@ -733,6 +772,11 @@ decl_storage! {
 		///
 		/// TWOX-NOTE: SAFE since `AccountId` is a secure hash.
 		pub ContractInfoOf: map hasher(twox_64_concat) T::AccountId => Option<ContractInfo<T>>;
+		/// Evicted contracts that await child trie deletion.
+		///
+		/// Child trie deletion is a heavy operation depending on the amount of storage items
+		/// stored in said trie. Therefore this operation is performed lazily in `on_initialize`.
+		pub DeletionQueue: Vec<storage::DeletedContract>;
 	}
 }
 

substrate/frame/contracts/src/rent.rs

@@ -20,13 +20,16 @@
 use crate::{
 	AliveContractInfo, BalanceOf, ContractInfo, ContractInfoOf, Module, RawEvent,
 	TombstoneContractInfo, Config, CodeHash, ConfigCache, Error,
+	storage::Storage,
 };
 use sp_std::prelude::*;
 use sp_io::hashing::blake2_256;
 use sp_core::crypto::UncheckedFrom;
-use frame_support::storage::child;
-use frame_support::traits::{Currency, ExistenceRequirement, Get, OnUnbalanced, WithdrawReasons};
-use frame_support::StorageMap;
+use frame_support::{
+	debug, StorageMap,
+	storage::child,
+	traits::{Currency, ExistenceRequirement, Get, OnUnbalanced, WithdrawReasons},
+};
 use pallet_contracts_primitives::{ContractAccessError, RentProjection, RentProjectionResult};
 use sp_runtime::{
 	DispatchError,
@@ -74,10 +77,6 @@ enum Verdict<T: Config> {
 	/// For example, it already paid its rent in the current block, or it has enough deposit for not
 	/// paying rent at all.
 	Exempt,
-	/// Funds dropped below the subsistence deposit.
-	///
-	/// Remove the contract along with it's storage.
-	Kill,
 	/// The contract cannot afford payment within its rent budget so it gets evicted. However,
 	/// because its balance is greater than the subsistence threshold it leaves a tombstone.
 	Evict {
@@ -181,11 +180,17 @@ where
 		let rent_budget = match Self::rent_budget(&total_balance, &free_balance, contract) {
 			Some(rent_budget) => rent_budget,
 			None => {
-				// The contract's total balance is already below subsistence threshold. That
-				// indicates that the contract cannot afford to leave a tombstone.
-				//
-				// So cleanly wipe the contract.
-				return Verdict::Kill;
+				// All functions that allow a contract to transfer balance enforce
+				// that the contract always stays above the subsistence threshold.
+				// We want the rent system to always leave a tombstone to prevent the
+				// accidental loss of a contract. Ony `seal_terminate` can remove a
+				// contract without a tombstone. Therefore this case should be never
+				// hit.
+				debug::error!(
+					"Tombstoned a contract that is below the subsistence threshold: {:?}",
+					account
+				);
+				0u32.into()
 			}
 		};
 
@@ -234,19 +239,19 @@ where
 		alive_contract_info: AliveContractInfo<T>,
 		current_block_number: T::BlockNumber,
 		verdict: Verdict<T>,
-	) -> Option<ContractInfo<T>> {
+		allow_eviction: bool,
+	) -> Result<Option<ContractInfo<T>>, DispatchError> {
 		match verdict {
-			Verdict::Exempt => return Some(ContractInfo::Alive(alive_contract_info)),
-			Verdict::Kill => {
-				<ContractInfoOf<T>>::remove(account);
-				child::kill_storage(
-					&alive_contract_info.child_trie_info(),
-					None,
-				);
-				<Module<T>>::deposit_event(RawEvent::Evicted(account.clone(), false));
-				None
+			Verdict::Exempt => return Ok(Some(ContractInfo::Alive(alive_contract_info))),
+			Verdict::Evict { amount: _ } if !allow_eviction => {
+				Ok(None)
 			}
 			Verdict::Evict { amount } => {
+				// We need to remove the trie first because it is the only operation
+				// that can fail and this function is called without a storage
+				// transaction when called through `claim_surcharge`.
+				Storage::<T>::queue_trie_for_deletion(&alive_contract_info)?;
+
 				if let Some(amount) = amount {
 					amount.withdraw(account);
 				}
@@ -262,14 +267,8 @@ where
 				);
 				let tombstone_info = ContractInfo::Tombstone(tombstone);
 				<ContractInfoOf<T>>::insert(account, &tombstone_info);
-
-				child::kill_storage(
-					&alive_contract_info.child_trie_info(),
-					None,
-				);
-
 				<Module<T>>::deposit_event(RawEvent::Evicted(account.clone(), true));
-				Some(tombstone_info)
+				Ok(Some(tombstone_info))
 			}
 			Verdict::Charge { amount } => {
 				let contract_info = ContractInfo::Alive(AliveContractInfo::<T> {
@@ -278,21 +277,21 @@ where
 					..alive_contract_info
 				});
 				<ContractInfoOf<T>>::insert(account, &contract_info);
-
 				amount.withdraw(account);
-				Some(contract_info)
+				Ok(Some(contract_info))
 			}
 		}
 	}
 
 	/// Make account paying the rent for the current block number
 	///
-	/// NOTE this function performs eviction eagerly. All changes are read and written directly to
-	/// storage.
-	pub fn collect(account: &T::AccountId) -> Option<ContractInfo<T>> {
+	/// This functions does **not** evict the contract. It returns `None` in case the
+	/// contract is in need of eviction. [`snitch_contract_should_be_evicted`] must
+	/// be called to perform the eviction.
+	pub fn charge(account: &T::AccountId) -> Result<Option<ContractInfo<T>>, DispatchError> {
 		let contract_info = <ContractInfoOf<T>>::get(account);
 		let alive_contract_info = match contract_info {
-			None | Some(ContractInfo::Tombstone(_)) => return contract_info,
+			None | Some(ContractInfo::Tombstone(_)) => return Ok(contract_info),
 			Some(ContractInfo::Alive(contract)) => contract,
 		};
 
@@ -303,7 +302,7 @@ where
 			Zero::zero(),
 			&alive_contract_info,
 		);
-		Self::enact_verdict(account, alive_contract_info, current_block_number, verdict)
+		Self::enact_verdict(account, alive_contract_info, current_block_number, verdict, false)
 	}
 
 	/// Process a report that a contract under the given address should be evicted.
@@ -321,10 +320,10 @@ where
 	pub fn snitch_contract_should_be_evicted(
 		account: &T::AccountId,
 		handicap: T::BlockNumber,
-	) -> bool {
-		let contract_info = <ContractInfoOf<T>>::get(account);
-		let alive_contract_info = match contract_info {
-			None | Some(ContractInfo::Tombstone(_)) => return false,
+	) -> Result<bool, DispatchError> {
+		let contract = <ContractInfoOf<T>>::get(account);
+		let contract = match contract {
+			None | Some(ContractInfo::Tombstone(_)) => return Ok(false),
 			Some(ContractInfo::Alive(contract)) => contract,
 		};
 		let current_block_number = <frame_system::Module<T>>::block_number();
@@ -332,16 +331,16 @@ where
 			account,
 			current_block_number,
 			handicap,
-			&alive_contract_info,
+			&contract,
 		);
 
 		// Enact the verdict only if the contract gets removed.
 		match verdict {
-			Verdict::Kill | Verdict::Evict { .. } => {
-				Self::enact_verdict(account, alive_contract_info, current_block_number, verdict);
-				true
+			Verdict::Evict { .. } => {
+				Self::enact_verdict(account, contract, current_block_number, verdict, true)?;
+				Ok(true)
 			}
-			_ => false,
+			_ => Ok(false),
 		}
 	}
 
@@ -359,9 +358,11 @@ where
 	pub fn compute_projection(
 		account: &T::AccountId,
 	) -> RentProjectionResult<T::BlockNumber> {
+		use ContractAccessError::IsTombstone;
+
 		let contract_info = <ContractInfoOf<T>>::get(account);
 		let alive_contract_info = match contract_info {
-			None | Some(ContractInfo::Tombstone(_)) => return Err(ContractAccessError::IsTombstone),
+			None | Some(ContractInfo::Tombstone(_)) => return Err(IsTombstone),
 			Some(ContractInfo::Alive(contract)) => contract,
 		};
 		let current_block_number = <frame_system::Module<T>>::block_number();
@@ -372,11 +373,11 @@ where
 			&alive_contract_info,
 		);
 		let new_contract_info =
-			Self::enact_verdict(account, alive_contract_info, current_block_number, verdict);
+			Self::enact_verdict(account, alive_contract_info, current_block_number, verdict, false);
 
 		// Check what happened after enaction of the verdict.
-		let alive_contract_info = match new_contract_info {
-			None | Some(ContractInfo::Tombstone(_)) => return Err(ContractAccessError::IsTombstone),
+		let alive_contract_info = match new_contract_info.map_err(|_| IsTombstone)? {
+			None | Some(ContractInfo::Tombstone(_)) => return Err(IsTombstone),
 			Some(ContractInfo::Alive(contract)) => contract,
 		};
 

substrate/frame/contracts/src/storage.rs

@@ -20,20 +20,37 @@
 use crate::{
 	exec::{AccountIdOf, StorageKey},
 	AliveContractInfo, BalanceOf, CodeHash, ContractInfo, ContractInfoOf, Config, TrieId,
-	AccountCounter,
+	AccountCounter, DeletionQueue, Error,
+	weights::WeightInfo,
 };
+use codec::{Encode, Decode};
 use sp_std::prelude::*;
 use sp_std::marker::PhantomData;
 use sp_io::hashing::blake2_256;
 use sp_runtime::traits::Bounded;
 use sp_core::crypto::UncheckedFrom;
-use frame_support::{storage::child, StorageMap};
+use frame_support::{
+	dispatch::DispatchResult,
+	StorageMap,
+	debug,
+	storage::{child::{self, KillOutcome}, StorageValue},
+	traits::Get,
+	weights::Weight,
+};
 
 /// An error that means that the account requested either doesn't exist or represents a tombstone
 /// account.
 #[cfg_attr(test, derive(PartialEq, Eq, Debug))]
 pub struct ContractAbsentError;
 
+#[derive(Encode, Decode)]
+pub struct DeletedContract {
+	pair_count: u32,
+	trie_id: TrieId,
+}
+
+
+
 pub struct Storage<T>(PhantomData<T>);
 
 impl<T> Storage<T>
@@ -191,18 +208,105 @@ where
 		})
 	}
 
-	/// Removes the contract and all the storage associated with it.
+	/// Push a contract's trie to the deletion queue for lazy removal.
 	///
-	/// This function doesn't affect the account.
-	pub fn destroy_contract(address: &AccountIdOf<T>, trie_id: &TrieId) {
-		<ContractInfoOf<T>>::remove(address);
-		child::kill_storage(&crate::child_trie_info(&trie_id), None);
+	/// You must make sure that the contract is also removed or converted into a tombstone
+	/// when queuing the trie for deletion.
+	pub fn queue_trie_for_deletion(contract: &AliveContractInfo<T>) -> DispatchResult {
+		if DeletionQueue::decode_len().unwrap_or(0) >= T::DeletionQueueDepth::get() as usize {
+			Err(Error::<T>::DeletionQueueFull.into())
+		} else {
+			DeletionQueue::append(DeletedContract {
+				pair_count: contract.total_pair_count,
+				trie_id: contract.trie_id.clone(),
+			});
+			Ok(())
+		}
+	}
+
+	/// Calculates the weight that is necessary to remove one key from the trie and how many
+	/// of those keys can be deleted from the deletion queue given the supplied queue length
+	/// and weight limit.
+	pub fn deletion_budget(queue_len: usize, weight_limit: Weight) -> (u64, u32) {
+		let base_weight = T::WeightInfo::on_initialize();
+		let weight_per_queue_item = T::WeightInfo::on_initialize_per_queue_item(1) -
+			T::WeightInfo::on_initialize_per_queue_item(0);
+		let weight_per_key = T::WeightInfo::on_initialize_per_trie_key(1) -
+			T::WeightInfo::on_initialize_per_trie_key(0);
+		let decoding_weight = weight_per_queue_item.saturating_mul(queue_len as Weight);
+
+		// `weight_per_key` being zero makes no sense and would constitute a failure to
+		// benchmark properly. We opt for not removing any keys at all in this case.
+		let key_budget = weight_limit
+			.saturating_sub(base_weight)
+			.saturating_sub(decoding_weight)
+			.checked_div(weight_per_key)
+			.unwrap_or(0) as u32;
+
+		(weight_per_key, key_budget)
+	}
+
+	/// Delete as many items from the deletion queue possible within the supplied weight limit.
+	///
+	/// It returns the amount of weight used for that task or `None` when no weight was used
+	/// apart from the base weight.
+	pub fn process_deletion_queue_batch(weight_limit: Weight) -> Weight {
+		let queue_len = DeletionQueue::decode_len().unwrap_or(0);
+		if queue_len == 0 {
+			return weight_limit;
+		}
+
+		let (weight_per_key, mut remaining_key_budget) = Self::deletion_budget(
+			queue_len,
+			weight_limit,
+		);
+
+		// We want to check whether we have enough weight to decode the queue before
+		// proceeding. Too little weight for decoding might happen during runtime upgrades
+		// which consume the whole block before the other `on_initialize` blocks are called.
+		if remaining_key_budget == 0 {
+			return weight_limit;
+		}
+
+		let mut queue = DeletionQueue::get();
+
+		while !queue.is_empty() && remaining_key_budget > 0 {
+			// Cannot panic due to loop condition
+			let trie = &mut queue[0];
+			let pair_count = trie.pair_count;
+			let outcome = child::kill_storage(
+				&crate::child_trie_info(&trie.trie_id),
+				Some(remaining_key_budget),
+			);
+			if pair_count > remaining_key_budget {
+				// Cannot underflow because of the if condition
+				trie.pair_count -= remaining_key_budget;
+			} else {
+				// We do not care to preserve order. The contract is deleted already and
+				// noone waits for the trie to be deleted.
+				let removed = queue.swap_remove(0);
+				match outcome {
+					// This should not happen as our budget was large enough to remove all keys.
+					KillOutcome::SomeRemaining => {
+						debug::error!(
+							"After deletion keys are remaining in this child trie: {:?}",
+							removed.trie_id,
+						);
+					},
+					KillOutcome::AllRemoved => (),
+				}
+			}
+			remaining_key_budget = remaining_key_budget
+				.saturating_sub(remaining_key_budget.min(pair_count));
+		}
+
+		DeletionQueue::put(queue);
+		weight_limit.saturating_sub(weight_per_key.saturating_mul(remaining_key_budget as Weight))
 	}
 
 	/// This generator uses inner counter for account id and applies the hash over `AccountId +
 	/// accountid_counter`.
 	pub fn generate_trie_id(account_id: &AccountIdOf<T>) -> TrieId {
-		use frame_support::StorageValue;
 		use sp_runtime::traits::Hash;
 		// Note that skipping a value due to error is not an issue here.
 		// We only need uniqueness, not sequence.
@@ -226,4 +330,15 @@ where
 			.and_then(|i| i.as_alive().map(|i| i.code_hash))
 			.ok_or(ContractAbsentError)
 	}
+
+	/// Fill up the queue in order to exercise the limits during testing.
+	#[cfg(test)]
+	pub fn fill_queue_with_dummies() {
+		let queue: Vec<_> = (0..T::DeletionQueueDepth::get()).map(|_| DeletedContract {
+			pair_count: 0,
+			trie_id: vec![],
+		})
+		.collect();
+		DeletionQueue::put(queue);
+	}
 }

substrate/frame/contracts/src/tests.rs

@@ -32,12 +32,14 @@ use sp_runtime::{
 	testing::{Header, H256},
 	AccountId32,
 };
+use sp_io::hashing::blake2_256;
 use frame_support::{
-	assert_ok, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event,
+	assert_ok, assert_err, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event,
 	impl_outer_origin, parameter_types, StorageMap,
-	traits::{Currency, ReservableCurrency},
-	weights::{Weight, PostDispatchInfo},
+	traits::{Currency, ReservableCurrency, OnInitialize},
+	weights::{Weight, PostDispatchInfo, DispatchClass, constants::WEIGHT_PER_SECOND},
 	dispatch::DispatchErrorWithPostInfo,
+	storage::child,
 };
 use frame_system::{self as system, EventRecord, Phase};
 
@@ -189,12 +191,12 @@ pub struct Test;
 parameter_types! {
 	pub const BlockHashCount: u64 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
-		frame_system::limits::BlockWeights::simple_max(1024);
+		frame_system::limits::BlockWeights::simple_max(2 * WEIGHT_PER_SECOND);
 	pub static ExistentialDeposit: u64 = 0;
 }
 impl frame_system::Config for Test {
 	type BaseCallFilter = ();
-	type BlockWeights = ();
+	type BlockWeights = BlockWeights;
 	type BlockLength = ();
 	type DbWeight = ();
 	type Origin = Origin;
@@ -243,6 +245,8 @@ parameter_types! {
 	pub const SurchargeReward: u64 = 150;
 	pub const MaxDepth: u32 = 100;
 	pub const MaxValueSize: u32 = 16_384;
+	pub const DeletionQueueDepth: u32 = 1024;
+	pub const DeletionWeightLimit: Weight = 500_000_000_000;
 }
 
 parameter_types! {
@@ -272,6 +276,8 @@ impl Config for Test {
 	type WeightPrice = Self;
 	type WeightInfo = ();
 	type ChainExtension = TestExtension;
+	type DeletionQueueDepth = DeletionQueueDepth;
+	type DeletionWeightLimit = DeletionWeightLimit;
 }
 
 type Balances = pallet_balances::Module<Test>;
@@ -859,15 +865,6 @@ fn deduct_blocks() {
 		});
 }
 
-#[test]
-fn call_contract_removals() {
-	removals(|addr| {
-		// Call on already-removed account might fail, and this is fine.
-		let _ = Contracts::call(Origin::signed(ALICE), addr, 0, GAS_LIMIT, call::null());
-		true
-	});
-}
-
 #[test]
 fn inherent_claim_surcharge_contract_removals() {
 	removals(|addr| Contracts::claim_surcharge(Origin::none(), addr, Some(ALICE)).is_ok());
@@ -918,7 +915,7 @@ fn claim_surcharge(blocks: u64, trigger_call: impl Fn(AccountIdOf<Test>) -> bool
 			initialize_block(blocks);
 
 			// Trigger rent through call
-			assert!(trigger_call(addr.clone()));
+			assert_eq!(trigger_call(addr.clone()), removes);
 
 			if removes {
 				assert!(ContractInfoOf::<Test>::get(&addr).unwrap().get_tombstone().is_some());
@@ -956,7 +953,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			let subsistence_threshold = 50 /*existential_deposit*/ + 16 /*tombstone_deposit*/;
 
 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_eq!(ContractInfoOf::<Test>::get(&addr).unwrap().get_alive().unwrap().rent_allowance, 1_000);
 			assert_eq!(Balances::free_balance(&addr), 100);
 
@@ -972,7 +969,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			initialize_block(20);
 
 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert!(ContractInfoOf::<Test>::get(&addr).unwrap().get_tombstone().is_some());
 			assert_eq!(Balances::free_balance(&addr), subsistence_threshold);
 		});
@@ -996,7 +993,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			let addr = Contracts::contract_address(&ALICE, &code_hash, &[]);
 
 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_eq!(
 				ContractInfoOf::<Test>::get(&addr)
 					.unwrap()
@@ -1023,7 +1020,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			initialize_block(20);
 
 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert!(ContractInfoOf::<Test>::get(&addr)
 				.unwrap()
 				.get_tombstone()
@@ -1052,7 +1049,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			let addr = Contracts::contract_address(&ALICE, &code_hash, &[]);
 
 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_eq!(
 				ContractInfoOf::<Test>::get(&addr)
 					.unwrap()
@@ -1096,7 +1093,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			initialize_block(20);
 
 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_matches!(ContractInfoOf::<Test>::get(&addr), Some(ContractInfo::Tombstone(_)));
 			assert_eq!(Balances::free_balance(&addr), subsistence_threshold);
 		});
@@ -1131,25 +1128,23 @@ fn call_removed_contract() {
 			// Advance blocks
 			initialize_block(10);
 
-			// Calling contract should remove contract and fail.
+			// Calling contract should deny access because rent cannot be paid.
 			assert_err_ignore_postinfo!(
 				Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, call::null()),
 				Error::<Test>::NotCallable
 			);
-			// Calling a contract that is about to evict shall emit an event.
-			assert_eq!(System::events(), vec![
-				EventRecord {
-					phase: Phase::Initialization,
-					event: MetaEvent::contracts(RawEvent::Evicted(addr.clone(), true)),
-					topics: vec![],
-				},
-			]);
+			// No event is generated because the contract is not actually removed.
+			assert_eq!(System::events(), vec![]);
 
 			// Subsequent contract calls should also fail.
 			assert_err_ignore_postinfo!(
-				Contracts::call(Origin::signed(ALICE), addr, 0, GAS_LIMIT, call::null()),
+				Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, call::null()),
 				Error::<Test>::NotCallable
 			);
+
+			// A snitch can now remove the contract
+			assert_ok!(Contracts::claim_surcharge(Origin::none(), addr.clone(), Some(ALICE)));
+			assert!(ContractInfoOf::<Test>::get(&addr).unwrap().get_tombstone().is_some());
 		})
 }
 
@@ -1278,13 +1273,17 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage:
 			initialize_block(5);
 
 			// Call `BOB`, which makes it pay rent. Since the rent allowance is set to 0
-			// we expect that it will get removed leaving tombstone.
+			// we expect that it is no longer callable but keeps existing until someone
+			// calls `claim_surcharge`.
 			assert_err_ignore_postinfo!(
 				Contracts::call(
 					Origin::signed(ALICE), addr_bob.clone(), 0, GAS_LIMIT, call::null()
 				),
 				Error::<Test>::NotCallable
 			);
+			assert!(System::events().is_empty());
+			assert!(ContractInfoOf::<Test>::get(&addr_bob).unwrap().get_alive().is_some());
+			assert_ok!(Contracts::claim_surcharge(Origin::none(), addr_bob.clone(), Some(ALICE)));
 			assert!(ContractInfoOf::<Test>::get(&addr_bob).unwrap().get_tombstone().is_some());
 			assert_eq!(System::events(), vec![
 				EventRecord {
@@ -2134,3 +2133,354 @@ fn chain_extension_works() {
 	});
 }
 
+#[test]
+fn lazy_removal_works() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+
+		// Put value into the contracts child trie
+		child::put(trie, &[99], &42);
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		assert_matches!(child::get(trie, &[99]), Some(42));
+
+		// Run the lazy removal
+		Contracts::on_initialize(Weight::max_value());
+
+		// Value should be gone now
+		assert_matches!(child::get::<i32>(trie, &[99]), None);
+	});
+}
+
+#[test]
+fn lazy_removal_partial_remove_works() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+
+	// We create a contract with some extra keys above the weight limit
+	let extra_keys = 7u32;
+	let weight_limit = 5_000_000_000;
+	let (_, max_keys) = Storage::<Test>::deletion_budget(1, weight_limit);
+	let vals: Vec<_> = (0..max_keys + extra_keys).map(|i| {
+		(blake2_256(&i.encode()), (i as u32), (i as u32).encode())
+	})
+	.collect();
+
+	let mut ext = ExtBuilder::default().existential_deposit(50).build();
+
+	let trie = ext.execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+
+		// Put value into the contracts child trie
+		for val in &vals {
+			Storage::<Test>::write(
+				&addr,
+				&info.trie_id,
+				&val.0,
+				Some(val.2.clone()),
+			).unwrap();
+		}
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		trie.clone()
+	});
+
+	// The lazy removal limit only applies to the backend but not to the overlay.
+	// This commits all keys from the overlay to the backend.
+	ext.commit_all().unwrap();
+
+	ext.execute_with(|| {
+		// Run the lazy removal
+		let weight_used = Storage::<Test>::process_deletion_queue_batch(weight_limit);
+
+		// Weight should be exhausted because we could not even delete all keys
+		assert_eq!(weight_used, weight_limit);
+
+		let mut num_deleted = 0u32;
+		let mut num_remaining = 0u32;
+
+		for val in &vals {
+			match child::get::<u32>(&trie, &blake2_256(&val.0)) {
+				None => num_deleted += 1,
+				Some(x) if x == val.1 => num_remaining += 1,
+				Some(_) => panic!("Unexpected value in contract storage"),
+			}
+		}
+
+		// All but one key is removed
+		assert_eq!(num_deleted + num_remaining, vals.len() as u32);
+		assert_eq!(num_deleted, max_keys);
+		assert_eq!(num_remaining, extra_keys);
+	});
+}
+
+#[test]
+fn lazy_removal_does_no_run_on_full_block() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+		let max_keys = 30;
+
+		// Create some storage items for the contract.
+		let vals: Vec<_> = (0..max_keys).map(|i| {
+			(blake2_256(&i.encode()), (i as u32), (i as u32).encode())
+		})
+		.collect();
+
+		// Put value into the contracts child trie
+		for val in &vals {
+			Storage::<Test>::write(
+				&addr,
+				&info.trie_id,
+				&val.0,
+				Some(val.2.clone()),
+			).unwrap();
+		}
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		// Fill up the block which should prevent the lazy storage removal from running.
+		System::register_extra_weight_unchecked(
+			<Test as system::Config>::BlockWeights::get().max_block,
+			DispatchClass::Mandatory,
+		);
+
+		// Run the lazy removal without any limit so that all keys would be removed if there
+		// had been some weight left in the block.
+		let weight_used = Contracts::on_initialize(Weight::max_value());
+		let base = <<Test as crate::Config>::WeightInfo as crate::WeightInfo>::on_initialize();
+		assert_eq!(weight_used, base);
+
+		// All the keys are still in place
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		// Run the lazy removal directly which disregards the block limits
+		Storage::<Test>::process_deletion_queue_batch(Weight::max_value());
+
+		// Now the keys should be gone
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), None);
+		}
+	});
+}
+
+
+#[test]
+fn lazy_removal_does_not_use_all_weight() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+		let weight_limit = 5_000_000_000;
+		let (weight_per_key, max_keys) = Storage::<Test>::deletion_budget(1, weight_limit);
+
+		// We create a contract with one less storage item than we can remove within the limit
+		let vals: Vec<_> = (0..max_keys - 1).map(|i| {
+			(blake2_256(&i.encode()), (i as u32), (i as u32).encode())
+		})
+		.collect();
+
+		// Put value into the contracts child trie
+		for val in &vals {
+			Storage::<Test>::write(
+				&addr,
+				&info.trie_id,
+				&val.0,
+				Some(val.2.clone()),
+			).unwrap();
+		}
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		// Run the lazy removal
+		let weight_used = Storage::<Test>::process_deletion_queue_batch(weight_limit);
+
+		// We have one less key in our trie than our weight limit suffices for
+		assert_eq!(weight_used, weight_limit - weight_per_key);
+
+		// All the keys are removed
+		for val in vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), None);
+		}
+	});
+}
+
+#[test]
+fn deletion_queue_full() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+
+		// fill the deletion queue up until its limit
+		Storage::<Test>::fill_queue_with_dummies();
+
+		// Terminate the contract should fail
+		assert_err_ignore_postinfo!(
+			Contracts::call(
+				Origin::signed(ALICE),
+				addr.clone(),
+				0,
+				GAS_LIMIT,
+				vec![],
+			),
+			Error::<Test>::DeletionQueueFull,
+		);
+
+		// Contract should be alive because removal failed
+		<ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+
+		// make the contract ripe for eviction
+		initialize_block(5);
+
+		// eviction should fail for the same reason as termination
+		assert_err!(
+			Contracts::claim_surcharge(Origin::none(), addr.clone(), Some(ALICE)),
+			Error::<Test>::DeletionQueueFull,
+		);
+
+		// Contract should be alive because removal failed
+		<ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+	});
+}

substrate/frame/contracts/src/wasm/runtime.rs

@@ -923,6 +923,8 @@ define_env!(Env, <E: Ext>,
 	// # Traps
 	//
 	// - The contract is live i.e is already on the call stack.
+	// - Failed to send the balance to the beneficiary.
+	// - The deletion queue is full.
 	seal_terminate(
 		ctx,
 		beneficiary_ptr: u32,