contracts: Lazy storage removal (#7740)

* Do not evict a contract from within a call stack We don't want to trigger contract eviction automatically when a contract is called. This is because those changes can be reverted due to how storage transactions are used at the moment. More Information: https://github.com/paritytech/substrate/issues/6439#issuecomment-648754324 It can be re-introduced once the linked issue is resolved. In the meantime `claim_surcharge` must be called to evict a contract. * Lazily delete storage in on_initialize instead of when removing the contract * Add missing documentation of new error * Make Module::claim_surcharge public It being the only dispatchable that is private is an oversight. * review: Add final newline * review: Simplify assert statement * Add test that checks that partial remove of a contract works * Premote warning to error * Added missing docs for seal_terminate * Lazy deletion should only take AVERAGE_ON_INITIALIZE_RATIO of the block * Added informational about the lazy deletion throughput * Avoid lazy deletion in case the block is already full * Prevent queue decoding in case of an already full block * Add test that checks that on_initialize honors block limits
2026-06-12 18:11:10 +00:00 · 2021-01-04 13:35:57 +01:00
parent f0b99dd2f2
commit 3ba8fdfc11
9 changed files with 1348 additions and 525 deletions
@@ -32,12 +32,14 @@ use sp_runtime::{
 	testing::{Header, H256},
 	AccountId32,
 };
+use sp_io::hashing::blake2_256;
 use frame_support::{
-	assert_ok, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event,
+	assert_ok, assert_err, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event,
 	impl_outer_origin, parameter_types, StorageMap,
-	traits::{Currency, ReservableCurrency},
-	weights::{Weight, PostDispatchInfo},
+	traits::{Currency, ReservableCurrency, OnInitialize},
+	weights::{Weight, PostDispatchInfo, DispatchClass, constants::WEIGHT_PER_SECOND},
 	dispatch::DispatchErrorWithPostInfo,
+	storage::child,
 };
 use frame_system::{self as system, EventRecord, Phase};

@@ -189,12 +191,12 @@ pub struct Test;
 parameter_types! {
 	pub const BlockHashCount: u64 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
-		frame_system::limits::BlockWeights::simple_max(1024);
+		frame_system::limits::BlockWeights::simple_max(2 * WEIGHT_PER_SECOND);
 	pub static ExistentialDeposit: u64 = 0;
 }
 impl frame_system::Config for Test {
 	type BaseCallFilter = ();
-	type BlockWeights = ();
+	type BlockWeights = BlockWeights;
 	type BlockLength = ();
 	type DbWeight = ();
 	type Origin = Origin;
@@ -243,6 +245,8 @@ parameter_types! {
 	pub const SurchargeReward: u64 = 150;
 	pub const MaxDepth: u32 = 100;
 	pub const MaxValueSize: u32 = 16_384;
+	pub const DeletionQueueDepth: u32 = 1024;
+	pub const DeletionWeightLimit: Weight = 500_000_000_000;
 }

 parameter_types! {
@@ -272,6 +276,8 @@ impl Config for Test {
 	type WeightPrice = Self;
 	type WeightInfo = ();
 	type ChainExtension = TestExtension;
+	type DeletionQueueDepth = DeletionQueueDepth;
+	type DeletionWeightLimit = DeletionWeightLimit;
 }

 type Balances = pallet_balances::Module<Test>;
@@ -859,15 +865,6 @@ fn deduct_blocks() {
 		});
 }

-#[test]
-fn call_contract_removals() {
-	removals(|addr| {
-		// Call on already-removed account might fail, and this is fine.
-		let _ = Contracts::call(Origin::signed(ALICE), addr, 0, GAS_LIMIT, call::null());
-		true
-	});
-}
-
 #[test]
 fn inherent_claim_surcharge_contract_removals() {
 	removals(|addr| Contracts::claim_surcharge(Origin::none(), addr, Some(ALICE)).is_ok());
@@ -918,7 +915,7 @@ fn claim_surcharge(blocks: u64, trigger_call: impl Fn(AccountIdOf<Test>) -> bool
 			initialize_block(blocks);

 			// Trigger rent through call
-			assert!(trigger_call(addr.clone()));
+			assert_eq!(trigger_call(addr.clone()), removes);

 			if removes {
 				assert!(ContractInfoOf::<Test>::get(&addr).unwrap().get_tombstone().is_some());
@@ -956,7 +953,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			let subsistence_threshold = 50 /*existential_deposit*/ + 16 /*tombstone_deposit*/;

 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_eq!(ContractInfoOf::<Test>::get(&addr).unwrap().get_alive().unwrap().rent_allowance, 1_000);
 			assert_eq!(Balances::free_balance(&addr), 100);

@@ -972,7 +969,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			initialize_block(20);

 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert!(ContractInfoOf::<Test>::get(&addr).unwrap().get_tombstone().is_some());
 			assert_eq!(Balances::free_balance(&addr), subsistence_threshold);
 		});
@@ -996,7 +993,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			let addr = Contracts::contract_address(&ALICE, &code_hash, &[]);

 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_eq!(
 				ContractInfoOf::<Test>::get(&addr)
 					.unwrap()
@@ -1023,7 +1020,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			initialize_block(20);

 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert!(ContractInfoOf::<Test>::get(&addr)
 				.unwrap()
 				.get_tombstone()
@@ -1052,7 +1049,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			let addr = Contracts::contract_address(&ALICE, &code_hash, &[]);

 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_eq!(
 				ContractInfoOf::<Test>::get(&addr)
 					.unwrap()
@@ -1096,7 +1093,7 @@ fn removals(trigger_call: impl Fn(AccountIdOf<Test>) -> bool) {
 			initialize_block(20);

 			// Trigger rent must have no effect
-			assert!(trigger_call(addr.clone()));
+			assert!(!trigger_call(addr.clone()));
 			assert_matches!(ContractInfoOf::<Test>::get(&addr), Some(ContractInfo::Tombstone(_)));
 			assert_eq!(Balances::free_balance(&addr), subsistence_threshold);
 		});
@@ -1131,25 +1128,23 @@ fn call_removed_contract() {
 			// Advance blocks
 			initialize_block(10);

-			// Calling contract should remove contract and fail.
+			// Calling contract should deny access because rent cannot be paid.
 			assert_err_ignore_postinfo!(
 				Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, call::null()),
 				Error::<Test>::NotCallable
 			);
-			// Calling a contract that is about to evict shall emit an event.
-			assert_eq!(System::events(), vec![
-				EventRecord {
-					phase: Phase::Initialization,
-					event: MetaEvent::contracts(RawEvent::Evicted(addr.clone(), true)),
-					topics: vec![],
-				},
-			]);
+			// No event is generated because the contract is not actually removed.
+			assert_eq!(System::events(), vec![]);

 			// Subsequent contract calls should also fail.
 			assert_err_ignore_postinfo!(
-				Contracts::call(Origin::signed(ALICE), addr, 0, GAS_LIMIT, call::null()),
+				Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, call::null()),
 				Error::<Test>::NotCallable
 			);
+
+			// A snitch can now remove the contract
+			assert_ok!(Contracts::claim_surcharge(Origin::none(), addr.clone(), Some(ALICE)));
+			assert!(ContractInfoOf::<Test>::get(&addr).unwrap().get_tombstone().is_some());
 		})
 }

@@ -1278,13 +1273,17 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage:
 			initialize_block(5);

 			// Call `BOB`, which makes it pay rent. Since the rent allowance is set to 0
-			// we expect that it will get removed leaving tombstone.
+			// we expect that it is no longer callable but keeps existing until someone
+			// calls `claim_surcharge`.
 			assert_err_ignore_postinfo!(
 				Contracts::call(
 					Origin::signed(ALICE), addr_bob.clone(), 0, GAS_LIMIT, call::null()
 				),
 				Error::<Test>::NotCallable
 			);
+			assert!(System::events().is_empty());
+			assert!(ContractInfoOf::<Test>::get(&addr_bob).unwrap().get_alive().is_some());
+			assert_ok!(Contracts::claim_surcharge(Origin::none(), addr_bob.clone(), Some(ALICE)));
 			assert!(ContractInfoOf::<Test>::get(&addr_bob).unwrap().get_tombstone().is_some());
 			assert_eq!(System::events(), vec![
 				EventRecord {
@@ -2134,3 +2133,354 @@ fn chain_extension_works() {
 	});
 }

+#[test]
+fn lazy_removal_works() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+
+		// Put value into the contracts child trie
+		child::put(trie, &[99], &42);
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		assert_matches!(child::get(trie, &[99]), Some(42));
+
+		// Run the lazy removal
+		Contracts::on_initialize(Weight::max_value());
+
+		// Value should be gone now
+		assert_matches!(child::get::<i32>(trie, &[99]), None);
+	});
+}
+
+#[test]
+fn lazy_removal_partial_remove_works() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+
+	// We create a contract with some extra keys above the weight limit
+	let extra_keys = 7u32;
+	let weight_limit = 5_000_000_000;
+	let (_, max_keys) = Storage::<Test>::deletion_budget(1, weight_limit);
+	let vals: Vec<_> = (0..max_keys + extra_keys).map(|i| {
+		(blake2_256(&i.encode()), (i as u32), (i as u32).encode())
+	})
+	.collect();
+
+	let mut ext = ExtBuilder::default().existential_deposit(50).build();
+
+	let trie = ext.execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+
+		// Put value into the contracts child trie
+		for val in &vals {
+			Storage::<Test>::write(
+				&addr,
+				&info.trie_id,
+				&val.0,
+				Some(val.2.clone()),
+			).unwrap();
+		}
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		trie.clone()
+	});
+
+	// The lazy removal limit only applies to the backend but not to the overlay.
+	// This commits all keys from the overlay to the backend.
+	ext.commit_all().unwrap();
+
+	ext.execute_with(|| {
+		// Run the lazy removal
+		let weight_used = Storage::<Test>::process_deletion_queue_batch(weight_limit);
+
+		// Weight should be exhausted because we could not even delete all keys
+		assert_eq!(weight_used, weight_limit);
+
+		let mut num_deleted = 0u32;
+		let mut num_remaining = 0u32;
+
+		for val in &vals {
+			match child::get::<u32>(&trie, &blake2_256(&val.0)) {
+				None => num_deleted += 1,
+				Some(x) if x == val.1 => num_remaining += 1,
+				Some(_) => panic!("Unexpected value in contract storage"),
+			}
+		}
+
+		// All but one key is removed
+		assert_eq!(num_deleted + num_remaining, vals.len() as u32);
+		assert_eq!(num_deleted, max_keys);
+		assert_eq!(num_remaining, extra_keys);
+	});
+}
+
+#[test]
+fn lazy_removal_does_no_run_on_full_block() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+		let max_keys = 30;
+
+		// Create some storage items for the contract.
+		let vals: Vec<_> = (0..max_keys).map(|i| {
+			(blake2_256(&i.encode()), (i as u32), (i as u32).encode())
+		})
+		.collect();
+
+		// Put value into the contracts child trie
+		for val in &vals {
+			Storage::<Test>::write(
+				&addr,
+				&info.trie_id,
+				&val.0,
+				Some(val.2.clone()),
+			).unwrap();
+		}
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		// Fill up the block which should prevent the lazy storage removal from running.
+		System::register_extra_weight_unchecked(
+			<Test as system::Config>::BlockWeights::get().max_block,
+			DispatchClass::Mandatory,
+		);
+
+		// Run the lazy removal without any limit so that all keys would be removed if there
+		// had been some weight left in the block.
+		let weight_used = Contracts::on_initialize(Weight::max_value());
+		let base = <<Test as crate::Config>::WeightInfo as crate::WeightInfo>::on_initialize();
+		assert_eq!(weight_used, base);
+
+		// All the keys are still in place
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		// Run the lazy removal directly which disregards the block limits
+		Storage::<Test>::process_deletion_queue_batch(Weight::max_value());
+
+		// Now the keys should be gone
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), None);
+		}
+	});
+}
+
+
+#[test]
+fn lazy_removal_does_not_use_all_weight() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+		let info = <ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+		let trie = &info.child_trie_info();
+		let weight_limit = 5_000_000_000;
+		let (weight_per_key, max_keys) = Storage::<Test>::deletion_budget(1, weight_limit);
+
+		// We create a contract with one less storage item than we can remove within the limit
+		let vals: Vec<_> = (0..max_keys - 1).map(|i| {
+			(blake2_256(&i.encode()), (i as u32), (i as u32).encode())
+		})
+		.collect();
+
+		// Put value into the contracts child trie
+		for val in &vals {
+			Storage::<Test>::write(
+				&addr,
+				&info.trie_id,
+				&val.0,
+				Some(val.2.clone()),
+			).unwrap();
+		}
+
+		// Terminate the contract
+		assert_ok!(Contracts::call(
+			Origin::signed(ALICE),
+			addr.clone(),
+			0,
+			GAS_LIMIT,
+			vec![],
+		));
+
+		// Contract info should be gone
+		assert!(!<ContractInfoOf::<Test>>::contains_key(&addr));
+
+		// But value should be still there as the lazy removal did not run, yet.
+		for val in &vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), Some(val.1));
+		}
+
+		// Run the lazy removal
+		let weight_used = Storage::<Test>::process_deletion_queue_batch(weight_limit);
+
+		// We have one less key in our trie than our weight limit suffices for
+		assert_eq!(weight_used, weight_limit - weight_per_key);
+
+		// All the keys are removed
+		for val in vals {
+			assert_eq!(child::get::<u32>(trie, &blake2_256(&val.0)), None);
+		}
+	});
+}
+
+#[test]
+fn deletion_queue_full() {
+	let (code, hash) = compile_module::<Test>("self_destruct").unwrap();
+	ExtBuilder::default().existential_deposit(50).build().execute_with(|| {
+		let subsistence = ConfigCache::<Test>::subsistence_threshold_uncached();
+		let _ = Balances::deposit_creating(&ALICE, 10 * subsistence);
+		assert_ok!(Contracts::put_code(Origin::signed(ALICE), code));
+
+		assert_ok!(
+			Contracts::instantiate(
+				Origin::signed(ALICE),
+				subsistence,
+				GAS_LIMIT,
+				hash.into(),
+				vec![],
+				vec![],
+			),
+		);
+
+		let addr = Contracts::contract_address(&ALICE, &hash, &[]);
+
+		// fill the deletion queue up until its limit
+		Storage::<Test>::fill_queue_with_dummies();
+
+		// Terminate the contract should fail
+		assert_err_ignore_postinfo!(
+			Contracts::call(
+				Origin::signed(ALICE),
+				addr.clone(),
+				0,
+				GAS_LIMIT,
+				vec![],
+			),
+			Error::<Test>::DeletionQueueFull,
+		);
+
+		// Contract should be alive because removal failed
+		<ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+
+		// make the contract ripe for eviction
+		initialize_block(5);
+
+		// eviction should fail for the same reason as termination
+		assert_err!(
+			Contracts::claim_surcharge(Origin::none(), addr.clone(), Some(ALICE)),
+			Error::<Test>::DeletionQueueFull,
+		);
+
+		// Contract should be alive because removal failed
+		<ContractInfoOf::<Test>>::get(&addr).unwrap().get_alive().unwrap();
+	});
+}