Move pruning strategy to runtime level (#128)

* move pruning strategy to runtim level

* cargo fmt --all

* Update modules/ethereum/src/lib.rs

Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>

* finalize_headers -> finalize_and_prune_headers

* PruningStrategy::default()

* fn import_of_non_best_block_may_finalize_blocks()

Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>
This commit is contained in:
Svyatoslav Nikolsky
2020-06-16 16:46:30 +03:00
committed by Bastian Köcher
parent 7513775676
commit a0c8206684
5 changed files with 215 additions and 64 deletions
+48 -1
View File
@@ -14,11 +14,16 @@
// You should have received a copy of the GNU General Public License
// along with Parity Bridges Common. If not, see <http://www.gnu.org/licenses/>.
use frame_support::RuntimeDebug;
use hex_literal::hex;
use pallet_bridge_eth_poa::{AuraConfiguration, ValidatorsConfiguration, ValidatorsSource};
use pallet_bridge_eth_poa::{AuraConfiguration, PruningStrategy, ValidatorsConfiguration, ValidatorsSource};
use sp_bridge_eth_poa::{Address, Header, U256};
use sp_std::prelude::*;
/// Max number of finalized headers to keep. It is equivalent of ~24 hours of
/// finalized blocks on current Kovan chain.
const FINALIZED_HEADERS_TO_KEEP: u64 = 20_000;
/// Aura engine configuration for Kovan chain.
pub fn kovan_aura_configuration() -> AuraConfiguration {
AuraConfiguration {
@@ -102,3 +107,45 @@ pub fn kovan_genesis_header() -> Header {
],
}
}
/// Kovan headers pruning strategy.
///
/// We do not prune unfinalized headers because exchange module only accepts
/// claims from finalized headers. And if we're pruning unfinalized headers, then
/// some claims may never be accepted.
#[derive(Default, RuntimeDebug)]
pub struct KovanPruningStrategy;
impl PruningStrategy for KovanPruningStrategy {
fn pruning_upper_bound(&mut self, _best_number: u64, best_finalized_number: u64) -> u64 {
best_finalized_number
.checked_sub(FINALIZED_HEADERS_TO_KEEP)
.unwrap_or(0)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pruning_strategy_keeps_enough_headers() {
assert_eq!(
KovanPruningStrategy::default().pruning_upper_bound(100_000, 10_000),
0,
"10_000 <= 20_000 => nothing should be pruned yet",
);
assert_eq!(
KovanPruningStrategy::default().pruning_upper_bound(100_000, 20_000),
0,
"20_000 <= 20_000 => nothing should be pruned yet",
);
assert_eq!(
KovanPruningStrategy::default().pruning_upper_bound(100_000, 30_000),
10_000,
"20_000 <= 30_000 => we're ready to prune first 10_000 headers",
);
}
}
+1
View File
@@ -225,6 +225,7 @@ impl pallet_bridge_eth_poa::Trait for Runtime {
type AuraConfiguration = KovanAuraConfiguration;
type FinalityVotesCachingInterval = FinalityVotesCachingInterval;
type ValidatorsConfiguration = KovanValidatorsConfiguration;
type PruningStrategy = kovan::KovanPruningStrategy;
type OnHeadersSubmitted = ();
}
+101 -35
View File
@@ -18,19 +18,10 @@ use crate::error::Error;
use crate::finality::finalize_blocks;
use crate::validators::{Validators, ValidatorsConfiguration};
use crate::verification::{is_importable_header, verify_aura_header};
use crate::{AuraConfiguration, ChangeToEnact, Storage};
use crate::{AuraConfiguration, ChangeToEnact, PruningStrategy, Storage};
use primitives::{Header, HeaderId, Receipt};
use sp_std::{collections::btree_map::BTreeMap, prelude::*};
/// Maximal number of headers behind best blocks that we are aiming to store. When there
/// are too many unfinalized headers, it slows down finalization tracking significantly.
/// That's why we won't consider imports/reorganizations to blocks of PRUNE_DEPTH age.
/// If there's more headers than that, we prune the oldest. The only exception is
/// when unfinalized header schedules validators set change. We can't compute finality
/// for pruned headers => we won't know when to enact validators set change. That's
/// why we never prune headers with scheduled changes.
pub(crate) const PRUNE_DEPTH: u64 = 4096;
/// Imports bunch of headers and updates blocks finality.
///
/// Transactions receipts must be provided if `header_import_requires_receipts()`
@@ -40,11 +31,11 @@ pub(crate) const PRUNE_DEPTH: u64 = 4096;
/// we have NOT imported.
/// Returns error if fatal error has occured during import. Some valid headers may be
/// imported in this case.
pub fn import_headers<S: Storage>(
pub fn import_headers<S: Storage, PS: PruningStrategy>(
storage: &mut S,
pruning_strategy: &mut PS,
aura_config: &AuraConfiguration,
validators_config: &ValidatorsConfiguration,
prune_depth: u64,
submitter: Option<S::Submitter>,
headers: Vec<(Header, Option<Vec<Receipt>>)>,
finalized_headers: &mut BTreeMap<S::Submitter, u64>,
@@ -54,9 +45,9 @@ pub fn import_headers<S: Storage>(
for (header, receipts) in headers {
let import_result = import_header(
storage,
pruning_strategy,
aura_config,
validators_config,
prune_depth,
submitter.clone(),
header,
receipts,
@@ -85,11 +76,11 @@ pub fn import_headers<S: Storage>(
/// has returned true.
///
/// Returns imported block id and list of all finalized headers.
pub fn import_header<S: Storage>(
pub fn import_header<S: Storage, PS: PruningStrategy>(
storage: &mut S,
pruning_strategy: &mut PS,
aura_config: &AuraConfiguration,
validators_config: &ValidatorsConfiguration,
prune_depth: u64,
submitter: Option<S::Submitter>,
header: Header,
receipts: Option<Vec<Receipt>>,
@@ -126,10 +117,9 @@ pub fn import_header<S: Storage>(
// (because otherwise we'll have inconsistent storage if transaction will fail)
// and finally insert the block
let (_, best_total_difficulty) = storage.best_block();
let (best_id, best_total_difficulty) = storage.best_block();
let total_difficulty = import_context.total_difficulty() + header.difficulty;
let is_best = total_difficulty > best_total_difficulty;
let header_number = header.number;
storage.insert_header(import_context.into_import_header(
is_best,
header_id,
@@ -140,15 +130,19 @@ pub fn import_header<S: Storage>(
finalized_blocks.votes,
));
// now mark finalized headers && prune old headers
storage.finalize_headers(
finalized_blocks.finalized_headers.last().map(|(id, _)| *id),
match is_best {
true => header_number.checked_sub(prune_depth),
false => None,
},
// compute upper border of updated pruning range
let new_best_block_id = if is_best { header_id } else { best_id };
let new_best_finalized_block_id = finalized_blocks.finalized_headers.last().map(|(id, _)| *id);
let pruning_upper_bound = pruning_strategy.pruning_upper_bound(
new_best_block_id.number,
new_best_finalized_block_id
.map(|id| id.number)
.unwrap_or(finalized_id.number),
);
// now mark finalized headers && prune old headers
storage.finalize_and_prune_headers(new_best_finalized_block_id, pruning_upper_bound);
Ok((header_id, finalized_blocks.finalized_headers))
}
@@ -169,7 +163,7 @@ mod tests {
use super::*;
use crate::mock::{
block_i, custom_block_i, custom_test_ext, genesis, signed_header, test_aura_config, test_validators_config,
validator, validators, validators_addresses, TestRuntime,
validator, validators, validators_addresses, KeepSomeHeadersBehindBest, TestRuntime, GENESIS_STEP,
};
use crate::validators::ValidatorsSource;
use crate::{BlocksToPrune, BridgeStorage, Headers, PruningRange};
@@ -179,19 +173,19 @@ mod tests {
fn rejects_finalized_block_competitors() {
custom_test_ext(genesis(), validators_addresses(3)).execute_with(|| {
let mut storage = BridgeStorage::<TestRuntime>::new();
storage.finalize_headers(
storage.finalize_and_prune_headers(
Some(HeaderId {
number: 100,
..Default::default()
}),
None,
0,
);
assert_eq!(
import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&test_validators_config(),
PRUNE_DEPTH,
None,
Default::default(),
None,
@@ -210,9 +204,9 @@ mod tests {
assert_eq!(
import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&test_validators_config(),
PRUNE_DEPTH,
None,
block.clone(),
None,
@@ -223,9 +217,9 @@ mod tests {
assert_eq!(
import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&test_validators_config(),
PRUNE_DEPTH,
None,
block,
None,
@@ -250,9 +244,9 @@ mod tests {
assert_eq!(
import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&validators_config,
PRUNE_DEPTH,
None,
header,
None
@@ -285,9 +279,9 @@ mod tests {
let header = block_i(i, &validators);
let (rolling_last_block_id, finalized_blocks) = import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&validators_config,
10,
Some(100),
header,
None,
@@ -316,9 +310,9 @@ mod tests {
});
let (rolling_last_block_id, finalized_blocks) = import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&validators_config,
10,
Some(101),
header11.clone(),
Some(vec![crate::validators::tests::validators_change_recept(
@@ -352,9 +346,9 @@ mod tests {
expected_blocks.push((header.compute_id(), Some(102)));
let (rolling_last_block_id, finalized_blocks) = import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&validators_config,
10,
Some(102),
header,
None,
@@ -387,9 +381,9 @@ mod tests {
let header = signed_header(&validators, header, step as _);
let (_, finalized_blocks) = import_header(
&mut storage,
&mut KeepSomeHeadersBehindBest::default(),
&test_aura_config(),
&validators_config,
10,
Some(103),
header,
None,
@@ -405,4 +399,76 @@ mod tests {
);
});
}
#[test]
fn import_of_non_best_block_may_finalize_blocks() {
const TOTAL_VALIDATORS: u8 = 3;
let validators_addresses = validators_addresses(TOTAL_VALIDATORS);
custom_test_ext(genesis(), validators_addresses.clone()).execute_with(move || {
let validators = validators(TOTAL_VALIDATORS);
let validators_config = ValidatorsConfiguration::Single(ValidatorsSource::Contract(
[0; 20].into(),
validators_addresses.clone(),
));
let mut storage = BridgeStorage::<TestRuntime>::new();
let mut pruning_strategy = KeepSomeHeadersBehindBest::default();
// insert headers (H1, validator1), (H2, validator1), (H3, validator1)
// making H3 the best header, without finalizing anything (we need 2 signatures)
let mut expected_best_block = Default::default();
for i in 1..4 {
let step = GENESIS_STEP + i * TOTAL_VALIDATORS as u64;
let header = custom_block_i(i, &validators, |header| {
header.author = validators_addresses[0];
header.seal[0][0] = step as u8;
});
let header = signed_header(&validators, header, step);
expected_best_block = header.compute_id();
import_header(
&mut storage,
&mut pruning_strategy,
&test_aura_config(),
&validators_config,
None,
header,
None,
)
.unwrap();
}
let (best_block, best_difficulty) = storage.best_block();
assert_eq!(best_block, expected_best_block);
assert_eq!(storage.finalized_block(), genesis().compute_id());
// insert headers (H1', validator1), (H2', validator2), finalizing H2, even though H3
// has better difficulty than H2' (because there are more steps involved)
let mut expected_finalized_block = Default::default();
let mut parent_hash = genesis().compute_hash();
for i in 1..3 {
let header = custom_block_i(i, &validators, |header| {
header.gas_limit += 1.into();
header.parent_hash = parent_hash;
});
let header = signed_header(&validators, header, GENESIS_STEP + i);
parent_hash = header.compute_hash();
if i == 1 {
expected_finalized_block = header.compute_id();
}
import_header(
&mut storage,
&mut pruning_strategy,
&test_aura_config(),
&validators_config,
None,
header,
None,
)
.unwrap();
}
let (new_best_block, new_best_difficulty) = storage.best_block();
assert_eq!(new_best_block, expected_best_block);
assert_eq!(new_best_difficulty, best_difficulty);
assert_eq!(storage.finalized_block(), expected_finalized_block);
});
}
}
+39 -21
View File
@@ -286,15 +286,31 @@ pub trait Storage {
fn scheduled_change(&self, hash: &H256) -> Option<ScheduledChange>;
/// Insert imported header.
fn insert_header(&mut self, header: HeaderToImport<Self::Submitter>);
/// Finalize given block and prune all headers with number < prune_end.
/// Finalize given block and schedules pruning of all headers
/// with number < prune_end.
///
/// The headers in the pruning range could be either finalized, or not.
/// It is the storage duty to ensure that unfinalized headers that have
/// scheduled changes won't be pruned until they or their competitors
/// are finalized.
fn finalize_headers(&mut self, finalized: Option<HeaderId>, prune_end: Option<u64>);
fn finalize_and_prune_headers(&mut self, finalized: Option<HeaderId>, prune_end: u64);
}
/// Decides whether the session should be ended.
/// Headers pruning strategy.
pub trait PruningStrategy: Default {
/// Return upper bound (exclusive) of headers pruning range.
///
/// Every value that is returned from this function, must be greater or equal to the
/// previous value. Otherwise it will be ignored (we can't revert pruning).
///
/// Module may prune both finalized and unfinalized blocks. But it can't give any
/// guarantees on when it will happen. Example: if some unfinalized block at height N
/// has scheduled validators set change, then the module won't prune any blocks with
/// number >= N even if strategy allows that.
fn pruning_upper_bound(&mut self, best_number: u64, best_finalized_number: u64) -> u64;
}
/// Callbacks for header submission rewards/penalties.
pub trait OnHeadersSubmitted<AccountId> {
/// Called when valid headers have been submitted.
///
@@ -322,6 +338,9 @@ impl<AccountId> OnHeadersSubmitted<AccountId> for () {
pub trait Trait: frame_system::Trait {
/// Aura configuration.
type AuraConfiguration: Get<AuraConfiguration>;
/// Validators configuration.
type ValidatorsConfiguration: Get<validators::ValidatorsConfiguration>;
/// Interval (in blocks) for for finality votes caching.
/// If None, cache is disabled.
///
@@ -329,8 +348,9 @@ pub trait Trait: frame_system::Trait {
/// be any significant finalization delays), or something that is bit larger
/// than average finalization delay.
type FinalityVotesCachingInterval: Get<Option<u64>>;
/// Validators configuration.
type ValidatorsConfiguration: Get<validators::ValidatorsConfiguration>;
/// Headers pruning strategy.
type PruningStrategy: PruningStrategy;
/// Handler for headers submission result.
type OnHeadersSubmitted: OnHeadersSubmitted<Self::AccountId>;
}
@@ -344,9 +364,9 @@ decl_module! {
import::import_header(
&mut BridgeStorage::<T>::new(),
&mut T::PruningStrategy::default(),
&T::AuraConfiguration::get(),
&T::ValidatorsConfiguration::get(),
crate::import::PRUNE_DEPTH,
None,
header,
receipts,
@@ -365,9 +385,9 @@ decl_module! {
let mut finalized_headers = BTreeMap::new();
let import_result = import::import_headers(
&mut BridgeStorage::<T>::new(),
&mut T::PruningStrategy::default(),
&T::AuraConfiguration::get(),
&T::ValidatorsConfiguration::get(),
crate::import::PRUNE_DEPTH,
Some(submitter.clone()),
headers_with_receipts,
&mut finalized_headers,
@@ -539,16 +559,14 @@ impl<T: Trait> BridgeStorage<T> {
}
/// Prune old blocks.
fn prune_blocks(&self, mut max_blocks_to_prune: u64, finalized_number: u64, prune_end: Option<u64>) {
fn prune_blocks(&self, mut max_blocks_to_prune: u64, finalized_number: u64, prune_end: u64) {
let pruning_range = BlocksToPrune::get();
let mut new_pruning_range = pruning_range.clone();
// update oldest block we want to keep
if let Some(prune_end) = prune_end {
if prune_end > new_pruning_range.oldest_block_to_keep {
new_pruning_range.oldest_block_to_keep = prune_end;
}
}
// start pruning blocks
let begin = new_pruning_range.oldest_unpruned_block;
@@ -770,7 +788,7 @@ impl<T: Trait> Storage for BridgeStorage<T> {
);
}
fn finalize_headers(&mut self, finalized: Option<HeaderId>, prune_end: Option<u64>) {
fn finalize_and_prune_headers(&mut self, finalized: Option<HeaderId>, prune_end: u64) {
// remember just finalized block
let finalized_number = finalized
.as_ref()
@@ -928,7 +946,7 @@ pub(crate) mod tests {
});
// try to prune blocks [5; 10)
storage.prune_blocks(0xFFFF, 10, Some(5));
storage.prune_blocks(0xFFFF, 10, 5);
assert_eq!(HeadersByNumber::get(&5).unwrap().len(), 5);
assert_eq!(
BlocksToPrune::get(),
@@ -949,7 +967,7 @@ pub(crate) mod tests {
});
// try to prune blocks [5; 10)
storage.prune_blocks(0xFFFF, 10, Some(3));
storage.prune_blocks(0xFFFF, 10, 3);
assert_eq!(
BlocksToPrune::get(),
PruningRange {
@@ -964,7 +982,7 @@ pub(crate) mod tests {
fn blocks_are_not_pruned_if_limit_is_zero() {
with_headers_to_prune(|storage| {
// try to prune blocks [0; 10)
storage.prune_blocks(0, 10, Some(10));
storage.prune_blocks(0, 10, 10);
assert!(HeadersByNumber::get(&0).is_some());
assert!(HeadersByNumber::get(&1).is_some());
assert!(HeadersByNumber::get(&2).is_some());
@@ -983,7 +1001,7 @@ pub(crate) mod tests {
fn blocks_are_pruned_if_limit_is_non_zero() {
with_headers_to_prune(|storage| {
// try to prune blocks [0; 10)
storage.prune_blocks(7, 10, Some(10));
storage.prune_blocks(7, 10, 10);
// 1 headers with number = 0 is pruned (1 total)
assert!(HeadersByNumber::get(&0).is_none());
// 5 headers with number = 1 are pruned (6 total)
@@ -999,7 +1017,7 @@ pub(crate) mod tests {
);
// try to prune blocks [2; 10)
storage.prune_blocks(11, 10, Some(10));
storage.prune_blocks(11, 10, 10);
// 4 headers with number = 2 are pruned (4 total)
assert!(HeadersByNumber::get(&2).is_none());
// 5 headers with number = 3 are pruned (9 total)
@@ -1023,7 +1041,7 @@ pub(crate) mod tests {
// last finalized block is 5
// and one of blocks#7 has scheduled change
// => we won't prune any block#7 at all
storage.prune_blocks(0xFFFF, 5, Some(10));
storage.prune_blocks(0xFFFF, 5, 10);
assert!(HeadersByNumber::get(&0).is_none());
assert!(HeadersByNumber::get(&1).is_none());
assert!(HeadersByNumber::get(&2).is_none());
@@ -1071,7 +1089,7 @@ pub(crate) mod tests {
oldest_unpruned_block: interval - 1,
oldest_block_to_keep: interval - 1,
});
storage.finalize_headers(None, Some(interval + 1));
storage.finalize_and_prune_headers(None, interval + 1);
assert_eq!(FinalityCache::<TestRuntime>::get(&header_with_entry_hash), None);
});
}
@@ -1152,7 +1170,7 @@ pub(crate) mod tests {
let mut storage = BridgeStorage::<TestRuntime>::new();
insert_header(&mut storage, example_header_parent());
insert_header(&mut storage, example_header());
storage.finalize_headers(Some(example_header().compute_id()), None);
storage.finalize_and_prune_headers(Some(example_header().compute_id()), 0);
assert_eq!(
verify_transaction_finalized(&storage, example_header_parent().compute_hash(), 0, &vec![example_tx()],),
true,
@@ -1206,7 +1224,7 @@ pub(crate) mod tests {
insert_header(&mut storage, example_header_parent());
insert_header(&mut storage, example_header());
insert_header(&mut storage, finalized_header_sibling);
storage.finalize_headers(Some(example_header().compute_id()), None);
storage.finalize_and_prune_headers(Some(example_header().compute_id()), 0);
assert_eq!(
verify_transaction_finalized(&storage, finalized_header_sibling_hash, 0, &vec![example_tx()],),
false,
@@ -1225,7 +1243,7 @@ pub(crate) mod tests {
insert_header(&mut storage, example_header_parent());
insert_header(&mut storage, finalized_header_uncle);
insert_header(&mut storage, example_header());
storage.finalize_headers(Some(example_header().compute_id()), None);
storage.finalize_and_prune_headers(Some(example_header().compute_id()), 0);
assert_eq!(
verify_transaction_finalized(&storage, finalized_header_uncle_hash, 0, &vec![example_tx()],),
false,
+24 -5
View File
@@ -16,7 +16,7 @@
use crate::finality::FinalityVotes;
use crate::validators::{ValidatorsConfiguration, ValidatorsSource};
use crate::{AuraConfiguration, GenesisConfig, HeaderToImport, HeadersByNumber, Storage, Trait};
use crate::{AuraConfiguration, GenesisConfig, HeaderToImport, HeadersByNumber, PruningStrategy, Storage, Trait};
use frame_support::StorageMap;
use frame_support::{impl_outer_origin, parameter_types, weights::Weight};
use parity_crypto::publickey::{sign, KeyPair, Secret};
@@ -78,11 +78,15 @@ parameter_types! {
impl Trait for TestRuntime {
type AuraConfiguration = TestAuraConfiguration;
type FinalityVotesCachingInterval = TestFinalityVotesCachingInterval;
type ValidatorsConfiguration = TestValidatorsConfiguration;
type FinalityVotesCachingInterval = TestFinalityVotesCachingInterval;
type PruningStrategy = KeepSomeHeadersBehindBest;
type OnHeadersSubmitted = ();
}
/// Step of genesis header.
pub const GENESIS_STEP: u64 = 42;
/// Aura configuration that is used in tests by default.
pub fn test_aura_config() -> AuraConfiguration {
AuraConfiguration {
@@ -105,7 +109,7 @@ pub fn test_validators_config() -> ValidatorsConfiguration {
/// Genesis header that is used in tests by default.
pub fn genesis() -> Header {
Header {
seal: vec![vec![42].into(), vec![].into()],
seal: vec![vec![GENESIS_STEP as _].into(), vec![].into()],
..Default::default()
}
}
@@ -123,12 +127,12 @@ pub fn custom_block_i(number: u64, validators: &[KeyPair], customize: impl FnOnc
parent_hash: HeadersByNumber::get(number - 1).unwrap()[0].clone(),
gas_limit: 0x2000.into(),
author: validator(validator_index).address(),
seal: vec![vec![number as u8 + 42].into(), vec![].into()],
seal: vec![vec![(number + GENESIS_STEP) as u8].into(), vec![].into()],
difficulty: number.into(),
..Default::default()
};
customize(&mut header);
signed_header(validators, header, number + 42)
signed_header(validators, header, number + GENESIS_STEP)
}
/// Build signed header from given header.
@@ -182,3 +186,18 @@ pub fn insert_header<S: Storage>(storage: &mut S, header: Header) {
finality_votes: FinalityVotes::default(),
});
}
/// Pruning strategy that keeps 10 headers behind best block.
pub struct KeepSomeHeadersBehindBest(pub u64);
impl Default for KeepSomeHeadersBehindBest {
fn default() -> KeepSomeHeadersBehindBest {
KeepSomeHeadersBehindBest(10)
}
}
impl PruningStrategy for KeepSomeHeadersBehindBest {
fn pruning_upper_bound(&mut self, best_number: u64, _: u64) -> u64 {
best_number.checked_sub(self.0).unwrap_or(0)
}
}