Offline fallback for GRANDPA (#1619)

Co-authored-by: André Silva <andre.beat@gmail.com>

* skeleton for finality tracker

* dispatch events when nothing finalized for a long time

* begin integrating finality tracker into grandpa

* add delay field to pending change

* add has_api_with function to sr_version for querying APIs

* partially integrate new force changes into grandpa

* implement forced changes

* get srml-grandpa compiling

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* remove explicit dependence on CoreApi

* increase node runtime version

* integrate grandpa forced changes into node runtime

* add some tests to finality-tracker

* integrate finality tracking into node-runtime

* test forced-change logic

* test forced changes in the authority-set handler

* kill some unneeded bounds in client

* test forced-changes in finality-grandpa and fix logic

* build wasm and finality-tracker is no-std

* restart voter on forced change

* allow returning custom error type from lock_import_and_run

* extract out most DB logic to aux_schema and use atomic client ops

* unify authority set writing

* implement set pausing

* bump runtime version

* note on DB when we pause.

* core: grandpa: integrate forced changes with multiple pending standard changes

* core: grandpa: fix AuthoritySet tests

* runtime: bump impl_version

* core: clear pending justification requests after forced change import

* srml: finality-tracker: use FinalizedInherentData

* core: log requests for clearing justification requests

* core, node: update runtimes

* core: grandpa: fix tests

* core: grandpa: remove todos and add comments

* core: grandpa: use has_api_with from ApiExt

* core: fix tests

* core: grandpa: remove unnecessary mut modifier

* core: replace PostImportActions bitflags with struct

* core: grandpa: restrict genesis on forced authority set change

* core: grandpa: add more docs

* core: grandpa: prevent safety violations in Environment::finalize_block

* core: grandpa: register finality tracker inherent data provider

* core: grandpa: fix tests

* node: update runtime blobs

* core: grandpa: remove outdated todo

* core: aura: fix typo in log message

* core: grandpa: check re-finalization is on canonical chain

* srml: finality-tracker: fix initialization

* node: update runtime wasm

* srml: finality-tracker: don't re-initialize config keys
This commit is contained in:
Robert Habermeier
2019-03-05 16:41:35 +01:00
committed by André Silva
parent 128d164f2b
commit dfb48a2405
31 changed files with 2217 additions and 516 deletions
@@ -39,17 +39,7 @@ impl<H, N> Clone for SharedAuthoritySet<H, N> {
}
}
impl<H, N> SharedAuthoritySet<H, N>
where H: PartialEq,
N: Ord,
{
/// The genesis authority set.
pub(crate) fn genesis(initial: Vec<(Ed25519AuthorityId, u64)>) -> Self {
SharedAuthoritySet {
inner: Arc::new(RwLock::new(AuthoritySet::genesis(initial)))
}
}
impl<H, N> SharedAuthoritySet<H, N> {
/// Acquire a reference to the inner read-write lock.
pub(crate) fn inner(&self) -> &RwLock<AuthoritySet<H, N>> {
&*self.inner
@@ -93,11 +83,18 @@ pub(crate) struct Status<H, N> {
}
/// A set of authorities.
#[derive(Debug, Clone, Encode, Decode)]
#[derive(Debug, Clone, Encode, Decode, PartialEq)]
pub(crate) struct AuthoritySet<H, N> {
current_authorities: Vec<(Ed25519AuthorityId, u64)>,
set_id: u64,
pending_changes: ForkTree<H, N, PendingChange<H, N>>,
pub(crate) current_authorities: Vec<(Ed25519AuthorityId, u64)>,
pub(crate) set_id: u64,
// Tree of pending standard changes across forks. Standard changes are
// enacted on finality and must be enacted (i.e. finalized) in-order across
// a given branch
pub(crate) pending_standard_changes: ForkTree<H, N, PendingChange<H, N>>,
// Pending forced changes across different forks (at most one per fork).
// Forced changes are enacted on block depth (not finality), for this reason
// only one forced change should exist per fork.
pub(crate) pending_forced_changes: Vec<PendingChange<H, N>>,
}
impl<H, N> AuthoritySet<H, N>
@@ -109,7 +106,8 @@ where H: PartialEq,
AuthoritySet {
current_authorities: initial,
set_id: 0,
pending_changes: ForkTree::new(),
pending_standard_changes: ForkTree::new(),
pending_forced_changes: Vec::new(),
}
}
@@ -124,12 +122,7 @@ where
N: Add<Output=N> + Ord + Clone + Debug,
H: Clone + Debug
{
/// Note an upcoming pending transition. Multiple pending changes on the
/// same branch can be added as long as they don't overlap. This method
/// assumes that changes on the same branch will be added in-order. The
/// given function `is_descendent_of` should return `true` if the second
/// hash (target) is a descendent of the first hash (base).
pub(crate) fn add_pending_change<F, E>(
fn add_standard_change<F, E>(
&mut self,
pending: PendingChange<H, N>,
is_descendent_of: &F,
@@ -140,65 +133,180 @@ where
let hash = pending.canon_hash.clone();
let number = pending.canon_height.clone();
debug!(target: "afg", "Inserting potential set change signalled at block {:?} \
debug!(target: "afg", "Inserting potential standard set change signalled at block {:?} \
(delayed by {:?} blocks).",
(&number, &hash), pending.finalization_depth);
(&number, &hash), pending.delay);
self.pending_changes.import(
self.pending_standard_changes.import(
hash.clone(),
number.clone(),
pending,
is_descendent_of,
)?;
debug!(target: "afg", "There are now {} alternatives for the next pending change (roots), \
and a total of {} pending changes (across all forks).",
self.pending_changes.roots().count(),
self.pending_changes.iter().count(),
debug!(target: "afg", "There are now {} alternatives for the next pending standard change (roots), \
and a total of {} pending standard changes (across all forks).",
self.pending_standard_changes.roots().count(),
self.pending_standard_changes.iter().count(),
);
Ok(())
}
/// Inspect pending changes. Pending changes in the tree are traversed in pre-order.
pub(crate) fn pending_changes(&self) -> impl Iterator<Item=&PendingChange<H, N>> {
self.pending_changes.iter().map(|(_, _, c)| c)
fn add_forced_change<F, E>(
&mut self,
pending: PendingChange<H, N>,
is_descendent_of: &F,
) -> Result<(), fork_tree::Error<E>> where
F: Fn(&H, &H) -> Result<bool, E>,
E: std::error::Error,
{
for change in self.pending_forced_changes.iter() {
if change.canon_hash == pending.canon_hash ||
is_descendent_of(&change.canon_hash, &pending.canon_hash)?
{
return Err(fork_tree::Error::UnfinalizedAncestor);
}
}
// ordered first by effective number and then by signal-block number.
let key = (pending.effective_number(), pending.canon_height.clone());
let idx = self.pending_forced_changes
.binary_search_by_key(&key, |change| (
change.effective_number(),
change.canon_height.clone(),
))
.unwrap_or_else(|i| i);
debug!(target: "afg", "Inserting potential forced set change at block {:?} \
(delayed by {:?} blocks).",
(&pending.canon_height, &pending.canon_hash), pending.delay);
self.pending_forced_changes.insert(idx, pending);
debug!(target: "afg", "There are now {} pending forced changes.", self.pending_forced_changes.len());
Ok(())
}
/// Get the earliest limit-block number, if any. If there are pending
/// changes across different forks, this method will return the earliest
/// effective number (across the different branches).
/// Note an upcoming pending transition. Multiple pending standard changes
/// on the same branch can be added as long as they don't overlap. Forced
/// changes are restricted to one per fork. This method assumes that changes
/// on the same branch will be added in-order. The given function
/// `is_descendent_of` should return `true` if the second hash (target) is a
/// descendent of the first hash (base).
pub(crate) fn add_pending_change<F, E>(
&mut self,
pending: PendingChange<H, N>,
is_descendent_of: &F,
) -> Result<(), fork_tree::Error<E>> where
F: Fn(&H, &H) -> Result<bool, E>,
E: std::error::Error,
{
match pending.delay_kind {
DelayKind::Best { .. } => {
self.add_forced_change(pending, is_descendent_of)
},
DelayKind::Finalized => {
self.add_standard_change(pending, is_descendent_of)
},
}
}
/// Inspect pending changes. Standard pending changes are iterated first,
/// and the changes in the tree are traversed in pre-order, afterwards all
/// forced changes are iterated.
pub(crate) fn pending_changes(&self) -> impl Iterator<Item=&PendingChange<H, N>> {
self.pending_standard_changes.iter().map(|(_, _, c)| c)
.chain(self.pending_forced_changes.iter())
}
/// Get the earliest limit-block number, if any. If there are pending changes across
/// different forks, this method will return the earliest effective number (across the
/// different branches). Only standard changes are taken into account for the current
/// limit, since any existing forced change should preclude the voter from voting.
pub(crate) fn current_limit(&self) -> Option<N> {
self.pending_changes.roots()
self.pending_standard_changes.roots()
.min_by_key(|&(_, _, c)| c.effective_number())
.map(|(_, _, c)| c.effective_number())
}
/// Apply or prune any pending transitions. This method ensures that if
/// there are multiple changes in the same branch, finalizing this block
/// won't finalize past multiple transitions (i.e. transitions must be
/// finalized in-order). The given function `is_descendent_of` should return
/// `true` if the second hash (target) is a descendent of the first hash
/// (base).
/// Apply or prune any pending transitions based on a best-block trigger.
///
/// Returns `Ok((median, new_set))` when a forced change has occurred. The
/// median represents the median last finalized block at the time the change
/// was signaled, and it should be used as the canon block when starting the
/// new grandpa voter. Only alters the internal state in this case.
///
/// These transitions are always forced and do not lead to justifications
/// which light clients can follow.
pub(crate) fn apply_forced_changes<F, E>(
&self,
best_hash: H,
best_number: N,
is_descendent_of: &F,
) -> Result<Option<(N, Self)>, E>
where F: Fn(&H, &H) -> Result<bool, E>,
{
let mut new_set = None;
for change in self.pending_forced_changes.iter()
.take_while(|c| c.effective_number() <= best_number) // to prevent iterating too far
.filter(|c| c.effective_number() == best_number)
{
// check if the given best block is in the same branch as the block that signalled the change.
if is_descendent_of(&change.canon_hash, &best_hash)? {
// apply this change: make the set canonical
info!(target: "finality", "Applying authority set change forced at block #{:?}",
change.canon_height);
let median_last_finalized = match change.delay_kind {
DelayKind::Best { ref median_last_finalized } => median_last_finalized.clone(),
_ => unreachable!("pending_forced_changes only contains forced changes; forced changes have delay kind Best; qed."),
};
new_set = Some((median_last_finalized, AuthoritySet {
current_authorities: change.next_authorities.clone(),
set_id: self.set_id + 1,
pending_standard_changes: ForkTree::new(), // new set, new changes.
pending_forced_changes: Vec::new(),
}));
break;
}
// we don't wipe forced changes until another change is
// applied
}
Ok(new_set)
}
/// Apply or prune any pending transitions based on a finality trigger. This
/// method ensures that if there are multiple changes in the same branch,
/// finalizing this block won't finalize past multiple transitions (i.e.
/// transitions must be finalized in-order). The given function
/// `is_descendent_of` should return `true` if the second hash (target) is a
/// descendent of the first hash (base).
///
/// When the set has changed, the return value will be `Ok(Some((H, N)))`
/// which is the canonical block where the set last changed (i.e. the given
/// hash and number).
pub(crate) fn apply_changes<F, E>(
pub(crate) fn apply_standard_changes<F, E>(
&mut self,
finalized_hash: H,
finalized_number: N,
is_descendent_of: &F,
) -> Result<Status<H, N>, fork_tree::Error<E>>
where F: Fn(&H, &H) -> Result<bool, E>,
E: std::error::Error,
where F: Fn(&H, &H) -> Result<bool, E>,
E: std::error::Error,
{
let mut status = Status {
changed: false,
new_set_block: None,
};
match self.pending_changes.finalize_with_descendent_if(
match self.pending_standard_changes.finalize_with_descendent_if(
&finalized_hash,
finalized_number.clone(),
is_descendent_of,
@@ -207,6 +315,10 @@ where
fork_tree::FinalizationResult::Changed(change) => {
status.changed = true;
// if we are able to finalize any standard change then we can
// discard all pending forced changes (on different forks)
self.pending_forced_changes.clear();
if let Some(change) = change {
info!(target: "finality", "Applying authority set change scheduled at block #{:?}",
change.canon_height);
@@ -226,13 +338,13 @@ where
Ok(status)
}
/// Check whether the given finalized block number enacts any authority set
/// change (without triggering it), ensuring that if there are multiple
/// changes in the same branch, finalizing this block won't finalize past
/// multiple transitions (i.e. transitions must be finalized in-order). The
/// given function `is_descendent_of` should return `true` if the second
/// hash (target) is a descendent of the first hash (base).
pub fn enacts_change<F, E>(
/// Check whether the given finalized block number enacts any standard
/// authority set change (without triggering it), ensuring that if there are
/// multiple changes in the same branch, finalizing this block won't
/// finalize past multiple transitions (i.e. transitions must be finalized
/// in-order). The given function `is_descendent_of` should return `true` if
/// the second hash (target) is a descendent of the first hash (base).
pub fn enacts_standard_change<F, E>(
&self,
finalized_hash: H,
finalized_number: N,
@@ -241,7 +353,7 @@ where
where F: Fn(&H, &H) -> Result<bool, E>,
E: std::error::Error,
{
self.pending_changes.finalizes_any_with_descendent_if(
self.pending_standard_changes.finalizes_any_with_descendent_if(
&finalized_hash,
finalized_number.clone(),
is_descendent_of,
@@ -250,27 +362,58 @@ where
}
}
/// Kinds of delays for pending changes.
#[derive(Debug, Clone, Encode, Decode, PartialEq)]
pub(crate) enum DelayKind<N> {
/// Depth in finalized chain.
Finalized,
/// Depth in best chain. The median last finalized block is calculated at the time the
/// change was signalled.
Best { median_last_finalized: N },
}
/// A pending change to the authority set.
///
/// This will be applied when the announcing block is at some depth within
/// the finalized chain.
#[derive(Debug, Clone, Encode, Decode, PartialEq)]
/// the finalized or unfinalized chain.
#[derive(Debug, Clone, Encode, PartialEq)]
pub(crate) struct PendingChange<H, N> {
/// The new authorities and weights to apply.
pub(crate) next_authorities: Vec<(Ed25519AuthorityId, u64)>,
/// How deep in the finalized chain the announcing block must be
/// How deep in the chain the announcing block must be
/// before the change is applied.
pub(crate) finalization_depth: N,
pub(crate) delay: N,
/// The announcing block's height.
pub(crate) canon_height: N,
/// The announcing block's hash.
pub(crate) canon_hash: H,
/// The delay kind.
pub(crate) delay_kind: DelayKind<N>,
}
impl<H: Decode, N: Decode> Decode for PendingChange<H, N> {
fn decode<I: parity_codec::Input>(value: &mut I) -> Option<Self> {
let next_authorities = Decode::decode(value)?;
let delay = Decode::decode(value)?;
let canon_height = Decode::decode(value)?;
let canon_hash = Decode::decode(value)?;
let delay_kind = DelayKind::decode(value).unwrap_or(DelayKind::Finalized);
Some(PendingChange {
next_authorities,
delay,
canon_height,
canon_hash,
delay_kind,
})
}
}
impl<H, N: Add<Output=N> + Clone> PendingChange<H, N> {
/// Returns the effective number this change will be applied at.
pub fn effective_number(&self) -> N {
self.canon_height.clone() + self.finalization_depth.clone()
self.canon_height.clone() + self.delay.clone()
}
}
@@ -295,28 +438,32 @@ mod tests {
let mut authorities = AuthoritySet {
current_authorities: Vec::new(),
set_id: 0,
pending_changes: ForkTree::new(),
pending_standard_changes: ForkTree::new(),
pending_forced_changes: Vec::new(),
};
let change_a = PendingChange {
next_authorities: Vec::new(),
finalization_depth: 10,
delay: 10,
canon_height: 5,
canon_hash: "hash_a",
delay_kind: DelayKind::Finalized,
};
let change_b = PendingChange {
next_authorities: Vec::new(),
finalization_depth: 0,
delay: 0,
canon_height: 5,
canon_hash: "hash_b",
delay_kind: DelayKind::Finalized,
};
let change_c = PendingChange {
next_authorities: Vec::new(),
finalization_depth: 5,
delay: 5,
canon_height: 10,
canon_hash: "hash_c",
delay_kind: DelayKind::Finalized,
};
authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(false)).unwrap();
@@ -327,9 +474,29 @@ mod tests {
_ => unreachable!(),
})).unwrap();
// forced changes are iterated last
let change_d = PendingChange {
next_authorities: Vec::new(),
delay: 2,
canon_height: 1,
canon_hash: "hash_d",
delay_kind: DelayKind::Best { median_last_finalized: 0 },
};
let change_e = PendingChange {
next_authorities: Vec::new(),
delay: 2,
canon_height: 0,
canon_hash: "hash_e",
delay_kind: DelayKind::Best { median_last_finalized: 0 },
};
authorities.add_pending_change(change_d.clone(), &static_is_descendent_of(false)).unwrap();
authorities.add_pending_change(change_e.clone(), &static_is_descendent_of(false)).unwrap();
assert_eq!(
authorities.pending_changes().collect::<Vec<_>>(),
vec![&change_b, &change_a, &change_c],
vec![&change_b, &change_a, &change_c, &change_e, &change_d],
);
}
@@ -338,7 +505,8 @@ mod tests {
let mut authorities = AuthoritySet {
current_authorities: Vec::new(),
set_id: 0,
pending_changes: ForkTree::new(),
pending_standard_changes: ForkTree::new(),
pending_forced_changes: Vec::new(),
};
let set_a = vec![([1; 32].into(), 5)];
@@ -347,16 +515,18 @@ mod tests {
// two competing changes at the same height on different forks
let change_a = PendingChange {
next_authorities: set_a.clone(),
finalization_depth: 10,
delay: 10,
canon_height: 5,
canon_hash: "hash_a",
delay_kind: DelayKind::Finalized,
};
let change_b = PendingChange {
next_authorities: set_b.clone(),
finalization_depth: 10,
delay: 10,
canon_height: 5,
canon_hash: "hash_b",
delay_kind: DelayKind::Finalized,
};
authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(true)).unwrap();
@@ -368,7 +538,7 @@ mod tests {
);
// finalizing "hash_c" won't enact the change signalled at "hash_a" but it will prune out "hash_b"
let status = authorities.apply_changes("hash_c", 11, &is_descendent_of(|base, hash| match (*base, *hash) {
let status = authorities.apply_standard_changes("hash_c", 11, &is_descendent_of(|base, hash| match (*base, *hash) {
("hash_a", "hash_c") => true,
("hash_b", "hash_c") => false,
_ => unreachable!(),
@@ -382,7 +552,7 @@ mod tests {
);
// finalizing "hash_d" will enact the change signalled at "hash_a"
let status = authorities.apply_changes("hash_d", 15, &is_descendent_of(|base, hash| match (*base, *hash) {
let status = authorities.apply_standard_changes("hash_d", 15, &is_descendent_of(|base, hash| match (*base, *hash) {
("hash_a", "hash_d") => true,
_ => unreachable!(),
})).unwrap();
@@ -400,7 +570,8 @@ mod tests {
let mut authorities = AuthoritySet {
current_authorities: Vec::new(),
set_id: 0,
pending_changes: ForkTree::new(),
pending_standard_changes: ForkTree::new(),
pending_forced_changes: Vec::new(),
};
let set_a = vec![([1; 32].into(), 5)];
@@ -409,16 +580,18 @@ mod tests {
// two competing changes at the same height on different forks
let change_a = PendingChange {
next_authorities: set_a.clone(),
finalization_depth: 10,
delay: 10,
canon_height: 5,
canon_hash: "hash_a",
delay_kind: DelayKind::Finalized,
};
let change_c = PendingChange {
next_authorities: set_c.clone(),
finalization_depth: 10,
delay: 10,
canon_height: 30,
canon_hash: "hash_c",
delay_kind: DelayKind::Finalized,
};
authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(true)).unwrap();
@@ -437,12 +610,12 @@ mod tests {
});
// trying to finalize past `change_c` without finalizing `change_a` first
match authorities.apply_changes("hash_d", 40, &is_descendent_of) {
match authorities.apply_standard_changes("hash_d", 40, &is_descendent_of) {
Err(fork_tree::Error::UnfinalizedAncestor) => {},
_ => unreachable!(),
}
let status = authorities.apply_changes("hash_b", 15, &is_descendent_of).unwrap();
let status = authorities.apply_standard_changes("hash_b", 15, &is_descendent_of).unwrap();
assert!(status.changed);
assert_eq!(status.new_set_block, Some(("hash_b", 15)));
@@ -450,7 +623,7 @@ mod tests {
assert_eq!(authorities.set_id, 1);
// after finalizing `change_a` it should be possible to finalize `change_c`
let status = authorities.apply_changes("hash_d", 40, &is_descendent_of).unwrap();
let status = authorities.apply_standard_changes("hash_d", 40, &is_descendent_of).unwrap();
assert!(status.changed);
assert_eq!(status.new_set_block, Some(("hash_d", 40)));
@@ -459,20 +632,22 @@ mod tests {
}
#[test]
fn enacts_change_works() {
fn enacts_standard_change_works() {
let mut authorities = AuthoritySet {
current_authorities: Vec::new(),
set_id: 0,
pending_changes: ForkTree::new(),
pending_standard_changes: ForkTree::new(),
pending_forced_changes: Vec::new(),
};
let set_a = vec![([1; 32].into(), 5)];
let change_a = PendingChange {
next_authorities: set_a.clone(),
finalization_depth: 10,
delay: 10,
canon_height: 5,
canon_hash: "hash_a",
delay_kind: DelayKind::Finalized,
};
authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(false)).unwrap();
@@ -484,12 +659,84 @@ mod tests {
});
// "hash_c" won't finalize the existing change since it isn't a descendent
assert!(!authorities.enacts_change("hash_c", 15, &is_descendent_of).unwrap());
assert!(!authorities.enacts_standard_change("hash_c", 15, &is_descendent_of).unwrap());
// "hash_b" at depth 14 won't work either
assert!(!authorities.enacts_change("hash_b", 14, &is_descendent_of).unwrap());
assert!(!authorities.enacts_standard_change("hash_b", 14, &is_descendent_of).unwrap());
// but it should work at depth 15 (change height + depth)
assert!(authorities.enacts_change("hash_b", 15, &is_descendent_of).unwrap());
assert!(authorities.enacts_standard_change("hash_b", 15, &is_descendent_of).unwrap());
}
#[test]
fn forced_changes() {
let mut authorities = AuthoritySet {
current_authorities: Vec::new(),
set_id: 0,
pending_standard_changes: ForkTree::new(),
pending_forced_changes: Vec::new(),
};
let set_a = vec![([1; 32].into(), 5)];
let set_b = vec![([2; 32].into(), 5)];
let change_a = PendingChange {
next_authorities: set_a.clone(),
delay: 10,
canon_height: 5,
canon_hash: "hash_a",
delay_kind: DelayKind::Best { median_last_finalized: 42 },
};
let change_b = PendingChange {
next_authorities: set_b.clone(),
delay: 10,
canon_height: 5,
canon_hash: "hash_b",
delay_kind: DelayKind::Best { median_last_finalized: 0 },
};
authorities.add_pending_change(change_a, &static_is_descendent_of(false)).unwrap();
authorities.add_pending_change(change_b, &static_is_descendent_of(false)).unwrap();
// there's an effective change triggered at block 15 but not a standard one.
// so this should do nothing.
assert!(!authorities.enacts_standard_change("hash_c", 15, &static_is_descendent_of(true)).unwrap());
// throw a standard change into the mix to prove that it's discarded
// for being on the same fork.
//
// NOTE: after https://github.com/paritytech/substrate/issues/1861
// this should still be rejected based on the "span" rule -- it overlaps
// with another change on the same fork.
let change_c = PendingChange {
next_authorities: set_b.clone(),
delay: 3,
canon_height: 8,
canon_hash: "hash_a8",
delay_kind: DelayKind::Best { median_last_finalized: 0 },
};
let is_descendent_of_a = is_descendent_of(|base: &&str, _| {
base.starts_with("hash_a")
});
assert!(authorities.add_pending_change(change_c, &is_descendent_of_a).is_err());
// too early.
assert!(authorities.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true)).unwrap().is_none());
// too late.
assert!(authorities.apply_forced_changes("hash_a16", 16, &static_is_descendent_of(true)).unwrap().is_none());
// on time -- chooses the right change.
assert_eq!(
authorities.apply_forced_changes("hash_a15", 15, &is_descendent_of_a).unwrap().unwrap(),
(42, AuthoritySet {
current_authorities: set_a,
set_id: 1,
pending_standard_changes: ForkTree::new(),
pending_forced_changes: Vec::new(),
})
);
}
}