Offline fallback for GRANDPA (#1619)

Co-authored-by: André Silva <andre.beat@gmail.com>

* skeleton for finality tracker

* dispatch events when nothing finalized for a long time

* begin integrating finality tracker into grandpa

* add delay field to pending change

* add has_api_with function to sr_version for querying APIs

* partially integrate new force changes into grandpa

* implement forced changes

* get srml-grandpa compiling

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* remove explicit dependence on CoreApi

* increase node runtime version

* integrate grandpa forced changes into node runtime

* add some tests to finality-tracker

* integrate finality tracking into node-runtime

* test forced-change logic

* test forced changes in the authority-set handler

* kill some unneeded bounds in client

* test forced-changes in finality-grandpa and fix logic

* build wasm and finality-tracker is no-std

* restart voter on forced change

* allow returning custom error type from lock_import_and_run

* extract out most DB logic to aux_schema and use atomic client ops

* unify authority set writing

* implement set pausing

* bump runtime version

* note on DB when we pause.

* core: grandpa: integrate forced changes with multiple pending standard changes

* core: grandpa: fix AuthoritySet tests

* runtime: bump impl_version

* core: clear pending justification requests after forced change import

* srml: finality-tracker: use FinalizedInherentData

* core: log requests for clearing justification requests

* core, node: update runtimes

* core: grandpa: fix tests

* core: grandpa: remove todos and add comments

* core: grandpa: use has_api_with from ApiExt

* core: fix tests

* core: grandpa: remove unnecessary mut modifier

* core: replace PostImportActions bitflags with struct

* core: grandpa: restrict genesis on forced authority set change

* core: grandpa: add more docs

* core: grandpa: prevent safety violations in Environment::finalize_block

* core: grandpa: register finality tracker inherent data provider

* core: grandpa: fix tests

* node: update runtime blobs

* core: grandpa: remove outdated todo

* core: aura: fix typo in log message

* core: grandpa: check re-finalization is on canonical chain

* srml: finality-tracker: fix initialization

* node: update runtime wasm

* srml: finality-tracker: don't re-initialize config keys

substrate/core/finality-grandpa/Cargo.toml

@@ -17,8 +17,10 @@ consensus_common = { package = "substrate-consensus-common", path = "../consensu
 substrate-primitives = { path = "../primitives" }
 substrate-telemetry = { path = "../telemetry" }
 client = { package = "substrate-client", path = "../client" }
+inherents = { package = "substrate-inherents", path = "../../core/inherents" }
 network = { package = "substrate-network", path = "../network" }
 service = { package = "substrate-service", path = "../service", optional = true }
+srml-finality-tracker = { path = "../../srml/finality-tracker" }
 fg_primitives = { package = "substrate-finality-grandpa-primitives", path = "primitives" }
 grandpa = { package = "finality-grandpa", version = "0.6.0", features = ["derive-codec"] }
 

substrate/core/finality-grandpa/primitives/src/lib.rs

@@ -61,6 +61,7 @@ decl_runtime_apis! {
 	/// applied in the runtime after those N blocks have passed.
 	///
 	/// The consensus protocol will coordinate the handoff externally.
+	#[api_version(2)]
 	pub trait GrandpaApi {
 		/// Check a digest for pending changes.
 		/// Return `None` if there are no pending changes.
@@ -77,6 +78,28 @@ decl_runtime_apis! {
 		fn grandpa_pending_change(digest: &DigestFor<Block>)
 			-> Option<ScheduledChange<NumberFor<Block>>>;
 
+		/// Check a digest for forced changes.
+		/// Return `None` if there are no forced changes. Otherwise, return a
+		/// tuple containing the pending change and the median last finalized
+		/// block number at the time the change was signalled.
+		///
+		/// Added in version 2.
+		///
+		/// Forced changes are applied after a delay of _imported_ blocks,
+		/// while pending changes are applied after a delay of _finalized_ blocks.
+		///
+		/// Precedence towards earlier or later digest items can be given
+		/// based on the rules of the chain.
+		///
+		/// No change should be scheduled if one is already and the delay has not
+		/// passed completely.
+		///
+		/// This should be a pure function: i.e. as long as the runtime can interpret
+		/// the digest type it should return the same result regardless of the current
+		/// state.
+		fn grandpa_forced_change(digest: &DigestFor<Block>)
+			-> Option<(NumberFor<Block>, ScheduledChange<NumberFor<Block>>)>;
+
 		/// Get the current GRANDPA authorities and weights. This should not change except
 		/// for when changes are scheduled and the corresponding delay has passed.
 		///

substrate/core/finality-grandpa/src/authorities.rs

@@ -39,17 +39,7 @@ impl<H, N> Clone for SharedAuthoritySet<H, N> {
 	}
 }
 
-impl<H, N> SharedAuthoritySet<H, N>
-where H: PartialEq,
-	  N: Ord,
-{
-	/// The genesis authority set.
-	pub(crate) fn genesis(initial: Vec<(Ed25519AuthorityId, u64)>) -> Self {
-		SharedAuthoritySet {
-			inner: Arc::new(RwLock::new(AuthoritySet::genesis(initial)))
-		}
-	}
-
+impl<H, N> SharedAuthoritySet<H, N> {
 	/// Acquire a reference to the inner read-write lock.
 	pub(crate) fn inner(&self) -> &RwLock<AuthoritySet<H, N>> {
 		&*self.inner
@@ -93,11 +83,18 @@ pub(crate) struct Status<H, N> {
 }
 
 /// A set of authorities.
-#[derive(Debug, Clone, Encode, Decode)]
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
 pub(crate) struct AuthoritySet<H, N> {
-	current_authorities: Vec<(Ed25519AuthorityId, u64)>,
-	set_id: u64,
-	pending_changes: ForkTree<H, N, PendingChange<H, N>>,
+	pub(crate) current_authorities: Vec<(Ed25519AuthorityId, u64)>,
+	pub(crate) set_id: u64,
+	// Tree of pending standard changes across forks. Standard changes are
+	// enacted on finality and must be enacted (i.e. finalized) in-order across
+	// a given branch
+	pub(crate) pending_standard_changes: ForkTree<H, N, PendingChange<H, N>>,
+	// Pending forced changes across different forks (at most one per fork).
+	// Forced changes are enacted on block depth (not finality), for this reason
+	// only one forced change should exist per fork.
+	pub(crate) pending_forced_changes: Vec<PendingChange<H, N>>,
 }
 
 impl<H, N> AuthoritySet<H, N>
@@ -109,7 +106,8 @@ where H: PartialEq,
 		AuthoritySet {
 			current_authorities: initial,
 			set_id: 0,
-			pending_changes: ForkTree::new(),
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
 		}
 	}
 
@@ -124,12 +122,7 @@ where
 	N: Add<Output=N> + Ord + Clone + Debug,
 	H: Clone + Debug
 {
-	/// Note an upcoming pending transition. Multiple pending changes on the
-	/// same branch can be added as long as they don't overlap. This method
-	/// assumes that changes on the same branch will be added in-order. The
-	/// given function `is_descendent_of` should return `true` if the second
-	/// hash (target) is a descendent of the first hash (base).
-	pub(crate) fn add_pending_change<F, E>(
+	fn add_standard_change<F, E>(
 		&mut self,
 		pending: PendingChange<H, N>,
 		is_descendent_of: &F,
@@ -140,65 +133,180 @@ where
 		let hash = pending.canon_hash.clone();
 		let number = pending.canon_height.clone();
 
-		debug!(target: "afg", "Inserting potential set change signalled at block {:?} \
+		debug!(target: "afg", "Inserting potential standard set change signalled at block {:?} \
 							   (delayed by {:?} blocks).",
-			   (&number, &hash), pending.finalization_depth);
+			   (&number, &hash), pending.delay);
 
-		self.pending_changes.import(
+		self.pending_standard_changes.import(
 			hash.clone(),
 			number.clone(),
 			pending,
 			is_descendent_of,
 		)?;
 
-		debug!(target: "afg", "There are now {} alternatives for the next pending change (roots), \
-							   and a total of {} pending changes (across all forks).",
-			self.pending_changes.roots().count(),
-			self.pending_changes.iter().count(),
+		debug!(target: "afg", "There are now {} alternatives for the next pending standard change (roots), \
+							   and a total of {} pending standard changes (across all forks).",
+			self.pending_standard_changes.roots().count(),
+			self.pending_standard_changes.iter().count(),
 		);
 
 		Ok(())
 	}
 
-	/// Inspect pending changes. Pending changes in the tree are traversed in pre-order.
-	pub(crate) fn pending_changes(&self) -> impl Iterator<Item=&PendingChange<H, N>> {
-		self.pending_changes.iter().map(|(_, _, c)| c)
+	fn add_forced_change<F, E>(
+		&mut self,
+		pending: PendingChange<H, N>,
+		is_descendent_of: &F,
+	) -> Result<(), fork_tree::Error<E>> where
+		F: Fn(&H, &H) -> Result<bool, E>,
+		E:  std::error::Error,
+	{
+		for change in self.pending_forced_changes.iter() {
+			if change.canon_hash == pending.canon_hash ||
+				is_descendent_of(&change.canon_hash, &pending.canon_hash)?
+			{
+				return Err(fork_tree::Error::UnfinalizedAncestor);
+			}
+		}
+
+		// ordered first by effective number and then by signal-block number.
+		let key = (pending.effective_number(), pending.canon_height.clone());
+		let idx = self.pending_forced_changes
+			.binary_search_by_key(&key, |change| (
+				change.effective_number(),
+				change.canon_height.clone(),
+			))
+			.unwrap_or_else(|i| i);
+
+		debug!(target: "afg", "Inserting potential forced set change at block {:?} \
+							   (delayed by {:?} blocks).",
+			   (&pending.canon_height, &pending.canon_hash), pending.delay);
+
+		self.pending_forced_changes.insert(idx, pending);
+
+		debug!(target: "afg", "There are now {} pending forced changes.", self.pending_forced_changes.len());
+
+		Ok(())
 	}
 
-	/// Get the earliest limit-block number, if any. If there are pending
-	/// changes across different forks, this method will return the earliest
-	/// effective number (across the different branches).
+	/// Note an upcoming pending transition. Multiple pending standard changes
+	/// on the same branch can be added as long as they don't overlap. Forced
+	/// changes are restricted to one per fork. This method assumes that changes
+	/// on the same branch will be added in-order. The given function
+	/// `is_descendent_of` should return `true` if the second hash (target) is a
+	/// descendent of the first hash (base).
+	pub(crate) fn add_pending_change<F, E>(
+		&mut self,
+		pending: PendingChange<H, N>,
+		is_descendent_of: &F,
+	) -> Result<(), fork_tree::Error<E>> where
+		F: Fn(&H, &H) -> Result<bool, E>,
+		E:  std::error::Error,
+	{
+		match pending.delay_kind {
+			DelayKind::Best { .. } => {
+				self.add_forced_change(pending, is_descendent_of)
+			},
+			DelayKind::Finalized => {
+				self.add_standard_change(pending, is_descendent_of)
+			},
+		}
+	}
+
+	/// Inspect pending changes. Standard pending changes are iterated first,
+	/// and the changes in the tree are traversed in pre-order, afterwards all
+	/// forced changes are iterated.
+	pub(crate) fn pending_changes(&self) -> impl Iterator<Item=&PendingChange<H, N>> {
+		self.pending_standard_changes.iter().map(|(_, _, c)| c)
+			.chain(self.pending_forced_changes.iter())
+	}
+
+	/// Get the earliest limit-block number, if any. If there are pending changes across
+	/// different forks, this method will return the earliest effective number (across the
+	/// different branches). Only standard changes are taken into account for the current
+	/// limit, since any existing forced change should preclude the voter from voting.
 	pub(crate) fn current_limit(&self) -> Option<N> {
-		self.pending_changes.roots()
+		self.pending_standard_changes.roots()
 			.min_by_key(|&(_, _, c)| c.effective_number())
 			.map(|(_, _, c)| c.effective_number())
 	}
 
-	/// Apply or prune any pending transitions. This method ensures that if
-	/// there are multiple changes in the same branch, finalizing this block
-	/// won't finalize past multiple transitions (i.e. transitions must be
-	/// finalized in-order). The given function `is_descendent_of` should return
-	/// `true` if the second hash (target) is a descendent of the first hash
-	/// (base).
+	/// Apply or prune any pending transitions based on a best-block trigger.
+	///
+	/// Returns `Ok((median, new_set))` when a forced change has occurred. The
+	/// median represents the median last finalized block at the time the change
+	/// was signaled, and it should be used as the canon block when starting the
+	/// new grandpa voter. Only alters the internal state in this case.
+	///
+	/// These transitions are always forced and do not lead to justifications
+	/// which light clients can follow.
+	pub(crate) fn apply_forced_changes<F, E>(
+		&self,
+		best_hash: H,
+		best_number: N,
+		is_descendent_of: &F,
+	) -> Result<Option<(N, Self)>, E>
+		where F: Fn(&H, &H) -> Result<bool, E>,
+	{
+		let mut new_set = None;
+
+		for change in self.pending_forced_changes.iter()
+			.take_while(|c| c.effective_number() <= best_number) // to prevent iterating too far
+			.filter(|c| c.effective_number() == best_number)
+		{
+			// check if the given best block is in the same branch as the block that signalled the change.
+			if is_descendent_of(&change.canon_hash, &best_hash)? {
+				// apply this change: make the set canonical
+				info!(target: "finality", "Applying authority set change forced at block #{:?}",
+					  change.canon_height);
+
+				let median_last_finalized = match change.delay_kind {
+					DelayKind::Best { ref median_last_finalized } => median_last_finalized.clone(),
+					_ => unreachable!("pending_forced_changes only contains forced changes; forced changes have delay kind Best; qed."),
+				};
+
+				new_set = Some((median_last_finalized, AuthoritySet {
+					current_authorities: change.next_authorities.clone(),
+					set_id: self.set_id + 1,
+					pending_standard_changes: ForkTree::new(), // new set, new changes.
+					pending_forced_changes: Vec::new(),
+				}));
+
+				break;
+			}
+
+			// we don't wipe forced changes until another change is
+			// applied
+		}
+
+		Ok(new_set)
+	}
+
+	/// Apply or prune any pending transitions based on a finality trigger. This
+	/// method ensures that if there are multiple changes in the same branch,
+	/// finalizing this block won't finalize past multiple transitions (i.e.
+	/// transitions must be finalized in-order). The given function
+	/// `is_descendent_of` should return `true` if the second hash (target) is a
+	/// descendent of the first hash (base).
 	///
 	/// When the set has changed, the return value will be `Ok(Some((H, N)))`
 	/// which is the canonical block where the set last changed (i.e. the given
 	/// hash and number).
-	pub(crate) fn apply_changes<F, E>(
+	pub(crate) fn apply_standard_changes<F, E>(
 		&mut self,
 		finalized_hash: H,
 		finalized_number: N,
 		is_descendent_of: &F,
 	) -> Result<Status<H, N>, fork_tree::Error<E>>
-	where F: Fn(&H, &H) -> Result<bool, E>,
-		  E: std::error::Error,
+		where F: Fn(&H, &H) -> Result<bool, E>,
+			  E: std::error::Error,
 	{
 		let mut status = Status {
 			changed: false,
 			new_set_block: None,
 		};
 
-		match self.pending_changes.finalize_with_descendent_if(
+		match self.pending_standard_changes.finalize_with_descendent_if(
 			&finalized_hash,
 			finalized_number.clone(),
 			is_descendent_of,
@@ -207,6 +315,10 @@ where
 			fork_tree::FinalizationResult::Changed(change) => {
 				status.changed = true;
 
+				// if we are able to finalize any standard change then we can
+				// discard all pending forced changes (on different forks)
+				self.pending_forced_changes.clear();
+
 				if let Some(change) = change {
 					info!(target: "finality", "Applying authority set change scheduled at block #{:?}",
 						  change.canon_height);
@@ -226,13 +338,13 @@ where
 		Ok(status)
 	}
 
-	/// Check whether the given finalized block number enacts any authority set
-	/// change (without triggering it), ensuring that if there are multiple
-	/// changes in the same branch, finalizing this block won't finalize past
-	/// multiple transitions (i.e. transitions must be finalized in-order). The
-	/// given function `is_descendent_of` should return `true` if the second
-	/// hash (target) is a descendent of the first hash (base).
-	pub fn enacts_change<F, E>(
+	/// Check whether the given finalized block number enacts any standard
+	/// authority set change (without triggering it), ensuring that if there are
+	/// multiple changes in the same branch, finalizing this block won't
+	/// finalize past multiple transitions (i.e. transitions must be finalized
+	/// in-order). The given function `is_descendent_of` should return `true` if
+	/// the second hash (target) is a descendent of the first hash (base).
+	pub fn enacts_standard_change<F, E>(
 		&self,
 		finalized_hash: H,
 		finalized_number: N,
@@ -241,7 +353,7 @@ where
 	where F: Fn(&H, &H) -> Result<bool, E>,
 		  E: std::error::Error,
 	{
-		self.pending_changes.finalizes_any_with_descendent_if(
+		self.pending_standard_changes.finalizes_any_with_descendent_if(
 			&finalized_hash,
 			finalized_number.clone(),
 			is_descendent_of,
@@ -250,27 +362,58 @@ where
 	}
 }
 
+/// Kinds of delays for pending changes.
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+pub(crate) enum DelayKind<N> {
+	/// Depth in finalized chain.
+	Finalized,
+	/// Depth in best chain. The median last finalized block is calculated at the time the
+	/// change was signalled.
+	Best { median_last_finalized: N },
+}
+
 /// A pending change to the authority set.
 ///
 /// This will be applied when the announcing block is at some depth within
-/// the finalized chain.
-#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+/// the finalized or unfinalized chain.
+#[derive(Debug, Clone, Encode, PartialEq)]
 pub(crate) struct PendingChange<H, N> {
 	/// The new authorities and weights to apply.
 	pub(crate) next_authorities: Vec<(Ed25519AuthorityId, u64)>,
-	/// How deep in the finalized chain the announcing block must be
+	/// How deep in the chain the announcing block must be
 	/// before the change is applied.
-	pub(crate) finalization_depth: N,
+	pub(crate) delay: N,
 	/// The announcing block's height.
 	pub(crate) canon_height: N,
 	/// The announcing block's hash.
 	pub(crate) canon_hash: H,
+	/// The delay kind.
+	pub(crate) delay_kind: DelayKind<N>,
+}
+
+impl<H: Decode, N: Decode> Decode for PendingChange<H, N> {
+	fn decode<I: parity_codec::Input>(value: &mut I) -> Option<Self> {
+		let next_authorities = Decode::decode(value)?;
+		let delay = Decode::decode(value)?;
+		let canon_height = Decode::decode(value)?;
+		let canon_hash = Decode::decode(value)?;
+
+		let delay_kind = DelayKind::decode(value).unwrap_or(DelayKind::Finalized);
+
+		Some(PendingChange {
+			next_authorities,
+			delay,
+			canon_height,
+			canon_hash,
+			delay_kind,
+		})
+	}
 }
 
 impl<H, N: Add<Output=N> + Clone> PendingChange<H, N> {
 	/// Returns the effective number this change will be applied at.
 	pub fn effective_number(&self) -> N {
-		self.canon_height.clone() + self.finalization_depth.clone()
+		self.canon_height.clone() + self.delay.clone()
 	}
 }
 
@@ -295,28 +438,32 @@ mod tests {
 		let mut authorities = AuthoritySet {
 			current_authorities: Vec::new(),
 			set_id: 0,
-			pending_changes: ForkTree::new(),
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
 		};
 
 		let change_a = PendingChange {
 			next_authorities: Vec::new(),
-			finalization_depth: 10,
+			delay: 10,
 			canon_height: 5,
 			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		let change_b = PendingChange {
 			next_authorities: Vec::new(),
-			finalization_depth: 0,
+			delay: 0,
 			canon_height: 5,
 			canon_hash: "hash_b",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		let change_c = PendingChange {
 			next_authorities: Vec::new(),
-			finalization_depth: 5,
+			delay: 5,
 			canon_height: 10,
 			canon_hash: "hash_c",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(false)).unwrap();
@@ -327,9 +474,29 @@ mod tests {
 			_ => unreachable!(),
 		})).unwrap();
 
+		// forced changes are iterated last
+		let change_d = PendingChange {
+			next_authorities: Vec::new(),
+			delay: 2,
+			canon_height: 1,
+			canon_hash: "hash_d",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		let change_e = PendingChange {
+			next_authorities: Vec::new(),
+			delay: 2,
+			canon_height: 0,
+			canon_hash: "hash_e",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		authorities.add_pending_change(change_d.clone(), &static_is_descendent_of(false)).unwrap();
+		authorities.add_pending_change(change_e.clone(), &static_is_descendent_of(false)).unwrap();
+
 		assert_eq!(
 			authorities.pending_changes().collect::<Vec<_>>(),
-			vec![&change_b, &change_a, &change_c],
+			vec![&change_b, &change_a, &change_c, &change_e, &change_d],
 		);
 	}
 
@@ -338,7 +505,8 @@ mod tests {
 		let mut authorities = AuthoritySet {
 			current_authorities: Vec::new(),
 			set_id: 0,
-			pending_changes: ForkTree::new(),
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
 		};
 
 		let set_a = vec![([1; 32].into(), 5)];
@@ -347,16 +515,18 @@ mod tests {
 		// two competing changes at the same height on different forks
 		let change_a = PendingChange {
 			next_authorities: set_a.clone(),
-			finalization_depth: 10,
+			delay: 10,
 			canon_height: 5,
 			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		let change_b = PendingChange {
 			next_authorities: set_b.clone(),
-			finalization_depth: 10,
+			delay: 10,
 			canon_height: 5,
 			canon_hash: "hash_b",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(true)).unwrap();
@@ -368,7 +538,7 @@ mod tests {
 		);
 
 		// finalizing "hash_c" won't enact the change signalled at "hash_a" but it will prune out "hash_b"
-		let status = authorities.apply_changes("hash_c", 11, &is_descendent_of(|base, hash| match (*base, *hash) {
+		let status = authorities.apply_standard_changes("hash_c", 11, &is_descendent_of(|base, hash| match (*base, *hash) {
 			("hash_a", "hash_c") => true,
 			("hash_b", "hash_c") => false,
 			_ => unreachable!(),
@@ -382,7 +552,7 @@ mod tests {
 		);
 
 		// finalizing "hash_d" will enact the change signalled at "hash_a"
-		let status = authorities.apply_changes("hash_d", 15, &is_descendent_of(|base, hash| match (*base, *hash) {
+		let status = authorities.apply_standard_changes("hash_d", 15, &is_descendent_of(|base, hash| match (*base, *hash) {
 			("hash_a", "hash_d") => true,
 			_ => unreachable!(),
 		})).unwrap();
@@ -400,7 +570,8 @@ mod tests {
 		let mut authorities = AuthoritySet {
 			current_authorities: Vec::new(),
 			set_id: 0,
-			pending_changes: ForkTree::new(),
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
 		};
 
 		let set_a = vec![([1; 32].into(), 5)];
@@ -409,16 +580,18 @@ mod tests {
 		// two competing changes at the same height on different forks
 		let change_a = PendingChange {
 			next_authorities: set_a.clone(),
-			finalization_depth: 10,
+			delay: 10,
 			canon_height: 5,
 			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		let change_c = PendingChange {
 			next_authorities: set_c.clone(),
-			finalization_depth: 10,
+			delay: 10,
 			canon_height: 30,
 			canon_hash: "hash_c",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(true)).unwrap();
@@ -437,12 +610,12 @@ mod tests {
 		});
 
 		// trying to finalize past `change_c` without finalizing `change_a` first
-		match authorities.apply_changes("hash_d", 40, &is_descendent_of) {
+		match authorities.apply_standard_changes("hash_d", 40, &is_descendent_of) {
 			Err(fork_tree::Error::UnfinalizedAncestor) => {},
 			_ => unreachable!(),
 		}
 
-		let status = authorities.apply_changes("hash_b", 15, &is_descendent_of).unwrap();
+		let status = authorities.apply_standard_changes("hash_b", 15, &is_descendent_of).unwrap();
 		assert!(status.changed);
 		assert_eq!(status.new_set_block, Some(("hash_b", 15)));
 
@@ -450,7 +623,7 @@ mod tests {
 		assert_eq!(authorities.set_id, 1);
 
 		// after finalizing `change_a` it should be possible to finalize `change_c`
-		let status = authorities.apply_changes("hash_d", 40, &is_descendent_of).unwrap();
+		let status = authorities.apply_standard_changes("hash_d", 40, &is_descendent_of).unwrap();
 		assert!(status.changed);
 		assert_eq!(status.new_set_block, Some(("hash_d", 40)));
 
@@ -459,20 +632,22 @@ mod tests {
 	}
 
 	#[test]
-	fn enacts_change_works() {
+	fn enacts_standard_change_works() {
 		let mut authorities = AuthoritySet {
 			current_authorities: Vec::new(),
 			set_id: 0,
-			pending_changes: ForkTree::new(),
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
 		};
 
 		let set_a = vec![([1; 32].into(), 5)];
 
 		let change_a = PendingChange {
 			next_authorities: set_a.clone(),
-			finalization_depth: 10,
+			delay: 10,
 			canon_height: 5,
 			canon_hash: "hash_a",
+			delay_kind: DelayKind::Finalized,
 		};
 
 		authorities.add_pending_change(change_a.clone(), &static_is_descendent_of(false)).unwrap();
@@ -484,12 +659,84 @@ mod tests {
 		});
 
 		// "hash_c" won't finalize the existing change since it isn't a descendent
-		assert!(!authorities.enacts_change("hash_c", 15, &is_descendent_of).unwrap());
-
+		assert!(!authorities.enacts_standard_change("hash_c", 15, &is_descendent_of).unwrap());
 		// "hash_b" at depth 14 won't work either
-		assert!(!authorities.enacts_change("hash_b", 14, &is_descendent_of).unwrap());
+		assert!(!authorities.enacts_standard_change("hash_b", 14, &is_descendent_of).unwrap());
 
 		// but it should work at depth 15 (change height + depth)
-		assert!(authorities.enacts_change("hash_b", 15, &is_descendent_of).unwrap());
+		assert!(authorities.enacts_standard_change("hash_b", 15, &is_descendent_of).unwrap());
+	}
+
+	#[test]
+	fn forced_changes() {
+		let mut authorities = AuthoritySet {
+			current_authorities: Vec::new(),
+			set_id: 0,
+			pending_standard_changes: ForkTree::new(),
+			pending_forced_changes: Vec::new(),
+		};
+
+		let set_a = vec![([1; 32].into(), 5)];
+		let set_b = vec![([2; 32].into(), 5)];
+
+		let change_a = PendingChange {
+			next_authorities: set_a.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_a",
+			delay_kind: DelayKind::Best { median_last_finalized: 42 },
+		};
+
+		let change_b = PendingChange {
+			next_authorities: set_b.clone(),
+			delay: 10,
+			canon_height: 5,
+			canon_hash: "hash_b",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		authorities.add_pending_change(change_a, &static_is_descendent_of(false)).unwrap();
+		authorities.add_pending_change(change_b, &static_is_descendent_of(false)).unwrap();
+
+		// there's an effective change triggered at block 15 but not a standard one.
+		// so this should do nothing.
+		assert!(!authorities.enacts_standard_change("hash_c", 15, &static_is_descendent_of(true)).unwrap());
+
+		// throw a standard change into the mix to prove that it's discarded
+		// for being on the same fork.
+		//
+		// NOTE: after https://github.com/paritytech/substrate/issues/1861
+		// this should still be rejected based on the "span" rule -- it overlaps
+		// with another change on the same fork.
+		let change_c = PendingChange {
+			next_authorities: set_b.clone(),
+			delay: 3,
+			canon_height: 8,
+			canon_hash: "hash_a8",
+			delay_kind: DelayKind::Best { median_last_finalized: 0 },
+		};
+
+		let is_descendent_of_a = is_descendent_of(|base: &&str, _| {
+			base.starts_with("hash_a")
+		});
+
+		assert!(authorities.add_pending_change(change_c, &is_descendent_of_a).is_err());
+
+		// too early.
+		assert!(authorities.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true)).unwrap().is_none());
+
+		// too late.
+		assert!(authorities.apply_forced_changes("hash_a16", 16, &static_is_descendent_of(true)).unwrap().is_none());
+
+		// on time -- chooses the right change.
+		assert_eq!(
+			authorities.apply_forced_changes("hash_a15", 15, &is_descendent_of_a).unwrap().unwrap(),
+			(42, AuthoritySet {
+				current_authorities: set_a,
+				set_id: 1,
+				pending_standard_changes: ForkTree::new(),
+				pending_forced_changes: Vec::new(),
+			})
+		);
 	}
 }

substrate/core/finality-grandpa/src/aux_schema.rs

@@ -0,0 +1,275 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Schema for stuff in the aux-db.
+
+use parity_codec::{Encode, Decode};
+use client::backend::AuxStore;
+use client::error::{Result as ClientResult, Error as ClientError, ErrorKind as ClientErrorKind};
+use fork_tree::ForkTree;
+use grandpa::round::State as RoundState;
+use substrate_primitives::Ed25519AuthorityId;
+use log::{info, warn};
+
+use crate::authorities::{AuthoritySet, SharedAuthoritySet, PendingChange, DelayKind};
+use crate::consensus_changes::{SharedConsensusChanges, ConsensusChanges};
+use crate::NewAuthoritySet;
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+const VERSION_KEY: &[u8] = b"grandpa_schema_version";
+const SET_STATE_KEY: &[u8] = b"grandpa_completed_round";
+const AUTHORITY_SET_KEY: &[u8] = b"grandpa_voters";
+const CONSENSUS_CHANGES_KEY: &[u8] = b"grandpa_consensus_changes";
+
+const CURRENT_VERSION: u32 = 1;
+
+/// The voter set state.
+#[derive(Clone, Encode, Decode)]
+pub enum VoterSetState<H, N> {
+	/// The voter set state, currently paused.
+	Paused(u64, RoundState<H, N>),
+	/// The voter set state, currently live.
+	Live(u64, RoundState<H, N>),
+}
+
+impl<H: Clone, N: Clone> VoterSetState<H, N> {
+	/// Yields the current state.
+	pub(crate) fn round(&self) -> (u64, RoundState<H, N>) {
+		match *self {
+			VoterSetState::Paused(n, ref s) => (n, s.clone()),
+			VoterSetState::Live(n, ref s) => (n, s.clone()),
+		}
+	}
+}
+
+type V0VoterSetState<H, N> = (u64, RoundState<H, N>);
+
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+struct V0PendingChange<H, N> {
+	next_authorities: Vec<(Ed25519AuthorityId, u64)>,
+	delay: N,
+	canon_height: N,
+	canon_hash: H,
+}
+
+#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+struct V0AuthoritySet<H, N> {
+	current_authorities: Vec<(Ed25519AuthorityId, u64)>,
+	set_id: u64,
+	pending_changes: Vec<V0PendingChange<H, N>>,
+}
+
+impl<H, N> Into<AuthoritySet<H, N>> for V0AuthoritySet<H, N>
+where H: Clone + Debug + PartialEq,
+	  N: Clone + Debug + Ord,
+{
+	fn into(self) -> AuthoritySet<H, N> {
+		let mut pending_standard_changes = ForkTree::new();
+
+		for old_change in self.pending_changes {
+			let new_change = PendingChange {
+				next_authorities: old_change.next_authorities,
+				delay: old_change.delay,
+				canon_height: old_change.canon_height,
+				canon_hash: old_change.canon_hash,
+				delay_kind: DelayKind::Finalized,
+			};
+
+			if let Err(err) = pending_standard_changes.import::<_, ClientError>(
+				new_change.canon_hash.clone(),
+				new_change.canon_height.clone(),
+				new_change,
+				// previously we only supported at most one pending change per fork
+				&|_, _| Ok(false),
+			) {
+				warn!(target: "afg", "Error migrating pending authority set change: {:?}.", err);
+				warn!(target: "afg", "Node is in a potentially inconsistent state.");
+			}
+		}
+
+		AuthoritySet {
+			current_authorities: self.current_authorities,
+			set_id: self.set_id,
+			pending_forced_changes: Vec::new(),
+			pending_standard_changes
+		}
+	}
+}
+
+fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<Option<T>> {
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..])
+			.ok_or_else(
+				|| ClientErrorKind::Backend(format!("GRANDPA DB is corrupted.")).into(),
+			)
+			.map(Some)
+	}
+}
+
+/// Persistent data kept between runs.
+pub(crate) struct PersistentData<H, N> {
+	pub(crate) authority_set: SharedAuthoritySet<H, N>,
+	pub(crate) consensus_changes: SharedConsensusChanges<H, N>,
+	pub(crate) set_state: VoterSetState<H, N>,
+}
+
+/// Load or initialize persistent data from backend.
+pub(crate) fn load_persistent<B, H, N, G>(
+	backend: &B,
+	genesis_hash: H,
+	genesis_number: N,
+	genesis_authorities: G,
+)
+	-> ClientResult<PersistentData<H, N>>
+	where
+		B: AuxStore,
+		H: Debug + Decode + Encode + Clone + PartialEq,
+		N: Debug + Decode + Encode + Clone + Ord,
+		G: FnOnce() -> ClientResult<Vec<(Ed25519AuthorityId, u64)>>
+{
+	let version: Option<u32> = load_decode(backend, VERSION_KEY)?;
+	let consensus_changes = load_decode(backend, CONSENSUS_CHANGES_KEY)?
+		.unwrap_or_else(ConsensusChanges::<H, N>::empty);
+
+	let make_genesis_round = move || RoundState::genesis((genesis_hash, genesis_number));
+
+	match version {
+		None => {
+			CURRENT_VERSION.using_encoded(|s|
+				backend.insert_aux(&[(VERSION_KEY, s)], &[])
+			)?;
+
+			if let Some(old_set) = load_decode::<_, V0AuthoritySet<H, N>>(backend, AUTHORITY_SET_KEY)? {
+				let new_set: AuthoritySet<H, N> = old_set.into();
+				backend.insert_aux(&[(AUTHORITY_SET_KEY, new_set.encode().as_slice())], &[])?;
+
+				let set_state = match load_decode::<_, V0VoterSetState<H, N>>(backend, SET_STATE_KEY)? {
+					Some((number, state)) => VoterSetState::Live(number, state),
+					None => VoterSetState::Live(0, make_genesis_round()),
+				};
+
+				return Ok(PersistentData {
+					authority_set: new_set.into(),
+					consensus_changes: Arc::new(consensus_changes.into()),
+					set_state,
+				});
+			}
+		}
+		Some(1) => {
+			if let Some(set) = load_decode::<_, AuthoritySet<H, N>>(backend, AUTHORITY_SET_KEY)? {
+				let set_state = match load_decode::<_, VoterSetState<H, N>>(backend, SET_STATE_KEY)? {
+					Some(state) => state,
+					None => VoterSetState::Live(0, make_genesis_round()),
+				};
+
+				return Ok(PersistentData {
+					authority_set: set.into(),
+					consensus_changes: Arc::new(consensus_changes.into()),
+					set_state,
+				});
+			}
+		}
+		Some(other) => return Err(ClientErrorKind::Backend(
+			format!("Unsupported GRANDPA DB version: {:?}", other)
+		).into()),
+	}
+
+	// genesis.
+	info!(target: "afg", "Loading GRANDPA authority set \
+		from genesis on what appears to be first startup.");
+
+	let genesis_set = AuthoritySet::genesis(genesis_authorities()?);
+	let genesis_state = VoterSetState::Live(0, make_genesis_round());
+	backend.insert_aux(
+		&[
+			(AUTHORITY_SET_KEY, genesis_set.encode().as_slice()),
+			(SET_STATE_KEY, genesis_state.encode().as_slice()),
+		],
+		&[],
+	)?;
+
+	Ok(PersistentData {
+		authority_set: genesis_set.into(),
+		set_state: genesis_state,
+		consensus_changes: Arc::new(consensus_changes.into()),
+	})
+}
+
+/// Update the authority set on disk after a change.
+pub(crate) fn update_authority_set<H, N, F, R>(
+	set: &AuthoritySet<H, N>,
+	new_set: Option<&NewAuthoritySet<H, N>>,
+	write_aux: F
+) -> R where
+	H: Encode + Clone,
+	N: Encode + Clone,
+	F: FnOnce(&[(&'static [u8], &[u8])]) -> R,
+{
+	// write new authority set state to disk.
+	let encoded_set = set.encode();
+
+	if let Some(new_set) = new_set {
+		// we also overwrite the "last completed round" entry with a blank slate
+		// because from the perspective of the finality gadget, the chain has
+		// reset.
+		let round_state = RoundState::genesis((
+			new_set.canon_hash.clone(),
+			new_set.canon_number.clone(),
+		));
+		let set_state = VoterSetState::Live(0, round_state);
+		let encoded = set_state.encode();
+
+		write_aux(&[
+			(AUTHORITY_SET_KEY, &encoded_set[..]),
+			(SET_STATE_KEY, &encoded[..]),
+		])
+	} else {
+		write_aux(&[(AUTHORITY_SET_KEY, &encoded_set[..])])
+	}
+}
+
+/// Write voter set state.
+pub(crate) fn write_voter_set_state<B, H, N>(backend: &B, state: &VoterSetState<H, N>)
+	-> ClientResult<()>
+	where B: AuxStore, H: Encode, N: Encode
+{
+	backend.insert_aux(
+		&[(SET_STATE_KEY, state.encode().as_slice())],
+		&[]
+	)
+}
+
+/// Update the consensus changes.
+pub(crate) fn update_consensus_changes<H, N, F, R>(
+	set: &ConsensusChanges<H, N>,
+	write_aux: F
+) -> R where
+	H: Encode + Clone,
+	N: Encode + Clone,
+	F: FnOnce(&[(&'static [u8], &[u8])]) -> R,
+{
+	write_aux(&[(CONSENSUS_CHANGES_KEY, set.encode().as_slice())])
+}
+
+#[cfg(test)]
+pub(crate) fn load_authorities<B: AuxStore, H: Decode, N: Decode>(backend: &B)
+	-> Option<AuthoritySet<H, N>> {
+	load_decode::<_, AuthoritySet<H, N>>(backend, AUTHORITY_SET_KEY)
+		.expect("backend error")
+}

substrate/core/finality-grandpa/src/consensus_changes.rs

@@ -23,11 +23,14 @@ pub(crate) struct ConsensusChanges<H, N> {
 	pending_changes: Vec<(N, H)>,
 }
 
-impl<H: Copy + PartialEq, N: Copy + Ord> ConsensusChanges<H, N> {
+impl<H, N> ConsensusChanges<H, N> {
 	/// Create empty consensus changes.
 	pub(crate) fn empty() -> Self {
 		ConsensusChanges { pending_changes: Vec::new(), }
 	}
+}
+
+impl<H: Copy + PartialEq, N: Copy + Ord> ConsensusChanges<H, N> {
 
 	/// Note unfinalized change of consensus-related data.
 	pub(crate) fn note_change(&mut self, at: (N, H)) {

substrate/core/finality-grandpa/src/environment.rs

@@ -14,7 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::fmt;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -22,6 +21,7 @@ use log::{debug, warn, info};
 use parity_codec::Encode;
 use futures::prelude::*;
 use tokio::timer::Delay;
+use parking_lot::RwLock;
 
 use client::{
 	backend::Backend, BlockchainEvents, CallExecutor, Client, error::Error as ClientError
@@ -36,14 +36,42 @@ use runtime_primitives::traits::{
 use substrate_primitives::{Blake2Hasher, ed25519,Ed25519AuthorityId, H256};
 
 use crate::{
-	AUTHORITY_SET_KEY, CONSENSUS_CHANGES_KEY, LAST_COMPLETED_KEY,
-	Commit, Config, Error, Network, Precommit, Prevote, LastCompleted,
+	Commit, Config, Error, Network, Precommit, Prevote,
+	CommandOrError, NewAuthoritySet, VoterCommand,
 };
-use crate::authorities::{AuthoritySet, SharedAuthoritySet};
+
+use crate::authorities::SharedAuthoritySet;
 use crate::consensus_changes::SharedConsensusChanges;
 use crate::justification::GrandpaJustification;
 use crate::until_imported::UntilVoteTargetImported;
 
+/// Data about a completed round.
+pub(crate) type CompletedRound<H, N> = (u64, RoundState<H, N>);
+
+/// A read-only view of the last completed round.
+pub(crate) struct LastCompletedRound<H, N> {
+	inner: RwLock<CompletedRound<H, N>>,
+}
+
+impl<H: Clone, N: Clone> LastCompletedRound<H, N> {
+	/// Create a new tracker based on some starting last-completed round.
+	pub(crate) fn new(round: CompletedRound<H, N>) -> Self {
+		LastCompletedRound { inner: RwLock::new(round) }
+	}
+
+	/// Read the last completed round.
+	pub(crate) fn read(&self) -> CompletedRound<H, N> {
+		self.inner.read().clone()
+	}
+
+	// NOTE: not exposed outside of this module intentionally.
+	fn with<F, R>(&self, f: F) -> R
+		where F: FnOnce(&mut CompletedRound<H, N>) -> R
+	{
+		f(&mut *self.inner.write())
+	}
+}
+
 /// The environment we run GRANDPA in.
 pub(crate) struct Environment<B, E, Block: BlockT, N: Network<Block>, RA> {
 	pub(crate) inner: Arc<Client<B, E, Block, RA>>,
@@ -53,6 +81,7 @@ pub(crate) struct Environment<B, E, Block: BlockT, N: Network<Block>, RA> {
 	pub(crate) consensus_changes: SharedConsensusChanges<Block::Hash, NumberFor<Block>>,
 	pub(crate) network: N,
 	pub(crate) set_id: u64,
+	pub(crate) last_completed: LastCompletedRound<Block::Hash, NumberFor<Block>>,
 }
 
 impl<Block: BlockT<Hash=H256>, B, E, N, RA> grandpa::Chain<Block::Hash, NumberFor<Block>> for Environment<B, E, Block, N, RA> where
@@ -166,54 +195,6 @@ impl<Block: BlockT<Hash=H256>, B, E, N, RA> grandpa::Chain<Block::Hash, NumberFo
 	}
 }
 
-/// A new authority set along with the canonical block it changed at.
-#[derive(Debug)]
-pub(crate) struct NewAuthoritySet<H, N> {
-	pub(crate) canon_number: N,
-	pub(crate) canon_hash: H,
-	pub(crate) set_id: u64,
-	pub(crate) authorities: Vec<(Ed25519AuthorityId, u64)>,
-}
-
-/// Signals either an early exit of a voter or an error.
-#[derive(Debug)]
-pub(crate) enum ExitOrError<H, N> {
-	/// An error occurred.
-	Error(Error),
-	/// Early exit of the voter: the new set ID and the new authorities along with respective weights.
-	AuthoritiesChanged(NewAuthoritySet<H, N>),
-}
-
-impl<H, N> From<Error> for ExitOrError<H, N> {
-	fn from(e: Error) -> Self {
-		ExitOrError::Error(e)
-	}
-}
-
-impl<H, N> From<ClientError> for ExitOrError<H, N> {
-	fn from(e: ClientError) -> Self {
-		ExitOrError::Error(Error::Client(e))
-	}
-}
-
-impl<H, N> From<grandpa::Error> for ExitOrError<H, N> {
-	fn from(e: grandpa::Error) -> Self {
-		ExitOrError::Error(Error::from(e))
-	}
-}
-
-impl<H: fmt::Debug, N: fmt::Debug> ::std::error::Error for ExitOrError<H, N> { }
-
-impl<H, N> fmt::Display for ExitOrError<H, N> {
-	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-		match *self {
-			ExitOrError::Error(ref e) => write!(f, "{:?}", e),
-			ExitOrError::AuthoritiesChanged(_) => write!(f, "restarting voter on new authorities"),
-		}
-	}
-}
-
-
 impl<B, E, Block: BlockT<Hash=H256>, N, RA> voter::Environment<Block::Hash, NumberFor<Block>> for Environment<B, E, Block, N, RA> where
 	Block: 'static,
 	B: Backend<Block, Blake2Hasher> + 'static,
@@ -237,7 +218,7 @@ impl<B, E, Block: BlockT<Hash=H256>, N, RA> voter::Environment<Block::Hash, Numb
 		SinkError = Self::Error,
 	> + Send>;
 
-	type Error = ExitOrError<Block::Hash, NumberFor<Block>>;
+	type Error = CommandOrError<Block::Hash, NumberFor<Block>>;
 
 	fn round_data(
 		&self,
@@ -295,17 +276,32 @@ impl<B, E, Block: BlockT<Hash=H256>, N, RA> voter::Environment<Block::Hash, Numb
 			state.finalized.as_ref().map(|e| e.1),
 		);
 
-		let encoded_state = (round, state).encode();
-		let res = Backend::insert_aux(&**self.inner.backend(), &[(LAST_COMPLETED_KEY, &encoded_state[..])], &[]);
-		if let Err(e) = res {
-			warn!(target: "afg", "Shutting down voter due to error bookkeeping last completed round in DB: {:?}", e);
-			Err(Error::Client(e).into())
-		} else {
+		self.last_completed.with(|last_completed| {
+			let set_state = crate::aux_schema::VoterSetState::Live(round, state.clone());
+			crate::aux_schema::write_voter_set_state(&**self.inner.backend(), &set_state)?;
+
+			*last_completed = (round, state); // after writing to DB successfully.
 			Ok(())
-		}
+		})
 	}
 
 	fn finalize_block(&self, hash: Block::Hash, number: NumberFor<Block>, round: u64, commit: Commit<Block>) -> Result<(), Self::Error> {
+		use client::blockchain::HeaderBackend;
+
+		let status = self.inner.backend().blockchain().info()?;
+		if number <= status.finalized_number && self.inner.backend().blockchain().hash(number)? == Some(hash) {
+			// This can happen after a forced change (triggered by the finality tracker when finality is stalled), since
+			// the voter will be restarted at the median last finalized block, which can be lower than the local best
+			// finalized block.
+			warn!(target: "afg", "Re-finalized block #{:?} ({:?}) in the canonical chain, current best finalized is #{:?}",
+				  hash,
+				  number,
+				  status.finalized_number,
+			);
+
+			return Ok(());
+		}
+
 		finalize_block(
 			&*self.inner,
 			&self.authority_set,
@@ -375,7 +371,7 @@ pub(crate) fn finalize_block<B, Block: BlockT<Hash=H256>, E, RA>(
 	hash: Block::Hash,
 	number: NumberFor<Block>,
 	justification_or_commit: JustificationOrCommit<Block>,
-) -> Result<(), ExitOrError<Block::Hash, NumberFor<Block>>> where
+) -> Result<(), CommandOrError<Block::Hash, NumberFor<Block>>> where
 	B: Backend<Block, Blake2Hasher>,
 	E: CallExecutor<Block, Blake2Hasher> + Send + Sync,
 	RA: Send + Sync,
@@ -385,72 +381,43 @@ pub(crate) fn finalize_block<B, Block: BlockT<Hash=H256>, E, RA>(
 
 	// FIXME #1483: clone only when changed
 	let old_authority_set = authority_set.clone();
-	// needed in case there is an authority set change, used for reverting in
-	// case of error
-	let mut old_last_completed = None;
-
-	let mut consensus_changes = consensus_changes.lock();
-	let status = authority_set.apply_changes(
-		hash,
-		number,
-		&is_descendent_of(client, None),
-	).map_err(|e| Error::Safety(e.to_string()))?;
-
-	if status.changed {
-		// write new authority set state to disk.
-		let encoded_set = authority_set.encode();
-
-		let write_result = if let Some((ref canon_hash, ref canon_number)) = status.new_set_block {
-			// we also overwrite the "last completed round" entry with a blank slate
-			// because from the perspective of the finality gadget, the chain has
-			// reset.
-			let round_state = RoundState::genesis((*canon_hash, *canon_number));
-			let last_completed: LastCompleted<_, _> = (0, round_state);
-			let encoded = last_completed.encode();
-
-			old_last_completed = Backend::get_aux(&**client.backend(), LAST_COMPLETED_KEY)?;
-
-			Backend::insert_aux(
-				&**client.backend(),
-				&[
-					(AUTHORITY_SET_KEY, &encoded_set[..]),
-					(LAST_COMPLETED_KEY, &encoded[..]),
-				],
-				&[]
-			)
-		} else {
-			Backend::insert_aux(&**client.backend(), &[(AUTHORITY_SET_KEY, &encoded_set[..])], &[])
-		};
-
-		if let Err(e) = write_result {
-			warn!(target: "finality", "Failed to write updated authority set to disk. Bailing.");
-			warn!(target: "finality", "Node is in a potentially inconsistent state.");
-
-			return Err(e.into());
-		}
-	}
-
-	// check if this is this is the first finalization of some consensus changes
-	let (alters_consensus_changes, finalizes_consensus_changes) = consensus_changes
-		.finalize((number, hash), |at_height| canonical_at_height(client, (hash, number), at_height))?;
-
 	// holds the old consensus changes in case it is changed below, needed for
 	// reverting in case of failure
 	let mut old_consensus_changes = None;
-	if alters_consensus_changes {
-		old_consensus_changes = Some(consensus_changes.clone());
 
-		let encoded = consensus_changes.encode();
-		let write_result = Backend::insert_aux(&**client.backend(), &[(CONSENSUS_CHANGES_KEY, &encoded[..])], &[]);
-		if let Err(e) = write_result {
-			warn!(target: "finality", "Failed to write updated consensus changes to disk. Bailing.");
-			warn!(target: "finality", "Node is in a potentially inconsistent state.");
+	let mut consensus_changes = consensus_changes.lock();
+	let canon_at_height = |canon_number| {
+		// "true" because the block is finalized
+		canonical_at_height(client, (hash, number), true, canon_number)
+	};
 
-			return Err(e.into());
+	let update_res: Result<_, Error> = client.lock_import_and_run(|import_op| {
+		let status = authority_set.apply_standard_changes(
+			hash,
+			number,
+			&is_descendent_of(client, None),
+		).map_err(|e| Error::Safety(e.to_string()))?;
+
+		// check if this is this is the first finalization of some consensus changes
+		let (alters_consensus_changes, finalizes_consensus_changes) = consensus_changes
+			.finalize((number, hash), &canon_at_height)?;
+
+		if alters_consensus_changes {
+			old_consensus_changes = Some(consensus_changes.clone());
+
+			let write_result = crate::aux_schema::update_consensus_changes(
+				&*consensus_changes,
+				|insert| client.apply_aux(import_op, insert, &[]),
+			);
+
+			if let Err(e) = write_result {
+				warn!(target: "finality", "Failed to write updated consensus changes to disk. Bailing.");
+				warn!(target: "finality", "Node is in a potentially inconsistent state.");
+
+				return Err(e.into());
+			}
 		}
-	}
 
-	let aux = |authority_set: &AuthoritySet<Block::Hash, NumberFor<Block>>| {
 		// NOTE: this code assumes that honest voters will never vote past a
 		// transition block, thus we don't have to worry about the case where
 		// we have a transition with `effective_block = N`, but we finalize
@@ -496,12 +463,12 @@ pub(crate) fn finalize_block<B, Block: BlockT<Hash=H256>, E, RA>(
 
 		// ideally some handle to a synchronization oracle would be used
 		// to avoid unconditionally notifying.
-		client.finalize_block(BlockId::Hash(hash), justification, true).map_err(|e| {
+		client.apply_finality(import_op, BlockId::Hash(hash), justification, true).map_err(|e| {
 			warn!(target: "finality", "Error applying finality to block {:?}: {:?}", (hash, number), e);
 			e
 		})?;
 
-		if let Some((canon_hash, canon_number)) = status.new_set_block {
+		let new_authorities = if let Some((canon_hash, canon_number)) = status.new_set_block {
 			// the authority set has changed.
 			let (new_id, set_ref) = authority_set.current();
 
@@ -511,54 +478,46 @@ pub(crate) fn finalize_block<B, Block: BlockT<Hash=H256>, E, RA>(
 				info!("Applying GRANDPA set change to new set {:?}", set_ref);
 			}
 
-			Err(ExitOrError::AuthoritiesChanged(NewAuthoritySet {
+			Some(NewAuthoritySet {
 				canon_hash,
 				canon_number,
 				set_id: new_id,
 				authorities: set_ref.to_vec(),
-			}))
+			})
 		} else {
-			Ok(())
-		}
-	};
+			None
+		};
 
-	match aux(&authority_set) {
-		Err(ExitOrError::Error(err)) => {
-			debug!(target: "afg", "Reverting authority set and/or consensus changes after block finalization error: {:?}", err);
-
-			let mut revert_aux = Vec::new();
-
-			if status.changed {
-				revert_aux.push((AUTHORITY_SET_KEY, old_authority_set.encode()));
-				if let Some(old_last_completed) = old_last_completed {
-					revert_aux.push((LAST_COMPLETED_KEY, old_last_completed));
-				}
-
-				*authority_set = old_authority_set.clone();
-			}
-
-			if let Some(old_consensus_changes) = old_consensus_changes {
-				revert_aux.push((CONSENSUS_CHANGES_KEY, old_consensus_changes.encode()));
-
-				*consensus_changes = old_consensus_changes;
-			}
-
-			let write_result = Backend::insert_aux(
-				&**client.backend(),
-				revert_aux.iter().map(|(k, v)| (*k, &**v)).collect::<Vec<_>>().iter(),
-				&[],
+		if status.changed {
+			let write_result = crate::aux_schema::update_authority_set(
+				&authority_set,
+				new_authorities.as_ref(),
+				|insert| client.apply_aux(import_op, insert, &[]),
 			);
 
 			if let Err(e) = write_result {
-				warn!(target: "finality", "Failed to revert consensus changes to disk. Bailing.");
+				warn!(target: "finality", "Failed to write updated authority set to disk. Bailing.");
 				warn!(target: "finality", "Node is in a potentially inconsistent state.");
 
 				return Err(e.into());
 			}
+		}
 
-			Err(ExitOrError::Error(err))
-		},
-		res => res,
+		Ok(new_authorities.map(VoterCommand::ChangeAuthorities))
+	});
+
+	match update_res {
+		Ok(Some(command)) => Err(CommandOrError::VoterCommand(command)),
+		Ok(None) => Ok(()),
+		Err(e) => {
+			*authority_set = old_authority_set;
+
+			if let Some(old_consensus_changes) = old_consensus_changes {
+				*consensus_changes = old_consensus_changes;
+			}
+
+			Err(CommandOrError::Error(e))
+		}
 	}
 }
 
@@ -567,27 +526,39 @@ pub(crate) fn finalize_block<B, Block: BlockT<Hash=H256>, E, RA>(
 pub(crate) fn canonical_at_height<B, E, Block: BlockT<Hash=H256>, RA>(
 	client: &Client<B, E, Block, RA>,
 	base: (Block::Hash, NumberFor<Block>),
+	base_is_canonical: bool,
 	height: NumberFor<Block>,
 ) -> Result<Option<Block::Hash>, ClientError> where
 	B: Backend<Block, Blake2Hasher>,
 	E: CallExecutor<Block, Blake2Hasher> + Send + Sync,
 {
-	use runtime_primitives::traits::{One, Zero};
+	use runtime_primitives::traits::{One, Zero, BlockNumberToHash};
 
 	if height > base.1 {
 		return Ok(None);
 	}
 
 	if height == base.1 {
-		return Ok(Some(base.0));
+		if base_is_canonical {
+			return Ok(Some(base.0));
+		} else {
+			return Ok(client.block_number_to_hash(height));
+		}
+	} else if base_is_canonical {
+		return Ok(client.block_number_to_hash(height));
 	}
 
-	let mut current = match client.header(&BlockId::Hash(base.0))? {
+	let one = NumberFor::<Block>::one();
+
+	// start by getting _canonical_ block with number at parent position and then iterating
+	// backwards by hash.
+	let mut current = match client.header(&BlockId::Number(base.1 - one))? {
 		Some(header) => header,
 		_ => return Ok(None),
 	};
 
-	let mut steps = base.1 - height;
+	// we've already checked that base > height above.
+	let mut steps = base.1 - height - one;
 
 	while steps > NumberFor::<Block>::zero() {
 		current = match client.header(&BlockId::Hash(*current.parent_hash()))? {
@@ -595,7 +566,7 @@ pub(crate) fn canonical_at_height<B, E, Block: BlockT<Hash=H256>, RA>(
 			_ => return Ok(None),
 		};
 
-		steps -= NumberFor::<Block>::one();
+		steps -= one;
 	}
 
 	Ok(Some(current.hash()))

substrate/core/finality-grandpa/src/import.rs

@@ -19,9 +19,12 @@ use std::sync::Arc;
 use log::{debug, trace, info};
 use parity_codec::Encode;
 use futures::sync::mpsc;
+use parking_lot::RwLockWriteGuard;
 
 use client::{blockchain, CallExecutor, Client};
+use client::blockchain::HeaderBackend;
 use client::backend::Backend;
+use client::runtime_api::ApiExt;
 use consensus_common::{
 	BlockImport, Error as ConsensusError, ErrorKind as ConsensusErrorKind,
 	ImportBlock, ImportResult, JustificationImport,
@@ -35,10 +38,10 @@ use runtime_primitives::traits::{
 };
 use substrate_primitives::{H256, Ed25519AuthorityId, Blake2Hasher};
 
-use crate::{AUTHORITY_SET_KEY, Error};
-use crate::authorities::SharedAuthoritySet;
+use crate::{Error, CommandOrError, NewAuthoritySet, VoterCommand};
+use crate::authorities::{AuthoritySet, SharedAuthoritySet, DelayKind, PendingChange};
 use crate::consensus_changes::SharedConsensusChanges;
-use crate::environment::{finalize_block, is_descendent_of, ExitOrError, NewAuthoritySet};
+use crate::environment::{finalize_block, is_descendent_of};
 use crate::justification::GrandpaJustification;
 
 /// A block-import handler for GRANDPA.
@@ -53,7 +56,7 @@ use crate::justification::GrandpaJustification;
 pub struct GrandpaBlockImport<B, E, Block: BlockT<Hash=H256>, RA, PRA> {
 	inner: Arc<Client<B, E, Block, RA>>,
 	authority_set: SharedAuthoritySet<Block::Hash, NumberFor<Block>>,
-	authority_set_change: mpsc::UnboundedSender<NewAuthoritySet<Block::Hash, NumberFor<Block>>>,
+	send_voter_commands: mpsc::UnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
 	consensus_changes: SharedConsensusChanges<Block::Hash, NumberFor<Block>>,
 	api: Arc<PRA>,
 }
@@ -80,7 +83,8 @@ impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> JustificationImport<Block>
 		// request justifications for all pending changes for which change blocks have already been imported
 		let authorities = self.authority_set.inner().read();
 		for pending_change in authorities.pending_changes() {
-			if pending_change.effective_number() > chain_info.finalized_number &&
+			if pending_change.delay_kind == DelayKind::Finalized &&
+				pending_change.effective_number() > chain_info.finalized_number &&
 				pending_change.effective_number() <= chain_info.best_number
 			{
 				let effective_block_hash = self.inner.best_containing(
@@ -109,6 +113,266 @@ impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> JustificationImport<Block>
 	}
 }
 
+enum AppliedChanges<H, N> {
+	Standard,
+	Forced(NewAuthoritySet<H, N>),
+	None,
+}
+
+impl<H, N> AppliedChanges<H, N> {
+	fn needs_justification(&self) -> bool {
+		match *self {
+			AppliedChanges::Standard => true,
+			AppliedChanges::Forced(_) | AppliedChanges::None => false,
+		}
+	}
+}
+
+struct PendingSetChanges<'a, Block: 'a + BlockT> {
+	just_in_case: Option<(
+		AuthoritySet<Block::Hash, NumberFor<Block>>,
+		RwLockWriteGuard<'a, AuthoritySet<Block::Hash, NumberFor<Block>>>,
+	)>,
+	applied_changes: AppliedChanges<Block::Hash, NumberFor<Block>>,
+	do_pause: bool,
+}
+
+impl<'a, Block: 'a + BlockT> PendingSetChanges<'a, Block> {
+	// revert the pending set change explicitly.
+	fn revert(self) { }
+
+	fn defuse(mut self) -> (AppliedChanges<Block::Hash, NumberFor<Block>>, bool) {
+		self.just_in_case = None;
+		let applied_changes = ::std::mem::replace(&mut self.applied_changes, AppliedChanges::None);
+		(applied_changes, self.do_pause)
+	}
+}
+
+impl<'a, Block: 'a + BlockT> Drop for PendingSetChanges<'a, Block> {
+	fn drop(&mut self) {
+		if let Some((old_set, mut authorities)) = self.just_in_case.take() {
+			*authorities = old_set;
+		}
+	}
+}
+
+impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> GrandpaBlockImport<B, E, Block, RA, PRA> where
+	NumberFor<Block>: grandpa::BlockNumberOps,
+	B: Backend<Block, Blake2Hasher> + 'static,
+	E: CallExecutor<Block, Blake2Hasher> + 'static + Clone + Send + Sync,
+	DigestFor<Block>: Encode,
+	DigestItemFor<Block>: DigestItem<AuthorityId=Ed25519AuthorityId>,
+	RA: Send + Sync,
+	PRA: ProvideRuntimeApi,
+	PRA::Api: GrandpaApi<Block>,
+{
+	// check for a new authority set change.
+	fn check_new_change(&self, header: &Block::Header, hash: Block::Hash)
+		-> Result<Option<PendingChange<Block::Hash, NumberFor<Block>>>, ConsensusError>
+	{
+		let at = BlockId::hash(*header.parent_hash());
+		let digest = header.digest();
+
+		let api = self.api.runtime_api();
+
+		// check for forced change.
+		{
+			let maybe_change = api.grandpa_forced_change(
+				&at,
+				digest,
+			);
+
+			match maybe_change {
+				Err(e) => match api.has_api_with::<GrandpaApi<Block>, _>(&at, |v| v >= 2) {
+					Err(e) => return Err(ConsensusErrorKind::ClientImport(e.to_string()).into()),
+					Ok(true) => {
+						// API version is high enough to support forced changes
+						// but got error, so it is legitimate.
+						return Err(ConsensusErrorKind::ClientImport(e.to_string()).into())
+					},
+					Ok(false) => {
+						// API version isn't high enough to support forced changes
+					},
+				},
+				Ok(None) => {},
+				Ok(Some((median_last_finalized, change))) => return Ok(Some(PendingChange {
+					next_authorities: change.next_authorities,
+					delay: change.delay,
+					canon_height: *header.number(),
+					canon_hash: hash,
+					delay_kind: DelayKind::Best { median_last_finalized },
+				})),
+			}
+		}
+
+		// check normal scheduled change.
+		{
+			let maybe_change = api.grandpa_pending_change(
+				&at,
+				digest,
+			);
+
+			match maybe_change {
+				Err(e) => Err(ConsensusErrorKind::ClientImport(e.to_string()).into()),
+				Ok(Some(change)) => Ok(Some(PendingChange {
+					next_authorities: change.next_authorities,
+					delay: change.delay,
+					canon_height: *header.number(),
+					canon_hash: hash,
+					delay_kind: DelayKind::Finalized,
+				})),
+				Ok(None) => Ok(None),
+			}
+		}
+	}
+
+	fn make_authorities_changes<'a>(&'a self, block: &mut ImportBlock<Block>, hash: Block::Hash)
+		-> Result<PendingSetChanges<'a, Block>, ConsensusError>
+	{
+		// when we update the authorities, we need to hold the lock
+		// until the block is written to prevent a race if we need to restore
+		// the old authority set on error or panic.
+		struct InnerGuard<'a, T: 'a> {
+			old: Option<T>,
+			guard: Option<RwLockWriteGuard<'a, T>>,
+		}
+
+		impl<'a, T: 'a> InnerGuard<'a, T> {
+			fn as_mut(&mut self) -> &mut T {
+				&mut **self.guard.as_mut().expect("only taken on deconstruction; qed")
+			}
+
+			fn set_old(&mut self, old: T) {
+				if self.old.is_none() {
+					// ignore "newer" old changes.
+					self.old = Some(old);
+				}
+			}
+
+			fn consume(mut self) -> Option<(T, RwLockWriteGuard<'a, T>)> {
+				if let Some(old) = self.old.take() {
+					Some((old, self.guard.take().expect("only taken on deconstruction; qed")))
+				} else {
+					None
+				}
+			}
+		}
+
+		impl<'a, T: 'a> Drop for InnerGuard<'a, T> {
+			fn drop(&mut self) {
+				if let (Some(mut guard), Some(old)) = (self.guard.take(), self.old.take()) {
+					*guard = old;
+				}
+			}
+		}
+
+		let number = block.header.number().clone();
+		let maybe_change = self.check_new_change(
+			&block.header,
+			hash,
+		)?;
+
+		// returns a function for checking whether a block is a descendent of another
+		// consistent with querying client directly after importing the block.
+		let parent_hash = *block.header.parent_hash();
+		let is_descendent_of = is_descendent_of(&self.inner, Some((&hash, &parent_hash)));
+
+		let mut guard = InnerGuard {
+			guard: Some(self.authority_set.inner().write()),
+			old: None,
+		};
+
+		// whether to pause the old authority set -- happens after import
+		// of a forced change block.
+		let mut do_pause = false;
+
+		// add any pending changes.
+		if let Some(change) = maybe_change {
+			let old = guard.as_mut().clone();
+			guard.set_old(old);
+
+			if let DelayKind::Best { .. } = change.delay_kind {
+				do_pause = true;
+			}
+
+			guard.as_mut().add_pending_change(
+				change,
+				&is_descendent_of,
+			).map_err(|e| ConsensusError::from(ConsensusErrorKind::ClientImport(e.to_string())))?;
+		}
+
+		let applied_changes = {
+			let forced_change_set = guard.as_mut().apply_forced_changes(hash, number, &is_descendent_of)
+				.map_err(|e| ConsensusErrorKind::ClientImport(e.to_string()))
+				.map_err(ConsensusError::from)?;
+
+			if let Some((median_last_finalized_number, new_set)) = forced_change_set {
+				let new_authorities = {
+					let (set_id, new_authorities) = new_set.current();
+
+					// we will use the median last finalized number as a hint
+					// for the canon block the new authority set should start
+					// with. we use the minimum between the median and the local
+					// best finalized block.
+					let best_finalized_number = self.inner.backend().blockchain().info()
+						.map_err(|e| ConsensusErrorKind::ClientImport(e.to_string()))?
+						.finalized_number;
+
+					let canon_number = best_finalized_number.min(median_last_finalized_number);
+
+					let canon_hash =
+						self.inner.backend().blockchain().header(BlockId::Number(canon_number))
+							.map_err(|e| ConsensusErrorKind::ClientImport(e.to_string()))?
+							.expect("the given block number is less or equal than the current best finalized number; \
+									 current best finalized number must exist in chain; qed.")
+							.hash();
+
+					NewAuthoritySet {
+						canon_number,
+						canon_hash,
+						set_id,
+						authorities: new_authorities.to_vec(),
+					}
+				};
+				let old = ::std::mem::replace(guard.as_mut(), new_set);
+				guard.set_old(old);
+
+				AppliedChanges::Forced(new_authorities)
+			} else {
+				let did_standard = guard.as_mut().enacts_standard_change(hash, number, &is_descendent_of)
+					.map_err(|e| ConsensusErrorKind::ClientImport(e.to_string()))
+					.map_err(ConsensusError::from)?;
+
+				if did_standard {
+					AppliedChanges::Standard
+				} else {
+					AppliedChanges::None
+				}
+			}
+		};
+
+		// consume the guard safely and write necessary changes.
+		let just_in_case = guard.consume();
+		if let Some((_, ref authorities)) = just_in_case {
+			let authorities_change = match applied_changes {
+				AppliedChanges::Forced(ref new) => Some(new),
+				AppliedChanges::Standard => None, // the change isn't actually applied yet.
+				AppliedChanges::None => None,
+			};
+
+			crate::aux_schema::update_authority_set(
+				authorities,
+				authorities_change,
+				|insert| block.auxiliary.extend(
+					insert.iter().map(|(k, v)| (k.to_vec(), Some(v.to_vec())))
+				)
+			);
+		}
+
+		Ok(PendingSetChanges { just_in_case, applied_changes, do_pause })
+	}
+}
+
 impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> BlockImport<Block>
 	for GrandpaBlockImport<B, E, Block, RA, PRA> where
 		NumberFor<Block>: grandpa::BlockNumberOps,
@@ -125,12 +389,8 @@ impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> BlockImport<Block>
 	fn import_block(&self, mut block: ImportBlock<Block>, new_authorities: Option<Vec<Ed25519AuthorityId>>)
 		-> Result<ImportResult, Self::Error>
 	{
-		use crate::authorities::PendingChange;
-		use client::blockchain::HeaderBackend;
-
 		let hash = block.post_header().hash();
-		let parent_hash = *block.header.parent_hash();
-		let number = *block.header.number();
+		let number = block.header.number().clone();
 
 		// early exit if block already in chain, otherwise the check for
 		// authority changes will error when trying to re-import a change block
@@ -140,82 +400,68 @@ impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> BlockImport<Block>
 			Err(e) => return Err(ConsensusErrorKind::ClientImport(e.to_string()).into()),
 		}
 
-		let maybe_change = self.api.runtime_api().grandpa_pending_change(
-			&BlockId::hash(parent_hash),
-			&block.header.digest().clone(),
-		);
-
-		let maybe_change = match maybe_change {
-			Err(e) => return Err(ConsensusErrorKind::ClientImport(e.to_string()).into()),
-			Ok(maybe) => maybe,
-		};
-
-		// when we update the authorities, we need to hold the lock
-		// until the block is written to prevent a race if we need to restore
-		// the old authority set on error.
-		let is_descendent_of = is_descendent_of(&self.inner, Some((&hash, &parent_hash)));
-		let just_in_case = if let Some(change) = maybe_change {
-			let mut authorities = self.authority_set.inner().write();
-			let old_set = authorities.clone();
-
-			authorities.add_pending_change(
-				PendingChange {
-					next_authorities: change.next_authorities,
-					finalization_depth: change.delay,
-					canon_height: number,
-					canon_hash: hash,
-				},
-				&is_descendent_of,
-			).map_err(|e| ConsensusError::from(ConsensusErrorKind::ClientImport(e.to_string())))?;
-
-			block.auxiliary.push((AUTHORITY_SET_KEY.to_vec(), Some(authorities.encode())));
-			Some((old_set, authorities))
-		} else {
-			None
-		};
+		let pending_changes = self.make_authorities_changes(&mut block, hash)?;
 
 		// we don't want to finalize on `inner.import_block`
 		let justification = block.justification.take();
 		let enacts_consensus_change = new_authorities.is_some();
 		let import_result = self.inner.import_block(block, new_authorities);
 
-		let import_result = {
-			// we scope this so that `just_in_case` is dropped eagerly and releases the authorities lock
-			let revert_authorities = || if let Some((old_set, mut authorities)) = just_in_case {
-				*authorities = old_set;
-			};
-
+		let mut imported_aux = {
 			match import_result {
-				Ok(ImportResult::Queued) => ImportResult::Queued,
+				Ok(ImportResult::Imported(aux)) => aux,
 				Ok(r) => {
 					debug!(target: "afg", "Restoring old authority set after block import result: {:?}", r);
-					revert_authorities();
+					pending_changes.revert();
 					return Ok(r);
 				},
 				Err(e) => {
 					debug!(target: "afg", "Restoring old authority set after block import error: {:?}", e);
-					revert_authorities();
+					pending_changes.revert();
 					return Err(ConsensusErrorKind::ClientImport(e.to_string()).into());
 				},
 			}
 		};
 
-		let enacts_change = self.authority_set.inner().read().enacts_change(
-			hash,
-			number,
-			&is_descendent_of,
-		).map_err(|e| ConsensusError::from(ConsensusErrorKind::ClientImport(e.to_string())))?;
+		let (applied_changes, do_pause) = pending_changes.defuse();
 
-		if !enacts_change && !enacts_consensus_change {
-			return Ok(import_result);
+		// Send the pause signal after import but BEFORE sending a `ChangeAuthorities` message.
+		if do_pause {
+			let _ = self.send_voter_commands.unbounded_send(
+				VoterCommand::Pause(format!("Forced change scheduled after inactivity"))
+			);
+		}
+
+		let needs_justification = applied_changes.needs_justification();
+		if let AppliedChanges::Forced(new) = applied_changes {
+			// NOTE: when we do a force change we are "discrediting" the old set so we
+			// ignore any justifications from them. this block may contain a justification
+			// which should be checked and imported below against the new authority
+			// triggered by this forced change. the new grandpa voter will start at the
+			// last median finalized block (which is before the block that enacts the
+			// change), full nodes syncing the chain will not be able to successfully
+			// import justifications for those blocks since their local authority set view
+			// is still of the set before the forced change was enacted, still after #1867
+			// they should import the block and discard the justification, and they will
+			// then request a justification from sync if it's necessary (which they should
+			// then be able to successfully validate).
+			let _ = self.send_voter_commands.unbounded_send(VoterCommand::ChangeAuthorities(new));
+
+			// we must clear all pending justifications requests, presumably they won't be
+			// finalized hence why this forced changes was triggered
+			imported_aux.clear_justification_requests = true;
+		}
+
+		if !needs_justification && !enacts_consensus_change {
+			return Ok(ImportResult::Imported(imported_aux));
 		}
 
 		match justification {
 			Some(justification) => {
-				self.import_justification(hash, number, justification, enacts_change)?;
+				self.import_justification(hash, number, justification, needs_justification)?;
 			},
 			None => {
-				if enacts_change {
+				if needs_justification {
 					trace!(
 						target: "finality",
 						"Imported unjustified block #{} that enacts authority set change, waiting for finality for enactment.",
@@ -229,11 +475,11 @@ impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> BlockImport<Block>
 					self.consensus_changes.lock().note_change((number, hash));
 				}
 
-				return Ok(ImportResult::NeedsJustification);
+				imported_aux.needs_justification = true;
 			}
 		}
 
-		Ok(import_result)
+		Ok(ImportResult::Imported(imported_aux))
 	}
 
 	fn check_block(
@@ -249,22 +495,22 @@ impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> GrandpaBlockImport<B, E, Block, RA
 	pub(crate) fn new(
 		inner: Arc<Client<B, E, Block, RA>>,
 		authority_set: SharedAuthoritySet<Block::Hash, NumberFor<Block>>,
-		authority_set_change: mpsc::UnboundedSender<NewAuthoritySet<Block::Hash, NumberFor<Block>>>,
+		send_voter_commands: mpsc::UnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
 		consensus_changes: SharedConsensusChanges<Block::Hash, NumberFor<Block>>,
 		api: Arc<PRA>,
 	) -> GrandpaBlockImport<B, E, Block, RA, PRA> {
 		GrandpaBlockImport {
 			inner,
 			authority_set,
-			authority_set_change,
+			send_voter_commands,
 			consensus_changes,
 			api,
 		}
 	}
 }
 
-impl<B, E, Block: BlockT<Hash=H256>, RA, PRA>
-	GrandpaBlockImport<B, E, Block, RA, PRA> where
+impl<B, E, Block: BlockT<Hash=H256>, RA, PRA> GrandpaBlockImport<B, E, Block, RA, PRA>
+	where
 		NumberFor<Block>: grandpa::BlockNumberOps,
 		B: Backend<Block, Blake2Hasher> + 'static,
 		E: CallExecutor<Block, Blake2Hasher> + 'static + Clone + Send + Sync,
@@ -304,13 +550,15 @@ impl<B, E, Block: BlockT<Hash=H256>, RA, PRA>
 		);
 
 		match result {
-			Err(ExitOrError::AuthoritiesChanged(new)) => {
-				info!(target: "finality", "Imported justification for block #{} that enacts authority set change, signalling voter.", number);
-				if let Err(e) = self.authority_set_change.unbounded_send(new) {
+			Err(CommandOrError::VoterCommand(command)) => {
+				info!(target: "finality", "Imported justification for block #{} that triggers \
+					command {}, signalling voter.", number, command);
+
+				if let Err(e) = self.send_voter_commands.unbounded_send(command) {
 					return Err(ConsensusErrorKind::ClientImport(e.to_string()).into());
 				}
 			},
-			Err(ExitOrError::Error(e)) => {
+			Err(CommandOrError::Error(e)) => {
 				return Err(match e {
 					Error::Grandpa(error) => ConsensusErrorKind::ClientImport(error.to_string()),
 					Error::Network(error) => ConsensusErrorKind::ClientImport(error),

substrate/core/finality-grandpa/src/lib.rs

@@ -66,21 +66,27 @@ use runtime_primitives::traits::{
 	DigestItemFor, DigestItem,
 };
 use fg_primitives::GrandpaApi;
+use inherents::InherentDataProviders;
 use runtime_primitives::generic::BlockId;
 use substrate_primitives::{ed25519, H256, Ed25519AuthorityId, Blake2Hasher};
 use substrate_telemetry::{telemetry, CONSENSUS_TRACE, CONSENSUS_DEBUG, CONSENSUS_WARN};
 
+use srml_finality_tracker;
+
 use grandpa::Error as GrandpaError;
 use grandpa::{voter, round::State as RoundState, BlockNumberOps, VoterSet};
 
 use network::Service as NetworkService;
 use network::consensus_gossip as network_gossip;
+
+use std::fmt;
 use std::sync::Arc;
 use std::time::Duration;
 
 pub use fg_primitives::ScheduledChange;
 
 mod authorities;
+mod aux_schema;
 mod communication;
 mod consensus_changes;
 mod environment;
@@ -94,9 +100,8 @@ mod service_integration;
 #[cfg(feature="service-integration")]
 pub use service_integration::{LinkHalfForService, BlockImportForService};
 
-use authorities::SharedAuthoritySet;
-use consensus_changes::{ConsensusChanges, SharedConsensusChanges};
-use environment::{Environment, ExitOrError, NewAuthoritySet};
+use aux_schema::{PersistentData, VoterSetState};
+use environment::Environment;
 pub use finality_proof::{prove_finality, check_finality_proof};
 use import::GrandpaBlockImport;
 use until_imported::UntilCommitBlocksImported;
@@ -104,17 +109,9 @@ use until_imported::UntilCommitBlocksImported;
 #[cfg(test)]
 mod tests;
 
-const LAST_COMPLETED_KEY: &[u8] = b"grandpa_completed_round";
-const AUTHORITY_SET_KEY: &[u8] = b"grandpa_voters";
-const CONSENSUS_CHANGES_KEY: &[u8] = b"grandpa_consensus_changes";
-
 const GRANDPA_ENGINE_ID: network::ConsensusEngineId = [b'a', b'f', b'g', b'1'];
-
 const MESSAGE_ROUND_TOLERANCE: u64 = 2;
 
-/// round-number, round-state
-type LastCompleted<H, N> = (u64, RoundState<H, N>);
-
 /// A GRANDPA message for a substrate chain.
 pub type Message<Block> = grandpa::Message<<Block as BlockT>::Hash, NumberFor<Block>>;
 /// A signed message.
@@ -557,13 +554,81 @@ impl<B, E, Block: BlockT<Hash=H256>, RA> BlockStatus<Block> for Arc<Client<B, E,
 	}
 }
 
-/// Half of a link between a block-import worker and a the background voter.
-// This should remain non-clone.
+/// A new authority set along with the canonical block it changed at.
+#[derive(Debug)]
+pub(crate) struct NewAuthoritySet<H, N> {
+	pub(crate) canon_number: N,
+	pub(crate) canon_hash: H,
+	pub(crate) set_id: u64,
+	pub(crate) authorities: Vec<(Ed25519AuthorityId, u64)>,
+}
+
+/// Commands issued to the voter.
+#[derive(Debug)]
+pub(crate) enum VoterCommand<H, N> {
+	/// Pause the voter for given reason.
+	Pause(String),
+	/// New authorities.
+	ChangeAuthorities(NewAuthoritySet<H, N>)
+}
+
+impl<H, N> fmt::Display for VoterCommand<H, N> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			VoterCommand::Pause(ref reason) => write!(f, "Pausing voter: {}", reason),
+			VoterCommand::ChangeAuthorities(_) => write!(f, "Changing authorities"),
+		}
+	}
+}
+
+/// Signals either an early exit of a voter or an error.
+#[derive(Debug)]
+pub(crate) enum CommandOrError<H, N> {
+	/// An error occurred.
+	Error(Error),
+	/// A command to the voter.
+	VoterCommand(VoterCommand<H, N>),
+}
+
+impl<H, N> From<Error> for CommandOrError<H, N> {
+	fn from(e: Error) -> Self {
+		CommandOrError::Error(e)
+	}
+}
+
+impl<H, N> From<ClientError> for CommandOrError<H, N> {
+	fn from(e: ClientError) -> Self {
+		CommandOrError::Error(Error::Client(e))
+	}
+}
+
+impl<H, N> From<grandpa::Error> for CommandOrError<H, N> {
+	fn from(e: grandpa::Error) -> Self {
+		CommandOrError::Error(Error::from(e))
+	}
+}
+
+impl<H, N> From<VoterCommand<H, N>> for CommandOrError<H, N> {
+	fn from(e: VoterCommand<H, N>) -> Self {
+		CommandOrError::VoterCommand(e)
+	}
+}
+
+impl<H: fmt::Debug, N: fmt::Debug> ::std::error::Error for CommandOrError<H, N> { }
+
+impl<H, N> fmt::Display for CommandOrError<H, N> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			CommandOrError::Error(ref e) => write!(f, "{:?}", e),
+			CommandOrError::VoterCommand(ref cmd) => write!(f, "{}", cmd),
+		}
+	}
+}
+
 pub struct LinkHalf<B, E, Block: BlockT<Hash=H256>, RA> {
 	client: Arc<Client<B, E, Block, RA>>,
-	authority_set: SharedAuthoritySet<Block::Hash, NumberFor<Block>>,
-	authority_set_change: mpsc::UnboundedReceiver<NewAuthoritySet<Block::Hash, NumberFor<Block>>>,
-	consensus_changes: SharedConsensusChanges<Block::Hash, NumberFor<Block>>,
+	persistent_data: PersistentData<Block::Hash, NumberFor<Block>>,
+	voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
 }
 
 /// Make block importer and link half necessary to tie the background voter
@@ -577,60 +642,41 @@ pub fn block_import<B, E, Block: BlockT<Hash=H256>, RA, PRA>(
 		E: CallExecutor<Block, Blake2Hasher> + 'static + Clone + Send + Sync,
 		RA: Send + Sync,
 		PRA: ProvideRuntimeApi,
-		PRA::Api: GrandpaApi<Block>
+		PRA::Api: GrandpaApi<Block>,
 {
 	use runtime_primitives::traits::Zero;
-	let authority_set = match Backend::get_aux(&**client.backend(), AUTHORITY_SET_KEY)? {
-		None => {
-			info!(target: "afg", "Loading GRANDPA authorities \
-				from genesis on what appears to be first startup.");
 
-			// no authority set on disk: fetch authorities from genesis state.
-			// if genesis state is not available, we may be a light client, but these
-			// are unsupported for following GRANDPA directly.
+	let chain_info = client.info()?;
+	let genesis_hash = chain_info.chain.genesis_hash;
+
+	let persistent_data = aux_schema::load_persistent(
+		&**client.backend(),
+		genesis_hash,
+		<NumberFor<Block>>::zero(),
+		|| {
 			let genesis_authorities = api.runtime_api()
 				.grandpa_authorities(&BlockId::number(Zero::zero()))?;
 			telemetry!(CONSENSUS_DEBUG; "afg.loading_authorities";
 				"authorities_len" => ?genesis_authorities.len()
 			);
-
-			let authority_set = SharedAuthoritySet::genesis(genesis_authorities);
-			let encoded = authority_set.inner().read().encode();
-			Backend::insert_aux(&**client.backend(), &[(AUTHORITY_SET_KEY, &encoded[..])], &[])?;
-
-			authority_set
+			Ok(genesis_authorities)
 		}
-		Some(raw) => crate::authorities::AuthoritySet::decode(&mut &raw[..])
-			.ok_or_else(|| ::client::error::ErrorKind::Backend(
-				format!("GRANDPA authority set kept in invalid format")
-			))?
-			.into(),
-	};
+	)?;
 
-	let consensus_changes = Backend::get_aux(&**client.backend(), CONSENSUS_CHANGES_KEY)?;
-	let consensus_changes = Arc::new(parking_lot::Mutex::new(match consensus_changes {
-		Some(raw) => ConsensusChanges::decode(&mut &raw[..])
-			.ok_or_else(|| ::client::error::ErrorKind::Backend(
-				format!("GRANDPA consensus changes kept in invalid format")
-			))?,
-		None => ConsensusChanges::empty(),
-	}));
-
-	let (authority_set_change_tx, authority_set_change_rx) = mpsc::unbounded();
+	let (voter_commands_tx, voter_commands_rx) = mpsc::unbounded();
 
 	Ok((
 		GrandpaBlockImport::new(
 			client.clone(),
-			authority_set.clone(),
-			authority_set_change_tx,
-			consensus_changes.clone(),
+			persistent_data.authority_set.clone(),
+			voter_commands_tx,
+			persistent_data.consensus_changes.clone(),
 			api,
 		),
 		LinkHalf {
 			client,
-			authority_set,
-			authority_set_change: authority_set_change_rx,
-			consensus_changes,
+			persistent_data,
+			voter_commands_rx,
 		},
 	))
 }
@@ -644,11 +690,11 @@ fn committer_communication<Block: BlockT<Hash=H256>, B, E, N, RA>(
 ) -> (
 	impl Stream<
 		Item = (u64, ::grandpa::CompactCommit<H256, NumberFor<Block>, ed25519::Signature, Ed25519AuthorityId>),
-		Error = ExitOrError<H256, NumberFor<Block>>,
+		Error = CommandOrError<H256, NumberFor<Block>>,
 	>,
 	impl Sink<
 		SinkItem = (u64, ::grandpa::Commit<H256, NumberFor<Block>, ed25519::Signature, Ed25519AuthorityId>),
-		SinkError = ExitOrError<H256, NumberFor<Block>>,
+		SinkError = CommandOrError<H256, NumberFor<Block>>,
 	>,
 ) where
 	B: Backend<Block, Blake2Hasher>,
@@ -687,12 +733,37 @@ fn committer_communication<Block: BlockT<Hash=H256>, B, E, N, RA>(
 	(commit_in, commit_out)
 }
 
+/// Register the finality tracker inherent data provider (which is used by
+/// GRANDPA), if not registered already.
+fn register_finality_tracker_inherent_data_provider<B, E, Block: BlockT<Hash=H256>, RA>(
+	client: Arc<Client<B, E, Block, RA>>,
+	inherent_data_providers: &InherentDataProviders,
+) -> Result<(), consensus_common::Error> where
+	B: Backend<Block, Blake2Hasher> + 'static,
+	E: CallExecutor<Block, Blake2Hasher> + Send + Sync + 'static,
+	RA: Send + Sync + 'static,
+{
+	if !inherent_data_providers.has_provider(&srml_finality_tracker::INHERENT_IDENTIFIER) {
+		inherent_data_providers
+			.register_provider(srml_finality_tracker::InherentDataProvider::new(move || {
+				match client.backend().blockchain().info() {
+					Err(e) => Err(std::borrow::Cow::Owned(e.to_string())),
+					Ok(info) => Ok(info.finalized_number),
+				}
+			}))
+			.map_err(|err| consensus_common::ErrorKind::InherentData(err.into()).into())
+	} else {
+		Ok(())
+	}
+}
+
 /// Run a GRANDPA voter as a task. Provide configuration and a link to a
 /// block import worker that has already been instantiated with `block_import`.
 pub fn run_grandpa<B, E, Block: BlockT<Hash=H256>, N, RA>(
 	config: Config,
 	link: LinkHalf<B, E, Block, RA>,
 	network: N,
+	inherent_data_providers: InherentDataProviders,
 	on_exit: impl Future<Item=(),Error=()> + Send + 'static,
 ) -> ::client::error::Result<impl Future<Item=(),Error=()> + Send + 'static> where
 	Block::Hash: Ord,
@@ -706,29 +777,18 @@ pub fn run_grandpa<B, E, Block: BlockT<Hash=H256>, N, RA>(
 	RA: Send + Sync + 'static,
 {
 	use futures::future::{self, Loop as FutureLoop};
-	use runtime_primitives::traits::Zero;
 
 	let LinkHalf {
 		client,
-		authority_set,
-		authority_set_change,
-		consensus_changes,
+		persistent_data,
+		voter_commands_rx,
 	} = link;
-
-	let chain_info = client.info()?;
-	let genesis_hash = chain_info.chain.genesis_hash;
-
 	// we shadow network with the wrapping/rebroadcasting network to avoid
 	// accidental reuse.
 	let (broadcast_worker, network) = communication::rebroadcasting_network(network);
+	let PersistentData { authority_set, set_state, consensus_changes } = persistent_data;
 
-	let (last_round_number, last_state) = match Backend::get_aux(&**client.backend(), LAST_COMPLETED_KEY)? {
-		None => (0, RoundState::genesis((genesis_hash, <NumberFor<Block>>::zero()))),
-		Some(raw) => LastCompleted::decode(&mut &raw[..])
-			.ok_or_else(|| ::client::error::ErrorKind::Backend(
-				format!("Last GRANDPA round state kept in invalid format")
-			))?
-	};
+	register_finality_tracker_inherent_data_provider(client.clone(), &inherent_data_providers)?;
 
 	let voters = authority_set.current_authorities();
 
@@ -740,95 +800,131 @@ pub fn run_grandpa<B, E, Block: BlockT<Hash=H256>, N, RA>(
 		set_id: authority_set.set_id(),
 		authority_set: authority_set.clone(),
 		consensus_changes: consensus_changes.clone(),
+		last_completed: environment::LastCompletedRound::new(set_state.round()),
 	});
 
-	let initial_state = (initial_environment, last_round_number, last_state, authority_set_change.into_future());
+	let initial_state = (initial_environment, set_state, voter_commands_rx.into_future());
 	let voter_work = future::loop_fn(initial_state, move |params| {
-		let (env, last_round_number, last_state, authority_set_change) = params;
+		let (env, set_state, voter_commands_rx) = params;
 		debug!(target: "afg", "{}: Starting new voter with set ID {}", config.name(), env.set_id);
 		telemetry!(CONSENSUS_DEBUG; "afg.starting_new_voter";
 			"name" => ?config.name(), "set_id" => ?env.set_id
 		);
 
-		let chain_info = match client.info() {
-			Ok(i) => i,
-			Err(e) => return future::Either::B(future::err(Error::Client(e))),
+		let mut maybe_voter = match set_state.clone() {
+			VoterSetState::Live(last_round_number, last_round_state) => {
+				let chain_info = match client.info() {
+					Ok(i) => i,
+					Err(e) => return future::Either::B(future::err(Error::Client(e))),
+				};
+
+				let last_finalized = (
+					chain_info.chain.finalized_hash,
+					chain_info.chain.finalized_number,
+				);
+
+				let committer_data = committer_communication(
+					config.local_key.clone(),
+					env.set_id,
+					&env.voters,
+					&client,
+					&network,
+				);
+
+				let voters = (*env.voters).clone();
+
+				Some(voter::Voter::new(
+					env.clone(),
+					voters,
+					committer_data,
+					last_round_number,
+					last_round_state,
+					last_finalized,
+				))
+			}
+			VoterSetState::Paused(_, _) => None,
 		};
 
-		let last_finalized = (
-			chain_info.chain.finalized_hash,
-			chain_info.chain.finalized_number,
-		);
+		// needs to be combined with another future otherwise it can deadlock.
+		let poll_voter = future::poll_fn(move || match maybe_voter {
+			Some(ref mut voter) => voter.poll(),
+			None => Ok(Async::NotReady),
+		});
 
-		let committer_data = committer_communication(
-			config.local_key.clone(),
-			env.set_id,
-			&env.voters,
-			&client,
-			&network,
-		);
-
-		let voters = (*env.voters).clone();
-
-		let voter = voter::Voter::new(
-			env,
-			voters,
-			committer_data,
-			last_round_number,
-			last_state,
-			last_finalized,
-		);
 		let client = client.clone();
 		let config = config.clone();
 		let network = network.clone();
 		let authority_set = authority_set.clone();
 		let consensus_changes = consensus_changes.clone();
 
-		let trigger_authority_set_change = |new: NewAuthoritySet<_, _>, authority_set_change| {
-			let env = Arc::new(Environment {
-				inner: client,
-				config,
-				voters: Arc::new(new.authorities.into_iter().collect()),
-				set_id: new.set_id,
-				network,
-				authority_set,
-				consensus_changes,
-			});
+		let handle_voter_command = move |command: VoterCommand<_, _>, voter_commands_rx| {
+			match command {
+				VoterCommand::ChangeAuthorities(new) => {
+					// start the new authority set using the block where the
+					// set changed (not where the signal happened!) as the base.
+					let genesis_state = RoundState::genesis((new.canon_hash, new.canon_number));
+					let env = Arc::new(Environment {
+						inner: client,
+						config,
+						voters: Arc::new(new.authorities.into_iter().collect()),
+						set_id: new.set_id,
+						network,
+						authority_set,
+						consensus_changes,
+						last_completed: environment::LastCompletedRound::new(
+							(0, genesis_state.clone())
+						),
+					});
 
-			// start the new authority set using the block where the
-			// set changed (not where the signal happened!) as the base.
-			Ok(FutureLoop::Continue((
-				env,
-				0, // always start at round 0 when changing sets.
-				RoundState::genesis((new.canon_hash, new.canon_number)),
-				authority_set_change,
-			)))
+
+					let set_state = VoterSetState::Live(
+						0, // always start at round 0 when changing sets.
+						genesis_state,
+					);
+
+					Ok(FutureLoop::Continue((env, set_state, voter_commands_rx)))
+				}
+				VoterCommand::Pause(reason) => {
+					info!(target: "afg", "Pausing old validator set: {}", reason);
+
+					// not racing because old voter is shut down.
+					let (last_round_number, last_round_state) = env.last_completed.read();
+					let set_state = VoterSetState::Paused(
+						last_round_number,
+						last_round_state,
+					);
+
+					aux_schema::write_voter_set_state(&**client.backend(), &set_state)?;
+
+					Ok(FutureLoop::Continue((env, set_state, voter_commands_rx)))
+				},
+			}
 		};
 
-		future::Either::A(voter.select2(authority_set_change).then(move |res| match res {
+		future::Either::A(poll_voter.select2(voter_commands_rx).then(move |res| match res {
 			Ok(future::Either::A(((), _))) => {
 				// voters don't conclude naturally; this could reasonably be an error.
 				Ok(FutureLoop::Break(()))
 			},
 			Err(future::Either::B(_)) => {
-				// the `authority_set_change` stream should not fail.
+				// the `voter_commands_rx` stream should not fail.
 				Ok(FutureLoop::Break(()))
 			},
 			Ok(future::Either::B(((None, _), _))) => {
-				// the `authority_set_change` stream should never conclude since it's never closed.
+				// the `voter_commands_rx` stream should never conclude since it's never closed.
 				Ok(FutureLoop::Break(()))
 			},
-			Err(future::Either::A((ExitOrError::Error(e), _))) => {
+			Err(future::Either::A((CommandOrError::Error(e), _))) => {
 				// return inner voter error
 				Err(e)
 			}
-			Ok(future::Either::B(((Some(new), authority_set_change), _))) => {
-				// authority set change triggered externally through the channel
-				trigger_authority_set_change(new, authority_set_change.into_future())
+			Ok(future::Either::B(((Some(command), voter_commands_rx), _))) => {
+				// some command issued externally.
+				handle_voter_command(command, voter_commands_rx.into_future())
 			}
-			Err(future::Either::A((ExitOrError::AuthoritiesChanged(new), authority_set_change))) => {
-				// authority set change triggered internally by finalizing a change block
-				trigger_authority_set_change(new, authority_set_change)
+			Err(future::Either::A((CommandOrError::VoterCommand(command), voter_commands_rx))) => {
+				// some command issued internally.
+				handle_voter_command(command, voter_commands_rx)
 			},
 		}))
 	});

substrate/core/finality-grandpa/src/tests.rs

@@ -29,8 +29,7 @@ use client::{
 	runtime_api::{Core, RuntimeVersion, ApiExt},
 };
 use test_client::{self, runtime::BlockNumber};
-use parity_codec::Decode;
-use consensus_common::{BlockOrigin, ForkChoiceStrategy, ImportBlock, ImportResult};
+use consensus_common::{BlockOrigin, ForkChoiceStrategy, ImportedAux, ImportBlock, ImportResult};
 use consensus_common::import_queue::{SharedBlockImport, SharedJustificationImport};
 use std::collections::{HashMap, HashSet};
 use std::result;
@@ -40,6 +39,7 @@ use runtime_primitives::ExecutionContext;
 use substrate_primitives::NativeOrEncoded;
 
 use authorities::AuthoritySet;
+use consensus_changes::ConsensusChanges;
 
 type PeerData =
 	Mutex<
@@ -240,6 +240,7 @@ impl Network<Block> for MessageRouting {
 struct TestApi {
 	genesis_authorities: Vec<(Ed25519AuthorityId, u64)>,
 	scheduled_changes: Arc<Mutex<HashMap<Hash, ScheduledChange<BlockNumber>>>>,
+	forced_changes: Arc<Mutex<HashMap<Hash, (BlockNumber, ScheduledChange<BlockNumber>)>>>,
 }
 
 impl TestApi {
@@ -247,6 +248,7 @@ impl TestApi {
 		TestApi {
 			genesis_authorities,
 			scheduled_changes: Arc::new(Mutex::new(HashMap::new())),
+			forced_changes: Arc::new(Mutex::new(HashMap::new())),
 		}
 	}
 }
@@ -349,6 +351,24 @@ impl GrandpaApi<Block> for RuntimeApi {
 		// extrinsics.
 		Ok(self.inner.scheduled_changes.lock().get(&parent_hash).map(|c| c.clone())).map(NativeOrEncoded::Native)
 	}
+
+	fn grandpa_forced_change_runtime_api_impl(
+		&self,
+		at: &BlockId<Block>,
+		_: ExecutionContext,
+		_: Option<(&DigestFor<Block>)>,
+		_: Vec<u8>,
+	)
+		-> Result<NativeOrEncoded<Option<(NumberFor<Block>, ScheduledChange<NumberFor<Block>>)>>> {
+		let parent_hash = match at {
+			&BlockId::Hash(at) => at,
+			_ => panic!("not requested by block hash!!"),
+		};
+
+		// we take only scheduled changes at given block number where there are no
+		// extrinsics.
+		Ok(self.inner.forced_changes.lock().get(&parent_hash).map(|c| c.clone())).map(NativeOrEncoded::Native)
+	}
 }
 
 const TEST_GOSSIP_DURATION: Duration = Duration::from_millis(500);
@@ -361,7 +381,14 @@ fn make_ids(keys: &[Keyring]) -> Vec<(Ed25519AuthorityId, u64)> {
 		.collect()
 }
 
-fn run_to_completion(blocks: u64, net: Arc<Mutex<GrandpaTestNet>>, peers: &[Keyring]) -> u64 {
+// run the voters to completion. provide a closure to be invoked after
+// the voters are spawned but before blocking on them.
+fn run_to_completion_with<F: FnOnce()>(
+	blocks: u64,
+	net: Arc<Mutex<GrandpaTestNet>>,
+	peers: &[Keyring],
+	before_waiting: F,
+) -> u64 {
 	use parking_lot::RwLock;
 
 	let mut finality_notifications = Vec::new();
@@ -402,6 +429,7 @@ fn run_to_completion(blocks: u64, net: Arc<Mutex<GrandpaTestNet>>, peers: &[Keyr
 			},
 			link,
 			MessageRouting::new(net.clone(), peer_id),
+			InherentDataProviders::new(),
 			futures::empty(),
 		).expect("all in order with client and network");
 
@@ -425,6 +453,8 @@ fn run_to_completion(blocks: u64, net: Arc<Mutex<GrandpaTestNet>>, peers: &[Keyr
 		.map(|_| ())
 		.map_err(|_| ());
 
+	(before_waiting)();
+
 	runtime.block_on(wait_for.select(drive_to_completion).map_err(|_| ())).unwrap();
 
 	let highest_finalized = *highest_finalized.read();
@@ -432,8 +462,13 @@ fn run_to_completion(blocks: u64, net: Arc<Mutex<GrandpaTestNet>>, peers: &[Keyr
 	highest_finalized
 }
 
+fn run_to_completion(blocks: u64, net: Arc<Mutex<GrandpaTestNet>>, peers: &[Keyring]) -> u64 {
+	run_to_completion_with(blocks, net, peers, || {})
+}
+
 #[test]
 fn finalize_3_voters_no_observers() {
+	let _ = env_logger::try_init();
 	let peers = &[Keyring::Alice, Keyring::Bob, Keyring::Charlie];
 	let voters = make_ids(peers);
 
@@ -495,6 +530,7 @@ fn finalize_3_voters_1_observer() {
 			},
 			link,
 			MessageRouting::new(net.clone(), peer_id),
+			InherentDataProviders::new(),
 			futures::empty(),
 		).expect("all in order with client and network");
 
@@ -552,8 +588,9 @@ fn transition_3_voters_twice_1_observer() {
 		assert_eq!(peer.client().info().unwrap().chain.best_number, 1,
 					"Peer #{} failed to sync", i);
 
-		let set_raw = peer.client().backend().get_aux(crate::AUTHORITY_SET_KEY).unwrap().unwrap();
-		let set = AuthoritySet::<Hash, BlockNumber>::decode(&mut &set_raw[..]).unwrap();
+		let set: AuthoritySet<Hash, BlockNumber> = crate::aux_schema::load_authorities(
+			&**peer.client().backend()
+		).unwrap();
 
 		assert_eq!(set.current(), (0, make_ids(peers_a).as_slice()));
 		assert_eq!(set.pending_changes().count(), 0);
@@ -638,8 +675,9 @@ fn transition_3_voters_twice_1_observer() {
 				.take_while(|n| Ok(n.header.number() < &30))
 				.for_each(move |_| Ok(()))
 				.map(move |()| {
-					let set_raw = client.backend().get_aux(crate::AUTHORITY_SET_KEY).unwrap().unwrap();
-					let set = AuthoritySet::<Hash, BlockNumber>::decode(&mut &set_raw[..]).unwrap();
+					let set: AuthoritySet<Hash, BlockNumber> = crate::aux_schema::load_authorities(
+						&**client.backend()
+					).unwrap();
 
 					assert_eq!(set.current(), (2, make_ids(peers_c).as_slice()));
 					assert_eq!(set.pending_changes().count(), 0);
@@ -654,6 +692,7 @@ fn transition_3_voters_twice_1_observer() {
 			},
 			link,
 			MessageRouting::new(net.clone(), peer_id),
+			InherentDataProviders::new(),
 			futures::empty(),
 		).expect("all in order with client and network");
 
@@ -784,7 +823,7 @@ fn sync_justifications_on_change_blocks() {
 
 #[test]
 fn finalizes_multiple_pending_changes_in_order() {
-	env_logger::init();
+	let _ = env_logger::try_init();
 
 	let peers_a = &[Keyring::Alice, Keyring::Bob, Keyring::Charlie];
 	let peers_b = &[Keyring::Dave, Keyring::Eve, Keyring::Ferdie];
@@ -865,6 +904,60 @@ fn doesnt_vote_on_the_tip_of_the_chain() {
 	assert_eq!(highest, 75);
 }
 
+#[test]
+fn force_change_to_new_set() {
+	// two of these guys are offline.
+	let genesis_authorities = &[Keyring::Alice, Keyring::Bob, Keyring::Charlie, Keyring::One, Keyring::Two];
+	let peers_a = &[Keyring::Alice, Keyring::Bob, Keyring::Charlie];
+	let api = TestApi::new(make_ids(genesis_authorities));
+
+	let voters = make_ids(peers_a);
+	let normal_transitions = api.scheduled_changes.clone();
+	let forced_transitions = api.forced_changes.clone();
+	let net = GrandpaTestNet::new(api, 3);
+	let net = Arc::new(Mutex::new(net));
+
+	let runner_net = net.clone();
+	let add_blocks = move || {
+		net.lock().peer(0).push_blocks(1, false);
+
+		{
+			// add a forced transition at block 12.
+			let parent_hash = net.lock().peer(0).client().info().unwrap().chain.best_hash;
+			forced_transitions.lock().insert(parent_hash, (0, ScheduledChange {
+				next_authorities: voters.clone(),
+				delay: 10,
+			}));
+
+			// add a normal transition too to ensure that forced changes take priority.
+			normal_transitions.lock().insert(parent_hash, ScheduledChange {
+				next_authorities: make_ids(genesis_authorities),
+				delay: 5,
+			});
+		}
+
+		net.lock().peer(0).push_blocks(25, false);
+		net.lock().sync();
+
+		for (i, peer) in net.lock().peers().iter().enumerate() {
+			assert_eq!(peer.client().info().unwrap().chain.best_number, 26,
+					"Peer #{} failed to sync", i);
+
+			let set: AuthoritySet<Hash, BlockNumber> = crate::aux_schema::load_authorities(
+				&**peer.client().backend()
+			).unwrap();
+
+			assert_eq!(set.current(), (1, voters.as_slice()));
+			assert_eq!(set.pending_changes().count(), 0);
+		}
+	};
+
+	// it will only finalize if the forced transition happens.
+	// we add_blocks after the voters are spawned because otherwise
+	// the link-halfs have the wrong AuthoritySet
+	run_to_completion_with(25, runner_net, peers_a, add_blocks);
+}
+
 #[test]
 fn allows_reimporting_change_blocks() {
 	let peers_a = &[Keyring::Alice, Keyring::Bob, Keyring::Charlie];
@@ -899,7 +992,7 @@ fn allows_reimporting_change_blocks() {
 
 	assert_eq!(
 		block_import.import_block(block(), None).unwrap(),
-		ImportResult::NeedsJustification
+		ImportResult::Imported(ImportedAux { needs_justification: true, clear_justification_requests: false }),
 	);
 
 	assert_eq!(