Offline fallback for GRANDPA (#1619)

Co-authored-by: André Silva <andre.beat@gmail.com>

* skeleton for finality tracker

* dispatch events when nothing finalized for a long time

* begin integrating finality tracker into grandpa

* add delay field to pending change

* add has_api_with function to sr_version for querying APIs

* partially integrate new force changes into grandpa

* implement forced changes

* get srml-grandpa compiling

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update core/finality-grandpa/src/authorities.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* remove explicit dependence on CoreApi

* increase node runtime version

* integrate grandpa forced changes into node runtime

* add some tests to finality-tracker

* integrate finality tracking into node-runtime

* test forced-change logic

* test forced changes in the authority-set handler

* kill some unneeded bounds in client

* test forced-changes in finality-grandpa and fix logic

* build wasm and finality-tracker is no-std

* restart voter on forced change

* allow returning custom error type from lock_import_and_run

* extract out most DB logic to aux_schema and use atomic client ops

* unify authority set writing

* implement set pausing

* bump runtime version

* note on DB when we pause.

* core: grandpa: integrate forced changes with multiple pending standard changes

* core: grandpa: fix AuthoritySet tests

* runtime: bump impl_version

* core: clear pending justification requests after forced change import

* srml: finality-tracker: use FinalizedInherentData

* core: log requests for clearing justification requests

* core, node: update runtimes

* core: grandpa: fix tests

* core: grandpa: remove todos and add comments

* core: grandpa: use has_api_with from ApiExt

* core: fix tests

* core: grandpa: remove unnecessary mut modifier

* core: replace PostImportActions bitflags with struct

* core: grandpa: restrict genesis on forced authority set change

* core: grandpa: add more docs

* core: grandpa: prevent safety violations in Environment::finalize_block

* core: grandpa: register finality tracker inherent data provider

* core: grandpa: fix tests

* node: update runtime blobs

* core: grandpa: remove outdated todo

* core: aura: fix typo in log message

* core: grandpa: check re-finalization is on canonical chain

* srml: finality-tracker: fix initialization

* node: update runtime wasm

* srml: finality-tracker: don't re-initialize config keys

substrate/srml/finality-tracker/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "srml-finality-tracker"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+hex-literal = "0.1.0"
+serde = { version = "1.0", default-features = false }
+serde_derive = { version = "1.0", optional = true }
+parity-codec = { version = "3.0", default-features = false }
+parity-codec-derive = { version = "3.0", default-features = false }
+substrate-inherents = { path = "../../core/inherents", default-features = false }
+sr-std = { path = "../../core/sr-std", default-features = false }
+sr-primitives = { path = "../../core/sr-primitives", default-features = false }
+srml-support = { path = "../support", default-features = false }
+srml-system = { path = "../system", default-features = false }
+srml-session = { path = "../session", default-features = false }
+
+[dev-dependencies]
+substrate-primitives = { path = "../../core/primitives", default-features = false }
+sr-io = { path = "../../core/sr-io", default-features = false }
+lazy_static = "1.0"
+parking_lot = "0.7"
+
+[features]
+default = ["std"]
+std = [
+	"serde/std",
+	"serde_derive",
+	"parity-codec/std",
+	"sr-std/std",
+	"srml-support/std",
+	"sr-primitives/std",
+	"srml-system/std",
+	"srml-session/std",
+	"substrate-inherents/std",
+]

substrate/srml/finality-tracker/src/lib.rs

@@ -0,0 +1,385 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! SRML module that tracks the last finalized block, as perceived by block authors.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[macro_use]
+extern crate srml_support;
+
+use substrate_inherents::{
+	RuntimeString, InherentIdentifier, ProvideInherent,
+	InherentData, MakeFatalError,
+};
+use srml_support::StorageValue;
+use sr_primitives::traits::{As, One, Zero};
+use sr_std::{prelude::*, result, cmp, vec};
+use parity_codec::Decode;
+use srml_system::{ensure_inherent, Trait as SystemTrait};
+
+#[cfg(feature = "std")]
+use parity_codec::Encode;
+
+const DEFAULT_WINDOW_SIZE: u64 = 101;
+const DEFAULT_DELAY: u64 = 1000;
+
+/// The identifier for the `finalnum` inherent.
+pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"finalnum";
+
+/// Auxiliary trait to extract finalized inherent data.
+pub trait FinalizedInherentData<N: Decode> {
+	/// Get finalized inherent data.
+	fn finalized_number(&self) -> Result<N, RuntimeString>;
+}
+
+impl<N: Decode> FinalizedInherentData<N> for InherentData {
+	fn finalized_number(&self) -> Result<N, RuntimeString> {
+		self.get_data(&INHERENT_IDENTIFIER)
+			.and_then(|r| r.ok_or_else(|| "Finalized number inherent data not found".into()))
+	}
+}
+
+/// Provider for inherent data.
+#[cfg(feature = "std")]
+pub struct InherentDataProvider<F, N> {
+	inner: F,
+	_marker: std::marker::PhantomData<N>,
+}
+
+#[cfg(feature = "std")]
+impl<F, N> InherentDataProvider<F, N> {
+	pub fn new(final_oracle: F) -> Self {
+		InherentDataProvider { inner: final_oracle, _marker: Default::default() }
+	}
+}
+
+#[cfg(feature = "std")]
+impl<F, N: Encode> substrate_inherents::ProvideInherentData for InherentDataProvider<F, N>
+	where F: Fn() -> Result<N, RuntimeString>
+{
+	fn inherent_identifier(&self) -> &'static InherentIdentifier {
+		&INHERENT_IDENTIFIER
+	}
+
+	fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), RuntimeString> {
+		(self.inner)()
+			.and_then(|n| inherent_data.put_data(INHERENT_IDENTIFIER, &n))
+	}
+
+	fn error_to_string(&self, _error: &[u8]) -> Option<String> {
+		Some(format!("no further information"))
+	}
+}
+
+
+pub trait Trait: SystemTrait {
+	/// Something which can be notified when the timestamp is set. Set this to `()` if not needed.
+	type OnFinalizationStalled: OnFinalizationStalled<Self::BlockNumber>;
+}
+
+decl_storage! {
+	trait Store for Module<T: Trait> as Timestamp {
+		/// Recent hints.
+		RecentHints get(recent_hints) build(|_| vec![T::BlockNumber::zero()]): Vec<T::BlockNumber>;
+		/// Ordered recent hints.
+		OrderedHints get(ordered_hints) build(|_| vec![T::BlockNumber::zero()]): Vec<T::BlockNumber>;
+		/// The median.
+		Median get(median) build(|_| T::BlockNumber::zero()): T::BlockNumber;
+		/// The number of recent samples to keep from this chain. Default is n-100
+		pub WindowSize get(window_size) config(window_size): T::BlockNumber = T::BlockNumber::sa(DEFAULT_WINDOW_SIZE);
+		/// The delay after which point things become suspicious.
+		pub ReportLatency get(report_latency) config(report_latency): T::BlockNumber = T::BlockNumber::sa(DEFAULT_DELAY);
+
+		/// Final hint to apply in the block. `None` means "same as parent".
+		Update: Option<T::BlockNumber>;
+
+		// when initialized through config this is set in the beginning.
+		Initialized get(initialized) build(|_| false): bool;
+	}
+}
+
+decl_module! {
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+		/// Hint that the author of this block thinks the best finalized
+		/// block is the given number.
+		fn final_hint(origin, #[compact] hint: T::BlockNumber) {
+			ensure_inherent(origin)?;
+			assert!(!<Self as Store>::Update::exists(), "Final hint must be updated only once in the block");
+			assert!(
+				srml_system::Module::<T>::block_number() >= hint,
+				"Finalized height above block number",
+			);
+			<Self as Store>::Update::put(hint);
+		}
+
+		fn on_finalise() {
+			Self::update_hint(<Self as Store>::Update::take())
+		}
+	}
+}
+
+impl<T: Trait> Module<T> {
+	fn update_hint(hint: Option<T::BlockNumber>) {
+		if !Self::initialized() {
+			<Self as Store>::RecentHints::put(vec![T::BlockNumber::zero()]);
+			<Self as Store>::OrderedHints::put(vec![T::BlockNumber::zero()]);
+			<Self as Store>::Median::put(T::BlockNumber::zero());
+
+			<Self as Store>::Initialized::put(true);
+		}
+
+		let mut recent = Self::recent_hints();
+		let mut ordered = Self::ordered_hints();
+		let window_size = cmp::max(T::BlockNumber::one(), Self::window_size());
+
+		let hint = hint.unwrap_or_else(|| recent.last()
+			.expect("always at least one recent sample; qed").clone()
+		);
+
+		// prune off the front of the list -- typically 1 except for when
+		// the sample size has just been shrunk.
+		{
+			// take into account the item we haven't pushed yet.
+			let to_prune = (recent.len() + 1).saturating_sub(window_size.as_() as usize);
+
+			for drained in recent.drain(..to_prune) {
+				let idx = ordered.binary_search(&drained)
+					.expect("recent and ordered contain the same items; qed");
+
+				ordered.remove(idx);
+			}
+		}
+
+		// find the position in the ordered list where the new item goes.
+		let ordered_idx = ordered.binary_search(&hint)
+			.unwrap_or_else(|idx| idx);
+
+		ordered.insert(ordered_idx, hint);
+		recent.push(hint);
+
+		let two = T::BlockNumber::one() + T::BlockNumber::one();
+
+		let median = {
+			let len = ordered.len();
+			assert!(len > 0, "pruning dictated by window_size which is always saturated at 1; qed");
+
+			if len % 2 == 0 {
+				let a = ordered[len / 2];
+				let b = ordered[(len / 2) - 1];
+
+				// compute average.
+				(a + b) / two
+			} else {
+				ordered[len / 2]
+			}
+		};
+
+		let our_window_size = recent.len();
+
+		<Self as Store>::RecentHints::put(recent);
+		<Self as Store>::OrderedHints::put(ordered);
+		<Self as Store>::Median::put(median);
+
+		if T::BlockNumber::sa(our_window_size as u64) == window_size {
+			let now = srml_system::Module::<T>::block_number();
+			let latency = Self::report_latency();
+
+			// the delay is the latency plus half the window size.
+			let delay = latency + (window_size / two);
+			// median may be at most n - delay
+			if median + delay <= now {
+				T::OnFinalizationStalled::on_stalled(window_size - T::BlockNumber::one());
+			}
+		}
+	}
+}
+
+/// Called when finalization stalled at a given number.
+pub trait OnFinalizationStalled<N> {
+	/// The parameter here is how many more blocks to wait before applying
+	/// changes triggered by finality stalling.
+	fn on_stalled(further_wait: N);
+}
+
+macro_rules! impl_on_stalled {
+	() => (
+		impl<N> OnFinalizationStalled<N> for () {
+			fn on_stalled(_: N) {}
+		}
+	);
+
+	( $($t:ident)* ) => {
+		impl<NUM: Clone, $($t: OnFinalizationStalled<NUM>),*> OnFinalizationStalled<NUM> for ($($t,)*) {
+			fn on_stalled(further_wait: NUM) {
+				$($t::on_stalled(further_wait.clone());)*
+			}
+		}
+	}
+}
+
+for_each_tuple!(impl_on_stalled);
+
+impl<T: Trait> ProvideInherent for Module<T> {
+	type Call = Call<T>;
+	type Error = MakeFatalError<()>;
+	const INHERENT_IDENTIFIER: InherentIdentifier = INHERENT_IDENTIFIER;
+
+	fn create_inherent(data: &InherentData) -> Option<Self::Call> {
+		let final_num =
+			data.finalized_number().expect("Gets and decodes final number inherent data");
+
+		// make hint only when not same as last to avoid bloat.
+		Self::recent_hints().last().and_then(|last| if last == &final_num {
+			None
+		} else {
+			Some(Call::final_hint(final_num))
+		})
+	}
+
+	fn check_inherent(_call: &Self::Call, _data: &InherentData) -> result::Result<(), Self::Error> {
+		Ok(())
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use sr_io::{with_externalities, TestExternalities};
+	use substrate_primitives::H256;
+	use sr_primitives::BuildStorage;
+	use sr_primitives::traits::{BlakeTwo256, IdentityLookup, OnFinalise, Header as HeaderT};
+	use sr_primitives::testing::{Digest, DigestItem, Header};
+	use srml_support::impl_outer_origin;
+	use srml_system as system;
+	use lazy_static::lazy_static;
+	use parking_lot::Mutex;
+
+	#[derive(Clone, PartialEq, Debug)]
+	pub struct StallEvent {
+		at: u64,
+		further_wait: u64,
+	}
+
+	macro_rules! make_test_context {
+		() => {
+			#[derive(Clone, Eq, PartialEq)]
+			pub struct Test;
+
+			impl_outer_origin! {
+				pub enum Origin for Test {}
+			}
+
+			impl system::Trait for Test {
+				type Origin = Origin;
+				type Index = u64;
+				type BlockNumber = u64;
+				type Hash = H256;
+				type Hashing = BlakeTwo256;
+				type Digest = Digest;
+				type AccountId = u64;
+				type Lookup = IdentityLookup<u64>;
+				type Header = Header;
+				type Event = ();
+				type Log = DigestItem;
+			}
+
+			type System = system::Module<Test>;
+
+			lazy_static! {
+				static ref NOTIFICATIONS: Mutex<Vec<StallEvent>> = Mutex::new(Vec::new());
+			}
+
+			pub struct StallTracker;
+			impl OnFinalizationStalled<u64> for StallTracker {
+				fn on_stalled(further_wait: u64) {
+					let now = System::block_number();
+					NOTIFICATIONS.lock().push(StallEvent { at: now, further_wait });
+				}
+			}
+
+			impl Trait for Test {
+				type OnFinalizationStalled = StallTracker;
+			}
+
+			type FinalityTracker = Module<Test>;
+		}
+	}
+
+	#[test]
+	fn median_works() {
+		make_test_context!();
+		let t = system::GenesisConfig::<Test>::default().build_storage().unwrap().0;
+
+		with_externalities(&mut TestExternalities::new(t), || {
+			FinalityTracker::update_hint(Some(500));
+			assert_eq!(FinalityTracker::median(), 250);
+			assert!(NOTIFICATIONS.lock().is_empty());
+		});
+	}
+
+	#[test]
+	fn notifies_when_stalled() {
+		make_test_context!();
+		let mut t = system::GenesisConfig::<Test>::default().build_storage().unwrap().0;
+		t.extend(GenesisConfig::<Test> {
+			window_size: 11,
+			report_latency: 100
+		}.build_storage().unwrap().0);
+
+		with_externalities(&mut TestExternalities::new(t), || {
+			let mut parent_hash = System::parent_hash();
+			for i in 2..106 {
+				System::initialise(&i, &parent_hash, &Default::default());
+				FinalityTracker::on_finalise(i);
+				let hdr = System::finalise();
+				parent_hash = hdr.hash();
+			}
+
+			assert_eq!(
+				NOTIFICATIONS.lock().to_vec(),
+				vec![StallEvent { at: 105, further_wait: 10 }]
+			)
+		});
+	}
+
+	#[test]
+	fn recent_notifications_prevent_stalling() {
+		make_test_context!();
+		let mut t = system::GenesisConfig::<Test>::default().build_storage().unwrap().0;
+		t.extend(GenesisConfig::<Test> {
+			window_size: 11,
+			report_latency: 100
+		}.build_storage().unwrap().0);
+
+		with_externalities(&mut TestExternalities::new(t), || {
+			let mut parent_hash = System::parent_hash();
+			for i in 2..106 {
+				System::initialise(&i, &parent_hash, &Default::default());
+				assert_ok!(FinalityTracker::dispatch(
+					Call::final_hint(i-1),
+					Origin::INHERENT,
+				));
+				FinalityTracker::on_finalise(i);
+				let hdr = System::finalise();
+				parent_hash = hdr.hash();
+			}
+
+			assert!(NOTIFICATIONS.lock().is_empty());
+		});
+	}
+}

substrate/srml/grandpa/Cargo.toml

@@ -16,6 +16,7 @@ primitives = { package = "sr-primitives", path = "../../core/sr-primitives", def
 srml-support = { path = "../support", default-features = false }
 system = { package = "srml-system", path = "../system", default-features = false }
 session = { package = "srml-session", path = "../session", default-features = false }
+finality-tracker = { package = "srml-finality-tracker", path = "../finality-tracker", default-features = false }
 
 [dev-dependencies]
 runtime_io = { package = "sr-io", path = "../../core/sr-io" }
@@ -33,4 +34,5 @@ std = [
 	"primitives/std",
 	"system/std",
 	"session/std",
+	"finality-tracker/std",
 ]

substrate/srml/grandpa/src/lib.rs

@@ -64,6 +64,8 @@ pub type Log<T> = RawLog<
 pub trait GrandpaChangeSignal<N> {
 	/// Try to cast the log entry as a contained signal.
 	fn as_signal(&self) -> Option<ScheduledChange<N>>;
+	/// Try to cast the log entry as a contained forced signal.
+	fn as_forced_signal(&self) -> Option<(N, ScheduledChange<N>)>;
 }
 
 /// A logs in this module.
@@ -71,15 +73,28 @@ pub trait GrandpaChangeSignal<N> {
 #[derive(Encode, Decode, PartialEq, Eq, Clone)]
 pub enum RawLog<N, SessionKey> {
 	/// Authorities set change has been signalled. Contains the new set of authorities
-	/// and the delay in blocks before applying.
+	/// and the delay in blocks _to finalize_ before applying.
 	AuthoritiesChangeSignal(N, Vec<(SessionKey, u64)>),
+	/// A forced authorities set change. Contains in this order: the median last
+	/// finalized block when the change was signaled, the delay in blocks _to import_
+	/// before applying and the new set of authorities.
+	ForcedAuthoritiesChangeSignal(N, N, Vec<(SessionKey, u64)>),
 }
 
 impl<N: Clone, SessionKey> RawLog<N, SessionKey> {
 	/// Try to cast the log entry as a contained signal.
 	pub fn as_signal(&self) -> Option<(N, &[(SessionKey, u64)])> {
 		match *self {
-			RawLog::AuthoritiesChangeSignal(ref n, ref signal) => Some((n.clone(), signal)),
+			RawLog::AuthoritiesChangeSignal(ref delay, ref signal) => Some((delay.clone(), signal)),
+			RawLog::ForcedAuthoritiesChangeSignal(_, _, _) => None,
+		}
+	}
+
+	/// Try to cast the log entry as a contained forced signal.
+	pub fn as_forced_signal(&self) -> Option<(N, N, &[(SessionKey, u64)])> {
+		match *self {
+			RawLog::ForcedAuthoritiesChangeSignal(ref median, ref delay, ref signal) => Some((median.clone(), delay.clone(), signal)),
+			RawLog::AuthoritiesChangeSignal(_, _) => None,
 		}
 	}
 }
@@ -96,6 +111,16 @@ impl<N, SessionKey> GrandpaChangeSignal<N> for RawLog<N, SessionKey>
 				.collect(),
 		})
 	}
+
+	fn as_forced_signal(&self) -> Option<(N, ScheduledChange<N>)> {
+		RawLog::as_forced_signal(self).map(|(median, delay, next_authorities)| (median, ScheduledChange {
+			delay,
+			next_authorities: next_authorities.iter()
+				.cloned()
+				.map(|(k, w)| (k.into(), w))
+				.collect(),
+		}))
+	}
 }
 
 pub trait Trait: system::Trait {
@@ -109,8 +134,21 @@ pub trait Trait: system::Trait {
 	type Event: From<Event<Self>> + Into<<Self as system::Trait>::Event>;
 }
 
-/// A stored pending change.
+/// A stored pending change, old format.
+// TODO: remove shim
+// https://github.com/paritytech/substrate/issues/1614
 #[derive(Encode, Decode)]
+pub struct OldStoredPendingChange<N, SessionKey> {
+	/// The block number this was scheduled at.
+	pub scheduled_at: N,
+	/// The delay in blocks until it will be applied.
+	pub delay: N,
+	/// The next authority set.
+	pub next_authorities: Vec<(SessionKey, u64)>,
+}
+
+/// A stored pending change.
+#[derive(Encode)]
 pub struct StoredPendingChange<N, SessionKey> {
 	/// The block number this was scheduled at.
 	pub scheduled_at: N,
@@ -118,6 +156,23 @@ pub struct StoredPendingChange<N, SessionKey> {
 	pub delay: N,
 	/// The next authority set.
 	pub next_authorities: Vec<(SessionKey, u64)>,
+	/// If defined it means the change was forced and the given block number
+	/// indicates the median last finalized block when the change was signaled.
+	pub forced: Option<N>,
+}
+
+impl<N: Decode, SessionKey: Decode> Decode for StoredPendingChange<N, SessionKey> {
+	fn decode<I: codec::Input>(value: &mut I) -> Option<Self> {
+		let old = OldStoredPendingChange::decode(value)?;
+		let forced = <Option<N>>::decode(value).unwrap_or(None);
+
+		Some(StoredPendingChange {
+			scheduled_at: old.scheduled_at,
+			delay: old.delay,
+			next_authorities: old.next_authorities,
+			forced,
+		})
+	}
 }
 
 /// GRANDPA events.
@@ -132,6 +187,8 @@ decl_storage! {
 	trait Store for Module<T: Trait> as GrandpaFinality {
 		// Pending change: (signalled at, scheduled change).
 		PendingChange get(pending_change): Option<StoredPendingChange<T::BlockNumber, T::SessionKey>>;
+		// next block number where we can force a change.
+		NextForced get(next_forced): Option<T::BlockNumber>;
 	}
 	add_extra_genesis {
 		config(authorities): Vec<(T::SessionKey, u64)>;
@@ -167,10 +224,18 @@ decl_module! {
 		fn on_finalise(block_number: T::BlockNumber) {
 			if let Some(pending_change) = <PendingChange<T>>::get() {
 				if block_number == pending_change.scheduled_at {
-					Self::deposit_log(RawLog::AuthoritiesChangeSignal(
-						pending_change.delay,
-						pending_change.next_authorities.clone(),
-					));
+					if let Some(median) = pending_change.forced {
+						Self::deposit_log(RawLog::ForcedAuthoritiesChangeSignal(
+							median,
+							pending_change.delay,
+							pending_change.next_authorities.clone(),
+						));
+					} else {
+						Self::deposit_log(RawLog::AuthoritiesChangeSignal(
+							pending_change.delay,
+							pending_change.next_authorities.clone(),
+						));
+					}
 				}
 
 				if block_number == pending_change.scheduled_at + pending_change.delay {
@@ -197,18 +262,39 @@ impl<T: Trait> Module<T> {
 	/// `in_blocks` after the current block. This value may be 0, in which
 	/// case the change is applied at the end of the current block.
 	///
+	/// If the `forced` parameter is defined, this indicates that the current
+	/// set has been synchronously determined to be offline and that after
+	/// `in_blocks` the given change should be applied. The given block number
+	/// indicates the median last finalized block number and it should be used
+	/// as the canon block when starting the new grandpa voter.
+	///
 	/// No change should be signalled while any change is pending. Returns
 	/// an error if a change is already pending.
 	pub fn schedule_change(
 		next_authorities: Vec<(T::SessionKey, u64)>,
 		in_blocks: T::BlockNumber,
+		forced: Option<T::BlockNumber>,
 	) -> Result {
+		use primitives::traits::As;
+
 		if Self::pending_change().is_none() {
 			let scheduled_at = system::ChainContext::<T>::default().current_height();
+
+			if let Some(_) = forced {
+				if Self::next_forced().map_or(false, |next| next > scheduled_at) {
+					return Err("Cannot signal forced change so soon after last.");
+				}
+
+				// only allow the next forced change when twice the window has passed since
+				// this one.
+				<NextForced<T>>::put(scheduled_at + in_blocks * T::BlockNumber::sa(2));
+			}
+
 			<PendingChange<T>>::put(StoredPendingChange {
 				delay: in_blocks,
 				scheduled_at,
 				next_authorities,
+				forced,
 			});
 
 			Ok(())
@@ -224,12 +310,19 @@ impl<T: Trait> Module<T> {
 }
 
 impl<T: Trait> Module<T> where Ed25519AuthorityId: core::convert::From<<T as Trait>::SessionKey> {
-	/// See if the digest contains any scheduled change.
+	/// See if the digest contains any standard scheduled change.
 	pub fn scrape_digest_change(log: &Log<T>)
 		-> Option<ScheduledChange<T::BlockNumber>>
 	{
 		<Log<T> as GrandpaChangeSignal<T::BlockNumber>>::as_signal(log)
 	}
+
+	/// See if the digest contains any forced scheduled change.
+	pub fn scrape_digest_forced_change(log: &Log<T>)
+		-> Option<(T::BlockNumber, ScheduledChange<T::BlockNumber>)>
+	{
+		<Log<T> as GrandpaChangeSignal<T::BlockNumber>>::as_forced_signal(log)
+	}
 }
 
 /// Helper for authorities being synchronized with the general session authorities.
@@ -266,7 +359,34 @@ impl<X, T> session::OnSessionChange<X> for SyncedAuthorities<T> where
 		// instant changes
 		let last_authorities = <Module<T>>::grandpa_authorities();
 		if next_authorities != last_authorities {
-			let _ = <Module<T>>::schedule_change(next_authorities, Zero::zero());
+			let _ = <Module<T>>::schedule_change(next_authorities, Zero::zero(), None);
 		}
 	}
 }
+
+impl<T> finality_tracker::OnFinalizationStalled<T::BlockNumber> for SyncedAuthorities<T> where
+	T: Trait,
+	T: session::Trait,
+	T: finality_tracker::Trait,
+	<T as session::Trait>::ConvertAccountIdToSessionKey: Convert<
+		<T as system::Trait>::AccountId,
+		<T as Trait>::SessionKey,
+	>,
+{
+	fn on_stalled(further_wait: T::BlockNumber) {
+		// when we record old authority sets, we can use `finality_tracker::median`
+		// to figure out _who_ failed. until then, we can't meaningfully guard
+		// against `next == last` the way that normal session changes do.
+
+		let next_authorities = <session::Module<T>>::validators()
+			.into_iter()
+			.map(T::ConvertAccountIdToSessionKey::convert)
+			.map(|key| (key, 1)) // evenly-weighted.
+			.collect::<Vec<(<T as Trait>::SessionKey, u64)>>();
+
+		let median = <finality_tracker::Module<T>>::median();
+
+		// schedule a change for `further_wait` blocks.
+		let _ = <Module<T>>::schedule_change(next_authorities, further_wait, Some(median));
+	}
+}

substrate/srml/grandpa/src/tests.rs

@@ -24,12 +24,14 @@ use runtime_io::with_externalities;
 use crate::mock::{Grandpa, System, new_test_ext};
 use system::{EventRecord, Phase};
 use crate::{RawLog, RawEvent};
+use codec::{Decode, Encode};
+use super::*;
 
 #[test]
 fn authorities_change_logged() {
 	with_externalities(&mut new_test_ext(vec![(1, 1), (2, 1), (3, 1)]), || {
 		System::initialise(&1, &Default::default(), &Default::default());
-		Grandpa::schedule_change(vec![(4, 1), (5, 1), (6, 1)], 0).unwrap();
+		Grandpa::schedule_change(vec![(4, 1), (5, 1), (6, 1)], 0, None).unwrap();
 
 		System::note_finished_extrinsics();
 		Grandpa::on_finalise(1);
@@ -54,7 +56,7 @@ fn authorities_change_logged() {
 fn authorities_change_logged_after_delay() {
 	with_externalities(&mut new_test_ext(vec![(1, 1), (2, 1), (3, 1)]), || {
 		System::initialise(&1, &Default::default(), &Default::default());
-		Grandpa::schedule_change(vec![(4, 1), (5, 1), (6, 1)], 1).unwrap();
+		Grandpa::schedule_change(vec![(4, 1), (5, 1), (6, 1)], 1, None).unwrap();
 		Grandpa::on_finalise(1);
 		let header = System::finalise();
 		assert_eq!(header.digest, testing::Digest {
@@ -84,25 +86,112 @@ fn authorities_change_logged_after_delay() {
 fn cannot_schedule_change_when_one_pending() {
 	with_externalities(&mut new_test_ext(vec![(1, 1), (2, 1), (3, 1)]), || {
 		System::initialise(&1, &Default::default(), &Default::default());
-		Grandpa::schedule_change(vec![(4, 1), (5, 1), (6, 1)], 1).unwrap();
+		Grandpa::schedule_change(vec![(4, 1), (5, 1), (6, 1)], 1, None).unwrap();
 		assert!(Grandpa::pending_change().is_some());
-		assert!(Grandpa::schedule_change(vec![(5, 1)], 1).is_err());
+		assert!(Grandpa::schedule_change(vec![(5, 1)], 1, None).is_err());
 
 		Grandpa::on_finalise(1);
 		let header = System::finalise();
 
 		System::initialise(&2, &header.hash(), &Default::default());
 		assert!(Grandpa::pending_change().is_some());
-		assert!(Grandpa::schedule_change(vec![(5, 1)], 1).is_err());
+		assert!(Grandpa::schedule_change(vec![(5, 1)], 1, None).is_err());
 
 		Grandpa::on_finalise(2);
 		let header = System::finalise();
 
 		System::initialise(&3, &header.hash(), &Default::default());
 		assert!(Grandpa::pending_change().is_none());
-		assert!(Grandpa::schedule_change(vec![(5, 1)], 1).is_ok());
+		assert!(Grandpa::schedule_change(vec![(5, 1)], 1, None).is_ok());
 
 		Grandpa::on_finalise(3);
 		let _header = System::finalise();
 	});
 }
+
+#[test]
+fn new_decodes_from_old() {
+	let old = OldStoredPendingChange {
+		scheduled_at: 5u32,
+		delay: 100u32,
+		next_authorities: vec![(1u64, 5), (2u64, 10), (3u64, 2)],
+	};
+
+	let encoded = old.encode();
+	let new = StoredPendingChange::<u32, u64>::decode(&mut &encoded[..]).unwrap();
+	assert!(new.forced.is_none());
+	assert_eq!(new.scheduled_at, old.scheduled_at);
+	assert_eq!(new.delay, old.delay);
+	assert_eq!(new.next_authorities, old.next_authorities);
+}
+
+#[test]
+fn dispatch_forced_change() {
+	with_externalities(&mut new_test_ext(vec![(1, 1), (2, 1), (3, 1)]), || {
+		System::initialise(&1, &Default::default(), &Default::default());
+		Grandpa::schedule_change(
+			vec![(4, 1), (5, 1), (6, 1)],
+			5,
+			Some(0),
+		).unwrap();
+
+		assert!(Grandpa::pending_change().is_some());
+		assert!(Grandpa::schedule_change(vec![(5, 1)], 1, Some(0)).is_err());
+
+		Grandpa::on_finalise(1);
+		let mut header = System::finalise();
+
+		for i in 2..7 {
+			System::initialise(&i, &header.hash(), &Default::default());
+			assert!(Grandpa::pending_change().unwrap().forced.is_some());
+			assert_eq!(Grandpa::next_forced(), Some(11));
+			assert!(Grandpa::schedule_change(vec![(5, 1)], 1, None).is_err());
+			assert!(Grandpa::schedule_change(vec![(5, 1)], 1, Some(0)).is_err());
+
+			Grandpa::on_finalise(i);
+			header = System::finalise();
+		}
+
+		// change has been applied at the end of block 6.
+		// add a normal change.
+		{
+			System::initialise(&7, &header.hash(), &Default::default());
+			assert!(Grandpa::pending_change().is_none());
+			assert_eq!(Grandpa::grandpa_authorities(), vec![(4, 1), (5, 1), (6, 1)]);
+			assert!(Grandpa::schedule_change(vec![(5, 1)], 1, None).is_ok());
+			Grandpa::on_finalise(7);
+			header = System::finalise();
+		}
+
+		// run the normal change.
+		{
+			System::initialise(&8, &header.hash(), &Default::default());
+			assert!(Grandpa::pending_change().is_some());
+			assert_eq!(Grandpa::grandpa_authorities(), vec![(4, 1), (5, 1), (6, 1)]);
+			assert!(Grandpa::schedule_change(vec![(5, 1)], 1, None).is_err());
+			Grandpa::on_finalise(8);
+			header = System::finalise();
+		}
+
+		// normal change applied. but we can't apply a new forced change for some
+		// time.
+		for i in 9..11 {
+			System::initialise(&i, &header.hash(), &Default::default());
+			assert!(Grandpa::pending_change().is_none());
+			assert_eq!(Grandpa::grandpa_authorities(), vec![(5, 1)]);
+			assert_eq!(Grandpa::next_forced(), Some(11));
+			assert!(Grandpa::schedule_change(vec![(5, 1), (6, 1)], 5, Some(0)).is_err());
+			Grandpa::on_finalise(i);
+			header = System::finalise();
+		}
+
+		{
+			System::initialise(&11, &header.hash(), &Default::default());
+			assert!(Grandpa::pending_change().is_none());
+			assert!(Grandpa::schedule_change(vec![(5, 1), (6, 1), (7, 1)], 5, Some(0)).is_ok());
+			assert_eq!(Grandpa::next_forced(), Some(21));
+			Grandpa::on_finalise(11);
+			header = System::finalise();
+		}
+	});
+}