Fixing BABE epochs to change between blocks (#3583)

* always fetch epoch from runtime

* node integration tests don't test light nodes

* give stand-in full node a FULL role

* rejig babe APIs

* introduce next-epoch-descriptor type

* overhaul srml-BABE epoch logic

* ensure VRF outputs end up in the right epoch-randomness

* rewrite `do_initialize` to remove unnecessary loop

* begin accounting for next epoch in epoch function

* slots passes header to epoch_data

* pass slot_number to SlotWorker::epoch_data

* begin extracting epoch-change logic into its own module

* aux methods for block weight

* aux methods for genesis configuration

* comment-out most, refactor header-check pipeline

* mostly flesh out verifier again

* reinstantiate babe BlockImport implementation

* reinstate import-queue instantiation

* reintroduce slot-worker implementation

* reinstate pretty much all the rest

* move fork-choice logic to BlockImport

* fix some, but not all errors

* patch test-runtime

* make is_descendent of slightly more generic

* get skeleton compiling when passing is_descendent_of

* make descendent-of-builder more succinct

* restore ordering of authority_index / slot_number

* start fiddling with tests

* fix warnings

* improve initialization architecture and handle genesis

* tests use correct block-import

* fix BABE tests

* fix some compiler errors

* fix node-cli compilation

* all crates compile

* bump runtime versions and fix some warnings

* tweak fork-tree search implementation

* do backtracking search in fork-tree

* node-cli integration tests now work

* fix broken assumption in test_connectivity

* babe tests fail for the right reasons.

* test genesis epoch logic for epoch_changes

* test that epochs can change between blocks

* First BABE SRML test

* Testing infrastructure for BABE

Also includes a trivial additional test.

* Apply suggestions from code review

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* A little more test progress

* More work on BABE testing

* Try to get the tests working

* Implement `UintAuthorityId`-based test mocks

* Fix compilation errors

* Adjust to upstream changes

* Block numbers are ignored in BABE epoch calculation

* authority_index() should ignore invalid authorities

* Fix compile error

* Add tests that session transitions happen

* Check if BABE produces logs

It currently does not.

* Fix test suite

This was really nasty, due to a type confusion that showed up as an
off-by-1 buffer error.

* Add additional tests

Most of these were derived from the current output, so they are only
useful to guard against regressions.

* Make the tests more readable

Also bump impl_version.

* Fix excessive line width

* Remove unused imports

* Update srml/babe/src/lib.rs

Co-Authored-By: André Silva <andre.beat@gmail.com>

* try to fix imports

* Fix build errors in test suite

* tests did not pass

* Try to get at least one digest to be output

Currently, the code emits either no digests (if I don’t call
`Session::rotate_session()` or two digests (if I do), which is wrong.

* More tests

They still don’t work, but this should help debugging.

* fix silly error

* Don’t even try to compile a broken test

* remove broken check_epoch test and add one for genesis epoch

* Check that the length of the pre-digests is correct

* Bump `impl_version`

* use epoch_for_descendent_of even for genesis

* account for competing block 1s

* finish srml-babe docs

Co-Authored-By: André Silva <andre.beat@gmail.com>

* address grumbles

substrate/core/service/test/src/lib.rs

@@ -20,7 +20,6 @@ use std::iter;
 use std::sync::{Arc, Mutex, MutexGuard};
 use std::net::Ipv4Addr;
 use std::time::Duration;
-use std::collections::HashMap;
 use log::info;
 use futures::{Future, Stream, Poll};
 use tempdir::TempDir;
@@ -36,7 +35,6 @@ use service::{
 use network::{multiaddr, Multiaddr};
 use network::config::{NetworkConfiguration, TransportConfig, NodeKeyConfig, Secret, NonReservedPeerMode};
 use sr_primitives::{generic::BlockId, traits::Block as BlockT};
-use consensus::{BlockImportParams, BlockImport};
 
 /// Maximum duration of single wait call.
 const MAX_WAIT_TIME: Duration = Duration::from_secs(60 * 3);
@@ -276,7 +274,13 @@ impl<G, F, L, U> TestNet<G, F, L, U> where
 	}
 }
 
-pub fn connectivity<G, Fb, F, Lb, L>(spec: ChainSpec<G>, full_builder: Fb, light_builder: Lb) where
+pub fn connectivity<G, Fb, F, Lb, L>(
+	spec: ChainSpec<G>,
+	full_builder: Fb,
+	light_builder: Lb,
+	light_node_interconnectivity: bool, // should normally be false, unless the light nodes
+	// aren't actually light.
+) where
 	Fb: Fn(Configuration<(), G>) -> Result<F, Error>,
 	F: AbstractService,
 	Lb: Fn(Configuration<(), G>) -> Result<L, Error>,
@@ -284,6 +288,14 @@ pub fn connectivity<G, Fb, F, Lb, L>(spec: ChainSpec<G>, full_builder: Fb, light
 {
 	const NUM_FULL_NODES: usize = 5;
 	const NUM_LIGHT_NODES: usize = 5;
+
+	let expected_full_connections = NUM_FULL_NODES - 1 + NUM_LIGHT_NODES;
+	let expected_light_connections = if light_node_interconnectivity {
+		expected_full_connections
+	} else {
+		NUM_FULL_NODES
+	};
+
 	{
 		let temp = TempDir::new("substrate-connectivity-test").expect("Error creating test dir");
 		let runtime = {
@@ -307,11 +319,14 @@ pub fn connectivity<G, Fb, F, Lb, L>(spec: ChainSpec<G>, full_builder: Fb, light
 				service.get().network().add_reserved_peer(first_address.to_string())
 					.expect("Error adding reserved peer");
 			}
+
 			network.run_until_all_full(
-				|_index, service| service.get().network().num_connected() == NUM_FULL_NODES - 1
-					+ NUM_LIGHT_NODES,
-				|_index, service| service.get().network().num_connected() == NUM_FULL_NODES,
+				move |_index, service| service.get().network().num_connected()
+					== expected_full_connections,
+				move |_index, service| service.get().network().num_connected()
+					== expected_light_connections,
 			);
+
 			network.runtime
 		};
 
@@ -350,10 +365,12 @@ pub fn connectivity<G, Fb, F, Lb, L>(spec: ChainSpec<G>, full_builder: Fb, light
 					address = node_id.clone();
 				}
 			}
+
 			network.run_until_all_full(
-				|_index, service| service.get().network().num_connected() == NUM_FULL_NODES - 1
-					+ NUM_LIGHT_NODES,
-				|_index, service| service.get().network().num_connected() == NUM_FULL_NODES,
+				move |_index, service| service.get().network().num_connected()
+					== expected_full_connections,
+				move |_index, service| service.get().network().num_connected()
+					== expected_light_connections,
 			);
 		}
 		temp.close().expect("Error removing temp dir");
@@ -364,14 +381,14 @@ pub fn sync<G, Fb, F, Lb, L, B, E, U>(
 	spec: ChainSpec<G>,
 	full_builder: Fb,
 	light_builder: Lb,
-	mut block_factory: B,
+	mut make_block_and_import: B,
 	mut extrinsic_factory: E
 ) where
 	Fb: Fn(Configuration<(), G>) -> Result<(F, U), Error>,
 	F: AbstractService,
 	Lb: Fn(Configuration<(), G>) -> Result<L, Error>,
 	L: AbstractService,
-	B: FnMut(&F, &U) -> BlockImportParams<F::Block>,
+	B: FnMut(&F, &mut U),
 	E: FnMut(&F, &U) -> <F::Block as BlockT>::Extrinsic,
 	U: Clone + Send + 'static,
 {
@@ -392,15 +409,13 @@ pub fn sync<G, Fb, F, Lb, L, B, E, U>(
 	);
 	info!("Checking block sync");
 	let first_address = {
-		let first_service = &network.full_nodes[0].1;
-		let first_user_data = &network.full_nodes[0].2;
-		let mut client = first_service.get().client();
+		let &mut (_, ref first_service, ref mut first_user_data, _) = &mut network.full_nodes[0];
 		for i in 0 .. NUM_BLOCKS {
 			if i % 128 == 0 {
-				info!("Generating #{}", i);
+				info!("Generating #{}", i + 1);
 			}
-			let import_data = block_factory(&first_service.get(), first_user_data);
-			client.import_block(import_data, HashMap::new()).expect("Error importing test block");
+
+			make_block_and_import(&first_service.get(), first_user_data);
 		}
 		network.full_nodes[0].3.clone()
 	};