epoch-changes: separate epoch header with epoch data (#4881)

* fork-tree: prune returns all pruned node data * epoch-changes: split EpochHeader vs epoch data * EpochChanges::viable_epoch and add missing comments * Incoperate the new epoch_changes interface for BABE * Fix BABE tests * Fix fork-tree pruning issue * Fix tests * Fix pruning algorithm * fork-tree: implement map function for mapping one value type to another * Add migration script for new epoch changes scheme * Update utils/fork-tree/src/lib.rs Co-Authored-By: André Silva <andre.beat@gmail.com> * Update client/consensus/slots/src/lib.rs Co-Authored-By: André Silva <andre.beat@gmail.com> * Remove authorities_len.is_none check, which is duplicate of unwrap_or(false) * Update client/consensus/epochs/src/lib.rs Co-Authored-By: André Silva <andre.beat@gmail.com> * Update client/consensus/epochs/src/lib.rs Co-Authored-By: André Silva <andre.beat@gmail.com> * No trailing ; for return statement * Use VERSION_KEY for migration * Fix issues that removed nodes are not properly added into removed list * Add comments indicating end_slot is non-inclusive * fork-tree: use &mut F for map type declaration * Add tests for v0 epoch_changes migration * Fix babe RPC tests Co-authored-by: André Silva <andre.beat@gmail.com>
2026-04-29 19:37:56 +00:00 · 2020-03-16 22:06:13 +01:00
parent abe391a0a7
commit 846a9ce8c6
10 changed files with 764 additions and 212 deletions
@@ -93,41 +93,77 @@ impl<H, N, V> ForkTree<H, N, V> where
 	/// node. Otherwise the tree remains unchanged. The given function
 	/// `is_descendent_of` should return `true` if the second hash (target) is a
 	/// descendent of the first hash (base).
+	///
+	/// Returns all pruned node data.
 	pub fn prune<F, E, P>(
 		&mut self,
 		hash: &H,
 		number: &N,
 		is_descendent_of: &F,
 		predicate: &P,
-	) -> Result<(), Error<E>>
+	) -> Result<impl Iterator<Item=(H, N, V)>, Error<E>>
 		where E: std::error::Error,
 			  F: Fn(&H, &H) -> Result<bool, E>,
 			  P: Fn(&V) -> bool,
 	{
-		let new_root = self.find_node_where(
+		let new_root_index = self.find_node_index_where(
 			hash,
 			number,
 			is_descendent_of,
 			predicate,
 		)?;

-		if let Some(root) = new_root {
-			let mut root = root.clone();
+		let removed = if let Some(mut root_index) = new_root_index {
+			let mut old_roots = std::mem::replace(&mut self.roots, Vec::new());
+
+			let mut root = None;
+			let mut cur_children = Some(&mut old_roots);
+
+			while let Some(cur_index) = root_index.pop() {
+				if let Some(children) = cur_children.take() {
+					if root_index.is_empty() {
+						root = Some(children.remove(cur_index));
+					} else {
+						cur_children = Some(&mut children[cur_index].children);
+					}
+				}
+			}
+
+			let mut root = root
+				.expect("find_node_index_where will return array with at least one index; \
+                         this results in at least one item in removed; qed");
+
+			let mut removed = old_roots;

 			// we found the deepest ancestor of the finalized block, so we prune
 			// out any children that don't include the finalized block.
-			let children = std::mem::replace(&mut root.children, Vec::new());
-			root.children = children.into_iter().filter(|node| {
-				node.number == *number && node.hash == *hash ||
-					node.number < *number && is_descendent_of(&node.hash, hash).unwrap_or(false)
-			}).take(1).collect();
+			let root_children = std::mem::replace(&mut root.children, Vec::new());
+			let mut is_first = true;
+
+			for child in root_children {
+				if is_first &&
+					(child.number == *number && child.hash == *hash ||
+					 child.number < *number && is_descendent_of(&child.hash, hash).unwrap_or(false))
+				{
+					root.children.push(child);
+					// assuming that the tree is well formed only one child should pass this requirement
+					// due to ancestry restrictions (i.e. they must be different forks).
+					is_first = false;
+				} else {
+					removed.push(child);
+				}
+			}

 			self.roots = vec![root];
-		}
+
+			removed
+		} else {
+			Vec::new()
+		};

 		self.rebalance();

-		Ok(())
+		Ok(RemovedIterator { stack: removed })
 	}
 }

@@ -250,6 +286,26 @@ impl<H, N, V> ForkTree<H, N, V> where
 		Ok(None)
 	}

+	/// Map fork tree into values of new types.
+	pub fn map<VT, F>(
+		self,
+		f: &mut F,
+	) -> ForkTree<H, N, VT> where
+		F: FnMut(&H, &N, V) -> VT,
+	{
+		let roots = self.roots
+			.into_iter()
+			.map(|root| {
+				root.map(f)
+			})
+			.collect();
+
+		ForkTree {
+			roots,
+			best_finalized_number: self.best_finalized_number,
+		}
+	}
+
 	/// Same as [`find_node_where`](Self::find_node_where), but returns mutable reference.
 	pub fn find_node_where_mut<F, E, P>(
 		&mut self,
@@ -275,6 +331,32 @@ impl<H, N, V> ForkTree<H, N, V> where
 		Ok(None)
 	}

+	/// Same as [`find_node_where`](Self::find_node_where), but returns indexes.
+	pub fn find_node_index_where<F, E, P>(
+		&self,
+		hash: &H,
+		number: &N,
+		is_descendent_of: &F,
+		predicate: &P,
+	) -> Result<Option<Vec<usize>>, Error<E>> where
+		E: std::error::Error,
+		F: Fn(&H, &H) -> Result<bool, E>,
+		P: Fn(&V) -> bool,
+	{
+		// search for node starting from all roots
+		for (index, root) in self.roots.iter().enumerate() {
+			let node = root.find_node_index_where(hash, number, is_descendent_of, predicate)?;
+
+			// found the node, early exit
+			if let FindOutcome::Found(mut node) = node {
+				node.push(index);
+				return Ok(Some(node));
+			}
+		}
+
+		Ok(None)
+	}
+
 	/// Finalize a root in the tree and return it, return `None` in case no root
 	/// with the given hash exists. All other roots are pruned, and the children
 	/// of the finalized node become the new roots.
@@ -588,6 +670,29 @@ mod node_implementation {
 			max + 1
 		}

+		/// Map node data into values of new types.
+		pub fn map<VT, F>(
+			self,
+			f: &mut F,
+		) -> Node<H, N, VT> where
+			F: FnMut(&H, &N, V) -> VT,
+		{
+			let children = self.children
+				.into_iter()
+				.map(|node| {
+					node.map(f)
+				})
+				.collect();
+
+			let vt = f(&self.hash, &self.number, self.data);
+			Node {
+				hash: self.hash,
+				number: self.number,
+				data: vt,
+				children,
+			}
+		}
+
 		pub fn import<F, E: std::error::Error>(
 			&mut self,
 			mut hash: H,
@@ -780,6 +885,27 @@ impl<'a, H, N, V> Iterator for ForkTreeIterator<'a, H, N, V> {
 	}
 }

+struct RemovedIterator<H, N, V> {
+	stack: Vec<Node<H, N, V>>,
+}
+
+impl<H, N, V> Iterator for RemovedIterator<H, N, V> {
+	type Item = (H, N, V);
+
+	fn next(&mut self) -> Option<Self::Item> {
+		self.stack.pop().map(|mut node| {
+			// child nodes are stored ordered by max branch height (decreasing),
+			// we want to keep this ordering while iterating but since we're
+			// using a stack for iterator state we need to reverse it.
+			let mut children = Vec::new();
+			std::mem::swap(&mut children, &mut node.children);
+
+			self.stack.extend(children.into_iter().rev());
+			(node.hash, node.number, node.data)
+		})
+	}
+}
+
 #[cfg(test)]
 mod test {
 	use super::{FinalizationResult, ForkTree, Error};
@@ -805,7 +931,7 @@ mod test {
 		//  /   /
 		// A - F - H - I
 		//          \
-		//           - L - M - N
+		//           - L - M
 		//              \
 		//               - O
 		//  \
@@ -813,22 +939,21 @@ mod test {
 		//
 		// (where N is not a part of fork tree)
 		let is_descendent_of = |base: &&str, block: &&str| -> Result<bool, TestError> {
-			let letters = vec!["B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"];
+			let letters = vec!["B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "O"];
 			match (*base, *block) {
 				("A", b) => Ok(letters.into_iter().any(|n| n == b)),
 				("B", b) => Ok(b == "C" || b == "D" || b == "E"),
 				("C", b) => Ok(b == "D" || b == "E"),
 				("D", b) => Ok(b == "E"),
 				("E", _) => Ok(false),
-				("F", b) => Ok(b == "G" || b == "H" || b == "I" || b == "L" || b == "M" || b == "N" || b == "O"),
+				("F", b) => Ok(b == "G" || b == "H" || b == "I" || b == "L" || b == "M" || b == "O"),
 				("G", _) => Ok(false),
 				("H", b) => Ok(b == "I" || b == "L" || b == "M" || b == "O"),
 				("I", _) => Ok(false),
 				("J", b) => Ok(b == "K"),
 				("K", _) => Ok(false),
-				("L", b) => Ok(b == "M" || b == "O" || b == "N"),
-				("M", b) => Ok(b == "N"),
-				("N", _) => Ok(false),
+				("L", b) => Ok(b == "M" || b == "O"),
+				("M", _) => Ok(false),
 				("O", _) => Ok(false),
 				("0", _) => Ok(true),
 				_ => Ok(false),
@@ -1324,11 +1449,18 @@ mod test {
 		assert_eq!(node.number, 3);
 	}

+	#[test]
+	fn map_works() {
+		let (tree, _is_descendent_of) = test_fork_tree();
+
+		let _tree = tree.map(&mut |_, _, _| ());
+	}
+
 	#[test]
 	fn prune_works() {
 		let (mut tree, is_descendent_of) = test_fork_tree();

-		tree.prune(
+		let removed = tree.prune(
 			&"C",
 			&3,
 			&is_descendent_of,
@@ -1345,7 +1477,12 @@ mod test {
 			vec!["B", "C", "D", "E"],
 		);

-		tree.prune(
+		assert_eq!(
+			removed.map(|(hash, _, _)| hash).collect::<Vec<_>>(),
+			vec!["A", "F", "G", "H", "I", "L", "M", "O", "J", "K"]
+		);
+
+		let removed = tree.prune(
 			&"E",
 			&5,
 			&is_descendent_of,
@@ -1361,6 +1498,11 @@ mod test {
 			tree.iter().map(|(hash, _, _)| *hash).collect::<Vec<_>>(),
 			vec!["D", "E"],
 		);
+
+		assert_eq!(
+			removed.map(|(hash, _, _)| hash).collect::<Vec<_>>(),
+			vec!["B", "C"]
+		);
 	}

 	#[test]