feat: Hide nodes that fail to sync a block in 15 minutes (#154)

* feat: Hide nodes that fail to sync a block in 15 minutes

* fix: Don't resend stale event if node is already stale

packages/backend/src/Chain.ts

@@ -38,6 +38,7 @@ export default class Chain {
     this.feeds.broadcast(Feed.addedNode(node));
 
     node.events.once('disconnect', () => this.removeNode(node));
+    node.events.once('stale', () => this.staleNode(node));
 
     node.events.on('block', () => this.updateBlock(node));
     node.events.on('finalized', () => this.updateFinalized(node));
@@ -89,6 +90,14 @@ export default class Chain {
     }
   }
 
+  public staleNode(node: Node) {
+    this.feeds.broadcast(Feed.removedNode(node));
+
+    if (this.height === node.best.number) {
+      this.downgradeBlock();
+    }
+  }
+
   public addFeed(feed: Feed) {
     this.feeds.add(feed);
 
@@ -104,6 +113,10 @@ export default class Chain {
     }
 
     for (const node of this.nodes.values()) {
+      if (node.isStale) {
+        continue;
+      }
+
       feed.sendMessage(Feed.addedNode(node));
       feed.sendMessage(Feed.finalized(node));
     }
@@ -152,7 +165,12 @@ export default class Chain {
       node.propagationTime = (node.blockTimestamp - this.blockTimestamp) as Types.PropagationTime;
     }
 
-    this.feeds.broadcast(Feed.imported(node));
+    if (node.isStale) {
+      node.isStale = false;
+      this.feeds.broadcast(Feed.addedNode(node));
+    } else {
+      this.feeds.broadcast(Feed.imported(node));
+    }
 
     console.log(`[${this.label}] ${node.name} imported ${height}, block time: ${node.blockTime / 1000}s, average: ${node.average / 1000}s | latency ${node.latency}`);
   }
@@ -162,6 +180,10 @@ export default class Chain {
     let finalized = Block.ZERO;
 
     for (const node of this.nodes) {
+      if (node.isStale) {
+        continue;
+      }
+
       if (this.height === node.best.number) {
         return;
       }

packages/backend/src/Node.ts

@@ -22,6 +22,7 @@ const BLOCK_TIME_HISTORY = 10;
 const MEMORY_RECORDS = 20;
 const CPU_RECORDS = 20;
 const TIMEOUT = (1000 * 60 * 1) as Types.Milliseconds; // 1 minute
+const NO_BLOCK_TIMEOUT = (1000 * 60 * 15) as Types.Milliseconds; // 15 minutes
 
 const nextId = idGenerator<Types.NodeId>();
 
@@ -52,6 +53,7 @@ export default class Node {
   public blockTime = 0 as Types.Milliseconds;
   public blockTimestamp = 0 as Types.Timestamp;
   public propagationTime: Maybe<Types.PropagationTime> = null;
+  public isStale = false;
 
   private peers = 0 as Types.PeerCount;
   private txcount = 0 as Types.TransactionCount;
@@ -190,6 +192,11 @@ export default class Node {
     if (this.lastMessage + TIMEOUT < now) {
       this.disconnect();
     } else {
+      if (!this.isStale && this.blockTimestamp + NO_BLOCK_TIMEOUT < now) {
+        this.isStale = true;
+        this.events.emit('stale');
+      }
+
       this.updateLatency(now);
     }
   }