Aggregate chains by network_id (#234)

* Aggregate chains by network_id

* Fix network_id handling

backend/Cargo.lock

@@ -92,7 +92,7 @@ dependencies = [
  "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -113,7 +113,7 @@ dependencies = [
  "http 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -242,7 +242,7 @@ dependencies = [
  "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -335,7 +335,7 @@ dependencies = [
  "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -453,7 +453,7 @@ dependencies = [
  "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -493,7 +493,7 @@ dependencies = [
  "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "publicsuffix 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
  "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -888,7 +888,7 @@ name = "impl-serde"
 version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1180,7 +1180,7 @@ dependencies = [
  "arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "bitvec 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byte-slice-cast 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1525,7 +1525,7 @@ dependencies = [
  "mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1632,15 +1632,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "serde"
-version = "1.0.102"
+version = "1.0.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.102"
+version = "1.0.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1655,7 +1655,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1665,7 +1665,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1676,7 +1676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1803,7 +1803,7 @@ dependencies = [
  "primitive-types 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
  "simple_logger 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2405,8 +2405,8 @@ dependencies = [
 "checksum security-framework-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c6680f163cacb6af55349fa4382a25b6d8f9c42b701e6f63dcce85a8f77e50fd"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-"checksum serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4b39bd9b0b087684013a792c59e3e07a46a01d2322518d8a1104641a0b1be0"
+"checksum serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)" = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff"
-"checksum serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "ca13fc1a832f793322228923fbb3aba9f3f44444898f835d31ad1b74fa0a2bf8"
+"checksum serde_derive 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)" = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8"
 "checksum serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)" = "2f72eb2a68a7dc3f9a691bfda9305a1c017a6215e5a4545c258500d2099a37c2"
 "checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a"
 "checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97"

backend/src/aggregator.rs

@@ -10,6 +10,7 @@ use crate::types::{NodeDetails, NodeId};
 
 pub struct Aggregator {
     labels: HashMap<Label, ChainId>,
+    networks: HashMap<Label, ChainId>,
     chains: DenseMap<ChainEntry>,
     feeds: DenseMap<Addr<FeedConnector>>,
     serializer: FeedMessageSerializer,
@@ -18,6 +19,7 @@ pub struct Aggregator {
 pub struct ChainEntry {
     addr: Addr<Chain>,
     label: Label,
+    network_id: Option<Label>,
     nodes: usize,
 }
 
@@ -25,6 +27,7 @@ impl Aggregator {
     pub fn new() -> Self {
         Aggregator {
             labels: HashMap::new(),
+            networks: HashMap::new(),
             chains: DenseMap::new(),
             feeds: DenseMap::new(),
             serializer: FeedMessageSerializer::new(),
@@ -33,33 +36,52 @@ impl Aggregator {
 
     /// Get an address to the chain actor by name. If the address is not found,
     /// or the address is disconnected (actor dropped), create a new one.
-    pub fn lazy_chain(&mut self, label: Label, ctx: &mut <Self as Actor>::Context) -> &mut ChainEntry {
+    pub fn lazy_chain(
-        let (cid, found) = self.labels
+        &mut self,
-            .get(&label)
+        label: &str,
-            .map(|&cid| (cid, true))
+        network: &Option<Label>,
-            .unwrap_or_else(|| {
+        ctx: &mut <Self as Actor>::Context,
+    ) -> ChainId {
+        let cid = match self.get_chain_id(label, network.as_ref()) {
+            Some(cid) => cid,
+            None => {
                 self.serializer.push(feed::AddedChain(&label, 1));
 
                 let addr = ctx.address();
-                let label = label.clone();
+                let label: Label = label.into();
-                let cid = self.chains.add_with(move |cid| {
+                let cid = self.chains.add_with(|cid| {
                     ChainEntry {
                         addr: Chain::new(cid, addr, label.clone()).start(),
-                        label,
+                        label: label.clone(),
+                        network_id: network.clone(),
                         nodes: 1,
                     }
                 });
 
+                self.labels.insert(label, cid);
+
+                if let Some(network) = network {
+                    self.networks.insert(network.clone(), cid);
+                }
+
                 self.broadcast();
 
-                (cid, false)
+                cid
-            });
+            }
+        };
 
-        if !found {
+        cid
-            self.labels.insert(label, cid);
+    }
+
+    fn get_chain_id(&self, label: &str, network: Option<&Label>) -> Option<ChainId> {
+        let labels = &self.labels;
+        let networks = &self.networks;
+
+        if let Some(network) = network {
+            networks.get(&**network).or_else(|| labels.get(label)).copied()
+        } else {
+            labels.get(label).copied()
         }
-
-        self.chains.get_mut(cid).expect("Entry just created above; qed")
     }
 
     fn get_chain(&mut self, label: &str) -> Option<&mut ChainEntry> {
@@ -84,13 +106,16 @@ impl Actor for Aggregator {
 #[derive(Message)]
 pub struct AddNode {
     pub node: NodeDetails,
-    pub chain: Label,
+    pub network_id: Option<Label>,
     pub rec: Recipient<Initialize>,
 }
 
 /// Message sent from the Chain to the Aggregator when the Chain loses all nodes
 #[derive(Message)]
-pub struct DropChain(pub Label);
+pub struct DropChain(pub ChainId);
+
+#[derive(Message)]
+pub struct RenameChain(pub ChainId, pub Label);
 
 /// Message sent from the FeedConnector to the Aggregator when subscribing to a new chain
 pub struct Subscribe {
@@ -146,9 +171,20 @@ impl Handler<AddNode> for Aggregator {
     type Result = ();
 
     fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
-        let AddNode { node, chain, rec } = msg;
+        let AddNode { node, network_id, rec } = msg;
 
-        self.lazy_chain(chain, ctx).addr.do_send(chain::AddNode {
+        let cid = self.lazy_chain(&node.chain, &network_id, ctx);
+        let chain = self.chains.get_mut(cid).expect("Entry just created above; qed");
+
+        if let Some(network_id) = network_id {
+            // Attach network id to the chain if it was not done yet
+            if chain.network_id.is_none() {
+                chain.network_id = Some(network_id.clone());
+                self.networks.insert(network_id, cid);
+            }
+        }
+
+        chain.addr.do_send(chain::AddNode {
             node,
             rec,
         });
@@ -159,15 +195,47 @@ impl Handler<DropChain> for Aggregator {
     type Result = ();
 
     fn handle(&mut self, msg: DropChain, _: &mut Self::Context) {
-        let DropChain(label) = msg;
+        let DropChain(cid) = msg;
 
-        if let Some(cid) = self.labels.remove(&label) {
+        if let Some(entry) = self.chains.remove(cid) {
-            self.chains.remove(cid);
+            let label = &entry.label;
-            self.serializer.push(feed::RemovedChain(&label));
+            self.labels.remove(label);
+            if let Some(network) = entry.network_id {
+                self.networks.remove(&network);
+            }
+
+            self.serializer.push(feed::RemovedChain(label));
+            info!("Dropped chain [{}] from the aggregator", label);
             self.broadcast();
         }
 
-        info!("Dropped chain [{}] from the aggregator", label);
+    }
+}
+
+impl Handler<RenameChain> for Aggregator {
+    type Result = ();
+
+    fn handle(&mut self, msg: RenameChain, _: &mut Self::Context) {
+        let RenameChain(cid, new) = msg;
+
+        if let Some(entry) = self.chains.get_mut(cid) {
+            if entry.label == new {
+                return;
+            }
+
+            // Update UI
+            self.serializer.push(feed::RemovedChain(&entry.label));
+            self.serializer.push(feed::AddedChain(&new, entry.nodes));
+
+            // Update labels -> cid map
+            self.labels.remove(&entry.label);
+            self.labels.insert(new.clone(), cid);
+
+            // Update entry
+            entry.label = new;
+
+            self.broadcast();
+        }
     }
 }
 

backend/src/chain.rs

@@ -1,9 +1,10 @@
+use std::collections::HashMap;
 use actix::prelude::*;
 use std::sync::Arc;
 use bytes::Bytes;
 use rustc_hash::FxHashMap;
 
-use crate::aggregator::{Aggregator, DropChain, NodeCount};
+use crate::aggregator::{Aggregator, DropChain, RenameChain, NodeCount};
 use crate::node::{Node, connector::Initialize, message::{NodeMessage, Details}};
 use crate::feed::connector::{FeedId, FeedConnector, Subscribed, Unsubscribed};
 use crate::feed::{self, FeedMessageSerializer};
@@ -19,8 +20,8 @@ pub struct Chain {
     cid: ChainId,
     /// Who to inform if we Chain drops itself
     aggregator: Addr<Aggregator>,
-    /// Label of this chain
+    /// Label of this chain, along with count of nodes that use this label
-    label: Label,
+    label: (Label, usize),
     /// Dense mapping of NodeId -> Node
     nodes: DenseMap<Node>,
     /// Dense mapping of FeedId -> Addr<FeedConnector>,
@@ -39,6 +40,8 @@ pub struct Chain {
     serializer: FeedMessageSerializer,
     /// When the best block first arrived
     timestamp: Option<Timestamp>,
+    /// Some nodes might manifest a different label, note them here
+    labels: HashMap<Label, usize>,
 }
 
 impl Chain {
@@ -48,7 +51,7 @@ impl Chain {
         Chain {
             cid,
             aggregator,
-            label,
+            label: (label, 0),
             nodes: DenseMap::new(),
             feeds: DenseMap::new(),
             finality_feeds: FxHashMap::default(),
@@ -58,9 +61,56 @@ impl Chain {
             average_block_time: None,
             serializer: FeedMessageSerializer::new(),
             timestamp: None,
+            labels: HashMap::default(),
         }
     }
 
+    fn increment_label_count(&mut self, label: &str) {
+        let count = match self.labels.get_mut(label) {
+            Some(count) => {
+                *count += 1;
+                *count
+            },
+            None => {
+                self.labels.insert(label.into(), 1);
+                1
+            },
+        };
+
+        if &*self.label.0 == label {
+            self.label.1 += 1;
+        } else {
+            if count > self.label.1 {
+                self.rename(label.into(), count);
+            }
+        }
+    }
+
+    fn decrement_label_count(&mut self, label: &str) {
+        match self.labels.get_mut(label) {
+            Some(count) => *count -= 1,
+            None => return,
+        };
+
+        if &*self.label.0 == label {
+            self.label.1 -= 1;
+
+            for (label, &count) in self.labels.iter() {
+                if count > self.label.1 {
+                    let label: Arc<_> = label.clone();
+                    self.rename(label, count);
+                    break;
+                }
+            }
+        }
+    }
+
+    fn rename(&mut self, label: Label, count: usize) {
+        self.label = (label, count);
+
+        self.aggregator.do_send(RenameChain(self.cid, self.label.0.clone()));
+    }
+
     fn broadcast(&mut self) {
         if let Some(msg) = self.serializer.finalize() {
             for (_, feed) in self.feeds.iter() {
@@ -132,7 +182,7 @@ impl Actor for Chain {
     type Context = Context<Self>;
 
     fn stopped(&mut self, _: &mut Self::Context) {
-        self.aggregator.do_send(DropChain(self.label.clone()));
+        self.aggregator.do_send(DropChain(self.cid));
 
         for (_, feed) in self.feeds.iter() {
             feed.do_send(Unsubscribed)
@@ -190,6 +240,8 @@ impl Handler<AddNode> for Chain {
     type Result = ();
 
     fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
+        self.increment_label_count(&msg.node.chain);
+
         let nid = self.nodes.add(Node::new(msg.node));
 
         if let Err(_) = msg.rec.do_send(Initialize(nid, ctx.address())) {
@@ -221,7 +273,7 @@ impl Chain {
                 self.best = *block;
                 info!(
                     "[{}] [{}/{}] new best block ({}) {:?}",
-                    self.label,
+                    self.label.0,
                     nodes_len,
                     self.feeds.len(),
                     self.best.height,
@@ -361,10 +413,12 @@ impl Handler<RemoveNode> for Chain {
     fn handle(&mut self, msg: RemoveNode, ctx: &mut Self::Context) {
         let RemoveNode(nid) = msg;
 
-        self.nodes.remove(nid);
+        if let Some(node) = self.nodes.remove(nid) {
+            self.decrement_label_count(&node.details().chain);
+        }
 
         if self.nodes.is_empty() {
-            info!("[{}] Lost all nodes, dropping...", self.label);
+            info!("[{}] Lost all nodes, dropping...", self.label.0);
             ctx.stop();
         }
 
@@ -383,7 +437,7 @@ impl Handler<Subscribe> for Chain {
 
         feed.do_send(Subscribed(fid, ctx.address().recipient()));
 
-        self.serializer.push(feed::SubscribedTo(&self.label));
+        self.serializer.push(feed::SubscribedTo(&self.label.0));
         self.serializer.push(feed::TimeSync(now()));
         self.serializer.push(feed::BestBlock(
             self.best.height,
@@ -445,7 +499,7 @@ impl Handler<Unsubscribe> for Chain {
         let Unsubscribe(fid) = msg;
 
         if let Some(feed) = self.feeds.get(fid) {
-            self.serializer.push(feed::UnsubscribedFrom(&self.label));
+            self.serializer.push(feed::UnsubscribedFrom(&self.label.0));
 
             if let Some(serialized) = self.serializer.finalize() {
                 feed.do_send(serialized);

backend/src/node/connector.rs

@@ -82,11 +82,11 @@ impl NodeConnector {
         }
 
         if let Details::SystemConnected(connected) = msg.details {
-            let SystemConnected { chain, node } = connected;
+            let SystemConnected { network_id, node } = connected;
             let rec = ctx.address().recipient();
-            let chain = chain.into();
+            let network_id = network_id.map(Into::into);
 
-            self.aggregator.do_send(AddNode { rec, chain, node });
+            self.aggregator.do_send(AddNode { rec, network_id, node });
         } else {
             if self.backlog.len() >= 10 {
                 self.backlog.remove(0);

backend/src/node/message.rs

@@ -56,7 +56,7 @@ pub enum Details {
 
 #[derive(Deserialize, Debug)]
 pub struct SystemConnected {
-    pub chain: Box<str>,
+    pub network_id: Option<Box<str>>,
     #[serde(flatten)]
     pub node: NodeDetails,
 }

backend/src/types.rs

@@ -11,6 +11,7 @@ pub use primitive_types::H256 as BlockHash;
 
 #[derive(Deserialize, Debug)]
 pub struct NodeDetails {
+    pub chain: Box<str>,
     pub name: Box<str>,
     pub implementation: Box<str>,
     pub version: Box<str>,