Merge remote-tracking branch 'origin/jsdw-sharding' into helm

.github/workflows/backend.yml

@@ -14,21 +14,30 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Build
+
+    - name: Build telemetry executables (in debug mode)
       working-directory: ./backend
-      run: cargo build --verbose
+      run: cargo build --bins --verbose
+
     - name: Run tests
       working-directory: ./backend
       run: cargo test --verbose
-    - name: Build release and call executable
+
+    - name: Build, release and call telemetry executable
       working-directory: ./backend
-      run: cargo run --release -- --help
+      run: cargo run --bin telemetry_core --release -- --help
+
+    - name: Build, release and call shard executable
+      working-directory: ./backend
+      run: cargo run --bin telemetry_shard --release -- --help
+
     - name: Login to Dockerhub
       uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
-    - name: Build and Push template image
+
+    - name: Build and push template image for tagged commit
       uses: docker/build-push-action@v2 # https://github.com/docker/build-push-action
       with:
         context: './backend'

backend/.dockerignore

@@ -1,3 +1,3 @@
 target
-Dockerfile
+*.Dockerfile
 .git

backend/Cargo.toml

@@ -1,6 +1,9 @@
 [workspace]
 members = [
-    "core",
+    "common",
+    "telemetry_core",
+    "telemetry_shard",
+    "test_utils"
 ]
 
 [profile.dev]
@@ -9,3 +12,7 @@ opt-level = 3
 [profile.release]
 lto = true
 panic = "abort"
+## Enabling these seems necessary to get
+## good debug info in Instruments:
+# debug = true
+# codegen-units = 1

backend/Dockerfile

@@ -10,15 +10,15 @@ RUN cargo build --${PROFILE} --bins
 # MAIN IMAGE FOR PEOPLE TO PULL --- small one#
 FROM docker.io/debian:buster-slim
 LABEL maintainer="Parity Technologies"
-LABEL description="Polkadot Telemetry backend, static build"
+LABEL description="Polkadot Telemetry backend shard/core binaries, static build"
 
 ARG PROFILE=release
 WORKDIR /usr/local/bin
 
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=builder /app/target/$PROFILE/telemetry /usr/local/bin
+COPY --from=builder /app/target/$PROFILE/telemetry_shard /usr/local/bin
+COPY --from=builder /app/target/$PROFILE/telemetry_core /usr/local/bin
 RUN apt-get -y update && apt-get -y install openssl && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/
 
 EXPOSE 8000
 
-ENTRYPOINT ["telemetry"]

backend/README.md

@@ -0,0 +1,19 @@
+# Backend Crates
+
+This folder contains the rust crates and documentation specific to the telemetry backend. A description of the folders:
+
+- [telemetry_core](./telemetry_core): The Telemetry Core. This aggregates data received from shards and allows UI feeds to connect and receive this information.
+- [telemetry_shard](./telemetry_shard): A Shard. It's expected that multiple of these will run. Nodes will connect to Shard instances and send JSON telemetry to them, and Shard instances will each connect to the Telemetry Core and relay on relevant data to it.
+- [common](./common): common code shared between the telemetry shard and core
+- [test_utils](./test_utils): Test utilities, primarily focused around making it easy to run end-to-end tests.
+- [docs](./docs): Material supporting the documentation lives here
+
+# Architecture
+
+As we move to a sharded version of this telemetry server, this set of architecture diagrams may be useful in helping to understand the current setup (middle diagram), previous setup (first diagram) and possible future setup if we need to scale further (last diagram):
+
+![Architecture Diagram](./docs/architecture.svg)
+
+# Deployment
+
+A `Dockerfile` exists which builds the Shard and Telemetry Core binaries into an image. A `docker-compose.yaml` in the root of the repository can serve as an example of these services, along with the UI, running together.

backend/common/Cargo.toml

@@ -1,29 +1,28 @@
 [package]
-name = "telemetry"
-version = "0.3.0"
+name = "common"
+version = "0.1.0"
 authors = ["Parity Technologies Ltd. <admin@parity.io>"]
 edition = "2018"
 license = "GPL-3.0"
 
 [dependencies]
-actix = "0.11.1"
-actix-web = { version = "4.0.0-beta.4", default-features = false }
-actix-web-actors = "4.0.0-beta.3"
-actix-http = "3.0.0-beta.4"
-bincode = "1.3.3"
+bimap = "0.6.1"
 bytes = "1.0.1"
-chrono = { version = "0.4", features = ["serde"] }
 fnv = "1.0.7"
+futures = "0.3.15"
 hex = "0.4.3"
+http = "0.2.4"
+log = "0.4"
+num-traits = "0.2"
+pin-project-lite = "0.2.7"
+primitive-types = { version = "0.9.0", features = ["serde"] }
+rustc-hash = "1.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["raw_value"] }
+soketto = "0.6.0"
 thiserror = "1.0.24"
-primitive-types = { version = "0.9.0", features = ["serde"] }
-log = "0.4"
-simple_logger = "1.11.0"
-num-traits = "0.2"
-parking_lot = "0.11"
-reqwest = { version = "0.11.1", features = ["blocking", "json"] }
-rustc-hash = "1.1.0"
-clap = "3.0.0-beta.2"
-ctor = "0.1.20"
+tokio = { version = "1.8.2", features = ["full"] }
+tokio-util = { version = "0.6", features = ["compat"] }
+
+[dev-dependencies]
+bincode = "1.3.3"

backend/common/src/assign_id.rs

@@ -0,0 +1,68 @@
+use bimap::BiMap;
+use std::hash::Hash;
+
+/// A struct that allows you to assign an Id to an arbitrary set of
+/// details (so long as they are Eq+Hash+Clone), and then access
+/// the assigned Id given those details or access the details given
+/// the Id.
+///
+/// The Id can be any type that's convertible to/from a `usize`. Using
+/// a custom type is recommended for increased type safety.
+#[derive(Debug)]
+pub struct AssignId<Id, Details> {
+    current_id: usize,
+    mapping: BiMap<usize, Details>,
+    _id_type: std::marker::PhantomData<Id>,
+}
+
+impl<Id, Details> AssignId<Id, Details>
+where
+    Details: Eq + Hash,
+    Id: From<usize> + Copy,
+    usize: From<Id>,
+{
+    pub fn new() -> Self {
+        Self {
+            current_id: 0,
+            mapping: BiMap::new(),
+            _id_type: std::marker::PhantomData,
+        }
+    }
+
+    pub fn assign_id(&mut self, details: Details) -> Id {
+        let this_id = self.current_id;
+        self.current_id += 1;
+        self.mapping.insert(this_id, details);
+        this_id.into()
+    }
+
+    pub fn get_details(&mut self, id: Id) -> Option<&Details> {
+        self.mapping.get_by_left(&id.into())
+    }
+
+    pub fn get_id(&mut self, details: &Details) -> Option<Id> {
+        self.mapping.get_by_right(details).map(|&id| id.into())
+    }
+
+    pub fn remove_by_id(&mut self, id: Id) -> Option<Details> {
+        self.mapping
+            .remove_by_left(&id.into())
+            .map(|(_, details)| details)
+    }
+
+    pub fn remove_by_details(&mut self, details: &Details) -> Option<Id> {
+        self.mapping
+            .remove_by_right(&details)
+            .map(|(id, _)| id.into())
+    }
+
+    pub fn clear(&mut self) {
+        *self = AssignId::new();
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (Id, &Details)> {
+        self.mapping
+            .iter()
+            .map(|(&id, details)| (id.into(), details))
+    }
+}

backend/common/src/dense_map.rs

@@ -1,17 +1,31 @@
-pub type Id = usize;
-
-pub struct DenseMap<T> {
+/// This stores items in contiguous memory, making a note of free
+/// slots when items are removed again so that they can be reused.
+///
+/// This is particularly efficient when items are often added and
+/// seldom removed.
+///
+/// Items are keyed by an Id, which can be any type you wish, but
+/// must be convertible to/from a `usize`. This promotes using a
+/// custom Id type to talk about items in the map.
+pub struct DenseMap<Id, T> {
     /// List of retired indexes that can be re-used
-    retired: Vec<Id>,
+    retired: Vec<usize>,
     /// All items
     items: Vec<Option<T>>,
+    /// Our ID type
+    _id_type: std::marker::PhantomData<Id>,
 }
 
-impl<T> DenseMap<T> {
+impl<Id, T> DenseMap<Id, T>
+where
+    Id: From<usize> + Copy,
+    usize: From<Id>,
+{
     pub fn new() -> Self {
         DenseMap {
             retired: Vec::new(),
             items: Vec::new(),
+            _id_type: std::marker::PhantomData,
         }
     }
 
@@ -25,11 +39,12 @@ impl<T> DenseMap<T> {
     {
         match self.retired.pop() {
             Some(id) => {
-                self.items[id] = Some(f(id));
-                id
+                let id_out = id.into();
+                self.items[id] = Some(f(id_out));
+                id_out
             }
             None => {
-                let id = self.items.len();
+                let id = self.items.len().into();
                 self.items.push(Some(f(id)));
                 id
             }
@@ -37,14 +52,17 @@ impl<T> DenseMap<T> {
     }
 
     pub fn get(&self, id: Id) -> Option<&T> {
+        let id: usize = id.into();
         self.items.get(id).and_then(|item| item.as_ref())
     }
 
     pub fn get_mut(&mut self, id: Id) -> Option<&mut T> {
+        let id: usize = id.into();
         self.items.get_mut(id).and_then(|item| item.as_mut())
     }
 
     pub fn remove(&mut self, id: Id) -> Option<T> {
+        let id: usize = id.into();
         let old = self.items.get_mut(id).and_then(|item| item.take());
 
         if old.is_some() {
@@ -60,14 +78,21 @@ impl<T> DenseMap<T> {
         self.items
             .iter()
             .enumerate()
-            .filter_map(|(id, item)| Some((id, item.as_ref()?)))
+            .filter_map(|(id, item)| Some((id.into(), item.as_ref()?)))
     }
 
     pub fn iter_mut(&mut self) -> impl Iterator<Item = (Id, &mut T)> + '_ {
         self.items
             .iter_mut()
             .enumerate()
-            .filter_map(|(id, item)| Some((id, item.as_mut()?)))
+            .filter_map(|(id, item)| Some((id.into(), item.as_mut()?)))
+    }
+
+    pub fn into_iter(self) -> impl Iterator<Item = (Id, T)> {
+        self.items
+            .into_iter()
+            .enumerate()
+            .filter_map(|(id, item)| Some((id.into(), item?)))
     }
 
     pub fn len(&self) -> usize {
@@ -77,4 +102,12 @@ impl<T> DenseMap<T> {
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
+
+    /// Return the next Id that will be assigned.
+    pub fn next_id(&self) -> usize {
+        match self.retired.last() {
+            Some(id) => *id,
+            None => self.items.len(),
+        }
+    }
 }

backend/common/src/id_type.rs

@@ -0,0 +1,74 @@
+/// Define a type that can be used as an ID, be converted from/to the inner type,
+/// and serialized/deserialized transparently into the inner type.
+#[macro_export]
+macro_rules! id_type {
+    ($( #[$attrs:meta] )* $vis:vis struct $ty:ident ( $inner:ident ) $(;)? ) => {
+        #[derive(Debug,Clone,Copy,PartialEq,Eq,Hash)]
+        $( #[$attrs] )*
+        $vis struct $ty($inner);
+
+        impl $ty {
+            #[allow(dead_code)]
+            pub fn new(inner: $inner) -> Self {
+                Self(inner)
+            }
+        }
+
+        impl From<$inner> for $ty {
+            fn from(inner: $inner) -> Self {
+                Self(inner)
+            }
+        }
+
+        impl From<$ty> for $inner {
+            fn from(ty: $ty) -> Self {
+                ty.0
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    //! Mostly we're just checking that everything compiles OK
+    //! when the macro is used as expected..
+
+    // A basic definition is possible:
+    id_type! {
+        struct Foo(usize)
+    }
+
+    // We can add a ';' on the end:
+    id_type! {
+        struct Bar(usize);
+    }
+
+    // Visibility qualifiers are allowed:
+    id_type! {
+        pub struct Wibble(u64)
+    }
+
+    // Doc strings are possible
+    id_type! {
+        /// We can have doc strings, too
+        pub(crate) struct Wobble(u16)
+    }
+
+    // In fact, any attributes can be added (common
+    // derives are added already):
+    id_type! {
+        /// We can have doc strings, too
+        #[derive(serde::Serialize)]
+        #[serde(transparent)]
+        pub(crate) struct Lark(u16)
+    }
+
+    #[test]
+    fn create_and_use_new_id_type() {
+        let _ = Foo::new(123);
+        let id = Foo::from(123);
+        let id_num: usize = id.into();
+
+        assert_eq!(id_num, 123);
+    }
+}

backend/common/src/internal_messages.rs

@@ -0,0 +1,51 @@
+//! Internal messages passed between the shard and telemetry core.
+
+use std::net::IpAddr;
+
+use crate::id_type;
+use crate::node_message::Payload;
+use crate::node_types::{BlockHash, NodeDetails};
+use serde::{Deserialize, Serialize};
+
+id_type! {
+    /// The shard-local ID of a given node, where a single connection
+    /// might send data on behalf of more than one chain.
+    #[derive(serde::Serialize, serde::Deserialize)]
+    pub struct ShardNodeId(usize);
+}
+
+/// Message sent from a telemetry shard to the telemetry core
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum FromShardAggregator {
+    /// Get information about a new node, including it's IP
+    /// address and chain genesis hash.
+    AddNode {
+        ip: Option<IpAddr>,
+        node: NodeDetails,
+        local_id: ShardNodeId,
+        genesis_hash: BlockHash,
+    },
+    /// A message payload with updated details for a node
+    UpdateNode {
+        local_id: ShardNodeId,
+        payload: Payload,
+    },
+    /// Inform the telemetry core that a node has been removed
+    RemoveNode { local_id: ShardNodeId },
+}
+
+/// Message sent form the telemetry core to a telemetry shard
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum FromTelemetryCore {
+    Mute {
+        local_id: ShardNodeId,
+        reason: MuteReason,
+    },
+}
+
+/// Why is the thing being muted?
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum MuteReason {
+    Overquota,
+    ChainNotAllowed,
+}

backend/common/src/lib.rs

@@ -0,0 +1,20 @@
+pub mod id_type;
+pub mod internal_messages;
+pub mod node_message;
+pub mod node_types;
+pub mod time;
+pub mod ws_client;
+pub mod ready_chunks_all;
+
+mod assign_id;
+mod dense_map;
+mod mean_list;
+mod most_seen;
+mod num_stats;
+
+// Export a bunch of common bits at the top level for ease of import:
+pub use assign_id::AssignId;
+pub use dense_map::DenseMap;
+pub use mean_list::MeanList;
+pub use most_seen::MostSeen;
+pub use num_stats::NumStats;

backend/common/src/most_seen.rs

@@ -0,0 +1,231 @@
+use std::collections::HashMap;
+use std::hash::Hash;
+
+/// Add items to this, and it will keep track of what the item
+/// seen the most is.
+#[derive(Debug)]
+pub struct MostSeen<T> {
+    current_best: T,
+    current_count: usize,
+    others: HashMap<T, usize>,
+}
+
+impl<T: Default> Default for MostSeen<T> {
+    fn default() -> Self {
+        Self {
+            current_best: T::default(),
+            current_count: 0,
+            others: HashMap::new(),
+        }
+    }
+}
+
+impl<T> MostSeen<T> {
+    pub fn new(item: T) -> Self {
+        Self {
+            current_best: item,
+            current_count: 1,
+            others: HashMap::new(),
+        }
+    }
+    pub fn best(&self) -> &T {
+        &self.current_best
+    }
+    pub fn best_count(&self) -> usize {
+        self.current_count
+    }
+}
+
+impl<T: Hash + Eq + Clone> MostSeen<T> {
+    pub fn insert(&mut self, item: &T) -> ChangeResult {
+        if &self.current_best == item {
+            // Item already the best one; bump count.
+            self.current_count += 1;
+            return ChangeResult::NoChange;
+        }
+
+        // Item not the best; increment count in map
+        let item_count = self.others.entry(item.clone()).or_default();
+        *item_count += 1;
+
+        // Is item now the best?
+        if *item_count > self.current_count {
+            let (mut item, mut count) = self.others.remove_entry(item).expect("item added above");
+
+            // Swap the current best for the new best:
+            std::mem::swap(&mut item, &mut self.current_best);
+            std::mem::swap(&mut count, &mut self.current_count);
+
+            // Insert the old best back into the map:
+            self.others.insert(item, count);
+
+            ChangeResult::NewMostSeenItem
+        } else {
+            ChangeResult::NoChange
+        }
+    }
+    pub fn remove(&mut self, item: &T) -> ChangeResult {
+        if &self.current_best == item {
+            // Item already the best one; reduce count (don't allow to drop below 0)
+            self.current_count = self.current_count.saturating_sub(1);
+
+            // Is there a new best?
+            let other_best = self.others.iter().max_by_key(|f| f.1);
+
+            let (other_item, &other_count) = match other_best {
+                Some(item) => item,
+                None => return ChangeResult::NoChange,
+            };
+
+            if other_count > self.current_count {
+                // Clone item to unborrow self.others so that we can remove
+                // the item from it. We could pre-emptively remove and reinsert
+                // instead, but most of the time there is no change, so I'm
+                // aiming to keep that path cheaper.
+                let other_item = other_item.clone();
+                let (mut other_item, mut other_count) = self
+                    .others
+                    .remove_entry(&other_item)
+                    .expect("item returned above, so def exists");
+
+                // Swap the current best for the new best:
+                std::mem::swap(&mut other_item, &mut self.current_best);
+                std::mem::swap(&mut other_count, &mut self.current_count);
+
+                // Insert the old best back into the map:
+                self.others.insert(other_item, other_count);
+
+                return ChangeResult::NewMostSeenItem;
+            } else {
+                return ChangeResult::NoChange;
+            }
+        }
+
+        // Item is in the map; not the best anyway. decrement count.
+        if let Some(count) = self.others.get_mut(item) {
+            *count += 1;
+        }
+        ChangeResult::NoChange
+    }
+}
+
+/// Record the result of adding/removing an entry
+#[derive(Clone, Copy)]
+pub enum ChangeResult {
+    /// The best item has remained the same.
+    NoChange,
+    /// There is a new best item now.
+    NewMostSeenItem,
+}
+
+impl ChangeResult {
+    pub fn has_changed(self) -> bool {
+        match self {
+            ChangeResult::NewMostSeenItem => true,
+            ChangeResult::NoChange => false,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn default_renames_instantly() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+        let res = a.insert(&"Hello");
+        assert_eq!(*a.best(), "Hello");
+        assert!(res.has_changed());
+    }
+
+    #[test]
+    fn new_renames_on_second_change() {
+        let mut a: MostSeen<&str> = MostSeen::new("First");
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "First");
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "Second");
+    }
+
+    #[test]
+    fn removing_doesnt_underflow() {
+        let mut a: MostSeen<&str> = MostSeen::new("First");
+        a.remove(&"First");
+        a.remove(&"First");
+        a.remove(&"Second");
+        a.remove(&"Third");
+    }
+
+    #[test]
+    fn keeps_track_of_best_count() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+        a.insert(&"First");
+        assert_eq!(a.best_count(), 1);
+
+        a.insert(&"First");
+        assert_eq!(a.best_count(), 2);
+
+        a.insert(&"First");
+        assert_eq!(a.best_count(), 3);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 2);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 1);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 0);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 0);
+    }
+
+    #[test]
+    fn it_tracks_best_on_insert() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+
+        a.insert(&"First");
+        assert_eq!(*a.best(), "First", "1");
+
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "First", "2");
+
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "Second", "3");
+
+        a.insert(&"First");
+        assert_eq!(*a.best(), "Second", "4");
+
+        a.insert(&"First");
+        assert_eq!(*a.best(), "First", "5");
+    }
+
+    #[test]
+    fn it_tracks_best() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+        a.insert(&"First");
+        a.insert(&"Second");
+        a.insert(&"Third"); // 1
+
+        a.insert(&"Second");
+        a.insert(&"Second"); // 3
+        a.insert(&"First"); // 2
+
+        assert_eq!(*a.best(), "Second");
+        assert_eq!(a.best_count(), 3);
+
+        let res = a.remove(&"Second");
+
+        assert!(!res.has_changed());
+        assert_eq!(a.best_count(), 2);
+        assert_eq!(*a.best(), "Second"); // Tied with "First"
+
+        let res = a.remove(&"Second");
+
+        assert!(res.has_changed());
+        assert_eq!(a.best_count(), 2);
+        assert_eq!(*a.best(), "First"); // First is now ahead
+    }
+}

backend/common/src/node_message.rs

@@ -0,0 +1,246 @@
+//! This is the internal represenation of telemetry messages sent from nodes.
+//! There is a separate JSON representation of these types, because internally we want to be
+//! able to serialize these messages to bincode, and various serde attribtues aren't compatible
+//! with this, hence this separate internal representation.
+
+use crate::node_types::{Block, BlockHash, BlockNumber, NodeDetails};
+use serde::{Deserialize, Serialize};
+
+pub type NodeMessageId = u64;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum NodeMessage {
+    V1 { payload: Payload },
+    V2 { id: NodeMessageId, payload: Payload },
+}
+
+impl NodeMessage {
+    /// Returns the ID associated with the node message, or 0
+    /// if the message has no ID.
+    pub fn id(&self) -> NodeMessageId {
+        match self {
+            NodeMessage::V1 { .. } => 0,
+            NodeMessage::V2 { id, .. } => *id,
+        }
+    }
+    /// Return the payload associated with the message.
+    pub fn into_payload(self) -> Payload {
+        match self {
+            NodeMessage::V1 { payload, .. } | NodeMessage::V2 { payload, .. } => payload,
+        }
+    }
+}
+
+impl From<NodeMessage> for Payload {
+    fn from(msg: NodeMessage) -> Payload {
+        msg.into_payload()
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum Payload {
+    SystemConnected(SystemConnected),
+    SystemInterval(SystemInterval),
+    BlockImport(Block),
+    NotifyFinalized(Finalized),
+    TxPoolImport,
+    AfgFinalized(AfgFinalized),
+    AfgReceivedPrecommit(AfgReceived),
+    AfgReceivedPrevote(AfgReceived),
+    AfgReceivedCommit(AfgReceived),
+    AfgAuthoritySet(AfgAuthoritySet),
+    AfgFinalizedBlocksUpTo,
+    AuraPreSealedBlock,
+    PreparedBlockForProposing,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct SystemConnected {
+    pub genesis_hash: BlockHash,
+    pub node: NodeDetails,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct SystemInterval {
+    pub peers: Option<u64>,
+    pub txcount: Option<u64>,
+    pub bandwidth_upload: Option<f64>,
+    pub bandwidth_download: Option<f64>,
+    pub finalized_height: Option<BlockNumber>,
+    pub finalized_hash: Option<BlockHash>,
+    pub block: Option<Block>,
+    pub used_state_cache_size: Option<f32>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Finalized {
+    pub hash: BlockHash,
+    pub height: Box<str>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct AfgFinalized {
+    pub finalized_hash: BlockHash,
+    pub finalized_number: Box<str>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct AfgReceived {
+    pub target_hash: BlockHash,
+    pub target_number: Box<str>,
+    pub voter: Option<Box<str>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct AfgAuthoritySet {
+    pub authority_id: Box<str>,
+    pub authorities: Box<str>,
+    pub authority_set_id: Box<str>,
+}
+
+impl Payload {
+    pub fn best_block(&self) -> Option<&Block> {
+        match self {
+            Payload::BlockImport(block) => Some(block),
+            Payload::SystemInterval(SystemInterval { block, .. }) => block.as_ref(),
+            _ => None,
+        }
+    }
+
+    pub fn finalized_block(&self) -> Option<Block> {
+        match self {
+            Payload::SystemInterval(ref interval) => Some(Block {
+                hash: interval.finalized_hash?,
+                height: interval.finalized_height?,
+            }),
+            Payload::NotifyFinalized(ref finalized) => Some(Block {
+                hash: finalized.hash,
+                height: finalized.height.parse().ok()?,
+            }),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bincode::Options;
+
+    // Without adding a derive macro and marker trait (and enforcing their use), we don't really
+    // know whether things can (de)serialize to bincode or not at runtime without failing unless
+    // we test the different types we want to (de)serialize ourselves. We just need to test each
+    // type, not each variant.
+    fn bincode_can_serialize_and_deserialize<'de, T>(item: T)
+    where
+        T: Serialize + serde::de::DeserializeOwned,
+    {
+        let bytes = bincode::serialize(&item).expect("Serialization should work");
+        let _: T = bincode::deserialize(&bytes).expect("Deserialization should work");
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_system_connected() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::SystemConnected(SystemConnected {
+                genesis_hash: BlockHash::zero(),
+                node: NodeDetails {
+                    chain: "foo".into(),
+                    name: "foo".into(),
+                    implementation: "foo".into(),
+                    version: "foo".into(),
+                    validator: None,
+                    network_id: None,
+                    startup_time: None,
+                },
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_system_interval() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::SystemInterval(SystemInterval {
+                peers: None,
+                txcount: None,
+                bandwidth_upload: None,
+                bandwidth_download: None,
+                finalized_height: None,
+                finalized_hash: None,
+                block: None,
+                used_state_cache_size: None,
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_block_import() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::BlockImport(Block {
+                hash: BlockHash([0; 32]),
+                height: 0,
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_notify_finalized() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::NotifyFinalized(Finalized {
+                hash: BlockHash::zero(),
+                height: "foo".into(),
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_tx_pool_import() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::TxPoolImport,
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_afg_finalized() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::AfgFinalized(AfgFinalized {
+                finalized_hash: BlockHash::zero(),
+                finalized_number: "foo".into(),
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_afg_received() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::AfgReceivedPrecommit(AfgReceived {
+                target_hash: BlockHash::zero(),
+                target_number: "foo".into(),
+                voter: None,
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_afg_authority_set() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::AfgAuthoritySet(AfgAuthoritySet {
+                authority_id: "foo".into(),
+                authorities: "foo".into(),
+                authority_set_id: "foo".into(),
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_block_zero() {
+        let raw = Block::zero();
+
+        let bytes = bincode::options().serialize(&raw).unwrap();
+
+        let deserialized: Block = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(raw.hash, deserialized.hash);
+        assert_eq!(raw.height, deserialized.height);
+    }
+}

backend/common/src/node_types.rs

@@ -1,16 +1,17 @@
-use serde::ser::{Serialize, SerializeTuple, Serializer};
-use serde::Deserialize;
+//! These types are partly used in [`crate::node_message`], but also stored and used
+//! more generally through the application.
 
-use crate::util::{now, MeanList};
+use serde::ser::{SerializeTuple, Serializer};
+use serde::{Deserialize, Serialize};
+
+use crate::{time, MeanList};
 
-pub type NodeId = usize;
-pub type ConnId = u64;
 pub type BlockNumber = u64;
 pub type Timestamp = u64;
-pub type Address = Box<str>;
 pub use primitive_types::H256 as BlockHash;
 
-#[derive(Deserialize, Debug, Clone)]
+///
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct NodeDetails {
     pub chain: Box<str>,
     pub name: Box<str>,
@@ -21,75 +22,13 @@ pub struct NodeDetails {
     pub startup_time: Option<Box<str>>,
 }
 
-#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)]
+///
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub struct NodeStats {
     pub peers: u64,
     pub txcount: u64,
 }
 
-#[derive(Default)]
-pub struct NodeIO {
-    pub used_state_cache_size: MeanList<f32>,
-}
-
-#[derive(Deserialize, Debug, Clone, Copy)]
-pub struct Block {
-    #[serde(rename = "best")]
-    pub hash: BlockHash,
-    pub height: BlockNumber,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct BlockDetails {
-    pub block: Block,
-    pub block_time: u64,
-    pub block_timestamp: u64,
-    pub propagation_time: Option<u64>,
-}
-
-impl Default for BlockDetails {
-    fn default() -> Self {
-        BlockDetails {
-            block: Block::zero(),
-            block_timestamp: now(),
-            block_time: 0,
-            propagation_time: None,
-        }
-    }
-}
-
-#[derive(Default)]
-pub struct NodeHardware {
-    /// Upload uses means
-    pub upload: MeanList<f64>,
-    /// Download uses means
-    pub download: MeanList<f64>,
-    /// Stampchange uses means
-    pub chart_stamps: MeanList<f64>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct NodeLocation {
-    pub latitude: f32,
-    pub longitude: f32,
-    pub city: Box<str>,
-}
-
-impl Serialize for NodeDetails {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(6)?;
-        tup.serialize_element(&self.name)?;
-        tup.serialize_element(&self.implementation)?;
-        tup.serialize_element(&self.version)?;
-        tup.serialize_element(&self.validator)?;
-        tup.serialize_element(&self.network_id)?;
-        tup.end()
-    }
-}
-
 impl Serialize for NodeStats {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -102,17 +41,130 @@ impl Serialize for NodeStats {
     }
 }
 
+impl<'de> Deserialize<'de> for NodeStats {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let (peers, txcount) = <(u64, u64)>::deserialize(deserializer)?;
+        Ok(NodeStats { peers, txcount })
+    }
+}
+
+///
+#[derive(Default)]
+pub struct NodeIO {
+    pub used_state_cache_size: MeanList<f32>,
+}
+
 impl Serialize for NodeIO {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
         S: Serializer,
     {
         let mut tup = serializer.serialize_tuple(1)?;
+        // This is "one-way": we can't deserialize again from this to a MeanList:
         tup.serialize_element(self.used_state_cache_size.slice())?;
         tup.end()
     }
 }
 
+///
+#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)]
+pub struct Block {
+    pub hash: BlockHash,
+    pub height: BlockNumber,
+}
+
+impl Block {
+    pub fn zero() -> Self {
+        Block {
+            hash: BlockHash::from([0; 32]),
+            height: 0,
+        }
+    }
+}
+
+///
+#[derive(Default)]
+pub struct NodeHardware {
+    /// Upload uses means
+    pub upload: MeanList<f64>,
+    /// Download uses means
+    pub download: MeanList<f64>,
+    /// Stampchange uses means
+    pub chart_stamps: MeanList<f64>,
+}
+
+impl Serialize for NodeHardware {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(3)?;
+        // These are "one-way": we can't deserialize again from them to MeanLists:
+        tup.serialize_element(self.upload.slice())?;
+        tup.serialize_element(self.download.slice())?;
+        tup.serialize_element(self.chart_stamps.slice())?;
+        tup.end()
+    }
+}
+
+///
+#[derive(Debug, Clone, PartialEq)]
+pub struct NodeLocation {
+    pub latitude: f32,
+    pub longitude: f32,
+    pub city: Box<str>,
+}
+
+impl Serialize for NodeLocation {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(3)?;
+        tup.serialize_element(&self.latitude)?;
+        tup.serialize_element(&self.longitude)?;
+        tup.serialize_element(&&*self.city)?;
+        tup.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for NodeLocation {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let (latitude, longitude, city) = <(f32, f32, Box<str>)>::deserialize(deserializer)?;
+        Ok(NodeLocation {
+            latitude,
+            longitude,
+            city,
+        })
+    }
+}
+
+///
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct BlockDetails {
+    pub block: Block,
+    pub block_time: u64,
+    pub block_timestamp: u64,
+    pub propagation_time: Option<u64>,
+}
+
+impl Default for BlockDetails {
+    fn default() -> Self {
+        BlockDetails {
+            block: Block::zero(),
+            block_timestamp: time::now(),
+            block_time: 0,
+            propagation_time: None,
+        }
+    }
+}
+
 impl Serialize for BlockDetails {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -128,28 +180,20 @@ impl Serialize for BlockDetails {
     }
 }
 
-impl Serialize for NodeLocation {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+impl<'de> Deserialize<'de> for BlockDetails {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
-        S: Serializer,
+        D: serde::Deserializer<'de>,
     {
-        let mut tup = serializer.serialize_tuple(3)?;
-        tup.serialize_element(&self.latitude)?;
-        tup.serialize_element(&self.longitude)?;
-        tup.serialize_element(&&*self.city)?;
-        tup.end()
-    }
-}
-
-impl Serialize for NodeHardware {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(3)?;
-        tup.serialize_element(self.upload.slice())?;
-        tup.serialize_element(self.download.slice())?;
-        tup.serialize_element(self.chart_stamps.slice())?;
-        tup.end()
+        let tup = <(u64, BlockHash, u64, u64, Option<u64>)>::deserialize(deserializer)?;
+        Ok(BlockDetails {
+            block: Block {
+                height: tup.0,
+                hash: tup.1,
+            },
+            block_time: tup.2,
+            block_timestamp: tup.3,
+            propagation_time: tup.4,
+        })
     }
 }

backend/common/src/ready_chunks_all.rs

@@ -0,0 +1,105 @@
+//! [`futures::StreamExt::ready_chunks()`] internally stores a vec with a certain capacity, and will buffer up
+//! up to that many items that are ready from the underlying stream before returning either when we run out of
+//! Poll::Ready items, or we hit the capacity.
+//!
+//! This variation has no fixed capacity, and will buffer everything it can up at each point to return. This is
+//! better when the amount of items varies a bunch (and we don't want to allocate a fixed capacity every time),
+//! and can help ensure that we process as many items as possible each time (rather than only up to capacity items).
+//!
+//! Code is adapted from the futures implementation
+//! (see [ready_chunks.rs](https://docs.rs/futures-util/0.3.15/src/futures_util/stream/stream/ready_chunks.rs.html)).
+
+use futures::stream::Fuse;
+use futures::StreamExt;
+use core::mem;
+use core::pin::Pin;
+use futures::stream::{FusedStream, Stream};
+use futures::task::{Context, Poll};
+use pin_project_lite::pin_project;
+
+pin_project! {
+    /// Buffer up all Ready items in the underlying stream each time
+    /// we attempt to retrieve items from it, and return a Vec of those
+    /// items.
+    #[derive(Debug)]
+    #[must_use = "streams do nothing unless polled"]
+    pub struct ReadyChunksAll<St: Stream> {
+        #[pin]
+        stream: Fuse<St>,
+        items: Vec<St::Item>,
+    }
+}
+
+impl<St: Stream> ReadyChunksAll<St>
+where
+    St: Stream,
+{
+    pub fn new(stream: St) -> Self {
+        Self {
+            stream: stream.fuse(),
+            items: Vec::new()
+        }
+    }
+}
+
+impl<St: Stream> Stream for ReadyChunksAll<St> {
+    type Item = Vec<St::Item>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let mut this = self.project();
+
+        loop {
+            match this.stream.as_mut().poll_next(cx) {
+                // Flush all collected data if underlying stream doesn't contain
+                // more ready values
+                Poll::Pending => {
+                    return if this.items.is_empty() {
+                        Poll::Pending
+                    } else {
+                        Poll::Ready(Some(mem::replace(this.items, Vec::new())))
+                    }
+                }
+
+                // Push the ready item into the buffer
+                Poll::Ready(Some(item)) => {
+                    this.items.push(item);
+                }
+
+                // Since the underlying stream ran out of values, return what we
+                // have buffered, if we have anything.
+                Poll::Ready(None) => {
+                    let last = if this.items.is_empty() {
+                        None
+                    } else {
+                        let full_buf = mem::replace(this.items, Vec::new());
+                        Some(full_buf)
+                    };
+
+                    return Poll::Ready(last);
+                }
+            }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        // Look at the underlying stream's size_hint. If we've
+        // buffered some items, we'll return at least that Vec,
+        // giving us a lower bound 1 greater than the underlying.
+        // The upper bound is, worst case, our vec + each individual
+        // item in the underlying stream.
+        let chunk_len = if self.items.is_empty() { 0 } else { 1 };
+        let (lower, upper) = self.stream.size_hint();
+        let lower = lower.saturating_add(chunk_len);
+        let upper = match upper {
+            Some(x) => x.checked_add(chunk_len),
+            None => None,
+        };
+        (lower, upper)
+    }
+}
+
+impl<St: FusedStream> FusedStream for ReadyChunksAll<St> {
+    fn is_terminated(&self) -> bool {
+        self.stream.is_terminated() && self.items.is_empty()
+    }
+}

backend/common/src/time.rs

@@ -0,0 +1,9 @@
+/// Returns current unix time in ms (compatible with JS Date.now())
+pub fn now() -> u64 {
+    use std::time::SystemTime;
+
+    SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .expect("System time must be configured to be post Unix Epoch start; qed")
+        .as_millis() as u64
+}

backend/common/src/ws_client.rs

@@ -0,0 +1,274 @@
+use futures::channel::mpsc;
+use futures::{Sink, SinkExt, Stream, StreamExt};
+use soketto::handshake::{Client, ServerResponse};
+use tokio::net::TcpStream;
+use tokio_util::compat::TokioAsyncReadCompatExt;
+
+/// Send messages into the connection
+#[derive(Clone)]
+pub struct Sender {
+    inner: mpsc::UnboundedSender<SentMessageInternal>,
+}
+
+impl Sender {
+    /// Ask the underlying Websocket connection to close.
+    pub async fn close(&mut self) -> Result<(), SendError> {
+        self.inner.send(SentMessageInternal::Close).await?;
+        Ok(())
+    }
+    /// Returns whether this channel is closed.
+    pub fn is_closed(&mut self) -> bool {
+        self.inner.is_closed()
+    }
+    /// Unbounded send will always queue the message and doesn't
+    /// need to be awaited.
+    pub fn unbounded_send(&self, msg: SentMessage) -> Result<(), SendError> {
+        self.inner
+            .unbounded_send(SentMessageInternal::Message(msg))
+            .map_err(|e| e.into_send_error())?;
+        Ok(())
+    }
+}
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum SendError {
+    #[error("Failed to send message: {0}")]
+    ChannelError(#[from] mpsc::SendError)
+}
+
+impl Sink<SentMessage> for Sender {
+    type Error = SendError;
+    fn poll_ready(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready_unpin(cx).map_err(|e| e.into())
+    }
+    fn start_send(mut self: std::pin::Pin<&mut Self>, item: SentMessage) -> Result<(), Self::Error> {
+        self.inner
+            .start_send_unpin(SentMessageInternal::Message(item))
+            .map_err(|e| e.into())
+    }
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.inner.poll_flush_unpin(cx).map_err(|e| e.into())
+    }
+    fn poll_close(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.inner.poll_close_unpin(cx).map_err(|e| e.into())
+    }
+}
+
+/// Receive messages out of a connection
+pub struct Receiver {
+    inner: mpsc::UnboundedReceiver<Result<RecvMessage, RecvError>>,
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum RecvError {
+    #[error("Text message contains invalid UTF8: {0}")]
+    InvalidUtf8(#[from] std::string::FromUtf8Error),
+    #[error("Stream finished")]
+    StreamFinished,
+}
+
+impl Stream for Receiver {
+    type Item = Result<RecvMessage, RecvError>;
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        self.inner.poll_next_unpin(cx).map_err(|e| e.into())
+    }
+}
+
+/// A message that can be received from the connection
+#[derive(Debug, Clone)]
+pub enum RecvMessage {
+    /// Send an owned string into the socket.
+    Text(String),
+    /// Send owned bytes into the socket.
+    Binary(Vec<u8>),
+}
+
+impl RecvMessage {
+    pub fn len(&self) -> usize {
+        match self {
+            RecvMessage::Binary(b) => b.len(),
+            RecvMessage::Text(s) => s.len(),
+        }
+    }
+}
+
+/// A message that can be sent into the connection
+#[derive(Debug, Clone)]
+pub enum SentMessage {
+    /// Being able to send static text is primarily useful for benchmarking,
+    /// so that we can avoid cloning an owned string and pass a static reference
+    /// (one such option here is using [`Box::leak`] to generate strings with
+    /// static lifetimes).
+    StaticText(&'static str),
+    /// Being able to send static bytes is primarily useful for benchmarking,
+    /// so that we can avoid cloning an owned string and pass a static reference
+    /// (one such option here is using [`Box::leak`] to generate bytes with
+    /// static lifetimes).
+    StaticBinary(&'static [u8]),
+    /// Send an owned string into the socket.
+    Text(String),
+    /// Send owned bytes into the socket.
+    Binary(Vec<u8>),
+}
+
+/// Sent messages can be anything publically visible, or a close message.
+#[derive(Debug, Clone)]
+enum SentMessageInternal {
+    Message(SentMessage),
+    Close,
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum ConnectError {
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("Handshake error: {0}")]
+    Handshake(#[from] soketto::handshake::Error),
+    #[error("Redirect not supported (status code: {status_code})")]
+    ConnectionFailedRedirect { status_code: u16 },
+    #[error("Connection rejected (status code: {status_code})")]
+    ConnectionFailedRejected { status_code: u16 },
+}
+
+/// Establish a websocket connection that you can send and receive messages from.
+/// A thin wrapper around Soketto that provides cancel-safe send/receive handles.
+///
+/// This must be called within the context of a tokio runtime.
+pub async fn connect(uri: &http::Uri) -> Result<(Sender, Receiver), ConnectError> {
+    let host = uri.host().unwrap_or("127.0.0.1");
+    let port = uri.port_u16().unwrap_or(80);
+    let path = uri.path();
+
+    let socket = TcpStream::connect((host, port)).await?;
+    socket.set_nodelay(true).expect("socket set_nodelay failed");
+
+    // Establish a WS connection:
+    let mut client = Client::new(socket.compat(), host, &path);
+    let (mut ws_to_connection, mut ws_from_connection) = match client.handshake().await? {
+        ServerResponse::Accepted { .. } => client.into_builder().finish(),
+        ServerResponse::Redirect { status_code, .. } => {
+            return Err(ConnectError::ConnectionFailedRedirect { status_code })
+        }
+        ServerResponse::Rejected { status_code } => {
+            return Err(ConnectError::ConnectionFailedRejected { status_code })
+        }
+    };
+
+    // Soketto sending/receiving isn't cancel safe, so we wrap the message stuff into spawned
+    // tasks and use channels (which are cancel safe) to send/recv messages atomically..
+
+    // Receive messages from the socket and post them out:
+    let (mut tx_to_external, rx_from_ws) = mpsc::unbounded();
+    tokio::spawn(async move {
+        let mut data = Vec::with_capacity(128);
+        loop {
+            // Clear the buffer and wait for the next message to arrive:
+            data.clear();
+
+            let message_data = match ws_from_connection.receive_data(&mut data).await {
+                Err(e) => {
+                    // Couldn't receive data may mean all senders are gone, so log
+                    // the error and shut this down:
+                    log::error!(
+                        "Shutting down websocket connection: Failed to receive data: {}",
+                        e
+                    );
+                    break;
+                }
+                Ok(data) => data,
+            };
+
+            let msg = match message_data {
+                soketto::Data::Text(_) => Ok(RecvMessage::Binary(data)),
+                soketto::Data::Binary(_) => String::from_utf8(data)
+                    .map(|s| RecvMessage::Text(s))
+                    .map_err(|e| e.into()),
+            };
+
+            data = Vec::with_capacity(128);
+
+            if let Err(e) = tx_to_external.send(msg).await {
+                // Failure to send likely means that the recv has been dropped,
+                // so let's drop this loop too.
+                log::error!(
+                    "Shutting down websocket connection: Failed to send data out: {}",
+                    e
+                );
+                break;
+            }
+        }
+    });
+
+    // Receive messages externally to send to the socket.
+    let (tx_to_ws, mut rx_from_external) = mpsc::unbounded();
+    tokio::spawn(async move {
+        while let Some(msg) = rx_from_external.next().await {
+            match msg {
+                SentMessageInternal::Message(SentMessage::Text(s)) => {
+                    if let Err(e) = ws_to_connection.send_text_owned(s).await {
+                        log::error!(
+                            "Shutting down websocket connection: Failed to send text data: {}",
+                            e
+                        );
+                        break;
+                    }
+                }
+                SentMessageInternal::Message(SentMessage::Binary(bytes)) => {
+                    if let Err(e) = ws_to_connection.send_binary_mut(bytes).await {
+                        log::error!(
+                            "Shutting down websocket connection: Failed to send binary data: {}",
+                            e
+                        );
+                        break;
+                    }
+                },
+                SentMessageInternal::Message(SentMessage::StaticText(s)) => {
+                    if let Err(e) = ws_to_connection.send_text(s).await {
+                        log::error!(
+                            "Shutting down websocket connection: Failed to send text data: {}",
+                            e
+                        );
+                        break;
+                    }
+                }
+                SentMessageInternal::Message(SentMessage::StaticBinary(bytes)) => {
+                    if let Err(e) = ws_to_connection.send_binary(bytes).await {
+                        log::error!(
+                            "Shutting down websocket connection: Failed to send binary data: {}",
+                            e
+                        );
+                        break;
+                    }
+                },
+                SentMessageInternal::Close => {
+                    if let Err(e) = ws_to_connection.close().await {
+                        log::error!("Error attempting to close connection: {}", e);
+                        break;
+                    }
+                }
+            }
+
+            if let Err(e) = ws_to_connection.flush().await {
+                log::error!(
+                    "Shutting down websocket connection: Failed to flush data: {}",
+                    e
+                );
+                break;
+            }
+        }
+    });
+
+    Ok((Sender { inner: tx_to_ws }, Receiver { inner: rx_from_ws }))
+}

backend/core/src/aggregator.rs

@@ -1,383 +0,0 @@
-use actix::prelude::*;
-use actix_web_actors::ws::{CloseCode, CloseReason};
-use ctor::ctor;
-use std::collections::{HashMap, HashSet};
-
-use crate::chain::{self, Chain, ChainId, Label};
-use crate::feed::connector::{Connected, FeedConnector, FeedId};
-use crate::feed::{self, FeedMessageSerializer};
-use crate::node::connector::{Mute, NodeConnector};
-use crate::types::{ConnId, NodeDetails};
-use crate::util::{DenseMap, Hash};
-
-pub struct Aggregator {
-    genesis_hashes: HashMap<Hash, ChainId>,
-    labels: HashMap<Label, ChainId>,
-    chains: DenseMap<ChainEntry>,
-    feeds: DenseMap<Addr<FeedConnector>>,
-    serializer: FeedMessageSerializer,
-    /// Denylist for networks we do not want to allow connecting.
-    denylist: HashSet<String>,
-}
-
-pub struct ChainEntry {
-    /// Address to the `Chain` agent
-    addr: Addr<Chain>,
-    /// Genesis [`Hash`] of the chain
-    genesis_hash: Hash,
-    /// String name of the chain
-    label: Label,
-    /// Node count
-    nodes: usize,
-    /// Maximum allowed nodes
-    max_nodes: usize,
-}
-
-#[ctor]
-/// Labels of chains we consider "first party". These chains allow any
-/// number of nodes to connect.
-static FIRST_PARTY_NETWORKS: HashSet<&'static str> = {
-    let mut set = HashSet::new();
-    set.insert("Polkadot");
-    set.insert("Kusama");
-    set.insert("Westend");
-    set.insert("Rococo");
-    set
-};
-
-/// Max number of nodes allowed to connect to the telemetry server.
-const THIRD_PARTY_NETWORKS_MAX_NODES: usize = 500;
-
-impl Aggregator {
-    pub fn new(denylist: HashSet<String>) -> Self {
-        Aggregator {
-            genesis_hashes: HashMap::new(),
-            labels: HashMap::new(),
-            chains: DenseMap::new(),
-            feeds: DenseMap::new(),
-            serializer: FeedMessageSerializer::new(),
-            denylist,
-        }
-    }
-
-    /// Get an address to the chain actor by name. If the address is not found,
-    /// or the address is disconnected (actor dropped), create a new one.
-    pub fn lazy_chain(
-        &mut self,
-        genesis_hash: Hash,
-        label: &str,
-        ctx: &mut <Self as Actor>::Context,
-    ) -> ChainId {
-        let cid = match self.genesis_hashes.get(&genesis_hash).copied() {
-            Some(cid) => cid,
-            None => {
-                self.serializer.push(feed::AddedChain(&label, 1));
-
-                let addr = ctx.address();
-                let max_nodes = max_nodes(label);
-                let label: Label = label.into();
-                let cid = self.chains.add_with(|cid| ChainEntry {
-                    addr: Chain::new(cid, addr, label.clone()).start(),
-                    genesis_hash,
-                    label: label.clone(),
-                    nodes: 1,
-                    max_nodes,
-                });
-
-                self.labels.insert(label, cid);
-                self.genesis_hashes.insert(genesis_hash, cid);
-
-                self.broadcast();
-
-                cid
-            }
-        };
-
-        cid
-    }
-
-    fn get_chain(&mut self, label: &str) -> Option<&mut ChainEntry> {
-        let chains = &mut self.chains;
-        self.labels
-            .get(label)
-            .and_then(move |&cid| chains.get_mut(cid))
-    }
-
-    fn broadcast(&mut self) {
-        if let Some(msg) = self.serializer.finalize() {
-            for (_, feed) in self.feeds.iter() {
-                feed.do_send(msg.clone());
-            }
-        }
-    }
-}
-
-impl Actor for Aggregator {
-    type Context = Context<Self>;
-}
-
-/// Message sent from the NodeConnector to the Aggregator upon getting all node details
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct AddNode {
-    /// Details of the node being added to the aggregator
-    pub node: NodeDetails,
-    /// Genesis [`Hash`] of the chain the node is being added to.
-    pub genesis_hash: Hash,
-    /// Connection id used by the node connector for multiplexing parachains
-    pub conn_id: ConnId,
-    /// Address of the NodeConnector actor
-    pub node_connector: Addr<NodeConnector>,
-}
-
-/// Message sent from the Chain to the Aggregator when the Chain loses all nodes
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct DropChain(pub ChainId);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct RenameChain(pub ChainId, pub Label);
-
-/// Message sent from the FeedConnector to the Aggregator when subscribing to a new chain
-#[derive(Message)]
-#[rtype(result = "bool")]
-pub struct Subscribe {
-    pub chain: Label,
-    pub feed: Addr<FeedConnector>,
-}
-
-/// Message sent from the FeedConnector to the Aggregator consensus requested
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct SendFinality {
-    pub chain: Label,
-    pub fid: FeedId,
-}
-
-/// Message sent from the FeedConnector to the Aggregator no more consensus required
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct NoMoreFinality {
-    pub chain: Label,
-    pub fid: FeedId,
-}
-
-/// Message sent from the FeedConnector to the Aggregator when first connected
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Connect(pub Addr<FeedConnector>);
-
-/// Message sent from the FeedConnector to the Aggregator when disconnecting
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Disconnect(pub FeedId);
-
-/// Message sent from the Chain to the Aggergator when the node count on the chain changes
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct NodeCount(pub ChainId, pub usize);
-
-/// Message sent to the Aggregator to get a health check
-#[derive(Message)]
-#[rtype(result = "usize")]
-pub struct GetHealth;
-
-impl Handler<AddNode> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
-        if self.denylist.contains(&*msg.node.chain) {
-            log::warn!(target: "Aggregator::AddNode", "'{}' is on the denylist.", msg.node.chain);
-            let AddNode { node_connector, .. } = msg;
-            let reason = CloseReason {
-                code: CloseCode::Abnormal,
-                description: Some("Denied".into()),
-            };
-            node_connector.do_send(Mute { reason });
-            return;
-        }
-        let AddNode {
-            node,
-            genesis_hash,
-            conn_id,
-            node_connector,
-        } = msg;
-        log::trace!(target: "Aggregator::AddNode", "New node connected. Chain '{}'", node.chain);
-
-        let cid = self.lazy_chain(genesis_hash, &node.chain, ctx);
-        let chain = self
-            .chains
-            .get_mut(cid)
-            .expect("Entry just created above; qed");
-        if chain.nodes < chain.max_nodes {
-            chain.addr.do_send(chain::AddNode {
-                node,
-                conn_id,
-                node_connector,
-            });
-        } else {
-            log::warn!(target: "Aggregator::AddNode", "Chain {} is over quota ({})", chain.label, chain.max_nodes);
-            let reason = CloseReason {
-                code: CloseCode::Again,
-                description: Some("Overquota".into()),
-            };
-            node_connector.do_send(Mute { reason });
-        }
-    }
-}
-
-impl Handler<DropChain> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: DropChain, _: &mut Self::Context) {
-        let DropChain(cid) = msg;
-
-        if let Some(entry) = self.chains.remove(cid) {
-            let label = &entry.label;
-            self.genesis_hashes.remove(&entry.genesis_hash);
-            self.labels.remove(label);
-            self.serializer.push(feed::RemovedChain(label));
-            log::info!("Dropped chain [{}] from the aggregator", label);
-            self.broadcast();
-        }
-    }
-}
-
-impl Handler<RenameChain> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: RenameChain, _: &mut Self::Context) {
-        let RenameChain(cid, new) = msg;
-
-        if let Some(entry) = self.chains.get_mut(cid) {
-            if entry.label == new {
-                return;
-            }
-
-            // Update UI
-            self.serializer.push(feed::RemovedChain(&entry.label));
-            self.serializer.push(feed::AddedChain(&new, entry.nodes));
-
-            // Update labels -> cid map
-            self.labels.remove(&entry.label);
-            self.labels.insert(new.clone(), cid);
-
-            // Update entry
-            entry.label = new;
-
-            self.broadcast();
-        }
-    }
-}
-
-impl Handler<Subscribe> for Aggregator {
-    type Result = bool;
-
-    fn handle(&mut self, msg: Subscribe, _: &mut Self::Context) -> bool {
-        let Subscribe { chain, feed } = msg;
-
-        if let Some(chain) = self.get_chain(&chain) {
-            chain.addr.do_send(chain::Subscribe(feed));
-            true
-        } else {
-            false
-        }
-    }
-}
-
-impl Handler<SendFinality> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: SendFinality, _: &mut Self::Context) {
-        let SendFinality { chain, fid } = msg;
-        if let Some(chain) = self.get_chain(&chain) {
-            chain.addr.do_send(chain::SendFinality(fid));
-        }
-    }
-}
-
-impl Handler<NoMoreFinality> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: NoMoreFinality, _: &mut Self::Context) {
-        let NoMoreFinality { chain, fid } = msg;
-        if let Some(chain) = self.get_chain(&chain) {
-            chain.addr.do_send(chain::NoMoreFinality(fid));
-        }
-    }
-}
-
-impl Handler<Connect> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: Connect, _: &mut Self::Context) {
-        let Connect(connector) = msg;
-
-        let fid = self.feeds.add(connector.clone());
-
-        log::info!("Feed #{} connected", fid);
-
-        connector.do_send(Connected(fid));
-
-        self.serializer.push(feed::Version(31));
-
-        // TODO: keep track on number of nodes connected to each chain
-        for (_, entry) in self.chains.iter() {
-            self.serializer
-                .push(feed::AddedChain(&entry.label, entry.nodes));
-        }
-
-        if let Some(msg) = self.serializer.finalize() {
-            connector.do_send(msg);
-        }
-    }
-}
-
-impl Handler<Disconnect> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: Disconnect, _: &mut Self::Context) {
-        let Disconnect(fid) = msg;
-
-        log::info!("Feed #{} disconnected", fid);
-
-        self.feeds.remove(fid);
-    }
-}
-
-impl Handler<NodeCount> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: NodeCount, _: &mut Self::Context) {
-        let NodeCount(cid, count) = msg;
-
-        if let Some(entry) = self.chains.get_mut(cid) {
-            entry.nodes = count;
-
-            if count != 0 {
-                self.serializer.push(feed::AddedChain(&entry.label, count));
-                self.broadcast();
-            }
-        }
-    }
-}
-
-impl Handler<GetHealth> for Aggregator {
-    type Result = usize;
-
-    fn handle(&mut self, _: GetHealth, _: &mut Self::Context) -> Self::Result {
-        self.chains.len()
-    }
-}
-
-/// First party networks (Polkadot, Kusama etc) are allowed any number of nodes.
-/// Third party networks are allowed `THIRD_PARTY_NETWORKS_MAX_NODES` nodes and
-/// no more.
-fn max_nodes(label: &str) -> usize {
-    if FIRST_PARTY_NETWORKS.contains(label) {
-        usize::MAX
-    } else {
-        THIRD_PARTY_NETWORKS_MAX_NODES
-    }
-}

backend/core/src/chain.rs

@@ -1,564 +0,0 @@
-use actix::prelude::*;
-use rustc_hash::FxHashMap;
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use crate::aggregator::{Aggregator, DropChain, NodeCount, RenameChain};
-use crate::feed::connector::{FeedConnector, FeedId, Subscribed, Unsubscribed};
-use crate::feed::{self, FeedMessageSerializer};
-use crate::node::{
-    connector::{Initialize, NodeConnector},
-    message::Payload,
-    Node,
-};
-use crate::types::{Block, BlockNumber, ConnId, NodeDetails, NodeId, NodeLocation, Timestamp};
-use crate::util::{now, DenseMap, NumStats};
-
-const STALE_TIMEOUT: u64 = 2 * 60 * 1000; // 2 minutes
-
-pub type ChainId = usize;
-pub type Label = Arc<str>;
-
-pub struct Chain {
-    cid: ChainId,
-    /// Who to inform if the Chain drops itself
-    aggregator: Addr<Aggregator>,
-    /// Label of this chain, along with count of nodes that use this label
-    label: (Label, usize),
-    /// Dense mapping of NodeId -> Node
-    nodes: DenseMap<Node>,
-    /// Dense mapping of FeedId -> Addr<FeedConnector>,
-    feeds: DenseMap<Addr<FeedConnector>>,
-    /// Mapping of FeedId -> Addr<FeedConnector> for feeds requiring finality info,
-    finality_feeds: FxHashMap<FeedId, Addr<FeedConnector>>,
-    /// Best block
-    best: Block,
-    /// Finalized block
-    finalized: Block,
-    /// Block times history, stored so we can calculate averages
-    block_times: NumStats<u64>,
-    /// Calculated average block time
-    average_block_time: Option<u64>,
-    /// Message serializer
-    serializer: FeedMessageSerializer,
-    /// When the best block first arrived
-    timestamp: Option<Timestamp>,
-    /// Some nodes might manifest a different label, note them here
-    labels: HashMap<Label, usize>,
-}
-
-impl Chain {
-    pub fn new(cid: ChainId, aggregator: Addr<Aggregator>, label: Label) -> Self {
-        log::info!("[{}] Created", label);
-
-        Chain {
-            cid,
-            aggregator,
-            label: (label, 0),
-            nodes: DenseMap::new(),
-            feeds: DenseMap::new(),
-            finality_feeds: FxHashMap::default(),
-            best: Block::zero(),
-            finalized: Block::zero(),
-            block_times: NumStats::new(50),
-            average_block_time: None,
-            serializer: FeedMessageSerializer::new(),
-            timestamp: None,
-            labels: HashMap::default(),
-        }
-    }
-
-    fn increment_label_count(&mut self, label: &str) {
-        let count = match self.labels.get_mut(label) {
-            Some(count) => {
-                *count += 1;
-                *count
-            }
-            None => {
-                self.labels.insert(label.into(), 1);
-                1
-            }
-        };
-
-        if &*self.label.0 == label {
-            self.label.1 += 1;
-        } else if count > self.label.1 {
-            self.rename(label.into(), count);
-        }
-    }
-
-    fn decrement_label_count(&mut self, label: &str) {
-        match self.labels.get_mut(label) {
-            Some(count) => *count -= 1,
-            None => return,
-        };
-
-        if &*self.label.0 == label {
-            self.label.1 -= 1;
-
-            for (label, &count) in self.labels.iter() {
-                if count > self.label.1 {
-                    let label: Arc<_> = label.clone();
-                    self.rename(label, count);
-                    break;
-                }
-            }
-        }
-    }
-
-    fn rename(&mut self, label: Label, count: usize) {
-        self.label = (label, count);
-
-        self.aggregator
-            .do_send(RenameChain(self.cid, self.label.0.clone()));
-    }
-
-    fn broadcast(&mut self) {
-        if let Some(msg) = self.serializer.finalize() {
-            for (_, feed) in self.feeds.iter() {
-                feed.do_send(msg.clone());
-            }
-        }
-    }
-
-    fn broadcast_finality(&mut self) {
-        if let Some(msg) = self.serializer.finalize() {
-            for feed in self.finality_feeds.values() {
-                feed.do_send(msg.clone());
-            }
-        }
-    }
-
-    /// Triggered when the number of nodes in this chain has changed, Aggregator will
-    /// propagate new counts to all connected feeds
-    fn update_count(&self) {
-        self.aggregator
-            .do_send(NodeCount(self.cid, self.nodes.len()));
-    }
-
-    /// Check if the chain is stale (has not received a new best block in a while).
-    /// If so, find a new best block, ignoring any stale nodes and marking them as such.
-    fn update_stale_nodes(&mut self, now: u64) {
-        let threshold = now - STALE_TIMEOUT;
-        let timestamp = match self.timestamp {
-            Some(ts) => ts,
-            None => return,
-        };
-
-        if timestamp > threshold {
-            // Timestamp is in range, nothing to do
-            return;
-        }
-
-        let mut best = Block::zero();
-        let mut finalized = Block::zero();
-        let mut timestamp = None;
-
-        for (nid, node) in self.nodes.iter_mut() {
-            if !node.update_stale(threshold) {
-                if node.best().height > best.height {
-                    best = *node.best();
-                    timestamp = Some(node.best_timestamp());
-                }
-
-                if node.finalized().height > finalized.height {
-                    finalized = *node.finalized();
-                }
-            } else {
-                self.serializer.push(feed::StaleNode(nid));
-            }
-        }
-
-        if self.best.height != 0 || self.finalized.height != 0 {
-            self.best = best;
-            self.finalized = finalized;
-            self.block_times.reset();
-            self.timestamp = timestamp;
-
-            self.serializer.push(feed::BestBlock(
-                self.best.height,
-                timestamp.unwrap_or(now),
-                None,
-            ));
-            self.serializer
-                .push(feed::BestFinalized(finalized.height, finalized.hash));
-        }
-    }
-}
-
-impl Actor for Chain {
-    type Context = Context<Self>;
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        self.aggregator.do_send(DropChain(self.cid));
-
-        for (_, feed) in self.feeds.iter() {
-            feed.do_send(Unsubscribed)
-        }
-    }
-}
-
-/// Message sent from the Aggregator to the Chain when new Node is connected
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct AddNode {
-    /// Details of the node being added to the aggregator
-    pub node: NodeDetails,
-    /// Connection id used by the node connector for multiplexing parachains
-    pub conn_id: ConnId,
-    /// Address of the NodeConnector actor to which we send [`Initialize`] or [`Mute`] messages.
-    pub node_connector: Addr<NodeConnector>,
-}
-
-/// Message sent from the NodeConnector to the Chain when it receives new telemetry data
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct UpdateNode {
-    pub nid: NodeId,
-    pub payload: Payload,
-}
-
-/// Message sent from the NodeConnector to the Chain when the connector disconnects
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct RemoveNode(pub NodeId);
-
-/// Message sent from the Aggregator to the Chain when the connector wants to subscribe to that chain
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Subscribe(pub Addr<FeedConnector>);
-
-/// Message sent from the FeedConnector before it subscribes to a new chain, or if it disconnects
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Unsubscribe(pub FeedId);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct SendFinality(pub FeedId);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct NoMoreFinality(pub FeedId);
-
-/// Message sent from the NodeConnector to the Chain when it receives location data
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct LocateNode {
-    pub nid: NodeId,
-    pub location: Arc<NodeLocation>,
-}
-
-impl Handler<AddNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
-        let AddNode {
-            node,
-            conn_id,
-            node_connector,
-        } = msg;
-        log::trace!(target: "Chain::AddNode", "New node connected. Chain '{}', node count goes from {} to {}", node.chain, self.nodes.len(), self.nodes.len() + 1);
-        self.increment_label_count(&node.chain);
-
-        let nid = self.nodes.add(Node::new(node));
-        let chain = ctx.address();
-
-        if node_connector
-            .try_send(Initialize {
-                nid,
-                conn_id,
-                chain,
-            })
-            .is_err()
-        {
-            self.nodes.remove(nid);
-        } else if let Some(node) = self.nodes.get(nid) {
-            self.serializer.push(feed::AddedNode(nid, node));
-            self.broadcast();
-        }
-
-        self.update_count();
-    }
-}
-
-impl Chain {
-    fn handle_block(&mut self, block: &Block, nid: NodeId) {
-        let mut propagation_time = None;
-        let now = now();
-        let nodes_len = self.nodes.len();
-
-        self.update_stale_nodes(now);
-
-        let node = match self.nodes.get_mut(nid) {
-            Some(node) => node,
-            None => return,
-        };
-
-        if node.update_block(*block) {
-            if block.height > self.best.height {
-                self.best = *block;
-                log::debug!(
-                    "[{}] [nodes={}/feeds={}] new best block={}/{:?}",
-                    self.label.0,
-                    nodes_len,
-                    self.feeds.len(),
-                    self.best.height,
-                    self.best.hash,
-                );
-                if let Some(timestamp) = self.timestamp {
-                    self.block_times.push(now - timestamp);
-                    self.average_block_time = Some(self.block_times.average());
-                }
-                self.timestamp = Some(now);
-                self.serializer.push(feed::BestBlock(
-                    self.best.height,
-                    now,
-                    self.average_block_time,
-                ));
-                propagation_time = Some(0);
-            } else if block.height == self.best.height {
-                if let Some(timestamp) = self.timestamp {
-                    propagation_time = Some(now - timestamp);
-                }
-            }
-
-            if let Some(details) = node.update_details(now, propagation_time) {
-                self.serializer.push(feed::ImportedBlock(nid, details));
-            }
-        }
-    }
-}
-
-impl Handler<UpdateNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: UpdateNode, _: &mut Self::Context) {
-        let UpdateNode { nid, payload } = msg;
-
-        if let Some(block) = payload.best_block() {
-            self.handle_block(block, nid);
-        }
-
-        if let Some(node) = self.nodes.get_mut(nid) {
-            match payload {
-                Payload::SystemInterval(ref interval) => {
-                    if node.update_hardware(interval) {
-                        self.serializer.push(feed::Hardware(nid, node.hardware()));
-                    }
-
-                    if let Some(stats) = node.update_stats(interval) {
-                        self.serializer.push(feed::NodeStatsUpdate(nid, stats));
-                    }
-
-                    if let Some(io) = node.update_io(interval) {
-                        self.serializer.push(feed::NodeIOUpdate(nid, io));
-                    }
-                }
-                Payload::AfgAuthoritySet(authority) => {
-                    node.set_validator_address(authority.authority_id.clone());
-                    self.broadcast();
-                    return;
-                }
-                Payload::AfgFinalized(finalized) => {
-                    if let Ok(finalized_number) = finalized.finalized_number.parse::<BlockNumber>()
-                    {
-                        if let Some(addr) = node.details().validator.clone() {
-                            self.serializer.push(feed::AfgFinalized(
-                                addr,
-                                finalized_number,
-                                finalized.finalized_hash,
-                            ));
-                            self.broadcast_finality();
-                        }
-                    }
-                    return;
-                }
-                Payload::AfgReceivedPrecommit(precommit) => {
-                    if let Ok(finalized_number) =
-                        precommit.received.target_number.parse::<BlockNumber>()
-                    {
-                        if let Some(addr) = node.details().validator.clone() {
-                            let voter = precommit.received.voter.clone();
-                            self.serializer.push(feed::AfgReceivedPrecommit(
-                                addr,
-                                finalized_number,
-                                precommit.received.target_hash,
-                                voter,
-                            ));
-                            self.broadcast_finality();
-                        }
-                    }
-                    return;
-                }
-                Payload::AfgReceivedPrevote(prevote) => {
-                    if let Ok(finalized_number) =
-                        prevote.received.target_number.parse::<BlockNumber>()
-                    {
-                        if let Some(addr) = node.details().validator.clone() {
-                            let voter = prevote.received.voter.clone();
-                            self.serializer.push(feed::AfgReceivedPrevote(
-                                addr,
-                                finalized_number,
-                                prevote.received.target_hash,
-                                voter,
-                            ));
-                            self.broadcast_finality();
-                        }
-                    }
-                    return;
-                }
-                Payload::AfgReceivedCommit(_) => {}
-                _ => (),
-            }
-
-            if let Some(block) = payload.finalized_block() {
-                if let Some(finalized) = node.update_finalized(block) {
-                    self.serializer.push(feed::FinalizedBlock(
-                        nid,
-                        finalized.height,
-                        finalized.hash,
-                    ));
-
-                    if finalized.height > self.finalized.height {
-                        self.finalized = *finalized;
-                        self.serializer
-                            .push(feed::BestFinalized(finalized.height, finalized.hash));
-                    }
-                }
-            }
-        }
-
-        self.broadcast();
-    }
-}
-
-impl Handler<LocateNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: LocateNode, _: &mut Self::Context) {
-        let LocateNode { nid, location } = msg;
-
-        if let Some(node) = self.nodes.get_mut(nid) {
-            self.serializer.push(feed::LocatedNode(
-                nid,
-                location.latitude,
-                location.longitude,
-                &location.city,
-            ));
-
-            node.update_location(location);
-        }
-    }
-}
-
-impl Handler<RemoveNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: RemoveNode, ctx: &mut Self::Context) {
-        let RemoveNode(nid) = msg;
-
-        if let Some(node) = self.nodes.remove(nid) {
-            self.decrement_label_count(&node.details().chain);
-        }
-
-        if self.nodes.is_empty() {
-            log::info!("[{}] Lost all nodes, dropping...", self.label.0);
-            ctx.stop();
-        }
-
-        self.serializer.push(feed::RemovedNode(nid));
-        self.broadcast();
-        self.update_count();
-    }
-}
-
-impl Handler<Subscribe> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: Subscribe, ctx: &mut Self::Context) {
-        let Subscribe(feed) = msg;
-        let fid = self.feeds.add(feed.clone());
-
-        feed.do_send(Subscribed(fid, ctx.address().recipient()));
-
-        self.serializer.push(feed::SubscribedTo(&self.label.0));
-        self.serializer.push(feed::TimeSync(now()));
-        self.serializer.push(feed::BestBlock(
-            self.best.height,
-            self.timestamp.unwrap_or(0),
-            self.average_block_time,
-        ));
-        self.serializer.push(feed::BestFinalized(
-            self.finalized.height,
-            self.finalized.hash,
-        ));
-
-        for (idx, (nid, node)) in self.nodes.iter().enumerate() {
-            // Send subscription confirmation and chain head before doing all the nodes,
-            // and continue sending batches of 32 nodes a time over the wire subsequently
-            if idx % 32 == 0 {
-                if let Some(serialized) = self.serializer.finalize() {
-                    feed.do_send(serialized);
-                }
-            }
-
-            self.serializer.push(feed::AddedNode(nid, node));
-            self.serializer.push(feed::FinalizedBlock(
-                nid,
-                node.finalized().height,
-                node.finalized().hash,
-            ));
-            if node.stale() {
-                self.serializer.push(feed::StaleNode(nid));
-            }
-        }
-
-        if let Some(serialized) = self.serializer.finalize() {
-            feed.do_send(serialized);
-        }
-    }
-}
-
-impl Handler<SendFinality> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: SendFinality, _ctx: &mut Self::Context) {
-        let SendFinality(fid) = msg;
-        if let Some(feed) = self.feeds.get(fid) {
-            self.finality_feeds.insert(fid, feed.clone());
-        }
-
-        // info!("Added new finality feed {}", fid);
-    }
-}
-
-impl Handler<NoMoreFinality> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: NoMoreFinality, _: &mut Self::Context) {
-        let NoMoreFinality(fid) = msg;
-
-        // info!("Removed finality feed {}", fid);
-        self.finality_feeds.remove(&fid);
-    }
-}
-
-impl Handler<Unsubscribe> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: Unsubscribe, _: &mut Self::Context) {
-        let Unsubscribe(fid) = msg;
-
-        if let Some(feed) = self.feeds.get(fid) {
-            self.serializer.push(feed::UnsubscribedFrom(&self.label.0));
-
-            if let Some(serialized) = self.serializer.finalize() {
-                feed.do_send(serialized);
-            }
-        }
-
-        self.feeds.remove(fid);
-        self.finality_feeds.remove(&fid);
-    }
-}

backend/core/src/feed.rs

@@ -1,195 +0,0 @@
-use serde::ser::{SerializeTuple, Serializer};
-use serde::Serialize;
-use std::mem;
-
-use crate::node::Node;
-use crate::types::{
-    Address, BlockDetails, BlockHash, BlockNumber, NodeHardware, NodeIO, NodeId, NodeStats,
-    Timestamp,
-};
-use serde_json::to_writer;
-
-pub mod connector;
-
-use connector::Serialized;
-
-pub trait FeedMessage: Serialize {
-    const ACTION: u8;
-}
-
-pub struct FeedMessageSerializer {
-    /// Current buffer,
-    buffer: Vec<u8>,
-}
-
-const BUFCAP: usize = 128;
-
-impl FeedMessageSerializer {
-    pub fn new() -> Self {
-        Self {
-            buffer: Vec::with_capacity(BUFCAP),
-        }
-    }
-
-    pub fn push<Message>(&mut self, msg: Message)
-    where
-        Message: FeedMessage,
-    {
-        let glue = match self.buffer.len() {
-            0 => b'[',
-            _ => b',',
-        };
-
-        self.buffer.push(glue);
-        let _ = to_writer(&mut self.buffer, &Message::ACTION);
-        self.buffer.push(b',');
-        let _ = to_writer(&mut self.buffer, &msg);
-    }
-
-    pub fn finalize(&mut self) -> Option<Serialized> {
-        if self.buffer.is_empty() {
-            return None;
-        }
-
-        self.buffer.push(b']');
-
-        let bytes = mem::replace(&mut self.buffer, Vec::with_capacity(BUFCAP)).into();
-
-        Some(Serialized(bytes))
-    }
-}
-
-macro_rules! actions {
-    ($($action:literal: $t:ty,)*) => {
-        $(
-            impl FeedMessage for $t {
-                const ACTION: u8 = $action;
-            }
-        )*
-    }
-}
-
-actions! {
-    0x00: Version,
-    0x01: BestBlock,
-    0x02: BestFinalized,
-    0x03: AddedNode<'_>,
-    0x04: RemovedNode,
-    0x05: LocatedNode<'_>,
-    0x06: ImportedBlock<'_>,
-    0x07: FinalizedBlock,
-    0x08: NodeStatsUpdate<'_>,
-    0x09: Hardware<'_>,
-    0x0A: TimeSync,
-    0x0B: AddedChain<'_>,
-    0x0C: RemovedChain<'_>,
-    0x0D: SubscribedTo<'_>,
-    0x0E: UnsubscribedFrom<'_>,
-    0x0F: Pong<'_>,
-    0x10: AfgFinalized,
-    0x11: AfgReceivedPrevote,
-    0x12: AfgReceivedPrecommit,
-    0x13: AfgAuthoritySet,
-    0x14: StaleNode,
-    0x15: NodeIOUpdate<'_>,
-}
-
-#[derive(Serialize)]
-pub struct Version(pub usize);
-
-#[derive(Serialize)]
-pub struct BestBlock(pub BlockNumber, pub Timestamp, pub Option<u64>);
-
-#[derive(Serialize)]
-pub struct BestFinalized(pub BlockNumber, pub BlockHash);
-
-pub struct AddedNode<'a>(pub NodeId, pub &'a Node);
-
-#[derive(Serialize)]
-pub struct RemovedNode(pub NodeId);
-
-#[derive(Serialize)]
-pub struct LocatedNode<'a>(pub NodeId, pub f32, pub f32, pub &'a str);
-
-#[derive(Serialize)]
-pub struct ImportedBlock<'a>(pub NodeId, pub &'a BlockDetails);
-
-#[derive(Serialize)]
-pub struct FinalizedBlock(pub NodeId, pub BlockNumber, pub BlockHash);
-
-#[derive(Serialize)]
-pub struct NodeStatsUpdate<'a>(pub NodeId, pub &'a NodeStats);
-
-#[derive(Serialize)]
-pub struct NodeIOUpdate<'a>(pub NodeId, pub &'a NodeIO);
-
-#[derive(Serialize)]
-pub struct Hardware<'a>(pub NodeId, pub &'a NodeHardware);
-
-#[derive(Serialize)]
-pub struct TimeSync(pub u64);
-
-#[derive(Serialize)]
-pub struct AddedChain<'a>(pub &'a str, pub usize);
-
-#[derive(Serialize)]
-pub struct RemovedChain<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct SubscribedTo<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct UnsubscribedFrom<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct Pong<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct AfgFinalized(pub Address, pub BlockNumber, pub BlockHash);
-
-#[derive(Serialize)]
-pub struct AfgReceivedPrevote(
-    pub Address,
-    pub BlockNumber,
-    pub BlockHash,
-    pub Option<Address>,
-);
-
-#[derive(Serialize)]
-pub struct AfgReceivedPrecommit(
-    pub Address,
-    pub BlockNumber,
-    pub BlockHash,
-    pub Option<Address>,
-);
-
-#[derive(Serialize)]
-pub struct AfgAuthoritySet(
-    pub Address,
-    pub Address,
-    pub Address,
-    pub BlockNumber,
-    pub BlockHash,
-);
-
-#[derive(Serialize)]
-pub struct StaleNode(pub NodeId);
-
-impl Serialize for AddedNode<'_> {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let AddedNode(nid, node) = self;
-        let mut tup = serializer.serialize_tuple(8)?;
-        tup.serialize_element(nid)?;
-        tup.serialize_element(node.details())?;
-        tup.serialize_element(node.stats())?;
-        tup.serialize_element(node.io())?;
-        tup.serialize_element(node.hardware())?;
-        tup.serialize_element(node.block_details())?;
-        tup.serialize_element(&node.location())?;
-        tup.serialize_element(&node.startup_time())?;
-        tup.end()
-    }
-}

backend/core/src/feed/connector.rs

@@ -1,217 +0,0 @@
-use crate::aggregator::{Aggregator, Connect, Disconnect, NoMoreFinality, SendFinality, Subscribe};
-use crate::chain::Unsubscribe;
-use crate::feed::{FeedMessageSerializer, Pong};
-use crate::util::fnv;
-use actix::prelude::*;
-use actix_web_actors::ws;
-use bytes::Bytes;
-use std::time::{Duration, Instant};
-
-pub type FeedId = usize;
-
-/// How often heartbeat pings are sent
-const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
-/// How long before lack of client response causes a timeout
-const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
-
-pub struct FeedConnector {
-    /// FeedId that Aggregator holds of this actor
-    fid_aggregator: FeedId,
-    /// FeedId that Chain holds of this actor
-    fid_chain: FeedId,
-    /// Client must send ping at least once per 10 seconds (CLIENT_TIMEOUT),
-    hb: Instant,
-    /// Aggregator actor address
-    aggregator: Addr<Aggregator>,
-    /// Chain actor address
-    chain: Option<Recipient<Unsubscribe>>,
-    /// FNV hash of the chain label, optimization to avoid double-subscribing
-    chain_label_hash: u64,
-    /// Message serializer
-    serializer: FeedMessageSerializer,
-}
-
-impl Actor for FeedConnector {
-    type Context = ws::WebsocketContext<Self>;
-
-    fn started(&mut self, ctx: &mut Self::Context) {
-        self.heartbeat(ctx);
-        self.aggregator.do_send(Connect(ctx.address()));
-    }
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        if let Some(chain) = self.chain.take() {
-            let _ = chain.do_send(Unsubscribe(self.fid_chain));
-        }
-
-        self.aggregator.do_send(Disconnect(self.fid_aggregator));
-    }
-}
-
-impl FeedConnector {
-    pub fn new(aggregator: Addr<Aggregator>) -> Self {
-        Self {
-            // Garbage id, will be replaced by the Connected message
-            fid_aggregator: !0,
-            // Garbage id, will be replaced by the Subscribed message
-            fid_chain: !0,
-            hb: Instant::now(),
-            aggregator,
-            chain: None,
-            chain_label_hash: 0,
-            serializer: FeedMessageSerializer::new(),
-        }
-    }
-
-    fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
-        ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
-            // check client heartbeats
-            if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
-                // stop actor
-                ctx.stop();
-            } else {
-                ctx.ping(b"")
-            }
-        });
-    }
-
-    fn handle_cmd(&mut self, cmd: &str, payload: &str, ctx: &mut <Self as Actor>::Context) {
-        match cmd {
-            "subscribe" => {
-                match fnv(payload) {
-                    hash if hash == self.chain_label_hash => return,
-                    hash => self.chain_label_hash = hash,
-                }
-
-                self.aggregator
-                    .send(Subscribe {
-                        chain: payload.into(),
-                        feed: ctx.address(),
-                    })
-                    .into_actor(self)
-                    .then(|res, actor, _| {
-                        match res {
-                            Ok(true) => (),
-                            // Chain not found, reset hash
-                            _ => actor.chain_label_hash = 0,
-                        }
-                        async {}.into_actor(actor)
-                    })
-                    .wait(ctx);
-            }
-            "send-finality" => {
-                self.aggregator.do_send(SendFinality {
-                    chain: payload.into(),
-                    fid: self.fid_chain,
-                });
-            }
-            "no-more-finality" => {
-                self.aggregator.do_send(NoMoreFinality {
-                    chain: payload.into(),
-                    fid: self.fid_chain,
-                });
-            }
-            "ping" => {
-                self.serializer.push(Pong(payload));
-                if let Some(serialized) = self.serializer.finalize() {
-                    ctx.binary(serialized.0);
-                }
-            }
-            _ => (),
-        }
-    }
-}
-
-/// Message sent form Chain to the FeedConnector upon successful subscription
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Subscribed(pub FeedId, pub Recipient<Unsubscribe>);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Unsubscribed;
-
-/// Message sent from Aggregator to FeedConnector upon successful connection
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Connected(pub FeedId);
-
-/// Message sent from either Aggregator or Chain to FeedConnector containing
-/// serialized message(s) for the frontend
-///
-/// Since Bytes is ARC'ed, this is cheap to clone
-#[derive(Message, Clone)]
-#[rtype(result = "()")]
-pub struct Serialized(pub Bytes);
-
-impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for FeedConnector {
-    fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
-        match msg {
-            Ok(ws::Message::Ping(msg)) => {
-                self.hb = Instant::now();
-                ctx.pong(&msg);
-            }
-            Ok(ws::Message::Pong(_)) => self.hb = Instant::now(),
-            Ok(ws::Message::Text(text)) => {
-                if let Some(idx) = text.find(':') {
-                    let cmd = &text[..idx];
-                    let payload = &text[idx + 1..];
-
-                    log::info!("New FEED message: {}", cmd);
-
-                    self.handle_cmd(cmd, payload, ctx);
-                }
-            }
-            Ok(ws::Message::Close(_)) => ctx.stop(),
-            Ok(_) => (),
-            Err(error) => {
-                log::error!("{:?}", error);
-                ctx.stop();
-            }
-        }
-    }
-}
-
-impl Handler<Subscribed> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Subscribed, _: &mut Self::Context) {
-        let Subscribed(fid_chain, chain) = msg;
-
-        if let Some(current) = self.chain.take() {
-            let _ = current.do_send(Unsubscribe(self.fid_chain));
-        }
-
-        self.fid_chain = fid_chain;
-        self.chain = Some(chain);
-    }
-}
-
-impl Handler<Unsubscribed> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, _: Unsubscribed, _: &mut Self::Context) {
-        self.chain = None;
-        self.chain_label_hash = 0;
-    }
-}
-
-impl Handler<Connected> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Connected, _: &mut Self::Context) {
-        let Connected(fid_aggregator) = msg;
-
-        self.fid_aggregator = fid_aggregator;
-    }
-}
-
-impl Handler<Serialized> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Serialized, ctx: &mut Self::Context) {
-        let Serialized(bytes) = msg;
-
-        ctx.binary(bytes);
-    }
-}

backend/core/src/main.rs

@@ -1,187 +0,0 @@
-use std::collections::HashSet;
-use std::iter::FromIterator;
-use std::net::Ipv4Addr;
-
-use actix::prelude::*;
-use actix_http::ws::Codec;
-use actix_web::{get, middleware, web, App, Error, HttpRequest, HttpResponse, HttpServer};
-use actix_web_actors::ws;
-use clap::Clap;
-use simple_logger::SimpleLogger;
-
-mod aggregator;
-mod chain;
-mod feed;
-mod node;
-mod shard;
-mod types;
-mod util;
-
-use aggregator::{Aggregator, GetHealth};
-use feed::connector::FeedConnector;
-use node::connector::NodeConnector;
-use shard::connector::ShardConnector;
-use util::{Locator, LocatorFactory};
-
-const VERSION: &str = env!("CARGO_PKG_VERSION");
-const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
-const NAME: &str = "Substrate Telemetry Backend";
-const ABOUT: &str = "This is the Telemetry Backend that injects and provide the data sent by Substrate/Polkadot nodes";
-
-#[derive(Clap, Debug)]
-#[clap(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
-struct Opts {
-    #[clap(
-        short = 'l',
-        long = "listen",
-        default_value = "127.0.0.1:8000",
-        about = "This is the socket address Telemetry is listening to. This is restricted to localhost (127.0.0.1) by default and should be fine for most use cases. If you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'"
-    )]
-    socket: std::net::SocketAddr,
-    #[clap(
-        required = false,
-        long = "denylist",
-        about = "Space delimited list of chains that are not allowed to connect to telemetry. Case sensitive."
-    )]
-    denylist: Vec<String>,
-    #[clap(
-        arg_enum,
-        required = false,
-        long = "log",
-        default_value = "info",
-        about = "Log level."
-    )]
-    log_level: LogLevel,
-}
-
-#[derive(Clap, Debug, PartialEq)]
-enum LogLevel {
-    Error,
-    Warn,
-    Info,
-    Debug,
-    Trace,
-}
-
-impl From<&LogLevel> for log::LevelFilter {
-    fn from(log_level: &LogLevel) -> Self {
-        match log_level {
-            LogLevel::Error => log::LevelFilter::Error,
-            LogLevel::Warn => log::LevelFilter::Warn,
-            LogLevel::Info => log::LevelFilter::Info,
-            LogLevel::Debug => log::LevelFilter::Debug,
-            LogLevel::Trace => log::LevelFilter::Trace,
-        }
-    }
-}
-
-/// Entry point for connecting nodes
-#[get("/submit")]
-async fn node_route(
-    req: HttpRequest,
-    stream: web::Payload,
-    aggregator: web::Data<Addr<Aggregator>>,
-    locator: web::Data<Addr<Locator>>,
-) -> Result<HttpResponse, Error> {
-    let ip = req
-        .connection_info()
-        .realip_remote_addr()
-        .and_then(|mut addr| {
-            if let Some(port_idx) = addr.find(':') {
-                addr = &addr[..port_idx];
-            }
-            addr.parse::<Ipv4Addr>().ok()
-        });
-
-    let mut res = ws::handshake(&req)?;
-    let aggregator = aggregator.get_ref().clone();
-    let locator = locator.get_ref().clone().recipient();
-
-    Ok(res.streaming(ws::WebsocketContext::with_codec(
-        NodeConnector::new(aggregator, locator, ip),
-        stream,
-        Codec::new().max_size(10 * 1024 * 1024), // 10mb frame limit
-    )))
-}
-
-#[get("/shard_submit/{chain_hash}")]
-async fn shard_route(
-    req: HttpRequest,
-    stream: web::Payload,
-    aggregator: web::Data<Addr<Aggregator>>,
-    path: web::Path<Box<str>>,
-) -> Result<HttpResponse, Error> {
-    let hash_str = path.into_inner();
-    let genesis_hash = hash_str.parse()?;
-
-    let mut res = ws::handshake(&req)?;
-
-    let aggregator = aggregator.get_ref().clone();
-
-    Ok(res.streaming(ws::WebsocketContext::with_codec(
-        ShardConnector::new(aggregator, genesis_hash),
-        stream,
-        Codec::new().max_size(10 * 1024 * 1024), // 10mb frame limit
-    )))
-}
-
-/// Entry point for connecting feeds
-#[get("/feed")]
-async fn feed_route(
-    req: HttpRequest,
-    stream: web::Payload,
-    aggregator: web::Data<Addr<Aggregator>>,
-) -> Result<HttpResponse, Error> {
-    ws::start(
-        FeedConnector::new(aggregator.get_ref().clone()),
-        &req,
-        stream,
-    )
-}
-
-/// Entry point for health check monitoring bots
-#[get("/health")]
-async fn health(aggregator: web::Data<Addr<Aggregator>>) -> Result<HttpResponse, Error> {
-    match aggregator.send(GetHealth).await {
-        Ok(count) => {
-            let body = format!("Connected chains: {}", count);
-
-            HttpResponse::Ok().body(body).await
-        }
-        Err(error) => {
-            log::error!("Health check mailbox error: {:?}", error);
-
-            HttpResponse::InternalServerError().await
-        }
-    }
-}
-
-/// Telemetry entry point. Listening by default on 127.0.0.1:8000.
-/// This can be changed using the `PORT` and `BIND` ENV variables.
-#[actix_web::main]
-async fn main() -> std::io::Result<()> {
-    let opts = Opts::parse();
-    let log_level = &opts.log_level;
-    SimpleLogger::new()
-        .with_level(log_level.into())
-        .init()
-        .expect("Must be able to start a logger");
-
-    let denylist = HashSet::from_iter(opts.denylist);
-    let aggregator = Aggregator::new(denylist).start();
-    let factory = LocatorFactory::new();
-    let locator = SyncArbiter::start(4, move || factory.create());
-    log::info!("Starting telemetry version: {}", env!("CARGO_PKG_VERSION"));
-    HttpServer::new(move || {
-        App::new()
-            .wrap(middleware::NormalizePath::default())
-            .data(aggregator.clone())
-            .data(locator.clone())
-            .service(node_route)
-            .service(feed_route)
-            .service(health)
-    })
-    .bind(opts.socket)?
-    .run()
-    .await
-}

backend/core/src/node/connector.rs

@@ -1,274 +0,0 @@
-use std::collections::BTreeMap;
-use std::mem;
-use std::net::Ipv4Addr;
-use std::time::{Duration, Instant};
-
-use crate::aggregator::{AddNode, Aggregator};
-use crate::chain::{Chain, RemoveNode, UpdateNode};
-use crate::node::message::{NodeMessage, Payload};
-use crate::node::NodeId;
-use crate::types::ConnId;
-use crate::util::LocateRequest;
-use actix::prelude::*;
-use actix_http::ws::Item;
-use actix_web_actors::ws::{self, CloseReason};
-use bytes::{Bytes, BytesMut};
-
-/// How often heartbeat pings are sent
-const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
-/// How long before lack of client response causes a timeout
-const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
-/// Continuation buffer limit, 10mb
-const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
-
-pub struct NodeConnector {
-    /// Multiplexing connections by id
-    multiplex: BTreeMap<ConnId, ConnMultiplex>,
-    /// Client must send ping at least once every 60 seconds (CLIENT_TIMEOUT),
-    hb: Instant,
-    /// Aggregator actor address
-    aggregator: Addr<Aggregator>,
-    /// IP address of the node this connector is responsible for
-    ip: Option<Ipv4Addr>,
-    /// Actix address of location services
-    locator: Recipient<LocateRequest>,
-    /// Buffer for constructing continuation messages
-    contbuf: BytesMut,
-}
-
-enum ConnMultiplex {
-    Connected {
-        /// Id of the node this multiplex connector is responsible for handling
-        nid: NodeId,
-        /// Chain address to which this multiplex connector is delegating messages
-        chain: Addr<Chain>,
-    },
-    Waiting {
-        /// Backlog of messages to be sent once we get a recipient handle to the chain
-        backlog: Vec<Payload>,
-    },
-}
-
-impl Default for ConnMultiplex {
-    fn default() -> Self {
-        ConnMultiplex::Waiting {
-            backlog: Vec::new(),
-        }
-    }
-}
-
-impl Actor for NodeConnector {
-    type Context = ws::WebsocketContext<Self>;
-
-    fn started(&mut self, ctx: &mut Self::Context) {
-        self.heartbeat(ctx);
-    }
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        for mx in self.multiplex.values() {
-            if let ConnMultiplex::Connected { chain, nid } = mx {
-                chain.do_send(RemoveNode(*nid));
-            }
-        }
-    }
-}
-
-impl NodeConnector {
-    pub fn new(
-        aggregator: Addr<Aggregator>,
-        locator: Recipient<LocateRequest>,
-        ip: Option<Ipv4Addr>,
-    ) -> Self {
-        Self {
-            multiplex: BTreeMap::new(),
-            hb: Instant::now(),
-            aggregator,
-            ip,
-            locator,
-            contbuf: BytesMut::new(),
-        }
-    }
-
-    fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
-        ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
-            // check client heartbeats
-            if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
-                // stop actor
-                ctx.close(Some(CloseReason {
-                    code: ws::CloseCode::Abnormal,
-                    description: Some("Missed heartbeat".into()),
-                }));
-                ctx.stop();
-            }
-        });
-    }
-
-    fn handle_message(
-        &mut self,
-        msg: NodeMessage,
-        ctx: &mut <Self as Actor>::Context,
-    ) {
-        let conn_id = msg.id();
-        let payload = msg.into();
-
-        match self.multiplex.entry(conn_id).or_default() {
-            ConnMultiplex::Connected { nid, chain } => {
-                chain.do_send(UpdateNode {
-                    nid: *nid,
-                    payload,
-                });
-            }
-            ConnMultiplex::Waiting { backlog } => {
-                if let Payload::SystemConnected(connected) = payload {
-                    self.aggregator.do_send(AddNode {
-                        node: connected.node,
-                        genesis_hash: connected.genesis_hash,
-                        conn_id,
-                        node_connector: ctx.address(),
-                    });
-                } else {
-                    if backlog.len() >= 10 {
-                        backlog.remove(0);
-                    }
-
-                    backlog.push(payload);
-                }
-            }
-        }
-    }
-
-    fn start_frame(&mut self, bytes: &[u8]) {
-        if !self.contbuf.is_empty() {
-            log::error!("Unused continuation buffer");
-            self.contbuf.clear();
-        }
-        self.continue_frame(bytes);
-    }
-
-    fn continue_frame(&mut self, bytes: &[u8]) {
-        if self.contbuf.len() + bytes.len() <= CONT_BUF_LIMIT {
-            self.contbuf.extend_from_slice(&bytes);
-        } else {
-            log::error!("Continuation buffer overflow");
-            self.contbuf = BytesMut::new();
-        }
-    }
-
-    fn finish_frame(&mut self) -> Bytes {
-        mem::replace(&mut self.contbuf, BytesMut::new()).freeze()
-    }
-}
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Mute {
-    pub reason: CloseReason,
-}
-
-impl Handler<Mute> for NodeConnector {
-    type Result = ();
-    fn handle(&mut self, msg: Mute, ctx: &mut Self::Context) {
-        let Mute { reason } = msg;
-        log::debug!(target: "NodeConnector::Mute", "Muting a node. Reason: {:?}", reason.description);
-
-        ctx.close(Some(reason));
-        ctx.stop();
-    }
-}
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Initialize {
-    pub nid: NodeId,
-    pub conn_id: ConnId,
-    pub chain: Addr<Chain>,
-}
-
-impl Handler<Initialize> for NodeConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Initialize, _: &mut Self::Context) {
-        let Initialize {
-            nid,
-            conn_id,
-            chain,
-        } = msg;
-        log::trace!(target: "NodeConnector::Initialize", "Initializing a node, nid={}, on conn_id={}", nid, conn_id);
-        let mx = self.multiplex.entry(conn_id).or_default();
-
-        if let ConnMultiplex::Waiting { backlog } = mx {
-            for payload in backlog.drain(..) {
-                chain.do_send(UpdateNode {
-                    nid,
-                    payload,
-                });
-            }
-
-            *mx = ConnMultiplex::Connected {
-                nid,
-                chain: chain.clone(),
-            };
-        };
-
-        // Acquire the node's physical location
-        if let Some(ip) = self.ip {
-            let _ = self.locator.do_send(LocateRequest { ip, nid, chain });
-        }
-    }
-}
-
-impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for NodeConnector {
-    fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
-        self.hb = Instant::now();
-
-        let data = match msg {
-            Ok(ws::Message::Ping(msg)) => {
-                ctx.pong(&msg);
-                return;
-            }
-            Ok(ws::Message::Pong(_)) => return,
-            Ok(ws::Message::Text(text)) => text.into_bytes(),
-            Ok(ws::Message::Binary(data)) => data,
-            Ok(ws::Message::Close(reason)) => {
-                ctx.close(reason);
-                ctx.stop();
-                return;
-            }
-            Ok(ws::Message::Nop) => return,
-            Ok(ws::Message::Continuation(cont)) => match cont {
-                Item::FirstText(bytes) | Item::FirstBinary(bytes) => {
-                    self.start_frame(&bytes);
-                    return;
-                }
-                Item::Continue(bytes) => {
-                    self.continue_frame(&bytes);
-                    return;
-                }
-                Item::Last(bytes) => {
-                    self.continue_frame(&bytes);
-                    self.finish_frame()
-                }
-            },
-            Err(error) => {
-                log::error!("{:?}", error);
-                ctx.stop();
-                return;
-            }
-        };
-
-        match serde_json::from_slice(&data) {
-            Ok(msg) => self.handle_message(msg, ctx),
-            #[cfg(debug)]
-            Err(err) => {
-                let data: &[u8] = data.get(..512).unwrap_or_else(|| &data);
-                log::warn!(
-                    "Failed to parse node message: {} {}",
-                    err,
-                    std::str::from_utf8(data).unwrap_or_else(|_| "INVALID UTF8")
-                )
-            }
-            #[cfg(not(debug))]
-            Err(_) => (),
-        }
-    }
-}

backend/core/src/node/message.rs

@@ -1,196 +0,0 @@
-use crate::node::NodeDetails;
-use crate::types::{Block, BlockHash, BlockNumber, ConnId};
-use crate::util::Hash;
-use actix::prelude::*;
-use serde::de::IgnoredAny;
-use serde::Deserialize;
-
-#[derive(Deserialize, Debug, Message)]
-#[rtype(result = "()")]
-#[serde(untagged)]
-pub enum NodeMessage {
-    V1 {
-        #[serde(flatten)]
-        payload: Payload,
-    },
-    V2 {
-        id: ConnId,
-        payload: Payload,
-    },
-}
-
-impl NodeMessage {
-    /// Returns the connection ID or 0 if there is no ID.
-    pub fn id(&self) -> ConnId {
-        match self {
-            NodeMessage::V1 { .. } => 0,
-            NodeMessage::V2 { id, .. } => *id,
-        }
-    }
-}
-
-impl From<NodeMessage> for Payload {
-    fn from(msg: NodeMessage) -> Payload {
-        match msg {
-            NodeMessage::V1 { payload, .. } | NodeMessage::V2 { payload, .. } => payload,
-        }
-    }
-}
-
-#[derive(Deserialize, Debug)]
-#[serde(tag = "msg")]
-pub enum Payload {
-    #[serde(rename = "system.connected")]
-    SystemConnected(SystemConnected),
-    #[serde(rename = "system.interval")]
-    SystemInterval(SystemInterval),
-    #[serde(rename = "block.import")]
-    BlockImport(Block),
-    #[serde(rename = "notify.finalized")]
-    NotifyFinalized(Finalized),
-    #[serde(rename = "txpool.import")]
-    TxPoolImport(IgnoredAny),
-    #[serde(rename = "afg.finalized")]
-    AfgFinalized(AfgFinalized),
-    #[serde(rename = "afg.received_precommit")]
-    AfgReceivedPrecommit(AfgReceivedPrecommit),
-    #[serde(rename = "afg.received_prevote")]
-    AfgReceivedPrevote(AfgReceivedPrevote),
-    #[serde(rename = "afg.received_commit")]
-    AfgReceivedCommit(AfgReceivedCommit),
-    #[serde(rename = "afg.authority_set")]
-    AfgAuthoritySet(AfgAuthoritySet),
-    #[serde(rename = "afg.finalized_blocks_up_to")]
-    AfgFinalizedBlocksUpTo(IgnoredAny),
-    #[serde(rename = "aura.pre_sealed_block")]
-    AuraPreSealedBlock(IgnoredAny),
-    #[serde(rename = "prepared_block_for_proposing")]
-    PreparedBlockForProposing(IgnoredAny),
-}
-
-#[derive(Deserialize, Debug)]
-pub struct SystemConnected {
-    pub genesis_hash: Hash,
-    #[serde(flatten)]
-    pub node: NodeDetails,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct SystemInterval {
-    pub peers: Option<u64>,
-    pub txcount: Option<u64>,
-    pub bandwidth_upload: Option<f64>,
-    pub bandwidth_download: Option<f64>,
-    pub finalized_height: Option<BlockNumber>,
-    pub finalized_hash: Option<BlockHash>,
-    #[serde(flatten)]
-    pub block: Option<Block>,
-    pub used_state_cache_size: Option<f32>,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct Finalized {
-    #[serde(rename = "best")]
-    pub hash: BlockHash,
-    pub height: Box<str>,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct AfgAuthoritySet {
-    pub authority_id: Box<str>,
-    pub authorities: Box<str>,
-    pub authority_set_id: Box<str>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgFinalized {
-    pub finalized_hash: BlockHash,
-    pub finalized_number: Box<str>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceived {
-    pub target_hash: BlockHash,
-    pub target_number: Box<str>,
-    pub voter: Option<Box<str>>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceivedPrecommit {
-    #[serde(flatten)]
-    pub received: AfgReceived,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceivedPrevote {
-    #[serde(flatten)]
-    pub received: AfgReceived,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceivedCommit {
-    #[serde(flatten)]
-    pub received: AfgReceived,
-}
-
-impl Block {
-    pub fn zero() -> Self {
-        Block {
-            hash: BlockHash::from([0; 32]),
-            height: 0,
-        }
-    }
-}
-
-impl Payload {
-    pub fn best_block(&self) -> Option<&Block> {
-        match self {
-            Payload::BlockImport(block) => Some(block),
-            Payload::SystemInterval(SystemInterval { block, .. }) => block.as_ref(),
-            _ => None,
-        }
-    }
-
-    pub fn finalized_block(&self) -> Option<Block> {
-        match self {
-            Payload::SystemInterval(ref interval) => Some(Block {
-                hash: interval.finalized_hash?,
-                height: interval.finalized_height?,
-            }),
-            Payload::NotifyFinalized(ref finalized) => Some(Block {
-                hash: finalized.hash,
-                height: finalized.height.parse().ok()?,
-            }),
-            _ => None,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn message_v1() {
-        let json = r#"{"msg":"notify.finalized","level":"INFO","ts":"2021-01-13T12:38:25.410794650+01:00","best":"0x031c3521ca2f9c673812d692fc330b9a18e18a2781e3f9976992f861fd3ea0cb","height":"50"}"#;
-        assert!(
-            matches!(
-                serde_json::from_str::<NodeMessage>(json).unwrap(),
-                NodeMessage::V1 { .. },
-            ),
-            "message did not match variant V1",
-        );
-    }
-
-    #[test]
-    fn message_v2() {
-        let json = r#"{"id":1,"ts":"2021-01-13T12:22:20.053527101+01:00","payload":{"best":"0xcc41708573f2acaded9dd75e07dac2d4163d136ca35b3061c558d7a35a09dd8d","height":"209","msg":"notify.finalized"}}"#;
-        assert!(
-            matches!(
-                serde_json::from_str::<NodeMessage>(json).unwrap(),
-                NodeMessage::V2 { .. },
-            ),
-            "message did not match variant V2",
-        );
-    }
-}

backend/core/src/shard.rs

@@ -1,13 +0,0 @@
-use crate::node::message::Payload;
-use serde::Deserialize;
-
-pub mod connector;
-
-/// Alias for the ID of the node connection
-type ShardConnId = usize;
-
-#[derive(Deserialize)]
-pub struct ShardMessage {
-    pub conn_id: ShardConnId,
-    pub payload: Payload,
-}

backend/core/src/shard/connector.rs

@@ -1,156 +0,0 @@
-use std::mem;
-use std::time::{Duration, Instant};
-
-use crate::aggregator::{AddNode, Aggregator};
-use crate::chain::{Chain, RemoveNode, UpdateNode};
-use crate::shard::ShardMessage;
-use crate::types::NodeId;
-use crate::util::{DenseMap, Hash};
-use actix::prelude::*;
-use actix_http::ws::Item;
-use actix_web_actors::ws::{self, CloseReason};
-use bincode::Options;
-use bytes::{Bytes, BytesMut};
-
-/// How often heartbeat pings are sent
-const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
-/// How long before lack of client response causes a timeout
-const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
-/// Continuation buffer limit, 10mb
-const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
-
-pub struct ShardConnector {
-    /// Client must send ping at least once every 60 seconds (CLIENT_TIMEOUT),
-    hb: Instant,
-    /// Aggregator actor address
-    aggregator: Addr<Aggregator>,
-    /// Genesis hash of the chain this connection will be submitting data for
-    genesis_hash: Hash,
-    /// Chain address to which this multiplex connector is delegating messages
-    chain: Option<Addr<Chain>>,
-    /// Mapping `ShardConnId` to `NodeId`
-    nodes: DenseMap<NodeId>,
-    /// Buffer for constructing continuation messages
-    contbuf: BytesMut,
-}
-
-impl Actor for ShardConnector {
-    type Context = ws::WebsocketContext<Self>;
-
-    fn started(&mut self, ctx: &mut Self::Context) {
-        self.heartbeat(ctx);
-    }
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        if let Some(ref chain) = self.chain {
-            for (_, nid) in self.nodes.iter() {
-                chain.do_send(RemoveNode(*nid))
-            }
-        }
-    }
-}
-
-impl ShardConnector {
-    pub fn new(aggregator: Addr<Aggregator>, genesis_hash: Hash) -> Self {
-        Self {
-            hb: Instant::now(),
-            aggregator,
-            genesis_hash,
-            chain: None,
-            nodes: DenseMap::new(),
-            contbuf: BytesMut::new(),
-        }
-    }
-
-    fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
-        ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
-            // check client heartbeats
-            if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
-                // stop actor
-                ctx.close(Some(CloseReason {
-                    code: ws::CloseCode::Abnormal,
-                    description: Some("Missed heartbeat".into()),
-                }));
-                ctx.stop();
-            }
-        });
-    }
-
-    fn handle_message(&mut self, msg: ShardMessage, ctx: &mut <Self as Actor>::Context) {
-        let ShardMessage { conn_id, payload } = msg;
-
-        // TODO: get `NodeId` for `ShardConnId` and proxy payload to `self.chain`.
-    }
-
-    fn start_frame(&mut self, bytes: &[u8]) {
-        if !self.contbuf.is_empty() {
-            log::error!("Unused continuation buffer");
-            self.contbuf.clear();
-        }
-        self.continue_frame(bytes);
-    }
-
-    fn continue_frame(&mut self, bytes: &[u8]) {
-        if self.contbuf.len() + bytes.len() <= CONT_BUF_LIMIT {
-            self.contbuf.extend_from_slice(&bytes);
-        } else {
-            log::error!("Continuation buffer overflow");
-            self.contbuf = BytesMut::new();
-        }
-    }
-
-    fn finish_frame(&mut self) -> Bytes {
-        mem::replace(&mut self.contbuf, BytesMut::new()).freeze()
-    }
-}
-
-impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for ShardConnector {
-    fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
-        self.hb = Instant::now();
-
-        let data = match msg {
-            Ok(ws::Message::Ping(msg)) => {
-                ctx.pong(&msg);
-                return;
-            }
-            Ok(ws::Message::Pong(_)) => return,
-            Ok(ws::Message::Text(text)) => text.into_bytes(),
-            Ok(ws::Message::Binary(data)) => data,
-            Ok(ws::Message::Close(reason)) => {
-                ctx.close(reason);
-                ctx.stop();
-                return;
-            }
-            Ok(ws::Message::Nop) => return,
-            Ok(ws::Message::Continuation(cont)) => match cont {
-                Item::FirstText(bytes) | Item::FirstBinary(bytes) => {
-                    self.start_frame(&bytes);
-                    return;
-                }
-                Item::Continue(bytes) => {
-                    self.continue_frame(&bytes);
-                    return;
-                }
-                Item::Last(bytes) => {
-                    self.continue_frame(&bytes);
-                    self.finish_frame()
-                }
-            },
-            Err(error) => {
-                log::error!("{:?}", error);
-                ctx.stop();
-                return;
-            }
-        };
-
-        match bincode::options().deserialize(&data) {
-            Ok(msg) => self.handle_message(msg, ctx),
-            #[cfg(debug)]
-            Err(err) => {
-                log::warn!("Failed to parse shard message: {}", err,)
-            }
-            #[cfg(not(debug))]
-            Err(_) => (),
-        }
-    }
-}

backend/core/src/util.rs

@@ -1,31 +0,0 @@
-mod dense_map;
-mod hash;
-mod location;
-mod mean_list;
-mod num_stats;
-
-pub use dense_map::DenseMap;
-pub use hash::Hash;
-pub use location::{LocateRequest, Locator, LocatorFactory};
-pub use mean_list::MeanList;
-pub use num_stats::NumStats;
-
-pub fn fnv<D: AsRef<[u8]>>(data: D) -> u64 {
-    use fnv::FnvHasher;
-    use std::hash::Hasher;
-
-    let mut hasher = FnvHasher::default();
-
-    hasher.write(data.as_ref());
-    hasher.finish()
-}
-
-/// Returns current unix time in ms (compatible with JS Date.now())
-pub fn now() -> u64 {
-    use std::time::SystemTime;
-
-    SystemTime::now()
-        .duration_since(SystemTime::UNIX_EPOCH)
-        .expect("System time must be configured to be post Unix Epoch start; qed")
-        .as_millis() as u64
-}

backend/core/src/util/hash.rs

@@ -1,89 +0,0 @@
-use std::fmt::{self, Debug, Display};
-use std::str::FromStr;
-
-use actix_web::error::ResponseError;
-use serde::de::{self, Deserialize, Deserializer, Unexpected, Visitor};
-
-const HASH_BYTES: usize = 32;
-
-/// Newtype wrapper for 32-byte hash values, implementing readable `Debug` and `serde::Deserialize`.
-// We could use primitive_types::H256 here, but opted for a custom type to avoid more dependencies.
-#[derive(Hash, PartialEq, Eq, Clone, Copy)]
-pub struct Hash([u8; HASH_BYTES]);
-
-struct HashVisitor;
-
-impl<'de> Visitor<'de> for HashVisitor {
-    type Value = Hash;
-
-    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-        formatter.write_str("hexidecimal string of 32 bytes beginning with 0x")
-    }
-
-    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
-    where
-        E: de::Error,
-    {
-        value
-            .parse()
-            .map_err(|_| de::Error::invalid_value(Unexpected::Str(value), &self))
-    }
-}
-
-impl FromStr for Hash {
-    type Err = HashParseError;
-
-    fn from_str(value: &str) -> Result<Self, Self::Err> {
-        if !value.starts_with("0x") {
-            return Err(HashParseError::InvalidPrefix);
-        }
-
-        let mut hash = [0; HASH_BYTES];
-
-        hex::decode_to_slice(&value[2..], &mut hash).map_err(HashParseError::HexError)?;
-
-        Ok(Hash(hash))
-    }
-}
-
-impl<'de> Deserialize<'de> for Hash {
-    fn deserialize<D>(deserializer: D) -> Result<Hash, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        deserializer.deserialize_str(HashVisitor)
-    }
-}
-
-impl Display for Hash {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.write_str("0x")?;
-
-        let mut ascii = [0; HASH_BYTES * 2];
-
-        hex::encode_to_slice(self.0, &mut ascii)
-            .expect("Encoding 32 bytes into 64 bytes of ascii; qed");
-
-        f.write_str(std::str::from_utf8(&ascii).expect("ASCII hex encoded bytes canot fail; qed"))
-    }
-}
-
-impl Debug for Hash {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        Display::fmt(self, f)
-    }
-}
-
-#[derive(thiserror::Error, Debug)]
-pub enum HashParseError {
-    HexError(hex::FromHexError),
-    InvalidPrefix,
-}
-
-impl Display for HashParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        Debug::fmt(self, f)
-    }
-}
-
-impl ResponseError for HashParseError {}

backend/core/src/util/location.rs

@@ -1,191 +0,0 @@
-use std::net::Ipv4Addr;
-use std::sync::Arc;
-
-use actix::prelude::*;
-use parking_lot::RwLock;
-use rustc_hash::FxHashMap;
-use serde::Deserialize;
-
-use crate::chain::{Chain, LocateNode};
-use crate::types::{NodeId, NodeLocation};
-
-#[derive(Clone)]
-pub struct Locator {
-    client: reqwest::blocking::Client,
-    cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
-}
-
-pub struct LocatorFactory {
-    cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
-}
-
-impl LocatorFactory {
-    pub fn new() -> Self {
-        let mut cache = FxHashMap::default();
-
-        // Default entry for localhost
-        cache.insert(
-            Ipv4Addr::new(127, 0, 0, 1),
-            Some(Arc::new(NodeLocation {
-                latitude: 52.516_6667,
-                longitude: 13.4,
-                city: "Berlin".into(),
-            })),
-        );
-
-        LocatorFactory {
-            cache: Arc::new(RwLock::new(cache)),
-        }
-    }
-
-    pub fn create(&self) -> Locator {
-        Locator {
-            client: reqwest::blocking::Client::new(),
-            cache: self.cache.clone(),
-        }
-    }
-}
-
-impl Actor for Locator {
-    type Context = SyncContext<Self>;
-}
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct LocateRequest {
-    pub ip: Ipv4Addr,
-    pub nid: NodeId,
-    pub chain: Addr<Chain>,
-}
-
-#[derive(Deserialize)]
-pub struct IPApiLocate {
-    city: Box<str>,
-    loc: Box<str>,
-}
-
-impl IPApiLocate {
-    fn into_node_location(self) -> Option<NodeLocation> {
-        let IPApiLocate { city, loc } = self;
-
-        let mut loc = loc.split(',').map(|n| n.parse());
-
-        let latitude = loc.next()?.ok()?;
-        let longitude = loc.next()?.ok()?;
-
-        // Guarantee that the iterator has been exhausted
-        if loc.next().is_some() {
-            return None;
-        }
-
-        Some(NodeLocation {
-            latitude,
-            longitude,
-            city,
-        })
-    }
-}
-
-impl Handler<LocateRequest> for Locator {
-    type Result = ();
-
-    fn handle(&mut self, msg: LocateRequest, _: &mut Self::Context) {
-        let LocateRequest { ip, nid, chain } = msg;
-
-        if let Some(item) = self.cache.read().get(&ip) {
-            if let Some(location) = item {
-                return chain.do_send(LocateNode {
-                    nid,
-                    location: location.clone(),
-                });
-            }
-
-            return;
-        }
-
-        let location = match self.iplocate(ip) {
-            Ok(location) => location,
-            Err(err) => return log::debug!("GET error for ip location: {:?}", err),
-        };
-
-        self.cache.write().insert(ip, location.clone());
-
-        if let Some(location) = location {
-            chain.do_send(LocateNode { nid, location });
-        }
-    }
-}
-
-impl Locator {
-    fn iplocate(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
-        let location = self.iplocate_ipapi_co(ip)?;
-
-        match location {
-            Some(location) => Ok(Some(location)),
-            None => self.iplocate_ipinfo_io(ip),
-        }
-    }
-
-    fn iplocate_ipapi_co(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
-        let location = self
-            .query(&format!("https://ipapi.co/{}/json", ip))?
-            .map(Arc::new);
-
-        Ok(location)
-    }
-
-    fn iplocate_ipinfo_io(
-        &self,
-        ip: Ipv4Addr,
-    ) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
-        let location = self
-            .query(&format!("https://ipinfo.io/{}/json", ip))?
-            .and_then(|loc: IPApiLocate| loc.into_node_location().map(Arc::new));
-
-        Ok(location)
-    }
-
-    fn query<T>(&self, url: &str) -> Result<Option<T>, reqwest::Error>
-    where
-        for<'de> T: Deserialize<'de>,
-    {
-        match self.client.get(url).send()?.json::<T>() {
-            Ok(result) => Ok(Some(result)),
-            Err(err) => {
-                log::debug!("JSON error for ip location: {:?}", err);
-                Ok(None)
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn ipapi_locate_to_node_location() {
-        let ipapi = IPApiLocate {
-            loc: "12.5,56.25".into(),
-            city: "Foobar".into(),
-        };
-
-        let location = ipapi.into_node_location().unwrap();
-
-        assert_eq!(location.latitude, 12.5);
-        assert_eq!(location.longitude, 56.25);
-        assert_eq!(&*location.city, "Foobar");
-    }
-
-    #[test]
-    fn ipapi_locate_to_node_location_too_many() {
-        let ipapi = IPApiLocate {
-            loc: "12.5,56.25,1.0".into(),
-            city: "Foobar".into(),
-        };
-
-        let location = ipapi.into_node_location();
-
-        assert!(location.is_none());
-    }
-}

backend/telemetry_core/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "telemetry_core"
+version = "0.1.0"
+authors = ["Parity Technologies Ltd. <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0"
+
+[dependencies]
+anyhow = "1.0.41"
+bimap = "0.6.1"
+bincode = "1.3.3"
+bytes = "1.0.1"
+common = { path = "../common" }
+futures = "0.3.15"
+hex = "0.4.3"
+http = "0.2.4"
+log = "0.4.14"
+once_cell = "1.8.0"
+parking_lot = "0.11.1"
+primitive-types = { version = "0.9.0", features = ["serde"] }
+reqwest = { version = "0.11.4", features = ["json"] }
+rustc-hash = "1.1.0"
+serde = { version = "1.0.126", features = ["derive"] }
+serde_json = "1.0.64"
+simple_logger = "1.11.0"
+smallvec = "1.6.1"
+soketto = "0.6.0"
+structopt = "0.3.21"
+thiserror = "1.0.25"
+tokio = { version = "1.7.0", features = ["full"] }
+tokio-util = { version = "0.6", features = ["compat"] }
+warp = "0.3.1"
+
+[dev-dependencies]
+criterion = { version = "0.3.4", features = ["async", "async_tokio"] }
+shellwords = "1.1.0"
+test_utils = { path = "../test_utils" }
+
+[[bench]]
+name = "throughput"
+harness = false

backend/telemetry_core/benches/throughput.rs

@@ -0,0 +1,98 @@
+use std::iter::FromIterator;
+
+use futures::StreamExt;
+use test_utils::workspace::start_server_release;
+use criterion::{criterion_group, criterion_main, Criterion};
+use tokio::runtime::Runtime;
+use serde_json::json;
+use common::node_types::BlockHash;
+
+pub fn benchmark_throughput_single_shard(c: &mut Criterion) {
+    /*
+    let rt = Runtime::new().expect("tokio runtime should start");
+
+    // Setup our server and node/feed connections first:
+    let (nodes, feeds) = rt.block_on(async {
+        let mut server = start_server_release().await;
+        let shard_id = server.add_shard().await.unwrap();
+
+        // Connect 1000 nodes to the shard:
+        let mut nodes = server
+            .get_shard(shard_id)
+            .unwrap()
+            .connect_multiple(1000)
+            .await
+            .expect("nodes can connect");
+
+        // Every node announces itself on the same chain:
+        for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
+            node_tx.send_json_text(json!({
+                "id":1, // message ID, not node ID. Can be the same for all.
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain":"Local Testnet",
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(1),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name": format!("Alice {}", idx),
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                }
+            })).await.unwrap();
+        }
+tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+        // Start 1000 feeds:
+        let mut feeds = server
+            .get_core()
+            .connect_multiple(1)
+            .await
+            .expect("feeds can connect");
+
+        // // Subscribe all feeds to the chain:
+        // for (feed_tx, _) in feeds.iter_mut() {
+        //     feed_tx.send_command("subscribe", "Local Testnet").await.unwrap();
+        // }
+
+println!("consuming feed");
+{
+
+    let mut msgs = futures::stream::FuturesUnordered::from_iter(
+        feeds
+        .iter_mut()
+        .map(|(_,rx)| rx.recv_feed_messages())
+    );
+
+    let mut n = 0;
+    while let Some(Ok(msg)) = msgs.next().await {
+        n += 1;
+        println!("Message {}: {:?}", n, msg);
+    }
+}
+
+        // // Consume any messages feeds have received so far (every feed should havea few at least):
+        // let feed_consumers = feeds
+        //     .iter_mut()
+        //     .map(|(_,rx)| rx.next());
+        // futures::future::join_all(feed_consumers).await;
+println!("feed consumed");
+        (nodes, feeds)
+    });
+
+    // Next, run criterion using the same tokio runtime to benchmark time taken to send
+    // messages to nodes and receive them from feeds.
+    c.bench_function(
+        "throughput time",
+        |b| b.to_async(&rt).iter(|| async {
+
+            // TODO: Actually implement the benchmark.
+
+        })
+    );
+    */
+}
+
+criterion_group!(benches, benchmark_throughput_single_shard);
+criterion_main!(benches);

backend/telemetry_core/src/aggregator/aggregator.rs

@@ -0,0 +1,117 @@
+use super::inner_loop;
+use crate::find_location::find_location;
+use crate::state::NodeId;
+use common::id_type;
+use futures::channel::mpsc;
+use futures::{future, Sink, SinkExt};
+use std::net::Ipv4Addr;
+use std::sync::atomic::AtomicU64;
+use std::sync::Arc;
+
+id_type! {
+    /// A unique Id is assigned per websocket connection (or more accurately,
+    /// per feed socket and per shard socket). This can be combined with the
+    /// [`LocalId`] of messages to give us a global ID.
+    pub struct ConnId(u64)
+}
+
+#[derive(Clone)]
+pub struct Aggregator(Arc<AggregatorInternal>);
+
+struct AggregatorInternal {
+    /// Shards that connect are each assigned a unique connection ID.
+    /// This helps us know who to send messages back to (especially in
+    /// conjunction with the [`LocalId`] that messages will come with).
+    shard_conn_id: AtomicU64,
+    /// Feeds that connect have their own unique connection ID, too.
+    feed_conn_id: AtomicU64,
+    /// Send messages in to the aggregator from the outside via this. This is
+    /// stored here so that anybody holding an `Aggregator` handle can
+    /// make use of it.
+    tx_to_aggregator: mpsc::UnboundedSender<inner_loop::ToAggregator>,
+}
+
+impl Aggregator {
+    /// Spawn a new Aggregator. This connects to the telemetry backend
+    pub async fn spawn(denylist: Vec<String>) -> anyhow::Result<Aggregator> {
+        let (tx_to_aggregator, rx_from_external) = mpsc::unbounded();
+
+        // Kick off a locator task to locate nodes, which hands back a channel to make location requests
+        let tx_to_locator = find_location(tx_to_aggregator.clone().with(|(node_id, msg)| {
+            future::ok::<_, mpsc::SendError>(inner_loop::ToAggregator::FromFindLocation(
+                node_id, msg,
+            ))
+        }));
+
+        // Handle any incoming messages in our handler loop:
+        tokio::spawn(Aggregator::handle_messages(
+            rx_from_external,
+            tx_to_locator,
+            denylist,
+        ));
+
+        // Return a handle to our aggregator:
+        Ok(Aggregator(Arc::new(AggregatorInternal {
+            shard_conn_id: AtomicU64::new(1),
+            feed_conn_id: AtomicU64::new(1),
+            tx_to_aggregator,
+        })))
+    }
+
+    // This is spawned into a separate task and handles any messages coming
+    // in to the aggregator. If nobody is tolding the tx side of the channel
+    // any more, this task will gracefully end.
+    async fn handle_messages(
+        rx_from_external: mpsc::UnboundedReceiver<inner_loop::ToAggregator>,
+        tx_to_aggregator: mpsc::UnboundedSender<(NodeId, Ipv4Addr)>,
+        denylist: Vec<String>,
+    ) {
+        inner_loop::InnerLoop::new(rx_from_external, tx_to_aggregator, denylist)
+            .handle()
+            .await;
+    }
+
+    /// Return a sink that a shard can send messages into to be handled by the aggregator.
+    pub fn subscribe_shard(
+        &self,
+    ) -> impl Sink<inner_loop::FromShardWebsocket, Error = anyhow::Error> + Unpin {
+        // Assign a unique aggregator-local ID to each connection that subscribes, and pass
+        // that along with every message to the aggregator loop:
+        let shard_conn_id = self
+            .0
+            .shard_conn_id
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let tx_to_aggregator = self.0.tx_to_aggregator.clone();
+
+        // Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
+        // but pinning by boxing is the easy solution for now:
+        Box::pin(tx_to_aggregator.with(move |msg| async move {
+            Ok(inner_loop::ToAggregator::FromShardWebsocket(
+                shard_conn_id.into(),
+                msg,
+            ))
+        }))
+    }
+
+    /// Return a sink that a feed can send messages into to be handled by the aggregator.
+    pub fn subscribe_feed(
+        &self,
+    ) -> impl Sink<inner_loop::FromFeedWebsocket, Error = anyhow::Error> + Unpin {
+        // Assign a unique aggregator-local ID to each connection that subscribes, and pass
+        // that along with every message to the aggregator loop:
+        let feed_conn_id = self
+            .0
+            .feed_conn_id
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let tx_to_aggregator = self.0.tx_to_aggregator.clone();
+
+        // Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
+        // but pinning by boxing is the easy solution for now:
+        Box::pin(tx_to_aggregator.with(move |msg| async move {
+            Ok(inner_loop::ToAggregator::FromFeedWebsocket(
+                feed_conn_id.into(),
+                msg,
+            ))
+        }))
+    }
+}

backend/telemetry_core/src/aggregator/inner_loop.rs

@@ -0,0 +1,611 @@
+use super::aggregator::ConnId;
+use crate::feed_message::{self, FeedMessageSerializer};
+use crate::find_location;
+use crate::state::{self, NodeId, State};
+use bimap::BiMap;
+use common::{
+    internal_messages::{self, MuteReason, ShardNodeId},
+    node_message,
+    node_types::BlockHash,
+    time,
+};
+use futures::channel::mpsc;
+use futures::StreamExt;
+use std::collections::{HashMap, HashSet};
+use std::{
+    net::{IpAddr, Ipv4Addr},
+    str::FromStr,
+};
+
+/// Incoming messages come via subscriptions, and end up looking like this.
+#[derive(Clone, Debug)]
+pub enum ToAggregator {
+    FromShardWebsocket(ConnId, FromShardWebsocket),
+    FromFeedWebsocket(ConnId, FromFeedWebsocket),
+    FromFindLocation(NodeId, find_location::Location),
+}
+
+/// An incoming shard connection can send these messages to the aggregator.
+#[derive(Clone, Debug)]
+pub enum FromShardWebsocket {
+    /// When the socket is opened, it'll send this first
+    /// so that we have a way to communicate back to it.
+    Initialize {
+        channel: mpsc::UnboundedSender<ToShardWebsocket>,
+    },
+    /// Tell the aggregator about a new node.
+    Add {
+        local_id: ShardNodeId,
+        ip: Option<std::net::IpAddr>,
+        node: common::node_types::NodeDetails,
+        genesis_hash: common::node_types::BlockHash,
+    },
+    /// Update/pass through details about a node.
+    Update {
+        local_id: ShardNodeId,
+        payload: node_message::Payload,
+    },
+    /// Tell the aggregator that a node has been removed when it disconnects.
+    Remove { local_id: ShardNodeId },
+    /// The shard is disconnected.
+    Disconnected,
+}
+
+/// The aggregator can these messages back to a shard connection.
+#[derive(Debug)]
+pub enum ToShardWebsocket {
+    /// Mute messages to the core by passing the shard-local ID of them.
+    Mute {
+        local_id: ShardNodeId,
+        reason: internal_messages::MuteReason,
+    },
+}
+
+/// An incoming feed connection can send these messages to the aggregator.
+#[derive(Clone, Debug)]
+pub enum FromFeedWebsocket {
+    /// When the socket is opened, it'll send this first
+    /// so that we have a way to communicate back to it.
+    /// Unbounded so that slow feeds don't block aggregato
+    /// progress.
+    Initialize {
+        channel: mpsc::UnboundedSender<ToFeedWebsocket>,
+    },
+    /// The feed can subscribe to a chain to receive
+    /// messages relating to it.
+    Subscribe { chain: Box<str> },
+    /// The feed wants finality info for the chain, too.
+    SendFinality,
+    /// The feed doesn't want any more finality info for the chain.
+    NoMoreFinality,
+    /// An explicit ping message.
+    Ping { value: Box<str> },
+    /// The feed is disconnected.
+    Disconnected,
+}
+
+// The frontend sends text based commands; parse them into these messages:
+impl FromStr for FromFeedWebsocket {
+    type Err = anyhow::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (cmd, value) = match s.find(':') {
+            Some(idx) => (&s[..idx], s[idx + 1..].into()),
+            None => return Err(anyhow::anyhow!("Expecting format `CMD:CHAIN_NAME`")),
+        };
+        match cmd {
+            "ping" => Ok(FromFeedWebsocket::Ping { value }),
+            "subscribe" => Ok(FromFeedWebsocket::Subscribe { chain: value }),
+            "send-finality" => Ok(FromFeedWebsocket::SendFinality),
+            "no-more-finality" => Ok(FromFeedWebsocket::NoMoreFinality),
+            _ => return Err(anyhow::anyhow!("Command {} not recognised", cmd)),
+        }
+    }
+}
+
+/// The aggregator can these messages back to a feed connection.
+#[derive(Clone, Debug)]
+pub enum ToFeedWebsocket {
+    Bytes(bytes::Bytes),
+}
+
+/// Instances of this are responsible for handling incoming and
+/// outgoing messages in the main aggregator loop.
+pub struct InnerLoop {
+    /// Messages from the outside world come into this:
+    rx_from_external: mpsc::UnboundedReceiver<ToAggregator>,
+
+    /// The state of our chains and nodes lives here:
+    node_state: State,
+    /// We maintain a mapping between NodeId and ConnId+LocalId, so that we know
+    /// which messages are about which nodes.
+    node_ids: BiMap<NodeId, (ConnId, ShardNodeId)>,
+
+    /// Keep track of how to send messages out to feeds.
+    feed_channels: HashMap<ConnId, mpsc::UnboundedSender<ToFeedWebsocket>>,
+    /// Keep track of how to send messages out to shards.
+    shard_channels: HashMap<ConnId, mpsc::UnboundedSender<ToShardWebsocket>>,
+
+    /// Which chain is a feed subscribed to?
+    /// Feed Connection ID -> Chain Genesis Hash
+    feed_conn_id_to_chain: HashMap<ConnId, BlockHash>,
+    /// Which feeds are subscribed to a given chain (needs to stay in sync with above)?
+    /// Chain Genesis Hash -> Feed Connection IDs
+    chain_to_feed_conn_ids: HashMap<BlockHash, HashSet<ConnId>>,
+
+    /// These feeds want finality info, too.
+    feed_conn_id_finality: HashSet<ConnId>,
+
+    /// Send messages here to make geographical location requests.
+    tx_to_locator: mpsc::UnboundedSender<(NodeId, Ipv4Addr)>,
+}
+
+impl InnerLoop {
+    /// Create a new inner loop handler with the various state it needs.
+    pub fn new(
+        rx_from_external: mpsc::UnboundedReceiver<ToAggregator>,
+        tx_to_locator: mpsc::UnboundedSender<(NodeId, Ipv4Addr)>,
+        denylist: Vec<String>,
+    ) -> Self {
+        InnerLoop {
+            rx_from_external,
+            node_state: State::new(denylist),
+            node_ids: BiMap::new(),
+            feed_channels: HashMap::new(),
+            shard_channels: HashMap::new(),
+            feed_conn_id_to_chain: HashMap::new(),
+            chain_to_feed_conn_ids: HashMap::new(),
+            feed_conn_id_finality: HashSet::new(),
+            tx_to_locator,
+        }
+    }
+
+    /// Start handling and responding to incoming messages. Owing to unbounded channels, we actually
+    /// only have a single `.await` (in this function). This helps to make it clear that the aggregator loop
+    /// will be able to make progress quickly without any potential yield points.
+    pub async fn handle(mut self) {
+        while let Some(msg) = self.rx_from_external.next().await {
+            match msg {
+                ToAggregator::FromFeedWebsocket(feed_conn_id, msg) => {
+                    self.handle_from_feed(feed_conn_id, msg)
+                }
+                ToAggregator::FromShardWebsocket(shard_conn_id, msg) => {
+                    self.handle_from_shard(shard_conn_id, msg)
+                }
+                ToAggregator::FromFindLocation(node_id, location) => {
+                    self.handle_from_find_location(node_id, location)
+                }
+            }
+        }
+    }
+
+    /// Handle messages that come from the node geographical locator.
+    fn handle_from_find_location(
+        &mut self,
+        node_id: NodeId,
+        location: find_location::Location,
+    ) {
+        self.node_state
+            .update_node_location(node_id, location.clone());
+
+        if let Some(loc) = location {
+            let mut feed_message_serializer = FeedMessageSerializer::new();
+            feed_message_serializer.push(feed_message::LocatedNode(
+                node_id.get_chain_node_id().into(),
+                loc.latitude,
+                loc.longitude,
+                &loc.city,
+            ));
+
+            let chain_genesis_hash = self
+                .node_state
+                .get_chain_by_node_id(node_id)
+                .map(|chain| *chain.genesis_hash());
+
+            if let Some(chain_genesis_hash) = chain_genesis_hash {
+                self.finalize_and_broadcast_to_chain_feeds(
+                    &chain_genesis_hash,
+                    feed_message_serializer,
+                );
+            }
+        }
+    }
+
+    /// Handle messages coming from shards.
+    fn handle_from_shard(&mut self, shard_conn_id: ConnId, msg: FromShardWebsocket) {
+        log::debug!("Message from shard ({:?}): {:?}", shard_conn_id, msg);
+
+        match msg {
+            FromShardWebsocket::Initialize { channel } => {
+                self.shard_channels.insert(shard_conn_id, channel);
+            }
+            FromShardWebsocket::Add {
+                local_id,
+                ip,
+                node,
+                genesis_hash,
+            } => {
+                match self.node_state.add_node(genesis_hash, node) {
+                    state::AddNodeResult::ChainOnDenyList => {
+                        if let Some(shard_conn) = self.shard_channels.get_mut(&shard_conn_id) {
+                            let _ = shard_conn
+                                .unbounded_send(ToShardWebsocket::Mute {
+                                    local_id,
+                                    reason: MuteReason::ChainNotAllowed,
+                                });
+                        }
+                    }
+                    state::AddNodeResult::ChainOverQuota => {
+                        if let Some(shard_conn) = self.shard_channels.get_mut(&shard_conn_id) {
+                            let _ = shard_conn
+                                .unbounded_send(ToShardWebsocket::Mute {
+                                    local_id,
+                                    reason: MuteReason::Overquota,
+                                });
+                        }
+                    }
+                    state::AddNodeResult::NodeAddedToChain(details) => {
+                        let node_id = details.id;
+
+                        // Record ID <-> (shardId,localId) for future messages:
+                        self.node_ids.insert(node_id, (shard_conn_id, local_id));
+
+                        // Don't hold onto details too long because we want &mut self later:
+                        let old_chain_label = details.old_chain_label.to_owned();
+                        let new_chain_label = details.new_chain_label.to_owned();
+                        let chain_node_count = details.chain_node_count;
+                        let has_chain_label_changed = details.has_chain_label_changed;
+
+                        // Tell chain subscribers about the node we've just added:
+                        let mut feed_messages_for_chain = FeedMessageSerializer::new();
+                        feed_messages_for_chain.push(feed_message::AddedNode(
+                            node_id.get_chain_node_id().into(),
+                            &details.node,
+                        ));
+                        self.finalize_and_broadcast_to_chain_feeds(
+                            &genesis_hash,
+                            feed_messages_for_chain,
+                        );
+                        // Tell everybody about the new node count and potential rename:
+                        let mut feed_messages_for_all = FeedMessageSerializer::new();
+                        if has_chain_label_changed {
+                            feed_messages_for_all
+                                .push(feed_message::RemovedChain(&old_chain_label));
+                        }
+                        feed_messages_for_all
+                            .push(feed_message::AddedChain(&new_chain_label, chain_node_count));
+                        self.finalize_and_broadcast_to_all_feeds(feed_messages_for_all);
+
+                        // Ask for the grographical location of the node.
+                        // Currently we only geographically locate IPV4 addresses so ignore IPV6.
+                        if let Some(IpAddr::V4(ip_v4)) = ip {
+                            let _ = self.tx_to_locator.unbounded_send((node_id, ip_v4));
+                        }
+                    }
+                }
+            }
+            FromShardWebsocket::Remove { local_id } => {
+                let node_id = match self.node_ids.remove_by_right(&(shard_conn_id, local_id)) {
+                    Some((node_id, _)) => node_id,
+                    None => {
+                        log::error!(
+                            "Cannot find ID for node with shard/connectionId of {:?}/{:?}",
+                            shard_conn_id,
+                            local_id
+                        );
+                        return;
+                    }
+                };
+                self.remove_nodes_and_broadcast_result(Some(node_id));
+            }
+            FromShardWebsocket::Update { local_id, payload } => {
+                let node_id = match self.node_ids.get_by_right(&(shard_conn_id, local_id)) {
+                    Some(id) => *id,
+                    None => {
+                        log::error!(
+                            "Cannot find ID for node with shard/connectionId of {:?}/{:?}",
+                            shard_conn_id,
+                            local_id
+                        );
+                        return;
+                    }
+                };
+
+                let mut feed_message_serializer = FeedMessageSerializer::new();
+                let broadcast_finality =
+                    self.node_state
+                        .update_node(node_id, payload, &mut feed_message_serializer);
+
+                if let Some(chain) = self.node_state.get_chain_by_node_id(node_id) {
+                    let genesis_hash = *chain.genesis_hash();
+                    if broadcast_finality {
+                        self.finalize_and_broadcast_to_chain_finality_feeds(
+                            &genesis_hash,
+                            feed_message_serializer,
+                        );
+                    } else {
+                        self.finalize_and_broadcast_to_chain_feeds(
+                            &genesis_hash,
+                            feed_message_serializer,
+                        );
+                    }
+                }
+            }
+            FromShardWebsocket::Disconnected => {
+                // Find all nodes associated with this shard connection ID:
+                let node_ids_to_remove: Vec<NodeId> = self
+                    .node_ids
+                    .iter()
+                    .filter(|(_, &(this_shard_conn_id, _))| shard_conn_id == this_shard_conn_id)
+                    .map(|(&node_id, _)| node_id)
+                    .collect();
+
+                // ... and remove them:
+                self.remove_nodes_and_broadcast_result(node_ids_to_remove);
+            }
+        }
+    }
+
+    /// Handle messages coming from feeds.
+    fn handle_from_feed(&mut self, feed_conn_id: ConnId, msg: FromFeedWebsocket) {
+        log::debug!("Message from feed ({:?}): {:?}", feed_conn_id, msg);
+        match msg {
+            FromFeedWebsocket::Initialize { channel } => {
+                self.feed_channels.insert(feed_conn_id, channel.clone());
+
+                // Tell the new feed subscription some basic things to get it going:
+                let mut feed_serializer = FeedMessageSerializer::new();
+                feed_serializer.push(feed_message::Version(31));
+                for chain in self.node_state.iter_chains() {
+                    feed_serializer
+                        .push(feed_message::AddedChain(chain.label(), chain.node_count()));
+                }
+
+                // Send this to the channel that subscribed:
+                if let Some(bytes) = feed_serializer.into_finalized() {
+                    let _ = channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                }
+            }
+            FromFeedWebsocket::Ping { value } => {
+                let feed_channel = match self.feed_channels.get_mut(&feed_conn_id) {
+                    Some(chan) => chan,
+                    None => return,
+                };
+
+                // Pong!
+                let mut feed_serializer = FeedMessageSerializer::new();
+                feed_serializer.push(feed_message::Pong(&value));
+                if let Some(bytes) = feed_serializer.into_finalized() {
+                    let _ = feed_channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                }
+            }
+            FromFeedWebsocket::Subscribe { chain } => {
+                let feed_channel = match self.feed_channels.get_mut(&feed_conn_id) {
+                    Some(chan) => chan,
+                    None => return,
+                };
+
+                // Unsubscribe from previous chain if subscribed to one:
+                let old_genesis_hash = self.feed_conn_id_to_chain.remove(&feed_conn_id);
+                if let Some(old_genesis_hash) = &old_genesis_hash {
+                    if let Some(map) = self.chain_to_feed_conn_ids.get_mut(old_genesis_hash) {
+                        map.remove(&feed_conn_id);
+                    }
+                }
+
+                // Untoggle request for finality feeds:
+                self.feed_conn_id_finality.remove(&feed_conn_id);
+
+                // Get old chain if there was one:
+                let node_state = &self.node_state;
+                let old_chain =
+                    old_genesis_hash.and_then(|hash| node_state.get_chain_by_genesis_hash(&hash));
+
+                // Get new chain, ignoring the rest if it doesn't exist.
+                let new_chain = match self.node_state.get_chain_by_label(&chain) {
+                    Some(chain) => chain,
+                    None => return,
+                };
+
+                // Send messages to the feed about this subscription:
+                let mut feed_serializer = FeedMessageSerializer::new();
+                if let Some(old_chain) = old_chain {
+                    feed_serializer.push(feed_message::UnsubscribedFrom(old_chain.label()));
+                }
+                feed_serializer.push(feed_message::SubscribedTo(new_chain.label()));
+                feed_serializer.push(feed_message::TimeSync(time::now()));
+                feed_serializer.push(feed_message::BestBlock(
+                    new_chain.best_block().height,
+                    new_chain.timestamp(),
+                    new_chain.average_block_time(),
+                ));
+                feed_serializer.push(feed_message::BestFinalized(
+                    new_chain.finalized_block().height,
+                    new_chain.finalized_block().hash,
+                ));
+                for (idx, (chain_node_id, node)) in new_chain.iter_nodes().enumerate() {
+                    let chain_node_id = chain_node_id.into();
+
+                    // Send subscription confirmation and chain head before doing all the nodes,
+                    // and continue sending batches of 32 nodes a time over the wire subsequently
+                    if idx % 32 == 0 {
+                        if let Some(bytes) = feed_serializer.finalize() {
+                            let _ = feed_channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                        }
+                    }
+                    feed_serializer.push(feed_message::AddedNode(chain_node_id, node));
+                    feed_serializer.push(feed_message::FinalizedBlock(
+                        chain_node_id,
+                        node.finalized().height,
+                        node.finalized().hash,
+                    ));
+                    if node.stale() {
+                        feed_serializer.push(feed_message::StaleNode(chain_node_id));
+                    }
+                }
+                if let Some(bytes) = feed_serializer.into_finalized() {
+                    let _ = feed_channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                }
+
+                // Actually make a note of the new chain subsciption:
+                let new_genesis_hash = *new_chain.genesis_hash();
+                self.feed_conn_id_to_chain
+                    .insert(feed_conn_id, new_genesis_hash);
+                self.chain_to_feed_conn_ids
+                    .entry(new_genesis_hash)
+                    .or_default()
+                    .insert(feed_conn_id);
+            }
+            FromFeedWebsocket::SendFinality => {
+                self.feed_conn_id_finality.insert(feed_conn_id);
+            }
+            FromFeedWebsocket::NoMoreFinality => {
+                self.feed_conn_id_finality.remove(&feed_conn_id);
+            }
+            FromFeedWebsocket::Disconnected => {
+                // The feed has disconnected; clean up references to it:
+                if let Some(chain) = self.feed_conn_id_to_chain.remove(&feed_conn_id) {
+                    self.chain_to_feed_conn_ids.remove(&chain);
+                }
+                self.feed_channels.remove(&feed_conn_id);
+                self.feed_conn_id_finality.remove(&feed_conn_id);
+            }
+        }
+    }
+
+    /// Remove all of the node IDs provided and broadcast messages to feeds as needed.
+    fn remove_nodes_and_broadcast_result(
+        &mut self,
+        node_ids: impl IntoIterator<Item = NodeId>,
+    ) {
+        // Group by chain to simplify the handling of feed messages:
+        let mut node_ids_per_chain: HashMap<BlockHash, Vec<NodeId>> = HashMap::new();
+        for node_id in node_ids.into_iter() {
+            if let Some(chain) = self.node_state.get_chain_by_node_id(node_id) {
+                node_ids_per_chain
+                    .entry(*chain.genesis_hash())
+                    .or_default()
+                    .push(node_id);
+            }
+        }
+
+        // Remove the nodes for each chain
+        let mut feed_messages_for_all = FeedMessageSerializer::new();
+        for (chain_label, node_ids) in node_ids_per_chain {
+            let mut feed_messages_for_chain = FeedMessageSerializer::new();
+            for node_id in node_ids {
+                self.remove_node(
+                    node_id,
+                    &mut feed_messages_for_chain,
+                    &mut feed_messages_for_all,
+                );
+            }
+            self.finalize_and_broadcast_to_chain_feeds(&chain_label, feed_messages_for_chain);
+        }
+        self.finalize_and_broadcast_to_all_feeds(feed_messages_for_all);
+    }
+
+    /// Remove a single node by its ID, pushing any messages we'd want to send
+    /// out to feeds onto the provided feed serializers. Doesn't actually send
+    /// anything to the feeds; just updates state as needed.
+    fn remove_node(
+        &mut self,
+        node_id: NodeId,
+        feed_for_chain: &mut FeedMessageSerializer,
+        feed_for_all: &mut FeedMessageSerializer,
+    ) {
+        // Remove our top level association (this may already have been done).
+        self.node_ids.remove_by_left(&node_id);
+
+        let removed_details = match self.node_state.remove_node(node_id) {
+            Some(remove_details) => remove_details,
+            None => {
+                log::error!("Could not find node {:?}", node_id);
+                return;
+            }
+        };
+
+        // The chain has been removed (no nodes left in it, or it was renamed):
+        if removed_details.chain_node_count == 0 || removed_details.has_chain_label_changed {
+            feed_for_all.push(feed_message::RemovedChain(&removed_details.old_chain_label));
+        }
+
+        // If the chain still exists, tell everybody about the new label or updated node count:
+        if removed_details.chain_node_count != 0 {
+            feed_for_all.push(feed_message::AddedChain(
+                &removed_details.new_chain_label,
+                removed_details.chain_node_count,
+            ));
+        }
+
+        // Assuming the chain hasn't gone away, tell chain subscribers about the node removal
+        if removed_details.chain_node_count != 0 {
+            feed_for_chain.push(feed_message::RemovedNode(
+                node_id.get_chain_node_id().into(),
+            ));
+        }
+    }
+
+    /// Finalize a [`FeedMessageSerializer`] and broadcast the result to feeds for the chain.
+    fn finalize_and_broadcast_to_chain_feeds(
+        &mut self,
+        genesis_hash: &BlockHash,
+        serializer: FeedMessageSerializer,
+    ) {
+        if let Some(bytes) = serializer.into_finalized() {
+            self.broadcast_to_chain_feeds(genesis_hash, ToFeedWebsocket::Bytes(bytes));
+        }
+    }
+
+    /// Send a message to all chain feeds.
+    fn broadcast_to_chain_feeds(&mut self, genesis_hash: &BlockHash, message: ToFeedWebsocket) {
+        if let Some(feeds) = self.chain_to_feed_conn_ids.get(genesis_hash) {
+            for &feed_id in feeds {
+                if let Some(chan) = self.feed_channels.get_mut(&feed_id) {
+                    let _ = chan.unbounded_send(message.clone());
+                }
+            }
+        }
+    }
+
+    /// Finalize a [`FeedMessageSerializer`] and broadcast the result to all feeds
+    fn finalize_and_broadcast_to_all_feeds(&mut self, serializer: FeedMessageSerializer) {
+        if let Some(bytes) = serializer.into_finalized() {
+            self.broadcast_to_all_feeds(ToFeedWebsocket::Bytes(bytes));
+        }
+    }
+
+    /// Send a message to everybody.
+    fn broadcast_to_all_feeds(&mut self, message: ToFeedWebsocket) {
+        for chan in self.feed_channels.values_mut() {
+            let _ = chan.unbounded_send(message.clone());
+        }
+    }
+
+    /// Finalize a [`FeedMessageSerializer`] and broadcast the result to chain finality feeds
+    fn finalize_and_broadcast_to_chain_finality_feeds(
+        &mut self,
+        genesis_hash: &BlockHash,
+        serializer: FeedMessageSerializer,
+    ) {
+        if let Some(bytes) = serializer.into_finalized() {
+            self.broadcast_to_chain_finality_feeds(genesis_hash, ToFeedWebsocket::Bytes(bytes));
+        }
+    }
+
+    /// Send a message to all chain finality feeds.
+    fn broadcast_to_chain_finality_feeds(
+        &mut self,
+        genesis_hash: &BlockHash,
+        message: ToFeedWebsocket,
+    ) {
+        if let Some(feeds) = self.chain_to_feed_conn_ids.get(genesis_hash) {
+            // Get all feeds for the chain, but only broadcast to those feeds that
+            // are also subscribed to receive finality updates.
+            for &feed_id in feeds.union(&self.feed_conn_id_finality) {
+                if let Some(chan) = self.feed_channels.get_mut(&feed_id) {
+                    let _ = chan.unbounded_send(message.clone());
+                }
+            }
+        }
+    }
+}

backend/telemetry_core/src/aggregator/mod.rs

@@ -0,0 +1,7 @@
+mod aggregator;
+mod inner_loop;
+
+// Expose the various message types that can be worked with externally:
+pub use inner_loop::{FromFeedWebsocket, FromShardWebsocket, ToFeedWebsocket, ToShardWebsocket};
+
+pub use aggregator::*;

backend/telemetry_core/src/feed_message.rs

@@ -0,0 +1,234 @@
+//! This module provides a way of encoding the various messages that we'll
+//! send to subscribed feeds (browsers).
+
+use serde::Serialize;
+use std::mem;
+
+use crate::state::Node;
+use common::node_types::{
+    BlockDetails, BlockHash, BlockNumber, NodeHardware, NodeIO, NodeStats, Timestamp,
+};
+use serde_json::to_writer;
+
+type Address = Box<str>;
+type FeedNodeId = usize;
+
+pub trait FeedMessage {
+    const ACTION: u8;
+}
+
+pub trait FeedMessageWrite: FeedMessage {
+    fn write_to_feed(&self, ser: &mut FeedMessageSerializer);
+}
+
+impl<T> FeedMessageWrite for T
+where
+    T: FeedMessage + Serialize,
+{
+    fn write_to_feed(&self, ser: &mut FeedMessageSerializer) {
+        ser.write(self)
+    }
+}
+
+pub struct FeedMessageSerializer {
+    /// Current buffer,
+    buffer: Vec<u8>,
+}
+
+const BUFCAP: usize = 128;
+
+impl FeedMessageSerializer {
+    pub fn new() -> Self {
+        Self {
+            buffer: Vec::with_capacity(BUFCAP),
+        }
+    }
+
+    pub fn push<Message>(&mut self, msg: Message)
+    where
+        Message: FeedMessageWrite,
+    {
+        let glue = match self.buffer.len() {
+            0 => b'[',
+            _ => b',',
+        };
+
+        self.buffer.push(glue);
+        self.write(&Message::ACTION);
+        self.buffer.push(b',');
+        msg.write_to_feed(self);
+    }
+
+    fn write<S>(&mut self, value: &S)
+    where
+        S: Serialize,
+    {
+        let _ = to_writer(&mut self.buffer, value);
+    }
+
+    /// Return the bytes we've serialized so far and prepare a new buffer. If you're
+    /// finished serializing data, prefer [`FeedMessageSerializer::into_finalized`]
+    pub fn finalize(&mut self) -> Option<bytes::Bytes> {
+        if self.buffer.is_empty() {
+            return None;
+        }
+
+        self.buffer.push(b']');
+
+        let bytes = mem::replace(&mut self.buffer, Vec::with_capacity(BUFCAP));
+
+        Some(bytes.into())
+    }
+
+    /// Return the bytes that we've serialized so far, consuming the serializer.
+    pub fn into_finalized(mut self) -> Option<bytes::Bytes> {
+        if self.buffer.is_empty() {
+            return None;
+        }
+
+        self.buffer.push(b']');
+        Some(self.buffer.into())
+    }
+}
+
+macro_rules! actions {
+    ($($action:literal: $t:ty,)*) => {
+        $(
+            impl FeedMessage for $t {
+                const ACTION: u8 = $action;
+            }
+        )*
+    }
+}
+
+actions! {
+     0: Version,
+     1: BestBlock,
+     2: BestFinalized,
+     3: AddedNode<'_>,
+     4: RemovedNode,
+     5: LocatedNode<'_>,
+     6: ImportedBlock<'_>,
+     7: FinalizedBlock,
+     8: NodeStatsUpdate<'_>,
+     9: Hardware<'_>,
+    10: TimeSync,
+    11: AddedChain<'_>,
+    12: RemovedChain<'_>,
+    13: SubscribedTo<'_>,
+    14: UnsubscribedFrom<'_>,
+    15: Pong<'_>,
+    16: AfgFinalized,
+    17: AfgReceivedPrevote,
+    18: AfgReceivedPrecommit,
+    19: AfgAuthoritySet,
+    20: StaleNode,
+    21: NodeIOUpdate<'_>,
+}
+
+#[derive(Serialize)]
+pub struct Version(pub usize);
+
+#[derive(Serialize)]
+pub struct BestBlock(pub BlockNumber, pub Timestamp, pub Option<u64>);
+
+#[derive(Serialize)]
+pub struct BestFinalized(pub BlockNumber, pub BlockHash);
+
+pub struct AddedNode<'a>(pub FeedNodeId, pub &'a Node);
+
+#[derive(Serialize)]
+pub struct RemovedNode(pub FeedNodeId);
+
+#[derive(Serialize)]
+pub struct LocatedNode<'a>(pub FeedNodeId, pub f32, pub f32, pub &'a str);
+
+#[derive(Serialize)]
+pub struct ImportedBlock<'a>(pub FeedNodeId, pub &'a BlockDetails);
+
+#[derive(Serialize)]
+pub struct FinalizedBlock(pub FeedNodeId, pub BlockNumber, pub BlockHash);
+
+#[derive(Serialize)]
+pub struct NodeStatsUpdate<'a>(pub FeedNodeId, pub &'a NodeStats);
+
+#[derive(Serialize)]
+pub struct NodeIOUpdate<'a>(pub FeedNodeId, pub &'a NodeIO);
+
+#[derive(Serialize)]
+pub struct Hardware<'a>(pub FeedNodeId, pub &'a NodeHardware);
+
+#[derive(Serialize)]
+pub struct TimeSync(pub u64);
+
+#[derive(Serialize)]
+pub struct AddedChain<'a>(pub &'a str, pub usize);
+
+#[derive(Serialize)]
+pub struct RemovedChain<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct SubscribedTo<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct UnsubscribedFrom<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct Pong<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct AfgFinalized(pub Address, pub BlockNumber, pub BlockHash);
+
+#[derive(Serialize)]
+pub struct AfgReceivedPrevote(
+    pub Address,
+    pub BlockNumber,
+    pub BlockHash,
+    pub Option<Address>,
+);
+
+#[derive(Serialize)]
+pub struct AfgReceivedPrecommit(
+    pub Address,
+    pub BlockNumber,
+    pub BlockHash,
+    pub Option<Address>,
+);
+
+#[derive(Serialize)]
+pub struct AfgAuthoritySet(
+    pub Address,
+    pub Address,
+    pub Address,
+    pub BlockNumber,
+    pub BlockHash,
+);
+
+#[derive(Serialize)]
+pub struct StaleNode(pub FeedNodeId);
+
+impl FeedMessageWrite for AddedNode<'_> {
+    fn write_to_feed(&self, ser: &mut FeedMessageSerializer) {
+        let AddedNode(nid, node) = self;
+
+        let details = node.details();
+        let details = (
+            &details.name,
+            &details.implementation,
+            &details.version,
+            &details.validator,
+            &details.network_id,
+        );
+
+        ser.write(&(
+            nid,
+            details,
+            node.stats(),
+            node.io(),
+            node.hardware(),
+            node.block_details(),
+            &node.location(),
+            &node.startup_time(),
+        ));
+    }
+}

backend/telemetry_core/src/find_location.rs

@@ -0,0 +1,211 @@
+use std::net::Ipv4Addr;
+use std::sync::Arc;
+
+use futures::channel::mpsc;
+use futures::{Sink, SinkExt, StreamExt};
+use parking_lot::RwLock;
+use rustc_hash::FxHashMap;
+use serde::Deserialize;
+
+use common::node_types::NodeLocation;
+use tokio::sync::Semaphore;
+
+/// The returned location is optional; it may be None if not found.
+pub type Location = Option<Arc<NodeLocation>>;
+
+/// This is responsible for taking an IP address and attempting
+/// to find a geographical location from this
+pub fn find_location<Id, R>(response_chan: R) -> mpsc::UnboundedSender<(Id, Ipv4Addr)>
+where
+    R: Sink<(Id, Option<Arc<NodeLocation>>)> + Unpin + Send + Clone + 'static,
+    Id: Clone + Send + 'static,
+{
+    let (tx, mut rx) = mpsc::unbounded();
+
+    // cache entries
+    let mut cache: FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>> = FxHashMap::default();
+
+    // Default entry for localhost
+    cache.insert(
+        Ipv4Addr::new(127, 0, 0, 1),
+        Some(Arc::new(NodeLocation {
+            latitude: 52.516_6667,
+            longitude: 13.4,
+            city: "Berlin".into(),
+        })),
+    );
+
+    // Create a locator with our cache. This is used to obtain locations.
+    let locator = Locator::new(cache);
+
+    // Spawn a loop to handle location requests
+    tokio::spawn(async move {
+        // Allow 4 requests at a time. acquiring a token will block while the
+        // number of concurrent location requests is more than this.
+        let semaphore = Arc::new(Semaphore::new(4));
+
+        loop {
+            while let Some((id, ip_address)) = rx.next().await {
+                let permit = semaphore.clone().acquire_owned().await.unwrap();
+                let mut response_chan = response_chan.clone();
+                let locator = locator.clone();
+
+                // Once we have acquired our permit, spawn a task to avoid
+                // blocking this loop so that we can handle concurrent requests.
+                tokio::spawn(async move {
+                    match locator.locate(ip_address).await {
+                        Ok(loc) => {
+                            let _ = response_chan.send((id, loc)).await;
+                        }
+                        Err(e) => {
+                            log::debug!("GET error for ip location: {:?}", e);
+                        }
+                    };
+
+                    // ensure permit is moved into task by dropping it explicitly:
+                    drop(permit);
+                });
+            }
+        }
+    });
+
+    tx
+}
+
+/// This struct can be used to make location requests, given
+/// an IPV4 address.
+#[derive(Clone)]
+struct Locator {
+    client: reqwest::Client,
+    cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
+}
+
+impl Locator {
+    pub fn new(cache: FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>) -> Self {
+        let client = reqwest::Client::new();
+
+        Locator {
+            client,
+            cache: Arc::new(RwLock::new(cache)),
+        }
+    }
+
+    pub async fn locate(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
+        // Return location quickly if it's cached:
+        let cached_loc = {
+            let cache_reader = self.cache.read();
+            cache_reader.get(&ip).map(|o| o.clone())
+        };
+        if let Some(loc) = cached_loc {
+            return Ok(loc);
+        }
+
+        // Look it up via the location services if not cached:
+        let location = self.iplocate_ipapi_co(ip).await?;
+        let location = match location {
+            Some(location) => Ok(Some(location)),
+            None => self.iplocate_ipinfo_io(ip).await,
+        }?;
+
+        self.cache.write().insert(ip, location.clone());
+        Ok(location)
+    }
+
+    async fn iplocate_ipapi_co(
+        &self,
+        ip: Ipv4Addr,
+    ) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
+        let location = self
+            .query(&format!("https://ipapi.co/{}/json", ip))
+            .await?
+            .map(Arc::new);
+
+        Ok(location)
+    }
+
+    async fn iplocate_ipinfo_io(
+        &self,
+        ip: Ipv4Addr,
+    ) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
+        let location = self
+            .query(&format!("https://ipinfo.io/{}/json", ip))
+            .await?
+            .and_then(|loc: IPApiLocate| loc.into_node_location().map(Arc::new));
+
+        Ok(location)
+    }
+
+    async fn query<T>(&self, url: &str) -> Result<Option<T>, reqwest::Error>
+    where
+        for<'de> T: Deserialize<'de>,
+    {
+        match self.client.get(url).send().await?.json::<T>().await {
+            Ok(result) => Ok(Some(result)),
+            Err(err) => {
+                log::debug!("JSON error for ip location: {:?}", err);
+                Ok(None)
+            }
+        }
+    }
+}
+
+/// This is the format returned from ipinfo.co, so we do
+/// a little conversion to get it into the shape we want.
+#[derive(Deserialize)]
+struct IPApiLocate {
+    city: Box<str>,
+    loc: Box<str>,
+}
+
+impl IPApiLocate {
+    fn into_node_location(self) -> Option<NodeLocation> {
+        let IPApiLocate { city, loc } = self;
+
+        let mut loc = loc.split(',').map(|n| n.parse());
+
+        let latitude = loc.next()?.ok()?;
+        let longitude = loc.next()?.ok()?;
+
+        // Guarantee that the iterator has been exhausted
+        if loc.next().is_some() {
+            return None;
+        }
+
+        Some(NodeLocation {
+            latitude,
+            longitude,
+            city,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ipapi_locate_to_node_location() {
+        let ipapi = IPApiLocate {
+            loc: "12.5,56.25".into(),
+            city: "Foobar".into(),
+        };
+
+        let location = ipapi.into_node_location().unwrap();
+
+        assert_eq!(location.latitude, 12.5);
+        assert_eq!(location.longitude, 56.25);
+        assert_eq!(&*location.city, "Foobar");
+    }
+
+    #[test]
+    fn ipapi_locate_to_node_location_too_many() {
+        let ipapi = IPApiLocate {
+            loc: "12.5,56.25,1.0".into(),
+            city: "Foobar".into(),
+        };
+
+        let location = ipapi.into_node_location();
+
+        assert!(location.is_none());
+    }
+}

backend/telemetry_core/src/main.rs

@@ -0,0 +1,340 @@
+mod aggregator;
+mod feed_message;
+mod find_location;
+mod state;
+
+use std::net::SocketAddr;
+use std::str::FromStr;
+
+use aggregator::{
+    Aggregator, FromFeedWebsocket, FromShardWebsocket, ToFeedWebsocket, ToShardWebsocket,
+};
+use bincode::Options;
+use common::internal_messages;
+use common::ready_chunks_all::ReadyChunksAll;
+use futures::{channel::mpsc, SinkExt, StreamExt};
+use simple_logger::SimpleLogger;
+use structopt::StructOpt;
+use warp::filters::ws;
+use warp::Filter;
+
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
+const NAME: &str = "Substrate Telemetry Backend Core";
+const ABOUT: &str = "This is the Telemetry Backend Core that receives telemetry messages \
+                     from Substrate/Polkadot nodes and provides the data to a subsribed feed";
+
+#[derive(StructOpt, Debug)]
+#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+struct Opts {
+    /// This is the socket address that Telemetryis listening to. This is restricted to
+    /// localhost (127.0.0.1) by default and should be fine for most use cases. If
+    /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
+    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8000")]
+    socket: std::net::SocketAddr,
+    /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
+    /// 'error' only logs errors and 'trace' logs everything.
+    #[structopt(required = false, long = "log", default_value = "info")]
+    log_level: log::LevelFilter,
+    /// Space delimited list of the names of chains that are not allowed to connect to
+    /// telemetry. Case sensitive.
+    #[structopt(required = false, long = "denylist")]
+    denylist: Vec<String>,
+}
+
+#[tokio::main]
+async fn main() {
+    let opts = Opts::from_args();
+
+    SimpleLogger::new()
+        .with_level(opts.log_level)
+        .init()
+        .expect("Must be able to start a logger");
+
+    log::info!("Starting Telemetry Core version: {}", VERSION);
+
+    if let Err(e) = start_server(opts).await {
+        log::error!("Error starting server: {}", e);
+    }
+}
+
+/// Declare our routes and start the server.
+async fn start_server(opts: Opts) -> anyhow::Result<()> {
+    let shard_aggregator = Aggregator::spawn(opts.denylist).await?;
+    let feed_aggregator = shard_aggregator.clone();
+
+    // Handle requests to /health by returning OK.
+    let health_route = warp::path("health").map(|| "OK");
+
+    // Handle websocket requests from shards.
+    let ws_shard_submit_route = warp::path("shard_submit")
+        .and(warp::ws())
+        .and(warp::filters::addr::remote())
+        .map(move |ws: ws::Ws, addr: Option<SocketAddr>| {
+            let tx_to_aggregator = shard_aggregator.subscribe_shard();
+            log::info!("Opening /shard_submit connection from {:?}", addr);
+            ws.on_upgrade(move |websocket| async move {
+                let (mut tx_to_aggregator, websocket) =
+                    handle_shard_websocket_connection(websocket, tx_to_aggregator).await;
+                log::info!("Closing /shard_submit connection from {:?}", addr);
+                // Tell the aggregator that this connection has closed, so it can tidy up.
+                let _ = tx_to_aggregator
+                    .send(FromShardWebsocket::Disconnected)
+                    .await;
+                let _ = websocket.close().await;
+            })
+        });
+
+    // Handle websocket requests from frontends.
+    let ws_feed_route = warp::path("feed")
+        .and(warp::ws())
+        .and(warp::filters::addr::remote())
+        .map(move |ws: ws::Ws, addr: Option<SocketAddr>| {
+            let tx_to_aggregator = feed_aggregator.subscribe_feed();
+            log::info!("Opening /feed connection from {:?}", addr);
+
+            // We can decide how many messages can be buffered to be sent, but not specifically how
+            // large those messages are cumulatively allowed to be:
+            ws.max_send_queue(1_000)
+                .on_upgrade(move |websocket| async move {
+                    let (mut tx_to_aggregator, websocket) =
+                        handle_feed_websocket_connection(websocket, tx_to_aggregator).await;
+                    log::info!("Closing /feed connection from {:?}", addr);
+                    // Tell the aggregator that this connection has closed, so it can tidy up.
+                    let _ = tx_to_aggregator.send(FromFeedWebsocket::Disconnected).await;
+                    let _ = websocket.close().await;
+                })
+        });
+
+    // Merge the routes and start our server:
+    let routes = ws_shard_submit_route.or(ws_feed_route).or(health_route);
+    warp::serve(routes).run(opts.socket).await;
+    Ok(())
+}
+
+/// This handles messages coming to/from a shard connection
+async fn handle_shard_websocket_connection<S>(
+    mut websocket: ws::WebSocket,
+    mut tx_to_aggregator: S,
+) -> (S, ws::WebSocket)
+where
+    S: futures::Sink<FromShardWebsocket, Error = anyhow::Error> + Unpin,
+{
+    let (tx_to_shard_conn, mut rx_from_aggregator) = mpsc::unbounded();
+
+    // Tell the aggregator about this new connection, and give it a way to send messages to us:
+    let init_msg = FromShardWebsocket::Initialize {
+        channel: tx_to_shard_conn,
+    };
+    if let Err(e) = tx_to_aggregator.send(init_msg).await {
+        log::error!("Error sending message to aggregator: {}", e);
+        return (tx_to_aggregator, websocket);
+    }
+
+    // Loop, handling new messages from the shard or from the aggregator:
+    loop {
+        tokio::select! {
+            // AGGREGATOR -> SHARD
+            msg = rx_from_aggregator.next() => {
+                // End the loop when connection from aggregator ends:
+                let msg = match msg {
+                    Some(msg) => msg,
+                    None => break
+                };
+
+                let internal_msg = match msg {
+                    ToShardWebsocket::Mute { local_id, reason } => {
+                        internal_messages::FromTelemetryCore::Mute { local_id, reason }
+                    }
+                };
+
+                let bytes = bincode::options()
+                    .serialize(&internal_msg)
+                    .expect("message to shard should serialize");
+
+                if let Err(e) = websocket.send(ws::Message::binary(bytes)).await {
+                    log::error!("Error sending message to shard; booting it: {}", e);
+                    break
+                }
+            }
+            // SHARD -> AGGREGATOR
+            msg = websocket.next() => {
+                // End the loop when connection from shard ends:
+                let msg = match msg {
+                    Some(msg) => msg,
+                    None => break
+                };
+
+                let msg = match msg {
+                    Err(e) => {
+                        log::error!("Error receiving message from shard; booting it: {}", e);
+                        break;
+                    },
+                    Ok(msg) => msg
+                };
+
+                // Close message? Break and allow connection to be dropped.
+                if msg.is_close() {
+                    break;
+                }
+
+                // If the message isn't something we want to handle, just ignore it.
+                // This includes system messages like "pings" and such, so don't log anything.
+                if !msg.is_binary() && !msg.is_text() {
+                    continue;
+                }
+
+                let bytes = msg.as_bytes();
+                let msg: internal_messages::FromShardAggregator = match bincode::options().deserialize(bytes) {
+                    Ok(msg) => msg,
+                    Err(e) => {
+                        log::error!("Failed to deserialize message from shard; booting it: {}", e);
+                        break;
+                    }
+                };
+
+                // Convert and send to the aggregator:
+                let aggregator_msg = match msg {
+                    internal_messages::FromShardAggregator::AddNode { ip, node, local_id, genesis_hash } => {
+                        FromShardWebsocket::Add { ip, node, genesis_hash, local_id }
+                    },
+                    internal_messages::FromShardAggregator::UpdateNode { payload, local_id } => {
+                        FromShardWebsocket::Update { local_id, payload }
+                    },
+                    internal_messages::FromShardAggregator::RemoveNode { local_id } => {
+                        FromShardWebsocket::Remove { local_id }
+                    },
+                };
+                if let Err(e) = tx_to_aggregator.send(aggregator_msg).await {
+                    log::error!("Failed to send message to aggregator; closing shard: {}", e);
+                    break;
+                }
+            }
+        }
+    }
+
+    // loop ended; give socket back to parent:
+    (tx_to_aggregator, websocket)
+}
+
+/// This handles messages coming from a feed connection
+async fn handle_feed_websocket_connection<S>(
+    mut websocket: ws::WebSocket,
+    mut tx_to_aggregator: S,
+) -> (S, ws::WebSocket)
+where
+    S: futures::Sink<FromFeedWebsocket, Error = anyhow::Error> + Unpin,
+{
+    // unbounded channel so that slow feeds don't block aggregator progress:
+    let (tx_to_feed_conn, rx_from_aggregator) = mpsc::unbounded();
+    let mut rx_from_aggregator_chunks = ReadyChunksAll::new(rx_from_aggregator);
+
+    // Tell the aggregator about this new connection, and give it a way to send messages to us:
+    let init_msg = FromFeedWebsocket::Initialize {
+        channel: tx_to_feed_conn,
+    };
+    if let Err(e) = tx_to_aggregator.send(init_msg).await {
+        log::error!("Error sending message to aggregator: {}", e);
+        return (tx_to_aggregator, websocket);
+    }
+
+    // Loop, handling new messages from the shard or from the aggregator:
+    loop {
+        // Without any special handling, if messages come in every ~2.5ms to each feed, the select! loop
+        // has to wake up 400 times a second to poll things. If we have 1000 feeds, that's 400,000 wakeups
+        // per second. Even without any work in the loop, that uses a bunch of CPU. As an example, try
+        // replacing the loop with this:
+        //
+        // ```
+        // let s = tokio::time::sleep(tokio::time::Duration::from_micros(2500));
+        // tokio::select! {
+        //     _ = s => {},
+        //     _ = websocket.next() => {}
+        // }
+        // continue;
+        // ```
+        //
+        // To combat this, we add a small wait to reduce how often the select loop will be woken up under high load. We
+        // buffer messages to feeds so that we do as much work as possible during each wakeup, and if the
+        // wakeup lasts longer than 75ms we don't wait before polling again. This knocks ~80% of a CPU worth of usage
+        // off on my machine running a soak test with 500 feeds, 4 shards and 100 nodes, doesn't seem to impact
+        // memory usage much, and still ensures that messages are delivered in a timely fashion.
+        //
+        // Increasing the wait to 100ms or more doesn't seem to have much more of a positive impact anyway.
+        let debounce = tokio::time::sleep_until(tokio::time::Instant::now() + std::time::Duration::from_millis(75));
+
+        tokio::select! {biased;
+
+            // FRONTEND -> AGGREGATOR (relay messages to the aggregator). Biased, so messages
+            // from the UI will have priority (especially important with our debounce delay).
+            msg = websocket.next() => {
+                // End the loop when connection from feed ends:
+                let msg = match msg {
+                    Some(msg) => msg,
+                    None => break
+                };
+
+                // If we see any errors, log them and end our loop:
+                let msg = match msg {
+                    Err(e) => {
+                        log::error!("Error in node websocket connection: {}", e);
+                        break;
+                    },
+                    Ok(msg) => msg
+                };
+
+                // Close message? Break and allow connection to be dropped.
+                if msg.is_close() {
+                    break;
+                }
+
+                // We ignore all but text messages from the frontend:
+                let text = match msg.to_str() {
+                    Ok(s) => s,
+                    Err(_) => continue
+                };
+
+                // Parse the message into a command we understand and send it to the aggregator:
+                let cmd = match FromFeedWebsocket::from_str(text) {
+                    Ok(cmd) => cmd,
+                    Err(e) => {
+                        log::warn!("Ignoring invalid command '{}' from the frontend: {}", text, e);
+                        continue
+                    }
+                };
+                if let Err(e) = tx_to_aggregator.send(cmd).await {
+                    log::error!("Failed to send message to aggregator; closing feed: {}", e);
+                    break;
+                }
+            }
+
+            // AGGREGATOR -> FRONTEND (buffer messages to the UI)
+            msgs = rx_from_aggregator_chunks.next() => {
+                // End the loop when connection from aggregator ends:
+                let msgs = match msgs {
+                    Some(msgs) => msgs,
+                    None => break
+                };
+
+                // There is only one message type at the mo; bytes to send
+                // to the websocket. collect them all up to dispatch in one shot.
+                let all_ws_msgs = msgs.into_iter().map(|msg| {
+                    let bytes = match msg {
+                        ToFeedWebsocket::Bytes(bytes) => bytes
+                    };
+                    Ok(ws::Message::binary(&*bytes))
+                });
+
+                if let Err(e) = websocket.send_all(&mut futures::stream::iter(all_ws_msgs)).await {
+                    log::warn!("Closing feed websocket due to error: {}", e);
+                    break;
+                }
+            }
+        }
+
+        debounce.await;
+    }
+
+    // loop ended; give socket back to parent:
+    (tx_to_aggregator, websocket)
+}

backend/telemetry_core/src/state/chain.rs

@@ -0,0 +1,370 @@
+use common::node_message::Payload;
+use common::node_types::{Block, Timestamp};
+use common::node_types::{BlockHash, BlockNumber};
+use common::{id_type, time, DenseMap, MostSeen, NumStats};
+use once_cell::sync::Lazy;
+use std::collections::HashSet;
+
+use crate::feed_message::{self, FeedMessageSerializer};
+use crate::find_location;
+
+use super::node::Node;
+
+id_type! {
+    /// A Node ID that is unique to the chain it's in.
+    pub struct ChainNodeId(usize)
+}
+
+pub type Label = Box<str>;
+
+const STALE_TIMEOUT: u64 = 2 * 60 * 1000; // 2 minutes
+
+pub struct Chain {
+    /// Labels that nodes use for this chain. We keep track of
+    /// the most commonly used label as nodes are added/removed.
+    labels: MostSeen<Label>,
+    /// Set of nodes that are in this chain
+    nodes: DenseMap<ChainNodeId, Node>,
+    /// Best block
+    best: Block,
+    /// Finalized block
+    finalized: Block,
+    /// Block times history, stored so we can calculate averages
+    block_times: NumStats<u64>,
+    /// Calculated average block time
+    average_block_time: Option<u64>,
+    /// When the best block first arrived
+    timestamp: Option<Timestamp>,
+    /// Genesis hash of this chain
+    genesis_hash: BlockHash,
+}
+
+pub enum AddNodeResult {
+    Overquota,
+    Added {
+        id: ChainNodeId,
+        chain_renamed: bool,
+    },
+}
+
+pub struct RemoveNodeResult {
+    pub chain_renamed: bool,
+}
+
+/// Labels of chains we consider "first party". These chains allow any
+/// number of nodes to connect.
+static FIRST_PARTY_NETWORKS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
+    let mut set = HashSet::new();
+    set.insert("Polkadot");
+    set.insert("Kusama");
+    set.insert("Westend");
+    set.insert("Rococo");
+    set
+});
+
+/// Max number of nodes allowed to connect to the telemetry server.
+const THIRD_PARTY_NETWORKS_MAX_NODES: usize = 500;
+
+impl Chain {
+    /// Create a new chain with an initial label.
+    pub fn new(genesis_hash: BlockHash) -> Self {
+        Chain {
+            labels: MostSeen::default(),
+            nodes: DenseMap::new(),
+            best: Block::zero(),
+            finalized: Block::zero(),
+            block_times: NumStats::new(50),
+            average_block_time: None,
+            timestamp: None,
+            genesis_hash,
+        }
+    }
+
+    /// Can we add a node? If not, it's because the chain is at its quota.
+    pub fn can_add_node(&self) -> bool {
+        // Dynamically determine the max nodes based on the most common
+        // label so far, in case it changes to something with a different limit.
+        self.nodes.len() < max_nodes(self.labels.best())
+    }
+
+    /// Assign a node to this chain.
+    pub fn add_node(&mut self, node: Node) -> AddNodeResult {
+        if !self.can_add_node() {
+            return AddNodeResult::Overquota;
+        }
+
+        let node_chain_label = &node.details().chain;
+        let label_result = self.labels.insert(node_chain_label);
+        let node_id = self.nodes.add(node);
+
+        AddNodeResult::Added {
+            id: node_id,
+            chain_renamed: label_result.has_changed(),
+        }
+    }
+
+    /// Remove a node from this chain.
+    pub fn remove_node(&mut self, node_id: ChainNodeId) -> RemoveNodeResult {
+        let node = match self.nodes.remove(node_id) {
+            Some(node) => node,
+            None => {
+                return RemoveNodeResult {
+                    chain_renamed: false,
+                }
+            }
+        };
+
+        let node_chain_label = &node.details().chain;
+        let label_result = self.labels.remove(node_chain_label);
+
+        RemoveNodeResult {
+            chain_renamed: label_result.has_changed(),
+        }
+    }
+
+    /// Attempt to update the best block seen in this chain.
+    /// Returns a boolean which denotes whether the output is for finalization feeds (true) or not (false).
+    pub fn update_node(
+        &mut self,
+        nid: ChainNodeId,
+        payload: Payload,
+        feed: &mut FeedMessageSerializer,
+    ) -> bool {
+        if let Some(block) = payload.best_block() {
+            self.handle_block(block, nid, feed);
+        }
+
+        if let Some(node) = self.nodes.get_mut(nid) {
+            match payload {
+                Payload::SystemInterval(ref interval) => {
+                    if node.update_hardware(interval) {
+                        feed.push(feed_message::Hardware(nid.into(), node.hardware()));
+                    }
+
+                    if let Some(stats) = node.update_stats(interval) {
+                        feed.push(feed_message::NodeStatsUpdate(nid.into(), stats));
+                    }
+
+                    if let Some(io) = node.update_io(interval) {
+                        feed.push(feed_message::NodeIOUpdate(nid.into(), io));
+                    }
+                }
+                Payload::AfgAuthoritySet(authority) => {
+                    node.set_validator_address(authority.authority_id.clone());
+                    return false;
+                }
+                Payload::AfgFinalized(finalized) => {
+                    if let Ok(finalized_number) = finalized.finalized_number.parse::<BlockNumber>()
+                    {
+                        if let Some(addr) = node.details().validator.clone() {
+                            feed.push(feed_message::AfgFinalized(
+                                addr,
+                                finalized_number,
+                                finalized.finalized_hash,
+                            ));
+                        }
+                    }
+                    return true;
+                }
+                Payload::AfgReceivedPrecommit(precommit) => {
+                    if let Ok(finalized_number) = precommit.target_number.parse::<BlockNumber>() {
+                        if let Some(addr) = node.details().validator.clone() {
+                            let voter = precommit.voter.clone();
+                            feed.push(feed_message::AfgReceivedPrecommit(
+                                addr,
+                                finalized_number,
+                                precommit.target_hash,
+                                voter,
+                            ));
+                        }
+                    }
+                    return true;
+                }
+                Payload::AfgReceivedPrevote(prevote) => {
+                    if let Ok(finalized_number) = prevote.target_number.parse::<BlockNumber>() {
+                        if let Some(addr) = node.details().validator.clone() {
+                            let voter = prevote.voter.clone();
+                            feed.push(feed_message::AfgReceivedPrevote(
+                                addr,
+                                finalized_number,
+                                prevote.target_hash,
+                                voter,
+                            ));
+                        }
+                    }
+                    return true;
+                }
+                Payload::AfgReceivedCommit(_) => {}
+                _ => (),
+            }
+
+            if let Some(block) = payload.finalized_block() {
+                if let Some(finalized) = node.update_finalized(block) {
+                    feed.push(feed_message::FinalizedBlock(
+                        nid.into(),
+                        finalized.height,
+                        finalized.hash,
+                    ));
+
+                    if finalized.height > self.finalized.height {
+                        self.finalized = *finalized;
+                        feed.push(feed_message::BestFinalized(
+                            finalized.height,
+                            finalized.hash,
+                        ));
+                    }
+                }
+            }
+        }
+
+        false
+    }
+
+    fn handle_block(&mut self, block: &Block, nid: ChainNodeId, feed: &mut FeedMessageSerializer) {
+        let mut propagation_time = None;
+        let now = time::now();
+        let nodes_len = self.nodes.len();
+
+        self.update_stale_nodes(now, feed);
+
+        let node = match self.nodes.get_mut(nid) {
+            Some(node) => node,
+            None => return,
+        };
+
+        if node.update_block(*block) {
+            if block.height > self.best.height {
+                self.best = *block;
+                log::debug!(
+                    "[{}] [nodes={}] new best block={}/{:?}",
+                    self.labels.best(),
+                    nodes_len,
+                    self.best.height,
+                    self.best.hash,
+                );
+                if let Some(timestamp) = self.timestamp {
+                    self.block_times.push(now - timestamp);
+                    self.average_block_time = Some(self.block_times.average());
+                }
+                self.timestamp = Some(now);
+                feed.push(feed_message::BestBlock(
+                    self.best.height,
+                    now,
+                    self.average_block_time,
+                ));
+                propagation_time = Some(0);
+            } else if block.height == self.best.height {
+                if let Some(timestamp) = self.timestamp {
+                    propagation_time = Some(now - timestamp);
+                }
+            }
+
+            if let Some(details) = node.update_details(now, propagation_time) {
+                feed.push(feed_message::ImportedBlock(nid.into(), details));
+            }
+        }
+    }
+
+    /// Check if the chain is stale (has not received a new best block in a while).
+    /// If so, find a new best block, ignoring any stale nodes and marking them as such.
+    fn update_stale_nodes(&mut self, now: u64, feed: &mut FeedMessageSerializer) {
+        let threshold = now - STALE_TIMEOUT;
+        let timestamp = match self.timestamp {
+            Some(ts) => ts,
+            None => return,
+        };
+
+        if timestamp > threshold {
+            // Timestamp is in range, nothing to do
+            return;
+        }
+
+        let mut best = Block::zero();
+        let mut finalized = Block::zero();
+        let mut timestamp = None;
+
+        for (nid, node) in self.nodes.iter_mut() {
+            if !node.update_stale(threshold) {
+                if node.best().height > best.height {
+                    best = *node.best();
+                    timestamp = Some(node.best_timestamp());
+                }
+
+                if node.finalized().height > finalized.height {
+                    finalized = *node.finalized();
+                }
+            } else {
+                feed.push(feed_message::StaleNode(nid.into()));
+            }
+        }
+
+        if self.best.height != 0 || self.finalized.height != 0 {
+            self.best = best;
+            self.finalized = finalized;
+            self.block_times.reset();
+            self.timestamp = timestamp;
+
+            feed.push(feed_message::BestBlock(
+                self.best.height,
+                timestamp.unwrap_or(now),
+                None,
+            ));
+            feed.push(feed_message::BestFinalized(
+                finalized.height,
+                finalized.hash,
+            ));
+        }
+    }
+
+    pub fn update_node_location(
+        &mut self,
+        node_id: ChainNodeId,
+        location: find_location::Location,
+    ) -> bool {
+        if let Some(node) = self.nodes.get_mut(node_id) {
+            node.update_location(location);
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn get_node(&self, id: ChainNodeId) -> Option<&Node> {
+        self.nodes.get(id)
+    }
+    pub fn iter_nodes(&self) -> impl Iterator<Item = (ChainNodeId, &Node)> {
+        self.nodes.iter()
+    }
+    pub fn label(&self) -> &str {
+        &self.labels.best()
+    }
+    pub fn node_count(&self) -> usize {
+        self.nodes.len()
+    }
+    pub fn best_block(&self) -> &Block {
+        &self.best
+    }
+    pub fn timestamp(&self) -> Option<Timestamp> {
+        self.timestamp
+    }
+    pub fn average_block_time(&self) -> Option<u64> {
+        self.average_block_time
+    }
+    pub fn finalized_block(&self) -> &Block {
+        &self.finalized
+    }
+    pub fn genesis_hash(&self) -> &BlockHash {
+        &self.genesis_hash
+    }
+}
+
+/// First party networks (Polkadot, Kusama etc) are allowed any number of nodes.
+/// Third party networks are allowed `THIRD_PARTY_NETWORKS_MAX_NODES` nodes and
+/// no more.
+fn max_nodes(label: &str) -> usize {
+    if FIRST_PARTY_NETWORKS.contains(label) {
+        usize::MAX
+    } else {
+        THIRD_PARTY_NETWORKS_MAX_NODES
+    }
+}

backend/telemetry_core/src/state/mod.rs

@@ -0,0 +1,7 @@
+mod chain;
+mod node;
+
+mod state;
+
+pub use node::Node;
+pub use state::*;

backend/telemetry_core/src/state/node.rs

@@ -1,15 +1,9 @@
-use std::sync::Arc;
-
-use crate::types::{
-    Block, BlockDetails, NodeDetails, NodeHardware, NodeIO, NodeId, NodeLocation, NodeStats,
-    Timestamp,
+use crate::find_location;
+use common::node_message::SystemInterval;
+use common::node_types::{
+    Block, BlockDetails, NodeDetails, NodeHardware, NodeIO, NodeLocation, NodeStats, Timestamp,
 };
-use crate::util::now;
-
-pub mod connector;
-pub mod message;
-
-use message::SystemInterval;
+use common::time;
 
 /// Minimum time between block below broadcasting updates to the browser gets throttled, in ms.
 const THROTTLE_THRESHOLD: u64 = 100;
@@ -32,7 +26,7 @@ pub struct Node {
     /// Hardware stats over time
     hardware: NodeHardware,
     /// Physical location details
-    location: Option<Arc<NodeLocation>>,
+    location: find_location::Location,
     /// Flag marking if the node is stale (not syncing or producing blocks)
     stale: bool,
     /// Unix timestamp for when node started up (falls back to connection time)
@@ -92,8 +86,8 @@ impl Node {
         self.location.as_deref()
     }
 
-    pub fn update_location(&mut self, location: Arc<NodeLocation>) {
-        self.location = Some(location);
+    pub fn update_location(&mut self, location: find_location::Location) {
+        self.location = location;
     }
 
     pub fn block_details(&self) -> &BlockDetails {
@@ -140,7 +134,7 @@ impl Node {
         if let Some(download) = interval.bandwidth_download {
             changed |= self.hardware.download.push(download);
         }
-        self.hardware.chart_stamps.push(now() as f64);
+        self.hardware.chart_stamps.push(time::now() as f64);
 
         changed
     }

backend/telemetry_core/src/state/state.rs

@@ -0,0 +1,414 @@
+use super::node::Node;
+use crate::feed_message::FeedMessageSerializer;
+use crate::find_location;
+use common::node_message::Payload;
+use common::node_types::{Block, BlockHash, NodeDetails, Timestamp};
+use common::{id_type, DenseMap};
+use std::collections::{HashMap, HashSet};
+use std::iter::IntoIterator;
+
+use super::chain::{self, Chain, ChainNodeId};
+
+id_type! {
+    /// A globally unique Chain ID.
+    pub struct ChainId(usize)
+}
+
+/// A "global" Node ID is a composite of the ID of the chain it's
+/// on, and it's chain local ID.
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct NodeId(ChainId, ChainNodeId);
+
+impl NodeId {
+    pub fn get_chain_node_id(&self) -> ChainNodeId {
+        self.1
+    }
+}
+
+/// Our state constains node and chain information
+pub struct State {
+    chains: DenseMap<ChainId, Chain>,
+
+    // Find the right chain given various details.
+    chains_by_genesis_hash: HashMap<BlockHash, ChainId>,
+    chains_by_label: HashMap<Box<str>, ChainId>,
+
+    /// Chain labels that we do not want to allow connecting.
+    denylist: HashSet<String>,
+}
+
+/// Adding a node to a chain leads to this node_idult
+pub enum AddNodeResult<'a> {
+    /// The chain is on the "deny list", so we can't add the node
+    ChainOnDenyList,
+    /// The chain is over quota (too many nodes connected), so can't add the node
+    ChainOverQuota,
+    /// The node was added to the chain
+    NodeAddedToChain(NodeAddedToChain<'a>),
+}
+
+#[cfg(test)]
+impl<'a> AddNodeResult<'a> {
+    pub fn unwrap_id(&self) -> NodeId {
+        match &self {
+            AddNodeResult::NodeAddedToChain(d) => d.id,
+            _ => panic!("Attempt to unwrap_id on AddNodeResult that did not succeed"),
+        }
+    }
+}
+
+pub struct NodeAddedToChain<'a> {
+    /// The ID assigned to this node.
+    pub id: NodeId,
+    /// The old label of the chain.
+    pub old_chain_label: Box<str>,
+    /// The new label of the chain.
+    pub new_chain_label: &'a str,
+    /// The node that was added.
+    pub node: &'a Node,
+    /// Number of nodes in the chain. If 1, the chain was just added.
+    pub chain_node_count: usize,
+    /// Has the chain label been updated?
+    pub has_chain_label_changed: bool,
+}
+
+/// if removing a node is successful, we get this information back.
+pub struct RemovedNode {
+    /// How many nodes remain on the chain (0 if the chain was removed)
+    pub chain_node_count: usize,
+    /// Has the chain label been updated?
+    pub has_chain_label_changed: bool,
+    /// The old label of the chain.
+    pub old_chain_label: Box<str>,
+    /// The new label of the chain.
+    pub new_chain_label: Box<str>,
+}
+
+impl State {
+    pub fn new<T: IntoIterator<Item = String>>(denylist: T) -> State {
+        State {
+            chains: DenseMap::new(),
+            chains_by_genesis_hash: HashMap::new(),
+            chains_by_label: HashMap::new(),
+            denylist: denylist.into_iter().collect(),
+        }
+    }
+
+    pub fn iter_chains(&self) -> impl Iterator<Item = StateChain<'_>> {
+        self.chains
+            .iter()
+            .map(move |(_, chain)| StateChain { chain })
+    }
+
+    pub fn get_chain_by_node_id(&self, node_id: NodeId) -> Option<StateChain<'_>> {
+        self.chains.get(node_id.0).map(|chain| StateChain { chain })
+    }
+
+    pub fn get_chain_by_genesis_hash(&self, genesis_hash: &BlockHash) -> Option<StateChain<'_>> {
+        self.chains_by_genesis_hash
+            .get(genesis_hash)
+            .and_then(|&chain_id| self.chains.get(chain_id))
+            .map(|chain| StateChain { chain })
+    }
+
+    pub fn get_chain_by_label(&self, label: &str) -> Option<StateChain<'_>> {
+        self.chains_by_label
+            .get(label)
+            .and_then(|&chain_id| self.chains.get(chain_id))
+            .map(|chain| StateChain { chain })
+    }
+
+    pub fn add_node(
+        &mut self,
+        genesis_hash: BlockHash,
+        node_details: NodeDetails,
+    ) -> AddNodeResult<'_> {
+        if self.denylist.contains(&*node_details.chain) {
+            return AddNodeResult::ChainOnDenyList;
+        }
+
+        // Get the chain ID, creating a new empty chain if one doesn't exist.
+        // If we create a chain here, we are expecting that it will allow at
+        // least this node to be added, because we don't currently try and clean it up
+        // if the add fails.
+        let chain_id = match self.chains_by_genesis_hash.get(&genesis_hash) {
+            Some(id) => *id,
+            None => {
+                let chain_id = self.chains.add(Chain::new(genesis_hash));
+                self.chains_by_genesis_hash.insert(genesis_hash, chain_id);
+                chain_id
+            }
+        };
+
+        // Get the chain.
+        let chain = self.chains.get_mut(chain_id).expect(
+            "should be known to exist after the above (unless chains_by_genesis_hash out of sync)",
+        );
+
+        let node = Node::new(node_details);
+        let old_chain_label = chain.label().into();
+
+        match chain.add_node(node) {
+            chain::AddNodeResult::Overquota => AddNodeResult::ChainOverQuota,
+            chain::AddNodeResult::Added { id, chain_renamed } => {
+                let chain = &*chain;
+
+                // Update the label we use to reference the chain if
+                // it changes (it'll always change first time a node's added):
+                if chain_renamed {
+                    self.chains_by_label.remove(&old_chain_label);
+                    self.chains_by_label.insert(chain.label().into(), chain_id);
+                }
+
+                AddNodeResult::NodeAddedToChain(NodeAddedToChain {
+                    id: NodeId(chain_id, id),
+                    node: chain.get_node(id).expect("node added above"),
+                    old_chain_label: old_chain_label,
+                    new_chain_label: chain.label(),
+                    chain_node_count: chain.node_count(),
+                    has_chain_label_changed: chain_renamed,
+                })
+            }
+        }
+    }
+
+    /// Remove a node
+    pub fn remove_node(&mut self, NodeId(chain_id, chain_node_id): NodeId) -> Option<RemovedNode> {
+        let chain = self.chains.get_mut(chain_id)?;
+        let old_chain_label = chain.label().into();
+
+        // Actually remove the node
+        let remove_result = chain.remove_node(chain_node_id);
+
+        // Get updated chain details.
+        let new_chain_label: Box<str> = chain.label().into();
+        let chain_node_count = chain.node_count();
+
+        // Is the chain empty? Remove if so and clean up indexes to it
+        if chain_node_count == 0 {
+            let genesis_hash = *chain.genesis_hash();
+            self.chains_by_label.remove(&old_chain_label);
+            self.chains_by_genesis_hash.remove(&genesis_hash);
+            self.chains.remove(chain_id);
+        }
+
+        // Make sure chains always referenced by their most common label:
+        if remove_result.chain_renamed {
+            self.chains_by_label.remove(&old_chain_label);
+            self.chains_by_label
+                .insert(new_chain_label.clone(), chain_id);
+        }
+
+        Some(RemovedNode {
+            old_chain_label,
+            new_chain_label,
+            chain_node_count: chain_node_count,
+            has_chain_label_changed: remove_result.chain_renamed,
+        })
+    }
+
+    /// Attempt to update the best block seen, given a node and block.
+    /// Returns a boolean which denotes whether the output is for finalization feeds (true) or not (false).
+    pub fn update_node(
+        &mut self,
+        NodeId(chain_id, chain_node_id): NodeId,
+        payload: Payload,
+        feed: &mut FeedMessageSerializer,
+    ) -> bool {
+        let chain = match self.chains.get_mut(chain_id) {
+            Some(chain) => chain,
+            None => {
+                log::error!("Cannot find chain for node with ID {:?}", chain_id);
+                return false;
+            }
+        };
+
+        chain.update_node(chain_node_id, payload, feed)
+    }
+
+    /// Update the location for a node. Return `false` if the node was not found.
+    pub fn update_node_location(
+        &mut self,
+        NodeId(chain_id, chain_node_id): NodeId,
+        location: find_location::Location,
+    ) -> bool {
+        if let Some(chain) = self.chains.get_mut(chain_id) {
+            chain.update_node_location(chain_node_id, location)
+        } else {
+            false
+        }
+    }
+}
+
+/// When we ask for a chain, we get this struct back. This ensures that we have
+/// a consistent public interface, and don't expose methods on [`Chain`] that
+/// aren't really intended for use outside of [`State`] methods. Any modification
+/// of a chain needs to go through [`State`].
+pub struct StateChain<'a> {
+    chain: &'a Chain,
+}
+
+impl<'a> StateChain<'a> {
+    pub fn label(&self) -> &'a str {
+        self.chain.label()
+    }
+    pub fn genesis_hash(&self) -> &'a BlockHash {
+        self.chain.genesis_hash()
+    }
+    pub fn node_count(&self) -> usize {
+        self.chain.node_count()
+    }
+    pub fn best_block(&self) -> &'a Block {
+        self.chain.best_block()
+    }
+    pub fn timestamp(&self) -> Timestamp {
+        self.chain.timestamp().unwrap_or(0)
+    }
+    pub fn average_block_time(&self) -> Option<u64> {
+        self.chain.average_block_time()
+    }
+    pub fn finalized_block(&self) -> &'a Block {
+        self.chain.finalized_block()
+    }
+    pub fn iter_nodes(&self) -> impl Iterator<Item = (ChainNodeId, &'a Node)> + 'a {
+        self.chain.iter_nodes()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn node(name: &str, chain: &str) -> NodeDetails {
+        NodeDetails {
+            chain: chain.into(),
+            name: name.into(),
+            implementation: "Bar".into(),
+            version: "0.1".into(),
+            validator: None,
+            network_id: None,
+            startup_time: None,
+        }
+    }
+
+    #[test]
+    fn adding_a_node_returns_expected_response() {
+        let mut state = State::new(None);
+
+        let chain1_genesis = BlockHash::from_low_u64_be(1);
+
+        let add_result = state.add_node(chain1_genesis, node("A", "Chain One"));
+
+        let add_node_result = match add_result {
+            AddNodeResult::ChainOnDenyList => panic!("Chain not on deny list"),
+            AddNodeResult::ChainOverQuota => panic!("Chain not Overquota"),
+            AddNodeResult::NodeAddedToChain(details) => details,
+        };
+
+        assert_eq!(add_node_result.id, NodeId(0.into(), 0.into()));
+        assert_eq!(&*add_node_result.old_chain_label, "");
+        assert_eq!(&*add_node_result.new_chain_label, "Chain One");
+        assert_eq!(add_node_result.chain_node_count, 1);
+        assert_eq!(add_node_result.has_chain_label_changed, true);
+
+        let add_result = state.add_node(chain1_genesis, node("A", "Chain One"));
+
+        let add_node_result = match add_result {
+            AddNodeResult::ChainOnDenyList => panic!("Chain not on deny list"),
+            AddNodeResult::ChainOverQuota => panic!("Chain not Overquota"),
+            AddNodeResult::NodeAddedToChain(details) => details,
+        };
+
+        assert_eq!(add_node_result.id, NodeId(0.into(), 1.into()));
+        assert_eq!(&*add_node_result.old_chain_label, "Chain One");
+        assert_eq!(&*add_node_result.new_chain_label, "Chain One");
+        assert_eq!(add_node_result.chain_node_count, 2);
+        assert_eq!(add_node_result.has_chain_label_changed, false);
+    }
+
+    #[test]
+    fn adding_and_removing_nodes_updates_chain_label_mapping() {
+        let mut state = State::new(None);
+
+        let chain1_genesis = BlockHash::from_low_u64_be(1);
+        let node_id0 = state
+            .add_node(chain1_genesis, node("A", "Chain One")) // 0
+            .unwrap_id();
+
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain One"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+
+        let node_id1 = state
+            .add_node(chain1_genesis, node("B", "Chain Two")) // 1
+            .unwrap_id();
+
+        // Chain name hasn't changed yet; "Chain One" as common as "Chain Two"..
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain One"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+
+        let node_id2 = state
+            .add_node(chain1_genesis, node("B", "Chain Two"))
+            .unwrap_id(); // 2
+
+        // Chain name has changed; "Chain Two" the winner now..
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain Two"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_none());
+        assert!(state.get_chain_by_label("Chain Two").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+
+        state.remove_node(node_id1).expect("Removal OK (id: 1)");
+        state.remove_node(node_id2).expect("Removal OK (id: 2)");
+
+        // Removed both "Chain Two" nodes; dominant name now "Chain One" again..
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain One"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_label("Chain Two").is_none());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+    }
+
+    #[test]
+    fn chain_removed_when_last_node_is() {
+        let mut state = State::new(None);
+
+        let chain1_genesis = BlockHash::from_low_u64_be(1);
+        let node_id = state
+            .add_node(chain1_genesis, node("A", "Chain One")) // 0
+            .unwrap_id();
+
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+        assert_eq!(state.iter_chains().count(), 1);
+
+        state.remove_node(node_id);
+
+        assert!(state.get_chain_by_label("Chain One").is_none());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_none());
+        assert_eq!(state.iter_chains().count(), 0);
+    }
+}

backend/telemetry_core/tests/e2e_tests.rs

@@ -0,0 +1,478 @@
+//! General end-to-end tests
+
+use common::node_types::BlockHash;
+use serde_json::json;
+use std::time::Duration;
+use test_utils::{
+    assert_contains_matches,
+    feed_message_de::{FeedMessage, NodeDetails},
+    workspace::start_server_debug
+};
+
+/// The simplest test we can run; the main benefit of this test (since we check similar)
+/// below) is just to give a feel for _how_ we can test basic feed related things.
+#[tokio::test]
+async fn feed_sent_version_on_connect() {
+    let server = start_server_debug().await;
+
+    // Connect a feed:
+    let (_feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    // Expect a version response of 31:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_eq!(
+        feed_messages,
+        vec![FeedMessage::Version(31)],
+        "expecting version"
+    );
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// Another very simple test: pings from feeds should be responded to by pongs
+/// with the same message content.
+#[tokio::test]
+async fn feed_ping_responded_to_with_pong() {
+    let server = start_server_debug().await;
+
+    // Connect a feed:
+    let (mut feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    // Ping it:
+    feed_tx.send_command("ping", "hello!").unwrap();
+
+    // Expect a pong response:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(
+        feed_messages.contains(&FeedMessage::Pong {
+            msg: "hello!".to_owned()
+        }),
+        "Expecting pong"
+    );
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+
+/// As a prelude to `lots_of_mute_messages_dont_cause_a_deadlock`, we can check that
+/// a lot of nodes can simultaneously subscribe and are all sent the expected response.
+#[tokio::test]
+async fn multiple_feeds_sent_version_on_connect() {
+    let server = start_server_debug().await;
+
+    // Connect a bunch of feeds:
+    let mut feeds = server
+        .get_core()
+        .connect_multiple_feeds(1000)
+        .await
+        .unwrap();
+
+    // Wait for responses all at once:
+    let responses = futures::future::join_all(
+        feeds.iter_mut()
+        .map(|(_, rx)| rx.recv_feed_messages())
+    );
+
+    let responses = tokio::time::timeout(Duration::from_secs(10), responses)
+        .await
+        .expect("we shouldn't hit a timeout waiting for responses");
+
+    // Expect a version response of 31 to all of them:
+    for feed_messages in responses {
+        assert_eq!(
+            feed_messages.expect("should have messages"),
+            vec![FeedMessage::Version(31)],
+            "expecting version"
+        );
+    }
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// When a lot (> ~700 in this case) of nodes are added, the chain becomes overquota.
+/// this leads to a load of messages being sent back to the shard. If bounded channels
+/// are used to send messages back to the shard, it's possible that we get into a situation
+/// where the shard is waiting trying to send the next "add node" message, while the
+/// telemetry core is waiting trying to send up to the shard the next "mute node" message,
+/// resulting in a deadlock. This test gives confidence that we don't run into such a deadlock.
+#[tokio::test]
+async fn lots_of_mute_messages_dont_cause_a_deadlock() {
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+
+    // Connect 1000 nodes to the shard:
+    let mut nodes = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_multiple_nodes(2000) // 1500 of these will be overquota.
+        .await
+        .expect("nodes can connect");
+
+    // Every node announces itself on the same chain:
+    for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
+        node_tx.send_json_text(json!({
+            "id":1, // message ID, not node ID. Can be the same for all.
+            "ts":"2021-07-12T10:37:47.714666+01:00",
+            "payload": {
+                "authority":true,
+                "chain":"Local Testnet",
+                "config":"",
+                "genesis_hash": BlockHash::from_low_u64_ne(1),
+                "implementation":"Substrate Node",
+                "msg":"system.connected",
+                "name": format!("Alice {}", idx),
+                "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                "startup_time":"1625565542717",
+                "version":"2.0.0-07a1af348-aarch64-macos"
+            }
+        })).unwrap();
+    }
+
+    // Wait a little time (just to let everything get deadlocked) before
+    // trying to have the aggregator send out feed messages.
+    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+
+    // Start a bunch of feeds. If deadlock has happened, none of them will
+    // receive any messages back.
+    let mut feeds = server
+        .get_core()
+        .connect_multiple_feeds(1)
+        .await
+        .expect("feeds can connect");
+
+    // Wait to see whether we get anything back:
+    let msgs_fut = futures::future::join_all(
+        feeds
+            .iter_mut()
+            .map(|(_,rx)| rx.recv_feed_messages())
+    );
+
+    // Give up after a timeout:
+    tokio::time::timeout(Duration::from_secs(10), msgs_fut)
+        .await
+        .expect("should not hit timeout waiting for messages (deadlock has happened)");
+}
+
+/// If a node is added, a connecting feed should be told about the new chain.
+/// If the node is removed, the feed should be told that the chain has gone.
+#[tokio::test]
+async fn feed_add_and_remove_node() {
+    // Connect server and add shard
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+
+    // Connect a node to the shard:
+    let (mut node_tx, _node_rx) = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_node()
+        .await
+        .expect("can connect to shard");
+
+    // Send a "system connected" message:
+    node_tx
+        .send_json_text(json!(
+            {
+                "id":1,
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain":"Local Testnet",
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(1),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name":"Alice",
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                },
+            }
+        ))
+        .unwrap();
+
+    // Wait a little for this message to propagate to the core
+    // (so that our feed connects after the core knows and not before).
+    tokio::time::sleep(Duration::from_millis(500)).await;
+
+    // Connect a feed.
+    let (_feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::AddedChain {
+        name: "Local Testnet".to_owned(),
+        node_count: 1
+    }));
+
+    // Disconnect the node:
+    node_tx.close().await.unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::RemovedChain {
+        name: "Local Testnet".to_owned(),
+    }));
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// If nodes connect and the chain name changes, feeds will be told about this
+/// and will keep receiving messages about the renamed chain (despite subscribing
+/// to it by name).
+#[tokio::test]
+async fn feeds_told_about_chain_rename_and_stay_subscribed() {
+    // Connect a node:
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+    let (mut node_tx, _node_rx) = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_node()
+        .await
+        .expect("can connect to shard");
+
+    let node_init_msg = |id, chain_name: &str, node_name: &str| json!({
+        "id":id,
+        "ts":"2021-07-12T10:37:47.714666+01:00",
+        "payload": {
+            "authority":true,
+            "chain": chain_name,
+            "config":"",
+            "genesis_hash": BlockHash::from_low_u64_ne(1),
+            "implementation":"Substrate Node",
+            "msg":"system.connected",
+            "name": node_name,
+            "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+            "startup_time":"1625565542717",
+            "version":"2.0.0-07a1af348-aarch64-macos"
+        },
+    });
+
+    // Subscribe a chain:
+    node_tx.send_json_text(node_init_msg(1, "Initial chain name", "Node 1")).unwrap();
+
+    // Connect a feed and subscribe to the above chain:
+    let (mut feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+    feed_tx.send_command("subscribe", "Initial chain name").unwrap();
+
+    // Feed is told about the chain, and the node on this chain:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedChain { name, node_count: 1 } if name == "Initial chain name",
+        FeedMessage::SubscribedTo { name } if name == "Initial chain name",
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 1",
+    );
+
+    // Subscribe another node. The chain doesn't rename yet but we are told about the new node
+    // count and the node that's been added.
+    node_tx.send_json_text(node_init_msg(2, "New chain name", "Node 2")).unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 2",
+        FeedMessage::AddedChain { name, node_count: 2 } if name == "Initial chain name",
+    );
+
+    // Subscribe a third node. The chain renames, so we're told about the new node but also
+    // about the chain rename.
+    node_tx.send_json_text(node_init_msg(3, "New chain name", "Node 3")).unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 3",
+        FeedMessage::RemovedChain { name } if name == "Initial chain name",
+        FeedMessage::AddedChain { name, node_count: 3 } if name == "New chain name",
+    );
+
+    // Just to be sure, subscribing a fourth node on this chain will still lead to updates
+    // to this feed.
+    node_tx.send_json_text(node_init_msg(4, "New chain name", "Node 4")).unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 4",
+        FeedMessage::AddedChain { name, node_count: 4 } if name == "New chain name",
+    );
+
+}
+
+/// If we add a couple of shards and a node for each, all feeds should be
+/// told about both node chains. If one shard goes away, we should get a
+/// "removed chain" message only for the node connected to that shard.
+#[tokio::test]
+async fn feed_add_and_remove_shard() {
+    let mut server = start_server_debug().await;
+
+    let mut shards = vec![];
+    for id in 1..=2 {
+        // Add a shard:
+        let shard_id = server.add_shard().await.unwrap();
+
+        // Connect a node to it:
+        let (mut node_tx, _node_rx) = server
+            .get_shard(shard_id)
+            .unwrap()
+            .connect_node()
+            .await
+            .expect("can connect to shard");
+
+        // Send a "system connected" message:
+        node_tx
+            .send_json_text(json!({
+                "id":id,
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain": format!("Local Testnet {}", id),
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(id),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name":"Alice",
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                },
+            }))
+            .unwrap();
+
+        // Keep what we need to to keep connection alive and let us kill a shard:
+        shards.push((shard_id, node_tx));
+    }
+
+    // Connect a feed.
+    let (_feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    // The feed should be told about both of the chains that we've sent info about:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::AddedChain {
+        name: "Local Testnet 1".to_owned(),
+        node_count: 1
+    }));
+    assert!(feed_messages.contains(&FeedMessage::AddedChain {
+        name: "Local Testnet 2".to_owned(),
+        node_count: 1
+    }));
+
+    // Disconnect the first shard:
+    server.kill_shard(shards[0].0).await;
+
+    // We should be told about the node connected to that shard disconnecting:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::RemovedChain {
+        name: "Local Testnet 1".to_owned(),
+    }));
+    assert!(!feed_messages.contains(
+        // Spot the "!"; this chain was not removed.
+        &FeedMessage::RemovedChain {
+            name: "Local Testnet 2".to_owned(),
+        }
+    ));
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// feeds can subscribe to one chain at a time. They should get the relevant
+/// messages for that chain and no other.
+#[tokio::test]
+async fn feed_can_subscribe_and_unsubscribe_from_chain() {
+    use FeedMessage::*;
+
+    // Start server, add shard, connect node:
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+    let (mut node_tx, _node_rx) = server.get_shard(shard_id).unwrap().connect_node().await.unwrap();
+
+    // Send a "system connected" message for a few nodes/chains:
+    for id in 1..=3 {
+        node_tx
+            .send_json_text(json!(
+                {
+                    "id":id,
+                    "ts":"2021-07-12T10:37:47.714666+01:00",
+                    "payload": {
+                        "authority":true,
+                        "chain":format!("Local Testnet {}", id),
+                        "config":"",
+                        "genesis_hash": BlockHash::from_low_u64_ne(id),
+                        "implementation":"Substrate Node",
+                        "msg":"system.connected",
+                        "name":format!("Alice {}", id),
+                        "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                        "startup_time":"1625565542717",
+                        "version":"2.0.0-07a1af348-aarch64-macos"
+                    },
+                }
+            ))
+            .unwrap();
+    }
+
+    // Connect a feed
+    let (mut feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(feed_messages, AddedChain { name, node_count: 1 } if name == "Local Testnet 1");
+
+    // Subscribe it to a chain
+    feed_tx.send_command("subscribe", "Local Testnet 1").unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        SubscribedTo { name } if name == "Local Testnet 1",
+        TimeSync {..},
+        BestBlock { block_number: 0, timestamp: 0, avg_block_time: None },
+        BestFinalized { block_number: 0, .. },
+        AddedNode { node_id: 0, node: NodeDetails { name, .. }, .. } if name == "Alice 1",
+        FinalizedBlock { node_id: 0, block_number: 0, .. }
+    );
+
+    // We receive updates relating to nodes on that chain:
+    node_tx.send_json_text(json!(
+        {"id":1, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:37:48.330433+01:00" }
+    )).unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_ne!(feed_messages.len(), 0);
+
+    // We don't receive anything for updates to nodes on other chains (wait a sec to ensure no messages are sent):
+    node_tx.send_json_text(json!(
+        {"id":2, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:37:48.330433+01:00" }
+    )).unwrap();
+
+    tokio::time::timeout(Duration::from_secs(1), feed_rx.recv_feed_messages())
+        .await
+        .expect_err("Timeout should elapse since no messages sent");
+
+    // We can change our subscription:
+    feed_tx.send_command("subscribe", "Local Testnet 2").unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+
+    // We are told about the subscription change and given similar on-subscribe messages to above.
+    assert_contains_matches!(
+        &feed_messages,
+        UnsubscribedFrom { name } if name == "Local Testnet 1",
+        SubscribedTo { name } if name == "Local Testnet 2",
+        TimeSync {..},
+        BestBlock {..},
+        BestFinalized {..},
+        AddedNode { node: NodeDetails { name, .. }, ..} if name == "Alice 2",
+        FinalizedBlock {..},
+    );
+
+    // We didn't get messages from this earlier, but we will now we've subscribed:
+    node_tx.send_json_text(json!(
+        {"id":2, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
+    )).unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_ne!(feed_messages.len(), 0);
+
+    // Tidy up:
+    server.shutdown().await;
+}

backend/telemetry_core/tests/soak_tests.rs

@@ -0,0 +1,211 @@
+/*!
+Soak tests. These are ignored by default, and are intended to be long runs
+of the core + shards(s) under different loads to get a feel for CPU/memory
+usage and general performance over time.
+
+Note that on MacOS inparticular, you may need to increase some limits to be
+able to open a large number of connections. Try commands like:
+
+```sh
+sudo sysctl -w kern.maxfiles=50000
+sudo sysctl -w kern.maxfilesperproc=50000
+ulimit -n 50000
+sudo sysctl -w kern.ipc.somaxconn=50000
+sudo sysctl -w kern.ipc.maxsockbuf=16777216
+```
+
+In general, if you run into issues, it may be better to run this on a linux
+box; MacOS seems to hit limits quicker in general.
+*/
+
+use futures::{ StreamExt };
+use structopt::StructOpt;
+use test_utils::workspace::start_server_release;
+use common::ws_client::{ SentMessage };
+use serde_json::json;
+use std::time::Duration;
+use std::sync::atomic::{ Ordering, AtomicUsize };
+use std::sync::Arc;
+use common::node_types::BlockHash;
+
+/// A configurable soak_test runner. Configure by providing the expected args as
+/// an environment variable. One example to run this test is:
+///
+/// ```sh
+/// SOAK_TEST_ARGS='--feeds 10 --nodes 100 --shards 4' cargo test -- soak_test --ignored --nocapture
+/// ```
+///
+/// You can also run this test against the pre-sharding actix binary with something like this:
+/// ```sh
+/// TELEMETRY_BIN=~/old_telemetry_binary SOAK_TEST_ARGS='--feeds 100 --nodes 100 --shards 4' cargo test -- soak_test --ignored --nocapture
+/// ```
+///
+/// Or, you can run it against existing processes with something like this:
+/// ```sh
+/// TELEMETRY_SUBMIT_HOSTS='127.0.0.1:8001' TELEMETRY_FEED_HOST='127.0.0.1:8000' SOAK_TEST_ARGS='--feeds 100 --nodes 100 --shards 4' cargo test -- soak_test --ignored --nocapture
+/// ```
+///
+/// Each will establish the same total number of connections and send the same messages.
+#[ignore]
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+pub async fn soak_test() {
+    let opts = get_soak_test_opts();
+    run_soak_test(opts).await;
+}
+
+/// The general soak test runner. This is called by tests.
+async fn run_soak_test(opts: SoakTestOpts) {
+    let mut server = start_server_release().await;
+
+    // Start up the shards we requested:
+    let mut shard_ids = vec![];
+    for _ in 0..opts.shards {
+        let shard_id = server.add_shard().await.expect("shard can't be added");
+        shard_ids.push(shard_id);
+    }
+
+    // Connect nodes to each shard:
+    let mut nodes = vec![];
+    for &shard_id in &shard_ids {
+        let mut conns = server
+            .get_shard(shard_id)
+            .unwrap()
+            .connect_multiple_nodes(opts.nodes)
+            .await
+            .expect("node connections failed");
+        nodes.append(&mut conns);
+    }
+
+    // Each node tells the shard about itself:
+    for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
+        node_tx.send_json_binary(json!({
+            "id":1, // Only needs to be unique per node
+            "ts":"2021-07-12T10:37:47.714666+01:00",
+            "payload": {
+                "authority":true,
+                "chain": "Polkadot", // <- so that we don't go over quota with lots of nodes.
+                "config":"",
+                "genesis_hash": BlockHash::from_low_u64_ne(1),
+                "implementation":"Substrate Node",
+                "msg":"system.connected",
+                "name": format!("Node #{}", idx),
+                "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                "startup_time":"1625565542717",
+                "version":"2.0.0-07a1af348-aarch64-macos"
+            },
+        })).unwrap();
+    }
+
+    // Connect feeds to the core:
+    let mut feeds = server
+        .get_core()
+        .connect_multiple_feeds(opts.feeds)
+        .await
+        .expect("feed connections failed");
+
+    // Every feed subscribes to the chain above to recv messages about it:
+    for (feed_tx, _) in &mut feeds {
+        feed_tx.send_command("subscribe", "Polkadot").unwrap();
+    }
+
+    // Start sending "update" messages from nodes at time intervals.
+    let bytes_in = Arc::new(AtomicUsize::new(0));
+    let bytes_in2 = Arc::clone(&bytes_in);
+    let send_handle = tokio::task::spawn(async move {
+        let msg = json!({
+            "id":1,
+            "payload":{
+                "bandwidth_download":576,
+                "bandwidth_upload":576,
+                "msg":"system.interval",
+                "peers":1
+            },
+            "ts":"2021-07-12T10:37:48.330433+01:00"
+        });
+        let msg_bytes: &'static [u8] = Box::new(serde_json::to_vec(&msg).unwrap()).leak();
+
+        loop {
+            // every ~1second we aim to have sent messages from all of the nodes. So we cycle through
+            // the node IDs and send a message from each at roughly 1s / number_of_nodes.
+            let mut interval = tokio::time::interval(Duration::from_secs_f64(1.0 / nodes.len() as f64));
+
+            for node_id in (0..nodes.len()).cycle() {
+                interval.tick().await;
+                let node_tx = &mut nodes[node_id].0;
+                node_tx.unbounded_send(SentMessage::StaticBinary(msg_bytes)).unwrap();
+                bytes_in2.fetch_add(msg_bytes.len(), Ordering::Relaxed);
+            }
+        }
+    });
+
+    // Also start receiving messages, counting the bytes received so far.
+    let bytes_out = Arc::new(AtomicUsize::new(0));
+    for (_, mut feed_rx) in feeds {
+        let bytes_out = Arc::clone(&bytes_out);
+        tokio::task::spawn(async move {
+            while let Some(msg) = feed_rx.next().await {
+                let msg = msg.expect("message could be received");
+                let num_bytes = msg.len();
+                bytes_out.fetch_add(num_bytes, Ordering::Relaxed);
+            }
+        });
+    }
+
+    // Periodically report on bytes out
+    tokio::task::spawn(async move {
+        let one_mb = 1024.0 * 1024.0;
+        let mut last_bytes_in = 0;
+        let mut last_bytes_out = 0;
+        let mut n = 1;
+        loop {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            let bytes_in_val = bytes_in.load(Ordering::Relaxed);
+            let bytes_out_val = bytes_out.load(Ordering::Relaxed);
+
+            println!("#{}: MB in/out per measurement: {:.4} / {:.4}, total bytes in/out: {} / {})",
+                n,
+                (bytes_in_val - last_bytes_in) as f64 / one_mb,
+                (bytes_out_val - last_bytes_out) as f64 / one_mb,
+                bytes_in_val,
+                bytes_out_val
+            );
+
+            n += 1;
+            last_bytes_in = bytes_in_val;
+            last_bytes_out = bytes_out_val;
+        }
+    });
+
+    // Wait for sending to finish before ending.
+    send_handle.await.unwrap();
+}
+
+/// General arguments that are used to start a soak test. Run `soak_test` as
+/// instructed by its documentation for full control over what is ran, or run
+/// preconfigured variants.
+#[derive(StructOpt, Debug)]
+struct SoakTestOpts {
+    /// The number of shards to run this test with
+    #[structopt(long)]
+    shards: usize,
+    /// The number of feeds to connect
+    #[structopt(long)]
+    feeds: usize,
+    /// The number of nodes to connect to each feed
+    #[structopt(long)]
+    nodes: usize
+}
+
+/// Get soak test args from an envvar and parse them via structopt.
+fn get_soak_test_opts() -> SoakTestOpts {
+    let arg_string = std::env::var("SOAK_TEST_ARGS")
+        .expect("Expecting args to be provided in the env var SOAK_TEST_ARGS");
+    let args = shellwords::split(&arg_string)
+        .expect("Could not parse SOAK_TEST_ARGS as shell arguments");
+
+    // The binary name is expected to be the first arg, so fake it:
+    let all_args = std::iter::once("soak_test".to_owned())
+        .chain(args.into_iter());
+
+    SoakTestOpts::from_iter(all_args)
+}

backend/telemetry_shard/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "telemetry_shard"
+version = "0.1.0"
+authors = ["Parity Technologies Ltd. <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0"
+
+[dependencies]
+anyhow = "1.0.41"
+bincode = "1.3.3"
+common = { path = "../common" }
+futures = "0.3.15"
+hex = "0.4.3"
+http = "0.2.4"
+log = "0.4.14"
+primitive-types = { version = "0.9.0", features = ["serde"] }
+serde = { version = "1.0.126", features = ["derive"] }
+serde_json = "1.0.64"
+simple_logger = "1.11.0"
+soketto = "0.6.0"
+structopt = "0.3.21"
+thiserror = "1.0.25"
+tokio = { version = "1.7.0", features = ["full"] }
+tokio-util = { version = "0.6", features = ["compat"] }
+warp = "0.3.1"

backend/telemetry_shard/src/aggregator.rs

@@ -0,0 +1,283 @@
+use crate::connection::{create_ws_connection_to_core, Message};
+use common::{
+    internal_messages::{self, ShardNodeId},
+    node_message,
+    node_types::BlockHash,
+    AssignId,
+};
+use futures::{channel::mpsc};
+use futures::{Sink, SinkExt, StreamExt};
+use std::collections::{HashMap, HashSet};
+use std::sync::atomic::AtomicU64;
+use std::sync::Arc;
+
+/// A unique Id is assigned per websocket connection (or more accurately,
+/// per thing-that-subscribes-to-the-aggregator). That connection might send
+/// data on behalf of multiple chains, so this ID is local to the aggregator,
+/// and a unique ID is assigned per batch of data too ([`internal_messages::LocalId`]).
+type ConnId = u64;
+
+/// Incoming messages are either from websocket connections or
+/// from the telemetry core. This can be private since the only
+/// external messages are via subscriptions that take
+/// [`FromWebsocket`] instances.
+#[derive(Clone, Debug)]
+enum ToAggregator {
+    /// Sent when the telemetry core is disconnected.
+    DisconnectedFromTelemetryCore,
+    /// Sent when the telemetry core (re)connects.
+    ConnectedToTelemetryCore,
+    /// Sent when a message comes in from a substrate node.
+    FromWebsocket(ConnId, FromWebsocket),
+    /// Send when a message comes in from the telemetry core.
+    FromTelemetryCore(internal_messages::FromTelemetryCore),
+}
+
+/// An incoming socket connection can provide these messages.
+/// Until a node has been Added via [`FromWebsocket::Add`],
+/// messages from it will be ignored.
+#[derive(Clone, Debug)]
+pub enum FromWebsocket {
+    /// Fire this when the connection is established.
+    Initialize {
+        /// When a message is sent back up this channel, we terminate
+        /// the websocket connection and force the node to reconnect
+        /// so that it sends its system info again incase the telemetry
+        /// core has restarted.
+        close_connection: mpsc::Sender<()>,
+    },
+    /// Tell the aggregator about a new node.
+    Add {
+        message_id: node_message::NodeMessageId,
+        ip: Option<std::net::IpAddr>,
+        node: common::node_types::NodeDetails,
+        genesis_hash: BlockHash,
+    },
+    /// Update/pass through details about a node.
+    Update {
+        message_id: node_message::NodeMessageId,
+        payload: node_message::Payload,
+    },
+    /// Make a note when the node disconnects.
+    Disconnected,
+}
+
+pub type FromAggregator = internal_messages::FromShardAggregator;
+
+/// The aggregator loop handles incoming messages from nodes, or from the telemetry core.
+/// this is where we decide what effect messages will have.
+#[derive(Clone)]
+pub struct Aggregator(Arc<AggregatorInternal>);
+
+struct AggregatorInternal {
+    /// Nodes that connect are each assigned a unique connection ID. Nodes
+    /// can send messages on behalf of more than one chain, and so this ID is
+    /// only really used inside the Aggregator in conjunction with a per-message
+    /// ID.
+    conn_id: AtomicU64,
+    /// Send messages to the aggregator from websockets via this. This is
+    /// stored here so that anybody holding an `Aggregator` handle can
+    /// make use of it.
+    tx_to_aggregator: mpsc::Sender<ToAggregator>,
+}
+
+impl Aggregator {
+    /// Spawn a new Aggregator. This connects to the telemetry backend
+    pub async fn spawn(telemetry_uri: http::Uri) -> anyhow::Result<Aggregator> {
+        let (tx_to_aggregator, rx_from_external) = mpsc::channel(10);
+
+        // Establish a resiliant connection to the core (this retries as needed):
+        let (tx_to_telemetry_core, mut rx_from_telemetry_core) =
+            create_ws_connection_to_core(telemetry_uri).await;
+
+        // Forward messages from the telemetry core into the aggregator:
+        let mut tx_to_aggregator2 = tx_to_aggregator.clone();
+        tokio::spawn(async move {
+            while let Some(msg) = rx_from_telemetry_core.next().await {
+                let msg_to_aggregator = match msg {
+                    Message::Connected => ToAggregator::ConnectedToTelemetryCore,
+                    Message::Disconnected => ToAggregator::DisconnectedFromTelemetryCore,
+                    Message::Data(data) => ToAggregator::FromTelemetryCore(data),
+                };
+                if let Err(_) = tx_to_aggregator2.send(msg_to_aggregator).await {
+                    // This will close the ws channels, which themselves log messages.
+                    break
+                }
+            }
+        });
+
+        // Start our aggregator loop, handling any incoming messages:
+        tokio::spawn(Aggregator::handle_messages(
+            rx_from_external,
+            tx_to_telemetry_core,
+        ));
+
+        // Return a handle to our aggregator so that we can send in messages to it:
+        Ok(Aggregator(Arc::new(AggregatorInternal {
+            conn_id: AtomicU64::new(1),
+            tx_to_aggregator,
+        })))
+    }
+
+    // This is spawned into a separate task and handles any messages coming
+    // in to the aggregator. If nobody is holding the tx side of the channel
+    // any more, this task will gracefully end.
+    async fn handle_messages(
+        mut rx_from_external: mpsc::Receiver<ToAggregator>,
+        mut tx_to_telemetry_core: mpsc::Sender<FromAggregator>,
+    ) {
+        use internal_messages::{FromShardAggregator, FromTelemetryCore};
+
+        // Just as an optimisation, we can keep track of whether we're connected to the backend
+        // or not, and ignore incoming messages while we aren't.
+        let mut connected_to_telemetry_core = false;
+
+        // A list of close channels for the currently connected substrate nodes. Send an empty
+        // tuple to these to ask the connections to be closed.
+        let mut close_connections: HashMap<ConnId, mpsc::Sender<()>> = HashMap::new();
+
+        // Maintain mappings from the connection ID and node message ID to the "local ID" which we
+        // broadcast to the telemetry core.
+        let mut to_local_id = AssignId::new();
+
+        // Any messages coming from nodes that have been muted are ignored:
+        let mut muted: HashSet<ShardNodeId> = HashSet::new();
+
+        // Now, loop and receive messages to handle.
+        while let Some(msg) = rx_from_external.next().await {
+            match msg {
+                ToAggregator::ConnectedToTelemetryCore => {
+                    // Take hold of the connection closers and run them all.
+                    let closers = close_connections;
+
+                    for (_, mut closer) in closers {
+                        // if this fails, it probably means the connection has died already anyway.
+                        let _ = closer.send(()).await;
+                    }
+
+                    // We've told everything to disconnect. Now, reset our state:
+                    close_connections = HashMap::new();
+                    to_local_id.clear();
+                    muted.clear();
+
+                    connected_to_telemetry_core = true;
+                    log::info!("Connected to telemetry core");
+                }
+                ToAggregator::DisconnectedFromTelemetryCore => {
+                    connected_to_telemetry_core = false;
+                    log::info!("Disconnected from telemetry core");
+                }
+                ToAggregator::FromWebsocket(
+                    conn_id,
+                    FromWebsocket::Initialize { close_connection },
+                ) => {
+                    // We boot all connections on a reconnect-to-core to force new systemconnected
+                    // messages to be sent. We could boot on muting, but need to be careful not to boot
+                    // connections where we mute one set of messages it sends and not others.
+                    close_connections.insert(conn_id, close_connection);
+                }
+                ToAggregator::FromWebsocket(
+                    conn_id,
+                    FromWebsocket::Add {
+                        message_id,
+                        ip,
+                        node,
+                        genesis_hash,
+                    },
+                ) => {
+                    // Don't bother doing anything else if we're disconnected, since we'll force the
+                    // node to reconnect anyway when the backend does:
+                    if !connected_to_telemetry_core {
+                        continue;
+                    }
+
+                    // Generate a new "local ID" for messages from this connection:
+                    let local_id = to_local_id.assign_id((conn_id, message_id));
+
+                    // Send the message to the telemetry core with this local ID:
+                    let _ = tx_to_telemetry_core
+                        .send(FromShardAggregator::AddNode {
+                            ip,
+                            node,
+                            genesis_hash,
+                            local_id,
+                        })
+                        .await;
+                }
+                ToAggregator::FromWebsocket(
+                    conn_id,
+                    FromWebsocket::Update {
+                        message_id,
+                        payload,
+                    },
+                ) => {
+                    // Ignore incoming messages if we're not connected to the backend:
+                    if !connected_to_telemetry_core {
+                        continue;
+                    }
+
+                    // Get the local ID, ignoring the message if none match:
+                    let local_id = match to_local_id.get_id(&(conn_id, message_id)) {
+                        Some(id) => id,
+                        None => continue,
+                    };
+
+                    // ignore the message if this node has been muted:
+                    if muted.contains(&local_id) {
+                        continue;
+                    }
+
+                    // Send the message to the telemetry core with this local ID:
+                    let _ = tx_to_telemetry_core
+                        .send(FromShardAggregator::UpdateNode { local_id, payload })
+                        .await;
+                }
+                ToAggregator::FromWebsocket(disconnected_conn_id, FromWebsocket::Disconnected) => {
+                    // Find all of the local IDs corresponding to the disconnected connection ID and
+                    // remove them, telling Telemetry Core about them too. This could be more efficient,
+                    // but the mapping isn't currently cached and it's not a super frequent op.
+                    let local_ids_disconnected: Vec<_> = to_local_id
+                        .iter()
+                        .filter(|(_, &(conn_id, _))| disconnected_conn_id == conn_id)
+                        .map(|(local_id, _)| local_id)
+                        .collect();
+
+                    close_connections.remove(&disconnected_conn_id);
+
+                    for local_id in local_ids_disconnected {
+                        to_local_id.remove_by_id(local_id);
+                        muted.remove(&local_id);
+                        let _ = tx_to_telemetry_core
+                            .send(FromShardAggregator::RemoveNode { local_id })
+                            .await;
+                    }
+                }
+                ToAggregator::FromTelemetryCore(FromTelemetryCore::Mute {
+                    local_id,
+                    reason: _,
+                }) => {
+                    // Mute the local ID we've been told to:
+                    muted.insert(local_id);
+                }
+            }
+        }
+    }
+
+    /// Return a sink that a node can send messages into to be handled by the aggregator.
+    pub fn subscribe_node(&self) -> impl Sink<FromWebsocket, Error = anyhow::Error> + Unpin {
+        // Assign a unique aggregator-local ID to each connection that subscribes, and pass
+        // that along with every message to the aggregator loop:
+        let conn_id: ConnId = self
+            .0
+            .conn_id
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let tx_to_aggregator = self.0.tx_to_aggregator.clone();
+
+        // Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
+        // but pinning by boxing is the easy solution for now:
+        Box::pin(
+            tx_to_aggregator
+                .with(move |msg| async move { Ok(ToAggregator::FromWebsocket(conn_id, msg)) }),
+        )
+    }
+}

backend/telemetry_shard/src/connection.rs

@@ -0,0 +1,129 @@
+use futures::channel::mpsc;
+use futures::{SinkExt, StreamExt};
+use common::ws_client;
+use bincode::Options;
+
+#[derive(Clone, Debug)]
+pub enum Message<Out> {
+    Connected,
+    Disconnected,
+    Data(Out),
+}
+
+/// Connect to the telemetry core, retrying the connection if we're disconnected.
+/// - Sends `Message::Connected` and `Message::Disconnected` when the connection goes up/down.
+/// - Returns a channel that allows you to send messages to the connection.
+/// - Messages are all encoded/decoded to/from bincode, and so need to support being (de)serialized from
+///   a non self-describing encoding.
+///
+/// Note: have a look at [`common::internal_messages`] to see the different message types exchanged
+/// between aggregator and core.
+pub async fn create_ws_connection_to_core<In, Out>(
+    telemetry_uri: http::Uri,
+) -> (mpsc::Sender<In>, mpsc::Receiver<Message<Out>>)
+where
+    In: serde::Serialize + Send + 'static,
+    Out: serde::de::DeserializeOwned + Send + 'static,
+{
+    let (tx_in, mut rx_in) = mpsc::channel(10);
+    let (mut tx_out, rx_out) = mpsc::channel(10);
+
+    let mut is_connected = false;
+
+    tokio::spawn(async move {
+        loop {
+            // Throw away any pending messages from the incoming channel so that it
+            // doesn't get filled up and begin blocking while we're looping and waiting
+            // for a reconnection.
+            while let Ok(Some(_)) = rx_in.try_next() {}
+
+            // Try to connect. If connection established, we serialize and forward messages
+            // to/from the core. If the external channels break, we end for good. If the internal
+            // channels break, we loop around and try connecting again.
+            match ws_client::connect(&telemetry_uri).await {
+                Ok((tx_to_core, mut rx_from_core)) => {
+                    is_connected = true;
+                    let mut tx_out = tx_out.clone();
+
+                    if let Err(e) = tx_out.send(Message::Connected).await {
+                        // If receiving end is closed, bail now.
+                        log::warn!("Aggregator is no longer receiving messages from core; disconnecting (permanently): {}", e);
+                        return
+                    }
+
+                    // Loop, forwarding messages to and from the core until something goes wrong.
+                    loop {
+                        tokio::select! {
+                            msg = rx_from_core.next() => {
+                                let msg = match msg {
+                                    Some(msg) => msg,
+                                    // No more messages from core? core WS is disconnected.
+                                    None => {
+                                        log::warn!("No more messages from core: shutting down connection (will reconnect)");
+                                        break
+                                    }
+                                };
+
+                                let bytes = match msg {
+                                    Ok(ws_client::RecvMessage::Binary(bytes)) => bytes,
+                                    Ok(ws_client::RecvMessage::Text(s)) => s.into_bytes(),
+                                    Err(e) => {
+                                        log::warn!("Unable to receive message from core: shutting down connection (will reconnect): {}", e);
+                                        break;
+                                    }
+                                };
+                                let msg = bincode::options()
+                                    .deserialize(&bytes)
+                                    .expect("internal messages must be deserializable");
+
+                                if let Err(e) = tx_out.send(Message::Data(msg)).await {
+                                    log::error!("Aggregator is no longer receiving messages from core; disconnecting (permanently): {}", e);
+                                    return;
+                                }
+                            },
+                            msg = rx_in.next() => {
+                                let msg = match msg {
+                                    Some(msg) => msg,
+                                    None => {
+                                        log::error!("Aggregator is no longer sending messages to core; disconnecting (permanently)");
+                                        return
+                                    }
+                                };
+
+                                let bytes = bincode::options()
+                                    .serialize(&msg)
+                                    .expect("internal messages must be serializable");
+                                let ws_msg = ws_client::SentMessage::Binary(bytes);
+
+                                if let Err(e) = tx_to_core.unbounded_send(ws_msg) {
+                                    log::warn!("Unable to send message to core; shutting down connection (will reconnect): {}", e);
+                                    break;
+                                }
+                            }
+                        };
+                    }
+                },
+                Err(connect_err) => {
+                    // Issue connecting? Wait and try again on the next loop iteration.
+                    log::error!(
+                        "Error connecting to websocker server (will reconnect): {}",
+                        connect_err
+                    );
+                }
+            }
+
+            if is_connected {
+                is_connected = false;
+                if let Err(e) = tx_out.send(Message::Disconnected).await {
+                    log::error!("Aggregator is no longer receiving messages from core; disconnecting (permanently): {}", e);
+                    return;
+                }
+            }
+
+            // Wait a little before we try to connect again.
+            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+        }
+    });
+
+    (tx_in, rx_out)
+}

backend/telemetry_shard/src/json_message/hash.rs

@@ -0,0 +1,219 @@
+//! A hash wrapper which can be deserialized from a hex string as well as from an array of bytes,
+//! so that it can deal with the sort of inputs we expect from substrate nodes.
+
+use serde::de::{self, Deserialize, Deserializer, SeqAccess, Unexpected, Visitor};
+use serde::ser::{Serialize, Serializer};
+use std::fmt::{self, Debug, Display};
+use std::str::FromStr;
+
+/// We assume that hashes are 32 bytes long, and in practise that's currently true,
+/// but in theory it doesn't need to be. We may need to be more dynamic here.
+const HASH_BYTES: usize = 32;
+
+/// Newtype wrapper for 32-byte hash values, implementing readable `Debug` and `serde::Deserialize`.
+/// This can deserialize from a JSON string or array.
+#[derive(Hash, PartialEq, Eq, Clone, Copy)]
+pub struct Hash([u8; HASH_BYTES]);
+
+impl From<Hash> for common::node_types::BlockHash {
+    fn from(hash: Hash) -> Self {
+        hash.0.into()
+    }
+}
+
+impl From<common::node_types::BlockHash> for Hash {
+    fn from(hash: common::node_types::BlockHash) -> Self {
+        Hash(hash.0)
+    }
+}
+
+struct HashVisitor;
+
+impl<'de> Visitor<'de> for HashVisitor {
+    type Value = Hash;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str(
+            "byte array of length 32, or hexidecimal string of 32 bytes beginning with 0x",
+        )
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        value
+            .parse()
+            .map_err(|_| de::Error::invalid_value(Unexpected::Str(value), &self))
+    }
+
+    fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.len() == HASH_BYTES {
+            let mut hash = [0; HASH_BYTES];
+
+            hash.copy_from_slice(value);
+
+            return Ok(Hash(hash));
+        }
+
+        Hash::from_ascii(value)
+            .map_err(|_| de::Error::invalid_value(Unexpected::Bytes(value), &self))
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: SeqAccess<'de>,
+    {
+        let mut hash = [0u8; HASH_BYTES];
+
+        for (i, byte) in hash.iter_mut().enumerate() {
+            match seq.next_element()? {
+                Some(b) => *byte = b,
+                None => return Err(de::Error::invalid_length(i, &"an array of 32 bytes")),
+            }
+        }
+
+        if seq.next_element::<u8>()?.is_some() {
+            return Err(de::Error::invalid_length(33, &"an array of 32 bytes"));
+        }
+
+        Ok(Hash(hash))
+    }
+}
+
+impl Hash {
+    pub fn from_ascii(value: &[u8]) -> Result<Self, HashParseError> {
+        if !value.starts_with(b"0x") {
+            return Err(HashParseError::InvalidPrefix);
+        }
+
+        let mut hash = [0; HASH_BYTES];
+
+        hex::decode_to_slice(&value[2..], &mut hash).map_err(HashParseError::HexError)?;
+
+        Ok(Hash(hash))
+    }
+}
+
+impl FromStr for Hash {
+    type Err = HashParseError;
+
+    fn from_str(value: &str) -> Result<Self, Self::Err> {
+        Hash::from_ascii(value.as_bytes())
+    }
+}
+
+impl<'de> Deserialize<'de> for Hash {
+    fn deserialize<D>(deserializer: D) -> Result<Hash, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_bytes(HashVisitor)
+    }
+}
+
+impl Serialize for Hash {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(&self.0)
+    }
+}
+
+impl Display for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str("0x")?;
+
+        let mut ascii = [0; HASH_BYTES * 2];
+
+        hex::encode_to_slice(self.0, &mut ascii)
+            .expect("Encoding 32 bytes into 64 bytes of ascii; qed");
+
+        f.write_str(std::str::from_utf8(&ascii).expect("ASCII hex encoded bytes can't fail; qed"))
+    }
+}
+
+impl Debug for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        Display::fmt(self, f)
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum HashParseError {
+    #[error("Error parsing string into hex: {0}")]
+    HexError(hex::FromHexError),
+    #[error("Invalid hex prefix: expected '0x'")]
+    InvalidPrefix,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Hash;
+    use bincode::Options;
+
+    const DUMMY: Hash = {
+        let mut hash = [0; 32];
+        hash[0] = 0xDE;
+        hash[1] = 0xAD;
+        hash[2] = 0xBE;
+        hash[3] = 0xEF;
+        Hash(hash)
+    };
+
+    #[test]
+    fn deserialize_json_hash_str() {
+        let json = r#""0xdeadBEEF00000000000000000000000000000000000000000000000000000000""#;
+
+        let hash: Hash = serde_json::from_str(json).unwrap();
+
+        assert_eq!(hash, DUMMY);
+    }
+
+    #[test]
+    fn deserialize_json_array() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let hash: Hash = serde_json::from_str(json).unwrap();
+
+        assert_eq!(hash, DUMMY);
+    }
+
+    #[test]
+    fn deserialize_json_array_too_short() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let res = serde_json::from_str::<Hash>(json);
+
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn deserialize_json_array_too_long() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let res = serde_json::from_str::<Hash>(json);
+
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn bincode() {
+        let bytes = bincode::options().serialize(&DUMMY).unwrap();
+
+        let mut expected = [0; 33];
+
+        expected[0] = 32; // length
+        expected[1..].copy_from_slice(&DUMMY.0);
+
+        assert_eq!(bytes, &expected);
+
+        let deserialized: Hash = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(DUMMY, deserialized);
+    }
+}

backend/telemetry_shard/src/json_message/mod.rs

@@ -0,0 +1,7 @@
+//! This module contains the types we need to deserialize JSON messages from nodes
+
+mod hash;
+mod node_message;
+
+pub use hash::Hash;
+pub use node_message::*;

backend/telemetry_shard/src/json_message/node_message.rs

@@ -0,0 +1,344 @@
+//! The structs and enums defined in this module are largely identical to those
+//! we'll use elsewhere internally, but are kept separate so that the JSON structure
+//! is defined (almost) from just this file, and we don't have to worry about breaking
+//! compatibility with the input data when we make changes to our internal data
+//! structures (for example, to support bincode better).
+use super::hash::Hash;
+use common::node_message as internal;
+use common::node_types;
+use serde::Deserialize;
+
+/// This struct represents a telemetry message sent from a node as
+/// a JSON payload. Since JSON is self describing, we can use attributes
+/// like serde(untagged) and serde(flatten) without issue.
+///
+/// Internally, we want to minimise the amount of data sent from shards to
+/// the core node. For that reason, we use a non-self-describing serialization
+/// format like bincode, which doesn't support things like `[serde(flatten)]` (which
+/// internally wants to serialize to a map of unknown length) or `[serde(tag/untagged)]`
+/// (which relies on the data to know which variant to deserialize to.)
+///
+/// So, this can be converted fairly cheaply into an enum we'll use internally
+/// which is compatible with formats like bincode.
+#[derive(Deserialize, Debug)]
+#[serde(untagged)]
+pub enum NodeMessage {
+    V1 {
+        #[serde(flatten)]
+        payload: Payload,
+    },
+    V2 {
+        id: NodeMessageId,
+        payload: Payload,
+    },
+}
+
+impl From<NodeMessage> for internal::NodeMessage {
+    fn from(msg: NodeMessage) -> Self {
+        match msg {
+            NodeMessage::V1 { payload } => internal::NodeMessage::V1 {
+                payload: payload.into(),
+            },
+            NodeMessage::V2 { id, payload } => internal::NodeMessage::V2 {
+                id,
+                payload: payload.into(),
+            },
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(tag = "msg")]
+pub enum Payload {
+    #[serde(rename = "system.connected")]
+    SystemConnected(SystemConnected),
+    #[serde(rename = "system.interval")]
+    SystemInterval(SystemInterval),
+    #[serde(rename = "block.import")]
+    BlockImport(Block),
+    #[serde(rename = "notify.finalized")]
+    NotifyFinalized(Finalized),
+    #[serde(rename = "txpool.import")]
+    TxPoolImport,
+    #[serde(rename = "afg.finalized")]
+    AfgFinalized(AfgFinalized),
+    #[serde(rename = "afg.received_precommit")]
+    AfgReceivedPrecommit(AfgReceived),
+    #[serde(rename = "afg.received_prevote")]
+    AfgReceivedPrevote(AfgReceived),
+    #[serde(rename = "afg.received_commit")]
+    AfgReceivedCommit(AfgReceived),
+    #[serde(rename = "afg.authority_set")]
+    AfgAuthoritySet(AfgAuthoritySet),
+    #[serde(rename = "afg.finalized_blocks_up_to")]
+    AfgFinalizedBlocksUpTo,
+    #[serde(rename = "aura.pre_sealed_block")]
+    AuraPreSealedBlock,
+    #[serde(rename = "prepared_block_for_proposing")]
+    PreparedBlockForProposing,
+}
+
+impl From<Payload> for internal::Payload {
+    fn from(msg: Payload) -> Self {
+        match msg {
+            Payload::SystemConnected(m) => internal::Payload::SystemConnected(m.into()),
+            Payload::SystemInterval(m) => internal::Payload::SystemInterval(m.into()),
+            Payload::BlockImport(m) => internal::Payload::BlockImport(m.into()),
+            Payload::NotifyFinalized(m) => internal::Payload::NotifyFinalized(m.into()),
+            Payload::TxPoolImport => internal::Payload::TxPoolImport,
+            Payload::AfgFinalized(m) => internal::Payload::AfgFinalized(m.into()),
+            Payload::AfgReceivedPrecommit(m) => internal::Payload::AfgReceivedPrecommit(m.into()),
+            Payload::AfgReceivedPrevote(m) => internal::Payload::AfgReceivedPrevote(m.into()),
+            Payload::AfgReceivedCommit(m) => internal::Payload::AfgReceivedCommit(m.into()),
+            Payload::AfgAuthoritySet(m) => internal::Payload::AfgAuthoritySet(m.into()),
+            Payload::AfgFinalizedBlocksUpTo => internal::Payload::AfgFinalizedBlocksUpTo,
+            Payload::AuraPreSealedBlock => internal::Payload::AuraPreSealedBlock,
+            Payload::PreparedBlockForProposing => internal::Payload::PreparedBlockForProposing,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct SystemConnected {
+    pub genesis_hash: Hash,
+    #[serde(flatten)]
+    pub node: NodeDetails,
+}
+
+impl From<SystemConnected> for internal::SystemConnected {
+    fn from(msg: SystemConnected) -> Self {
+        internal::SystemConnected {
+            genesis_hash: msg.genesis_hash.into(),
+            node: msg.node.into(),
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct SystemInterval {
+    pub peers: Option<u64>,
+    pub txcount: Option<u64>,
+    pub bandwidth_upload: Option<f64>,
+    pub bandwidth_download: Option<f64>,
+    pub finalized_height: Option<BlockNumber>,
+    pub finalized_hash: Option<Hash>,
+    #[serde(flatten)]
+    pub block: Option<Block>,
+    pub used_state_cache_size: Option<f32>,
+}
+
+impl From<SystemInterval> for internal::SystemInterval {
+    fn from(msg: SystemInterval) -> Self {
+        internal::SystemInterval {
+            peers: msg.peers,
+            txcount: msg.txcount,
+            bandwidth_upload: msg.bandwidth_upload,
+            bandwidth_download: msg.bandwidth_download,
+            finalized_height: msg.finalized_height,
+            finalized_hash: msg.finalized_hash.map(|h| h.into()),
+            block: msg.block.map(|b| b.into()),
+            used_state_cache_size: msg.used_state_cache_size,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct Finalized {
+    #[serde(rename = "best")]
+    pub hash: Hash,
+    pub height: Box<str>,
+}
+
+impl From<Finalized> for internal::Finalized {
+    fn from(msg: Finalized) -> Self {
+        internal::Finalized {
+            hash: msg.hash.into(),
+            height: msg.height,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct AfgAuthoritySet {
+    pub authority_id: Box<str>,
+    pub authorities: Box<str>,
+    pub authority_set_id: Box<str>,
+}
+
+impl From<AfgAuthoritySet> for internal::AfgAuthoritySet {
+    fn from(msg: AfgAuthoritySet) -> Self {
+        internal::AfgAuthoritySet {
+            authority_id: msg.authority_id,
+            authorities: msg.authorities,
+            authority_set_id: msg.authority_set_id,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct AfgFinalized {
+    pub finalized_hash: Hash,
+    pub finalized_number: Box<str>,
+}
+
+impl From<AfgFinalized> for internal::AfgFinalized {
+    fn from(msg: AfgFinalized) -> Self {
+        internal::AfgFinalized {
+            finalized_hash: msg.finalized_hash.into(),
+            finalized_number: msg.finalized_number,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct AfgReceived {
+    pub target_hash: Hash,
+    pub target_number: Box<str>,
+    pub voter: Option<Box<str>>,
+}
+
+impl From<AfgReceived> for internal::AfgReceived {
+    fn from(msg: AfgReceived) -> Self {
+        internal::AfgReceived {
+            target_hash: msg.target_hash.into(),
+            target_number: msg.target_number,
+            voter: msg.voter,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone, Copy)]
+pub struct Block {
+    #[serde(rename = "best")]
+    pub hash: Hash,
+    pub height: BlockNumber,
+}
+
+impl From<Block> for node_types::Block {
+    fn from(block: Block) -> Self {
+        node_types::Block {
+            hash: block.hash.into(),
+            height: block.height,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct NodeDetails {
+    pub chain: Box<str>,
+    pub name: Box<str>,
+    pub implementation: Box<str>,
+    pub version: Box<str>,
+    pub validator: Option<Box<str>>,
+    pub network_id: Option<Box<str>>,
+    pub startup_time: Option<Box<str>>,
+}
+
+impl From<NodeDetails> for node_types::NodeDetails {
+    fn from(details: NodeDetails) -> Self {
+        node_types::NodeDetails {
+            chain: details.chain,
+            name: details.name,
+            implementation: details.implementation,
+            version: details.version,
+            validator: details.validator,
+            network_id: details.network_id,
+            startup_time: details.startup_time,
+        }
+    }
+}
+
+type NodeMessageId = u64;
+type BlockNumber = u64;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn message_v1() {
+        let json = r#"{
+            "msg":"notify.finalized",
+            "level":"INFO",
+            "ts":"2021-01-13T12:38:25.410794650+01:00",
+            "best":"0x031c3521ca2f9c673812d692fc330b9a18e18a2781e3f9976992f861fd3ea0cb",
+            "height":"50"
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V1 { .. },
+            ),
+            "message did not match variant V1",
+        );
+    }
+
+    #[test]
+    fn message_v2() {
+        let json = r#"{
+            "id":1,
+            "ts":"2021-01-13T12:22:20.053527101+01:00",
+            "payload":{
+                "best":"0xcc41708573f2acaded9dd75e07dac2d4163d136ca35b3061c558d7a35a09dd8d",
+                "height":"209",
+                "msg":"notify.finalized"
+            }
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V2 { .. },
+            ),
+            "message did not match variant V2",
+        );
+    }
+
+    #[test]
+    fn message_v2_received_precommit() {
+        let json = r#"{
+            "id":1,
+            "ts":"2021-01-13T12:22:20.053527101+01:00",
+            "payload":{
+                "target_hash":"0xcc41708573f2acaded9dd75e07dac2d4163d136ca35b3061c558d7a35a09dd8d",
+                "target_number":"209",
+                "voter":"foo",
+                "msg":"afg.received_precommit"
+            }
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V2 {
+                    payload: Payload::AfgReceivedPrecommit(..),
+                    ..
+                },
+            ),
+            "message did not match the expected output",
+        );
+    }
+
+    #[test]
+    fn message_v2_tx_pool_import() {
+        // We should happily ignore any fields we don't care about.
+        let json = r#"{
+            "id":1,
+            "ts":"2021-01-13T12:22:20.053527101+01:00",
+            "payload":{
+                "foo":"Something",
+                "bar":123,
+                "wibble":"wobble",
+                "msg":"txpool.import"
+            }
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V2 {
+                    payload: Payload::TxPoolImport,
+                    ..
+                },
+            ),
+            "message did not match the expected output",
+        );
+    }
+}

backend/telemetry_shard/src/main.rs

@@ -0,0 +1,197 @@
+#[warn(missing_docs)]
+
+mod aggregator;
+mod connection;
+mod json_message;
+mod real_ip;
+
+use std::net::IpAddr;
+
+use aggregator::{Aggregator, FromWebsocket};
+use common::node_message;
+use futures::{channel::mpsc, SinkExt, StreamExt};
+use http::Uri;
+use real_ip::real_ip;
+use simple_logger::SimpleLogger;
+use structopt::StructOpt;
+use warp::filters::ws;
+use warp::Filter;
+
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
+const NAME: &str = "Substrate Telemetry Backend Shard";
+const ABOUT: &str = "This is the Telemetry Backend Shard that forwards the \
+                     data sent by Substrate/Polkadot nodes to the Backend Core";
+
+#[derive(StructOpt, Debug)]
+#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+struct Opts {
+    /// This is the socket address that this shard is listening to. This is restricted to
+    /// localhost (127.0.0.1) by default and should be fine for most use cases. If
+    /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
+    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8001")]
+    socket: std::net::SocketAddr,
+    /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
+    /// 'error' only logs errors and 'trace' logs everything.
+    #[structopt(required = false, long = "log", default_value = "info")]
+    log_level: log::LevelFilter,
+    /// Url to the Backend Core endpoint accepting shard connections
+    #[structopt(
+        short = "c",
+        long = "core",
+        default_value = "ws://127.0.0.1:8000/shard_submit/"
+    )]
+    core_url: Uri,
+}
+
+#[tokio::main]
+async fn main() {
+    let opts = Opts::from_args();
+
+    SimpleLogger::new()
+        .with_level(opts.log_level)
+        .init()
+        .expect("Must be able to start a logger");
+
+    log::info!("Starting Telemetry Shard version: {}", VERSION);
+
+    if let Err(e) = start_server(opts).await {
+        log::error!("Error starting server: {}", e);
+    }
+}
+
+/// Declare our routes and start the server.
+async fn start_server(opts: Opts) -> anyhow::Result<()> {
+    let aggregator = Aggregator::spawn(opts.core_url).await?;
+
+    // Handle requests to /health by returning OK.
+    let health_route = warp::path("health").map(|| "OK");
+
+    // Handle websocket requests to /submit.
+    let ws_route = warp::path("submit").and(warp::ws()).and(real_ip()).map(
+        move |ws: ws::Ws, addr: Option<IpAddr>| {
+            // Send messages from the websocket connection to this sink
+            // to have them pass to the aggregator.
+            let tx_to_aggregator = aggregator.subscribe_node();
+            log::info!("Opening /submit connection from {:?}", addr);
+            ws.on_upgrade(move |websocket| async move {
+                let (mut tx_to_aggregator, websocket) =
+                    handle_node_websocket_connection(websocket, tx_to_aggregator, addr).await;
+                log::info!("Closing /submit connection from {:?}", addr);
+                // Tell the aggregator that this connection has closed, so it can tidy up.
+                let _ = tx_to_aggregator.send(FromWebsocket::Disconnected).await;
+                // Note: IF we want to close with a status code and reason, we need to construct
+                // a ws::Message using `ws::Message::close_with`, rather than using this method:
+                let _ = websocket.close().await;
+            })
+        },
+    );
+
+    // Merge the routes and start our server:
+    let routes = ws_route.or(health_route);
+    warp::serve(routes).run(opts.socket).await;
+    Ok(())
+}
+
+/// This takes care of handling messages from an established socket connection.
+async fn handle_node_websocket_connection<S>(
+    mut websocket: ws::WebSocket,
+    mut tx_to_aggregator: S,
+    addr: Option<IpAddr>,
+) -> (S, ws::WebSocket)
+where
+    S: futures::Sink<FromWebsocket, Error = anyhow::Error> + Unpin,
+{
+    // This could be a oneshot channel, but it's useful to be able to clone
+    // messages, and we can't clone oneshot channel senders.
+    let (close_connection_tx, mut close_connection_rx) = mpsc::channel(0);
+
+    // Tell the aggregator about this new connection, and give it a way to close this connection:
+    let init_msg = FromWebsocket::Initialize {
+        close_connection: close_connection_tx,
+    };
+    if let Err(e) = tx_to_aggregator.send(init_msg).await {
+        log::error!("Error sending message to aggregator: {}", e);
+        return (tx_to_aggregator, websocket);
+    }
+
+    // Now we've "initialized", wait for messages from the node. Messages will
+    // either be `SystemConnected` type messages that inform us that a new set
+    // of messages with some message ID will be sent (a node could have more
+    // than one of these), or updates linked to a specific message_id.
+    loop {
+        tokio::select! {
+            // The close channel has fired, so end the loop:
+            _ = close_connection_rx.next() => {
+                log::info!("connection to {:?} being closed by aggregator", addr);
+                break
+            },
+            // A message was received; handle it:
+            msg = websocket.next() => {
+                let msg = match msg {
+                    Some(msg) => msg,
+                    None => { log::warn!("Websocket connection from {:?} closed", addr); break }
+                };
+
+                // If we see any errors, log them and end our loop:
+                let msg = match msg {
+                    Err(e) => { log::error!("Error in node websocket connection: {}", e); break },
+                    Ok(msg) => msg,
+                };
+
+                // Close message? Break to close connection.
+                if msg.is_close() {
+                    break;
+                }
+
+                // If the message isn't something we want to handle, just ignore it.
+                // This includes system messages like "pings" and such, so don't log anything.
+                if !msg.is_binary() && !msg.is_text() {
+                    continue;
+                }
+                // Deserialize from JSON, warning in debug mode if deserialization fails:
+                let bytes = msg.as_bytes();
+                let node_message: json_message::NodeMessage = match serde_json::from_slice(bytes) {
+                    Ok(node_message) => node_message,
+                    #[cfg(debug)]
+                    Err(e) => {
+                        let bytes: &[u8] = bytes.get(..512).unwrap_or_else(|| &bytes);
+                        let msg_start = std::str::from_utf8(bytes).unwrap_or_else(|_| "INVALID UTF8");
+                        log::warn!("Failed to parse node message ({}): {}", msg_start, e);
+                        continue;
+                    },
+                    #[cfg(not(debug))]
+                    Err(_) => {
+                        continue;
+                    }
+                };
+
+                // Pull relevant details from the message:
+                let node_message: node_message::NodeMessage = node_message.into();
+                let message_id = node_message.id();
+                let payload = node_message.into_payload();
+
+                // Until the aggregator receives an `Add` message, which we can create once
+                // we see one of these SystemConnected ones, it will ignore messages with
+                // the corresponding message_id.
+                if let node_message::Payload::SystemConnected(info) = payload {
+                    let _ = tx_to_aggregator.send(FromWebsocket::Add {
+                        message_id,
+                        ip: addr,
+                        node: info.node,
+                        genesis_hash: info.genesis_hash,
+                    }).await;
+                }
+                // Anything that's not an "Add" is an Update. The aggregator will ignore
+                // updates against a message_id that hasn't first been Added, above.
+                else if let Err(e) = tx_to_aggregator.send(FromWebsocket::Update { message_id, payload } ).await {
+                    log::error!("Failed to send node message to aggregator: {}", e);
+                    continue;
+                }
+            }
+        }
+    }
+
+    // Return what we need to close the connection gracefully:
+    (tx_to_aggregator, websocket)
+}

backend/telemetry_shard/src/real_ip.rs

@@ -0,0 +1,136 @@
+use std::net::{IpAddr, SocketAddr};
+use warp::filters::addr;
+use warp::filters::header;
+use warp::Filter;
+
+/**
+A warp filter to extract the "real" IP address of the connection by looking at headers
+set by proxies (this is inspired by Actix Web's implementation of the feature).
+
+First, check for the standardised "Forwarded" header. This looks something like:
+
+"Forwarded: for=12.34.56.78;host=example.com;proto=https, for=23.45.67.89"
+
+Each proxy can append to this comma separated list of forwarded-details. We'll look for
+the first "for" address and try to decode that.
+
+If this doesn't yield a result, look for the non-standard but common X-Forwarded-For header,
+which contains a comma separated list of addresses; each proxy in the potential chain possibly
+appending one to the end. So, take the first of these if it exists.
+
+If still no luck, look for the X-Real-IP header, which we expect to contain a single IP address.
+
+If that _still_ doesn't work, fall back to the socket address of the connection.
+
+Return `None` if all of this fails to yield an address.
+*/
+pub fn real_ip() -> impl warp::Filter<Extract = (Option<IpAddr>,), Error = warp::Rejection> + Clone
+{
+    header::optional("forwarded")
+        .and(header::optional("x-forwarded-for"))
+        .and(header::optional("x-real-ip"))
+        .and(addr::remote())
+        .map(pick_best_ip_from_options)
+}
+
+fn pick_best_ip_from_options(
+    // Forwarded header value (if present)
+    forwarded: Option<String>,
+    // X-Forwarded-For header value (if present)
+    forwarded_for: Option<String>,
+    // X-Real-IP header value (if present)
+    real_ip: Option<String>,
+    // socket address (if known)
+    addr: Option<SocketAddr>,
+) -> Option<IpAddr> {
+    let realip = forwarded
+        .as_ref()
+        .and_then(|val| get_first_addr_from_forwarded_header(val))
+        .or_else(|| {
+            // fall back to X-Forwarded-For
+            forwarded_for
+                .as_ref()
+                .and_then(|val| get_first_addr_from_x_forwarded_for_header(val))
+        })
+        .or_else(|| {
+            // fall back to X-Real-IP
+            real_ip.as_ref().map(|val| val.trim())
+        })
+        .and_then(|ip| {
+            // Try parsing assuming it may have a port first,
+            // and then assuming it doesn't.
+            ip.parse::<SocketAddr>()
+                .map(|s| s.ip())
+                .or_else(|_| ip.parse::<IpAddr>())
+                .ok()
+        })
+        // Fall back to local IP address if the above fails
+        .or(addr.map(|a| a.ip()));
+
+    realip
+}
+
+/// Follow https://datatracker.ietf.org/doc/html/rfc7239 to decode the Forwarded header value.
+/// Roughly, proxies can add new sets of values by appending a comma to the existing list
+/// (so we have something like "values1, values2, values3" from proxy1, proxy2 and proxy3 for
+/// instance) and then the valeus themselves are ';' separated name=value pairs. The value in each
+/// pair may or may not be surrounded in double quotes.
+///
+/// Examples from the RFC:
+///
+///   Forwarded: for="_gazonk"
+///   Forwarded: For="[2001:db8:cafe::17]:4711"
+///   Forwarded: for=192.0.2.60;proto=http;by=203.0.113.43
+///   Forwarded: for=192.0.2.43, for=198.51.100.17
+fn get_first_addr_from_forwarded_header(value: &str) -> Option<&str> {
+    let first_values = value.split(',').next()?;
+
+    for pair in first_values.split(';') {
+        let mut parts = pair.trim().splitn(2, '=');
+        let key = parts.next()?;
+        let value = parts.next()?;
+
+        if key.to_lowercase() == "for" {
+            // trim double quotes if they surround the value:
+            let value = if value.starts_with('"') && value.ends_with('"') {
+                &value[1..value.len() - 1]
+            } else {
+                value
+            };
+            return Some(value);
+        }
+    }
+
+    None
+}
+
+fn get_first_addr_from_x_forwarded_for_header(value: &str) -> Option<&str> {
+    value.split(",").map(|val| val.trim()).next()
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn get_addr_from_forwarded_rfc_examples() {
+        let examples = vec![
+            (r#"for="_gazonk""#, "_gazonk"),
+            (
+                r#"For="[2001:db8:cafe::17]:4711""#,
+                "[2001:db8:cafe::17]:4711",
+            ),
+            (r#"for=192.0.2.60;proto=http;by=203.0.113.43"#, "192.0.2.60"),
+            (r#"for=192.0.2.43, for=198.51.100.17"#, "192.0.2.43"),
+        ];
+
+        for (value, expected) in examples {
+            assert_eq!(
+                get_first_addr_from_forwarded_header(value),
+                Some(expected),
+                "Header value: {}",
+                value
+            );
+        }
+    }
+}

backend/test_utils/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "test_utils"
+version = "0.1.0"
+authors = ["James Wilson <james@jsdw.me>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.41"
+futures = "0.3.15"
+http = "0.2.4"
+log = "0.4.14"
+serde_json = "1.0.64"
+soketto = "0.6.0"
+thiserror = "1.0.25"
+tokio = { version = "1.7.1", features = ["full"] }
+tokio-util = { version = "0.6.7", features = ["full"] }
+common = { path = "../common" }

backend/test_utils/src/contains_matches.rs

@@ -0,0 +1,94 @@
+/**
+This macro checks to see whether an iterable container contains each of the
+match items given, in the order that they are given in (but not necessarily
+contiguous, ie other items may be interspersed between the ones we're looking
+to match).
+
+Similar to `matches!`.
+
+```
+enum Item {
+    Foo { a: usize },
+    Bar(bool),
+    Wibble
+}
+
+use Item::*;
+
+let does_contain: bool = test_utils::contains_matches!(
+    vec![Foo { a: 2 }, Wibble, Bar(true), Foo { a: 100 }],
+    Foo { a: 2 } | Foo { a: 3 },
+    Bar(true),
+    Foo {..}
+);
+
+assert!(does_contain);
+```
+*/
+#[macro_export]
+macro_rules! contains_matches {
+    ($expression:expr, $( $( $pattern:pat )|+ $( if $guard:expr )? ),+ $(,)?) => {{
+        let mut items = $expression.into_iter();
+
+        // For each pattern we want to match, we consume items until
+        // we find the first match, and then break the loop and do the
+        // same again with the next pattern. If we run out of items, we
+        // set the validity to false and stop trying to match. Else, we
+        // match againse each of the patterns and return true.
+        let mut is_valid = true;
+        $(
+            while is_valid {
+                let item = match items.next() {
+                    Some(item) => item,
+                    None => {
+                        is_valid = false;
+                        break;
+                    }
+                };
+
+                match item {
+                    $( $pattern )|+ $( if $guard )? => break,
+                    _ => continue
+                }
+            }
+        )+
+
+        is_valid
+    }}
+}
+
+/**
+This macro checks to see whether an iterable container contains each of the
+match items given, in the order that they are given in (but not necessarily
+contiguous, ie other items may be interspersed between the ones we're looking
+to match).
+
+Panics if this is not the case.
+```
+enum Item {
+    Foo { a: usize },
+    Bar(bool),
+    Wibble
+}
+
+use Item::*;
+
+test_utils::assert_contains_matches!(
+    vec![Foo { a: 2 }, Wibble, Bar(true), Foo { a: 100 }],
+    Foo { a: 2 },
+    Bar(true),
+    Foo {..}
+);
+```
+*/
+#[macro_export]
+macro_rules! assert_contains_matches {
+    ($expression:expr, $( $( $pattern:pat )|+ $( if $guard:expr )? ),+ $(,)?) => {
+        let does_contain_matches = $crate::contains_matches!(
+            $expression,
+            $( $( $pattern )|+ $( if $guard )? ),+
+        );
+
+        assert!(does_contain_matches);
+    }
+}

backend/test_utils/src/feed_message_de.rs

@@ -0,0 +1,370 @@
+use anyhow::Context;
+use common::node_types::{
+    BlockDetails, BlockHash, BlockNumber, NodeLocation, NodeStats, Timestamp,
+};
+use serde_json::value::RawValue;
+
+#[derive(Debug, PartialEq)]
+pub enum FeedMessage {
+    Version(usize),
+    BestBlock {
+        block_number: BlockNumber,
+        timestamp: Timestamp,
+        avg_block_time: Option<u64>,
+    },
+    BestFinalized {
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    AddedNode {
+        node_id: usize,
+        node: NodeDetails,
+        stats: NodeStats,
+        // io: NodeIO, // can't losslessly deserialize
+        // hardware: NodeHardware, // can't losslessly deserialize
+        block_details: BlockDetails,
+        location: Option<NodeLocation>,
+        startup_time: Option<Timestamp>,
+    },
+    RemovedNode {
+        node_id: usize,
+    },
+    LocatedNode {
+        node_id: usize,
+        lat: f32,
+        long: f32,
+        city: String,
+    },
+    ImportedBlock {
+        node_id: usize,
+        block_details: BlockDetails,
+    },
+    FinalizedBlock {
+        node_id: usize,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    NodeStatsUpdate {
+        node_id: usize,
+        stats: NodeStats,
+    },
+    Hardware {
+        node_id: usize,
+        // hardware: NodeHardware, // Can't losslessly deserialize
+    },
+    TimeSync {
+        time: Timestamp,
+    },
+    AddedChain {
+        name: String,
+        node_count: usize,
+    },
+    RemovedChain {
+        name: String,
+    },
+    SubscribedTo {
+        name: String,
+    },
+    UnsubscribedFrom {
+        name: String,
+    },
+    Pong {
+        msg: String,
+    },
+    AfgFinalized {
+        address: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    AfgReceivedPrevote {
+        address: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+        voter: Option<String>,
+    },
+    AfgReceivedPrecommit {
+        address: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+        voter: Option<String>,
+    },
+    AfgAuthoritySet {
+        // Not used currently; not sure what "address" params are:
+        a1: String,
+        a2: String,
+        a3: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    StaleNode {
+        node_id: usize,
+    },
+    NodeIOUpdate {
+        node_id: usize,
+        // details: NodeIO, // can't losslessly deserialize
+    },
+    /// A "special" case when we don't know how to decode an action:
+    UnknownValue {
+        action: u8,
+        value: String,
+    },
+}
+
+#[derive(Debug, PartialEq)]
+pub struct NodeDetails {
+    pub name: String,
+    pub implementation: String,
+    pub version: String,
+    pub validator: Option<String>,
+    pub network_id: Option<String>,
+}
+
+impl FeedMessage {
+    /// Decode a slice of bytes into a vector of feed messages
+    pub fn from_bytes(bytes: &[u8]) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        let v: Vec<&RawValue> = serde_json::from_slice(bytes)?;
+
+        let mut feed_messages = vec![];
+        for raw_keyval in v.chunks(2) {
+            let raw_key = raw_keyval[0];
+            let raw_val = raw_keyval[1];
+            let action: u8 = serde_json::from_str(raw_key.get())?;
+            let msg = FeedMessage::decode(action, raw_val)
+                .with_context(|| format!("Failed to decode message with action {}", action))?;
+
+            feed_messages.push(msg);
+        }
+
+        Ok(feed_messages)
+    }
+
+    // Deserialize the feed message to a value based on the "action" key
+    fn decode(action: u8, raw_val: &RawValue) -> Result<FeedMessage, anyhow::Error> {
+        let feed_message = match action {
+            // Version:
+            0 => {
+                let version = serde_json::from_str(raw_val.get())?;
+                FeedMessage::Version(version)
+            }
+            // BestBlock
+            1 => {
+                let (block_number, timestamp, avg_block_time) =
+                    serde_json::from_str(raw_val.get())?;
+                FeedMessage::BestBlock {
+                    block_number,
+                    timestamp,
+                    avg_block_time,
+                }
+            }
+            // BestFinalized
+            2 => {
+                let (block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::BestFinalized {
+                    block_number,
+                    block_hash,
+                }
+            }
+            // AddNode
+            3 => {
+                let (
+                    node_id,
+                    (name, implementation, version, validator, network_id),
+                    stats,
+                    io,
+                    hardware,
+                    block_details,
+                    location,
+                    startup_time,
+                ) = serde_json::from_str(raw_val.get())?;
+
+                // Give these two types but don't use the results:
+                let (_, _): (&RawValue, &RawValue) = (io, hardware);
+
+                FeedMessage::AddedNode {
+                    node_id,
+                    node: NodeDetails {
+                        name,
+                        implementation,
+                        version,
+                        validator,
+                        network_id,
+                    },
+                    stats,
+                    block_details,
+                    location,
+                    startup_time,
+                }
+            }
+            // RemoveNode
+            4 => {
+                let node_id = serde_json::from_str(raw_val.get())?;
+                FeedMessage::RemovedNode { node_id }
+            }
+            // LocatedNode
+            5 => {
+                let (node_id, lat, long, city) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::LocatedNode {
+                    node_id,
+                    lat,
+                    long,
+                    city,
+                }
+            }
+            // ImportedBlock
+            6 => {
+                let (node_id, block_details) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::ImportedBlock {
+                    node_id,
+                    block_details,
+                }
+            }
+            // FinalizedBlock
+            7 => {
+                let (node_id, block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::FinalizedBlock {
+                    node_id,
+                    block_number,
+                    block_hash,
+                }
+            }
+            // NodeStatsUpdate
+            8 => {
+                let (node_id, stats) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::NodeStatsUpdate { node_id, stats }
+            }
+            // Hardware
+            9 => {
+                let (node_id, _hardware): (_, &RawValue) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::Hardware { node_id }
+            }
+            // TimeSync
+            10 => {
+                let time = serde_json::from_str(raw_val.get())?;
+                FeedMessage::TimeSync { time }
+            }
+            // AddedChain
+            11 => {
+                let (name, node_count) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::AddedChain { name, node_count }
+            }
+            // RemovedChain
+            12 => {
+                let name = serde_json::from_str(raw_val.get())?;
+                FeedMessage::RemovedChain { name }
+            }
+            // SubscribedTo
+            13 => {
+                let name = serde_json::from_str(raw_val.get())?;
+                FeedMessage::SubscribedTo { name }
+            }
+            // UnsubscribedFrom
+            14 => {
+                let name = serde_json::from_str(raw_val.get())?;
+                FeedMessage::UnsubscribedFrom { name }
+            }
+            // Pong
+            15 => {
+                let msg = serde_json::from_str(raw_val.get())?;
+                FeedMessage::Pong { msg }
+            }
+            // AfgFinalized
+            16 => {
+                let (address, block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgFinalized {
+                    address,
+                    block_number,
+                    block_hash,
+                }
+            }
+            // AfgReceivedPrevote
+            17 => {
+                let (address, block_number, block_hash, voter) =
+                    serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgReceivedPrevote {
+                    address,
+                    block_number,
+                    block_hash,
+                    voter,
+                }
+            }
+            // AfgReceivedPrecommit
+            18 => {
+                let (address, block_number, block_hash, voter) =
+                    serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgReceivedPrecommit {
+                    address,
+                    block_number,
+                    block_hash,
+                    voter,
+                }
+            }
+            // AfgAuthoritySet
+            19 => {
+                let (a1, a2, a3, block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgAuthoritySet {
+                    a1,
+                    a2,
+                    a3,
+                    block_number,
+                    block_hash,
+                }
+            }
+            // StaleNode
+            20 => {
+                let node_id = serde_json::from_str(raw_val.get())?;
+                FeedMessage::StaleNode { node_id }
+            }
+            // NodeIOUpdate
+            21 => {
+                // ignore NodeIO for now:
+                let (node_id, _node_io): (_, &RawValue) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::NodeIOUpdate { node_id }
+            }
+            // A catchall for messages we don't know/care about yet:
+            _ => {
+                let value = raw_val.to_string();
+                FeedMessage::UnknownValue { action, value }
+            }
+        };
+
+        Ok(feed_message)
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+
+    #[test]
+    fn decode_remove_node_msg() {
+        // "remove chain ''":
+        let msg = r#"[12,""]"#;
+
+        assert_eq!(
+            FeedMessage::from_bytes(msg.as_bytes()).unwrap(),
+            vec![FeedMessage::RemovedChain {
+                name: "".to_owned()
+            }]
+        );
+    }
+
+    #[test]
+    fn decode_remove_then_add_node_msg() {
+        // "remove chain '', then add chain 'Local Testnet' with 1 node":
+        let msg = r#"[12,"",11,["Local Testnet",1]]"#;
+
+        assert_eq!(
+            FeedMessage::from_bytes(msg.as_bytes()).unwrap(),
+            vec![
+                FeedMessage::RemovedChain {
+                    name: "".to_owned()
+                },
+                FeedMessage::AddedChain {
+                    name: "Local Testnet".to_owned(),
+                    node_count: 1
+                },
+            ]
+        );
+    }
+}

backend/test_utils/src/lib.rs

@@ -0,0 +1,14 @@
+/// Create/connect to a server consisting of shards and a core process that we can interact with.
+pub mod server;
+
+/// Test support for deserializing feed messages from the feed processes. This basically
+/// is the slightly-lossy inverse of the custom serialization we do to feed messages.
+pub mod feed_message_de;
+
+/// A couple of macros to make it easier to test for the presense of things (mainly, feed messages)
+/// in an iterable container.
+#[macro_use]
+pub mod contains_matches;
+
+/// Utilities to help with running tests from within this current workspace.
+pub mod workspace;

backend/test_utils/src/server/channels.rs

@@ -0,0 +1,253 @@
+use std::{ops::{Deref, DerefMut}, time::Duration};
+
+use crate::feed_message_de::FeedMessage;
+use common::ws_client;
+use futures::{Sink, SinkExt, Stream, StreamExt};
+
+/// Wrap a `ws_client::Sender` with convenient utility methods for shard connections
+pub struct ShardSender(ws_client::Sender);
+
+impl From<ws_client::Sender> for ShardSender {
+    fn from(c: ws_client::Sender) -> Self {
+        ShardSender(c)
+    }
+}
+
+impl Sink<ws_client::SentMessage> for ShardSender {
+    type Error = ws_client::SendError;
+    fn poll_ready(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_ready_unpin(cx)
+    }
+    fn start_send(
+        mut self: std::pin::Pin<&mut Self>,
+        item: ws_client::SentMessage,
+    ) -> Result<(), Self::Error> {
+        self.0.start_send_unpin(item)
+    }
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_flush_unpin(cx)
+    }
+    fn poll_close(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_close_unpin(cx)
+    }
+}
+
+impl ShardSender {
+    /// Send JSON as a binary websocket message
+    pub fn send_json_binary(
+        &mut self,
+        json: serde_json::Value,
+    ) -> Result<(), ws_client::SendError> {
+        let bytes = serde_json::to_vec(&json).expect("valid bytes");
+        self.unbounded_send(ws_client::SentMessage::Binary(bytes))
+    }
+    /// Send JSON as a textual websocket message
+    pub fn send_json_text(
+        &mut self,
+        json: serde_json::Value,
+    ) -> Result<(), ws_client::SendError> {
+        let s = serde_json::to_string(&json).expect("valid string");
+        self.unbounded_send(ws_client::SentMessage::Text(s))
+    }
+}
+
+impl Deref for ShardSender {
+    type Target = ws_client::Sender;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for ShardSender {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+/// Wrap a `ws_client::Receiver` with convenient utility methods for shard connections
+pub struct ShardReceiver(ws_client::Receiver);
+
+impl From<ws_client::Receiver> for ShardReceiver {
+    fn from(c: ws_client::Receiver) -> Self {
+        ShardReceiver(c)
+    }
+}
+
+impl Stream for ShardReceiver {
+    type Item = Result<ws_client::RecvMessage, ws_client::RecvError>;
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        self.0.poll_next_unpin(cx)
+    }
+}
+
+impl Deref for ShardReceiver {
+    type Target = ws_client::Receiver;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for ShardReceiver {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+/// Wrap a `ws_client::Sender` with convenient utility methods for feed connections
+pub struct FeedSender(ws_client::Sender);
+
+impl From<ws_client::Sender> for FeedSender {
+    fn from(c: ws_client::Sender) -> Self {
+        FeedSender(c)
+    }
+}
+
+impl Sink<ws_client::SentMessage> for FeedSender {
+    type Error = ws_client::SendError;
+    fn poll_ready(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_ready_unpin(cx)
+    }
+    fn start_send(
+        mut self: std::pin::Pin<&mut Self>,
+        item: ws_client::SentMessage,
+    ) -> Result<(), Self::Error> {
+        self.0.start_send_unpin(item)
+    }
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_flush_unpin(cx)
+    }
+    fn poll_close(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_close_unpin(cx)
+    }
+}
+
+impl Deref for FeedSender {
+    type Target = ws_client::Sender;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for FeedSender {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl FeedSender {
+    /// Send a command into the feed. A command consists of a string
+    /// "command" part, and another string "parameter" part.
+    pub fn send_command<S: AsRef<str>>(
+        &mut self,
+        command: S,
+        param: S,
+    ) -> Result<(), ws_client::SendError> {
+        self.unbounded_send(ws_client::SentMessage::Text(format!(
+            "{}:{}",
+            command.as_ref(),
+            param.as_ref()
+        )))
+    }
+}
+
+
+/// Wrap a `ws_client::Receiver` with convenient utility methods for feed connections
+pub struct FeedReceiver(ws_client::Receiver);
+
+impl From<ws_client::Receiver> for FeedReceiver {
+    fn from(c: ws_client::Receiver) -> Self {
+        FeedReceiver(c)
+    }
+}
+
+impl Stream for FeedReceiver {
+    type Item = Result<ws_client::RecvMessage, ws_client::RecvError>;
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        self.0.poll_next_unpin(cx).map_err(|e| e.into())
+    }
+}
+
+impl Deref for FeedReceiver {
+    type Target = ws_client::Receiver;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for FeedReceiver {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl FeedReceiver {
+    /// Wait for the next set of feed messages to arrive. Returns an error if the connection
+    /// is closed, or the messages that come back cannot be properly decoded.
+    ///
+    /// Prefer [`FeedReceiver::recv_feed_messages`]; tests should generally be
+    /// robust in assuming that messages may not all be delivered at once (unless we are
+    /// specifically testing which messages are buffered together).
+    pub async fn recv_feed_messages_once(&mut self) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        let msg = self
+            .0
+            .next()
+            .await
+            .ok_or_else(|| anyhow::anyhow!("Stream closed: no more messages"))??;
+
+        match msg {
+            ws_client::RecvMessage::Binary(data) => {
+                let messages = FeedMessage::from_bytes(&data)?;
+                Ok(messages)
+            },
+            ws_client::RecvMessage::Text(text) => {
+                let messages = FeedMessage::from_bytes(text.as_bytes())?;
+                Ok(messages)
+            },
+        }
+    }
+
+    /// Wait for feed messages to be sent back, building up a list of output messages until
+    /// the channel goes quiet for a short while.
+    pub async fn recv_feed_messages(&mut self) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        // Block as long as needed for messages to start coming in:
+        let mut feed_messages = self.recv_feed_messages_once().await?;
+        // Then, loop a little to make sure we catch any additional messages that are sent soon after:
+        loop {
+            match tokio::time::timeout(Duration::from_millis(250), self.recv_feed_messages_once())
+                .await
+            {
+                // Timeout elapsed; return the messages we have so far
+                Err(_) => {
+                    break Ok(feed_messages);
+                }
+                // Append messages that come back to our vec
+                Ok(Ok(mut msgs)) => {
+                    feed_messages.append(&mut msgs);
+                }
+                // Error came back receiving messages; return it
+                Ok(Err(e)) => break Err(e),
+            }
+        }
+    }
+}

backend/test_utils/src/server/mod.rs

@@ -0,0 +1,5 @@
+mod server;
+mod utils;
+
+pub mod channels;
+pub use server::*;

backend/test_utils/src/server/server.rs

@@ -0,0 +1,421 @@
+use super::{channels, utils};
+use common::ws_client;
+use common::{id_type, DenseMap};
+use std::ffi::OsString;
+use std::marker::PhantomData;
+use tokio::process::{self, Command as TokioCommand};
+
+id_type! {
+    /// The ID of a running process. Cannot be constructed externally.
+    pub struct ProcessId(usize);
+}
+
+pub enum StartOpts {
+    /// Start a single core process that is expected
+    /// to have both `/feed` and `/submit` endpoints
+    SingleProcess {
+        /// Command to run to start the process.
+        /// The `--listen` and `--log` arguments will be appended within and shouldn't be provided.
+        command: Command,
+    },
+    /// Start a core process with a `/feed` andpoint as well as (optionally)
+    /// multiple shard processes with `/submit` endpoints.
+    ShardAndCore {
+        /// Command to run to start a shard.
+        /// The `--listen` and `--log` arguments will be appended within and shouldn't be provided.
+        shard_command: Command,
+        /// Command to run to start a telemetry core process.
+        /// The `--listen` and `--log` arguments will be appended within and shouldn't be provided.
+        core_command: Command,
+    },
+    /// Connect to existing process(es).
+    ConnectToExisting {
+        /// Where are the processes that we can `/submit` things to?
+        /// Eg: `vec![127.0.0.1:12345, 127.0.0.1:9091]`
+        submit_hosts: Vec<String>,
+        /// Where is the process that we can subscribe to the `/feed` of?
+        /// Eg: `127.0.0.1:3000`
+        feed_host: String,
+    }
+}
+
+/// This represents a telemetry server. It can be in different modes
+/// depending on how it was started, but the interface is similar in every case
+/// so that tests are somewhat compatible with multiple configurations.
+pub enum Server {
+    SingleProcessMode {
+        /// A virtual shard that we can hand out.
+        virtual_shard: ShardProcess,
+        /// Core process that we can connect to.
+        core: CoreProcess
+    },
+    ShardAndCoreMode {
+        /// Command to run to start a new shard.
+        shard_command: Command,
+        /// Shard processes that we can connect to.
+        shards: DenseMap<ProcessId, ShardProcess>,
+        /// Core process that we can connect to.
+        core: CoreProcess,
+    },
+    ConnectToExistingMode {
+        /// The hosts that we can connect to to submit things.
+        submit_hosts: Vec<String>,
+        /// Which host do we use next (we'll cycle around them
+        /// as shards are "added").
+        next_submit_host_idx: usize,
+        /// Shard processes that we can connect to.
+        shards: DenseMap<ProcessId, ShardProcess>,
+        /// Core process that we can connect to.
+        core: CoreProcess,
+    }
+}
+
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Can't establsih connection: {0}")]
+    ConnectionError(#[from] ws_client::ConnectError),
+    #[error("Can't establsih connection: {0}")]
+    JoinError(#[from] tokio::task::JoinError),
+    #[error("Can't establsih connection: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("Could not obtain port for process as the line we waited for in log output didn't show up: {0}")]
+    ErrorObtainingPort(anyhow::Error),
+    #[error("Whoops; attempt to kill a process we didn't start (and so have no handle to)")]
+    CannotKillNoHandle,
+    #[error(
+        "Can't add a shard: command not provided, or we are not in charge of spawning processes"
+    )]
+    CannotAddShard,
+    #[error("The URI provided was invalid: {0}")]
+    InvalidUri(#[from] http::uri::InvalidUri)
+}
+
+impl Server {
+    pub fn get_core(&self) -> &CoreProcess {
+        match self {
+            Server::SingleProcessMode { core, .. } => core,
+            Server::ShardAndCoreMode { core, ..} => core,
+            Server::ConnectToExistingMode { core, .. } => core
+        }
+    }
+
+    pub fn get_shard(&self, id: ProcessId) -> Option<&ShardProcess> {
+        match self {
+            Server::SingleProcessMode { virtual_shard, .. } => Some(virtual_shard),
+            Server::ShardAndCoreMode { shards, ..} => shards.get(id),
+            Server::ConnectToExistingMode { shards, .. } => shards.get(id)
+        }
+    }
+
+    pub async fn kill_shard(&mut self, id: ProcessId) -> bool {
+        let shard = match self {
+            // Can't remove the pretend shard:
+            Server::SingleProcessMode { .. } => return false,
+            Server::ShardAndCoreMode { shards, ..} => shards.remove(id),
+            Server::ConnectToExistingMode { shards, .. } => shards.remove(id)
+        };
+
+        let shard = match shard {
+            Some(shard) => shard,
+            None => return false,
+        };
+
+        // With this, killing will complete even if the promise returned is cancelled
+        // (it should regardless, but just to play it safe..)
+        let _ = tokio::spawn(async move {
+            let _ = shard.kill().await;
+        })
+        .await;
+
+        true
+    }
+
+    /// Kill everything and tidy up
+    pub async fn shutdown(self) {
+        // Spawn so we don't need to await cleanup if we don't care.
+        // Run all kill futs simultaneously.
+        let handle = tokio::spawn(async move {
+            let (core, shards) = match self {
+                Server::SingleProcessMode { core, .. }
+                    => (core, DenseMap::new()),
+                Server::ShardAndCoreMode { core, shards, ..}
+                    => (core, shards),
+                Server::ConnectToExistingMode { core, shards, .. }
+                    => (core, shards)
+            };
+
+            let shard_kill_futs = shards.into_iter().map(|(_, s)| s.kill());
+            let _ = tokio::join!(futures::future::join_all(shard_kill_futs), core.kill());
+        });
+
+        // You can wait for cleanup but aren't obliged to:
+        let _ = handle.await;
+    }
+
+    /// Connect a new shard and return a process that you can interact with:
+    pub async fn add_shard(&mut self) -> Result<ProcessId, Error> {
+        match self {
+            // Always get back the same "virtual" shard; we're always just talking to the core anyway.
+            Server::SingleProcessMode { virtual_shard, .. } => {
+                Ok(virtual_shard.id)
+            },
+            // We're connecting to an existing process. Find the next host we've been told about
+            // round-robin style and use that as our new virtual shard.
+            Server::ConnectToExistingMode { submit_hosts, next_submit_host_idx, shards, .. } => {
+                let host = match submit_hosts.get(*next_submit_host_idx % submit_hosts.len()) {
+                    Some(host) => host,
+                    None => return Err(Error::CannotAddShard)
+                };
+                *next_submit_host_idx += 1;
+
+                let pid = shards.add_with(|id| Process {
+                    id,
+                    host: format!("{}", host),
+                    handle: None,
+                    _channel_type: PhantomData,
+                });
+
+                Ok(pid)
+            },
+            // Start a new process and return that.
+            Server::ShardAndCoreMode { shard_command, shards, core } => {
+                // Where is the URI we'll want to submit things to?
+                let core_shard_submit_uri = format!("http://{}/shard_submit", core.host);
+
+                let mut shard_cmd: TokioCommand = shard_command.clone().into();
+                shard_cmd
+                    .arg("--listen")
+                    .arg("127.0.0.1:0") // 0 to have a port picked by the kernel
+                    .arg("--log")
+                    .arg("info")
+                    .arg("--core")
+                    .arg(core_shard_submit_uri)
+                    .kill_on_drop(true)
+                    .stdout(std::process::Stdio::piped())
+                    .stdin(std::process::Stdio::piped());
+
+                let mut shard_process = shard_cmd.spawn()?;
+                let mut child_stdout = shard_process.stdout.take().expect("shard stdout");
+                let shard_port = utils::get_port(&mut child_stdout)
+                    .await
+                    .map_err(|e| Error::ErrorObtainingPort(e))?;
+
+                // Attempt to wait until we've received word that the shard is connected to the
+                // core before continuing. If we don't wait for this, the connection may happen
+                // after we've attempted to connect node sockets, and they would be booted and
+                // made to reconnect, which we don't want to deal with in general.
+                let _ = utils::wait_for_line_containing(
+                    &mut child_stdout,
+                    |s| s.contains("Connected to telemetry core"),
+                    std::time::Duration::from_secs(5),
+                )
+                .await;
+
+                // Since we're piping stdout from the child process, we need somewhere for it to go
+                // else the process will get stuck when it tries to produce output:
+                utils::drain(child_stdout, tokio::io::stderr());
+
+                let pid = shards.add_with(|id| Process {
+                    id,
+                    host: format!("127.0.0.1:{}", shard_port),
+                    handle: Some(shard_process),
+                    _channel_type: PhantomData,
+                });
+
+                Ok(pid)
+            },
+        }
+    }
+
+    /// Start a server.
+    pub async fn start(opts: StartOpts) -> Result<Server, Error> {
+        let server = match opts {
+            StartOpts::SingleProcess { command } => {
+                let core_process = Server::start_core(command).await?;
+                let virtual_shard_host = core_process.host.clone();
+                Server::SingleProcessMode {
+                    core: core_process,
+                    virtual_shard: Process {
+                        id: ProcessId(0),
+                        host: virtual_shard_host,
+                        handle: None,
+                        _channel_type: PhantomData
+                    }
+                }
+            },
+            StartOpts::ShardAndCore { core_command, shard_command } => {
+                let core_process = Server::start_core(core_command).await?;
+                Server::ShardAndCoreMode {
+                    core: core_process,
+                    shard_command,
+                    shards: DenseMap::new()
+                }
+            },
+            StartOpts::ConnectToExisting { feed_host, submit_hosts } => {
+                Server::ConnectToExistingMode {
+                    submit_hosts,
+                    next_submit_host_idx: 0,
+                    shards: DenseMap::new(),
+                    core: Process {
+                        id: ProcessId(0),
+                        host: feed_host,
+                        handle: None,
+                        _channel_type: PhantomData,
+                    },
+                }
+            }
+        };
+
+        Ok(server)
+    }
+
+    /// Start up a core process and return it.
+    async fn start_core(command: Command) -> Result<CoreProcess, Error> {
+        let mut tokio_core_cmd: TokioCommand = command.into();
+        let mut child = tokio_core_cmd
+            .arg("--listen")
+            .arg("127.0.0.1:0") // 0 to have a port picked by the kernel
+            .arg("--log")
+            .arg("info")
+            .kill_on_drop(true)
+            .stdout(std::process::Stdio::piped())
+            .stdin(std::process::Stdio::piped())
+            .spawn()?;
+
+        // Find out the port that this is running on
+        let mut child_stdout = child.stdout.take().expect("core stdout");
+        let core_port = utils::get_port(&mut child_stdout)
+            .await
+            .map_err(|e| Error::ErrorObtainingPort(e))?;
+
+        // Since we're piping stdout from the child process, we need somewhere for it to go
+        // else the process will get stuck when it tries to produce output:
+        utils::drain(child_stdout, tokio::io::stderr());
+
+        let core_process = Process {
+            id: ProcessId(0),
+            host: format!("127.0.0.1:{}", core_port),
+            handle: Some(child),
+            _channel_type: PhantomData,
+        };
+
+        Ok(core_process)
+    }
+}
+
+/// This represents a running process that we can connect to, which
+/// may be either a `telemetry_shard` or `telemetry_core`.
+pub struct Process<Channel> {
+    id: ProcessId,
+    /// Host that the process is running on (eg 127.0.0.1:8080).
+    host: String,
+    /// If we started the processes ourselves, we'll have a handle to
+    /// them which we can use to kill them. Else, we may not.
+    handle: Option<process::Child>,
+    /// The kind of the process (lets us add methods specific to shard/core).
+    _channel_type: PhantomData<Channel>,
+}
+
+/// A shard process with shard-specific methods.
+pub type ShardProcess = Process<(channels::ShardSender, channels::ShardReceiver)>;
+
+/// A core process with core-specific methods.
+pub type CoreProcess = Process<(channels::FeedSender, channels::FeedReceiver)>;
+
+impl<Channel> Process<Channel> {
+    /// Get the ID of this process
+    pub fn id(&self) -> ProcessId {
+        self.id
+    }
+
+    /// Kill the process and wait for this to complete
+    /// Not public: Klling done via Server.
+    async fn kill(self) -> Result<(), Error> {
+        match self.handle {
+            Some(mut handle) => Ok(handle.kill().await?),
+            None => Err(Error::CannotKillNoHandle),
+        }
+    }
+}
+
+impl<Send: From<ws_client::Sender>, Recv: From<ws_client::Receiver>> Process<(Send, Recv)> {
+    /// Establish a connection to the process
+    async fn connect_to_uri(&self, uri: &http::Uri) -> Result<(Send, Recv), Error> {
+        ws_client::connect(uri)
+            .await
+            .map(|(s, r)| (s.into(), r.into()))
+            .map_err(|e| e.into())
+    }
+
+    /// Establish multiple connections to the process
+    async fn connect_multiple_to_uri(
+        &self,
+        uri: &http::Uri,
+        num_connections: usize,
+    ) -> Result<Vec<(Send, Recv)>, Error> {
+        utils::connect_multiple_to_uri(uri, num_connections)
+            .await
+            .map(|v| v.into_iter().map(|(s, r)| (s.into(), r.into())).collect())
+            .map_err(|e| e.into())
+    }
+}
+
+impl ShardProcess {
+    /// Establish a connection to the process
+    pub async fn connect_node(&self) -> Result<(channels::ShardSender, channels::ShardReceiver), Error> {
+        let uri = format!("http://{}/submit", self.host).parse()?;
+        self.connect_to_uri(&uri).await
+    }
+
+    /// Establish multiple connections to the process
+    pub async fn connect_multiple_nodes(&self, num_connections: usize) -> Result<Vec<(channels::ShardSender, channels::ShardReceiver)>, Error> {
+        let uri = format!("http://{}/submit", self.host).parse()?;
+        self.connect_multiple_to_uri(&uri, num_connections).await
+    }
+}
+
+impl CoreProcess {
+    /// Establish a connection to the process
+    pub async fn connect_feed(&self) -> Result<(channels::FeedSender, channels::FeedReceiver), Error> {
+        let uri = format!("http://{}/feed", self.host).parse()?;
+        self.connect_to_uri(&uri).await
+    }
+
+    /// Establish multiple connections to the process
+    pub async fn connect_multiple_feeds(&self, num_connections: usize) -> Result<Vec<(channels::FeedSender, channels::FeedReceiver)>, Error> {
+        let uri = format!("http://{}/feed", self.host).parse()?;
+        self.connect_multiple_to_uri(&uri, num_connections).await
+    }
+}
+
+/// This defines a command to run. This exists because [`tokio::process::Command`]
+/// cannot be cloned, but we need to be able to clone our command to spawn multiple
+/// processes with it.
+#[derive(Clone, Debug)]
+pub struct Command {
+    command: OsString,
+    args: Vec<OsString>,
+}
+
+impl Command {
+    pub fn new<S: Into<OsString>>(command: S) -> Command {
+        Command {
+            command: command.into(),
+            args: Vec::new(),
+        }
+    }
+
+    pub fn arg<S: Into<OsString>>(mut self, arg: S) -> Command {
+        self.args.push(arg.into());
+        self
+    }
+}
+
+impl Into<TokioCommand> for Command {
+    fn into(self) -> TokioCommand {
+        let mut cmd = TokioCommand::new(self.command);
+        cmd.args(self.args);
+        cmd
+    }
+}

backend/test_utils/src/server/utils.rs

@@ -0,0 +1,97 @@
+use common::ws_client;
+use anyhow::{anyhow, Context};
+use tokio::io::BufReader;
+use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite};
+use tokio::time::Duration;
+
+/// Reads from the stdout of the shard/core process to extract the port that was assigned to it,
+/// with the side benefit that we'll wait for it to start listening before returning. We do this
+/// because we want to allow the kernel to assign ports and so don't specify a port as an arg.
+pub async fn get_port<R: AsyncRead + Unpin>(reader: R) -> Result<u16, anyhow::Error> {
+    // For the new service:
+    let new_expected_text = "listening on http://127.0.0.1:";
+    // For the older non-sharded actix based service:
+    let old_expected_text = "service on 127.0.0.1:";
+
+    let is_text = |s: &str| s.contains(new_expected_text) || s.contains(old_expected_text);
+    wait_for_line_containing(reader, is_text, Duration::from_secs(240))
+        .await
+        .and_then(|line| {
+            // The line must match one of our expected strings:
+            let (_, port_str) = line
+                .rsplit_once(new_expected_text)
+                .unwrap_or_else(|| line.rsplit_once(old_expected_text).unwrap());
+            // Grab the port after the string:
+            port_str
+                .trim()
+                .parse()
+                .with_context(|| format!("Could not parse output to port: {}", port_str))
+        })
+}
+
+/// Wait for a line of output containing the text given. Also provide a timeout,
+/// such that if we don't see a new line of output within the timeout we bail out
+/// and return an error.
+pub async fn wait_for_line_containing<R: AsyncRead + Unpin, F: Fn(&str) -> bool>(
+    reader: R,
+    is_match: F,
+    max_wait_between_lines: Duration,
+) -> Result<String, anyhow::Error> {
+    let reader = BufReader::new(reader);
+    let mut reader_lines = reader.lines();
+
+    loop {
+        let line = tokio::time::timeout(max_wait_between_lines, reader_lines.next_line()).await;
+
+        let line = match line {
+            // timeout expired; couldn't get port:
+            Err(_) => {
+                return Err(anyhow!("Timeout elapsed waiting for text match"))
+            }
+            // Something went wrong reading line; bail:
+            Ok(Err(e)) => {
+                return Err(anyhow!("Could not read line from stdout: {}", e))
+            },
+            // No more output; process ended? bail:
+            Ok(Ok(None)) => {
+                return Err(anyhow!(
+                    "No more output from stdout; has the process ended?"
+                ))
+            }
+            // All OK, and a line is given back; phew!
+            Ok(Ok(Some(line))) => line,
+        };
+
+        if is_match(&line) {
+            return Ok(line);
+        }
+    }
+}
+
+/// Establish multiple connections to a URI and return them all.
+pub async fn connect_multiple_to_uri(
+    uri: &http::Uri,
+    num_connections: usize,
+) -> Result<Vec<(ws_client::Sender, ws_client::Receiver)>, ws_client::ConnectError> {
+    // Previous versions of this used future::join_all to concurrently establish all of the
+    // connections we want. However, trying to do that with more than say ~130 connections on
+    // MacOS led to hitting "Connection reset by peer" errors, so let's do it one-at-a-time.
+    // (Side note: on Ubuntu the concurrency seemed to be no issue up to at least 10k connections).
+    let mut sockets = vec![];
+    for _ in 0..num_connections {
+        sockets.push(ws_client::connect(uri).await?);
+    }
+    Ok(sockets)
+}
+
+/// Drain output from a reader to stdout. After acquiring port details from spawned processes,
+/// they expect their stdout to be continue to be consumed, and so we do this here.
+pub fn drain<R, W>(mut reader: R, mut writer: W)
+where
+    R: AsyncRead + Unpin + Send + 'static,
+    W: AsyncWrite + Unpin + Send + 'static,
+{
+    tokio::spawn(async move {
+        let _ = tokio::io::copy(&mut reader, &mut writer).await;
+    });
+}

backend/test_utils/src/workspace/commands.rs

@@ -0,0 +1,46 @@
+//! Commands that we can use when running `cargo test` style tests in this workspace
+//! that want to test the current code.
+use crate::server::Command;
+use std::path::PathBuf;
+
+/// Runs `cargo run` in the current workspace to start up a telemetry shard process.
+///
+/// Note: The CWD must be somewhere within this backend workspace for the command to work.
+pub fn cargo_run_telemetry_shard(release_mode: bool) -> Result<Command, std::io::Error> {
+    telemetry_command("telemetry_shard", release_mode)
+}
+
+/// Runs `cargo run` in the current workspace to start up a telemetry core process.
+///
+/// Note: The CWD must be somewhere within this backend workspace for the command to work.
+pub fn cargo_run_telemetry_core(release_mode: bool) -> Result<Command, std::io::Error> {
+    telemetry_command("telemetry_core", release_mode)
+}
+
+fn telemetry_command(bin: &'static str, release_mode: bool) -> Result<Command, std::io::Error> {
+    let mut workspace_dir = try_find_workspace_dir()?;
+    workspace_dir.push("Cargo.toml");
+
+    let mut cmd = Command::new("cargo").arg("run");
+
+    // Release mode?
+    if release_mode {
+        cmd = cmd.arg("--release");
+    }
+
+    cmd = cmd.arg("--bin")
+        .arg(bin)
+        .arg("--manifest-path")
+        .arg(workspace_dir)
+        .arg("--");
+
+    Ok(cmd)
+}
+
+/// A _very_ naive way to find the workspace ("backend") directory
+/// from the current path (which is assumed to be inside it).
+fn try_find_workspace_dir() -> Result<PathBuf, std::io::Error> {
+    let mut dir = std::env::current_dir()?;
+    while !dir.ends_with("backend") && dir.pop() {}
+    Ok(dir)
+}

backend/test_utils/src/workspace/mod.rs

@@ -0,0 +1,4 @@
+mod commands;
+mod start_server;
+
+pub use start_server::{ start_server_debug, start_server_release };

backend/test_utils/src/workspace/start_server.rs

@@ -0,0 +1,62 @@
+use super::commands;
+use crate::server::{self, Server, Command};
+
+/// Start a telemetry server. We'll use `cargo run` by default, but you can also provide
+/// env vars to configure the binary that runs for the shard and core process. Either:
+///
+/// - `TELEMETRY_BIN` - path to the telemetry binary (which can function as shard _and_ core)
+///
+/// Or alternately neither/one/both of:
+///
+/// - `TELEMETRY_SHARD_BIN` - path to telemetry_shard binary
+/// - `TELEMETRY_CORE_BIN` - path to telemetry_core binary
+///
+/// (Whatever is not provided will be substituted with a `cargo run` variant instead)
+///
+/// Or alternately alternately, we can connect to a running instance by providing:
+///
+/// - `TELEMETRY_SUBMIT_HOSTS` - hosts (comma separated) to connect to for telemetry `/submit`s.
+/// - `TELEMETRY_FEED_HOST` - host to connect to for feeds (eg 127.0.0.1:3000)
+///
+pub async fn start_server(release_mode: bool) -> Server {
+    // Start to a single process:
+    if let Ok(bin) = std::env::var("TELEMETRY_BIN") {
+        return Server::start(server::StartOpts::SingleProcess {
+            command: Command::new(bin)
+        }).await.unwrap();
+    }
+
+    // Connect to a running instance:
+    if let Ok(feed_host) = std::env::var("TELEMETRY_FEED_HOST") {
+        let feed_host = feed_host.trim().into();
+        let submit_hosts: Vec<_> = std::env::var("TELEMETRY_SUBMIT_HOSTS")
+            .map(|var| var.split(",").map(|var| var.trim().into()).collect())
+            .unwrap_or(Vec::new());
+        return Server::start(server::StartOpts::ConnectToExisting {
+            feed_host,
+            submit_hosts,
+        }).await.unwrap();
+    }
+
+    // Start a shard and core process:
+    let shard_command = std::env::var("TELEMETRY_SHARD_BIN")
+        .map(|val| Command::new(val))
+        .unwrap_or_else(|_| commands::cargo_run_telemetry_shard(release_mode).expect("must be in rust workspace to run shard command"));
+    let core_command = std::env::var("TELEMETRY_CORE_BIN")
+        .map(|val| Command::new(val))
+        .unwrap_or_else(|_| commands::cargo_run_telemetry_core(release_mode).expect("must be in rust workspace to run core command"));
+    Server::start(server::StartOpts::ShardAndCore {
+        shard_command,
+        core_command
+    }).await.unwrap()
+}
+
+/// Start a telemetry core server in debug mode. see [`start_server`] for details.
+pub async fn start_server_debug() -> Server {
+    start_server(false).await
+}
+
+/// Start a telemetry core server in release mode. see [`start_server`] for details.
+pub async fn start_server_release() -> Server {
+    start_server(true).await
+}

docker-compose.yml

@@ -11,12 +11,29 @@ services:
       - ./packages:/app/packages
     ports:
       - 3000:80
-  telemetry-backend:
+    expose:
+      - 3000
+  telemetry-backend-shard:
     build:
       dockerfile: Dockerfile
       context: ./backend/
-    environment:
-      - PORT=8000
+    command: [
+      'telemetry_shard',
+      '--listen', '0.0.0.0:8001',
+      '--core', 'http://telemetry-backend-core:8000/shard_submit'
+    ]
+    ports:
+      - 8001:8001
+    expose:
+      - 8001
+  telemetry-backend-core:
+    build:
+      dockerfile: Dockerfile
+      context: ./backend/
+    command: [
+      'telemetry_core',
+      '--listen', '0.0.0.0:8000'
+    ]
     ports:
       - 8000:8000
     expose: