Merge pull request #349 from paritytech/jsdw-sharding

Sharded Telemetry Server

.github/workflows/backend.yml

@@ -14,21 +14,30 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Build
+
+    - name: Build telemetry executables (in debug mode)
       working-directory: ./backend
-      run: cargo build --verbose
+      run: cargo build --bins --verbose
+
     - name: Run tests
       working-directory: ./backend
       run: cargo test --verbose
-    - name: Build release and call executable
+
+    - name: Build, release and call telemetry executable
       working-directory: ./backend
-      run: cargo run --release -- --help
+      run: cargo run --bin telemetry_core --release -- --help
+
+    - name: Build, release and call shard executable
+      working-directory: ./backend
+      run: cargo run --bin telemetry_shard --release -- --help
+
     - name: Login to Dockerhub
       uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
-    - name: Build and Push template image
+
+    - name: Build and push template image for tagged commit
       uses: docker/build-push-action@v2 # https://github.com/docker/build-push-action
       with:
         context: './backend'

.gitlab-ci.yml

@@ -0,0 +1,88 @@
+variables:
+  CONTAINER_REGISTRY:             "docker.io/parity"
+  BACKEND_CONTAINER_REPO:         "substrate-telemetry-backend"
+  FRONTEND_CONTAINER_REPO:        "substrate-telemetry-frontend"
+  KUBE_NAMESPACE:                 "substrate-telemetry"
+  BACKEND_IMAGE_FULL_NAME:        "${CONTAINER_REGISTRY}/${BACKEND_CONTAINER_REPO}:${CI_COMMIT_SHORT_SHA}"
+  FRONTEND_IMAGE_FULL_NAME:       "${CONTAINER_REGISTRY}/${FRONTEND_CONTAINER_REPO}:${CI_COMMIT_SHORT_SHA}"
+
+stages:
+  - dockerize
+  - staging
+
+#.delete_deployment:       &delete_deployment
+#  - helm uninstall -n "$KUBE_NAMESPACE" "$KUBE_NAMESPACE"
+
+.dockerize:               &dockerize
+  stage:                  dockerize
+  image:                  quay.io/buildah/stable
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "jsdw-sharding"'
+      when: manual
+  tags:
+    - kubernetes-parity-build
+
+.deploy-k8s:              &deploy-k8s
+  image:                  paritytech/kubetools:3.5.3
+  script:
+    - echo "Deploying using image $BACKEND_IMAGE_FULL_NAME"
+    - echo "Using Helm `helm version --short`"
+    - export KUBERNETES_VERSION_TAG="$CI_PIPELINE_ID"
+    - |-
+      sed -i "s/version:.*/version: $KUBERNETES_VERSION_TAG/" ./helm/Chart.yaml
+    - |-
+      sed -i "s/appVersion:.*/appVersion: $KUBERNETES_VERSION_TAG/" ./helm/Chart.yaml
+    # validate the chart
+    - helm --debug template
+        --create-namespace
+        --namespace $KUBE_NAMESPACE
+        --set image.backend.repository="${CONTAINER_REGISTRY}/${BACKEND_CONTAINER_REPO}"
+        --set image.backend.tag="${CI_COMMIT_SHORT_SHA}"
+        --set image.frontend.repository="${CONTAINER_REGISTRY}/${FRONTEND_CONTAINER_REPO}"
+        --set image.frontend.tag="${CI_COMMIT_SHORT_SHA}"
+        $KUBE_NAMESPACE ./helm/
+    # install the chart into the relevant cluster
+    - helm --debug upgrade
+        --install
+        --atomic
+        --timeout 120s
+        --create-namespace
+        --namespace $KUBE_NAMESPACE
+        --set image.backend.repository="${CONTAINER_REGISTRY}/${BACKEND_CONTAINER_REPO}"
+        --set image.backend.tag="${CI_COMMIT_SHORT_SHA}"
+        --set image.frontend.repository="${CONTAINER_REGISTRY}/${FRONTEND_CONTAINER_REPO}"
+        --set image.frontend.tag="${CI_COMMIT_SHORT_SHA}"
+        $KUBE_NAMESPACE ./helm/
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "jsdw-sharding"'
+      when: manual
+  tags:
+    - kubernetes-parity-build
+
+dockerize-backend:
+  <<:                     *dockerize
+  script:
+    - echo "Building image $BACKEND_IMAGE_FULL_NAME"
+    - buildah bud
+      --format=docker
+      --tag "$BACKEND_IMAGE_FULL_NAME" ./backend/
+    - echo ${Docker_Hub_Pass_Parity} |
+        buildah login --username ${Docker_Hub_User_Parity} --password-stdin docker.io
+    - buildah push --format=v2s2 "$BACKEND_IMAGE_FULL_NAME"
+
+dockerize-frontend:
+  <<:                     *dockerize
+  script:
+    - echo "Building image $FRONTEND_IMAGE_FULL_NAME"
+    - buildah bud
+      --format=docker
+      --tag "$FRONTEND_IMAGE_FULL_NAME" ./frontend/
+    - echo ${Docker_Hub_Pass_Parity} |
+        buildah login --username ${Docker_Hub_User_Parity} --password-stdin docker.io
+    - buildah push --format=v2s2 "$FRONTEND_IMAGE_FULL_NAME"
+
+deploy-parity-stg:
+  stage:                  staging
+  <<:                     *deploy-k8s
+  environment:
+    name:                 parity-stg

README.md

@@ -5,7 +5,7 @@
 
 ## Overview
 
-This repository contains both the backend ingestion server for Substrate Telemetry as well as the Frontend you typically see running at [telemetry.polkadot.io](https://telemetry.polkadot.io/).
+This repository contains the backend ingestion server for Substrate Telemetry (which itself is comprised of two binaries; `telemetry_shard` and `telemetry_core`) as well as the Frontend you typically see running at [telemetry.polkadot.io](https://telemetry.polkadot.io/).
 
 The backend is a Rust project and the frontend is React/Typescript project.
 
@@ -20,23 +20,32 @@ nvm install stable
 yarn
 ```
 
-### Terminal 1 - Backend
+### Terminal 1 & 2 - Backend
+
+Build the backend binaries by running the following:
 
 ```
 cd backend
 cargo build --release
-./target/release/telemetry --help
 ```
 
-By default, telemetry will listen on the local interface only (127.0.0.1) on port 8000. You may change both those values with the `--listen` flag as shown below:
+And then, in two different terminals, run:
 
 ```
-telemetry --listen 0.0.0.0:8888
+./target/release/telemetry_core
 ```
 
-This example listen on all interfaces and on port :8888
+and
 
-### Terminal 2 - Frontend
+```
+./target/release/telemetry_shard
+```
+
+Use `--help` on either binary to see the available options.
+
+By default, `telemetry_core` will listen on 127.0.0.1:8000, and `telemetry_shard` will listen on 127.0.0.1:8001, and expect the `telemetry_core` to be listening on its default address. To listen on different addresses, use the `--listen` option on either binary, for example `--listen 0.0.0.0:8000`. The `telemetry_shard` also needs to be told where the core is, so if the core is configured with `--listen 127.0.0.1:9090`, remember to pass `--core 127.0.0.1:9090` to the shard, too.
+
+### Terminal 3 - Frontend
 
 ```sh
 cd frontend
@@ -44,69 +53,100 @@ yarn install
 yarn start
 ```
 
-### Terminal 3 - Node
+Once this is running, you'll be able to navigate to [http://localhost:3000](http://localhost:3000) to view the UI.
+
+### Terminal 4 - Node
 
 Follow up installation instructions from the [Polkadot repo](https://github.com/paritytech/polkadot)
 
+If you started the backend binaries with their default arguments, you can connect a node to the shard by running:
+
 ```sh
-polkadot --dev --telemetry-url ws://localhost:8000/submit
+polkadot --dev --telemetry-url 'ws://localhost:8001/submit 0'
 ```
 
+**Note:** The "0" at the end of the URL is a verbosity level, and not part of the URL itself. Verbosity levels range from 0-9, with 0 denoting the lowest verbosity.
+
 ## Docker
 
-*Pre-built docker images are available at `parity/substrate-telemetry-frontend` and `parity/substrate-telemetry-backend`.*
+### Building images
 
-### Run the backend and frontend
+To build the backend docker image, navigate into the `backend` folder of this repository and run:
 
-Obviously, the frontend need to be aware of the backend. In a similar way, your node will need to connect to the backend.
-For the sake of brevity below, I will name the containers `backend` and `frontend`. In a complex environment, you will want to use names such as `telemetry-backend` for instance to avoid conflicts with other `backend` containers.
-
-Let's start the backend first. We will be using the published [chevdor](https://hub.docker.com/u/chevdor) images here, feel free to replace with your own image.
-
-```sh
-docker run --rm -i --name backend -p 8000:8000 \
-  chevdor/substrate-telemetry-backend -l 0.0.0.0:8000
+```
+docker build -t substrate-telemetry-backend .
 ```
 
-Let's now start the frontend:
+The backend image contains both the `telemetry_core` and `telemetry_shard` binaries.
 
-```sh
-docker run --rm -i --name frontend --link backend -p 80:80 \
-  -e SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed \
-  chevdor/substrate-telemetry-frontend
+To build the frontend docker image, navigate into the `frontend` folder and run:
+
+```
+docker build -t substrate-telemetry-frontend .
 ```
 
-WARNING: Do not forget the `/feed` part of the URL...
+### Run the backend and frontend using `docker-compose`
 
-NOTE: Here we used `SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed`. This will work if you test with everything running locally on your machine but NOT if your backend runs on a remote server. Keep in mind that the frontend docker image is serving a static site running your browser. The `SUBSTRATE_TELEMETRY_URL` is the WebSocket url that your browser will use to reach the backend. Say your backend runs on a remore server at `192.168.0.100`, you will need to set the IP/url accordingly in `SUBSTRATE_TELEMETRY_URL`.
+The easiest way to run the backend and frontend images is to use `docker-compose`. To do this, run `docker-compose up` in the root of this repository to build and run the images. Once running, you can view the UI by navigating a browser to `http://localhost:3000`.
 
-At that point, you can already open your browser at [http://localhost](http://localhost/) and see that telemetry is waiting for data.
+To connect a substrate node and have it send telemetry to this running instance, you have to tell it where to send telemetry by appending the argument `--telemetry-url 'ws://localhost:8001/submit 0'` (see "Terminal 4 - Node" above).
 
-Let's bring some data in with  a node:
+### Run the backend and frontend using `docker`
+
+If you'd like to get things runing manually using Docker, you can do the following. This assumes that you've built the images as per the above, and have two images named `substrate-telemetry-backend` and `substrate-telemetry-frontend`.
+
+1. Create a new shared network so that the various containers can communicate with eachother:
+
+   ```
+   docker network create telemetry
+   ```
+
+2. Start up the backend core process. We expose port 8000 so that a UI running in a host browser can connect to the `/feed` endpoint.
+
+   ```
+   docker run --rm -it --network=telemetry \
+       --name backend-core \
+       -p 8000:8000 \
+       substrate-telemetry-backend \
+       telemetry_core -l 0.0.0.0:8000
+   ```
+
+3. In another terminal, start up the backend shard process. We tell it where it can reach the core to send messages (possible because it has been started on the same network), and we listen on and expose port 8001 so that nodes running in the host can connect and send telemetry to it.
+
+   ```
+   docker run --rm -it --network=telemetry \
+       --name backend-shard \
+       -p 8001:8001 \
+       substrate-telemetry-backend \
+       telemetry_shard -l 0.0.0.0:8001 -c http://backend-core:8000/shard_submit
+   ```
+
+4. In another terminal, start up the frontend server. We pass a `SUBSTRATE_TELEMETRY_URL` env var to tell the UI how to connect to the core process to receive telemetry. This is relative to the host machine, since that is where the browser and UI will be running.
+
+   ```
+   docker run --rm -it --network=telemetry \
+       --name frontend \
+       -p 3000:80 \
+       -e SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed \
+       substrate-telemetry-frontend
+   ```
+
+   **NOTE:** Here we used `SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed`. This will work if you test with everything running locally on your machine but NOT if your backend runs on a remote server. Keep in mind that the frontend docker image is serving a static site running your browser. The `SUBSTRATE_TELEMETRY_URL` is the WebSocket url that your browser will use to reach the backend. Say your backend runs on a remote server at `foo.example.com`, you will need to set the IP/url accordingly in `SUBSTRATE_TELEMETRY_URL` (in this case, to `ws://foo.example.com/feed`).
+
+With these running, you'll be able to navigate to [http://localhost:3000](http://localhost:3000) to view the UI. If you'd like to connect a node and have it send telemetry to your running shard, you can run the following:
 
 ```sh
-docker run --rm -i --name substrate --link backend -p 9944:9944 \
-  chevdor/substrate substrate --dev --telemetry-url 'ws://backend:8000/submit 0'
+docker run --rm -it --network=telemetry \
+  --name substrate \
+  -p 9944:9944 \
+  chevdor/substrate \
+  substrate --dev --telemetry-url 'ws://backend-shard:8001/submit 0'
 ```
 
-You should now see your node showing up in your local [telemetry frontend](http://localhost/):
+You should now see your node showing up in your local [telemetry frontend](http://localhost:3000/):
+
 ![image](doc/screenshot01.png)
 
-### Run via docker-compose
-
-To run via docker make sure that you have Docker Desktop.
-If you don't you can download for you OS here [Docker Desktop](https://www.docker.com/products/docker-desktop)
-
-```sh
-docker-compose up --build -d
-```
-
-- `-d` stands for detach, if you would like to see logs I recommend using [Kitmatic](https://kitematic.com/) or don't use the `-d`
-- `--build` will build the images and rebuild, but this is not required every time
-- If you want to makes UI changes, there is no need to rebuild the image as the files are being copied in via volumes.
-
-Now navigate to [http://localhost:3000](http://localhost:3000/) in your browser to view the app.
-
 ### Build & Publish the Frontend docker image
 
 The building process is standard. You just need to notice that the Dockerfile is in ./packages/frontend/ and tell docker about it. The context must remain the repository's root though.

backend/.dockerignore

@@ -1,3 +1,3 @@
 target
-Dockerfile
+*.Dockerfile
 .git

backend/Cargo.toml

@@ -1,6 +1,9 @@
 [workspace]
 members = [
-    "core",
+    "common",
+    "telemetry_core",
+    "telemetry_shard",
+    "test_utils"
 ]
 
 [profile.dev]
@@ -9,3 +12,7 @@ opt-level = 3
 [profile.release]
 lto = true
 panic = "abort"
+## Enabling these seems necessary to get
+## good debug info in Instruments:
+# debug = true
+# codegen-units = 1

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM paritytech/ci-linux:production as builder
+FROM docker.io/paritytech/ci-linux:production as builder
 
 ARG PROFILE=release
 WORKDIR /app
@@ -8,17 +8,17 @@ COPY . .
 RUN cargo build --${PROFILE} --bins
 
 # MAIN IMAGE FOR PEOPLE TO PULL --- small one#
-FROM debian:buster-slim
+FROM docker.io/debian:buster-slim
 LABEL maintainer="Parity Technologies"
-LABEL description="Polkadot Telemetry backend, static build"
+LABEL description="Polkadot Telemetry backend shard/core binaries, static build"
 
 ARG PROFILE=release
 WORKDIR /usr/local/bin
 
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=builder /app/target/$PROFILE/telemetry /usr/local/bin
+COPY --from=builder /app/target/$PROFILE/telemetry_shard /usr/local/bin
+COPY --from=builder /app/target/$PROFILE/telemetry_core /usr/local/bin
 RUN apt-get -y update && apt-get -y install openssl && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/
 
 EXPOSE 8000
 
-ENTRYPOINT ["telemetry"]

backend/README.md

@@ -0,0 +1,19 @@
+# Backend Crates
+
+This folder contains the rust crates and documentation specific to the telemetry backend. A description of the folders:
+
+- [telemetry_core](./telemetry_core): The Telemetry Core. This aggregates data received from shards and allows UI feeds to connect and receive this information.
+- [telemetry_shard](./telemetry_shard): A Shard. It's expected that multiple of these will run. Nodes will connect to Shard instances and send JSON telemetry to them, and Shard instances will each connect to the Telemetry Core and relay on relevant data to it.
+- [common](./common): common code shared between the telemetry shard and core
+- [test_utils](./test_utils): Test utilities, primarily focused around making it easy to run end-to-end tests.
+- [docs](./docs): Material supporting the documentation lives here
+
+# Architecture
+
+As we move to a sharded version of this telemetry server, this set of architecture diagrams may be useful in helping to understand the current setup (middle diagram), previous setup (first diagram) and possible future setup if we need to scale further (last diagram):
+
+![Architecture Diagram](./docs/architecture.svg)
+
+# Deployment
+
+A `Dockerfile` exists which builds the Shard and Telemetry Core binaries into an image. A `docker-compose.yaml` in the root of the repository can serve as an example of these services, along with the UI, running together.

backend/common/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "common"
+version = "0.1.0"
+authors = ["Parity Technologies Ltd. <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0"
+
+[dependencies]
+anyhow = "1.0.42"
+base64 = { default-features = false, features = ["alloc"], version = "0.13" }
+bimap = "0.6.1"
+bytes = "1.0.1"
+fnv = "1.0.7"
+futures = "0.3.15"
+hex = "0.4.3"
+http = "0.2.4"
+hyper = { version = "0.14.11", features = ["full"] }
+log = "0.4"
+num-traits = "0.2"
+pin-project-lite = "0.2.7"
+primitive-types = { version = "0.9.0", features = ["serde"] }
+rustc-hash = "1.1.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0", features = ["raw_value"] }
+sha-1 = { default-features = false, version = "0.9" }
+soketto = "0.6.0"
+thiserror = "1.0.24"
+tokio = { version = "1.8.2", features = ["full"] }
+tokio-util = { version = "0.6", features = ["compat"] }
+
+[dev-dependencies]
+bincode = "1.3.3"

backend/common/src/assign_id.rs

@@ -0,0 +1,84 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use bimap::BiMap;
+use std::hash::Hash;
+
+/// A struct that allows you to assign an Id to an arbitrary set of
+/// details (so long as they are Eq+Hash+Clone), and then access
+/// the assigned Id given those details or access the details given
+/// the Id.
+///
+/// The Id can be any type that's convertible to/from a `usize`. Using
+/// a custom type is recommended for increased type safety.
+#[derive(Debug)]
+pub struct AssignId<Id, Details> {
+    current_id: usize,
+    mapping: BiMap<usize, Details>,
+    _id_type: std::marker::PhantomData<Id>,
+}
+
+impl<Id, Details> AssignId<Id, Details>
+where
+    Details: Eq + Hash,
+    Id: From<usize> + Copy,
+    usize: From<Id>,
+{
+    pub fn new() -> Self {
+        Self {
+            current_id: 0,
+            mapping: BiMap::new(),
+            _id_type: std::marker::PhantomData,
+        }
+    }
+
+    pub fn assign_id(&mut self, details: Details) -> Id {
+        let this_id = self.current_id;
+        self.current_id += 1;
+        self.mapping.insert(this_id, details);
+        this_id.into()
+    }
+
+    pub fn get_details(&mut self, id: Id) -> Option<&Details> {
+        self.mapping.get_by_left(&id.into())
+    }
+
+    pub fn get_id(&mut self, details: &Details) -> Option<Id> {
+        self.mapping.get_by_right(details).map(|&id| id.into())
+    }
+
+    pub fn remove_by_id(&mut self, id: Id) -> Option<Details> {
+        self.mapping
+            .remove_by_left(&id.into())
+            .map(|(_, details)| details)
+    }
+
+    pub fn remove_by_details(&mut self, details: &Details) -> Option<Id> {
+        self.mapping
+            .remove_by_right(&details)
+            .map(|(id, _)| id.into())
+    }
+
+    pub fn clear(&mut self) {
+        *self = AssignId::new();
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (Id, &Details)> {
+        self.mapping
+            .iter()
+            .map(|(&id, details)| (id.into(), details))
+    }
+}

backend/common/src/byte_size.rs

@@ -0,0 +1,108 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::{anyhow, Error};
+
+#[derive(Copy, Clone, Debug)]
+pub struct ByteSize(usize);
+
+impl ByteSize {
+    pub fn new(bytes: usize) -> ByteSize {
+        ByteSize(bytes)
+    }
+    /// Return the number of bytes stored within.
+    pub fn num_bytes(self) -> usize {
+        self.0
+    }
+}
+
+impl From<ByteSize> for usize {
+    fn from(b: ByteSize) -> Self {
+        b.0
+    }
+}
+
+impl std::str::FromStr for ByteSize {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let s = s.trim();
+        match s.find(|c| !char::is_ascii_digit(&c)) {
+            // No non-numeric chars; assume bytes then
+            None => Ok(ByteSize(s.parse().expect("all ascii digits"))),
+            // First non-numeric char
+            Some(idx) => {
+                let n = s[..idx].parse().expect("all ascii digits");
+                let suffix = s[idx..].trim();
+                let n = match suffix {
+                    "B" | "b" => n,
+                    "kB" | "K" | "k" => n * 1000,
+                    "MB" | "M" | "m" => n * 1000 * 1000,
+                    "GB" | "G" | "g" => n * 1000 * 1000 * 1000,
+                    "KiB" | "Ki" => n * 1024,
+                    "MiB" | "Mi" => n * 1024 * 1024,
+                    "GiB" | "Gi" => n * 1024 * 1024 * 1024,
+                    _ => {
+                        return Err(anyhow!(
+                            "\
+                        Cannot parse into bytes; suffix is '{}', but expecting one of \
+                        B,b, kB,K,k, MB,M,m, GB,G,g, KiB,Ki, MiB,Mi, GiB,Gi",
+                            suffix
+                        ))
+                    }
+                };
+                Ok(ByteSize(n))
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::byte_size::ByteSize;
+
+    #[test]
+    fn can_parse_valid_strings() {
+        let cases = vec![
+            ("100", 100),
+            ("100B", 100),
+            ("100b", 100),
+            ("20kB", 20 * 1000),
+            ("20 kB", 20 * 1000),
+            ("20K", 20 * 1000),
+            (" 20k", 20 * 1000),
+            ("1MB", 1 * 1000 * 1000),
+            ("1M", 1 * 1000 * 1000),
+            ("1m", 1 * 1000 * 1000),
+            ("1 m", 1 * 1000 * 1000),
+            ("1GB", 1 * 1000 * 1000 * 1000),
+            ("1G", 1 * 1000 * 1000 * 1000),
+            ("1g", 1 * 1000 * 1000 * 1000),
+            ("1KiB", 1 * 1024),
+            ("1Ki", 1 * 1024),
+            ("1MiB", 1 * 1024 * 1024),
+            ("1Mi", 1 * 1024 * 1024),
+            ("1GiB", 1 * 1024 * 1024 * 1024),
+            ("1Gi", 1 * 1024 * 1024 * 1024),
+            (" 1 Gi ", 1 * 1024 * 1024 * 1024),
+        ];
+
+        for (s, expected) in cases {
+            let b: ByteSize = s.parse().unwrap();
+            assert_eq!(b.num_bytes(), expected);
+        }
+    }
+}

backend/common/src/dense_map.rs

@@ -0,0 +1,133 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/// This stores items in contiguous memory, making a note of free
+/// slots when items are removed again so that they can be reused.
+///
+/// This is particularly efficient when items are often added and
+/// seldom removed.
+///
+/// Items are keyed by an Id, which can be any type you wish, but
+/// must be convertible to/from a `usize`. This promotes using a
+/// custom Id type to talk about items in the map.
+pub struct DenseMap<Id, T> {
+    /// List of retired indexes that can be re-used
+    retired: Vec<usize>,
+    /// All items
+    items: Vec<Option<T>>,
+    /// Our ID type
+    _id_type: std::marker::PhantomData<Id>,
+}
+
+impl<Id, T> DenseMap<Id, T>
+where
+    Id: From<usize> + Copy,
+    usize: From<Id>,
+{
+    pub fn new() -> Self {
+        DenseMap {
+            retired: Vec::new(),
+            items: Vec::new(),
+            _id_type: std::marker::PhantomData,
+        }
+    }
+
+    pub fn add(&mut self, item: T) -> Id {
+        self.add_with(|_| item)
+    }
+
+    pub fn as_slice(&self) -> &[Option<T>] {
+        &self.items
+    }
+
+    pub fn add_with<F>(&mut self, f: F) -> Id
+    where
+        F: FnOnce(Id) -> T,
+    {
+        match self.retired.pop() {
+            Some(id) => {
+                let id_out = id.into();
+                self.items[id] = Some(f(id_out));
+                id_out
+            }
+            None => {
+                let id = self.items.len().into();
+                self.items.push(Some(f(id)));
+                id
+            }
+        }
+    }
+
+    pub fn get(&self, id: Id) -> Option<&T> {
+        let id: usize = id.into();
+        self.items.get(id).and_then(|item| item.as_ref())
+    }
+
+    pub fn get_mut(&mut self, id: Id) -> Option<&mut T> {
+        let id: usize = id.into();
+        self.items.get_mut(id).and_then(|item| item.as_mut())
+    }
+
+    pub fn remove(&mut self, id: Id) -> Option<T> {
+        let id: usize = id.into();
+        let old = self.items.get_mut(id).and_then(|item| item.take());
+
+        if old.is_some() {
+            // something was actually removed, so lets add the id to
+            // the list of retired ids!
+            self.retired.push(id);
+        }
+
+        old
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (Id, &T)> + '_ {
+        self.items
+            .iter()
+            .enumerate()
+            .filter_map(|(id, item)| Some((id.into(), item.as_ref()?)))
+    }
+
+    pub fn iter_mut(&mut self) -> impl Iterator<Item = (Id, &mut T)> + '_ {
+        self.items
+            .iter_mut()
+            .enumerate()
+            .filter_map(|(id, item)| Some((id.into(), item.as_mut()?)))
+    }
+
+    pub fn into_iter(self) -> impl Iterator<Item = (Id, T)> {
+        self.items
+            .into_iter()
+            .enumerate()
+            .filter_map(|(id, item)| Some((id.into(), item?)))
+    }
+
+    pub fn len(&self) -> usize {
+        self.items.len() - self.retired.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Return the next Id that will be assigned.
+    pub fn next_id(&self) -> usize {
+        match self.retired.last() {
+            Some(id) => *id,
+            None => self.items.len(),
+        }
+    }
+}

backend/common/src/either_sink.rs

@@ -0,0 +1,66 @@
+use futures::sink::Sink;
+use pin_project_lite::pin_project;
+
+pin_project! {
+    #[project = EitherSinkInner]
+    pub enum EitherSink<A, B> {
+        A { #[pin] inner: A },
+        B { #[pin] inner: B }
+    }
+}
+
+/// A simple enum that delegates implementation to one of
+/// the two possible sinks contained within.
+impl<A, B> EitherSink<A, B> {
+    pub fn a(val: A) -> Self {
+        EitherSink::A { inner: val }
+    }
+    pub fn b(val: B) -> Self {
+        EitherSink::B { inner: val }
+    }
+}
+
+impl<Item, Error, A, B> Sink<Item> for EitherSink<A, B>
+where
+    A: Sink<Item, Error = Error>,
+    B: Sink<Item, Error = Error>,
+{
+    type Error = Error;
+
+    fn poll_ready(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        match self.project() {
+            EitherSinkInner::A { inner } => inner.poll_ready(cx),
+            EitherSinkInner::B { inner } => inner.poll_ready(cx),
+        }
+    }
+
+    fn start_send(self: std::pin::Pin<&mut Self>, item: Item) -> Result<(), Self::Error> {
+        match self.project() {
+            EitherSinkInner::A { inner } => inner.start_send(item),
+            EitherSinkInner::B { inner } => inner.start_send(item),
+        }
+    }
+
+    fn poll_flush(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        match self.project() {
+            EitherSinkInner::A { inner } => inner.poll_flush(cx),
+            EitherSinkInner::B { inner } => inner.poll_flush(cx),
+        }
+    }
+
+    fn poll_close(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        match self.project() {
+            EitherSinkInner::A { inner } => inner.poll_close(cx),
+            EitherSinkInner::B { inner } => inner.poll_close(cx),
+        }
+    }
+}

backend/common/src/http_utils.rs

@@ -0,0 +1,172 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use futures::io::{BufReader, BufWriter};
+use hyper::server::conn::AddrStream;
+use hyper::{Body, Request, Response, Server};
+use std::future::Future;
+use std::net::SocketAddr;
+use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
+
+/// A convenience function to start up a Hyper server and handle requests.
+pub async fn start_server<H, F>(addr: SocketAddr, handler: H) -> Result<(), anyhow::Error>
+where
+    H: Clone + Send + Sync + 'static + FnMut(SocketAddr, Request<Body>) -> F,
+    F: Send + 'static + Future<Output = Result<Response<Body>, anyhow::Error>>,
+{
+    let service = hyper::service::make_service_fn(move |addr: &AddrStream| {
+        let mut handler = handler.clone();
+        let addr = addr.remote_addr();
+        async move { Ok::<_, hyper::Error>(hyper::service::service_fn(move |r| handler(addr, r))) }
+    });
+    let server = Server::bind(&addr).serve(service);
+
+    log::info!("listening on http://{}", server.local_addr());
+    server.await?;
+
+    Ok(())
+}
+
+type WsStream = BufReader<BufWriter<Compat<hyper::upgrade::Upgraded>>>;
+pub type WsSender = soketto::connection::Sender<WsStream>;
+pub type WsReceiver = soketto::connection::Receiver<WsStream>;
+
+/// A convenience function to upgrade a Hyper request into a Soketto Websocket.
+pub fn upgrade_to_websocket<H, F>(req: Request<Body>, on_upgrade: H) -> hyper::Response<Body>
+where
+    H: 'static + Send + FnOnce(WsSender, WsReceiver) -> F,
+    F: Send + Future<Output = ()>,
+{
+    if !is_upgrade_request(&req) {
+        return basic_response(400, "Expecting WebSocket upgrade headers");
+    }
+
+    let key = match req.headers().get("Sec-WebSocket-Key") {
+        Some(key) => key,
+        None => {
+            return basic_response(
+                400,
+                "Upgrade to websocket connection failed; Sec-WebSocket-Key header not provided",
+            )
+        }
+    };
+
+    if req
+        .headers()
+        .get("Sec-WebSocket-Version")
+        .map(|v| v.as_bytes())
+        != Some(b"13")
+    {
+        return basic_response(
+            400,
+            "Sec-WebSocket-Version header should have a value of 13",
+        );
+    }
+
+    // Just a little ceremony we need to go to to return the correct response key:
+    let mut accept_key_buf = [0; 32];
+    let accept_key = generate_websocket_accept_key(key.as_bytes(), &mut accept_key_buf);
+
+    // Tell the client that we accept the upgrade-to-WS request:
+    let response = Response::builder()
+        .status(hyper::StatusCode::SWITCHING_PROTOCOLS)
+        .header(hyper::header::CONNECTION, "upgrade")
+        .header(hyper::header::UPGRADE, "websocket")
+        .header("Sec-WebSocket-Accept", accept_key)
+        .body(Body::empty())
+        .expect("bug: failed to build response");
+
+    // Spawn our handler to work with the WS connection:
+    tokio::spawn(async move {
+        // Get our underlying TCP stream:
+        let stream = match hyper::upgrade::on(req).await {
+            Ok(stream) => stream,
+            Err(e) => {
+                log::error!("Error upgrading connection to websocket: {}", e);
+                return;
+            }
+        };
+
+        // Start a Soketto server with it:
+        let server =
+            soketto::handshake::Server::new(BufReader::new(BufWriter::new(stream.compat())));
+
+        // Get hold of a way to send and receive messages:
+        let (sender, receiver) = server.into_builder().finish();
+
+        // Pass these to our when-upgraded handler:
+        on_upgrade(sender, receiver).await;
+    });
+
+    response
+}
+
+/// A helper to return a basic HTTP response with a code and text body.
+fn basic_response(code: u16, msg: impl AsRef<str>) -> Response<Body> {
+    Response::builder()
+        .status(code)
+        .body(Body::from(msg.as_ref().to_owned()))
+        .expect("bug: failed to build response body")
+}
+
+/// Defined in RFC 6455. this is how we convert the Sec-WebSocket-Key in a request into a
+/// Sec-WebSocket-Accept that we return in the response.
+fn generate_websocket_accept_key<'a>(key: &[u8], buf: &'a mut [u8; 32]) -> &'a [u8] {
+    // Defined in RFC 6455, we append this to the key to generate the response:
+    const KEY: &[u8] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
+
+    use sha1::{Digest, Sha1};
+    let mut digest = Sha1::new();
+    digest.update(key);
+    digest.update(KEY);
+    let d = digest.finalize();
+
+    let n = base64::encode_config_slice(&d, base64::STANDARD, buf);
+    &buf[..n]
+}
+
+/// Check if a request is a websocket upgrade request.
+fn is_upgrade_request<B>(request: &hyper::Request<B>) -> bool {
+    header_contains_value(request.headers(), hyper::header::CONNECTION, b"upgrade")
+        && header_contains_value(request.headers(), hyper::header::UPGRADE, b"websocket")
+}
+
+/// Check if there is a header of the given name containing the wanted value.
+fn header_contains_value(
+    headers: &hyper::HeaderMap,
+    header: hyper::header::HeaderName,
+    value: &[u8],
+) -> bool {
+    pub fn trim(x: &[u8]) -> &[u8] {
+        let from = match x.iter().position(|x| !x.is_ascii_whitespace()) {
+            Some(i) => i,
+            None => return &[],
+        };
+        let to = x.iter().rposition(|x| !x.is_ascii_whitespace()).unwrap();
+        &x[from..=to]
+    }
+
+    for header in headers.get_all(header) {
+        if header
+            .as_bytes()
+            .split(|&c| c == b',')
+            .any(|x| trim(x).eq_ignore_ascii_case(value))
+        {
+            return true;
+        }
+    }
+    false
+}

backend/common/src/id_type.rs

@@ -0,0 +1,90 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/// Define a type that can be used as an ID, be converted from/to the inner type,
+/// and serialized/deserialized transparently into the inner type.
+#[macro_export]
+macro_rules! id_type {
+    ($( #[$attrs:meta] )* $vis:vis struct $ty:ident ( $inner:ident ) $(;)? ) => {
+        #[derive(Debug,Clone,Copy,PartialEq,Eq,Hash)]
+        $( #[$attrs] )*
+        $vis struct $ty($inner);
+
+        impl $ty {
+            #[allow(dead_code)]
+            pub fn new(inner: $inner) -> Self {
+                Self(inner)
+            }
+        }
+
+        impl From<$inner> for $ty {
+            fn from(inner: $inner) -> Self {
+                Self(inner)
+            }
+        }
+
+        impl From<$ty> for $inner {
+            fn from(ty: $ty) -> Self {
+                ty.0
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    // Mostly we're just checking that everything compiles OK
+    // when the macro is used as expected..
+
+    // A basic definition is possible:
+    id_type! {
+        struct Foo(usize)
+    }
+
+    // We can add a ';' on the end:
+    id_type! {
+        struct Bar(usize);
+    }
+
+    // Visibility qualifiers are allowed:
+    id_type! {
+        pub struct Wibble(u64)
+    }
+
+    // Doc strings are possible
+    id_type! {
+        /// We can have doc strings, too
+        pub(crate) struct Wobble(u16)
+    }
+
+    // In fact, any attributes can be added (common
+    // derives are added already):
+    id_type! {
+        /// We can have doc strings, too
+        #[derive(serde::Serialize)]
+        #[serde(transparent)]
+        pub(crate) struct Lark(u16)
+    }
+
+    #[test]
+    fn create_and_use_new_id_type() {
+        let _ = Foo::new(123);
+        let id = Foo::from(123);
+        let id_num: usize = id.into();
+
+        assert_eq!(id_num, 123);
+    }
+}

backend/common/src/internal_messages.rs

@@ -0,0 +1,67 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Internal messages passed between the shard and telemetry core.
+
+use std::net::IpAddr;
+
+use crate::id_type;
+use crate::node_message::Payload;
+use crate::node_types::{BlockHash, NodeDetails};
+use serde::{Deserialize, Serialize};
+
+id_type! {
+    /// The shard-local ID of a given node, where a single connection
+    /// might send data on behalf of more than one chain.
+    #[derive(serde::Serialize, serde::Deserialize)]
+    pub struct ShardNodeId(usize);
+}
+
+/// Message sent from a telemetry shard to the telemetry core
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum FromShardAggregator {
+    /// Get information about a new node, including it's IP
+    /// address and chain genesis hash.
+    AddNode {
+        ip: IpAddr,
+        node: NodeDetails,
+        local_id: ShardNodeId,
+        genesis_hash: BlockHash,
+    },
+    /// A message payload with updated details for a node
+    UpdateNode {
+        local_id: ShardNodeId,
+        payload: Payload,
+    },
+    /// Inform the telemetry core that a node has been removed
+    RemoveNode { local_id: ShardNodeId },
+}
+
+/// Message sent form the telemetry core to a telemetry shard
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum FromTelemetryCore {
+    Mute {
+        local_id: ShardNodeId,
+        reason: MuteReason,
+    },
+}
+
+/// Why is the thing being muted?
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum MuteReason {
+    Overquota,
+    ChainNotAllowed,
+}

backend/common/src/lib.rs

@@ -0,0 +1,41 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub mod byte_size;
+pub mod http_utils;
+pub mod id_type;
+pub mod internal_messages;
+pub mod node_message;
+pub mod node_types;
+pub mod ready_chunks_all;
+pub mod rolling_total;
+pub mod time;
+pub mod ws_client;
+
+mod assign_id;
+mod dense_map;
+mod either_sink;
+mod mean_list;
+mod most_seen;
+mod num_stats;
+
+// Export a bunch of common bits at the top level for ease of import:
+pub use assign_id::AssignId;
+pub use dense_map::DenseMap;
+pub use either_sink::EitherSink;
+pub use mean_list::MeanList;
+pub use most_seen::MostSeen;
+pub use num_stats::NumStats;

backend/common/src/mean_list.rs

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 use num_traits::{Float, Zero};
 use std::ops::AddAssign;
 

backend/common/src/most_seen.rs

@@ -0,0 +1,252 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::HashMap;
+use std::hash::Hash;
+
+/// Add items to this, and it will keep track of what the item
+/// seen the most is.
+#[derive(Debug)]
+pub struct MostSeen<T> {
+    current_best: T,
+    current_count: usize,
+    others: HashMap<T, usize>,
+}
+
+impl<T: Default> Default for MostSeen<T> {
+    fn default() -> Self {
+        // This sets the "most seen item" to the default value for the type,
+        // and notes that nobody has actually seen it yet (current_count is 0).
+        Self {
+            current_best: T::default(),
+            current_count: 0,
+            others: HashMap::new(),
+        }
+    }
+}
+
+impl<T> MostSeen<T> {
+    pub fn new(item: T) -> Self {
+        // This starts us off with an item that we've seen. This item is set as
+        // the "most seen item" and the current_count is set to 1, as we've seen it
+        // once by virtue of providing it here.
+        Self {
+            current_best: item,
+            current_count: 1,
+            others: HashMap::new(),
+        }
+    }
+    pub fn best(&self) -> &T {
+        &self.current_best
+    }
+    pub fn best_count(&self) -> usize {
+        self.current_count
+    }
+}
+
+impl<T: Hash + Eq + Clone> MostSeen<T> {
+    pub fn insert(&mut self, item: &T) -> ChangeResult {
+        if &self.current_best == item {
+            // Item already the best one; bump count.
+            self.current_count += 1;
+            return ChangeResult::NoChange;
+        }
+
+        // Item not the best; increment count in map
+        let item_count = self.others.entry(item.clone()).or_default();
+        *item_count += 1;
+
+        // Is item now the best?
+        if *item_count > self.current_count {
+            let (mut item, mut count) = self.others.remove_entry(item).expect("item added above");
+
+            // Swap the current best for the new best:
+            std::mem::swap(&mut item, &mut self.current_best);
+            std::mem::swap(&mut count, &mut self.current_count);
+
+            // Insert the old best back into the map:
+            self.others.insert(item, count);
+
+            ChangeResult::NewMostSeenItem
+        } else {
+            ChangeResult::NoChange
+        }
+    }
+    pub fn remove(&mut self, item: &T) -> ChangeResult {
+        if &self.current_best == item {
+            // Item already the best one; reduce count (don't allow to drop below 0)
+            self.current_count = self.current_count.saturating_sub(1);
+
+            // Is there a new best?
+            let other_best = self.others.iter().max_by_key(|f| f.1);
+
+            let (other_item, &other_count) = match other_best {
+                Some(item) => item,
+                None => return ChangeResult::NoChange,
+            };
+
+            if other_count > self.current_count {
+                // Clone item to unborrow self.others so that we can remove
+                // the item from it. We could pre-emptively remove and reinsert
+                // instead, but most of the time there is no change, so I'm
+                // aiming to keep that path cheaper.
+                let other_item = other_item.clone();
+                let (mut other_item, mut other_count) = self
+                    .others
+                    .remove_entry(&other_item)
+                    .expect("item returned above, so def exists");
+
+                // Swap the current best for the new best:
+                std::mem::swap(&mut other_item, &mut self.current_best);
+                std::mem::swap(&mut other_count, &mut self.current_count);
+
+                // Insert the old best back into the map:
+                self.others.insert(other_item, other_count);
+
+                return ChangeResult::NewMostSeenItem;
+            } else {
+                return ChangeResult::NoChange;
+            }
+        }
+
+        // Item is in the map; not the best anyway. decrement count.
+        if let Some(count) = self.others.get_mut(item) {
+            *count += 1;
+        }
+        ChangeResult::NoChange
+    }
+}
+
+/// Record the result of adding/removing an entry
+#[derive(Clone, Copy)]
+pub enum ChangeResult {
+    /// The best item has remained the same.
+    NoChange,
+    /// There is a new best item now.
+    NewMostSeenItem,
+}
+
+impl ChangeResult {
+    pub fn has_changed(self) -> bool {
+        match self {
+            ChangeResult::NewMostSeenItem => true,
+            ChangeResult::NoChange => false,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn default_renames_instantly() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+        let res = a.insert(&"Hello");
+        assert_eq!(*a.best(), "Hello");
+        assert!(res.has_changed());
+    }
+
+    #[test]
+    fn new_renames_on_second_change() {
+        let mut a: MostSeen<&str> = MostSeen::new("First");
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "First");
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "Second");
+    }
+
+    #[test]
+    fn removing_doesnt_underflow() {
+        let mut a: MostSeen<&str> = MostSeen::new("First");
+        a.remove(&"First");
+        a.remove(&"First");
+        a.remove(&"Second");
+        a.remove(&"Third");
+    }
+
+    #[test]
+    fn keeps_track_of_best_count() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+        a.insert(&"First");
+        assert_eq!(a.best_count(), 1);
+
+        a.insert(&"First");
+        assert_eq!(a.best_count(), 2);
+
+        a.insert(&"First");
+        assert_eq!(a.best_count(), 3);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 2);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 1);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 0);
+
+        a.remove(&"First");
+        assert_eq!(a.best_count(), 0);
+    }
+
+    #[test]
+    fn it_tracks_best_on_insert() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+
+        a.insert(&"First");
+        assert_eq!(*a.best(), "First", "1");
+
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "First", "2");
+
+        a.insert(&"Second");
+        assert_eq!(*a.best(), "Second", "3");
+
+        a.insert(&"First");
+        assert_eq!(*a.best(), "Second", "4");
+
+        a.insert(&"First");
+        assert_eq!(*a.best(), "First", "5");
+    }
+
+    #[test]
+    fn it_tracks_best() {
+        let mut a: MostSeen<&str> = MostSeen::default();
+        a.insert(&"First");
+        a.insert(&"Second");
+        a.insert(&"Third"); // 1
+
+        a.insert(&"Second");
+        a.insert(&"Second"); // 3
+        a.insert(&"First"); // 2
+
+        assert_eq!(*a.best(), "Second");
+        assert_eq!(a.best_count(), 3);
+
+        let res = a.remove(&"Second");
+
+        assert!(!res.has_changed());
+        assert_eq!(a.best_count(), 2);
+        assert_eq!(*a.best(), "Second"); // Tied with "First"
+
+        let res = a.remove(&"Second");
+
+        assert!(res.has_changed());
+        assert_eq!(a.best_count(), 2);
+        assert_eq!(*a.best(), "First"); // First is now ahead
+    }
+}

backend/common/src/node_message.rs

@@ -0,0 +1,262 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! This is the internal represenation of telemetry messages sent from nodes.
+//! There is a separate JSON representation of these types, because internally we want to be
+//! able to serialize these messages to bincode, and various serde attribtues aren't compatible
+//! with this, hence this separate internal representation.
+
+use crate::node_types::{Block, BlockHash, BlockNumber, NodeDetails};
+use serde::{Deserialize, Serialize};
+
+pub type NodeMessageId = u64;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum NodeMessage {
+    V1 { payload: Payload },
+    V2 { id: NodeMessageId, payload: Payload },
+}
+
+impl NodeMessage {
+    /// Returns the ID associated with the node message, or 0
+    /// if the message has no ID.
+    pub fn id(&self) -> NodeMessageId {
+        match self {
+            NodeMessage::V1 { .. } => 0,
+            NodeMessage::V2 { id, .. } => *id,
+        }
+    }
+    /// Return the payload associated with the message.
+    pub fn into_payload(self) -> Payload {
+        match self {
+            NodeMessage::V1 { payload, .. } | NodeMessage::V2 { payload, .. } => payload,
+        }
+    }
+}
+
+impl From<NodeMessage> for Payload {
+    fn from(msg: NodeMessage) -> Payload {
+        msg.into_payload()
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum Payload {
+    SystemConnected(SystemConnected),
+    SystemInterval(SystemInterval),
+    BlockImport(Block),
+    NotifyFinalized(Finalized),
+    TxPoolImport,
+    AfgFinalized(AfgFinalized),
+    AfgReceivedPrecommit(AfgReceived),
+    AfgReceivedPrevote(AfgReceived),
+    AfgReceivedCommit(AfgReceived),
+    AfgAuthoritySet(AfgAuthoritySet),
+    AfgFinalizedBlocksUpTo,
+    AuraPreSealedBlock,
+    PreparedBlockForProposing,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct SystemConnected {
+    pub genesis_hash: BlockHash,
+    pub node: NodeDetails,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct SystemInterval {
+    pub peers: Option<u64>,
+    pub txcount: Option<u64>,
+    pub bandwidth_upload: Option<f64>,
+    pub bandwidth_download: Option<f64>,
+    pub finalized_height: Option<BlockNumber>,
+    pub finalized_hash: Option<BlockHash>,
+    pub block: Option<Block>,
+    pub used_state_cache_size: Option<f32>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Finalized {
+    pub hash: BlockHash,
+    pub height: Box<str>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct AfgFinalized {
+    pub finalized_hash: BlockHash,
+    pub finalized_number: Box<str>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct AfgReceived {
+    pub target_hash: BlockHash,
+    pub target_number: Box<str>,
+    pub voter: Option<Box<str>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct AfgAuthoritySet {
+    pub authority_id: Box<str>,
+    pub authorities: Box<str>,
+    pub authority_set_id: Box<str>,
+}
+
+impl Payload {
+    pub fn best_block(&self) -> Option<&Block> {
+        match self {
+            Payload::BlockImport(block) => Some(block),
+            Payload::SystemInterval(SystemInterval { block, .. }) => block.as_ref(),
+            _ => None,
+        }
+    }
+
+    pub fn finalized_block(&self) -> Option<Block> {
+        match self {
+            Payload::SystemInterval(ref interval) => Some(Block {
+                hash: interval.finalized_hash?,
+                height: interval.finalized_height?,
+            }),
+            Payload::NotifyFinalized(ref finalized) => Some(Block {
+                hash: finalized.hash,
+                height: finalized.height.parse().ok()?,
+            }),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bincode::Options;
+
+    // Without adding a derive macro and marker trait (and enforcing their use), we don't really
+    // know whether things can (de)serialize to bincode or not at runtime without failing unless
+    // we test the different types we want to (de)serialize ourselves. We just need to test each
+    // type, not each variant.
+    fn bincode_can_serialize_and_deserialize<'de, T>(item: T)
+    where
+        T: Serialize + serde::de::DeserializeOwned,
+    {
+        let bytes = bincode::serialize(&item).expect("Serialization should work");
+        let _: T = bincode::deserialize(&bytes).expect("Deserialization should work");
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_system_connected() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::SystemConnected(SystemConnected {
+                genesis_hash: BlockHash::zero(),
+                node: NodeDetails {
+                    chain: "foo".into(),
+                    name: "foo".into(),
+                    implementation: "foo".into(),
+                    version: "foo".into(),
+                    validator: None,
+                    network_id: None,
+                    startup_time: None,
+                },
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_system_interval() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::SystemInterval(SystemInterval {
+                peers: None,
+                txcount: None,
+                bandwidth_upload: None,
+                bandwidth_download: None,
+                finalized_height: None,
+                finalized_hash: None,
+                block: None,
+                used_state_cache_size: None,
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_block_import() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::BlockImport(Block {
+                hash: BlockHash([0; 32]),
+                height: 0,
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_notify_finalized() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::NotifyFinalized(Finalized {
+                hash: BlockHash::zero(),
+                height: "foo".into(),
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_tx_pool_import() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::TxPoolImport,
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_afg_finalized() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::AfgFinalized(AfgFinalized {
+                finalized_hash: BlockHash::zero(),
+                finalized_number: "foo".into(),
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_afg_received() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::AfgReceivedPrecommit(AfgReceived {
+                target_hash: BlockHash::zero(),
+                target_number: "foo".into(),
+                voter: None,
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_can_serialize_and_deserialize_node_message_afg_authority_set() {
+        bincode_can_serialize_and_deserialize(NodeMessage::V1 {
+            payload: Payload::AfgAuthoritySet(AfgAuthoritySet {
+                authority_id: "foo".into(),
+                authorities: "foo".into(),
+                authority_set_id: "foo".into(),
+            }),
+        });
+    }
+
+    #[test]
+    fn bincode_block_zero() {
+        let raw = Block::zero();
+
+        let bytes = bincode::options().serialize(&raw).unwrap();
+
+        let deserialized: Block = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(raw.hash, deserialized.hash);
+        assert_eq!(raw.height, deserialized.height);
+    }
+}

backend/common/src/node_types.rs

@@ -0,0 +1,224 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! These types are partly used in [`crate::node_message`], but also stored and used
+//! more generally through the application.
+
+use serde::ser::{SerializeTuple, Serializer};
+use serde::{Deserialize, Serialize};
+
+use crate::{time, MeanList};
+
+pub type BlockNumber = u64;
+pub type Timestamp = u64;
+pub use primitive_types::H256 as BlockHash;
+
+/// Basic node details.
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct NodeDetails {
+    pub chain: Box<str>,
+    pub name: Box<str>,
+    pub implementation: Box<str>,
+    pub version: Box<str>,
+    pub validator: Option<Box<str>>,
+    pub network_id: Option<Box<str>>,
+    pub startup_time: Option<Box<str>>,
+}
+
+/// A couple of node statistics.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct NodeStats {
+    pub peers: u64,
+    pub txcount: u64,
+}
+
+// # A note about serialization/deserialization of types in this file:
+//
+// Some of the types here are sent to UI feeds. In an effort to keep the
+// amount of bytes sent to a minimum, we have written custom serializers
+// for those types.
+//
+// For testing purposes, it's useful to be able to deserialize from some
+// of these types so that we can test message feed things, so custom
+// deserializers exist to undo the work of the custom serializers.
+impl Serialize for NodeStats {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(2)?;
+        tup.serialize_element(&self.peers)?;
+        tup.serialize_element(&self.txcount)?;
+        tup.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for NodeStats {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let (peers, txcount) = <(u64, u64)>::deserialize(deserializer)?;
+        Ok(NodeStats { peers, txcount })
+    }
+}
+
+/// Node IO details.
+#[derive(Default)]
+pub struct NodeIO {
+    pub used_state_cache_size: MeanList<f32>,
+}
+
+impl Serialize for NodeIO {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(1)?;
+        // This is "one-way": we can't deserialize again from this to a MeanList:
+        tup.serialize_element(self.used_state_cache_size.slice())?;
+        tup.end()
+    }
+}
+
+/// Concise block details
+#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)]
+pub struct Block {
+    pub hash: BlockHash,
+    pub height: BlockNumber,
+}
+
+impl Block {
+    pub fn zero() -> Self {
+        Block {
+            hash: BlockHash::from([0; 32]),
+            height: 0,
+        }
+    }
+}
+
+/// Node hardware details.
+#[derive(Default)]
+pub struct NodeHardware {
+    /// Upload uses means
+    pub upload: MeanList<f64>,
+    /// Download uses means
+    pub download: MeanList<f64>,
+    /// Stampchange uses means
+    pub chart_stamps: MeanList<f64>,
+}
+
+impl Serialize for NodeHardware {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(3)?;
+        // These are "one-way": we can't deserialize again from them to MeanLists:
+        tup.serialize_element(self.upload.slice())?;
+        tup.serialize_element(self.download.slice())?;
+        tup.serialize_element(self.chart_stamps.slice())?;
+        tup.end()
+    }
+}
+
+/// Node location details
+#[derive(Debug, Clone, PartialEq)]
+pub struct NodeLocation {
+    pub latitude: f32,
+    pub longitude: f32,
+    pub city: Box<str>,
+}
+
+impl Serialize for NodeLocation {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(3)?;
+        tup.serialize_element(&self.latitude)?;
+        tup.serialize_element(&self.longitude)?;
+        tup.serialize_element(&&*self.city)?;
+        tup.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for NodeLocation {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let (latitude, longitude, city) = <(f32, f32, Box<str>)>::deserialize(deserializer)?;
+        Ok(NodeLocation {
+            latitude,
+            longitude,
+            city,
+        })
+    }
+}
+
+/// Verbose block details
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct BlockDetails {
+    pub block: Block,
+    pub block_time: u64,
+    pub block_timestamp: u64,
+    pub propagation_time: Option<u64>,
+}
+
+impl Default for BlockDetails {
+    fn default() -> Self {
+        BlockDetails {
+            block: Block::zero(),
+            block_timestamp: time::now(),
+            block_time: 0,
+            propagation_time: None,
+        }
+    }
+}
+
+impl Serialize for BlockDetails {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(5)?;
+        tup.serialize_element(&self.block.height)?;
+        tup.serialize_element(&self.block.hash)?;
+        tup.serialize_element(&self.block_time)?;
+        tup.serialize_element(&self.block_timestamp)?;
+        tup.serialize_element(&self.propagation_time)?;
+        tup.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for BlockDetails {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let tup = <(u64, BlockHash, u64, u64, Option<u64>)>::deserialize(deserializer)?;
+        Ok(BlockDetails {
+            block: Block {
+                height: tup.0,
+                hash: tup.1,
+            },
+            block_time: tup.2,
+            block_timestamp: tup.3,
+            propagation_time: tup.4,
+        })
+    }
+}

backend/common/src/num_stats.rs

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 use num_traits::{Bounded, NumOps, Zero};
 use std::convert::TryFrom;
 use std::iter::Sum;

backend/common/src/ready_chunks_all.rs

@@ -0,0 +1,121 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! [`futures::StreamExt::ready_chunks()`] internally stores a vec with a certain capacity, and will buffer up
+//! up to that many items that are ready from the underlying stream before returning either when we run out of
+//! Poll::Ready items, or we hit the capacity.
+//!
+//! This variation has no fixed capacity, and will buffer everything it can up at each point to return. This is
+//! better when the amount of items varies a bunch (and we don't want to allocate a fixed capacity every time),
+//! and can help ensure that we process as many items as possible each time (rather than only up to capacity items).
+//!
+//! Code is adapted from the futures implementation
+//! (see [ready_chunks.rs](https://docs.rs/futures-util/0.3.15/src/futures_util/stream/stream/ready_chunks.rs.html)).
+
+use core::mem;
+use core::pin::Pin;
+use futures::stream::Fuse;
+use futures::stream::{FusedStream, Stream};
+use futures::task::{Context, Poll};
+use futures::StreamExt;
+use pin_project_lite::pin_project;
+
+pin_project! {
+    /// Buffer up all Ready items in the underlying stream each time
+    /// we attempt to retrieve items from it, and return a Vec of those
+    /// items.
+    #[derive(Debug)]
+    #[must_use = "streams do nothing unless polled"]
+    pub struct ReadyChunksAll<St: Stream> {
+        #[pin]
+        stream: Fuse<St>,
+        items: Vec<St::Item>,
+    }
+}
+
+impl<St: Stream> ReadyChunksAll<St>
+where
+    St: Stream,
+{
+    pub fn new(stream: St) -> Self {
+        Self {
+            stream: stream.fuse(),
+            items: Vec::new(),
+        }
+    }
+}
+
+impl<St: Stream> Stream for ReadyChunksAll<St> {
+    type Item = Vec<St::Item>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let mut this = self.project();
+
+        loop {
+            match this.stream.as_mut().poll_next(cx) {
+                // Flush all collected data if underlying stream doesn't contain
+                // more ready values
+                Poll::Pending => {
+                    return if this.items.is_empty() {
+                        Poll::Pending
+                    } else {
+                        Poll::Ready(Some(mem::take(this.items)))
+                    }
+                }
+
+                // Push the ready item into the buffer
+                Poll::Ready(Some(item)) => {
+                    this.items.push(item);
+                }
+
+                // Since the underlying stream ran out of values, return what we
+                // have buffered, if we have anything.
+                Poll::Ready(None) => {
+                    let last = if this.items.is_empty() {
+                        None
+                    } else {
+                        let full_buf = mem::take(this.items);
+                        Some(full_buf)
+                    };
+
+                    return Poll::Ready(last);
+                }
+            }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        // Look at the underlying stream's size_hint. If we've
+        // buffered some items, we'll return at least that Vec,
+        // giving us a lower bound 1 greater than the underlying.
+        // The upper bound is, worst case, our vec + each individual
+        // item in the underlying stream.
+        let chunk_len = if self.items.is_empty() { 0 } else { 1 };
+        let (lower, upper) = self.stream.size_hint();
+        let lower = lower.saturating_add(chunk_len);
+        let upper = match upper {
+            Some(x) => x.checked_add(chunk_len),
+            None => None,
+        };
+        (lower, upper)
+    }
+}
+
+impl<St: FusedStream> FusedStream for ReadyChunksAll<St> {
+    fn is_terminated(&self) -> bool {
+        self.stream.is_terminated() && self.items.is_empty()
+    }
+}

backend/common/src/rolling_total.rs

@@ -0,0 +1,337 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use num_traits::{SaturatingAdd, SaturatingSub, Zero};
+use std::collections::VecDeque;
+use std::time::{Duration, Instant};
+
+/// Build an object responsible for keeping track of a rolling total.
+/// It does this in constant time and using memory proportional to the
+/// granularity * window size multiple that we set.
+pub struct RollingTotalBuilder<Time: TimeSource = SystemTimeSource> {
+    window_size_multiple: usize,
+    granularity: Duration,
+    time_source: Time,
+}
+
+impl RollingTotalBuilder {
+    /// Build a [`RollingTotal`] struct. By default,
+    /// the window_size is 10s, the granularity is 1s,
+    /// and system time is used.
+    pub fn new() -> RollingTotalBuilder<SystemTimeSource> {
+        Self {
+            window_size_multiple: 10,
+            granularity: Duration::from_secs(1),
+            time_source: SystemTimeSource,
+        }
+    }
+
+    /// Set the source of time we'll use. By default, we use system time.
+    pub fn time_source<Time: TimeSource>(self, val: Time) -> RollingTotalBuilder<Time> {
+        RollingTotalBuilder {
+            window_size_multiple: self.window_size_multiple,
+            granularity: self.granularity,
+            time_source: val,
+        }
+    }
+
+    /// Set the size of the window of time that we'll look back on
+    /// to sum up values over to give us the current total. The size
+    /// is set as a multiple of the granularity; a granulatiry of 1s
+    /// and a size of 10 means the window size will be 10 seconds.
+    pub fn window_size_multiple(mut self, val: usize) -> Self {
+        self.window_size_multiple = val;
+        self
+    }
+
+    /// What is the granulatiry of our windows of time. For example, a
+    /// granularity of 5 seconds means that every 5 seconds the window
+    /// that we look at shifts forward to the next 5 seconds worth of data.
+    /// A larger granularity is more efficient but less accurate than a
+    /// smaller one.
+    pub fn granularity(mut self, val: Duration) -> Self {
+        self.granularity = val;
+        self
+    }
+}
+
+impl<Time: TimeSource> RollingTotalBuilder<Time> {
+    /// Create a [`RollingTotal`] with these setings, starting from the
+    /// instant provided.
+    pub fn start<T>(self) -> RollingTotal<T, Time>
+    where
+        T: Zero + SaturatingAdd + SaturatingSub,
+    {
+        let mut averages = VecDeque::new();
+        averages.push_back((self.time_source.now(), T::zero()));
+
+        RollingTotal {
+            window_size_multiple: self.window_size_multiple,
+            time_source: self.time_source,
+            granularity: self.granularity,
+            averages,
+            total: T::zero(),
+        }
+    }
+}
+
+pub struct RollingTotal<Val, Time = SystemTimeSource> {
+    window_size_multiple: usize,
+    time_source: Time,
+    granularity: Duration,
+    averages: VecDeque<(Instant, Val)>,
+    total: Val,
+}
+
+impl<Val, Time: TimeSource> RollingTotal<Val, Time>
+where
+    Val: SaturatingAdd + SaturatingSub + Copy + std::fmt::Debug,
+    Time: TimeSource,
+{
+    /// Add a new value at some time.
+    pub fn push(&mut self, value: Val) {
+        let time = self.time_source.now();
+        let (last_time, last_val) = self.averages.back_mut().expect("always 1 value");
+
+        let since_last_nanos = time.duration_since(*last_time).as_nanos();
+        let granularity_nanos = self.granularity.as_nanos();
+
+        if since_last_nanos >= granularity_nanos {
+            // New time doesn't fit into last bucket; create a new bucket with a time
+            // that is some number of granularity steps from the last, and add the
+            // value to that.
+
+            // This rounds down, eg 7 / 5 = 1. Find the number of granularity steps
+            // to jump from the last time such that the jump can fit this new value.
+            let steps = since_last_nanos / granularity_nanos;
+
+            // Create a new time this number of jumps forward, and push it.
+            let new_time =
+                *last_time + Duration::from_nanos(granularity_nanos as u64) * steps as u32;
+            self.total = self.total.saturating_add(&value);
+            self.averages.push_back((new_time, value));
+
+            // Remove any old times/values no longer within our window size. If window_size_multiple
+            // is 1, then we only keep the just-pushed time, hence the "-1". Remember to keep our
+            // cached total up to date if we remove things.
+            let oldest_time_in_window =
+                new_time - (self.granularity * (self.window_size_multiple - 1) as u32);
+            while self.averages.front().expect("always 1 value").0 < oldest_time_in_window {
+                let value = self.averages.pop_front().expect("always 1 value").1;
+                self.total = self.total.saturating_sub(&value);
+            }
+        } else {
+            // New time fits into our last bucket, so just add it on. We don't need to worry
+            // about bucket cleanup since number/times of buckets hasn't changed.
+            *last_val = last_val.saturating_add(&value);
+            self.total = self.total.saturating_add(&value);
+        }
+    }
+
+    /// Fetch the current rolling total that we've accumulated. Note that this
+    /// is based on the last seen times and values, and is not affected by the time
+    /// that it is called.
+    pub fn total(&self) -> Val {
+        self.total
+    }
+
+    /// Fetch the current time source, in case we need to modify it.
+    pub fn time_source(&mut self) -> &mut Time {
+        &mut self.time_source
+    }
+
+    #[cfg(test)]
+    pub fn averages(&self) -> &VecDeque<(Instant, Val)> {
+        &self.averages
+    }
+}
+
+/// A source of time that we can use in our rolling total.
+/// This allows us to avoid explicitly mentioning time when pushing
+/// new values, and makes it a little harder to accidentally pass
+/// an older time and cause a panic.
+pub trait TimeSource {
+    fn now(&self) -> Instant;
+}
+
+pub struct SystemTimeSource;
+impl TimeSource for SystemTimeSource {
+    fn now(&self) -> Instant {
+        Instant::now()
+    }
+}
+
+pub struct UserTimeSource(Instant);
+impl UserTimeSource {
+    pub fn new(time: Instant) -> Self {
+        UserTimeSource(time)
+    }
+    pub fn set_time(&mut self, time: Instant) {
+        self.0 = time;
+    }
+    pub fn increment_by(&mut self, duration: Duration) {
+        self.0 += duration;
+    }
+}
+impl TimeSource for UserTimeSource {
+    fn now(&self) -> Instant {
+        self.0
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+
+    #[test]
+    fn times_grouped_by_granularity_spacing() {
+        let start_time = Instant::now();
+        let granularity = Duration::from_secs(1);
+        let mut rolling_total = RollingTotalBuilder::new()
+            .granularity(granularity)
+            .window_size_multiple(10)
+            .time_source(UserTimeSource(start_time))
+            .start();
+
+        rolling_total.push(1);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_millis(1210)); // 1210; bucket 1
+        rolling_total.push(2);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_millis(2500)); // 3710: bucket 3
+        rolling_total.push(3);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_millis(1100)); // 4810: bucket 4
+        rolling_total.push(4);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_millis(190)); // 5000: bucket 5
+        rolling_total.push(5);
+
+        // Regardless of the exact time that's elapsed, we'll end up with buckets that
+        // are exactly granularity spacing (or multiples of) apart.
+        assert_eq!(
+            rolling_total
+                .averages()
+                .into_iter()
+                .copied()
+                .collect::<Vec<_>>(),
+            vec![
+                (start_time, 1),
+                (start_time + granularity, 2),
+                (start_time + granularity * 3, 3),
+                (start_time + granularity * 4, 4),
+                (start_time + granularity * 5, 5),
+            ]
+        )
+    }
+
+    #[test]
+    fn gets_correct_total_within_granularity() {
+        let start_time = Instant::now();
+        let mut rolling_total = RollingTotalBuilder::new()
+            .granularity(Duration::from_secs(1))
+            .window_size_multiple(10)
+            .time_source(UserTimeSource(start_time))
+            .start();
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_millis(300));
+        rolling_total.push(1);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_millis(300));
+        rolling_total.push(10);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_millis(300));
+        rolling_total.push(-5);
+
+        assert_eq!(rolling_total.total(), 6);
+        assert_eq!(rolling_total.averages().len(), 1);
+    }
+
+    #[test]
+    fn gets_correct_total_within_window() {
+        let start_time = Instant::now();
+        let mut rolling_total = RollingTotalBuilder::new()
+            .granularity(Duration::from_secs(1))
+            .window_size_multiple(10)
+            .time_source(UserTimeSource(start_time))
+            .start();
+
+        rolling_total.push(4);
+
+        assert_eq!(rolling_total.averages().len(), 1);
+        assert_eq!(rolling_total.total(), 4);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_secs(3));
+        rolling_total.push(1);
+
+        assert_eq!(rolling_total.averages().len(), 2);
+        assert_eq!(rolling_total.total(), 5);
+
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_secs(1));
+        rolling_total.push(10);
+
+        assert_eq!(rolling_total.averages().len(), 3);
+        assert_eq!(rolling_total.total(), 15);
+
+        // Jump precisely to the end of the window. Now, pushing a
+        // value will displace the first one (4). Note: if no value
+        // is pushed, this time change will have no effect.
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_secs(8));
+        rolling_total.push(20);
+
+        assert_eq!(rolling_total.averages().len(), 3);
+        assert_eq!(rolling_total.total(), 15 + 20 - 4);
+
+        // Jump so that only the last value is still within the window:
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_secs(9));
+        rolling_total.push(1);
+
+        assert_eq!(rolling_total.averages().len(), 2);
+        assert_eq!(rolling_total.total(), 21);
+
+        // Jump so that everything is out of scope (just about!):
+        rolling_total
+            .time_source()
+            .increment_by(Duration::from_secs(10));
+        rolling_total.push(1);
+
+        assert_eq!(rolling_total.averages().len(), 1);
+        assert_eq!(rolling_total.total(), 1);
+    }
+}

backend/common/src/time.rs

@@ -0,0 +1,25 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/// Returns current unix time in ms (compatible with JS Date.now())
+pub fn now() -> u64 {
+    use std::time::SystemTime;
+
+    SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .expect("System time must be configured to be post Unix Epoch start; qed")
+        .as_millis() as u64
+}

backend/common/src/ws_client/connect.rs

@@ -0,0 +1,254 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use super::on_close::OnClose;
+use futures::channel::mpsc;
+use futures::{SinkExt, StreamExt};
+use soketto::handshake::{Client, ServerResponse};
+use std::sync::Arc;
+use tokio::net::TcpStream;
+use tokio_util::compat::TokioAsyncReadCompatExt;
+
+use super::{
+    receiver::{Receiver, RecvMessage},
+    sender::{Sender, SentMessage},
+};
+
+/// The send side of a Soketto WebSocket connection
+pub type RawSender = soketto::connection::Sender<tokio_util::compat::Compat<tokio::net::TcpStream>>;
+
+/// The receive side of a Soketto WebSocket connection
+pub type RawReceiver =
+    soketto::connection::Receiver<tokio_util::compat::Compat<tokio::net::TcpStream>>;
+
+/// A websocket connection. From this, we can either expose the raw connection
+/// or expose a cancel-safe interface to it.
+pub struct Connection {
+    tx: RawSender,
+    rx: RawReceiver,
+}
+
+impl Connection {
+    /// Get hold of the raw send/receive interface for this connection.
+    /// These are not cancel-safe, but can be more performant than the
+    /// cancel-safe channel based interface.
+    pub fn into_raw(self) -> (RawSender, RawReceiver) {
+        (self.tx, self.rx)
+    }
+
+    /// Get hold of send and receive channels for this connection.
+    /// These channels are cancel-safe.
+    ///
+    /// This spawns a couple of tasks for pulling/pushing messages onto the
+    /// connection, and so messages will be pushed onto the receiving channel
+    /// without any further polling. use [`Connection::into_raw`] if you need
+    /// more precise control over when messages are pulled from the socket.
+    ///
+    /// # Panics
+    ///
+    /// This will panic if not called within the context of a tokio runtime.
+    ///
+    pub fn into_channels(self) -> (Sender, Receiver) {
+        let (mut ws_to_connection, mut ws_from_connection) = (self.tx, self.rx);
+
+        // Shut everything down when we're told to close, which will be either when
+        // we hit an error trying to receive data on the socket, or when both the send
+        // and recv channels that we hand out are dropped. Notably, we allow either recv or
+        // send alone to be dropped and still keep the socket open (we may only care about
+        // one way communication).
+        let (tx_closed1, mut rx_closed1) = tokio::sync::broadcast::channel::<()>(1);
+        let tx_closed2 = tx_closed1.clone();
+        let mut rx_closed2 = tx_closed1.subscribe();
+
+        // Receive messages from the socket:
+        let (mut tx_to_external, rx_from_ws) = mpsc::unbounded();
+        tokio::spawn(async move {
+            let mut send_to_external = true;
+            loop {
+                let mut data = Vec::new();
+
+                // Wait for messages, or bail entirely if asked to close.
+                let message_data = tokio::select! {
+                    msg_data = ws_from_connection.receive_data(&mut data) => { msg_data },
+                    _ = rx_closed1.recv() => { break }
+                };
+
+                let message_data = match message_data {
+                    Err(e) => {
+                        // The socket had an error, so notify interested parties that we should
+                        // shut the connection down and bail out of this receive loop.
+                        log::error!(
+                            "Shutting down websocket connection: Failed to receive data: {}",
+                            e
+                        );
+                        let _ = tx_closed1.send(());
+                        break;
+                    }
+                    Ok(data) => data,
+                };
+
+                // if we hit an error sending, we keep receiving messages and reacting
+                // to recv issues, but we stop trying to send them anywhere.
+                if !send_to_external {
+                    continue;
+                }
+
+                let msg = match message_data {
+                    soketto::Data::Binary(_) => Ok(RecvMessage::Binary(data)),
+                    soketto::Data::Text(_) => String::from_utf8(data)
+                        .map(|s| RecvMessage::Text(s))
+                        .map_err(|e| e.into()),
+                };
+
+                if let Err(e) = tx_to_external.send(msg).await {
+                    // Our external channel may have closed or errored, but the socket hasn't
+                    // been closed, so keep receiving in order to allow the socket to continue to
+                    // function properly (we may be happy just sending messages to it), but stop
+                    // trying to hand back messages we've received from the socket.
+                    log::warn!("Failed to send data out: {}", e);
+                    send_to_external = false;
+                }
+            }
+        });
+
+        // Send messages to the socket:
+        let (tx_to_ws, mut rx_from_external) = mpsc::unbounded();
+        tokio::spawn(async move {
+            loop {
+                // Wait for messages, or bail entirely if asked to close.
+                let msg = tokio::select! {
+                    msg = rx_from_external.next() => { msg },
+                    _ = rx_closed2.recv() => {
+                        // attempt to gracefully end the connection.
+                        let _ = ws_to_connection.close().await;
+                        break
+                    }
+                };
+
+                // No more messages; channel closed. End this loop. Unlike the recv side which
+                // needs to keep receiving data for the WS connection to stay open, there's no
+                // reason to keep this side of the loop open if our channel is closed.
+                let msg = match msg {
+                    None => break,
+                    Some(msg) => msg,
+                };
+
+                // We don't explicitly shut down the channel if we hit send errors. Why? Because the
+                // receive side of the channel will react to socket errors as well, and close things
+                // down from there.
+                match msg {
+                    SentMessage::Text(s) => {
+                        if let Err(e) = ws_to_connection.send_text_owned(s).await {
+                            log::error!(
+                                "Shutting down websocket connection: Failed to send text data: {}",
+                                e
+                            );
+                            break;
+                        }
+                    }
+                    SentMessage::Binary(bytes) => {
+                        if let Err(e) = ws_to_connection.send_binary_mut(bytes).await {
+                            log::error!(
+                                "Shutting down websocket connection: Failed to send binary data: {}",
+                                e
+                            );
+                            break;
+                        }
+                    }
+                    SentMessage::StaticText(s) => {
+                        if let Err(e) = ws_to_connection.send_text(s).await {
+                            log::error!(
+                                "Shutting down websocket connection: Failed to send text data: {}",
+                                e
+                            );
+                            break;
+                        }
+                    }
+                    SentMessage::StaticBinary(bytes) => {
+                        if let Err(e) = ws_to_connection.send_binary(bytes).await {
+                            log::error!(
+                                "Shutting down websocket connection: Failed to send binary data: {}",
+                                e
+                            );
+                            break;
+                        }
+                    }
+                }
+
+                if let Err(e) = ws_to_connection.flush().await {
+                    log::error!(
+                        "Shutting down websocket connection: Failed to flush data: {}",
+                        e
+                    );
+                    break;
+                }
+            }
+        });
+
+        // Keep track of whether one of sender or received have
+        // been dropped. If both have, we close the socket connection.
+        let on_close = Arc::new(OnClose(tx_closed2));
+
+        (
+            Sender {
+                inner: tx_to_ws,
+                closer: Arc::clone(&on_close),
+            },
+            Receiver {
+                inner: rx_from_ws,
+                closer: on_close,
+            },
+        )
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum ConnectError {
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("Handshake error: {0}")]
+    Handshake(#[from] soketto::handshake::Error),
+    #[error("Redirect not supported (status code: {status_code})")]
+    ConnectionFailedRedirect { status_code: u16 },
+    #[error("Connection rejected (status code: {status_code})")]
+    ConnectionFailedRejected { status_code: u16 },
+}
+
+/// Establish a websocket connection that you can send and receive messages from.
+pub async fn connect(uri: &http::Uri) -> Result<Connection, ConnectError> {
+    let host = uri.host().unwrap_or("127.0.0.1");
+    let port = uri.port_u16().unwrap_or(80);
+    let path = uri.path();
+
+    let socket = TcpStream::connect((host, port)).await?;
+    socket.set_nodelay(true).expect("socket set_nodelay failed");
+
+    // Establish a WS connection:
+    let mut client = Client::new(socket.compat(), host, &path);
+    let (ws_to_connection, ws_from_connection) = match client.handshake().await? {
+        ServerResponse::Accepted { .. } => client.into_builder().finish(),
+        ServerResponse::Redirect { status_code, .. } => {
+            return Err(ConnectError::ConnectionFailedRedirect { status_code })
+        }
+        ServerResponse::Rejected { status_code } => {
+            return Err(ConnectError::ConnectionFailedRejected { status_code })
+        }
+    };
+
+    Ok(Connection {
+        tx: ws_to_connection,
+        rx: ws_from_connection,
+    })
+}

backend/common/src/ws_client/mod.rs

@@ -0,0 +1,28 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/// Functionality to establish a connection
+mod connect;
+/// A close helper that we use in sender/receiver.
+mod on_close;
+/// The channel based receive interface
+mod receiver;
+/// The channel based send interface
+mod sender;
+
+pub use connect::{connect, ConnectError, Connection, RawReceiver, RawSender};
+pub use receiver::{Receiver, RecvError, RecvMessage};
+pub use sender::{SendError, Sender, SentMessage};

backend/common/src/ws_client/on_close.rs

@@ -0,0 +1,26 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tokio::sync::broadcast;
+
+/// A small helper to fire the "close" channel when it's dropped.
+pub struct OnClose(pub broadcast::Sender<()>);
+
+impl Drop for OnClose {
+    fn drop(&mut self) {
+        let _ = self.0.send(());
+    }
+}

backend/common/src/ws_client/receiver.rs

@@ -0,0 +1,72 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::on_close::OnClose;
+use futures::channel::mpsc;
+use futures::{Stream, StreamExt};
+use std::sync::Arc;
+
+/// Receive messages out of a connection
+pub struct Receiver {
+    pub(super) inner: mpsc::UnboundedReceiver<Result<RecvMessage, RecvError>>,
+    pub(super) closer: Arc<OnClose>,
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum RecvError {
+    #[error("Text message contains invalid UTF8: {0}")]
+    InvalidUtf8(#[from] std::string::FromUtf8Error),
+    #[error("Stream finished")]
+    StreamFinished,
+    #[error("Failed to send close message")]
+    CloseError,
+}
+
+impl Receiver {
+    /// Ask the underlying Websocket connection to close.
+    pub async fn close(&mut self) -> Result<(), RecvError> {
+        self.closer.0.send(()).map_err(|_| RecvError::CloseError)?;
+        Ok(())
+    }
+}
+
+impl Stream for Receiver {
+    type Item = Result<RecvMessage, RecvError>;
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        self.inner.poll_next_unpin(cx).map_err(|e| e.into())
+    }
+}
+
+/// A message that can be received from the channel interface
+#[derive(Debug, Clone)]
+pub enum RecvMessage {
+    /// Send an owned string into the socket.
+    Text(String),
+    /// Send owned bytes into the socket.
+    Binary(Vec<u8>),
+}
+
+impl RecvMessage {
+    pub fn len(&self) -> usize {
+        match self {
+            RecvMessage::Binary(b) => b.len(),
+            RecvMessage::Text(s) => s.len(),
+        }
+    }
+}

backend/common/src/ws_client/sender.rs

@@ -0,0 +1,102 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::on_close::OnClose;
+use futures::channel::mpsc;
+use futures::{Sink, SinkExt};
+use std::sync::Arc;
+
+/// A message that can be sent into the channel interface
+#[derive(Debug, Clone)]
+pub enum SentMessage {
+    /// Being able to send static text is primarily useful for benchmarking,
+    /// so that we can avoid cloning an owned string and pass a static reference
+    /// (one such option here is using [`Box::leak`] to generate strings with
+    /// static lifetimes).
+    StaticText(&'static str),
+    /// Being able to send static bytes is primarily useful for benchmarking,
+    /// so that we can avoid cloning an owned string and pass a static reference
+    /// (one such option here is using [`Box::leak`] to generate bytes with
+    /// static lifetimes).
+    StaticBinary(&'static [u8]),
+    /// Send an owned string into the socket.
+    Text(String),
+    /// Send owned bytes into the socket.
+    Binary(Vec<u8>),
+}
+
+/// Send messages into the connection
+#[derive(Clone)]
+pub struct Sender {
+    pub(super) inner: mpsc::UnboundedSender<SentMessage>,
+    pub(super) closer: Arc<OnClose>,
+}
+
+impl Sender {
+    /// Ask the underlying Websocket connection to close.
+    pub async fn close(&mut self) -> Result<(), SendError> {
+        self.closer.0.send(()).map_err(|_| SendError::CloseError)?;
+        Ok(())
+    }
+    /// Returns whether this channel is closed.
+    pub fn is_closed(&self) -> bool {
+        self.inner.is_closed()
+    }
+    /// Unbounded send will always queue the message and doesn't
+    /// need to be awaited.
+    pub fn unbounded_send(&self, msg: SentMessage) -> Result<(), SendError> {
+        self.inner
+            .unbounded_send(msg)
+            .map_err(|e| e.into_send_error())?;
+        Ok(())
+    }
+}
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum SendError {
+    #[error("Failed to send message: {0}")]
+    ChannelError(#[from] mpsc::SendError),
+    #[error("Failed to send close message")]
+    CloseError,
+}
+
+impl Sink<SentMessage> for Sender {
+    type Error = SendError;
+    fn poll_ready(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready_unpin(cx).map_err(|e| e.into())
+    }
+    fn start_send(
+        mut self: std::pin::Pin<&mut Self>,
+        item: SentMessage,
+    ) -> Result<(), Self::Error> {
+        self.inner.start_send_unpin(item).map_err(|e| e.into())
+    }
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.inner.poll_flush_unpin(cx).map_err(|e| e.into())
+    }
+    fn poll_close(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.inner.poll_close_unpin(cx).map_err(|e| e.into())
+    }
+}

backend/core/Cargo.toml

@@ -1,29 +0,0 @@
-[package]
-name = "telemetry"
-version = "0.3.0"
-authors = ["Parity Technologies Ltd. <admin@parity.io>"]
-edition = "2018"
-license = "GPL-3.0"
-
-[dependencies]
-actix = "0.11.1"
-actix-web = { version = "4.0.0-beta.4", default-features = false }
-actix-web-actors = "4.0.0-beta.3"
-actix-http = "3.0.0-beta.4"
-bincode = "1.3.3"
-bytes = "1.0.1"
-chrono = { version = "0.4", features = ["serde"] }
-fnv = "1.0.7"
-hex = "0.4.3"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0", features = ["raw_value"] }
-thiserror = "1.0.24"
-primitive-types = { version = "0.9.0", features = ["serde"] }
-log = "0.4"
-simple_logger = "1.11.0"
-num-traits = "0.2"
-parking_lot = "0.11"
-reqwest = { version = "0.11.1", features = ["blocking", "json"] }
-rustc-hash = "1.1.0"
-clap = "3.0.0-beta.2"
-ctor = "0.1.20"

backend/core/src/aggregator.rs

@@ -1,383 +0,0 @@
-use actix::prelude::*;
-use actix_web_actors::ws::{CloseCode, CloseReason};
-use ctor::ctor;
-use std::collections::{HashMap, HashSet};
-
-use crate::chain::{self, Chain, ChainId, Label};
-use crate::feed::connector::{Connected, FeedConnector, FeedId};
-use crate::feed::{self, FeedMessageSerializer};
-use crate::node::connector::{Mute, NodeConnector};
-use crate::types::{ConnId, NodeDetails};
-use crate::util::{DenseMap, Hash};
-
-pub struct Aggregator {
-    genesis_hashes: HashMap<Hash, ChainId>,
-    labels: HashMap<Label, ChainId>,
-    chains: DenseMap<ChainEntry>,
-    feeds: DenseMap<Addr<FeedConnector>>,
-    serializer: FeedMessageSerializer,
-    /// Denylist for networks we do not want to allow connecting.
-    denylist: HashSet<String>,
-}
-
-pub struct ChainEntry {
-    /// Address to the `Chain` agent
-    addr: Addr<Chain>,
-    /// Genesis [`Hash`] of the chain
-    genesis_hash: Hash,
-    /// String name of the chain
-    label: Label,
-    /// Node count
-    nodes: usize,
-    /// Maximum allowed nodes
-    max_nodes: usize,
-}
-
-#[ctor]
-/// Labels of chains we consider "first party". These chains allow any
-/// number of nodes to connect.
-static FIRST_PARTY_NETWORKS: HashSet<&'static str> = {
-    let mut set = HashSet::new();
-    set.insert("Polkadot");
-    set.insert("Kusama");
-    set.insert("Westend");
-    set.insert("Rococo");
-    set
-};
-
-/// Max number of nodes allowed to connect to the telemetry server.
-const THIRD_PARTY_NETWORKS_MAX_NODES: usize = 500;
-
-impl Aggregator {
-    pub fn new(denylist: HashSet<String>) -> Self {
-        Aggregator {
-            genesis_hashes: HashMap::new(),
-            labels: HashMap::new(),
-            chains: DenseMap::new(),
-            feeds: DenseMap::new(),
-            serializer: FeedMessageSerializer::new(),
-            denylist,
-        }
-    }
-
-    /// Get an address to the chain actor by name. If the address is not found,
-    /// or the address is disconnected (actor dropped), create a new one.
-    pub fn lazy_chain(
-        &mut self,
-        genesis_hash: Hash,
-        label: &str,
-        ctx: &mut <Self as Actor>::Context,
-    ) -> ChainId {
-        let cid = match self.genesis_hashes.get(&genesis_hash).copied() {
-            Some(cid) => cid,
-            None => {
-                self.serializer.push(feed::AddedChain(&label, 1));
-
-                let addr = ctx.address();
-                let max_nodes = max_nodes(label);
-                let label: Label = label.into();
-                let cid = self.chains.add_with(|cid| ChainEntry {
-                    addr: Chain::new(cid, addr, label.clone()).start(),
-                    genesis_hash,
-                    label: label.clone(),
-                    nodes: 1,
-                    max_nodes,
-                });
-
-                self.labels.insert(label, cid);
-                self.genesis_hashes.insert(genesis_hash, cid);
-
-                self.broadcast();
-
-                cid
-            }
-        };
-
-        cid
-    }
-
-    fn get_chain(&mut self, label: &str) -> Option<&mut ChainEntry> {
-        let chains = &mut self.chains;
-        self.labels
-            .get(label)
-            .and_then(move |&cid| chains.get_mut(cid))
-    }
-
-    fn broadcast(&mut self) {
-        if let Some(msg) = self.serializer.finalize() {
-            for (_, feed) in self.feeds.iter() {
-                feed.do_send(msg.clone());
-            }
-        }
-    }
-}
-
-impl Actor for Aggregator {
-    type Context = Context<Self>;
-}
-
-/// Message sent from the NodeConnector to the Aggregator upon getting all node details
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct AddNode {
-    /// Details of the node being added to the aggregator
-    pub node: NodeDetails,
-    /// Genesis [`Hash`] of the chain the node is being added to.
-    pub genesis_hash: Hash,
-    /// Connection id used by the node connector for multiplexing parachains
-    pub conn_id: ConnId,
-    /// Address of the NodeConnector actor
-    pub node_connector: Addr<NodeConnector>,
-}
-
-/// Message sent from the Chain to the Aggregator when the Chain loses all nodes
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct DropChain(pub ChainId);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct RenameChain(pub ChainId, pub Label);
-
-/// Message sent from the FeedConnector to the Aggregator when subscribing to a new chain
-#[derive(Message)]
-#[rtype(result = "bool")]
-pub struct Subscribe {
-    pub chain: Label,
-    pub feed: Addr<FeedConnector>,
-}
-
-/// Message sent from the FeedConnector to the Aggregator consensus requested
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct SendFinality {
-    pub chain: Label,
-    pub fid: FeedId,
-}
-
-/// Message sent from the FeedConnector to the Aggregator no more consensus required
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct NoMoreFinality {
-    pub chain: Label,
-    pub fid: FeedId,
-}
-
-/// Message sent from the FeedConnector to the Aggregator when first connected
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Connect(pub Addr<FeedConnector>);
-
-/// Message sent from the FeedConnector to the Aggregator when disconnecting
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Disconnect(pub FeedId);
-
-/// Message sent from the Chain to the Aggergator when the node count on the chain changes
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct NodeCount(pub ChainId, pub usize);
-
-/// Message sent to the Aggregator to get a health check
-#[derive(Message)]
-#[rtype(result = "usize")]
-pub struct GetHealth;
-
-impl Handler<AddNode> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
-        if self.denylist.contains(&*msg.node.chain) {
-            log::warn!(target: "Aggregator::AddNode", "'{}' is on the denylist.", msg.node.chain);
-            let AddNode { node_connector, .. } = msg;
-            let reason = CloseReason {
-                code: CloseCode::Abnormal,
-                description: Some("Denied".into()),
-            };
-            node_connector.do_send(Mute { reason });
-            return;
-        }
-        let AddNode {
-            node,
-            genesis_hash,
-            conn_id,
-            node_connector,
-        } = msg;
-        log::trace!(target: "Aggregator::AddNode", "New node connected. Chain '{}'", node.chain);
-
-        let cid = self.lazy_chain(genesis_hash, &node.chain, ctx);
-        let chain = self
-            .chains
-            .get_mut(cid)
-            .expect("Entry just created above; qed");
-        if chain.nodes < chain.max_nodes {
-            chain.addr.do_send(chain::AddNode {
-                node,
-                conn_id,
-                node_connector,
-            });
-        } else {
-            log::warn!(target: "Aggregator::AddNode", "Chain {} is over quota ({})", chain.label, chain.max_nodes);
-            let reason = CloseReason {
-                code: CloseCode::Again,
-                description: Some("Overquota".into()),
-            };
-            node_connector.do_send(Mute { reason });
-        }
-    }
-}
-
-impl Handler<DropChain> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: DropChain, _: &mut Self::Context) {
-        let DropChain(cid) = msg;
-
-        if let Some(entry) = self.chains.remove(cid) {
-            let label = &entry.label;
-            self.genesis_hashes.remove(&entry.genesis_hash);
-            self.labels.remove(label);
-            self.serializer.push(feed::RemovedChain(label));
-            log::info!("Dropped chain [{}] from the aggregator", label);
-            self.broadcast();
-        }
-    }
-}
-
-impl Handler<RenameChain> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: RenameChain, _: &mut Self::Context) {
-        let RenameChain(cid, new) = msg;
-
-        if let Some(entry) = self.chains.get_mut(cid) {
-            if entry.label == new {
-                return;
-            }
-
-            // Update UI
-            self.serializer.push(feed::RemovedChain(&entry.label));
-            self.serializer.push(feed::AddedChain(&new, entry.nodes));
-
-            // Update labels -> cid map
-            self.labels.remove(&entry.label);
-            self.labels.insert(new.clone(), cid);
-
-            // Update entry
-            entry.label = new;
-
-            self.broadcast();
-        }
-    }
-}
-
-impl Handler<Subscribe> for Aggregator {
-    type Result = bool;
-
-    fn handle(&mut self, msg: Subscribe, _: &mut Self::Context) -> bool {
-        let Subscribe { chain, feed } = msg;
-
-        if let Some(chain) = self.get_chain(&chain) {
-            chain.addr.do_send(chain::Subscribe(feed));
-            true
-        } else {
-            false
-        }
-    }
-}
-
-impl Handler<SendFinality> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: SendFinality, _: &mut Self::Context) {
-        let SendFinality { chain, fid } = msg;
-        if let Some(chain) = self.get_chain(&chain) {
-            chain.addr.do_send(chain::SendFinality(fid));
-        }
-    }
-}
-
-impl Handler<NoMoreFinality> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: NoMoreFinality, _: &mut Self::Context) {
-        let NoMoreFinality { chain, fid } = msg;
-        if let Some(chain) = self.get_chain(&chain) {
-            chain.addr.do_send(chain::NoMoreFinality(fid));
-        }
-    }
-}
-
-impl Handler<Connect> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: Connect, _: &mut Self::Context) {
-        let Connect(connector) = msg;
-
-        let fid = self.feeds.add(connector.clone());
-
-        log::info!("Feed #{} connected", fid);
-
-        connector.do_send(Connected(fid));
-
-        self.serializer.push(feed::Version(31));
-
-        // TODO: keep track on number of nodes connected to each chain
-        for (_, entry) in self.chains.iter() {
-            self.serializer
-                .push(feed::AddedChain(&entry.label, entry.nodes));
-        }
-
-        if let Some(msg) = self.serializer.finalize() {
-            connector.do_send(msg);
-        }
-    }
-}
-
-impl Handler<Disconnect> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: Disconnect, _: &mut Self::Context) {
-        let Disconnect(fid) = msg;
-
-        log::info!("Feed #{} disconnected", fid);
-
-        self.feeds.remove(fid);
-    }
-}
-
-impl Handler<NodeCount> for Aggregator {
-    type Result = ();
-
-    fn handle(&mut self, msg: NodeCount, _: &mut Self::Context) {
-        let NodeCount(cid, count) = msg;
-
-        if let Some(entry) = self.chains.get_mut(cid) {
-            entry.nodes = count;
-
-            if count != 0 {
-                self.serializer.push(feed::AddedChain(&entry.label, count));
-                self.broadcast();
-            }
-        }
-    }
-}
-
-impl Handler<GetHealth> for Aggregator {
-    type Result = usize;
-
-    fn handle(&mut self, _: GetHealth, _: &mut Self::Context) -> Self::Result {
-        self.chains.len()
-    }
-}
-
-/// First party networks (Polkadot, Kusama etc) are allowed any number of nodes.
-/// Third party networks are allowed `THIRD_PARTY_NETWORKS_MAX_NODES` nodes and
-/// no more.
-fn max_nodes(label: &str) -> usize {
-    if FIRST_PARTY_NETWORKS.contains(label) {
-        usize::MAX
-    } else {
-        THIRD_PARTY_NETWORKS_MAX_NODES
-    }
-}

backend/core/src/chain.rs

@@ -1,564 +0,0 @@
-use actix::prelude::*;
-use rustc_hash::FxHashMap;
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use crate::aggregator::{Aggregator, DropChain, NodeCount, RenameChain};
-use crate::feed::connector::{FeedConnector, FeedId, Subscribed, Unsubscribed};
-use crate::feed::{self, FeedMessageSerializer};
-use crate::node::{
-    connector::{Initialize, NodeConnector},
-    message::Payload,
-    Node,
-};
-use crate::types::{Block, BlockNumber, ConnId, NodeDetails, NodeId, NodeLocation, Timestamp};
-use crate::util::{now, DenseMap, NumStats};
-
-const STALE_TIMEOUT: u64 = 2 * 60 * 1000; // 2 minutes
-
-pub type ChainId = usize;
-pub type Label = Arc<str>;
-
-pub struct Chain {
-    cid: ChainId,
-    /// Who to inform if the Chain drops itself
-    aggregator: Addr<Aggregator>,
-    /// Label of this chain, along with count of nodes that use this label
-    label: (Label, usize),
-    /// Dense mapping of NodeId -> Node
-    nodes: DenseMap<Node>,
-    /// Dense mapping of FeedId -> Addr<FeedConnector>,
-    feeds: DenseMap<Addr<FeedConnector>>,
-    /// Mapping of FeedId -> Addr<FeedConnector> for feeds requiring finality info,
-    finality_feeds: FxHashMap<FeedId, Addr<FeedConnector>>,
-    /// Best block
-    best: Block,
-    /// Finalized block
-    finalized: Block,
-    /// Block times history, stored so we can calculate averages
-    block_times: NumStats<u64>,
-    /// Calculated average block time
-    average_block_time: Option<u64>,
-    /// Message serializer
-    serializer: FeedMessageSerializer,
-    /// When the best block first arrived
-    timestamp: Option<Timestamp>,
-    /// Some nodes might manifest a different label, note them here
-    labels: HashMap<Label, usize>,
-}
-
-impl Chain {
-    pub fn new(cid: ChainId, aggregator: Addr<Aggregator>, label: Label) -> Self {
-        log::info!("[{}] Created", label);
-
-        Chain {
-            cid,
-            aggregator,
-            label: (label, 0),
-            nodes: DenseMap::new(),
-            feeds: DenseMap::new(),
-            finality_feeds: FxHashMap::default(),
-            best: Block::zero(),
-            finalized: Block::zero(),
-            block_times: NumStats::new(50),
-            average_block_time: None,
-            serializer: FeedMessageSerializer::new(),
-            timestamp: None,
-            labels: HashMap::default(),
-        }
-    }
-
-    fn increment_label_count(&mut self, label: &str) {
-        let count = match self.labels.get_mut(label) {
-            Some(count) => {
-                *count += 1;
-                *count
-            }
-            None => {
-                self.labels.insert(label.into(), 1);
-                1
-            }
-        };
-
-        if &*self.label.0 == label {
-            self.label.1 += 1;
-        } else if count > self.label.1 {
-            self.rename(label.into(), count);
-        }
-    }
-
-    fn decrement_label_count(&mut self, label: &str) {
-        match self.labels.get_mut(label) {
-            Some(count) => *count -= 1,
-            None => return,
-        };
-
-        if &*self.label.0 == label {
-            self.label.1 -= 1;
-
-            for (label, &count) in self.labels.iter() {
-                if count > self.label.1 {
-                    let label: Arc<_> = label.clone();
-                    self.rename(label, count);
-                    break;
-                }
-            }
-        }
-    }
-
-    fn rename(&mut self, label: Label, count: usize) {
-        self.label = (label, count);
-
-        self.aggregator
-            .do_send(RenameChain(self.cid, self.label.0.clone()));
-    }
-
-    fn broadcast(&mut self) {
-        if let Some(msg) = self.serializer.finalize() {
-            for (_, feed) in self.feeds.iter() {
-                feed.do_send(msg.clone());
-            }
-        }
-    }
-
-    fn broadcast_finality(&mut self) {
-        if let Some(msg) = self.serializer.finalize() {
-            for feed in self.finality_feeds.values() {
-                feed.do_send(msg.clone());
-            }
-        }
-    }
-
-    /// Triggered when the number of nodes in this chain has changed, Aggregator will
-    /// propagate new counts to all connected feeds
-    fn update_count(&self) {
-        self.aggregator
-            .do_send(NodeCount(self.cid, self.nodes.len()));
-    }
-
-    /// Check if the chain is stale (has not received a new best block in a while).
-    /// If so, find a new best block, ignoring any stale nodes and marking them as such.
-    fn update_stale_nodes(&mut self, now: u64) {
-        let threshold = now - STALE_TIMEOUT;
-        let timestamp = match self.timestamp {
-            Some(ts) => ts,
-            None => return,
-        };
-
-        if timestamp > threshold {
-            // Timestamp is in range, nothing to do
-            return;
-        }
-
-        let mut best = Block::zero();
-        let mut finalized = Block::zero();
-        let mut timestamp = None;
-
-        for (nid, node) in self.nodes.iter_mut() {
-            if !node.update_stale(threshold) {
-                if node.best().height > best.height {
-                    best = *node.best();
-                    timestamp = Some(node.best_timestamp());
-                }
-
-                if node.finalized().height > finalized.height {
-                    finalized = *node.finalized();
-                }
-            } else {
-                self.serializer.push(feed::StaleNode(nid));
-            }
-        }
-
-        if self.best.height != 0 || self.finalized.height != 0 {
-            self.best = best;
-            self.finalized = finalized;
-            self.block_times.reset();
-            self.timestamp = timestamp;
-
-            self.serializer.push(feed::BestBlock(
-                self.best.height,
-                timestamp.unwrap_or(now),
-                None,
-            ));
-            self.serializer
-                .push(feed::BestFinalized(finalized.height, finalized.hash));
-        }
-    }
-}
-
-impl Actor for Chain {
-    type Context = Context<Self>;
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        self.aggregator.do_send(DropChain(self.cid));
-
-        for (_, feed) in self.feeds.iter() {
-            feed.do_send(Unsubscribed)
-        }
-    }
-}
-
-/// Message sent from the Aggregator to the Chain when new Node is connected
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct AddNode {
-    /// Details of the node being added to the aggregator
-    pub node: NodeDetails,
-    /// Connection id used by the node connector for multiplexing parachains
-    pub conn_id: ConnId,
-    /// Address of the NodeConnector actor to which we send [`Initialize`] or [`Mute`] messages.
-    pub node_connector: Addr<NodeConnector>,
-}
-
-/// Message sent from the NodeConnector to the Chain when it receives new telemetry data
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct UpdateNode {
-    pub nid: NodeId,
-    pub payload: Payload,
-}
-
-/// Message sent from the NodeConnector to the Chain when the connector disconnects
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct RemoveNode(pub NodeId);
-
-/// Message sent from the Aggregator to the Chain when the connector wants to subscribe to that chain
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Subscribe(pub Addr<FeedConnector>);
-
-/// Message sent from the FeedConnector before it subscribes to a new chain, or if it disconnects
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Unsubscribe(pub FeedId);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct SendFinality(pub FeedId);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct NoMoreFinality(pub FeedId);
-
-/// Message sent from the NodeConnector to the Chain when it receives location data
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct LocateNode {
-    pub nid: NodeId,
-    pub location: Arc<NodeLocation>,
-}
-
-impl Handler<AddNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
-        let AddNode {
-            node,
-            conn_id,
-            node_connector,
-        } = msg;
-        log::trace!(target: "Chain::AddNode", "New node connected. Chain '{}', node count goes from {} to {}", node.chain, self.nodes.len(), self.nodes.len() + 1);
-        self.increment_label_count(&node.chain);
-
-        let nid = self.nodes.add(Node::new(node));
-        let chain = ctx.address();
-
-        if node_connector
-            .try_send(Initialize {
-                nid,
-                conn_id,
-                chain,
-            })
-            .is_err()
-        {
-            self.nodes.remove(nid);
-        } else if let Some(node) = self.nodes.get(nid) {
-            self.serializer.push(feed::AddedNode(nid, node));
-            self.broadcast();
-        }
-
-        self.update_count();
-    }
-}
-
-impl Chain {
-    fn handle_block(&mut self, block: &Block, nid: NodeId) {
-        let mut propagation_time = None;
-        let now = now();
-        let nodes_len = self.nodes.len();
-
-        self.update_stale_nodes(now);
-
-        let node = match self.nodes.get_mut(nid) {
-            Some(node) => node,
-            None => return,
-        };
-
-        if node.update_block(*block) {
-            if block.height > self.best.height {
-                self.best = *block;
-                log::debug!(
-                    "[{}] [nodes={}/feeds={}] new best block={}/{:?}",
-                    self.label.0,
-                    nodes_len,
-                    self.feeds.len(),
-                    self.best.height,
-                    self.best.hash,
-                );
-                if let Some(timestamp) = self.timestamp {
-                    self.block_times.push(now - timestamp);
-                    self.average_block_time = Some(self.block_times.average());
-                }
-                self.timestamp = Some(now);
-                self.serializer.push(feed::BestBlock(
-                    self.best.height,
-                    now,
-                    self.average_block_time,
-                ));
-                propagation_time = Some(0);
-            } else if block.height == self.best.height {
-                if let Some(timestamp) = self.timestamp {
-                    propagation_time = Some(now - timestamp);
-                }
-            }
-
-            if let Some(details) = node.update_details(now, propagation_time) {
-                self.serializer.push(feed::ImportedBlock(nid, details));
-            }
-        }
-    }
-}
-
-impl Handler<UpdateNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: UpdateNode, _: &mut Self::Context) {
-        let UpdateNode { nid, payload } = msg;
-
-        if let Some(block) = payload.best_block() {
-            self.handle_block(block, nid);
-        }
-
-        if let Some(node) = self.nodes.get_mut(nid) {
-            match payload {
-                Payload::SystemInterval(ref interval) => {
-                    if node.update_hardware(interval) {
-                        self.serializer.push(feed::Hardware(nid, node.hardware()));
-                    }
-
-                    if let Some(stats) = node.update_stats(interval) {
-                        self.serializer.push(feed::NodeStatsUpdate(nid, stats));
-                    }
-
-                    if let Some(io) = node.update_io(interval) {
-                        self.serializer.push(feed::NodeIOUpdate(nid, io));
-                    }
-                }
-                Payload::AfgAuthoritySet(authority) => {
-                    node.set_validator_address(authority.authority_id.clone());
-                    self.broadcast();
-                    return;
-                }
-                Payload::AfgFinalized(finalized) => {
-                    if let Ok(finalized_number) = finalized.finalized_number.parse::<BlockNumber>()
-                    {
-                        if let Some(addr) = node.details().validator.clone() {
-                            self.serializer.push(feed::AfgFinalized(
-                                addr,
-                                finalized_number,
-                                finalized.finalized_hash,
-                            ));
-                            self.broadcast_finality();
-                        }
-                    }
-                    return;
-                }
-                Payload::AfgReceivedPrecommit(precommit) => {
-                    if let Ok(finalized_number) =
-                        precommit.received.target_number.parse::<BlockNumber>()
-                    {
-                        if let Some(addr) = node.details().validator.clone() {
-                            let voter = precommit.received.voter.clone();
-                            self.serializer.push(feed::AfgReceivedPrecommit(
-                                addr,
-                                finalized_number,
-                                precommit.received.target_hash,
-                                voter,
-                            ));
-                            self.broadcast_finality();
-                        }
-                    }
-                    return;
-                }
-                Payload::AfgReceivedPrevote(prevote) => {
-                    if let Ok(finalized_number) =
-                        prevote.received.target_number.parse::<BlockNumber>()
-                    {
-                        if let Some(addr) = node.details().validator.clone() {
-                            let voter = prevote.received.voter.clone();
-                            self.serializer.push(feed::AfgReceivedPrevote(
-                                addr,
-                                finalized_number,
-                                prevote.received.target_hash,
-                                voter,
-                            ));
-                            self.broadcast_finality();
-                        }
-                    }
-                    return;
-                }
-                Payload::AfgReceivedCommit(_) => {}
-                _ => (),
-            }
-
-            if let Some(block) = payload.finalized_block() {
-                if let Some(finalized) = node.update_finalized(block) {
-                    self.serializer.push(feed::FinalizedBlock(
-                        nid,
-                        finalized.height,
-                        finalized.hash,
-                    ));
-
-                    if finalized.height > self.finalized.height {
-                        self.finalized = *finalized;
-                        self.serializer
-                            .push(feed::BestFinalized(finalized.height, finalized.hash));
-                    }
-                }
-            }
-        }
-
-        self.broadcast();
-    }
-}
-
-impl Handler<LocateNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: LocateNode, _: &mut Self::Context) {
-        let LocateNode { nid, location } = msg;
-
-        if let Some(node) = self.nodes.get_mut(nid) {
-            self.serializer.push(feed::LocatedNode(
-                nid,
-                location.latitude,
-                location.longitude,
-                &location.city,
-            ));
-
-            node.update_location(location);
-        }
-    }
-}
-
-impl Handler<RemoveNode> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: RemoveNode, ctx: &mut Self::Context) {
-        let RemoveNode(nid) = msg;
-
-        if let Some(node) = self.nodes.remove(nid) {
-            self.decrement_label_count(&node.details().chain);
-        }
-
-        if self.nodes.is_empty() {
-            log::info!("[{}] Lost all nodes, dropping...", self.label.0);
-            ctx.stop();
-        }
-
-        self.serializer.push(feed::RemovedNode(nid));
-        self.broadcast();
-        self.update_count();
-    }
-}
-
-impl Handler<Subscribe> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: Subscribe, ctx: &mut Self::Context) {
-        let Subscribe(feed) = msg;
-        let fid = self.feeds.add(feed.clone());
-
-        feed.do_send(Subscribed(fid, ctx.address().recipient()));
-
-        self.serializer.push(feed::SubscribedTo(&self.label.0));
-        self.serializer.push(feed::TimeSync(now()));
-        self.serializer.push(feed::BestBlock(
-            self.best.height,
-            self.timestamp.unwrap_or(0),
-            self.average_block_time,
-        ));
-        self.serializer.push(feed::BestFinalized(
-            self.finalized.height,
-            self.finalized.hash,
-        ));
-
-        for (idx, (nid, node)) in self.nodes.iter().enumerate() {
-            // Send subscription confirmation and chain head before doing all the nodes,
-            // and continue sending batches of 32 nodes a time over the wire subsequently
-            if idx % 32 == 0 {
-                if let Some(serialized) = self.serializer.finalize() {
-                    feed.do_send(serialized);
-                }
-            }
-
-            self.serializer.push(feed::AddedNode(nid, node));
-            self.serializer.push(feed::FinalizedBlock(
-                nid,
-                node.finalized().height,
-                node.finalized().hash,
-            ));
-            if node.stale() {
-                self.serializer.push(feed::StaleNode(nid));
-            }
-        }
-
-        if let Some(serialized) = self.serializer.finalize() {
-            feed.do_send(serialized);
-        }
-    }
-}
-
-impl Handler<SendFinality> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: SendFinality, _ctx: &mut Self::Context) {
-        let SendFinality(fid) = msg;
-        if let Some(feed) = self.feeds.get(fid) {
-            self.finality_feeds.insert(fid, feed.clone());
-        }
-
-        // info!("Added new finality feed {}", fid);
-    }
-}
-
-impl Handler<NoMoreFinality> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: NoMoreFinality, _: &mut Self::Context) {
-        let NoMoreFinality(fid) = msg;
-
-        // info!("Removed finality feed {}", fid);
-        self.finality_feeds.remove(&fid);
-    }
-}
-
-impl Handler<Unsubscribe> for Chain {
-    type Result = ();
-
-    fn handle(&mut self, msg: Unsubscribe, _: &mut Self::Context) {
-        let Unsubscribe(fid) = msg;
-
-        if let Some(feed) = self.feeds.get(fid) {
-            self.serializer.push(feed::UnsubscribedFrom(&self.label.0));
-
-            if let Some(serialized) = self.serializer.finalize() {
-                feed.do_send(serialized);
-            }
-        }
-
-        self.feeds.remove(fid);
-        self.finality_feeds.remove(&fid);
-    }
-}

backend/core/src/feed.rs

@@ -1,195 +0,0 @@
-use serde::ser::{SerializeTuple, Serializer};
-use serde::Serialize;
-use std::mem;
-
-use crate::node::Node;
-use crate::types::{
-    Address, BlockDetails, BlockHash, BlockNumber, NodeHardware, NodeIO, NodeId, NodeStats,
-    Timestamp,
-};
-use serde_json::to_writer;
-
-pub mod connector;
-
-use connector::Serialized;
-
-pub trait FeedMessage: Serialize {
-    const ACTION: u8;
-}
-
-pub struct FeedMessageSerializer {
-    /// Current buffer,
-    buffer: Vec<u8>,
-}
-
-const BUFCAP: usize = 128;
-
-impl FeedMessageSerializer {
-    pub fn new() -> Self {
-        Self {
-            buffer: Vec::with_capacity(BUFCAP),
-        }
-    }
-
-    pub fn push<Message>(&mut self, msg: Message)
-    where
-        Message: FeedMessage,
-    {
-        let glue = match self.buffer.len() {
-            0 => b'[',
-            _ => b',',
-        };
-
-        self.buffer.push(glue);
-        let _ = to_writer(&mut self.buffer, &Message::ACTION);
-        self.buffer.push(b',');
-        let _ = to_writer(&mut self.buffer, &msg);
-    }
-
-    pub fn finalize(&mut self) -> Option<Serialized> {
-        if self.buffer.is_empty() {
-            return None;
-        }
-
-        self.buffer.push(b']');
-
-        let bytes = mem::replace(&mut self.buffer, Vec::with_capacity(BUFCAP)).into();
-
-        Some(Serialized(bytes))
-    }
-}
-
-macro_rules! actions {
-    ($($action:literal: $t:ty,)*) => {
-        $(
-            impl FeedMessage for $t {
-                const ACTION: u8 = $action;
-            }
-        )*
-    }
-}
-
-actions! {
-    0x00: Version,
-    0x01: BestBlock,
-    0x02: BestFinalized,
-    0x03: AddedNode<'_>,
-    0x04: RemovedNode,
-    0x05: LocatedNode<'_>,
-    0x06: ImportedBlock<'_>,
-    0x07: FinalizedBlock,
-    0x08: NodeStatsUpdate<'_>,
-    0x09: Hardware<'_>,
-    0x0A: TimeSync,
-    0x0B: AddedChain<'_>,
-    0x0C: RemovedChain<'_>,
-    0x0D: SubscribedTo<'_>,
-    0x0E: UnsubscribedFrom<'_>,
-    0x0F: Pong<'_>,
-    0x10: AfgFinalized,
-    0x11: AfgReceivedPrevote,
-    0x12: AfgReceivedPrecommit,
-    0x13: AfgAuthoritySet,
-    0x14: StaleNode,
-    0x15: NodeIOUpdate<'_>,
-}
-
-#[derive(Serialize)]
-pub struct Version(pub usize);
-
-#[derive(Serialize)]
-pub struct BestBlock(pub BlockNumber, pub Timestamp, pub Option<u64>);
-
-#[derive(Serialize)]
-pub struct BestFinalized(pub BlockNumber, pub BlockHash);
-
-pub struct AddedNode<'a>(pub NodeId, pub &'a Node);
-
-#[derive(Serialize)]
-pub struct RemovedNode(pub NodeId);
-
-#[derive(Serialize)]
-pub struct LocatedNode<'a>(pub NodeId, pub f32, pub f32, pub &'a str);
-
-#[derive(Serialize)]
-pub struct ImportedBlock<'a>(pub NodeId, pub &'a BlockDetails);
-
-#[derive(Serialize)]
-pub struct FinalizedBlock(pub NodeId, pub BlockNumber, pub BlockHash);
-
-#[derive(Serialize)]
-pub struct NodeStatsUpdate<'a>(pub NodeId, pub &'a NodeStats);
-
-#[derive(Serialize)]
-pub struct NodeIOUpdate<'a>(pub NodeId, pub &'a NodeIO);
-
-#[derive(Serialize)]
-pub struct Hardware<'a>(pub NodeId, pub &'a NodeHardware);
-
-#[derive(Serialize)]
-pub struct TimeSync(pub u64);
-
-#[derive(Serialize)]
-pub struct AddedChain<'a>(pub &'a str, pub usize);
-
-#[derive(Serialize)]
-pub struct RemovedChain<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct SubscribedTo<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct UnsubscribedFrom<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct Pong<'a>(pub &'a str);
-
-#[derive(Serialize)]
-pub struct AfgFinalized(pub Address, pub BlockNumber, pub BlockHash);
-
-#[derive(Serialize)]
-pub struct AfgReceivedPrevote(
-    pub Address,
-    pub BlockNumber,
-    pub BlockHash,
-    pub Option<Address>,
-);
-
-#[derive(Serialize)]
-pub struct AfgReceivedPrecommit(
-    pub Address,
-    pub BlockNumber,
-    pub BlockHash,
-    pub Option<Address>,
-);
-
-#[derive(Serialize)]
-pub struct AfgAuthoritySet(
-    pub Address,
-    pub Address,
-    pub Address,
-    pub BlockNumber,
-    pub BlockHash,
-);
-
-#[derive(Serialize)]
-pub struct StaleNode(pub NodeId);
-
-impl Serialize for AddedNode<'_> {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let AddedNode(nid, node) = self;
-        let mut tup = serializer.serialize_tuple(8)?;
-        tup.serialize_element(nid)?;
-        tup.serialize_element(node.details())?;
-        tup.serialize_element(node.stats())?;
-        tup.serialize_element(node.io())?;
-        tup.serialize_element(node.hardware())?;
-        tup.serialize_element(node.block_details())?;
-        tup.serialize_element(&node.location())?;
-        tup.serialize_element(&node.startup_time())?;
-        tup.end()
-    }
-}

backend/core/src/feed/connector.rs

@@ -1,217 +0,0 @@
-use crate::aggregator::{Aggregator, Connect, Disconnect, NoMoreFinality, SendFinality, Subscribe};
-use crate::chain::Unsubscribe;
-use crate::feed::{FeedMessageSerializer, Pong};
-use crate::util::fnv;
-use actix::prelude::*;
-use actix_web_actors::ws;
-use bytes::Bytes;
-use std::time::{Duration, Instant};
-
-pub type FeedId = usize;
-
-/// How often heartbeat pings are sent
-const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
-/// How long before lack of client response causes a timeout
-const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
-
-pub struct FeedConnector {
-    /// FeedId that Aggregator holds of this actor
-    fid_aggregator: FeedId,
-    /// FeedId that Chain holds of this actor
-    fid_chain: FeedId,
-    /// Client must send ping at least once per 10 seconds (CLIENT_TIMEOUT),
-    hb: Instant,
-    /// Aggregator actor address
-    aggregator: Addr<Aggregator>,
-    /// Chain actor address
-    chain: Option<Recipient<Unsubscribe>>,
-    /// FNV hash of the chain label, optimization to avoid double-subscribing
-    chain_label_hash: u64,
-    /// Message serializer
-    serializer: FeedMessageSerializer,
-}
-
-impl Actor for FeedConnector {
-    type Context = ws::WebsocketContext<Self>;
-
-    fn started(&mut self, ctx: &mut Self::Context) {
-        self.heartbeat(ctx);
-        self.aggregator.do_send(Connect(ctx.address()));
-    }
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        if let Some(chain) = self.chain.take() {
-            let _ = chain.do_send(Unsubscribe(self.fid_chain));
-        }
-
-        self.aggregator.do_send(Disconnect(self.fid_aggregator));
-    }
-}
-
-impl FeedConnector {
-    pub fn new(aggregator: Addr<Aggregator>) -> Self {
-        Self {
-            // Garbage id, will be replaced by the Connected message
-            fid_aggregator: !0,
-            // Garbage id, will be replaced by the Subscribed message
-            fid_chain: !0,
-            hb: Instant::now(),
-            aggregator,
-            chain: None,
-            chain_label_hash: 0,
-            serializer: FeedMessageSerializer::new(),
-        }
-    }
-
-    fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
-        ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
-            // check client heartbeats
-            if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
-                // stop actor
-                ctx.stop();
-            } else {
-                ctx.ping(b"")
-            }
-        });
-    }
-
-    fn handle_cmd(&mut self, cmd: &str, payload: &str, ctx: &mut <Self as Actor>::Context) {
-        match cmd {
-            "subscribe" => {
-                match fnv(payload) {
-                    hash if hash == self.chain_label_hash => return,
-                    hash => self.chain_label_hash = hash,
-                }
-
-                self.aggregator
-                    .send(Subscribe {
-                        chain: payload.into(),
-                        feed: ctx.address(),
-                    })
-                    .into_actor(self)
-                    .then(|res, actor, _| {
-                        match res {
-                            Ok(true) => (),
-                            // Chain not found, reset hash
-                            _ => actor.chain_label_hash = 0,
-                        }
-                        async {}.into_actor(actor)
-                    })
-                    .wait(ctx);
-            }
-            "send-finality" => {
-                self.aggregator.do_send(SendFinality {
-                    chain: payload.into(),
-                    fid: self.fid_chain,
-                });
-            }
-            "no-more-finality" => {
-                self.aggregator.do_send(NoMoreFinality {
-                    chain: payload.into(),
-                    fid: self.fid_chain,
-                });
-            }
-            "ping" => {
-                self.serializer.push(Pong(payload));
-                if let Some(serialized) = self.serializer.finalize() {
-                    ctx.binary(serialized.0);
-                }
-            }
-            _ => (),
-        }
-    }
-}
-
-/// Message sent form Chain to the FeedConnector upon successful subscription
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Subscribed(pub FeedId, pub Recipient<Unsubscribe>);
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Unsubscribed;
-
-/// Message sent from Aggregator to FeedConnector upon successful connection
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Connected(pub FeedId);
-
-/// Message sent from either Aggregator or Chain to FeedConnector containing
-/// serialized message(s) for the frontend
-///
-/// Since Bytes is ARC'ed, this is cheap to clone
-#[derive(Message, Clone)]
-#[rtype(result = "()")]
-pub struct Serialized(pub Bytes);
-
-impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for FeedConnector {
-    fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
-        match msg {
-            Ok(ws::Message::Ping(msg)) => {
-                self.hb = Instant::now();
-                ctx.pong(&msg);
-            }
-            Ok(ws::Message::Pong(_)) => self.hb = Instant::now(),
-            Ok(ws::Message::Text(text)) => {
-                if let Some(idx) = text.find(':') {
-                    let cmd = &text[..idx];
-                    let payload = &text[idx + 1..];
-
-                    log::info!("New FEED message: {}", cmd);
-
-                    self.handle_cmd(cmd, payload, ctx);
-                }
-            }
-            Ok(ws::Message::Close(_)) => ctx.stop(),
-            Ok(_) => (),
-            Err(error) => {
-                log::error!("{:?}", error);
-                ctx.stop();
-            }
-        }
-    }
-}
-
-impl Handler<Subscribed> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Subscribed, _: &mut Self::Context) {
-        let Subscribed(fid_chain, chain) = msg;
-
-        if let Some(current) = self.chain.take() {
-            let _ = current.do_send(Unsubscribe(self.fid_chain));
-        }
-
-        self.fid_chain = fid_chain;
-        self.chain = Some(chain);
-    }
-}
-
-impl Handler<Unsubscribed> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, _: Unsubscribed, _: &mut Self::Context) {
-        self.chain = None;
-        self.chain_label_hash = 0;
-    }
-}
-
-impl Handler<Connected> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Connected, _: &mut Self::Context) {
-        let Connected(fid_aggregator) = msg;
-
-        self.fid_aggregator = fid_aggregator;
-    }
-}
-
-impl Handler<Serialized> for FeedConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Serialized, ctx: &mut Self::Context) {
-        let Serialized(bytes) = msg;
-
-        ctx.binary(bytes);
-    }
-}

backend/core/src/main.rs

@@ -1,187 +0,0 @@
-use std::collections::HashSet;
-use std::iter::FromIterator;
-use std::net::Ipv4Addr;
-
-use actix::prelude::*;
-use actix_http::ws::Codec;
-use actix_web::{get, middleware, web, App, Error, HttpRequest, HttpResponse, HttpServer};
-use actix_web_actors::ws;
-use clap::Clap;
-use simple_logger::SimpleLogger;
-
-mod aggregator;
-mod chain;
-mod feed;
-mod node;
-mod shard;
-mod types;
-mod util;
-
-use aggregator::{Aggregator, GetHealth};
-use feed::connector::FeedConnector;
-use node::connector::NodeConnector;
-use shard::connector::ShardConnector;
-use util::{Locator, LocatorFactory};
-
-const VERSION: &str = env!("CARGO_PKG_VERSION");
-const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
-const NAME: &str = "Substrate Telemetry Backend";
-const ABOUT: &str = "This is the Telemetry Backend that injects and provide the data sent by Substrate/Polkadot nodes";
-
-#[derive(Clap, Debug)]
-#[clap(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
-struct Opts {
-    #[clap(
-        short = 'l',
-        long = "listen",
-        default_value = "127.0.0.1:8000",
-        about = "This is the socket address Telemetry is listening to. This is restricted to localhost (127.0.0.1) by default and should be fine for most use cases. If you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'"
-    )]
-    socket: std::net::SocketAddr,
-    #[clap(
-        required = false,
-        long = "denylist",
-        about = "Space delimited list of chains that are not allowed to connect to telemetry. Case sensitive."
-    )]
-    denylist: Vec<String>,
-    #[clap(
-        arg_enum,
-        required = false,
-        long = "log",
-        default_value = "info",
-        about = "Log level."
-    )]
-    log_level: LogLevel,
-}
-
-#[derive(Clap, Debug, PartialEq)]
-enum LogLevel {
-    Error,
-    Warn,
-    Info,
-    Debug,
-    Trace,
-}
-
-impl From<&LogLevel> for log::LevelFilter {
-    fn from(log_level: &LogLevel) -> Self {
-        match log_level {
-            LogLevel::Error => log::LevelFilter::Error,
-            LogLevel::Warn => log::LevelFilter::Warn,
-            LogLevel::Info => log::LevelFilter::Info,
-            LogLevel::Debug => log::LevelFilter::Debug,
-            LogLevel::Trace => log::LevelFilter::Trace,
-        }
-    }
-}
-
-/// Entry point for connecting nodes
-#[get("/submit")]
-async fn node_route(
-    req: HttpRequest,
-    stream: web::Payload,
-    aggregator: web::Data<Addr<Aggregator>>,
-    locator: web::Data<Addr<Locator>>,
-) -> Result<HttpResponse, Error> {
-    let ip = req
-        .connection_info()
-        .realip_remote_addr()
-        .and_then(|mut addr| {
-            if let Some(port_idx) = addr.find(':') {
-                addr = &addr[..port_idx];
-            }
-            addr.parse::<Ipv4Addr>().ok()
-        });
-
-    let mut res = ws::handshake(&req)?;
-    let aggregator = aggregator.get_ref().clone();
-    let locator = locator.get_ref().clone().recipient();
-
-    Ok(res.streaming(ws::WebsocketContext::with_codec(
-        NodeConnector::new(aggregator, locator, ip),
-        stream,
-        Codec::new().max_size(10 * 1024 * 1024), // 10mb frame limit
-    )))
-}
-
-#[get("/shard_submit/{chain_hash}")]
-async fn shard_route(
-    req: HttpRequest,
-    stream: web::Payload,
-    aggregator: web::Data<Addr<Aggregator>>,
-    path: web::Path<Box<str>>,
-) -> Result<HttpResponse, Error> {
-    let hash_str = path.into_inner();
-    let genesis_hash = hash_str.parse()?;
-
-    let mut res = ws::handshake(&req)?;
-
-    let aggregator = aggregator.get_ref().clone();
-
-    Ok(res.streaming(ws::WebsocketContext::with_codec(
-        ShardConnector::new(aggregator, genesis_hash),
-        stream,
-        Codec::new().max_size(10 * 1024 * 1024), // 10mb frame limit
-    )))
-}
-
-/// Entry point for connecting feeds
-#[get("/feed")]
-async fn feed_route(
-    req: HttpRequest,
-    stream: web::Payload,
-    aggregator: web::Data<Addr<Aggregator>>,
-) -> Result<HttpResponse, Error> {
-    ws::start(
-        FeedConnector::new(aggregator.get_ref().clone()),
-        &req,
-        stream,
-    )
-}
-
-/// Entry point for health check monitoring bots
-#[get("/health")]
-async fn health(aggregator: web::Data<Addr<Aggregator>>) -> Result<HttpResponse, Error> {
-    match aggregator.send(GetHealth).await {
-        Ok(count) => {
-            let body = format!("Connected chains: {}", count);
-
-            HttpResponse::Ok().body(body).await
-        }
-        Err(error) => {
-            log::error!("Health check mailbox error: {:?}", error);
-
-            HttpResponse::InternalServerError().await
-        }
-    }
-}
-
-/// Telemetry entry point. Listening by default on 127.0.0.1:8000.
-/// This can be changed using the `PORT` and `BIND` ENV variables.
-#[actix_web::main]
-async fn main() -> std::io::Result<()> {
-    let opts = Opts::parse();
-    let log_level = &opts.log_level;
-    SimpleLogger::new()
-        .with_level(log_level.into())
-        .init()
-        .expect("Must be able to start a logger");
-
-    let denylist = HashSet::from_iter(opts.denylist);
-    let aggregator = Aggregator::new(denylist).start();
-    let factory = LocatorFactory::new();
-    let locator = SyncArbiter::start(4, move || factory.create());
-    log::info!("Starting telemetry version: {}", env!("CARGO_PKG_VERSION"));
-    HttpServer::new(move || {
-        App::new()
-            .wrap(middleware::NormalizePath::default())
-            .data(aggregator.clone())
-            .data(locator.clone())
-            .service(node_route)
-            .service(feed_route)
-            .service(health)
-    })
-    .bind(opts.socket)?
-    .run()
-    .await
-}

backend/core/src/node/connector.rs

@@ -1,274 +0,0 @@
-use std::collections::BTreeMap;
-use std::mem;
-use std::net::Ipv4Addr;
-use std::time::{Duration, Instant};
-
-use crate::aggregator::{AddNode, Aggregator};
-use crate::chain::{Chain, RemoveNode, UpdateNode};
-use crate::node::message::{NodeMessage, Payload};
-use crate::node::NodeId;
-use crate::types::ConnId;
-use crate::util::LocateRequest;
-use actix::prelude::*;
-use actix_http::ws::Item;
-use actix_web_actors::ws::{self, CloseReason};
-use bytes::{Bytes, BytesMut};
-
-/// How often heartbeat pings are sent
-const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
-/// How long before lack of client response causes a timeout
-const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
-/// Continuation buffer limit, 10mb
-const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
-
-pub struct NodeConnector {
-    /// Multiplexing connections by id
-    multiplex: BTreeMap<ConnId, ConnMultiplex>,
-    /// Client must send ping at least once every 60 seconds (CLIENT_TIMEOUT),
-    hb: Instant,
-    /// Aggregator actor address
-    aggregator: Addr<Aggregator>,
-    /// IP address of the node this connector is responsible for
-    ip: Option<Ipv4Addr>,
-    /// Actix address of location services
-    locator: Recipient<LocateRequest>,
-    /// Buffer for constructing continuation messages
-    contbuf: BytesMut,
-}
-
-enum ConnMultiplex {
-    Connected {
-        /// Id of the node this multiplex connector is responsible for handling
-        nid: NodeId,
-        /// Chain address to which this multiplex connector is delegating messages
-        chain: Addr<Chain>,
-    },
-    Waiting {
-        /// Backlog of messages to be sent once we get a recipient handle to the chain
-        backlog: Vec<Payload>,
-    },
-}
-
-impl Default for ConnMultiplex {
-    fn default() -> Self {
-        ConnMultiplex::Waiting {
-            backlog: Vec::new(),
-        }
-    }
-}
-
-impl Actor for NodeConnector {
-    type Context = ws::WebsocketContext<Self>;
-
-    fn started(&mut self, ctx: &mut Self::Context) {
-        self.heartbeat(ctx);
-    }
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        for mx in self.multiplex.values() {
-            if let ConnMultiplex::Connected { chain, nid } = mx {
-                chain.do_send(RemoveNode(*nid));
-            }
-        }
-    }
-}
-
-impl NodeConnector {
-    pub fn new(
-        aggregator: Addr<Aggregator>,
-        locator: Recipient<LocateRequest>,
-        ip: Option<Ipv4Addr>,
-    ) -> Self {
-        Self {
-            multiplex: BTreeMap::new(),
-            hb: Instant::now(),
-            aggregator,
-            ip,
-            locator,
-            contbuf: BytesMut::new(),
-        }
-    }
-
-    fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
-        ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
-            // check client heartbeats
-            if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
-                // stop actor
-                ctx.close(Some(CloseReason {
-                    code: ws::CloseCode::Abnormal,
-                    description: Some("Missed heartbeat".into()),
-                }));
-                ctx.stop();
-            }
-        });
-    }
-
-    fn handle_message(
-        &mut self,
-        msg: NodeMessage,
-        ctx: &mut <Self as Actor>::Context,
-    ) {
-        let conn_id = msg.id();
-        let payload = msg.into();
-
-        match self.multiplex.entry(conn_id).or_default() {
-            ConnMultiplex::Connected { nid, chain } => {
-                chain.do_send(UpdateNode {
-                    nid: *nid,
-                    payload,
-                });
-            }
-            ConnMultiplex::Waiting { backlog } => {
-                if let Payload::SystemConnected(connected) = payload {
-                    self.aggregator.do_send(AddNode {
-                        node: connected.node,
-                        genesis_hash: connected.genesis_hash,
-                        conn_id,
-                        node_connector: ctx.address(),
-                    });
-                } else {
-                    if backlog.len() >= 10 {
-                        backlog.remove(0);
-                    }
-
-                    backlog.push(payload);
-                }
-            }
-        }
-    }
-
-    fn start_frame(&mut self, bytes: &[u8]) {
-        if !self.contbuf.is_empty() {
-            log::error!("Unused continuation buffer");
-            self.contbuf.clear();
-        }
-        self.continue_frame(bytes);
-    }
-
-    fn continue_frame(&mut self, bytes: &[u8]) {
-        if self.contbuf.len() + bytes.len() <= CONT_BUF_LIMIT {
-            self.contbuf.extend_from_slice(&bytes);
-        } else {
-            log::error!("Continuation buffer overflow");
-            self.contbuf = BytesMut::new();
-        }
-    }
-
-    fn finish_frame(&mut self) -> Bytes {
-        mem::replace(&mut self.contbuf, BytesMut::new()).freeze()
-    }
-}
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Mute {
-    pub reason: CloseReason,
-}
-
-impl Handler<Mute> for NodeConnector {
-    type Result = ();
-    fn handle(&mut self, msg: Mute, ctx: &mut Self::Context) {
-        let Mute { reason } = msg;
-        log::debug!(target: "NodeConnector::Mute", "Muting a node. Reason: {:?}", reason.description);
-
-        ctx.close(Some(reason));
-        ctx.stop();
-    }
-}
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct Initialize {
-    pub nid: NodeId,
-    pub conn_id: ConnId,
-    pub chain: Addr<Chain>,
-}
-
-impl Handler<Initialize> for NodeConnector {
-    type Result = ();
-
-    fn handle(&mut self, msg: Initialize, _: &mut Self::Context) {
-        let Initialize {
-            nid,
-            conn_id,
-            chain,
-        } = msg;
-        log::trace!(target: "NodeConnector::Initialize", "Initializing a node, nid={}, on conn_id={}", nid, conn_id);
-        let mx = self.multiplex.entry(conn_id).or_default();
-
-        if let ConnMultiplex::Waiting { backlog } = mx {
-            for payload in backlog.drain(..) {
-                chain.do_send(UpdateNode {
-                    nid,
-                    payload,
-                });
-            }
-
-            *mx = ConnMultiplex::Connected {
-                nid,
-                chain: chain.clone(),
-            };
-        };
-
-        // Acquire the node's physical location
-        if let Some(ip) = self.ip {
-            let _ = self.locator.do_send(LocateRequest { ip, nid, chain });
-        }
-    }
-}
-
-impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for NodeConnector {
-    fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
-        self.hb = Instant::now();
-
-        let data = match msg {
-            Ok(ws::Message::Ping(msg)) => {
-                ctx.pong(&msg);
-                return;
-            }
-            Ok(ws::Message::Pong(_)) => return,
-            Ok(ws::Message::Text(text)) => text.into_bytes(),
-            Ok(ws::Message::Binary(data)) => data,
-            Ok(ws::Message::Close(reason)) => {
-                ctx.close(reason);
-                ctx.stop();
-                return;
-            }
-            Ok(ws::Message::Nop) => return,
-            Ok(ws::Message::Continuation(cont)) => match cont {
-                Item::FirstText(bytes) | Item::FirstBinary(bytes) => {
-                    self.start_frame(&bytes);
-                    return;
-                }
-                Item::Continue(bytes) => {
-                    self.continue_frame(&bytes);
-                    return;
-                }
-                Item::Last(bytes) => {
-                    self.continue_frame(&bytes);
-                    self.finish_frame()
-                }
-            },
-            Err(error) => {
-                log::error!("{:?}", error);
-                ctx.stop();
-                return;
-            }
-        };
-
-        match serde_json::from_slice(&data) {
-            Ok(msg) => self.handle_message(msg, ctx),
-            #[cfg(debug)]
-            Err(err) => {
-                let data: &[u8] = data.get(..512).unwrap_or_else(|| &data);
-                log::warn!(
-                    "Failed to parse node message: {} {}",
-                    err,
-                    std::str::from_utf8(data).unwrap_or_else(|_| "INVALID UTF8")
-                )
-            }
-            #[cfg(not(debug))]
-            Err(_) => (),
-        }
-    }
-}

backend/core/src/node/message.rs

@@ -1,196 +0,0 @@
-use crate::node::NodeDetails;
-use crate::types::{Block, BlockHash, BlockNumber, ConnId};
-use crate::util::Hash;
-use actix::prelude::*;
-use serde::de::IgnoredAny;
-use serde::Deserialize;
-
-#[derive(Deserialize, Debug, Message)]
-#[rtype(result = "()")]
-#[serde(untagged)]
-pub enum NodeMessage {
-    V1 {
-        #[serde(flatten)]
-        payload: Payload,
-    },
-    V2 {
-        id: ConnId,
-        payload: Payload,
-    },
-}
-
-impl NodeMessage {
-    /// Returns the connection ID or 0 if there is no ID.
-    pub fn id(&self) -> ConnId {
-        match self {
-            NodeMessage::V1 { .. } => 0,
-            NodeMessage::V2 { id, .. } => *id,
-        }
-    }
-}
-
-impl From<NodeMessage> for Payload {
-    fn from(msg: NodeMessage) -> Payload {
-        match msg {
-            NodeMessage::V1 { payload, .. } | NodeMessage::V2 { payload, .. } => payload,
-        }
-    }
-}
-
-#[derive(Deserialize, Debug)]
-#[serde(tag = "msg")]
-pub enum Payload {
-    #[serde(rename = "system.connected")]
-    SystemConnected(SystemConnected),
-    #[serde(rename = "system.interval")]
-    SystemInterval(SystemInterval),
-    #[serde(rename = "block.import")]
-    BlockImport(Block),
-    #[serde(rename = "notify.finalized")]
-    NotifyFinalized(Finalized),
-    #[serde(rename = "txpool.import")]
-    TxPoolImport(IgnoredAny),
-    #[serde(rename = "afg.finalized")]
-    AfgFinalized(AfgFinalized),
-    #[serde(rename = "afg.received_precommit")]
-    AfgReceivedPrecommit(AfgReceivedPrecommit),
-    #[serde(rename = "afg.received_prevote")]
-    AfgReceivedPrevote(AfgReceivedPrevote),
-    #[serde(rename = "afg.received_commit")]
-    AfgReceivedCommit(AfgReceivedCommit),
-    #[serde(rename = "afg.authority_set")]
-    AfgAuthoritySet(AfgAuthoritySet),
-    #[serde(rename = "afg.finalized_blocks_up_to")]
-    AfgFinalizedBlocksUpTo(IgnoredAny),
-    #[serde(rename = "aura.pre_sealed_block")]
-    AuraPreSealedBlock(IgnoredAny),
-    #[serde(rename = "prepared_block_for_proposing")]
-    PreparedBlockForProposing(IgnoredAny),
-}
-
-#[derive(Deserialize, Debug)]
-pub struct SystemConnected {
-    pub genesis_hash: Hash,
-    #[serde(flatten)]
-    pub node: NodeDetails,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct SystemInterval {
-    pub peers: Option<u64>,
-    pub txcount: Option<u64>,
-    pub bandwidth_upload: Option<f64>,
-    pub bandwidth_download: Option<f64>,
-    pub finalized_height: Option<BlockNumber>,
-    pub finalized_hash: Option<BlockHash>,
-    #[serde(flatten)]
-    pub block: Option<Block>,
-    pub used_state_cache_size: Option<f32>,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct Finalized {
-    #[serde(rename = "best")]
-    pub hash: BlockHash,
-    pub height: Box<str>,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct AfgAuthoritySet {
-    pub authority_id: Box<str>,
-    pub authorities: Box<str>,
-    pub authority_set_id: Box<str>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgFinalized {
-    pub finalized_hash: BlockHash,
-    pub finalized_number: Box<str>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceived {
-    pub target_hash: BlockHash,
-    pub target_number: Box<str>,
-    pub voter: Option<Box<str>>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceivedPrecommit {
-    #[serde(flatten)]
-    pub received: AfgReceived,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceivedPrevote {
-    #[serde(flatten)]
-    pub received: AfgReceived,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct AfgReceivedCommit {
-    #[serde(flatten)]
-    pub received: AfgReceived,
-}
-
-impl Block {
-    pub fn zero() -> Self {
-        Block {
-            hash: BlockHash::from([0; 32]),
-            height: 0,
-        }
-    }
-}
-
-impl Payload {
-    pub fn best_block(&self) -> Option<&Block> {
-        match self {
-            Payload::BlockImport(block) => Some(block),
-            Payload::SystemInterval(SystemInterval { block, .. }) => block.as_ref(),
-            _ => None,
-        }
-    }
-
-    pub fn finalized_block(&self) -> Option<Block> {
-        match self {
-            Payload::SystemInterval(ref interval) => Some(Block {
-                hash: interval.finalized_hash?,
-                height: interval.finalized_height?,
-            }),
-            Payload::NotifyFinalized(ref finalized) => Some(Block {
-                hash: finalized.hash,
-                height: finalized.height.parse().ok()?,
-            }),
-            _ => None,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn message_v1() {
-        let json = r#"{"msg":"notify.finalized","level":"INFO","ts":"2021-01-13T12:38:25.410794650+01:00","best":"0x031c3521ca2f9c673812d692fc330b9a18e18a2781e3f9976992f861fd3ea0cb","height":"50"}"#;
-        assert!(
-            matches!(
-                serde_json::from_str::<NodeMessage>(json).unwrap(),
-                NodeMessage::V1 { .. },
-            ),
-            "message did not match variant V1",
-        );
-    }
-
-    #[test]
-    fn message_v2() {
-        let json = r#"{"id":1,"ts":"2021-01-13T12:22:20.053527101+01:00","payload":{"best":"0xcc41708573f2acaded9dd75e07dac2d4163d136ca35b3061c558d7a35a09dd8d","height":"209","msg":"notify.finalized"}}"#;
-        assert!(
-            matches!(
-                serde_json::from_str::<NodeMessage>(json).unwrap(),
-                NodeMessage::V2 { .. },
-            ),
-            "message did not match variant V2",
-        );
-    }
-}

backend/core/src/shard.rs

@@ -1,13 +0,0 @@
-use crate::node::message::Payload;
-use serde::Deserialize;
-
-pub mod connector;
-
-/// Alias for the ID of the node connection
-type ShardConnId = usize;
-
-#[derive(Deserialize)]
-pub struct ShardMessage {
-    pub conn_id: ShardConnId,
-    pub payload: Payload,
-}

backend/core/src/shard/connector.rs

@@ -1,156 +0,0 @@
-use std::mem;
-use std::time::{Duration, Instant};
-
-use crate::aggregator::{AddNode, Aggregator};
-use crate::chain::{Chain, RemoveNode, UpdateNode};
-use crate::shard::ShardMessage;
-use crate::types::NodeId;
-use crate::util::{DenseMap, Hash};
-use actix::prelude::*;
-use actix_http::ws::Item;
-use actix_web_actors::ws::{self, CloseReason};
-use bincode::Options;
-use bytes::{Bytes, BytesMut};
-
-/// How often heartbeat pings are sent
-const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
-/// How long before lack of client response causes a timeout
-const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
-/// Continuation buffer limit, 10mb
-const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
-
-pub struct ShardConnector {
-    /// Client must send ping at least once every 60 seconds (CLIENT_TIMEOUT),
-    hb: Instant,
-    /// Aggregator actor address
-    aggregator: Addr<Aggregator>,
-    /// Genesis hash of the chain this connection will be submitting data for
-    genesis_hash: Hash,
-    /// Chain address to which this multiplex connector is delegating messages
-    chain: Option<Addr<Chain>>,
-    /// Mapping `ShardConnId` to `NodeId`
-    nodes: DenseMap<NodeId>,
-    /// Buffer for constructing continuation messages
-    contbuf: BytesMut,
-}
-
-impl Actor for ShardConnector {
-    type Context = ws::WebsocketContext<Self>;
-
-    fn started(&mut self, ctx: &mut Self::Context) {
-        self.heartbeat(ctx);
-    }
-
-    fn stopped(&mut self, _: &mut Self::Context) {
-        if let Some(ref chain) = self.chain {
-            for (_, nid) in self.nodes.iter() {
-                chain.do_send(RemoveNode(*nid))
-            }
-        }
-    }
-}
-
-impl ShardConnector {
-    pub fn new(aggregator: Addr<Aggregator>, genesis_hash: Hash) -> Self {
-        Self {
-            hb: Instant::now(),
-            aggregator,
-            genesis_hash,
-            chain: None,
-            nodes: DenseMap::new(),
-            contbuf: BytesMut::new(),
-        }
-    }
-
-    fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
-        ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
-            // check client heartbeats
-            if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
-                // stop actor
-                ctx.close(Some(CloseReason {
-                    code: ws::CloseCode::Abnormal,
-                    description: Some("Missed heartbeat".into()),
-                }));
-                ctx.stop();
-            }
-        });
-    }
-
-    fn handle_message(&mut self, msg: ShardMessage, ctx: &mut <Self as Actor>::Context) {
-        let ShardMessage { conn_id, payload } = msg;
-
-        // TODO: get `NodeId` for `ShardConnId` and proxy payload to `self.chain`.
-    }
-
-    fn start_frame(&mut self, bytes: &[u8]) {
-        if !self.contbuf.is_empty() {
-            log::error!("Unused continuation buffer");
-            self.contbuf.clear();
-        }
-        self.continue_frame(bytes);
-    }
-
-    fn continue_frame(&mut self, bytes: &[u8]) {
-        if self.contbuf.len() + bytes.len() <= CONT_BUF_LIMIT {
-            self.contbuf.extend_from_slice(&bytes);
-        } else {
-            log::error!("Continuation buffer overflow");
-            self.contbuf = BytesMut::new();
-        }
-    }
-
-    fn finish_frame(&mut self) -> Bytes {
-        mem::replace(&mut self.contbuf, BytesMut::new()).freeze()
-    }
-}
-
-impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for ShardConnector {
-    fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
-        self.hb = Instant::now();
-
-        let data = match msg {
-            Ok(ws::Message::Ping(msg)) => {
-                ctx.pong(&msg);
-                return;
-            }
-            Ok(ws::Message::Pong(_)) => return,
-            Ok(ws::Message::Text(text)) => text.into_bytes(),
-            Ok(ws::Message::Binary(data)) => data,
-            Ok(ws::Message::Close(reason)) => {
-                ctx.close(reason);
-                ctx.stop();
-                return;
-            }
-            Ok(ws::Message::Nop) => return,
-            Ok(ws::Message::Continuation(cont)) => match cont {
-                Item::FirstText(bytes) | Item::FirstBinary(bytes) => {
-                    self.start_frame(&bytes);
-                    return;
-                }
-                Item::Continue(bytes) => {
-                    self.continue_frame(&bytes);
-                    return;
-                }
-                Item::Last(bytes) => {
-                    self.continue_frame(&bytes);
-                    self.finish_frame()
-                }
-            },
-            Err(error) => {
-                log::error!("{:?}", error);
-                ctx.stop();
-                return;
-            }
-        };
-
-        match bincode::options().deserialize(&data) {
-            Ok(msg) => self.handle_message(msg, ctx),
-            #[cfg(debug)]
-            Err(err) => {
-                log::warn!("Failed to parse shard message: {}", err,)
-            }
-            #[cfg(not(debug))]
-            Err(_) => (),
-        }
-    }
-}

backend/core/src/types.rs

@@ -1,155 +0,0 @@
-use serde::ser::{Serialize, SerializeTuple, Serializer};
-use serde::Deserialize;
-
-use crate::util::{now, MeanList};
-
-pub type NodeId = usize;
-pub type ConnId = u64;
-pub type BlockNumber = u64;
-pub type Timestamp = u64;
-pub type Address = Box<str>;
-pub use primitive_types::H256 as BlockHash;
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct NodeDetails {
-    pub chain: Box<str>,
-    pub name: Box<str>,
-    pub implementation: Box<str>,
-    pub version: Box<str>,
-    pub validator: Option<Box<str>>,
-    pub network_id: Option<Box<str>>,
-    pub startup_time: Option<Box<str>>,
-}
-
-#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)]
-pub struct NodeStats {
-    pub peers: u64,
-    pub txcount: u64,
-}
-
-#[derive(Default)]
-pub struct NodeIO {
-    pub used_state_cache_size: MeanList<f32>,
-}
-
-#[derive(Deserialize, Debug, Clone, Copy)]
-pub struct Block {
-    #[serde(rename = "best")]
-    pub hash: BlockHash,
-    pub height: BlockNumber,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct BlockDetails {
-    pub block: Block,
-    pub block_time: u64,
-    pub block_timestamp: u64,
-    pub propagation_time: Option<u64>,
-}
-
-impl Default for BlockDetails {
-    fn default() -> Self {
-        BlockDetails {
-            block: Block::zero(),
-            block_timestamp: now(),
-            block_time: 0,
-            propagation_time: None,
-        }
-    }
-}
-
-#[derive(Default)]
-pub struct NodeHardware {
-    /// Upload uses means
-    pub upload: MeanList<f64>,
-    /// Download uses means
-    pub download: MeanList<f64>,
-    /// Stampchange uses means
-    pub chart_stamps: MeanList<f64>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct NodeLocation {
-    pub latitude: f32,
-    pub longitude: f32,
-    pub city: Box<str>,
-}
-
-impl Serialize for NodeDetails {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(6)?;
-        tup.serialize_element(&self.name)?;
-        tup.serialize_element(&self.implementation)?;
-        tup.serialize_element(&self.version)?;
-        tup.serialize_element(&self.validator)?;
-        tup.serialize_element(&self.network_id)?;
-        tup.end()
-    }
-}
-
-impl Serialize for NodeStats {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(2)?;
-        tup.serialize_element(&self.peers)?;
-        tup.serialize_element(&self.txcount)?;
-        tup.end()
-    }
-}
-
-impl Serialize for NodeIO {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(1)?;
-        tup.serialize_element(self.used_state_cache_size.slice())?;
-        tup.end()
-    }
-}
-
-impl Serialize for BlockDetails {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(5)?;
-        tup.serialize_element(&self.block.height)?;
-        tup.serialize_element(&self.block.hash)?;
-        tup.serialize_element(&self.block_time)?;
-        tup.serialize_element(&self.block_timestamp)?;
-        tup.serialize_element(&self.propagation_time)?;
-        tup.end()
-    }
-}
-
-impl Serialize for NodeLocation {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(3)?;
-        tup.serialize_element(&self.latitude)?;
-        tup.serialize_element(&self.longitude)?;
-        tup.serialize_element(&&*self.city)?;
-        tup.end()
-    }
-}
-
-impl Serialize for NodeHardware {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut tup = serializer.serialize_tuple(3)?;
-        tup.serialize_element(self.upload.slice())?;
-        tup.serialize_element(self.download.slice())?;
-        tup.serialize_element(self.chart_stamps.slice())?;
-        tup.end()
-    }
-}

backend/core/src/util.rs

@@ -1,31 +0,0 @@
-mod dense_map;
-mod hash;
-mod location;
-mod mean_list;
-mod num_stats;
-
-pub use dense_map::DenseMap;
-pub use hash::Hash;
-pub use location::{LocateRequest, Locator, LocatorFactory};
-pub use mean_list::MeanList;
-pub use num_stats::NumStats;
-
-pub fn fnv<D: AsRef<[u8]>>(data: D) -> u64 {
-    use fnv::FnvHasher;
-    use std::hash::Hasher;
-
-    let mut hasher = FnvHasher::default();
-
-    hasher.write(data.as_ref());
-    hasher.finish()
-}
-
-/// Returns current unix time in ms (compatible with JS Date.now())
-pub fn now() -> u64 {
-    use std::time::SystemTime;
-
-    SystemTime::now()
-        .duration_since(SystemTime::UNIX_EPOCH)
-        .expect("System time must be configured to be post Unix Epoch start; qed")
-        .as_millis() as u64
-}

backend/core/src/util/dense_map.rs

@@ -1,80 +0,0 @@
-pub type Id = usize;
-
-pub struct DenseMap<T> {
-    /// List of retired indexes that can be re-used
-    retired: Vec<Id>,
-    /// All items
-    items: Vec<Option<T>>,
-}
-
-impl<T> DenseMap<T> {
-    pub fn new() -> Self {
-        DenseMap {
-            retired: Vec::new(),
-            items: Vec::new(),
-        }
-    }
-
-    pub fn add(&mut self, item: T) -> Id {
-        self.add_with(|_| item)
-    }
-
-    pub fn add_with<F>(&mut self, f: F) -> Id
-    where
-        F: FnOnce(Id) -> T,
-    {
-        match self.retired.pop() {
-            Some(id) => {
-                self.items[id] = Some(f(id));
-                id
-            }
-            None => {
-                let id = self.items.len();
-                self.items.push(Some(f(id)));
-                id
-            }
-        }
-    }
-
-    pub fn get(&self, id: Id) -> Option<&T> {
-        self.items.get(id).and_then(|item| item.as_ref())
-    }
-
-    pub fn get_mut(&mut self, id: Id) -> Option<&mut T> {
-        self.items.get_mut(id).and_then(|item| item.as_mut())
-    }
-
-    pub fn remove(&mut self, id: Id) -> Option<T> {
-        let old = self.items.get_mut(id).and_then(|item| item.take());
-
-        if old.is_some() {
-            // something was actually removed, so lets add the id to
-            // the list of retired ids!
-            self.retired.push(id);
-        }
-
-        old
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = (Id, &T)> + '_ {
-        self.items
-            .iter()
-            .enumerate()
-            .filter_map(|(id, item)| Some((id, item.as_ref()?)))
-    }
-
-    pub fn iter_mut(&mut self) -> impl Iterator<Item = (Id, &mut T)> + '_ {
-        self.items
-            .iter_mut()
-            .enumerate()
-            .filter_map(|(id, item)| Some((id, item.as_mut()?)))
-    }
-
-    pub fn len(&self) -> usize {
-        self.items.len() - self.retired.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-}

backend/core/src/util/hash.rs

@@ -1,89 +0,0 @@
-use std::fmt::{self, Debug, Display};
-use std::str::FromStr;
-
-use actix_web::error::ResponseError;
-use serde::de::{self, Deserialize, Deserializer, Unexpected, Visitor};
-
-const HASH_BYTES: usize = 32;
-
-/// Newtype wrapper for 32-byte hash values, implementing readable `Debug` and `serde::Deserialize`.
-// We could use primitive_types::H256 here, but opted for a custom type to avoid more dependencies.
-#[derive(Hash, PartialEq, Eq, Clone, Copy)]
-pub struct Hash([u8; HASH_BYTES]);
-
-struct HashVisitor;
-
-impl<'de> Visitor<'de> for HashVisitor {
-    type Value = Hash;
-
-    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-        formatter.write_str("hexidecimal string of 32 bytes beginning with 0x")
-    }
-
-    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
-    where
-        E: de::Error,
-    {
-        value
-            .parse()
-            .map_err(|_| de::Error::invalid_value(Unexpected::Str(value), &self))
-    }
-}
-
-impl FromStr for Hash {
-    type Err = HashParseError;
-
-    fn from_str(value: &str) -> Result<Self, Self::Err> {
-        if !value.starts_with("0x") {
-            return Err(HashParseError::InvalidPrefix);
-        }
-
-        let mut hash = [0; HASH_BYTES];
-
-        hex::decode_to_slice(&value[2..], &mut hash).map_err(HashParseError::HexError)?;
-
-        Ok(Hash(hash))
-    }
-}
-
-impl<'de> Deserialize<'de> for Hash {
-    fn deserialize<D>(deserializer: D) -> Result<Hash, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        deserializer.deserialize_str(HashVisitor)
-    }
-}
-
-impl Display for Hash {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.write_str("0x")?;
-
-        let mut ascii = [0; HASH_BYTES * 2];
-
-        hex::encode_to_slice(self.0, &mut ascii)
-            .expect("Encoding 32 bytes into 64 bytes of ascii; qed");
-
-        f.write_str(std::str::from_utf8(&ascii).expect("ASCII hex encoded bytes canot fail; qed"))
-    }
-}
-
-impl Debug for Hash {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        Display::fmt(self, f)
-    }
-}
-
-#[derive(thiserror::Error, Debug)]
-pub enum HashParseError {
-    HexError(hex::FromHexError),
-    InvalidPrefix,
-}
-
-impl Display for HashParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        Debug::fmt(self, f)
-    }
-}
-
-impl ResponseError for HashParseError {}

backend/core/src/util/location.rs

@@ -1,191 +0,0 @@
-use std::net::Ipv4Addr;
-use std::sync::Arc;
-
-use actix::prelude::*;
-use parking_lot::RwLock;
-use rustc_hash::FxHashMap;
-use serde::Deserialize;
-
-use crate::chain::{Chain, LocateNode};
-use crate::types::{NodeId, NodeLocation};
-
-#[derive(Clone)]
-pub struct Locator {
-    client: reqwest::blocking::Client,
-    cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
-}
-
-pub struct LocatorFactory {
-    cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
-}
-
-impl LocatorFactory {
-    pub fn new() -> Self {
-        let mut cache = FxHashMap::default();
-
-        // Default entry for localhost
-        cache.insert(
-            Ipv4Addr::new(127, 0, 0, 1),
-            Some(Arc::new(NodeLocation {
-                latitude: 52.516_6667,
-                longitude: 13.4,
-                city: "Berlin".into(),
-            })),
-        );
-
-        LocatorFactory {
-            cache: Arc::new(RwLock::new(cache)),
-        }
-    }
-
-    pub fn create(&self) -> Locator {
-        Locator {
-            client: reqwest::blocking::Client::new(),
-            cache: self.cache.clone(),
-        }
-    }
-}
-
-impl Actor for Locator {
-    type Context = SyncContext<Self>;
-}
-
-#[derive(Message)]
-#[rtype(result = "()")]
-pub struct LocateRequest {
-    pub ip: Ipv4Addr,
-    pub nid: NodeId,
-    pub chain: Addr<Chain>,
-}
-
-#[derive(Deserialize)]
-pub struct IPApiLocate {
-    city: Box<str>,
-    loc: Box<str>,
-}
-
-impl IPApiLocate {
-    fn into_node_location(self) -> Option<NodeLocation> {
-        let IPApiLocate { city, loc } = self;
-
-        let mut loc = loc.split(',').map(|n| n.parse());
-
-        let latitude = loc.next()?.ok()?;
-        let longitude = loc.next()?.ok()?;
-
-        // Guarantee that the iterator has been exhausted
-        if loc.next().is_some() {
-            return None;
-        }
-
-        Some(NodeLocation {
-            latitude,
-            longitude,
-            city,
-        })
-    }
-}
-
-impl Handler<LocateRequest> for Locator {
-    type Result = ();
-
-    fn handle(&mut self, msg: LocateRequest, _: &mut Self::Context) {
-        let LocateRequest { ip, nid, chain } = msg;
-
-        if let Some(item) = self.cache.read().get(&ip) {
-            if let Some(location) = item {
-                return chain.do_send(LocateNode {
-                    nid,
-                    location: location.clone(),
-                });
-            }
-
-            return;
-        }
-
-        let location = match self.iplocate(ip) {
-            Ok(location) => location,
-            Err(err) => return log::debug!("GET error for ip location: {:?}", err),
-        };
-
-        self.cache.write().insert(ip, location.clone());
-
-        if let Some(location) = location {
-            chain.do_send(LocateNode { nid, location });
-        }
-    }
-}
-
-impl Locator {
-    fn iplocate(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
-        let location = self.iplocate_ipapi_co(ip)?;
-
-        match location {
-            Some(location) => Ok(Some(location)),
-            None => self.iplocate_ipinfo_io(ip),
-        }
-    }
-
-    fn iplocate_ipapi_co(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
-        let location = self
-            .query(&format!("https://ipapi.co/{}/json", ip))?
-            .map(Arc::new);
-
-        Ok(location)
-    }
-
-    fn iplocate_ipinfo_io(
-        &self,
-        ip: Ipv4Addr,
-    ) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
-        let location = self
-            .query(&format!("https://ipinfo.io/{}/json", ip))?
-            .and_then(|loc: IPApiLocate| loc.into_node_location().map(Arc::new));
-
-        Ok(location)
-    }
-
-    fn query<T>(&self, url: &str) -> Result<Option<T>, reqwest::Error>
-    where
-        for<'de> T: Deserialize<'de>,
-    {
-        match self.client.get(url).send()?.json::<T>() {
-            Ok(result) => Ok(Some(result)),
-            Err(err) => {
-                log::debug!("JSON error for ip location: {:?}", err);
-                Ok(None)
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn ipapi_locate_to_node_location() {
-        let ipapi = IPApiLocate {
-            loc: "12.5,56.25".into(),
-            city: "Foobar".into(),
-        };
-
-        let location = ipapi.into_node_location().unwrap();
-
-        assert_eq!(location.latitude, 12.5);
-        assert_eq!(location.longitude, 56.25);
-        assert_eq!(&*location.city, "Foobar");
-    }
-
-    #[test]
-    fn ipapi_locate_to_node_location_too_many() {
-        let ipapi = IPApiLocate {
-            loc: "12.5,56.25,1.0".into(),
-            city: "Foobar".into(),
-        };
-
-        let location = ipapi.into_node_location();
-
-        assert!(location.is_none());
-    }
-}

backend/telemetry_core/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "telemetry_core"
+version = "0.1.0"
+authors = ["Parity Technologies Ltd. <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0"
+
+[dependencies]
+anyhow = "1.0.41"
+bimap = "0.6.1"
+bincode = "1.3.3"
+bytes = "1.0.1"
+common = { path = "../common" }
+futures = "0.3.15"
+hex = "0.4.3"
+http = "0.2.4"
+hyper = "0.14.11"
+log = "0.4.14"
+num_cpus = "1.13.0"
+once_cell = "1.8.0"
+parking_lot = "0.11.1"
+primitive-types = { version = "0.9.0", features = ["serde"] }
+rayon = "1.5.1"
+reqwest = { version = "0.11.4", features = ["json"] }
+rustc-hash = "1.1.0"
+serde = { version = "1.0.126", features = ["derive"] }
+serde_json = "1.0.64"
+simple_logger = "1.11.0"
+smallvec = "1.6.1"
+soketto = "0.6.0"
+structopt = "0.3.21"
+thiserror = "1.0.25"
+tokio = { version = "1.7.0", features = ["full"] }
+tokio-util = { version = "0.6", features = ["compat"] }
+
+[dev-dependencies]
+shellwords = "1.1.0"
+test_utils = { path = "../test_utils" }
+criterion = { version = "0.3.4", features = ["async", "async_tokio"] }
+
+[[bench]]
+name = "subscribe"
+harness = false

backend/telemetry_core/benches/subscribe.rs

@@ -0,0 +1,120 @@
+use common::node_types::BlockHash;
+use criterion::{criterion_group, criterion_main, Criterion};
+use serde_json::json;
+use std::time::{Duration, Instant};
+use test_utils::feed_message_de::FeedMessage;
+use test_utils::workspace::{start_server, CoreOpts, ServerOpts, ShardOpts};
+use tokio::runtime::Runtime;
+
+/// This benchmark roughly times the subscribe function. Note that there's a lot of
+/// overhead in other areas, so even with the entire subscribe function commented out
+/// By benchmark timings are ~50ms (whereas they are ~320ms with the version of the
+/// subscribe handler at the time of writing).
+///
+/// If you want to use this benchmark, it's therefore worth commenting out the subscribe
+/// logic entirely and running this to give yourself a "baseline".
+pub fn benchmark_subscribe_speed(c: &mut Criterion) {
+    const NUMBER_OF_FEEDS: usize = 100;
+    const NUMBER_OF_NODES: usize = 10_000;
+
+    let rt = Runtime::new().expect("tokio runtime should start");
+
+    c.bench_function("subscribe speed: time till pong", move |b| {
+        b.to_async(&rt).iter_custom(|iters| async move {
+            // Now, see how quickly a feed is subscribed. Criterion controls the number of
+            // iters performed here, but a lot of the time that number is "1".
+            let mut total_time = Duration::ZERO;
+            for _n in 0..iters {
+                // Start a server:
+                let mut server = start_server(
+                    ServerOpts {
+                        release_mode: true,
+                        log_output: false,
+                    },
+                    CoreOpts {
+                        worker_threads: Some(16),
+                        num_aggregators: Some(1),
+                        ..Default::default()
+                    },
+                    ShardOpts {
+                        max_nodes_per_connection: Some(usize::MAX),
+                        max_node_data_per_second: Some(usize::MAX),
+                        worker_threads: Some(2),
+                        ..Default::default()
+                    },
+                )
+                .await;
+                let shard_id = server.add_shard().await.unwrap();
+
+                // Connect a shard:
+                let (mut node_tx, _) = server
+                    .get_shard(shard_id)
+                    .unwrap()
+                    .connect_node()
+                    .await
+                    .expect("node can connect");
+
+                // Add a bunch of actual nodes on the same chain:
+                for n in 0..NUMBER_OF_NODES {
+                    node_tx
+                        .send_json_text(json!({
+                            "id":n,
+                            "ts":"2021-07-12T10:37:47.714666+01:00",
+                            "payload": {
+                                "authority":true,
+                                "chain":"Polkadot", // No limit to #nodes on this network.
+                                "config":"",
+                                "genesis_hash": BlockHash::from_low_u64_ne(1),
+                                "implementation":"Substrate Node",
+                                "msg":"system.connected",
+                                "name": format!("Node {}", n),
+                                "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                                "startup_time":"1625565542717",
+                                "version":"2.0.0-07a1af348-aarch64-macos"
+                            }
+                        }))
+                        .unwrap();
+                }
+
+                // Give those messages a chance to be handled. This, of course,
+                // assumes that those messages _can_ be handled this quickly. If not,
+                // we'll start to skew benchmark results with the "time taken to add node".
+                tokio::time::sleep(Duration::from_millis(250)).await;
+
+                // Start a bunch of feeds:
+                let mut feeds = server
+                    .get_core()
+                    .connect_multiple_feeds(NUMBER_OF_FEEDS)
+                    .await
+                    .expect("feeds can connect");
+
+                // Subscribe every feed to the chain:
+                for (feed_tx, _) in feeds.iter() {
+                    feed_tx.send_command("subscribe", "Polkadot").unwrap();
+                }
+
+                // Then, Ping a feed:
+                feeds[0].0.send_command("ping", "Finished!").unwrap();
+                let finished = FeedMessage::Pong {
+                    msg: "Finished!".to_owned(),
+                };
+
+                // Wait and see how long it takes to get a pong back:
+                let start = Instant::now();
+                loop {
+                    let msgs = feeds[0].1.recv_feed_messages_once().await.unwrap();
+                    if msgs.iter().find(|&m| m == &finished).is_some() {
+                        break;
+                    }
+                }
+                total_time += start.elapsed();
+            }
+
+            // The total time spent waiting for subscribes:
+            total_time
+        })
+    });
+}
+
+criterion_group!(benches, benchmark_subscribe_speed);
+criterion_main!(benches);

backend/telemetry_core/src/aggregator/aggregator.rs

@@ -0,0 +1,140 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::inner_loop;
+use crate::find_location::find_location;
+use crate::state::NodeId;
+use common::id_type;
+use futures::channel::mpsc;
+use futures::{future, Sink, SinkExt};
+use std::net::Ipv4Addr;
+use std::sync::atomic::AtomicU64;
+use std::sync::Arc;
+
+id_type! {
+    /// A unique Id is assigned per websocket connection (or more accurately,
+    /// per feed socket and per shard socket). This can be combined with the
+    /// [`LocalId`] of messages to give us a global ID.
+    pub struct ConnId(u64)
+}
+
+#[derive(Clone)]
+pub struct Aggregator(Arc<AggregatorInternal>);
+
+struct AggregatorInternal {
+    /// Shards that connect are each assigned a unique connection ID.
+    /// This helps us know who to send messages back to (especially in
+    /// conjunction with the `ShardNodeId` that messages will come with).
+    shard_conn_id: AtomicU64,
+    /// Feeds that connect have their own unique connection ID, too.
+    feed_conn_id: AtomicU64,
+    /// Send messages in to the aggregator from the outside via this. This is
+    /// stored here so that anybody holding an `Aggregator` handle can
+    /// make use of it.
+    tx_to_aggregator: mpsc::UnboundedSender<inner_loop::ToAggregator>,
+}
+
+impl Aggregator {
+    /// Spawn a new Aggregator. This connects to the telemetry backend
+    pub async fn spawn(denylist: Vec<String>) -> anyhow::Result<Aggregator> {
+        let (tx_to_aggregator, rx_from_external) = mpsc::unbounded();
+
+        // Kick off a locator task to locate nodes, which hands back a channel to make location requests
+        let tx_to_locator = find_location(tx_to_aggregator.clone().with(|(node_id, msg)| {
+            future::ok::<_, mpsc::SendError>(inner_loop::ToAggregator::FromFindLocation(
+                node_id, msg,
+            ))
+        }));
+
+        // Handle any incoming messages in our handler loop:
+        tokio::spawn(Aggregator::handle_messages(
+            rx_from_external,
+            tx_to_locator,
+            denylist,
+        ));
+
+        // Return a handle to our aggregator:
+        Ok(Aggregator(Arc::new(AggregatorInternal {
+            shard_conn_id: AtomicU64::new(1),
+            feed_conn_id: AtomicU64::new(1),
+            tx_to_aggregator,
+        })))
+    }
+
+    // This is spawned into a separate task and handles any messages coming
+    // in to the aggregator. If nobody is tolding the tx side of the channel
+    // any more, this task will gracefully end.
+    async fn handle_messages(
+        rx_from_external: mpsc::UnboundedReceiver<inner_loop::ToAggregator>,
+        tx_to_aggregator: mpsc::UnboundedSender<(NodeId, Ipv4Addr)>,
+        denylist: Vec<String>,
+    ) {
+        inner_loop::InnerLoop::new(rx_from_external, tx_to_aggregator, denylist)
+            .handle()
+            .await;
+    }
+
+    /// Return a sink that a shard can send messages into to be handled by the aggregator.
+    pub fn subscribe_shard(
+        &self,
+    ) -> impl Sink<inner_loop::FromShardWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static
+    {
+        // Assign a unique aggregator-local ID to each connection that subscribes, and pass
+        // that along with every message to the aggregator loop:
+        let shard_conn_id = self
+            .0
+            .shard_conn_id
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let tx_to_aggregator = self.0.tx_to_aggregator.clone();
+
+        // Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
+        // but pinning by boxing is the easy solution for now:
+        Box::pin(tx_to_aggregator.with(move |msg| async move {
+            Ok(inner_loop::ToAggregator::FromShardWebsocket(
+                shard_conn_id.into(),
+                msg,
+            ))
+        }))
+    }
+
+    /// Return a sink that a feed can send messages into to be handled by the aggregator.
+    pub fn subscribe_feed(
+        &self,
+    ) -> (
+        u64,
+        impl Sink<inner_loop::FromFeedWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static,
+    ) {
+        // Assign a unique aggregator-local ID to each connection that subscribes, and pass
+        // that along with every message to the aggregator loop:
+        let feed_conn_id = self
+            .0
+            .feed_conn_id
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let tx_to_aggregator = self.0.tx_to_aggregator.clone();
+
+        // Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
+        // but pinning by boxing is the easy solution for now:
+        (
+            feed_conn_id,
+            Box::pin(tx_to_aggregator.with(move |msg| async move {
+                Ok(inner_loop::ToAggregator::FromFeedWebsocket(
+                    feed_conn_id.into(),
+                    msg,
+                ))
+            })),
+        )
+    }
+}

backend/telemetry_core/src/aggregator/aggregator_set.rs

@@ -0,0 +1,85 @@
+use super::aggregator::Aggregator;
+use super::inner_loop;
+use common::EitherSink;
+use futures::{Sink, SinkExt, StreamExt};
+use inner_loop::FromShardWebsocket;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct AggregatorSet(Arc<AggregatorSetInner>);
+
+pub struct AggregatorSetInner {
+    aggregators: Vec<Aggregator>,
+    next_idx: AtomicUsize,
+}
+
+impl AggregatorSet {
+    /// Spawn the number of aggregators we're asked to.
+    pub async fn spawn(
+        num_aggregators: usize,
+        denylist: Vec<String>,
+    ) -> anyhow::Result<AggregatorSet> {
+        assert_ne!(num_aggregators, 0, "You must have 1 or more aggregator");
+
+        let aggregators = futures::future::try_join_all(
+            (0..num_aggregators).map(|_| Aggregator::spawn(denylist.clone())),
+        )
+        .await?;
+
+        Ok(AggregatorSet(Arc::new(AggregatorSetInner {
+            aggregators,
+            next_idx: AtomicUsize::new(0),
+        })))
+    }
+
+    /// Return a sink that a shard can send messages into to be handled by all aggregators.
+    pub fn subscribe_shard(
+        &self,
+    ) -> impl Sink<inner_loop::FromShardWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static
+    {
+        // Special case 1 aggregator to avoid the extra indirection and so on
+        // if we don't actually need it.
+        if self.0.aggregators.len() == 1 {
+            let sub = self.0.aggregators[0].subscribe_shard();
+            return EitherSink::a(sub);
+        }
+
+        let mut conns: Vec<_> = self
+            .0
+            .aggregators
+            .iter()
+            .map(|a| a.subscribe_shard())
+            .collect();
+
+        // Send every incoming message to all aggregators.
+        let (tx, mut rx) = futures::channel::mpsc::unbounded::<FromShardWebsocket>();
+        tokio::spawn(async move {
+            while let Some(msg) = rx.next().await {
+                for conn in &mut conns {
+                    // Unbounded channel under the hood, so this await
+                    // shouldn't ever need to yield.
+                    if let Err(e) = conn.send(msg.clone()).await {
+                        log::error!("Aggregator connection has failed: {}", e);
+                        return;
+                    }
+                }
+            }
+        });
+
+        EitherSink::b(tx.sink_map_err(|e| anyhow::anyhow!("{}", e)))
+    }
+
+    /// Return a sink that a feed can send messages into to be handled by a single aggregator.
+    pub fn subscribe_feed(
+        &self,
+    ) -> (
+        u64,
+        impl Sink<inner_loop::FromFeedWebsocket, Error = anyhow::Error> + Send + Sync + Unpin + 'static,
+    ) {
+        let last_val = self.0.next_idx.fetch_add(1, Ordering::Relaxed);
+        let this_idx = (last_val + 1) % self.0.aggregators.len();
+
+        self.0.aggregators[this_idx].subscribe_feed()
+    }
+}

backend/telemetry_core/src/aggregator/inner_loop.rs

@@ -0,0 +1,630 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::aggregator::ConnId;
+use crate::feed_message::{self, FeedMessageSerializer};
+use crate::find_location;
+use crate::state::{self, NodeId, State};
+use bimap::BiMap;
+use common::{
+    internal_messages::{self, MuteReason, ShardNodeId},
+    node_message,
+    node_types::BlockHash,
+    time,
+};
+use futures::channel::mpsc;
+use futures::StreamExt;
+use std::collections::{HashMap, HashSet};
+use std::{
+    net::{IpAddr, Ipv4Addr},
+    str::FromStr,
+};
+
+/// Incoming messages come via subscriptions, and end up looking like this.
+#[derive(Clone, Debug)]
+pub enum ToAggregator {
+    FromShardWebsocket(ConnId, FromShardWebsocket),
+    FromFeedWebsocket(ConnId, FromFeedWebsocket),
+    FromFindLocation(NodeId, find_location::Location),
+}
+
+/// An incoming shard connection can send these messages to the aggregator.
+#[derive(Clone, Debug)]
+pub enum FromShardWebsocket {
+    /// When the socket is opened, it'll send this first
+    /// so that we have a way to communicate back to it.
+    Initialize {
+        channel: mpsc::UnboundedSender<ToShardWebsocket>,
+    },
+    /// Tell the aggregator about a new node.
+    Add {
+        local_id: ShardNodeId,
+        ip: std::net::IpAddr,
+        node: common::node_types::NodeDetails,
+        genesis_hash: common::node_types::BlockHash,
+    },
+    /// Update/pass through details about a node.
+    Update {
+        local_id: ShardNodeId,
+        payload: node_message::Payload,
+    },
+    /// Tell the aggregator that a node has been removed when it disconnects.
+    Remove { local_id: ShardNodeId },
+    /// The shard is disconnected.
+    Disconnected,
+}
+
+/// The aggregator can these messages back to a shard connection.
+#[derive(Debug)]
+pub enum ToShardWebsocket {
+    /// Mute messages to the core by passing the shard-local ID of them.
+    Mute {
+        local_id: ShardNodeId,
+        reason: internal_messages::MuteReason,
+    },
+}
+
+/// An incoming feed connection can send these messages to the aggregator.
+#[derive(Clone, Debug)]
+pub enum FromFeedWebsocket {
+    /// When the socket is opened, it'll send this first
+    /// so that we have a way to communicate back to it.
+    /// Unbounded so that slow feeds don't block aggregato
+    /// progress.
+    Initialize {
+        channel: mpsc::UnboundedSender<ToFeedWebsocket>,
+    },
+    /// The feed can subscribe to a chain to receive
+    /// messages relating to it.
+    Subscribe { chain: Box<str> },
+    /// The feed wants finality info for the chain, too.
+    SendFinality,
+    /// The feed doesn't want any more finality info for the chain.
+    NoMoreFinality,
+    /// An explicit ping message.
+    Ping { value: Box<str> },
+    /// The feed is disconnected.
+    Disconnected,
+}
+
+// The frontend sends text based commands; parse them into these messages:
+impl FromStr for FromFeedWebsocket {
+    type Err = anyhow::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (cmd, value) = match s.find(':') {
+            Some(idx) => (&s[..idx], s[idx + 1..].into()),
+            None => return Err(anyhow::anyhow!("Expecting format `CMD:CHAIN_NAME`")),
+        };
+        match cmd {
+            "ping" => Ok(FromFeedWebsocket::Ping { value }),
+            "subscribe" => Ok(FromFeedWebsocket::Subscribe { chain: value }),
+            "send-finality" => Ok(FromFeedWebsocket::SendFinality),
+            "no-more-finality" => Ok(FromFeedWebsocket::NoMoreFinality),
+            _ => return Err(anyhow::anyhow!("Command {} not recognised", cmd)),
+        }
+    }
+}
+
+/// The aggregator can these messages back to a feed connection.
+#[derive(Clone, Debug)]
+pub enum ToFeedWebsocket {
+    Bytes(bytes::Bytes),
+}
+
+/// Instances of this are responsible for handling incoming and
+/// outgoing messages in the main aggregator loop.
+pub struct InnerLoop {
+    /// Messages from the outside world come into this:
+    rx_from_external: mpsc::UnboundedReceiver<ToAggregator>,
+
+    /// The state of our chains and nodes lives here:
+    node_state: State,
+    /// We maintain a mapping between NodeId and ConnId+LocalId, so that we know
+    /// which messages are about which nodes.
+    node_ids: BiMap<NodeId, (ConnId, ShardNodeId)>,
+
+    /// Keep track of how to send messages out to feeds.
+    feed_channels: HashMap<ConnId, mpsc::UnboundedSender<ToFeedWebsocket>>,
+    /// Keep track of how to send messages out to shards.
+    shard_channels: HashMap<ConnId, mpsc::UnboundedSender<ToShardWebsocket>>,
+
+    /// Which chain is a feed subscribed to?
+    /// Feed Connection ID -> Chain Genesis Hash
+    feed_conn_id_to_chain: HashMap<ConnId, BlockHash>,
+    /// Which feeds are subscribed to a given chain (needs to stay in sync with above)?
+    /// Chain Genesis Hash -> Feed Connection IDs
+    chain_to_feed_conn_ids: HashMap<BlockHash, HashSet<ConnId>>,
+
+    /// These feeds want finality info, too.
+    feed_conn_id_finality: HashSet<ConnId>,
+
+    /// Send messages here to make geographical location requests.
+    tx_to_locator: mpsc::UnboundedSender<(NodeId, Ipv4Addr)>,
+}
+
+impl InnerLoop {
+    /// Create a new inner loop handler with the various state it needs.
+    pub fn new(
+        rx_from_external: mpsc::UnboundedReceiver<ToAggregator>,
+        tx_to_locator: mpsc::UnboundedSender<(NodeId, Ipv4Addr)>,
+        denylist: Vec<String>,
+    ) -> Self {
+        InnerLoop {
+            rx_from_external,
+            node_state: State::new(denylist),
+            node_ids: BiMap::new(),
+            feed_channels: HashMap::new(),
+            shard_channels: HashMap::new(),
+            feed_conn_id_to_chain: HashMap::new(),
+            chain_to_feed_conn_ids: HashMap::new(),
+            feed_conn_id_finality: HashSet::new(),
+            tx_to_locator,
+        }
+    }
+
+    /// Start handling and responding to incoming messages. Owing to unbounded channels, we actually
+    /// only have a single `.await` (in this function). This helps to make it clear that the aggregator loop
+    /// will be able to make progress quickly without any potential yield points.
+    pub async fn handle(mut self) {
+        while let Some(msg) = self.rx_from_external.next().await {
+            match msg {
+                ToAggregator::FromFeedWebsocket(feed_conn_id, msg) => {
+                    self.handle_from_feed(feed_conn_id, msg)
+                }
+                ToAggregator::FromShardWebsocket(shard_conn_id, msg) => {
+                    self.handle_from_shard(shard_conn_id, msg)
+                }
+                ToAggregator::FromFindLocation(node_id, location) => {
+                    self.handle_from_find_location(node_id, location)
+                }
+            }
+        }
+    }
+
+    /// Handle messages that come from the node geographical locator.
+    fn handle_from_find_location(&mut self, node_id: NodeId, location: find_location::Location) {
+        self.node_state
+            .update_node_location(node_id, location.clone());
+
+        if let Some(loc) = location {
+            let mut feed_message_serializer = FeedMessageSerializer::new();
+            feed_message_serializer.push(feed_message::LocatedNode(
+                node_id.get_chain_node_id().into(),
+                loc.latitude,
+                loc.longitude,
+                &loc.city,
+            ));
+
+            let chain_genesis_hash = self
+                .node_state
+                .get_chain_by_node_id(node_id)
+                .map(|chain| *chain.genesis_hash());
+
+            if let Some(chain_genesis_hash) = chain_genesis_hash {
+                self.finalize_and_broadcast_to_chain_feeds(
+                    &chain_genesis_hash,
+                    feed_message_serializer,
+                );
+            }
+        }
+    }
+
+    /// Handle messages coming from shards.
+    fn handle_from_shard(&mut self, shard_conn_id: ConnId, msg: FromShardWebsocket) {
+        match msg {
+            FromShardWebsocket::Initialize { channel } => {
+                self.shard_channels.insert(shard_conn_id, channel);
+            }
+            FromShardWebsocket::Add {
+                local_id,
+                ip,
+                node,
+                genesis_hash,
+            } => {
+                match self.node_state.add_node(genesis_hash, node) {
+                    state::AddNodeResult::ChainOnDenyList => {
+                        if let Some(shard_conn) = self.shard_channels.get_mut(&shard_conn_id) {
+                            let _ = shard_conn.unbounded_send(ToShardWebsocket::Mute {
+                                local_id,
+                                reason: MuteReason::ChainNotAllowed,
+                            });
+                        }
+                    }
+                    state::AddNodeResult::ChainOverQuota => {
+                        if let Some(shard_conn) = self.shard_channels.get_mut(&shard_conn_id) {
+                            let _ = shard_conn.unbounded_send(ToShardWebsocket::Mute {
+                                local_id,
+                                reason: MuteReason::Overquota,
+                            });
+                        }
+                    }
+                    state::AddNodeResult::NodeAddedToChain(details) => {
+                        let node_id = details.id;
+
+                        // Record ID <-> (shardId,localId) for future messages:
+                        self.node_ids.insert(node_id, (shard_conn_id, local_id));
+
+                        // Don't hold onto details too long because we want &mut self later:
+                        let old_chain_label = details.old_chain_label.to_owned();
+                        let new_chain_label = details.new_chain_label.to_owned();
+                        let chain_node_count = details.chain_node_count;
+                        let has_chain_label_changed = details.has_chain_label_changed;
+
+                        // Tell chain subscribers about the node we've just added:
+                        let mut feed_messages_for_chain = FeedMessageSerializer::new();
+                        feed_messages_for_chain.push(feed_message::AddedNode(
+                            node_id.get_chain_node_id().into(),
+                            &details.node,
+                        ));
+                        self.finalize_and_broadcast_to_chain_feeds(
+                            &genesis_hash,
+                            feed_messages_for_chain,
+                        );
+                        // Tell everybody about the new node count and potential rename:
+                        let mut feed_messages_for_all = FeedMessageSerializer::new();
+                        if has_chain_label_changed {
+                            feed_messages_for_all
+                                .push(feed_message::RemovedChain(&old_chain_label));
+                        }
+                        feed_messages_for_all
+                            .push(feed_message::AddedChain(&new_chain_label, chain_node_count));
+                        self.finalize_and_broadcast_to_all_feeds(feed_messages_for_all);
+
+                        // Ask for the grographical location of the node.
+                        // Currently we only geographically locate IPV4 addresses so ignore IPV6.
+                        if let IpAddr::V4(ip_v4) = ip {
+                            let _ = self.tx_to_locator.unbounded_send((node_id, ip_v4));
+                        }
+                    }
+                }
+            }
+            FromShardWebsocket::Remove { local_id } => {
+                let node_id = match self.node_ids.remove_by_right(&(shard_conn_id, local_id)) {
+                    Some((node_id, _)) => node_id,
+                    None => {
+                        log::error!(
+                            "Cannot find ID for node with shard/connectionId of {:?}/{:?}",
+                            shard_conn_id,
+                            local_id
+                        );
+                        return;
+                    }
+                };
+                self.remove_nodes_and_broadcast_result(Some(node_id));
+            }
+            FromShardWebsocket::Update { local_id, payload } => {
+                let node_id = match self.node_ids.get_by_right(&(shard_conn_id, local_id)) {
+                    Some(id) => *id,
+                    None => {
+                        log::error!(
+                            "Cannot find ID for node with shard/connectionId of {:?}/{:?}",
+                            shard_conn_id,
+                            local_id
+                        );
+                        return;
+                    }
+                };
+
+                let mut feed_message_serializer = FeedMessageSerializer::new();
+                let broadcast_finality =
+                    self.node_state
+                        .update_node(node_id, payload, &mut feed_message_serializer);
+
+                if let Some(chain) = self.node_state.get_chain_by_node_id(node_id) {
+                    let genesis_hash = *chain.genesis_hash();
+                    if broadcast_finality {
+                        self.finalize_and_broadcast_to_chain_finality_feeds(
+                            &genesis_hash,
+                            feed_message_serializer,
+                        );
+                    } else {
+                        self.finalize_and_broadcast_to_chain_feeds(
+                            &genesis_hash,
+                            feed_message_serializer,
+                        );
+                    }
+                }
+            }
+            FromShardWebsocket::Disconnected => {
+                // Find all nodes associated with this shard connection ID:
+                let node_ids_to_remove: Vec<NodeId> = self
+                    .node_ids
+                    .iter()
+                    .filter(|(_, &(this_shard_conn_id, _))| shard_conn_id == this_shard_conn_id)
+                    .map(|(&node_id, _)| node_id)
+                    .collect();
+
+                // ... and remove them:
+                self.remove_nodes_and_broadcast_result(node_ids_to_remove);
+            }
+        }
+    }
+
+    /// Handle messages coming from feeds.
+    fn handle_from_feed(&mut self, feed_conn_id: ConnId, msg: FromFeedWebsocket) {
+        match msg {
+            FromFeedWebsocket::Initialize { channel } => {
+                self.feed_channels.insert(feed_conn_id, channel.clone());
+
+                // Tell the new feed subscription some basic things to get it going:
+                let mut feed_serializer = FeedMessageSerializer::new();
+                feed_serializer.push(feed_message::Version(31));
+                for chain in self.node_state.iter_chains() {
+                    feed_serializer
+                        .push(feed_message::AddedChain(chain.label(), chain.node_count()));
+                }
+
+                // Send this to the channel that subscribed:
+                if let Some(bytes) = feed_serializer.into_finalized() {
+                    let _ = channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                }
+            }
+            FromFeedWebsocket::Ping { value } => {
+                let feed_channel = match self.feed_channels.get_mut(&feed_conn_id) {
+                    Some(chan) => chan,
+                    None => return,
+                };
+
+                // Pong!
+                let mut feed_serializer = FeedMessageSerializer::new();
+                feed_serializer.push(feed_message::Pong(&value));
+                if let Some(bytes) = feed_serializer.into_finalized() {
+                    let _ = feed_channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                }
+            }
+            FromFeedWebsocket::Subscribe { chain } => {
+                let feed_channel = match self.feed_channels.get_mut(&feed_conn_id) {
+                    Some(chan) => chan,
+                    None => return,
+                };
+
+                // Unsubscribe from previous chain if subscribed to one:
+                let old_genesis_hash = self.feed_conn_id_to_chain.remove(&feed_conn_id);
+                if let Some(old_genesis_hash) = &old_genesis_hash {
+                    if let Some(map) = self.chain_to_feed_conn_ids.get_mut(old_genesis_hash) {
+                        map.remove(&feed_conn_id);
+                    }
+                }
+
+                // Untoggle request for finality feeds:
+                self.feed_conn_id_finality.remove(&feed_conn_id);
+
+                // Get old chain if there was one:
+                let node_state = &self.node_state;
+                let old_chain =
+                    old_genesis_hash.and_then(|hash| node_state.get_chain_by_genesis_hash(&hash));
+
+                // Get new chain, ignoring the rest if it doesn't exist.
+                let new_chain = match self.node_state.get_chain_by_label(&chain) {
+                    Some(chain) => chain,
+                    None => return,
+                };
+
+                // Send messages to the feed about this subscription:
+                let mut feed_serializer = FeedMessageSerializer::new();
+                if let Some(old_chain) = old_chain {
+                    feed_serializer.push(feed_message::UnsubscribedFrom(old_chain.label()));
+                }
+                feed_serializer.push(feed_message::SubscribedTo(new_chain.label()));
+                feed_serializer.push(feed_message::TimeSync(time::now()));
+                feed_serializer.push(feed_message::BestBlock(
+                    new_chain.best_block().height,
+                    new_chain.timestamp(),
+                    new_chain.average_block_time(),
+                ));
+                feed_serializer.push(feed_message::BestFinalized(
+                    new_chain.finalized_block().height,
+                    new_chain.finalized_block().hash,
+                ));
+                if let Some(bytes) = feed_serializer.into_finalized() {
+                    let _ = feed_channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                }
+
+                // If many (eg 10k) nodes are connected, serializing all of their info takes time.
+                // So, parallelise this with Rayon, but we still send out messages for each node in order
+                // (which is helpful for the UI as it tries to maintain a sorted list of nodes). The chunk
+                // size is the max number of node info we fit into 1 message; smaller messages allow the UI
+                // to react a little faster and not have to wait for a larger update to come in. A chunk size
+                // of 64 means each message is ~32k.
+                use rayon::prelude::*;
+                let all_feed_messages: Vec<_> = new_chain
+                    .nodes_slice()
+                    .par_iter()
+                    .enumerate()
+                    .chunks(64)
+                    .filter_map(|nodes| {
+                        let mut feed_serializer = FeedMessageSerializer::new();
+                        for (node_id, node) in nodes
+                            .iter()
+                            .filter_map(|&(idx, n)| n.as_ref().map(|n| (idx, n)))
+                        {
+                            feed_serializer.push(feed_message::AddedNode(node_id, node));
+                            feed_serializer.push(feed_message::FinalizedBlock(
+                                node_id,
+                                node.finalized().height,
+                                node.finalized().hash,
+                            ));
+                            if node.stale() {
+                                feed_serializer.push(feed_message::StaleNode(node_id));
+                            }
+                        }
+                        feed_serializer.into_finalized()
+                    })
+                    .collect();
+                for bytes in all_feed_messages {
+                    let _ = feed_channel.unbounded_send(ToFeedWebsocket::Bytes(bytes));
+                }
+
+                // Actually make a note of the new chain subsciption:
+                let new_genesis_hash = *new_chain.genesis_hash();
+                self.feed_conn_id_to_chain
+                    .insert(feed_conn_id, new_genesis_hash);
+                self.chain_to_feed_conn_ids
+                    .entry(new_genesis_hash)
+                    .or_default()
+                    .insert(feed_conn_id);
+            }
+            FromFeedWebsocket::SendFinality => {
+                self.feed_conn_id_finality.insert(feed_conn_id);
+            }
+            FromFeedWebsocket::NoMoreFinality => {
+                self.feed_conn_id_finality.remove(&feed_conn_id);
+            }
+            FromFeedWebsocket::Disconnected => {
+                // The feed has disconnected; clean up references to it:
+                if let Some(chain) = self.feed_conn_id_to_chain.remove(&feed_conn_id) {
+                    self.chain_to_feed_conn_ids.remove(&chain);
+                }
+                self.feed_channels.remove(&feed_conn_id);
+                self.feed_conn_id_finality.remove(&feed_conn_id);
+            }
+        }
+    }
+
+    /// Remove all of the node IDs provided and broadcast messages to feeds as needed.
+    fn remove_nodes_and_broadcast_result(&mut self, node_ids: impl IntoIterator<Item = NodeId>) {
+        // Group by chain to simplify the handling of feed messages:
+        let mut node_ids_per_chain: HashMap<BlockHash, Vec<NodeId>> = HashMap::new();
+        for node_id in node_ids.into_iter() {
+            if let Some(chain) = self.node_state.get_chain_by_node_id(node_id) {
+                node_ids_per_chain
+                    .entry(*chain.genesis_hash())
+                    .or_default()
+                    .push(node_id);
+            }
+        }
+
+        // Remove the nodes for each chain
+        let mut feed_messages_for_all = FeedMessageSerializer::new();
+        for (chain_label, node_ids) in node_ids_per_chain {
+            let mut feed_messages_for_chain = FeedMessageSerializer::new();
+            for node_id in node_ids {
+                self.remove_node(
+                    node_id,
+                    &mut feed_messages_for_chain,
+                    &mut feed_messages_for_all,
+                );
+            }
+            self.finalize_and_broadcast_to_chain_feeds(&chain_label, feed_messages_for_chain);
+        }
+        self.finalize_and_broadcast_to_all_feeds(feed_messages_for_all);
+    }
+
+    /// Remove a single node by its ID, pushing any messages we'd want to send
+    /// out to feeds onto the provided feed serializers. Doesn't actually send
+    /// anything to the feeds; just updates state as needed.
+    fn remove_node(
+        &mut self,
+        node_id: NodeId,
+        feed_for_chain: &mut FeedMessageSerializer,
+        feed_for_all: &mut FeedMessageSerializer,
+    ) {
+        // Remove our top level association (this may already have been done).
+        self.node_ids.remove_by_left(&node_id);
+
+        let removed_details = match self.node_state.remove_node(node_id) {
+            Some(remove_details) => remove_details,
+            None => {
+                log::error!("Could not find node {:?}", node_id);
+                return;
+            }
+        };
+
+        // The chain has been removed (no nodes left in it, or it was renamed):
+        if removed_details.chain_node_count == 0 || removed_details.has_chain_label_changed {
+            feed_for_all.push(feed_message::RemovedChain(&removed_details.old_chain_label));
+        }
+
+        // If the chain still exists, tell everybody about the new label or updated node count:
+        if removed_details.chain_node_count != 0 {
+            feed_for_all.push(feed_message::AddedChain(
+                &removed_details.new_chain_label,
+                removed_details.chain_node_count,
+            ));
+        }
+
+        // Assuming the chain hasn't gone away, tell chain subscribers about the node removal
+        if removed_details.chain_node_count != 0 {
+            feed_for_chain.push(feed_message::RemovedNode(
+                node_id.get_chain_node_id().into(),
+            ));
+        }
+    }
+
+    /// Finalize a [`FeedMessageSerializer`] and broadcast the result to feeds for the chain.
+    fn finalize_and_broadcast_to_chain_feeds(
+        &mut self,
+        genesis_hash: &BlockHash,
+        serializer: FeedMessageSerializer,
+    ) {
+        if let Some(bytes) = serializer.into_finalized() {
+            self.broadcast_to_chain_feeds(genesis_hash, ToFeedWebsocket::Bytes(bytes));
+        }
+    }
+
+    /// Send a message to all chain feeds.
+    fn broadcast_to_chain_feeds(&mut self, genesis_hash: &BlockHash, message: ToFeedWebsocket) {
+        if let Some(feeds) = self.chain_to_feed_conn_ids.get(genesis_hash) {
+            for &feed_id in feeds {
+                if let Some(chan) = self.feed_channels.get_mut(&feed_id) {
+                    let _ = chan.unbounded_send(message.clone());
+                }
+            }
+        }
+    }
+
+    /// Finalize a [`FeedMessageSerializer`] and broadcast the result to all feeds
+    fn finalize_and_broadcast_to_all_feeds(&mut self, serializer: FeedMessageSerializer) {
+        if let Some(bytes) = serializer.into_finalized() {
+            self.broadcast_to_all_feeds(ToFeedWebsocket::Bytes(bytes));
+        }
+    }
+
+    /// Send a message to everybody.
+    fn broadcast_to_all_feeds(&mut self, message: ToFeedWebsocket) {
+        for chan in self.feed_channels.values_mut() {
+            let _ = chan.unbounded_send(message.clone());
+        }
+    }
+
+    /// Finalize a [`FeedMessageSerializer`] and broadcast the result to chain finality feeds
+    fn finalize_and_broadcast_to_chain_finality_feeds(
+        &mut self,
+        genesis_hash: &BlockHash,
+        serializer: FeedMessageSerializer,
+    ) {
+        if let Some(bytes) = serializer.into_finalized() {
+            self.broadcast_to_chain_finality_feeds(genesis_hash, ToFeedWebsocket::Bytes(bytes));
+        }
+    }
+
+    /// Send a message to all chain finality feeds.
+    fn broadcast_to_chain_finality_feeds(
+        &mut self,
+        genesis_hash: &BlockHash,
+        message: ToFeedWebsocket,
+    ) {
+        if let Some(feeds) = self.chain_to_feed_conn_ids.get(genesis_hash) {
+            // Get all feeds for the chain, but only broadcast to those feeds that
+            // are also subscribed to receive finality updates.
+            for &feed_id in feeds.union(&self.feed_conn_id_finality) {
+                if let Some(chan) = self.feed_channels.get_mut(&feed_id) {
+                    let _ = chan.unbounded_send(message.clone());
+                }
+            }
+        }
+    }
+}

backend/telemetry_core/src/aggregator/mod.rs

@@ -0,0 +1,24 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod aggregator;
+mod aggregator_set;
+mod inner_loop;
+
+// Expose the various message types that can be worked with externally:
+pub use inner_loop::{FromFeedWebsocket, FromShardWebsocket, ToFeedWebsocket, ToShardWebsocket};
+
+pub use aggregator_set::*;

backend/telemetry_core/src/feed_message.rs

@@ -0,0 +1,235 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! This module provides a way of encoding the various messages that we'll
+//! send to subscribed feeds (browsers).
+
+use serde::Serialize;
+
+use crate::state::Node;
+use common::node_types::{
+    BlockDetails, BlockHash, BlockNumber, NodeHardware, NodeIO, NodeStats, Timestamp,
+};
+use serde_json::to_writer;
+
+type Address = Box<str>;
+type FeedNodeId = usize;
+
+pub trait FeedMessage {
+    const ACTION: u8;
+}
+
+pub trait FeedMessageWrite: FeedMessage {
+    fn write_to_feed(&self, ser: &mut FeedMessageSerializer);
+}
+
+impl<T> FeedMessageWrite for T
+where
+    T: FeedMessage + Serialize,
+{
+    fn write_to_feed(&self, ser: &mut FeedMessageSerializer) {
+        ser.write(self)
+    }
+}
+
+pub struct FeedMessageSerializer {
+    /// Current buffer,
+    buffer: Vec<u8>,
+}
+
+const BUFCAP: usize = 128;
+
+impl FeedMessageSerializer {
+    pub fn new() -> Self {
+        Self {
+            buffer: Vec::with_capacity(BUFCAP),
+        }
+    }
+
+    pub fn push<Message>(&mut self, msg: Message)
+    where
+        Message: FeedMessageWrite,
+    {
+        let glue = match self.buffer.len() {
+            0 => b'[',
+            _ => b',',
+        };
+
+        self.buffer.push(glue);
+        self.write(&Message::ACTION);
+        self.buffer.push(b',');
+        msg.write_to_feed(self);
+    }
+
+    fn write<S>(&mut self, value: &S)
+    where
+        S: Serialize,
+    {
+        let _ = to_writer(&mut self.buffer, value);
+    }
+
+    /// Return the bytes that we've serialized so far, consuming the serializer.
+    pub fn into_finalized(mut self) -> Option<bytes::Bytes> {
+        if self.buffer.is_empty() {
+            return None;
+        }
+
+        self.buffer.push(b']');
+        Some(self.buffer.into())
+    }
+}
+
+macro_rules! actions {
+    ($($action:literal: $t:ty,)*) => {
+        $(
+            impl FeedMessage for $t {
+                const ACTION: u8 = $action;
+            }
+        )*
+    }
+}
+
+actions! {
+     0: Version,
+     1: BestBlock,
+     2: BestFinalized,
+     3: AddedNode<'_>,
+     4: RemovedNode,
+     5: LocatedNode<'_>,
+     6: ImportedBlock<'_>,
+     7: FinalizedBlock,
+     8: NodeStatsUpdate<'_>,
+     9: Hardware<'_>,
+    10: TimeSync,
+    11: AddedChain<'_>,
+    12: RemovedChain<'_>,
+    13: SubscribedTo<'_>,
+    14: UnsubscribedFrom<'_>,
+    15: Pong<'_>,
+    16: AfgFinalized,
+    17: AfgReceivedPrevote,
+    18: AfgReceivedPrecommit,
+    19: AfgAuthoritySet,
+    20: StaleNode,
+    21: NodeIOUpdate<'_>,
+}
+
+#[derive(Serialize)]
+pub struct Version(pub usize);
+
+#[derive(Serialize)]
+pub struct BestBlock(pub BlockNumber, pub Timestamp, pub Option<u64>);
+
+#[derive(Serialize)]
+pub struct BestFinalized(pub BlockNumber, pub BlockHash);
+
+pub struct AddedNode<'a>(pub FeedNodeId, pub &'a Node);
+
+#[derive(Serialize)]
+pub struct RemovedNode(pub FeedNodeId);
+
+#[derive(Serialize)]
+pub struct LocatedNode<'a>(pub FeedNodeId, pub f32, pub f32, pub &'a str);
+
+#[derive(Serialize)]
+pub struct ImportedBlock<'a>(pub FeedNodeId, pub &'a BlockDetails);
+
+#[derive(Serialize)]
+pub struct FinalizedBlock(pub FeedNodeId, pub BlockNumber, pub BlockHash);
+
+#[derive(Serialize)]
+pub struct NodeStatsUpdate<'a>(pub FeedNodeId, pub &'a NodeStats);
+
+#[derive(Serialize)]
+pub struct NodeIOUpdate<'a>(pub FeedNodeId, pub &'a NodeIO);
+
+#[derive(Serialize)]
+pub struct Hardware<'a>(pub FeedNodeId, pub &'a NodeHardware);
+
+#[derive(Serialize)]
+pub struct TimeSync(pub u64);
+
+#[derive(Serialize)]
+pub struct AddedChain<'a>(pub &'a str, pub usize);
+
+#[derive(Serialize)]
+pub struct RemovedChain<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct SubscribedTo<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct UnsubscribedFrom<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct Pong<'a>(pub &'a str);
+
+#[derive(Serialize)]
+pub struct AfgFinalized(pub Address, pub BlockNumber, pub BlockHash);
+
+#[derive(Serialize)]
+pub struct AfgReceivedPrevote(
+    pub Address,
+    pub BlockNumber,
+    pub BlockHash,
+    pub Option<Address>,
+);
+
+#[derive(Serialize)]
+pub struct AfgReceivedPrecommit(
+    pub Address,
+    pub BlockNumber,
+    pub BlockHash,
+    pub Option<Address>,
+);
+
+#[derive(Serialize)]
+pub struct AfgAuthoritySet(
+    pub Address,
+    pub Address,
+    pub Address,
+    pub BlockNumber,
+    pub BlockHash,
+);
+
+#[derive(Serialize)]
+pub struct StaleNode(pub FeedNodeId);
+
+impl FeedMessageWrite for AddedNode<'_> {
+    fn write_to_feed(&self, ser: &mut FeedMessageSerializer) {
+        let AddedNode(nid, node) = self;
+
+        let details = node.details();
+        let details = (
+            &details.name,
+            &details.implementation,
+            &details.version,
+            &details.validator,
+            &details.network_id,
+        );
+
+        ser.write(&(
+            nid,
+            details,
+            node.stats(),
+            node.io(),
+            node.hardware(),
+            node.block_details(),
+            &node.location(),
+            &node.startup_time(),
+        ));
+    }
+}

backend/telemetry_core/src/find_location.rs

@@ -0,0 +1,227 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::net::Ipv4Addr;
+use std::sync::Arc;
+
+use futures::channel::mpsc;
+use futures::{Sink, SinkExt, StreamExt};
+use parking_lot::RwLock;
+use rustc_hash::FxHashMap;
+use serde::Deserialize;
+
+use common::node_types::NodeLocation;
+use tokio::sync::Semaphore;
+
+/// The returned location is optional; it may be None if not found.
+pub type Location = Option<Arc<NodeLocation>>;
+
+/// This is responsible for taking an IP address and attempting
+/// to find a geographical location from this
+pub fn find_location<Id, R>(response_chan: R) -> mpsc::UnboundedSender<(Id, Ipv4Addr)>
+where
+    R: Sink<(Id, Option<Arc<NodeLocation>>)> + Unpin + Send + Clone + 'static,
+    Id: Clone + Send + 'static,
+{
+    let (tx, mut rx) = mpsc::unbounded();
+
+    // cache entries
+    let mut cache: FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>> = FxHashMap::default();
+
+    // Default entry for localhost
+    cache.insert(
+        Ipv4Addr::new(127, 0, 0, 1),
+        Some(Arc::new(NodeLocation {
+            latitude: 52.516_6667,
+            longitude: 13.4,
+            city: "Berlin".into(),
+        })),
+    );
+
+    // Create a locator with our cache. This is used to obtain locations.
+    let locator = Locator::new(cache);
+
+    // Spawn a loop to handle location requests
+    tokio::spawn(async move {
+        // Allow 4 requests at a time. acquiring a token will block while the
+        // number of concurrent location requests is more than this.
+        let semaphore = Arc::new(Semaphore::new(4));
+
+        loop {
+            while let Some((id, ip_address)) = rx.next().await {
+                let permit = semaphore.clone().acquire_owned().await.unwrap();
+                let mut response_chan = response_chan.clone();
+                let locator = locator.clone();
+
+                // Once we have acquired our permit, spawn a task to avoid
+                // blocking this loop so that we can handle concurrent requests.
+                tokio::spawn(async move {
+                    match locator.locate(ip_address).await {
+                        Ok(loc) => {
+                            let _ = response_chan.send((id, loc)).await;
+                        }
+                        Err(e) => {
+                            log::debug!("GET error for ip location: {:?}", e);
+                        }
+                    };
+
+                    // ensure permit is moved into task by dropping it explicitly:
+                    drop(permit);
+                });
+            }
+        }
+    });
+
+    tx
+}
+
+/// This struct can be used to make location requests, given
+/// an IPV4 address.
+#[derive(Clone)]
+struct Locator {
+    client: reqwest::Client,
+    cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
+}
+
+impl Locator {
+    pub fn new(cache: FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>) -> Self {
+        let client = reqwest::Client::new();
+
+        Locator {
+            client,
+            cache: Arc::new(RwLock::new(cache)),
+        }
+    }
+
+    pub async fn locate(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
+        // Return location quickly if it's cached:
+        let cached_loc = {
+            let cache_reader = self.cache.read();
+            cache_reader.get(&ip).cloned()
+        };
+        if let Some(loc) = cached_loc {
+            return Ok(loc);
+        }
+
+        // Look it up via the location services if not cached:
+        let location = self.iplocate_ipapi_co(ip).await?;
+        let location = match location {
+            Some(location) => Ok(Some(location)),
+            None => self.iplocate_ipinfo_io(ip).await,
+        }?;
+
+        self.cache.write().insert(ip, location.clone());
+        Ok(location)
+    }
+
+    async fn iplocate_ipapi_co(
+        &self,
+        ip: Ipv4Addr,
+    ) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
+        let location = self
+            .query(&format!("https://ipapi.co/{}/json", ip))
+            .await?
+            .map(Arc::new);
+
+        Ok(location)
+    }
+
+    async fn iplocate_ipinfo_io(
+        &self,
+        ip: Ipv4Addr,
+    ) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
+        let location = self
+            .query(&format!("https://ipinfo.io/{}/json", ip))
+            .await?
+            .and_then(|loc: IPApiLocate| loc.into_node_location().map(Arc::new));
+
+        Ok(location)
+    }
+
+    async fn query<T>(&self, url: &str) -> Result<Option<T>, reqwest::Error>
+    where
+        for<'de> T: Deserialize<'de>,
+    {
+        match self.client.get(url).send().await?.json::<T>().await {
+            Ok(result) => Ok(Some(result)),
+            Err(err) => {
+                log::debug!("JSON error for ip location: {:?}", err);
+                Ok(None)
+            }
+        }
+    }
+}
+
+/// This is the format returned from ipinfo.co, so we do
+/// a little conversion to get it into the shape we want.
+#[derive(Deserialize)]
+struct IPApiLocate {
+    city: Box<str>,
+    loc: Box<str>,
+}
+
+impl IPApiLocate {
+    fn into_node_location(self) -> Option<NodeLocation> {
+        let IPApiLocate { city, loc } = self;
+
+        let mut loc = loc.split(',').map(|n| n.parse());
+
+        let latitude = loc.next()?.ok()?;
+        let longitude = loc.next()?.ok()?;
+
+        // Guarantee that the iterator has been exhausted
+        if loc.next().is_some() {
+            return None;
+        }
+
+        Some(NodeLocation {
+            latitude,
+            longitude,
+            city,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ipapi_locate_to_node_location() {
+        let ipapi = IPApiLocate {
+            loc: "12.5,56.25".into(),
+            city: "Foobar".into(),
+        };
+
+        let location = ipapi.into_node_location().unwrap();
+
+        assert_eq!(location.latitude, 12.5);
+        assert_eq!(location.longitude, 56.25);
+        assert_eq!(&*location.city, "Foobar");
+    }
+
+    #[test]
+    fn ipapi_locate_to_node_location_too_many() {
+        let ipapi = IPApiLocate {
+            loc: "12.5,56.25,1.0".into(),
+            city: "Foobar".into(),
+        };
+
+        let location = ipapi.into_node_location();
+
+        assert!(location.is_none());
+    }
+}

backend/telemetry_core/src/main.rs

@@ -0,0 +1,468 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod aggregator;
+mod feed_message;
+mod find_location;
+mod state;
+use std::str::FromStr;
+use tokio::time::{Duration, Instant};
+
+use aggregator::{
+    AggregatorSet, FromFeedWebsocket, FromShardWebsocket, ToFeedWebsocket, ToShardWebsocket,
+};
+use bincode::Options;
+use common::http_utils;
+use common::internal_messages;
+use common::ready_chunks_all::ReadyChunksAll;
+use futures::{channel::mpsc, SinkExt, StreamExt};
+use hyper::{Method, Response};
+use simple_logger::SimpleLogger;
+use structopt::StructOpt;
+
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
+const NAME: &str = "Substrate Telemetry Backend Core";
+const ABOUT: &str = "This is the Telemetry Backend Core that receives telemetry messages \
+                     from Substrate/Polkadot nodes and provides the data to a subsribed feed";
+
+#[derive(StructOpt, Debug)]
+#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+struct Opts {
+    /// This is the socket address that Telemetry is listening to. This is restricted to
+    /// localhost (127.0.0.1) by default and should be fine for most use cases. If
+    /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
+    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8000")]
+    socket: std::net::SocketAddr,
+    /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
+    /// 'error' only logs errors and 'trace' logs everything.
+    #[structopt(long = "log", default_value = "info")]
+    log_level: log::LevelFilter,
+    /// Space delimited list of the names of chains that are not allowed to connect to
+    /// telemetry. Case sensitive.
+    #[structopt(long, required = false)]
+    denylist: Vec<String>,
+    /// If it takes longer than this number of seconds to send the current batch of messages
+    /// to a feed, the feed connection will be closed.
+    #[structopt(long, default_value = "10")]
+    feed_timeout: u64,
+    /// Number of worker threads to spawn. If "0" is given, use the number of CPUs available
+    /// on the machine. If no value is given, use an internal default that we have deemed sane.
+    #[structopt(long)]
+    worker_threads: Option<usize>,
+    /// Each aggregator keeps track of the entire node state. Feed subscriptions are split across
+    /// aggregators.
+    #[structopt(long)]
+    num_aggregators: Option<usize>,
+}
+
+fn main() {
+    let opts = Opts::from_args();
+
+    SimpleLogger::new()
+        .with_level(opts.log_level)
+        .init()
+        .expect("Must be able to start a logger");
+
+    log::info!("Starting Telemetry Core version: {}", VERSION);
+
+    let worker_threads = match opts.worker_threads {
+        Some(0) => num_cpus::get(),
+        Some(n) => n,
+        // By default, use a max of 8 worker threads, as perf
+        // testing has found that to be a good sweet spot.
+        None => usize::min(num_cpus::get(), 8),
+    };
+
+    let num_aggregators = match opts.num_aggregators {
+        Some(0) => num_cpus::get(),
+        Some(n) => n,
+        // For now, we just have 1 aggregator loop by default,
+        // but we may want to be smarter here eventually.
+        None => 1,
+    };
+
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .worker_threads(worker_threads)
+        .thread_name("telemetry_core_worker")
+        .build()
+        .unwrap()
+        .block_on(async {
+            if let Err(e) = start_server(num_aggregators, opts).await {
+                log::error!("Error starting server: {}", e);
+            }
+        });
+}
+
+/// Declare our routes and start the server.
+async fn start_server(num_aggregators: usize, opts: Opts) -> anyhow::Result<()> {
+    let aggregator = AggregatorSet::spawn(num_aggregators, opts.denylist).await?;
+    let socket_addr = opts.socket;
+    let feed_timeout = opts.feed_timeout;
+
+    let server = http_utils::start_server(socket_addr, move |addr, req| {
+        let aggregator = aggregator.clone();
+        async move {
+            match (req.method(), req.uri().path().trim_end_matches('/')) {
+                // Check that the server is up and running:
+                (&Method::GET, "/health") => Ok(Response::new("OK".into())),
+                // Subscribe to feed messages:
+                (&Method::GET, "/feed") => {
+                    log::info!("Opening /feed connection from {:?}", addr);
+                    Ok(http_utils::upgrade_to_websocket(
+                        req,
+                        move |ws_send, ws_recv| async move {
+                            let (feed_id, tx_to_aggregator) = aggregator.subscribe_feed();
+                            let (mut tx_to_aggregator, mut ws_send) =
+                                handle_feed_websocket_connection(
+                                    ws_send,
+                                    ws_recv,
+                                    tx_to_aggregator,
+                                    feed_timeout,
+                                    feed_id,
+                                )
+                                .await;
+                            log::info!("Closing /feed connection from {:?}", addr);
+                            // Tell the aggregator that this connection has closed, so it can tidy up.
+                            let _ = tx_to_aggregator.send(FromFeedWebsocket::Disconnected).await;
+                            let _ = ws_send.close().await;
+                        },
+                    ))
+                }
+                // Subscribe to shard messages:
+                (&Method::GET, "/shard_submit") => {
+                    Ok(http_utils::upgrade_to_websocket(
+                        req,
+                        move |ws_send, ws_recv| async move {
+                            log::info!("Opening /shard_submit connection from {:?}", addr);
+                            let tx_to_aggregator = aggregator.subscribe_shard();
+                            let (mut tx_to_aggregator, mut ws_send) =
+                                handle_shard_websocket_connection(
+                                    ws_send,
+                                    ws_recv,
+                                    tx_to_aggregator,
+                                )
+                                .await;
+                            log::info!("Closing /shard_submit connection from {:?}", addr);
+                            // Tell the aggregator that this connection has closed, so it can tidy up.
+                            let _ = tx_to_aggregator
+                                .send(FromShardWebsocket::Disconnected)
+                                .await;
+                            let _ = ws_send.close().await;
+                        },
+                    ))
+                }
+                // 404 for anything else:
+                _ => Ok(Response::builder()
+                    .status(404)
+                    .body("Not found".into())
+                    .unwrap()),
+            }
+        }
+    });
+
+    server.await?;
+    Ok(())
+}
+
+/// This handles messages coming to/from a shard connection
+async fn handle_shard_websocket_connection<S>(
+    mut ws_send: http_utils::WsSender,
+    mut ws_recv: http_utils::WsReceiver,
+    mut tx_to_aggregator: S,
+) -> (S, http_utils::WsSender)
+where
+    S: futures::Sink<FromShardWebsocket, Error = anyhow::Error> + Unpin + Send + 'static,
+{
+    let (tx_to_shard_conn, mut rx_from_aggregator) = mpsc::unbounded();
+
+    // Tell the aggregator about this new connection, and give it a way to send messages to us:
+    let init_msg = FromShardWebsocket::Initialize {
+        channel: tx_to_shard_conn,
+    };
+    if let Err(e) = tx_to_aggregator.send(init_msg).await {
+        log::error!("Error sending message to aggregator: {}", e);
+        return (tx_to_aggregator, ws_send);
+    }
+
+    // Channels to notify each loop if the other closes:
+    let (recv_closer_tx, mut recv_closer_rx) = tokio::sync::oneshot::channel::<()>();
+    let (send_closer_tx, mut send_closer_rx) = tokio::sync::oneshot::channel::<()>();
+
+    // Receive messages from a shard:
+    let recv_handle = tokio::spawn(async move {
+        loop {
+            let mut bytes = Vec::new();
+
+            // Receive a message, or bail if closer called. We don't care about cancel safety;
+            // if we're halfway through receiving a message, no biggie since we're closing the
+            // connection anyway.
+            let msg_info = tokio::select! {
+                msg_info = ws_recv.receive_data(&mut bytes) => msg_info,
+                _ = &mut recv_closer_rx => break
+            };
+
+            // Handle the socket closing, or errors receiving the message.
+            if let Err(soketto::connection::Error::Closed) = msg_info {
+                break;
+            }
+            if let Err(e) = msg_info {
+                log::error!(
+                    "Shutting down websocket connection: Failed to receive data: {}",
+                    e
+                );
+                break;
+            }
+
+            let msg: internal_messages::FromShardAggregator =
+                match bincode::options().deserialize(&bytes) {
+                    Ok(msg) => msg,
+                    Err(e) => {
+                        log::error!(
+                            "Failed to deserialize message from shard; booting it: {}",
+                            e
+                        );
+                        break;
+                    }
+                };
+
+            // Convert and send to the aggregator:
+            let aggregator_msg = match msg {
+                internal_messages::FromShardAggregator::AddNode {
+                    ip,
+                    node,
+                    local_id,
+                    genesis_hash,
+                } => FromShardWebsocket::Add {
+                    ip,
+                    node,
+                    genesis_hash,
+                    local_id,
+                },
+                internal_messages::FromShardAggregator::UpdateNode { payload, local_id } => {
+                    FromShardWebsocket::Update { local_id, payload }
+                }
+                internal_messages::FromShardAggregator::RemoveNode { local_id } => {
+                    FromShardWebsocket::Remove { local_id }
+                }
+            };
+
+            if let Err(e) = tx_to_aggregator.send(aggregator_msg).await {
+                log::error!("Failed to send message to aggregator; closing shard: {}", e);
+                break;
+            }
+        }
+
+        drop(send_closer_tx); // Kill the send task if this recv task ends
+        tx_to_aggregator
+    });
+
+    // Send messages to the shard:
+    let send_handle = tokio::spawn(async move {
+        loop {
+            let msg = tokio::select! {
+                msg = rx_from_aggregator.next() => msg,
+                _ = &mut send_closer_rx => { break }
+            };
+
+            let msg = match msg {
+                Some(msg) => msg,
+                None => break,
+            };
+
+            let internal_msg = match msg {
+                ToShardWebsocket::Mute { local_id, reason } => {
+                    internal_messages::FromTelemetryCore::Mute { local_id, reason }
+                }
+            };
+
+            let bytes = bincode::options()
+                .serialize(&internal_msg)
+                .expect("message to shard should serialize");
+
+            if let Err(e) = ws_send.send_binary(bytes).await {
+                log::error!("Failed to send message to aggregator; closing shard: {}", e)
+            }
+            if let Err(e) = ws_send.flush().await {
+                log::error!(
+                    "Failed to flush message to aggregator; closing shard: {}",
+                    e
+                )
+            }
+        }
+
+        drop(recv_closer_tx); // Kill the recv task if this send task ends
+        ws_send
+    });
+
+    // If our send/recv tasks are stopped (if one of them dies, they both will),
+    // collect the bits we need to hand back from them:
+    let ws_send = send_handle.await.unwrap();
+    let tx_to_aggregator = recv_handle.await.unwrap();
+
+    // loop ended; give socket back to parent:
+    (tx_to_aggregator, ws_send)
+}
+
+/// This handles messages coming from a feed connection
+async fn handle_feed_websocket_connection<S>(
+    mut ws_send: http_utils::WsSender,
+    mut ws_recv: http_utils::WsReceiver,
+    mut tx_to_aggregator: S,
+    feed_timeout: u64,
+    _feed_id: u64, // <- can be useful for debugging purposes.
+) -> (S, http_utils::WsSender)
+where
+    S: futures::Sink<FromFeedWebsocket, Error = anyhow::Error> + Unpin + Send + 'static,
+{
+    // unbounded channel so that slow feeds don't block aggregator progress:
+    let (tx_to_feed_conn, rx_from_aggregator) = mpsc::unbounded();
+    let mut rx_from_aggregator_chunks = ReadyChunksAll::new(rx_from_aggregator);
+
+    // Tell the aggregator about this new connection, and give it a way to send messages to us:
+    let init_msg = FromFeedWebsocket::Initialize {
+        channel: tx_to_feed_conn,
+    };
+    if let Err(e) = tx_to_aggregator.send(init_msg).await {
+        log::error!("Error sending message to aggregator: {}", e);
+        return (tx_to_aggregator, ws_send);
+    }
+
+    // Channels to notify each loop if the other closes:
+    let (recv_closer_tx, mut recv_closer_rx) = tokio::sync::oneshot::channel::<()>();
+    let (send_closer_tx, mut send_closer_rx) = tokio::sync::oneshot::channel::<()>();
+
+    // Receive messages from the feed:
+    let recv_handle = tokio::spawn(async move {
+        loop {
+            let mut bytes = Vec::new();
+
+            // Receive a message, or bail if closer called. We don't care about cancel safety;
+            // if we're halfway through receiving a message, no biggie since we're closing the
+            // connection anyway.
+            let msg_info = tokio::select! {
+                msg_info = ws_recv.receive_data(&mut bytes) => msg_info,
+                _ = &mut recv_closer_rx => { break }
+            };
+
+            // Handle the socket closing, or errors receiving the message.
+            if let Err(soketto::connection::Error::Closed) = msg_info {
+                break;
+            }
+            if let Err(e) = msg_info {
+                log::error!(
+                    "Shutting down websocket connection: Failed to receive data: {}",
+                    e
+                );
+                break;
+            }
+
+            // We ignore all but valid UTF8 text messages from the frontend:
+            let text = match String::from_utf8(bytes) {
+                Ok(s) => s,
+                Err(_) => continue,
+            };
+
+            // Parse the message into a command we understand and send it to the aggregator:
+            let cmd = match FromFeedWebsocket::from_str(&text) {
+                Ok(cmd) => cmd,
+                Err(e) => {
+                    log::warn!(
+                        "Ignoring invalid command '{}' from the frontend: {}",
+                        text,
+                        e
+                    );
+                    continue;
+                }
+            };
+            if let Err(e) = tx_to_aggregator.send(cmd).await {
+                log::error!("Failed to send message to aggregator; closing feed: {}", e);
+                break;
+            }
+        }
+
+        drop(send_closer_tx); // Kill the send task if this recv task ends
+        tx_to_aggregator
+    });
+
+    // Send messages to the feed:
+    let send_handle = tokio::spawn(async move {
+        'outer: loop {
+            let debounce = tokio::time::sleep_until(Instant::now() + Duration::from_millis(75));
+
+            let msgs = tokio::select! {
+                msgs = rx_from_aggregator_chunks.next() => msgs,
+                _ = &mut send_closer_rx => { break }
+            };
+
+            // End the loop when connection from aggregator ends:
+            let msgs = match msgs {
+                Some(msgs) => msgs,
+                None => break,
+            };
+
+            // There is only one message type at the mo; bytes to send
+            // to the websocket. collect them all up to dispatch in one shot.
+            let all_msg_bytes = msgs.into_iter().map(|msg| match msg {
+                ToFeedWebsocket::Bytes(bytes) => bytes,
+            });
+
+            // If the feed is too slow to receive the current batch of messages, we'll drop it.
+            let message_send_deadline = Instant::now() + Duration::from_secs(feed_timeout);
+
+            for bytes in all_msg_bytes {
+                match tokio::time::timeout_at(message_send_deadline, ws_send.send_binary(&bytes))
+                    .await
+                {
+                    Err(_) => {
+                        log::warn!("Closing feed websocket that was too slow to keep up (too slow to send messages)");
+                        break 'outer;
+                    }
+                    Ok(Err(e)) => {
+                        log::warn!("Closing feed websocket due to error sending data: {}", e);
+                        break 'outer;
+                    }
+                    Ok(_) => {}
+                }
+            }
+            match tokio::time::timeout_at(message_send_deadline, ws_send.flush()).await {
+                Err(_) => {
+                    log::warn!("Closing feed websocket that was too slow to keep up (too slow to flush messages)");
+                    break;
+                }
+                Ok(Err(e)) => {
+                    log::warn!("Closing feed websocket due to error flushing data: {}", e);
+                    break;
+                }
+                Ok(_) => {}
+            }
+
+            debounce.await;
+        }
+
+        drop(recv_closer_tx); // Kill the recv task if this send task ends
+        ws_send
+    });
+
+    // If our send/recv tasks are stopped (if one of them dies, they both will),
+    // collect the bits we need to hand back from them:
+    let ws_send = send_handle.await.unwrap();
+    let tx_to_aggregator = recv_handle.await.unwrap();
+
+    // loop ended; give socket back to parent:
+    (tx_to_aggregator, ws_send)
+}

backend/telemetry_core/src/state/chain.rs

@@ -0,0 +1,386 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use common::node_message::Payload;
+use common::node_types::{Block, Timestamp};
+use common::node_types::{BlockHash, BlockNumber};
+use common::{id_type, time, DenseMap, MostSeen, NumStats};
+use once_cell::sync::Lazy;
+use std::collections::HashSet;
+
+use crate::feed_message::{self, FeedMessageSerializer};
+use crate::find_location;
+
+use super::node::Node;
+
+id_type! {
+    /// A Node ID that is unique to the chain it's in.
+    pub struct ChainNodeId(usize)
+}
+
+pub type Label = Box<str>;
+
+const STALE_TIMEOUT: u64 = 2 * 60 * 1000; // 2 minutes
+
+pub struct Chain {
+    /// Labels that nodes use for this chain. We keep track of
+    /// the most commonly used label as nodes are added/removed.
+    labels: MostSeen<Label>,
+    /// Set of nodes that are in this chain
+    nodes: DenseMap<ChainNodeId, Node>,
+    /// Best block
+    best: Block,
+    /// Finalized block
+    finalized: Block,
+    /// Block times history, stored so we can calculate averages
+    block_times: NumStats<u64>,
+    /// Calculated average block time
+    average_block_time: Option<u64>,
+    /// When the best block first arrived
+    timestamp: Option<Timestamp>,
+    /// Genesis hash of this chain
+    genesis_hash: BlockHash,
+}
+
+pub enum AddNodeResult {
+    Overquota,
+    Added {
+        id: ChainNodeId,
+        chain_renamed: bool,
+    },
+}
+
+pub struct RemoveNodeResult {
+    pub chain_renamed: bool,
+}
+
+/// Labels of chains we consider "first party". These chains allow any
+/// number of nodes to connect.
+static FIRST_PARTY_NETWORKS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
+    let mut set = HashSet::new();
+    set.insert("Polkadot");
+    set.insert("Kusama");
+    set.insert("Westend");
+    set.insert("Rococo");
+    set
+});
+
+/// Max number of nodes allowed to connect to the telemetry server.
+const THIRD_PARTY_NETWORKS_MAX_NODES: usize = 500;
+
+impl Chain {
+    /// Create a new chain with an initial label.
+    pub fn new(genesis_hash: BlockHash) -> Self {
+        Chain {
+            labels: MostSeen::default(),
+            nodes: DenseMap::new(),
+            best: Block::zero(),
+            finalized: Block::zero(),
+            block_times: NumStats::new(50),
+            average_block_time: None,
+            timestamp: None,
+            genesis_hash,
+        }
+    }
+
+    /// Can we add a node? If not, it's because the chain is at its quota.
+    pub fn is_overquota(&self) -> bool {
+        // Dynamically determine the max nodes based on the most common
+        // label so far, in case it changes to something with a different limit.
+        self.nodes.len() < max_nodes(self.labels.best())
+    }
+
+    /// Assign a node to this chain.
+    pub fn add_node(&mut self, node: Node) -> AddNodeResult {
+        if !self.is_overquota() {
+            return AddNodeResult::Overquota;
+        }
+
+        let node_chain_label = &node.details().chain;
+        let label_result = self.labels.insert(node_chain_label);
+        let node_id = self.nodes.add(node);
+
+        AddNodeResult::Added {
+            id: node_id,
+            chain_renamed: label_result.has_changed(),
+        }
+    }
+
+    /// Remove a node from this chain.
+    pub fn remove_node(&mut self, node_id: ChainNodeId) -> RemoveNodeResult {
+        let node = match self.nodes.remove(node_id) {
+            Some(node) => node,
+            None => {
+                return RemoveNodeResult {
+                    chain_renamed: false,
+                }
+            }
+        };
+
+        let node_chain_label = &node.details().chain;
+        let label_result = self.labels.remove(node_chain_label);
+
+        RemoveNodeResult {
+            chain_renamed: label_result.has_changed(),
+        }
+    }
+
+    /// Attempt to update the best block seen in this chain.
+    /// Returns a boolean which denotes whether the output is for finalization feeds (true) or not (false).
+    pub fn update_node(
+        &mut self,
+        nid: ChainNodeId,
+        payload: Payload,
+        feed: &mut FeedMessageSerializer,
+    ) -> bool {
+        if let Some(block) = payload.best_block() {
+            self.handle_block(block, nid, feed);
+        }
+
+        if let Some(node) = self.nodes.get_mut(nid) {
+            match payload {
+                Payload::SystemInterval(ref interval) => {
+                    if node.update_hardware(interval) {
+                        feed.push(feed_message::Hardware(nid.into(), node.hardware()));
+                    }
+
+                    if let Some(stats) = node.update_stats(interval) {
+                        feed.push(feed_message::NodeStatsUpdate(nid.into(), stats));
+                    }
+
+                    if let Some(io) = node.update_io(interval) {
+                        feed.push(feed_message::NodeIOUpdate(nid.into(), io));
+                    }
+                }
+                Payload::AfgAuthoritySet(authority) => {
+                    node.set_validator_address(authority.authority_id.clone());
+                    return false;
+                }
+                Payload::AfgFinalized(finalized) => {
+                    if let Ok(finalized_number) = finalized.finalized_number.parse::<BlockNumber>()
+                    {
+                        if let Some(addr) = node.details().validator.clone() {
+                            feed.push(feed_message::AfgFinalized(
+                                addr,
+                                finalized_number,
+                                finalized.finalized_hash,
+                            ));
+                        }
+                    }
+                    return true;
+                }
+                Payload::AfgReceivedPrecommit(precommit) => {
+                    if let Ok(finalized_number) = precommit.target_number.parse::<BlockNumber>() {
+                        if let Some(addr) = node.details().validator.clone() {
+                            let voter = precommit.voter.clone();
+                            feed.push(feed_message::AfgReceivedPrecommit(
+                                addr,
+                                finalized_number,
+                                precommit.target_hash,
+                                voter,
+                            ));
+                        }
+                    }
+                    return true;
+                }
+                Payload::AfgReceivedPrevote(prevote) => {
+                    if let Ok(finalized_number) = prevote.target_number.parse::<BlockNumber>() {
+                        if let Some(addr) = node.details().validator.clone() {
+                            let voter = prevote.voter.clone();
+                            feed.push(feed_message::AfgReceivedPrevote(
+                                addr,
+                                finalized_number,
+                                prevote.target_hash,
+                                voter,
+                            ));
+                        }
+                    }
+                    return true;
+                }
+                Payload::AfgReceivedCommit(_) => {}
+                _ => (),
+            }
+
+            if let Some(block) = payload.finalized_block() {
+                if let Some(finalized) = node.update_finalized(block) {
+                    feed.push(feed_message::FinalizedBlock(
+                        nid.into(),
+                        finalized.height,
+                        finalized.hash,
+                    ));
+
+                    if finalized.height > self.finalized.height {
+                        self.finalized = *finalized;
+                        feed.push(feed_message::BestFinalized(
+                            finalized.height,
+                            finalized.hash,
+                        ));
+                    }
+                }
+            }
+        }
+
+        false
+    }
+
+    fn handle_block(&mut self, block: &Block, nid: ChainNodeId, feed: &mut FeedMessageSerializer) {
+        let mut propagation_time = None;
+        let now = time::now();
+        let nodes_len = self.nodes.len();
+
+        self.update_stale_nodes(now, feed);
+
+        let node = match self.nodes.get_mut(nid) {
+            Some(node) => node,
+            None => return,
+        };
+
+        if node.update_block(*block) {
+            if block.height > self.best.height {
+                self.best = *block;
+                log::debug!(
+                    "[{}] [nodes={}] new best block={}/{:?}",
+                    self.labels.best(),
+                    nodes_len,
+                    self.best.height,
+                    self.best.hash,
+                );
+                if let Some(timestamp) = self.timestamp {
+                    self.block_times.push(now - timestamp);
+                    self.average_block_time = Some(self.block_times.average());
+                }
+                self.timestamp = Some(now);
+                feed.push(feed_message::BestBlock(
+                    self.best.height,
+                    now,
+                    self.average_block_time,
+                ));
+                propagation_time = Some(0);
+            } else if block.height == self.best.height {
+                if let Some(timestamp) = self.timestamp {
+                    propagation_time = Some(now - timestamp);
+                }
+            }
+
+            if let Some(details) = node.update_details(now, propagation_time) {
+                feed.push(feed_message::ImportedBlock(nid.into(), details));
+            }
+        }
+    }
+
+    /// Check if the chain is stale (has not received a new best block in a while).
+    /// If so, find a new best block, ignoring any stale nodes and marking them as such.
+    fn update_stale_nodes(&mut self, now: u64, feed: &mut FeedMessageSerializer) {
+        let threshold = now - STALE_TIMEOUT;
+        let timestamp = match self.timestamp {
+            Some(ts) => ts,
+            None => return,
+        };
+
+        if timestamp > threshold {
+            // Timestamp is in range, nothing to do
+            return;
+        }
+
+        let mut best = Block::zero();
+        let mut finalized = Block::zero();
+        let mut timestamp = None;
+
+        for (nid, node) in self.nodes.iter_mut() {
+            if !node.update_stale(threshold) {
+                if node.best().height > best.height {
+                    best = *node.best();
+                    timestamp = Some(node.best_timestamp());
+                }
+
+                if node.finalized().height > finalized.height {
+                    finalized = *node.finalized();
+                }
+            } else {
+                feed.push(feed_message::StaleNode(nid.into()));
+            }
+        }
+
+        if self.best.height != 0 || self.finalized.height != 0 {
+            self.best = best;
+            self.finalized = finalized;
+            self.block_times.reset();
+            self.timestamp = timestamp;
+
+            feed.push(feed_message::BestBlock(
+                self.best.height,
+                timestamp.unwrap_or(now),
+                None,
+            ));
+            feed.push(feed_message::BestFinalized(
+                finalized.height,
+                finalized.hash,
+            ));
+        }
+    }
+
+    pub fn update_node_location(
+        &mut self,
+        node_id: ChainNodeId,
+        location: find_location::Location,
+    ) -> bool {
+        if let Some(node) = self.nodes.get_mut(node_id) {
+            node.update_location(location);
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn get_node(&self, id: ChainNodeId) -> Option<&Node> {
+        self.nodes.get(id)
+    }
+    pub fn nodes_slice(&self) -> &[Option<Node>] {
+        self.nodes.as_slice()
+    }
+    pub fn label(&self) -> &str {
+        &self.labels.best()
+    }
+    pub fn node_count(&self) -> usize {
+        self.nodes.len()
+    }
+    pub fn best_block(&self) -> &Block {
+        &self.best
+    }
+    pub fn timestamp(&self) -> Option<Timestamp> {
+        self.timestamp
+    }
+    pub fn average_block_time(&self) -> Option<u64> {
+        self.average_block_time
+    }
+    pub fn finalized_block(&self) -> &Block {
+        &self.finalized
+    }
+    pub fn genesis_hash(&self) -> &BlockHash {
+        &self.genesis_hash
+    }
+}
+
+/// First party networks (Polkadot, Kusama etc) are allowed any number of nodes.
+/// Third party networks are allowed `THIRD_PARTY_NETWORKS_MAX_NODES` nodes and
+/// no more.
+fn max_nodes(label: &str) -> usize {
+    if FIRST_PARTY_NETWORKS.contains(label) {
+        usize::MAX
+    } else {
+        THIRD_PARTY_NETWORKS_MAX_NODES
+    }
+}

backend/telemetry_core/src/state/mod.rs

@@ -0,0 +1,23 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod chain;
+mod node;
+
+mod state;
+
+pub use node::Node;
+pub use state::*;

backend/telemetry_core/src/state/node.rs

@@ -1,15 +1,25 @@
-use std::sync::Arc;
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use crate::types::{
-    Block, BlockDetails, NodeDetails, NodeHardware, NodeIO, NodeId, NodeLocation, NodeStats,
-    Timestamp,
+use crate::find_location;
+use common::node_message::SystemInterval;
+use common::node_types::{
+    Block, BlockDetails, NodeDetails, NodeHardware, NodeIO, NodeLocation, NodeStats, Timestamp,
 };
-use crate::util::now;
-
-pub mod connector;
-pub mod message;
-
-use message::SystemInterval;
+use common::time;
 
 /// Minimum time between block below broadcasting updates to the browser gets throttled, in ms.
 const THROTTLE_THRESHOLD: u64 = 100;
@@ -32,7 +42,7 @@ pub struct Node {
     /// Hardware stats over time
     hardware: NodeHardware,
     /// Physical location details
-    location: Option<Arc<NodeLocation>>,
+    location: find_location::Location,
     /// Flag marking if the node is stale (not syncing or producing blocks)
     stale: bool,
     /// Unix timestamp for when node started up (falls back to connection time)
@@ -92,8 +102,8 @@ impl Node {
         self.location.as_deref()
     }
 
-    pub fn update_location(&mut self, location: Arc<NodeLocation>) {
-        self.location = Some(location);
+    pub fn update_location(&mut self, location: find_location::Location) {
+        self.location = location;
     }
 
     pub fn block_details(&self) -> &BlockDetails {
@@ -140,7 +150,7 @@ impl Node {
         if let Some(download) = interval.bandwidth_download {
             changed |= self.hardware.download.push(download);
         }
-        self.hardware.chart_stamps.push(now() as f64);
+        self.hardware.chart_stamps.push(time::now() as f64);
 
         changed
     }

backend/telemetry_core/src/state/state.rs

@@ -0,0 +1,430 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::node::Node;
+use crate::feed_message::FeedMessageSerializer;
+use crate::find_location;
+use common::node_message::Payload;
+use common::node_types::{Block, BlockHash, NodeDetails, Timestamp};
+use common::{id_type, DenseMap};
+use std::collections::{HashMap, HashSet};
+use std::iter::IntoIterator;
+
+use super::chain::{self, Chain, ChainNodeId};
+
+id_type! {
+    /// A globally unique Chain ID.
+    pub struct ChainId(usize)
+}
+
+/// A "global" Node ID is a composite of the ID of the chain it's
+/// on, and it's chain local ID.
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct NodeId(ChainId, ChainNodeId);
+
+impl NodeId {
+    pub fn get_chain_node_id(&self) -> ChainNodeId {
+        self.1
+    }
+}
+
+/// Our state constains node and chain information
+pub struct State {
+    chains: DenseMap<ChainId, Chain>,
+
+    // Find the right chain given various details.
+    chains_by_genesis_hash: HashMap<BlockHash, ChainId>,
+    chains_by_label: HashMap<Box<str>, ChainId>,
+
+    /// Chain labels that we do not want to allow connecting.
+    denylist: HashSet<String>,
+}
+
+/// Adding a node to a chain leads to this node_idult
+pub enum AddNodeResult<'a> {
+    /// The chain is on the "deny list", so we can't add the node
+    ChainOnDenyList,
+    /// The chain is over quota (too many nodes connected), so can't add the node
+    ChainOverQuota,
+    /// The node was added to the chain
+    NodeAddedToChain(NodeAddedToChain<'a>),
+}
+
+#[cfg(test)]
+impl<'a> AddNodeResult<'a> {
+    pub fn unwrap_id(&self) -> NodeId {
+        match &self {
+            AddNodeResult::NodeAddedToChain(d) => d.id,
+            _ => panic!("Attempt to unwrap_id on AddNodeResult that did not succeed"),
+        }
+    }
+}
+
+pub struct NodeAddedToChain<'a> {
+    /// The ID assigned to this node.
+    pub id: NodeId,
+    /// The old label of the chain.
+    pub old_chain_label: Box<str>,
+    /// The new label of the chain.
+    pub new_chain_label: &'a str,
+    /// The node that was added.
+    pub node: &'a Node,
+    /// Number of nodes in the chain. If 1, the chain was just added.
+    pub chain_node_count: usize,
+    /// Has the chain label been updated?
+    pub has_chain_label_changed: bool,
+}
+
+/// if removing a node is successful, we get this information back.
+pub struct RemovedNode {
+    /// How many nodes remain on the chain (0 if the chain was removed)
+    pub chain_node_count: usize,
+    /// Has the chain label been updated?
+    pub has_chain_label_changed: bool,
+    /// The old label of the chain.
+    pub old_chain_label: Box<str>,
+    /// The new label of the chain.
+    pub new_chain_label: Box<str>,
+}
+
+impl State {
+    pub fn new<T: IntoIterator<Item = String>>(denylist: T) -> State {
+        State {
+            chains: DenseMap::new(),
+            chains_by_genesis_hash: HashMap::new(),
+            chains_by_label: HashMap::new(),
+            denylist: denylist.into_iter().collect(),
+        }
+    }
+
+    pub fn iter_chains(&self) -> impl Iterator<Item = StateChain<'_>> {
+        self.chains
+            .iter()
+            .map(move |(_, chain)| StateChain { chain })
+    }
+
+    pub fn get_chain_by_node_id(&self, node_id: NodeId) -> Option<StateChain<'_>> {
+        self.chains.get(node_id.0).map(|chain| StateChain { chain })
+    }
+
+    pub fn get_chain_by_genesis_hash(&self, genesis_hash: &BlockHash) -> Option<StateChain<'_>> {
+        self.chains_by_genesis_hash
+            .get(genesis_hash)
+            .and_then(|&chain_id| self.chains.get(chain_id))
+            .map(|chain| StateChain { chain })
+    }
+
+    pub fn get_chain_by_label(&self, label: &str) -> Option<StateChain<'_>> {
+        self.chains_by_label
+            .get(label)
+            .and_then(|&chain_id| self.chains.get(chain_id))
+            .map(|chain| StateChain { chain })
+    }
+
+    pub fn add_node(
+        &mut self,
+        genesis_hash: BlockHash,
+        node_details: NodeDetails,
+    ) -> AddNodeResult<'_> {
+        if self.denylist.contains(&*node_details.chain) {
+            return AddNodeResult::ChainOnDenyList;
+        }
+
+        // Get the chain ID, creating a new empty chain if one doesn't exist.
+        // If we create a chain here, we are expecting that it will allow at
+        // least this node to be added, because we don't currently try and clean it up
+        // if the add fails.
+        let chain_id = match self.chains_by_genesis_hash.get(&genesis_hash) {
+            Some(id) => *id,
+            None => {
+                let chain_id = self.chains.add(Chain::new(genesis_hash));
+                self.chains_by_genesis_hash.insert(genesis_hash, chain_id);
+                chain_id
+            }
+        };
+
+        // Get the chain.
+        let chain = self.chains.get_mut(chain_id).expect(
+            "should be known to exist after the above (unless chains_by_genesis_hash out of sync)",
+        );
+
+        let node = Node::new(node_details);
+        let old_chain_label = chain.label().into();
+
+        match chain.add_node(node) {
+            chain::AddNodeResult::Overquota => AddNodeResult::ChainOverQuota,
+            chain::AddNodeResult::Added { id, chain_renamed } => {
+                let chain = &*chain;
+
+                // Update the label we use to reference the chain if
+                // it changes (it'll always change first time a node's added):
+                if chain_renamed {
+                    self.chains_by_label.remove(&old_chain_label);
+                    self.chains_by_label.insert(chain.label().into(), chain_id);
+                }
+
+                AddNodeResult::NodeAddedToChain(NodeAddedToChain {
+                    id: NodeId(chain_id, id),
+                    node: chain.get_node(id).expect("node added above"),
+                    old_chain_label: old_chain_label,
+                    new_chain_label: chain.label(),
+                    chain_node_count: chain.node_count(),
+                    has_chain_label_changed: chain_renamed,
+                })
+            }
+        }
+    }
+
+    /// Remove a node
+    pub fn remove_node(&mut self, NodeId(chain_id, chain_node_id): NodeId) -> Option<RemovedNode> {
+        let chain = self.chains.get_mut(chain_id)?;
+        let old_chain_label = chain.label().into();
+
+        // Actually remove the node
+        let remove_result = chain.remove_node(chain_node_id);
+
+        // Get updated chain details.
+        let new_chain_label: Box<str> = chain.label().into();
+        let chain_node_count = chain.node_count();
+
+        // Is the chain empty? Remove if so and clean up indexes to it
+        if chain_node_count == 0 {
+            let genesis_hash = *chain.genesis_hash();
+            self.chains_by_label.remove(&old_chain_label);
+            self.chains_by_genesis_hash.remove(&genesis_hash);
+            self.chains.remove(chain_id);
+        }
+
+        // Make sure chains always referenced by their most common label:
+        if remove_result.chain_renamed {
+            self.chains_by_label.remove(&old_chain_label);
+            self.chains_by_label
+                .insert(new_chain_label.clone(), chain_id);
+        }
+
+        Some(RemovedNode {
+            old_chain_label,
+            new_chain_label,
+            chain_node_count: chain_node_count,
+            has_chain_label_changed: remove_result.chain_renamed,
+        })
+    }
+
+    /// Attempt to update the best block seen, given a node and block.
+    /// Returns a boolean which denotes whether the output is for finalization feeds (true) or not (false).
+    pub fn update_node(
+        &mut self,
+        NodeId(chain_id, chain_node_id): NodeId,
+        payload: Payload,
+        feed: &mut FeedMessageSerializer,
+    ) -> bool {
+        let chain = match self.chains.get_mut(chain_id) {
+            Some(chain) => chain,
+            None => {
+                log::error!("Cannot find chain for node with ID {:?}", chain_id);
+                return false;
+            }
+        };
+
+        chain.update_node(chain_node_id, payload, feed)
+    }
+
+    /// Update the location for a node. Return `false` if the node was not found.
+    pub fn update_node_location(
+        &mut self,
+        NodeId(chain_id, chain_node_id): NodeId,
+        location: find_location::Location,
+    ) -> bool {
+        if let Some(chain) = self.chains.get_mut(chain_id) {
+            chain.update_node_location(chain_node_id, location)
+        } else {
+            false
+        }
+    }
+}
+
+/// When we ask for a chain, we get this struct back. This ensures that we have
+/// a consistent public interface, and don't expose methods on [`Chain`] that
+/// aren't really intended for use outside of [`State`] methods. Any modification
+/// of a chain needs to go through [`State`].
+pub struct StateChain<'a> {
+    chain: &'a Chain,
+}
+
+impl<'a> StateChain<'a> {
+    pub fn label(&self) -> &'a str {
+        self.chain.label()
+    }
+    pub fn genesis_hash(&self) -> &'a BlockHash {
+        self.chain.genesis_hash()
+    }
+    pub fn node_count(&self) -> usize {
+        self.chain.node_count()
+    }
+    pub fn best_block(&self) -> &'a Block {
+        self.chain.best_block()
+    }
+    pub fn timestamp(&self) -> Timestamp {
+        self.chain.timestamp().unwrap_or(0)
+    }
+    pub fn average_block_time(&self) -> Option<u64> {
+        self.chain.average_block_time()
+    }
+    pub fn finalized_block(&self) -> &'a Block {
+        self.chain.finalized_block()
+    }
+    pub fn nodes_slice(&self) -> &[Option<Node>] {
+        self.chain.nodes_slice()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn node(name: &str, chain: &str) -> NodeDetails {
+        NodeDetails {
+            chain: chain.into(),
+            name: name.into(),
+            implementation: "Bar".into(),
+            version: "0.1".into(),
+            validator: None,
+            network_id: None,
+            startup_time: None,
+        }
+    }
+
+    #[test]
+    fn adding_a_node_returns_expected_response() {
+        let mut state = State::new(None);
+
+        let chain1_genesis = BlockHash::from_low_u64_be(1);
+
+        let add_result = state.add_node(chain1_genesis, node("A", "Chain One"));
+
+        let add_node_result = match add_result {
+            AddNodeResult::ChainOnDenyList => panic!("Chain not on deny list"),
+            AddNodeResult::ChainOverQuota => panic!("Chain not Overquota"),
+            AddNodeResult::NodeAddedToChain(details) => details,
+        };
+
+        assert_eq!(add_node_result.id, NodeId(0.into(), 0.into()));
+        assert_eq!(&*add_node_result.old_chain_label, "");
+        assert_eq!(&*add_node_result.new_chain_label, "Chain One");
+        assert_eq!(add_node_result.chain_node_count, 1);
+        assert_eq!(add_node_result.has_chain_label_changed, true);
+
+        let add_result = state.add_node(chain1_genesis, node("A", "Chain One"));
+
+        let add_node_result = match add_result {
+            AddNodeResult::ChainOnDenyList => panic!("Chain not on deny list"),
+            AddNodeResult::ChainOverQuota => panic!("Chain not Overquota"),
+            AddNodeResult::NodeAddedToChain(details) => details,
+        };
+
+        assert_eq!(add_node_result.id, NodeId(0.into(), 1.into()));
+        assert_eq!(&*add_node_result.old_chain_label, "Chain One");
+        assert_eq!(&*add_node_result.new_chain_label, "Chain One");
+        assert_eq!(add_node_result.chain_node_count, 2);
+        assert_eq!(add_node_result.has_chain_label_changed, false);
+    }
+
+    #[test]
+    fn adding_and_removing_nodes_updates_chain_label_mapping() {
+        let mut state = State::new(None);
+
+        let chain1_genesis = BlockHash::from_low_u64_be(1);
+        let node_id0 = state
+            .add_node(chain1_genesis, node("A", "Chain One")) // 0
+            .unwrap_id();
+
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain One"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+
+        let node_id1 = state
+            .add_node(chain1_genesis, node("B", "Chain Two")) // 1
+            .unwrap_id();
+
+        // Chain name hasn't changed yet; "Chain One" as common as "Chain Two"..
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain One"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+
+        let node_id2 = state
+            .add_node(chain1_genesis, node("B", "Chain Two"))
+            .unwrap_id(); // 2
+
+        // Chain name has changed; "Chain Two" the winner now..
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain Two"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_none());
+        assert!(state.get_chain_by_label("Chain Two").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+
+        state.remove_node(node_id1).expect("Removal OK (id: 1)");
+        state.remove_node(node_id2).expect("Removal OK (id: 2)");
+
+        // Removed both "Chain Two" nodes; dominant name now "Chain One" again..
+        assert_eq!(
+            state
+                .get_chain_by_node_id(node_id0)
+                .expect("Chain should exist")
+                .label(),
+            "Chain One"
+        );
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_label("Chain Two").is_none());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+    }
+
+    #[test]
+    fn chain_removed_when_last_node_is() {
+        let mut state = State::new(None);
+
+        let chain1_genesis = BlockHash::from_low_u64_be(1);
+        let node_id = state
+            .add_node(chain1_genesis, node("A", "Chain One")) // 0
+            .unwrap_id();
+
+        assert!(state.get_chain_by_label("Chain One").is_some());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_some());
+        assert_eq!(state.iter_chains().count(), 1);
+
+        state.remove_node(node_id);
+
+        assert!(state.get_chain_by_label("Chain One").is_none());
+        assert!(state.get_chain_by_genesis_hash(&chain1_genesis).is_none());
+        assert_eq!(state.iter_chains().count(), 0);
+    }
+}

backend/telemetry_core/tests/e2e_tests.rs

@@ -0,0 +1,818 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/*!
+General end-to-end tests
+
+Note that on MacOS inparticular, you may need to increase some limits to be
+able to open a large number of connections and run some of the tests.
+Try running these:
+
+```sh
+sudo sysctl -w kern.maxfiles=100000
+sudo sysctl -w kern.maxfilesperproc=100000
+ulimit -n 100000
+sudo sysctl -w kern.ipc.somaxconn=100000
+sudo sysctl -w kern.ipc.maxsockbuf=16777216
+```
+*/
+
+use common::node_types::BlockHash;
+use common::ws_client::SentMessage;
+use serde_json::json;
+use std::time::Duration;
+use test_utils::{
+    assert_contains_matches,
+    feed_message_de::{FeedMessage, NodeDetails},
+    workspace::{start_server, start_server_debug, CoreOpts, ServerOpts, ShardOpts},
+};
+
+/// The simplest test we can run; the main benefit of this test (since we check similar)
+/// below) is just to give a feel for _how_ we can test basic feed related things.
+#[tokio::test]
+async fn feed_sent_version_on_connect() {
+    let server = start_server_debug().await;
+
+    // Connect a feed:
+    let (_feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    // Expect a version response of 31:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_eq!(
+        feed_messages,
+        vec![FeedMessage::Version(31)],
+        "expecting version"
+    );
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// Another very simple test: pings from feeds should be responded to by pongs
+/// with the same message content.
+#[tokio::test]
+async fn feed_ping_responded_to_with_pong() {
+    let server = start_server_debug().await;
+
+    // Connect a feed:
+    let (feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    // Ping it:
+    feed_tx.send_command("ping", "hello!").unwrap();
+
+    // Expect a pong response:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(
+        feed_messages.contains(&FeedMessage::Pong {
+            msg: "hello!".to_owned()
+        }),
+        "Expecting pong"
+    );
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// As a prelude to `lots_of_mute_messages_dont_cause_a_deadlock`, we can check that
+/// a lot of nodes can simultaneously subscribe and are all sent the expected response.
+#[tokio::test]
+async fn multiple_feeds_sent_version_on_connect() {
+    let server = start_server_debug().await;
+
+    // Connect a bunch of feeds:
+    let mut feeds = server
+        .get_core()
+        .connect_multiple_feeds(1000)
+        .await
+        .unwrap();
+
+    // Wait for responses all at once:
+    let responses =
+        futures::future::join_all(feeds.iter_mut().map(|(_, rx)| rx.recv_feed_messages()));
+
+    let responses = tokio::time::timeout(Duration::from_secs(10), responses)
+        .await
+        .expect("we shouldn't hit a timeout waiting for responses");
+
+    // Expect a version response of 31 to all of them:
+    for feed_messages in responses {
+        assert_eq!(
+            feed_messages.expect("should have messages"),
+            vec![FeedMessage::Version(31)],
+            "expecting version"
+        );
+    }
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// When a lot of nodes are added, the chain becomes overquota.
+/// This leads to a load of messages being sent back to the shard. If bounded channels
+/// are used to send messages back to the shard, it's possible that we get into a situation
+/// where the shard is waiting trying to send the next "add node" message, while the
+/// telemetry core is waiting trying to send up to the shard the next "mute node" message,
+/// resulting in a deadlock. This test gives confidence that we don't run into such a deadlock.
+#[tokio::test]
+async fn lots_of_mute_messages_dont_cause_a_deadlock() {
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+
+    // Connect 1000 nodes to the shard:
+    let mut nodes = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_multiple_nodes(2000) // 1500 of these will be overquota.
+        .await
+        .expect("nodes can connect");
+
+    // Every node announces itself on the same chain:
+    for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
+        node_tx
+            .send_json_text(json!({
+                "id":1, // message ID, not node ID. Can be the same for all.
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain":"Local Testnet",
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(1),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name": format!("Alice {}", idx),
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                }
+            }))
+            .unwrap();
+    }
+
+    // Wait a little time (just to let everything get deadlocked) before
+    // trying to have the aggregator send out feed messages.
+    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+
+    // Start a feed. If deadlock has happened, it won't receive
+    // any messages.
+    let (_, mut feed_rx) = server
+        .get_core()
+        .connect_feed()
+        .await
+        .expect("feed can connect");
+
+    // Give up after a timeout:
+    tokio::time::timeout(Duration::from_secs(10), feed_rx.recv_feed_messages())
+        .await
+        .expect("should not hit timeout waiting for messages (deadlock has happened)")
+        .expect("shouldn't run into error receiving messages");
+}
+
+/// If a node is added, a connecting feed should be told about the new chain.
+/// If the node is removed, the feed should be told that the chain has gone.
+#[tokio::test]
+async fn feed_add_and_remove_node() {
+    // Connect server and add shard
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+
+    // Connect a node to the shard:
+    let (mut node_tx, _node_rx) = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_node()
+        .await
+        .expect("can connect to shard");
+
+    // Send a "system connected" message:
+    node_tx
+        .send_json_text(json!(
+            {
+                "id":1,
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain":"Local Testnet",
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(1),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name":"Alice",
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                },
+            }
+        ))
+        .unwrap();
+
+    // Wait a little for this message to propagate to the core
+    // (so that our feed connects after the core knows and not before).
+    tokio::time::sleep(Duration::from_millis(500)).await;
+
+    // Connect a feed.
+    let (_feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::AddedChain {
+        name: "Local Testnet".to_owned(),
+        node_count: 1
+    }));
+
+    // Disconnect the node:
+    node_tx.close().await.unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::RemovedChain {
+        name: "Local Testnet".to_owned(),
+    }));
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// If nodes connect and the chain name changes, feeds will be told about this
+/// and will keep receiving messages about the renamed chain (despite subscribing
+/// to it by name).
+#[tokio::test]
+async fn feeds_told_about_chain_rename_and_stay_subscribed() {
+    // Connect a node:
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+    let (mut node_tx, _node_rx) = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_node()
+        .await
+        .expect("can connect to shard");
+
+    let node_init_msg = |id, chain_name: &str, node_name: &str| {
+        json!({
+            "id":id,
+            "ts":"2021-07-12T10:37:47.714666+01:00",
+            "payload": {
+                "authority":true,
+                "chain": chain_name,
+                "config":"",
+                "genesis_hash": BlockHash::from_low_u64_ne(1),
+                "implementation":"Substrate Node",
+                "msg":"system.connected",
+                "name": node_name,
+                "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                "startup_time":"1625565542717",
+                "version":"2.0.0-07a1af348-aarch64-macos"
+            },
+        })
+    };
+
+    // Subscribe a chain:
+    node_tx
+        .send_json_text(node_init_msg(1, "Initial chain name", "Node 1"))
+        .unwrap();
+
+    // Connect a feed and subscribe to the above chain:
+    let (feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+    feed_tx
+        .send_command("subscribe", "Initial chain name")
+        .unwrap();
+
+    // Feed is told about the chain, and the node on this chain:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedChain { name, node_count: 1 } if name == "Initial chain name",
+        FeedMessage::SubscribedTo { name } if name == "Initial chain name",
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 1",
+    );
+
+    // Subscribe another node. The chain doesn't rename yet but we are told about the new node
+    // count and the node that's been added.
+    node_tx
+        .send_json_text(node_init_msg(2, "New chain name", "Node 2"))
+        .unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 2",
+        FeedMessage::AddedChain { name, node_count: 2 } if name == "Initial chain name",
+    );
+
+    // Subscribe a third node. The chain renames, so we're told about the new node but also
+    // about the chain rename.
+    node_tx
+        .send_json_text(node_init_msg(3, "New chain name", "Node 3"))
+        .unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 3",
+        FeedMessage::RemovedChain { name } if name == "Initial chain name",
+        FeedMessage::AddedChain { name, node_count: 3 } if name == "New chain name",
+    );
+
+    // Just to be sure, subscribing a fourth node on this chain will still lead to updates
+    // to this feed.
+    node_tx
+        .send_json_text(node_init_msg(4, "New chain name", "Node 4"))
+        .unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        FeedMessage::AddedNode { node: NodeDetails { name: node_name, .. }, ..} if node_name == "Node 4",
+        FeedMessage::AddedChain { name, node_count: 4 } if name == "New chain name",
+    );
+}
+
+/// If we add a couple of shards and a node for each, all feeds should be
+/// told about both node chains. If one shard goes away, we should get a
+/// "removed chain" message only for the node connected to that shard.
+#[tokio::test]
+async fn feed_add_and_remove_shard() {
+    let mut server = start_server_debug().await;
+
+    let mut shards = vec![];
+    for id in 1..=2 {
+        // Add a shard:
+        let shard_id = server.add_shard().await.unwrap();
+
+        // Connect a node to it:
+        let (mut node_tx, _node_rx) = server
+            .get_shard(shard_id)
+            .unwrap()
+            .connect_node()
+            .await
+            .expect("can connect to shard");
+
+        // Send a "system connected" message:
+        node_tx
+            .send_json_text(json!({
+                "id":id,
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain": format!("Local Testnet {}", id),
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(id),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name":"Alice",
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                },
+            }))
+            .unwrap();
+
+        // Keep what we need to to keep connection alive and let us kill a shard:
+        shards.push((shard_id, node_tx));
+    }
+
+    // Connect a feed.
+    let (_feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    // The feed should be told about both of the chains that we've sent info about:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::AddedChain {
+        name: "Local Testnet 1".to_owned(),
+        node_count: 1
+    }));
+    assert!(feed_messages.contains(&FeedMessage::AddedChain {
+        name: "Local Testnet 2".to_owned(),
+        node_count: 1
+    }));
+
+    // Disconnect the first shard:
+    server.kill_shard(shards[0].0).await;
+
+    // We should be told about the node connected to that shard disconnecting:
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert!(feed_messages.contains(&FeedMessage::RemovedChain {
+        name: "Local Testnet 1".to_owned(),
+    }));
+    assert!(!feed_messages.contains(
+        // Spot the "!"; this chain was not removed.
+        &FeedMessage::RemovedChain {
+            name: "Local Testnet 2".to_owned(),
+        }
+    ));
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// feeds can subscribe to one chain at a time. They should get the relevant
+/// messages for that chain and no other.
+#[tokio::test]
+async fn feed_can_subscribe_and_unsubscribe_from_chain() {
+    use FeedMessage::*;
+
+    // Start server, add shard, connect node:
+    let mut server = start_server_debug().await;
+    let shard_id = server.add_shard().await.unwrap();
+    let (mut node_tx, _node_rx) = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_node()
+        .await
+        .unwrap();
+
+    // Send a "system connected" message for a few nodes/chains:
+    for id in 1..=3 {
+        node_tx
+            .send_json_text(json!(
+                {
+                    "id":id,
+                    "ts":"2021-07-12T10:37:47.714666+01:00",
+                    "payload": {
+                        "authority":true,
+                        "chain":format!("Local Testnet {}", id),
+                        "config":"",
+                        "genesis_hash": BlockHash::from_low_u64_ne(id),
+                        "implementation":"Substrate Node",
+                        "msg":"system.connected",
+                        "name":format!("Alice {}", id),
+                        "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                        "startup_time":"1625565542717",
+                        "version":"2.0.0-07a1af348-aarch64-macos"
+                    },
+                }
+            ))
+            .unwrap();
+    }
+
+    // Connect a feed
+    let (feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(feed_messages, AddedChain { name, node_count: 1 } if name == "Local Testnet 1");
+
+    // Subscribe it to a chain
+    feed_tx
+        .send_command("subscribe", "Local Testnet 1")
+        .unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_contains_matches!(
+        feed_messages,
+        SubscribedTo { name } if name == "Local Testnet 1",
+        TimeSync {..},
+        BestBlock { block_number: 0, timestamp: 0, avg_block_time: None },
+        BestFinalized { block_number: 0, .. },
+        AddedNode { node_id: 0, node: NodeDetails { name, .. }, .. } if name == "Alice 1",
+        FinalizedBlock { node_id: 0, block_number: 0, .. }
+    );
+
+    // We receive updates relating to nodes on that chain:
+    node_tx.send_json_text(json!(
+        {"id":1, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:37:48.330433+01:00" }
+    )).unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_ne!(feed_messages.len(), 0);
+
+    // We don't receive anything for updates to nodes on other chains (wait a sec to ensure no messages are sent):
+    node_tx.send_json_text(json!(
+        {"id":2, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:37:48.330433+01:00" }
+    )).unwrap();
+
+    tokio::time::timeout(Duration::from_secs(1), feed_rx.recv_feed_messages())
+        .await
+        .expect_err("Timeout should elapse since no messages sent");
+
+    // We can change our subscription:
+    feed_tx
+        .send_command("subscribe", "Local Testnet 2")
+        .unwrap();
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+
+    // We are told about the subscription change and given similar on-subscribe messages to above.
+    assert_contains_matches!(
+        &feed_messages,
+        UnsubscribedFrom { name } if name == "Local Testnet 1",
+        SubscribedTo { name } if name == "Local Testnet 2",
+        TimeSync {..},
+        BestBlock {..},
+        BestFinalized {..},
+        AddedNode { node: NodeDetails { name, .. }, ..} if name == "Alice 2",
+        FinalizedBlock {..},
+    );
+
+    // We didn't get messages from this earlier, but we will now we've subscribed:
+    node_tx.send_json_text(json!(
+        {"id":2, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
+    )).unwrap();
+
+    let feed_messages = feed_rx.recv_feed_messages().await.unwrap();
+    assert_ne!(feed_messages.len(), 0);
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// If a node sends more than some rolling average amount of data, it'll be booted.
+#[tokio::test]
+async fn node_banned_if_it_sends_too_much_data() {
+    async fn try_send_data(max_bytes: usize, send_msgs: usize, bytes_per_msg: usize) -> bool {
+        let mut server = start_server(
+            ServerOpts::default(),
+            CoreOpts::default(),
+            ShardOpts {
+                // Remember, this is (currently) averaged over the last 10 seconds,
+                // so we need to send 10x this amount of data for an imemdiate ban:
+                max_node_data_per_second: Some(max_bytes),
+                ..Default::default()
+            },
+        )
+        .await;
+
+        // Give us a shard to talk to:
+        let shard_id = server.add_shard().await.unwrap();
+        let (node_tx, _node_rx) = server
+            .get_shard(shard_id)
+            .unwrap()
+            .connect_node()
+            .await
+            .unwrap();
+
+        // Send the data requested to the shard:
+        for _ in 0..send_msgs {
+            node_tx
+                .unbounded_send(SentMessage::Binary(vec![1; bytes_per_msg]))
+                .unwrap();
+        }
+
+        // Wait a little for the shard to react and cut off the connection (or not):
+        tokio::time::sleep(Duration::from_millis(250)).await;
+
+        // Has the connection been closed?
+        node_tx.is_closed()
+    }
+
+    assert_eq!(
+        try_send_data(1000, 10, 1000).await,
+        false,
+        "shouldn't be closed; we didn't exceed 10x threshold"
+    );
+    assert_eq!(
+        try_send_data(999, 10, 1000).await,
+        true,
+        "should be closed; we sent just over 10x the block threshold"
+    );
+}
+
+/// Feeds will be disconnected if they can't receive messages quickly enough.
+#[tokio::test]
+async fn slow_feeds_are_disconnected() {
+    let mut server = start_server(
+        ServerOpts::default(),
+        // Timeout faster so the test can be quicker:
+        CoreOpts {
+            feed_timeout: Some(1),
+            ..Default::default()
+        },
+        // Allow us to send more messages in more easily:
+        ShardOpts {
+            max_nodes_per_connection: Some(100_000),
+            // Prevent the shard being being banned when it sends a load of data at once:
+            max_node_data_per_second: Some(100_000_000),
+            ..Default::default()
+        },
+    )
+    .await;
+
+    // Give us a shard to talk to:
+    let shard_id = server.add_shard().await.unwrap();
+    let (mut node_tx, _node_rx) = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_node()
+        .await
+        .unwrap();
+
+    // Add a load of nodes from this shard so there's plenty of data to give to a feed.
+    // We want to exhaust any buffers between core and feed (eg BufWriters). If the number
+    // is too low, data will happily be sent into a buffer and the connection won't need to
+    // be closed.
+    for n in 1..50_000 {
+        node_tx
+            .send_json_text(json!({
+                "id":n,
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain":"Polkadot",
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(1),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name": format!("Alice {}", n),
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                }
+            }))
+            .unwrap();
+    }
+
+    // Connect a raw feed so that we can control how fast we consume data from the websocket
+    let (mut raw_feed_tx, mut raw_feed_rx) = server.get_core().connect_feed_raw().await.unwrap();
+
+    // Subscribe the feed:
+    raw_feed_tx.send_text("subscribe:Polkadot").await.unwrap();
+
+    // Wait a little.. the feed hasn't been receiving messages so it should
+    // be booted after ~a second.
+    tokio::time::sleep(Duration::from_secs(2)).await;
+
+    // Drain anything out and expect to hit a "closed" error, rather than get stuck
+    // waiting to receive mroe data (or see some other error).
+    loop {
+        let mut v = Vec::new();
+        let data =
+            tokio::time::timeout(Duration::from_secs(1), raw_feed_rx.receive_data(&mut v)).await;
+
+        match data {
+            Ok(Ok(_)) => {
+                continue; // Drain data
+            }
+            Ok(Err(soketto::connection::Error::Closed)) => {
+                break; // End loop; success!
+            }
+            Ok(Err(e)) => {
+                panic!("recv should be closed but instead we saw this error: {}", e);
+            }
+            Err(_) => {
+                panic!("recv should be closed but seems to be happy waiting for more data");
+            }
+        }
+    }
+
+    // Tidy up:
+    server.shutdown().await;
+}
+
+/// If something connects to the `/submit` endpoint, there is a limit to the number
+/// of different messags IDs it can send telemetry about, to prevent a malicious actor from
+/// spamming a load of message IDs and exhausting our memory.
+#[tokio::test]
+async fn max_nodes_per_connection_is_enforced() {
+    let mut server = start_server(
+        ServerOpts::default(),
+        CoreOpts::default(),
+        // Limit max nodes per connection to 2; any other msgs should be ignored.
+        ShardOpts {
+            max_nodes_per_connection: Some(2),
+            ..Default::default()
+        },
+    )
+    .await;
+
+    // Connect to a shard
+    let shard_id = server.add_shard().await.unwrap();
+    let (mut node_tx, _node_rx) = server
+        .get_shard(shard_id)
+        .unwrap()
+        .connect_node()
+        .await
+        .unwrap();
+
+    // Connect a feed.
+    let (feed_tx, mut feed_rx) = server.get_core().connect_feed().await.unwrap();
+
+    // We'll send these messages from the node:
+    let json_msg = |n| {
+        json!({
+            "id":n,
+            "ts":"2021-07-12T10:37:47.714666+01:00",
+            "payload": {
+                "authority":true,
+                "chain":"Test Chain",
+                "config":"",
+                "genesis_hash": BlockHash::from_low_u64_ne(1),
+                "implementation":"Polkadot",
+                "msg":"system.connected",
+                "name": format!("Alice {}", n),
+                "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                "startup_time":"1625565542717",
+                "version":"2.0.0-07a1af348-aarch64-macos"
+            }
+        })
+    };
+
+    // First message ID should lead to feed messages:
+    node_tx.send_json_text(json_msg(1)).unwrap();
+    assert_ne!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    // Second message ID should lead to feed messages as well:
+    node_tx.send_json_text(json_msg(2)).unwrap();
+    assert_ne!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    // Third message ID should be ignored:
+    node_tx.send_json_text(json_msg(3)).unwrap();
+    assert_eq!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    // Forth message ID should be ignored as well:
+    node_tx.send_json_text(json_msg(4)).unwrap();
+    assert_eq!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    // (now that the chain "Test Chain" is known about, subscribe to it for update messages.
+    // This wasn't needed to receive messages re the above since everybody hears about node
+    // count changes)
+    feed_tx.send_command("subscribe", "Test Chain").unwrap();
+    feed_rx.recv_feed_messages().await.unwrap();
+
+    // Update about non-ignored IDs should still lead to feed output:
+
+    node_tx.send_json_text(json!(
+        {"id":1, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
+    )).unwrap();
+    assert_ne!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    node_tx.send_json_text(json!(
+        {"id":2, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
+    )).unwrap();
+    assert_ne!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    // Updates about ignored IDs are still ignored:
+
+    node_tx.send_json_text(json!(
+        {"id":3, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
+    )).unwrap();
+    assert_eq!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    node_tx.send_json_text(json!(
+        {"id":4, "payload":{ "bandwidth_download":576,"bandwidth_upload":576,"msg":"system.interval","peers":1},"ts":"2021-07-12T10:38:48.330433+01:00" }
+    )).unwrap();
+    assert_eq!(
+        feed_rx
+            .recv_feed_messages_timeout(Duration::from_secs(1))
+            .await
+            .unwrap()
+            .len(),
+        0
+    );
+
+    // Tidy up:
+    server.shutdown().await;
+}

backend/telemetry_core/tests/soak_tests.rs

@@ -0,0 +1,432 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/*!
+Soak tests. These are ignored by default, and are intended to be long runs
+of the core + shards(s) under different loads to get a feel for CPU/memory
+usage and general performance over time.
+
+Note that on MacOS inparticular, you may need to increase some limits to be
+able to open a large number of connections. Try commands like:
+
+```sh
+sudo sysctl -w kern.maxfiles=100000
+sudo sysctl -w kern.maxfilesperproc=100000
+ulimit -n 100000
+sudo sysctl -w kern.ipc.somaxconn=100000
+sudo sysctl -w kern.ipc.maxsockbuf=16777216
+```
+
+In general, if you run into issues, it may be better to run this on a linux
+box; MacOS seems to hit limits quicker in general.
+*/
+
+use common::node_types::BlockHash;
+use common::ws_client::SentMessage;
+use futures::{future, StreamExt};
+use serde_json::json;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+use structopt::StructOpt;
+use test_utils::workspace::{start_server, CoreOpts, ServerOpts, ShardOpts};
+
+/// A configurable soak_test runner. Configure by providing the expected args as
+/// an environment variable. One example to run this test is:
+///
+/// ```sh
+/// SOAK_TEST_ARGS='--feeds 10 --nodes 100 --shards 4' cargo test --release -- soak_test --ignored --nocapture
+/// ```
+///
+/// You can also run this test against the pre-sharding actix binary with something like this:
+/// ```sh
+/// TELEMETRY_BIN=~/old_telemetry_binary SOAK_TEST_ARGS='--feeds 100 --nodes 100 --shards 4' cargo test --release -- soak_test --ignored --nocapture
+/// ```
+///
+/// Or, you can run it against existing processes with something like this:
+/// ```sh
+/// TELEMETRY_SUBMIT_HOSTS='127.0.0.1:8001' TELEMETRY_FEED_HOST='127.0.0.1:8000' SOAK_TEST_ARGS='--feeds 100 --nodes 100 --shards 4' cargo test --release -- soak_test --ignored --nocapture
+/// ```
+///
+/// Each will establish the same total number of connections and send the same messages.
+#[ignore]
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+pub async fn soak_test() {
+    let opts = get_soak_test_opts();
+    run_soak_test(opts).await;
+}
+
+/// A general soak test runner.
+/// This test sends the same message over and over, and so
+/// the results should be pretty reproducible.
+async fn run_soak_test(opts: SoakTestOpts) {
+    let mut server = start_server(
+        ServerOpts {
+            release_mode: true,
+            log_output: opts.log_output,
+            ..Default::default()
+        },
+        CoreOpts {
+            worker_threads: opts.core_worker_threads,
+            ..Default::default()
+        },
+        ShardOpts {
+            worker_threads: opts.shard_worker_threads,
+            ..Default::default()
+        },
+    )
+    .await;
+    println!("Telemetry core running at {}", server.get_core().host());
+
+    // Start up the shards we requested:
+    let mut shard_ids = vec![];
+    for _ in 0..opts.shards {
+        let shard_id = server.add_shard().await.expect("shard can't be added");
+        shard_ids.push(shard_id);
+    }
+
+    // Connect nodes to each shard:
+    let mut nodes = vec![];
+    for &shard_id in &shard_ids {
+        let mut conns = server
+            .get_shard(shard_id)
+            .unwrap()
+            .connect_multiple_nodes(opts.nodes)
+            .await
+            .expect("node connections failed");
+        nodes.append(&mut conns);
+    }
+
+    // Each node tells the shard about itself:
+    for (idx, (node_tx, _)) in nodes.iter_mut().enumerate() {
+        node_tx
+            .send_json_binary(json!({
+                "id":1, // Only needs to be unique per node
+                "ts":"2021-07-12T10:37:47.714666+01:00",
+                "payload": {
+                    "authority":true,
+                    "chain": "Polkadot", // <- so that we don't go over quota with lots of nodes.
+                    "config":"",
+                    "genesis_hash": BlockHash::from_low_u64_ne(1),
+                    "implementation":"Substrate Node",
+                    "msg":"system.connected",
+                    "name": format!("Node #{}", idx),
+                    "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                    "startup_time":"1625565542717",
+                    "version":"2.0.0-07a1af348-aarch64-macos"
+                },
+            }))
+            .unwrap();
+    }
+
+    // Connect feeds to the core:
+    let mut feeds = server
+        .get_core()
+        .connect_multiple_feeds(opts.feeds)
+        .await
+        .expect("feed connections failed");
+
+    // Every feed subscribes to the chain above to recv messages about it:
+    for (feed_tx, _) in &mut feeds {
+        feed_tx.send_command("subscribe", "Polkadot").unwrap();
+    }
+
+    // Start sending "update" messages from nodes at time intervals.
+    let bytes_in = Arc::new(AtomicUsize::new(0));
+    let bytes_in2 = Arc::clone(&bytes_in);
+    tokio::task::spawn(async move {
+        let msg = json!({
+            "id":1,
+            "payload":{
+                "bandwidth_download":576,
+                "bandwidth_upload":576,
+                "msg":"system.interval",
+                "peers":1
+            },
+            "ts":"2021-07-12T10:37:48.330433+01:00"
+        });
+        let msg_bytes: &'static [u8] = Box::new(serde_json::to_vec(&msg).unwrap()).leak();
+
+        loop {
+            // every ~1second we aim to have sent messages from all of the nodes. So we cycle through
+            // the node IDs and send a message from each at roughly 1s / number_of_nodes.
+            let mut interval =
+                tokio::time::interval(Duration::from_secs_f64(1.0 / nodes.len() as f64));
+
+            for node_id in (0..nodes.len()).cycle() {
+                interval.tick().await;
+                let node_tx = &mut nodes[node_id].0;
+                node_tx
+                    .unbounded_send(SentMessage::StaticBinary(msg_bytes))
+                    .unwrap();
+                bytes_in2.fetch_add(msg_bytes.len(), Ordering::Relaxed);
+            }
+        }
+    });
+
+    // Also start receiving messages, counting the bytes received so far.
+    let bytes_out = Arc::new(AtomicUsize::new(0));
+    let msgs_out = Arc::new(AtomicUsize::new(0));
+    for (_, mut feed_rx) in feeds {
+        let bytes_out = Arc::clone(&bytes_out);
+        let msgs_out = Arc::clone(&msgs_out);
+        tokio::task::spawn(async move {
+            while let Some(msg) = feed_rx.next().await {
+                let msg = msg.expect("message could be received");
+                let num_bytes = msg.len();
+                bytes_out.fetch_add(num_bytes, Ordering::Relaxed);
+                msgs_out.fetch_add(1, Ordering::Relaxed);
+            }
+            eprintln!("Error: feed has been closed unexpectedly");
+        });
+    }
+
+    // Periodically report on bytes out
+    tokio::task::spawn(async move {
+        let one_mb = 1024.0 * 1024.0;
+        let mut last_bytes_in = 0;
+        let mut last_bytes_out = 0;
+        let mut last_msgs_out = 0;
+        let mut n = 1;
+        loop {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            let bytes_in_val = bytes_in.load(Ordering::Relaxed);
+            let bytes_out_val = bytes_out.load(Ordering::Relaxed);
+            let msgs_out_val = msgs_out.load(Ordering::Relaxed);
+
+            println!(
+                "#{}: MB in/out per measurement: {:.4} / {:.4}, total bytes in/out: {} / {}, msgs out: {}, total msgs out: {})",
+                n,
+                (bytes_in_val - last_bytes_in) as f64 / one_mb,
+                (bytes_out_val - last_bytes_out) as f64 / one_mb,
+                bytes_in_val,
+                bytes_out_val,
+                (msgs_out_val - last_msgs_out),
+                msgs_out_val
+            );
+
+            n += 1;
+            last_bytes_in = bytes_in_val;
+            last_bytes_out = bytes_out_val;
+            last_msgs_out = msgs_out_val;
+        }
+    });
+
+    // Wait forever.
+    future::pending().await
+}
+
+/// Identical to `soak_test`, except that we try to send realistic messages from fake nodes.
+/// This means it's potentially less reproducable, but presents a more accurate picture of
+/// the load, and lets us see the UI working more or less.
+///
+/// We can provide the same arguments as we would to `soak_test`:
+///
+/// ```sh
+/// SOAK_TEST_ARGS='--feeds 10 --nodes 100 --shards 4' cargo test --release -- realistic_soak_test --ignored --nocapture
+/// ```
+///
+/// You can also run this test against the pre-sharding actix binary with something like this:
+/// ```sh
+/// TELEMETRY_BIN=~/old_telemetry_binary SOAK_TEST_ARGS='--feeds 100 --nodes 100 --shards 4' cargo test --release -- realistic_soak_test --ignored --nocapture
+/// ```
+///
+/// Or, you can run it against existing processes with something like this:
+/// ```sh
+/// TELEMETRY_SUBMIT_HOSTS='127.0.0.1:8001' TELEMETRY_FEED_HOST='127.0.0.1:8000' SOAK_TEST_ARGS='--feeds 100 --nodes 100 --shards 4' cargo test --release -- realistic_soak_test --ignored --nocapture
+/// ```
+///
+#[ignore]
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+pub async fn realistic_soak_test() {
+    let opts = get_soak_test_opts();
+    run_realistic_soak_test(opts).await;
+}
+
+/// A general soak test runner.
+/// This test sends realistic messages from connected nodes
+/// so that we can see how things react under more normal
+/// circumstances
+async fn run_realistic_soak_test(opts: SoakTestOpts) {
+    let mut server = start_server(
+        ServerOpts {
+            release_mode: true,
+            log_output: opts.log_output,
+            ..Default::default()
+        },
+        CoreOpts {
+            worker_threads: opts.core_worker_threads,
+            ..Default::default()
+        },
+        ShardOpts {
+            worker_threads: opts.shard_worker_threads,
+            ..Default::default()
+        },
+    )
+    .await;
+    println!("Telemetry core running at {}", server.get_core().host());
+
+    // Start up the shards we requested:
+    let mut shard_ids = vec![];
+    for _ in 0..opts.shards {
+        let shard_id = server.add_shard().await.expect("shard can't be added");
+        shard_ids.push(shard_id);
+    }
+
+    // Connect nodes to each shard:
+    let mut nodes = vec![];
+    for &shard_id in &shard_ids {
+        let mut conns = server
+            .get_shard(shard_id)
+            .unwrap()
+            .connect_multiple_nodes(opts.nodes)
+            .await
+            .expect("node connections failed");
+        nodes.append(&mut conns);
+    }
+
+    // Start nodes talking to the shards:
+    let bytes_in = Arc::new(AtomicUsize::new(0));
+    for node in nodes.into_iter().enumerate() {
+        let bytes_in = Arc::clone(&bytes_in);
+        tokio::spawn(async move {
+            let (idx, (tx, _)) = node;
+
+            let telemetry = test_utils::fake_telemetry::FakeTelemetry::new(
+                Duration::from_secs(3),
+                format!("Node {}", idx + 1),
+                "Polkadot".to_owned(),
+                idx + 1,
+            );
+
+            let res = telemetry
+                .start(|msg| async {
+                    bytes_in.fetch_add(msg.len(), Ordering::Relaxed);
+                    tx.unbounded_send(SentMessage::Binary(msg))?;
+                    Ok::<_, anyhow::Error>(())
+                })
+                .await;
+
+            if let Err(e) = res {
+                log::error!("Telemetry Node #{} has died with error: {}", idx, e);
+            }
+        });
+    }
+
+    // Connect feeds to the core:
+    let mut feeds = server
+        .get_core()
+        .connect_multiple_feeds(opts.feeds)
+        .await
+        .expect("feed connections failed");
+
+    // Every feed subscribes to the chain above to recv messages about it:
+    for (feed_tx, _) in &mut feeds {
+        feed_tx.send_command("subscribe", "Polkadot").unwrap();
+    }
+
+    // Also start receiving messages, counting the bytes received so far.
+    let bytes_out = Arc::new(AtomicUsize::new(0));
+    let msgs_out = Arc::new(AtomicUsize::new(0));
+    for (_, mut feed_rx) in feeds {
+        let bytes_out = Arc::clone(&bytes_out);
+        let msgs_out = Arc::clone(&msgs_out);
+        tokio::task::spawn(async move {
+            while let Some(msg) = feed_rx.next().await {
+                let msg = msg.expect("message could be received");
+                let num_bytes = msg.len();
+                bytes_out.fetch_add(num_bytes, Ordering::Relaxed);
+                msgs_out.fetch_add(1, Ordering::Relaxed);
+            }
+            eprintln!("Error: feed has been closed unexpectedly");
+        });
+    }
+
+    // Periodically report on bytes out
+    tokio::task::spawn(async move {
+        let one_mb = 1024.0 * 1024.0;
+        let mut last_bytes_in = 0;
+        let mut last_bytes_out = 0;
+        let mut last_msgs_out = 0;
+        let mut n = 1;
+        loop {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            let bytes_in_val = bytes_in.load(Ordering::Relaxed);
+            let bytes_out_val = bytes_out.load(Ordering::Relaxed);
+            let msgs_out_val = msgs_out.load(Ordering::Relaxed);
+
+            println!(
+                "#{}: MB in/out per measurement: {:.4} / {:.4}, total bytes in/out: {} / {}, msgs out: {}, total msgs out: {})",
+                n,
+                (bytes_in_val - last_bytes_in) as f64 / one_mb,
+                (bytes_out_val - last_bytes_out) as f64 / one_mb,
+                bytes_in_val,
+                bytes_out_val,
+                (msgs_out_val - last_msgs_out),
+                msgs_out_val
+            );
+
+            n += 1;
+            last_bytes_in = bytes_in_val;
+            last_bytes_out = bytes_out_val;
+            last_msgs_out = msgs_out_val;
+        }
+    });
+
+    // Wait forever.
+    future::pending().await
+}
+
+/// General arguments that are used to start a soak test. Run `soak_test` as
+/// instructed by its documentation for full control over what is ran, or run
+/// preconfigured variants.
+#[derive(StructOpt, Debug)]
+struct SoakTestOpts {
+    /// The number of shards to run this test with
+    #[structopt(long)]
+    shards: usize,
+    /// The number of feeds to connect
+    #[structopt(long)]
+    feeds: usize,
+    /// The number of nodes to connect to each feed
+    #[structopt(long)]
+    nodes: usize,
+    /// Number of aggregator loops to use in the core
+    #[structopt(long)]
+    core_num_aggregators: Option<usize>,
+    /// Number of worker threads the core will use
+    #[structopt(long)]
+    core_worker_threads: Option<usize>,
+    /// Number of worker threads each shard will use
+    #[structopt(long)]
+    shard_worker_threads: Option<usize>,
+    /// Should we log output from the core/shards to stdout?
+    #[structopt(long)]
+    log_output: bool,
+}
+
+/// Get soak test args from an envvar and parse them via structopt.
+fn get_soak_test_opts() -> SoakTestOpts {
+    let arg_string = std::env::var("SOAK_TEST_ARGS")
+        .expect("Expecting args to be provided in the env var SOAK_TEST_ARGS");
+    let args =
+        shellwords::split(&arg_string).expect("Could not parse SOAK_TEST_ARGS as shell arguments");
+
+    // The binary name is expected to be the first arg, so fake it:
+    let all_args = std::iter::once("soak_test".to_owned()).chain(args.into_iter());
+
+    SoakTestOpts::from_iter(all_args)
+}

backend/telemetry_shard/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "telemetry_shard"
+version = "0.1.0"
+authors = ["Parity Technologies Ltd. <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0"
+
+[dependencies]
+anyhow = "1.0.41"
+bincode = "1.3.3"
+common = { path = "../common" }
+futures = "0.3.15"
+hex = "0.4.3"
+http = "0.2.4"
+hyper = "0.14.11"
+log = "0.4.14"
+num_cpus = "1.13.0"
+primitive-types = { version = "0.9.0", features = ["serde"] }
+serde = { version = "1.0.126", features = ["derive"] }
+serde_json = "1.0.64"
+simple_logger = "1.11.0"
+soketto = "0.6.0"
+structopt = "0.3.21"
+thiserror = "1.0.25"
+tokio = { version = "1.7.0", features = ["full"] }
+tokio-util = { version = "0.6", features = ["compat"] }

backend/telemetry_shard/src/aggregator.rs

@@ -0,0 +1,299 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::connection::{create_ws_connection_to_core, Message};
+use common::{
+    internal_messages::{self, ShardNodeId},
+    node_message,
+    node_types::BlockHash,
+    AssignId,
+};
+use futures::channel::mpsc;
+use futures::{Sink, SinkExt, StreamExt};
+use std::collections::{HashMap, HashSet};
+use std::sync::atomic::AtomicU64;
+use std::sync::Arc;
+
+/// A unique Id is assigned per websocket connection (or more accurately,
+/// per thing-that-subscribes-to-the-aggregator). That connection might send
+/// data on behalf of multiple chains, so this ID is local to the aggregator,
+/// and a unique ID is assigned per batch of data too ([`internal_messages::ShardNodeId`]).
+type ConnId = u64;
+
+/// Incoming messages are either from websocket connections or
+/// from the telemetry core. This can be private since the only
+/// external messages are via subscriptions that take
+/// [`FromWebsocket`] instances.
+#[derive(Clone, Debug)]
+enum ToAggregator {
+    /// Sent when the telemetry core is disconnected.
+    DisconnectedFromTelemetryCore,
+    /// Sent when the telemetry core (re)connects.
+    ConnectedToTelemetryCore,
+    /// Sent when a message comes in from a substrate node.
+    FromWebsocket(ConnId, FromWebsocket),
+    /// Send when a message comes in from the telemetry core.
+    FromTelemetryCore(internal_messages::FromTelemetryCore),
+}
+
+/// An incoming socket connection can provide these messages.
+/// Until a node has been Added via [`FromWebsocket::Add`],
+/// messages from it will be ignored.
+#[derive(Clone, Debug)]
+pub enum FromWebsocket {
+    /// Fire this when the connection is established.
+    Initialize {
+        /// When a message is sent back up this channel, we terminate
+        /// the websocket connection and force the node to reconnect
+        /// so that it sends its system info again incase the telemetry
+        /// core has restarted.
+        close_connection: mpsc::Sender<()>,
+    },
+    /// Tell the aggregator about a new node.
+    Add {
+        message_id: node_message::NodeMessageId,
+        ip: std::net::IpAddr,
+        node: common::node_types::NodeDetails,
+        genesis_hash: BlockHash,
+    },
+    /// Update/pass through details about a node.
+    Update {
+        message_id: node_message::NodeMessageId,
+        payload: node_message::Payload,
+    },
+    /// Make a note when the node disconnects.
+    Disconnected,
+}
+
+pub type FromAggregator = internal_messages::FromShardAggregator;
+
+/// The aggregator loop handles incoming messages from nodes, or from the telemetry core.
+/// this is where we decide what effect messages will have.
+#[derive(Clone)]
+pub struct Aggregator(Arc<AggregatorInternal>);
+
+struct AggregatorInternal {
+    /// Nodes that connect are each assigned a unique connection ID. Nodes
+    /// can send messages on behalf of more than one chain, and so this ID is
+    /// only really used inside the Aggregator in conjunction with a per-message
+    /// ID.
+    conn_id: AtomicU64,
+    /// Send messages to the aggregator from websockets via this. This is
+    /// stored here so that anybody holding an `Aggregator` handle can
+    /// make use of it.
+    tx_to_aggregator: mpsc::Sender<ToAggregator>,
+}
+
+impl Aggregator {
+    /// Spawn a new Aggregator. This connects to the telemetry backend
+    pub async fn spawn(telemetry_uri: http::Uri) -> anyhow::Result<Aggregator> {
+        let (tx_to_aggregator, rx_from_external) = mpsc::channel(10);
+
+        // Establish a resiliant connection to the core (this retries as needed):
+        let (tx_to_telemetry_core, mut rx_from_telemetry_core) =
+            create_ws_connection_to_core(telemetry_uri).await;
+
+        // Forward messages from the telemetry core into the aggregator:
+        let mut tx_to_aggregator2 = tx_to_aggregator.clone();
+        tokio::spawn(async move {
+            while let Some(msg) = rx_from_telemetry_core.next().await {
+                let msg_to_aggregator = match msg {
+                    Message::Connected => ToAggregator::ConnectedToTelemetryCore,
+                    Message::Disconnected => ToAggregator::DisconnectedFromTelemetryCore,
+                    Message::Data(data) => ToAggregator::FromTelemetryCore(data),
+                };
+                if let Err(_) = tx_to_aggregator2.send(msg_to_aggregator).await {
+                    // This will close the ws channels, which themselves log messages.
+                    break;
+                }
+            }
+        });
+
+        // Start our aggregator loop, handling any incoming messages:
+        tokio::spawn(Aggregator::handle_messages(
+            rx_from_external,
+            tx_to_telemetry_core,
+        ));
+
+        // Return a handle to our aggregator so that we can send in messages to it:
+        Ok(Aggregator(Arc::new(AggregatorInternal {
+            conn_id: AtomicU64::new(1),
+            tx_to_aggregator,
+        })))
+    }
+
+    // This is spawned into a separate task and handles any messages coming
+    // in to the aggregator. If nobody is holding the tx side of the channel
+    // any more, this task will gracefully end.
+    async fn handle_messages(
+        mut rx_from_external: mpsc::Receiver<ToAggregator>,
+        mut tx_to_telemetry_core: mpsc::Sender<FromAggregator>,
+    ) {
+        use internal_messages::{FromShardAggregator, FromTelemetryCore};
+
+        // Just as an optimisation, we can keep track of whether we're connected to the backend
+        // or not, and ignore incoming messages while we aren't.
+        let mut connected_to_telemetry_core = false;
+
+        // A list of close channels for the currently connected substrate nodes. Send an empty
+        // tuple to these to ask the connections to be closed.
+        let mut close_connections: HashMap<ConnId, mpsc::Sender<()>> = HashMap::new();
+
+        // Maintain mappings from the connection ID and node message ID to the "local ID" which we
+        // broadcast to the telemetry core.
+        let mut to_local_id = AssignId::new();
+
+        // Any messages coming from nodes that have been muted are ignored:
+        let mut muted: HashSet<ShardNodeId> = HashSet::new();
+
+        // Now, loop and receive messages to handle.
+        while let Some(msg) = rx_from_external.next().await {
+            match msg {
+                ToAggregator::ConnectedToTelemetryCore => {
+                    // Take hold of the connection closers and run them all.
+                    let closers = close_connections;
+
+                    for (_, mut closer) in closers {
+                        // if this fails, it probably means the connection has died already anyway.
+                        let _ = closer.send(()).await;
+                    }
+
+                    // We've told everything to disconnect. Now, reset our state:
+                    close_connections = HashMap::new();
+                    to_local_id.clear();
+                    muted.clear();
+
+                    connected_to_telemetry_core = true;
+                    log::info!("Connected to telemetry core");
+                }
+                ToAggregator::DisconnectedFromTelemetryCore => {
+                    connected_to_telemetry_core = false;
+                    log::info!("Disconnected from telemetry core");
+                }
+                ToAggregator::FromWebsocket(
+                    conn_id,
+                    FromWebsocket::Initialize { close_connection },
+                ) => {
+                    // We boot all connections on a reconnect-to-core to force new systemconnected
+                    // messages to be sent. We could boot on muting, but need to be careful not to boot
+                    // connections where we mute one set of messages it sends and not others.
+                    close_connections.insert(conn_id, close_connection);
+                }
+                ToAggregator::FromWebsocket(
+                    conn_id,
+                    FromWebsocket::Add {
+                        message_id,
+                        ip,
+                        node,
+                        genesis_hash,
+                    },
+                ) => {
+                    // Don't bother doing anything else if we're disconnected, since we'll force the
+                    // node to reconnect anyway when the backend does:
+                    if !connected_to_telemetry_core {
+                        continue;
+                    }
+
+                    // Generate a new "local ID" for messages from this connection:
+                    let local_id = to_local_id.assign_id((conn_id, message_id));
+
+                    // Send the message to the telemetry core with this local ID:
+                    let _ = tx_to_telemetry_core
+                        .send(FromShardAggregator::AddNode {
+                            ip,
+                            node,
+                            genesis_hash,
+                            local_id,
+                        })
+                        .await;
+                }
+                ToAggregator::FromWebsocket(
+                    conn_id,
+                    FromWebsocket::Update {
+                        message_id,
+                        payload,
+                    },
+                ) => {
+                    // Ignore incoming messages if we're not connected to the backend:
+                    if !connected_to_telemetry_core {
+                        continue;
+                    }
+
+                    // Get the local ID, ignoring the message if none match:
+                    let local_id = match to_local_id.get_id(&(conn_id, message_id)) {
+                        Some(id) => id,
+                        None => continue,
+                    };
+
+                    // ignore the message if this node has been muted:
+                    if muted.contains(&local_id) {
+                        continue;
+                    }
+
+                    // Send the message to the telemetry core with this local ID:
+                    let _ = tx_to_telemetry_core
+                        .send(FromShardAggregator::UpdateNode { local_id, payload })
+                        .await;
+                }
+                ToAggregator::FromWebsocket(disconnected_conn_id, FromWebsocket::Disconnected) => {
+                    // Find all of the local IDs corresponding to the disconnected connection ID and
+                    // remove them, telling Telemetry Core about them too. This could be more efficient,
+                    // but the mapping isn't currently cached and it's not a super frequent op.
+                    let local_ids_disconnected: Vec<_> = to_local_id
+                        .iter()
+                        .filter(|(_, &(conn_id, _))| disconnected_conn_id == conn_id)
+                        .map(|(local_id, _)| local_id)
+                        .collect();
+
+                    close_connections.remove(&disconnected_conn_id);
+
+                    for local_id in local_ids_disconnected {
+                        to_local_id.remove_by_id(local_id);
+                        muted.remove(&local_id);
+                        let _ = tx_to_telemetry_core
+                            .send(FromShardAggregator::RemoveNode { local_id })
+                            .await;
+                    }
+                }
+                ToAggregator::FromTelemetryCore(FromTelemetryCore::Mute {
+                    local_id,
+                    reason: _,
+                }) => {
+                    // Mute the local ID we've been told to:
+                    muted.insert(local_id);
+                }
+            }
+        }
+    }
+
+    /// Return a sink that a node can send messages into to be handled by the aggregator.
+    pub fn subscribe_node(&self) -> impl Sink<FromWebsocket, Error = anyhow::Error> + Unpin {
+        // Assign a unique aggregator-local ID to each connection that subscribes, and pass
+        // that along with every message to the aggregator loop:
+        let conn_id: ConnId = self
+            .0
+            .conn_id
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let tx_to_aggregator = self.0.tx_to_aggregator.clone();
+
+        // Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
+        // but pinning by boxing is the easy solution for now:
+        Box::pin(
+            tx_to_aggregator
+                .with(move |msg| async move { Ok(ToAggregator::FromWebsocket(conn_id, msg)) }),
+        )
+    }
+}

backend/telemetry_shard/src/blocked_addrs.rs

@@ -0,0 +1,66 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::HashMap;
+use std::net::IpAddr;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+/// Keep track of nodes that have been blocked.
+#[derive(Debug, Clone)]
+pub struct BlockedAddrs(Arc<BlockAddrsInner>);
+
+#[derive(Debug)]
+struct BlockAddrsInner {
+    block_duration: Duration,
+    inner: Mutex<HashMap<IpAddr, (&'static str, Instant)>>,
+}
+
+impl BlockedAddrs {
+    /// Create a new block list. Nodes are blocked for the duration
+    /// provided here.
+    pub fn new(block_duration: Duration) -> BlockedAddrs {
+        BlockedAddrs(Arc::new(BlockAddrsInner {
+            block_duration,
+            inner: Mutex::new(HashMap::new()),
+        }))
+    }
+
+    /// Block a new address
+    pub fn block_addr(&self, addr: IpAddr, reason: &'static str) {
+        let now = Instant::now();
+        self.0.inner.lock().unwrap().insert(addr, (reason, now));
+    }
+
+    /// Find out whether an address has been blocked. If it has, a reason
+    /// will be returned. Else, we'll get None back. This function may also
+    /// perform cleanup if the item was blocked and the block has expired.
+    pub fn blocked_reason(&self, addr: &IpAddr) -> Option<&'static str> {
+        let mut map = self.0.inner.lock().unwrap();
+
+        let (reason, time) = match map.get(addr) {
+            Some(&(reason, time)) => (reason, time),
+            None => return None,
+        };
+
+        if time + self.0.block_duration < Instant::now() {
+            map.remove(addr);
+            None
+        } else {
+            Some(reason)
+        }
+    }
+}

backend/telemetry_shard/src/connection.rs

@@ -0,0 +1,146 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use bincode::Options;
+use common::ws_client;
+use futures::channel::mpsc;
+use futures::{SinkExt, StreamExt};
+
+#[derive(Clone, Debug)]
+pub enum Message<Out> {
+    Connected,
+    Disconnected,
+    Data(Out),
+}
+
+/// Connect to the telemetry core, retrying the connection if we're disconnected.
+/// - Sends `Message::Connected` and `Message::Disconnected` when the connection goes up/down.
+/// - Returns a channel that allows you to send messages to the connection.
+/// - Messages are all encoded/decoded to/from bincode, and so need to support being (de)serialized from
+///   a non self-describing encoding.
+///
+/// Note: have a look at [`common::internal_messages`] to see the different message types exchanged
+/// between aggregator and core.
+pub async fn create_ws_connection_to_core<In, Out>(
+    telemetry_uri: http::Uri,
+) -> (mpsc::Sender<In>, mpsc::Receiver<Message<Out>>)
+where
+    In: serde::Serialize + Send + 'static,
+    Out: serde::de::DeserializeOwned + Send + 'static,
+{
+    let (tx_in, mut rx_in) = mpsc::channel(10);
+    let (mut tx_out, rx_out) = mpsc::channel(10);
+
+    let mut is_connected = false;
+
+    tokio::spawn(async move {
+        loop {
+            // Throw away any pending messages from the incoming channel so that it
+            // doesn't get filled up and begin blocking while we're looping and waiting
+            // for a reconnection.
+            while let Ok(Some(_)) = rx_in.try_next() {}
+
+            // Try to connect. If connection established, we serialize and forward messages
+            // to/from the core. If the external channels break, we end for good. If the internal
+            // channels break, we loop around and try connecting again.
+            match ws_client::connect(&telemetry_uri).await {
+                Ok(connection) => {
+                    let (tx_to_core, mut rx_from_core) = connection.into_channels();
+                    is_connected = true;
+                    let mut tx_out = tx_out.clone();
+
+                    if let Err(e) = tx_out.send(Message::Connected).await {
+                        // If receiving end is closed, bail now.
+                        log::warn!("Aggregator is no longer receiving messages from core; disconnecting (permanently): {}", e);
+                        return;
+                    }
+
+                    // Loop, forwarding messages to and from the core until something goes wrong.
+                    loop {
+                        tokio::select! {
+                            msg = rx_from_core.next() => {
+                                let msg = match msg {
+                                    Some(msg) => msg,
+                                    // No more messages from core? core WS is disconnected.
+                                    None => {
+                                        log::warn!("No more messages from core: shutting down connection (will reconnect)");
+                                        break
+                                    }
+                                };
+
+                                let bytes = match msg {
+                                    Ok(ws_client::RecvMessage::Binary(bytes)) => bytes,
+                                    Ok(ws_client::RecvMessage::Text(s)) => s.into_bytes(),
+                                    Err(e) => {
+                                        log::warn!("Unable to receive message from core: shutting down connection (will reconnect): {}", e);
+                                        break;
+                                    }
+                                };
+                                let msg = bincode::options()
+                                    .deserialize(&bytes)
+                                    .expect("internal messages must be deserializable");
+
+                                if let Err(e) = tx_out.send(Message::Data(msg)).await {
+                                    log::error!("Aggregator is no longer receiving messages from core; disconnecting (permanently): {}", e);
+                                    return;
+                                }
+                            },
+                            msg = rx_in.next() => {
+                                let msg = match msg {
+                                    Some(msg) => msg,
+                                    None => {
+                                        log::error!("Aggregator is no longer sending messages to core; disconnecting (permanently)");
+                                        return
+                                    }
+                                };
+
+                                let bytes = bincode::options()
+                                    .serialize(&msg)
+                                    .expect("internal messages must be serializable");
+                                let ws_msg = ws_client::SentMessage::Binary(bytes);
+
+                                if let Err(e) = tx_to_core.unbounded_send(ws_msg) {
+                                    log::warn!("Unable to send message to core; shutting down connection (will reconnect): {}", e);
+                                    break;
+                                }
+                            }
+                        };
+                    }
+                }
+                Err(connect_err) => {
+                    // Issue connecting? Wait and try again on the next loop iteration.
+                    log::error!(
+                        "Error connecting to websocker server (will reconnect): {}",
+                        connect_err
+                    );
+                }
+            }
+
+            if is_connected {
+                is_connected = false;
+                if let Err(e) = tx_out.send(Message::Disconnected).await {
+                    log::error!("Aggregator is no longer receiving messages from core; disconnecting (permanently): {}", e);
+                    return;
+                }
+            }
+
+            // Wait a little before we try to connect again.
+            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+        }
+    });
+
+    (tx_in, rx_out)
+}

backend/telemetry_shard/src/json_message/hash.rs

@@ -0,0 +1,235 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! A hash wrapper which can be deserialized from a hex string as well as from an array of bytes,
+//! so that it can deal with the sort of inputs we expect from substrate nodes.
+
+use serde::de::{self, Deserialize, Deserializer, SeqAccess, Unexpected, Visitor};
+use serde::ser::{Serialize, Serializer};
+use std::fmt::{self, Debug, Display};
+use std::str::FromStr;
+
+/// We assume that hashes are 32 bytes long, and in practise that's currently true,
+/// but in theory it doesn't need to be. We may need to be more dynamic here.
+const HASH_BYTES: usize = 32;
+
+/// Newtype wrapper for 32-byte hash values, implementing readable `Debug` and `serde::Deserialize`.
+/// This can deserialize from a JSON string or array.
+#[derive(Hash, PartialEq, Eq, Clone, Copy)]
+pub struct Hash([u8; HASH_BYTES]);
+
+impl From<Hash> for common::node_types::BlockHash {
+    fn from(hash: Hash) -> Self {
+        hash.0.into()
+    }
+}
+
+impl From<common::node_types::BlockHash> for Hash {
+    fn from(hash: common::node_types::BlockHash) -> Self {
+        Hash(hash.0)
+    }
+}
+
+struct HashVisitor;
+
+impl<'de> Visitor<'de> for HashVisitor {
+    type Value = Hash;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str(
+            "byte array of length 32, or hexidecimal string of 32 bytes beginning with 0x",
+        )
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        value
+            .parse()
+            .map_err(|_| de::Error::invalid_value(Unexpected::Str(value), &self))
+    }
+
+    fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.len() == HASH_BYTES {
+            let mut hash = [0; HASH_BYTES];
+
+            hash.copy_from_slice(value);
+
+            return Ok(Hash(hash));
+        }
+
+        Hash::from_ascii(value)
+            .map_err(|_| de::Error::invalid_value(Unexpected::Bytes(value), &self))
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: SeqAccess<'de>,
+    {
+        let mut hash = [0u8; HASH_BYTES];
+
+        for (i, byte) in hash.iter_mut().enumerate() {
+            match seq.next_element()? {
+                Some(b) => *byte = b,
+                None => return Err(de::Error::invalid_length(i, &"an array of 32 bytes")),
+            }
+        }
+
+        if seq.next_element::<u8>()?.is_some() {
+            return Err(de::Error::invalid_length(33, &"an array of 32 bytes"));
+        }
+
+        Ok(Hash(hash))
+    }
+}
+
+impl Hash {
+    pub fn from_ascii(value: &[u8]) -> Result<Self, HashParseError> {
+        if !value.starts_with(b"0x") {
+            return Err(HashParseError::InvalidPrefix);
+        }
+
+        let mut hash = [0; HASH_BYTES];
+
+        hex::decode_to_slice(&value[2..], &mut hash).map_err(HashParseError::HexError)?;
+
+        Ok(Hash(hash))
+    }
+}
+
+impl FromStr for Hash {
+    type Err = HashParseError;
+
+    fn from_str(value: &str) -> Result<Self, Self::Err> {
+        Hash::from_ascii(value.as_bytes())
+    }
+}
+
+impl<'de> Deserialize<'de> for Hash {
+    fn deserialize<D>(deserializer: D) -> Result<Hash, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_bytes(HashVisitor)
+    }
+}
+
+impl Serialize for Hash {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(&self.0)
+    }
+}
+
+impl Display for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str("0x")?;
+
+        let mut ascii = [0; HASH_BYTES * 2];
+
+        hex::encode_to_slice(self.0, &mut ascii)
+            .expect("Encoding 32 bytes into 64 bytes of ascii; qed");
+
+        f.write_str(std::str::from_utf8(&ascii).expect("ASCII hex encoded bytes can't fail; qed"))
+    }
+}
+
+impl Debug for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        Display::fmt(self, f)
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum HashParseError {
+    #[error("Error parsing string into hex: {0}")]
+    HexError(hex::FromHexError),
+    #[error("Invalid hex prefix: expected '0x'")]
+    InvalidPrefix,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Hash;
+    use bincode::Options;
+
+    const DUMMY: Hash = {
+        let mut hash = [0; 32];
+        hash[0] = 0xDE;
+        hash[1] = 0xAD;
+        hash[2] = 0xBE;
+        hash[3] = 0xEF;
+        Hash(hash)
+    };
+
+    #[test]
+    fn deserialize_json_hash_str() {
+        let json = r#""0xdeadBEEF00000000000000000000000000000000000000000000000000000000""#;
+
+        let hash: Hash = serde_json::from_str(json).unwrap();
+
+        assert_eq!(hash, DUMMY);
+    }
+
+    #[test]
+    fn deserialize_json_array() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let hash: Hash = serde_json::from_str(json).unwrap();
+
+        assert_eq!(hash, DUMMY);
+    }
+
+    #[test]
+    fn deserialize_json_array_too_short() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let res = serde_json::from_str::<Hash>(json);
+
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn deserialize_json_array_too_long() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let res = serde_json::from_str::<Hash>(json);
+
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn bincode() {
+        let bytes = bincode::options().serialize(&DUMMY).unwrap();
+
+        let mut expected = [0; 33];
+
+        expected[0] = 32; // length
+        expected[1..].copy_from_slice(&DUMMY.0);
+
+        assert_eq!(bytes, &expected);
+
+        let deserialized: Hash = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(DUMMY, deserialized);
+    }
+}

backend/telemetry_shard/src/json_message/mod.rs

@@ -0,0 +1,23 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! This module contains the types we need to deserialize JSON messages from nodes
+
+mod hash;
+mod node_message;
+
+pub use hash::Hash;
+pub use node_message::*;

backend/telemetry_shard/src/json_message/node_message.rs

@@ -0,0 +1,360 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! The structs and enums defined in this module are largely identical to those
+//! we'll use elsewhere internally, but are kept separate so that the JSON structure
+//! is defined (almost) from just this file, and we don't have to worry about breaking
+//! compatibility with the input data when we make changes to our internal data
+//! structures (for example, to support bincode better).
+use super::hash::Hash;
+use common::node_message as internal;
+use common::node_types;
+use serde::Deserialize;
+
+/// This struct represents a telemetry message sent from a node as
+/// a JSON payload. Since JSON is self describing, we can use attributes
+/// like serde(untagged) and serde(flatten) without issue.
+///
+/// Internally, we want to minimise the amount of data sent from shards to
+/// the core node. For that reason, we use a non-self-describing serialization
+/// format like bincode, which doesn't support things like `[serde(flatten)]` (which
+/// internally wants to serialize to a map of unknown length) or `[serde(tag/untagged)]`
+/// (which relies on the data to know which variant to deserialize to.)
+///
+/// So, this can be converted fairly cheaply into an enum we'll use internally
+/// which is compatible with formats like bincode.
+#[derive(Deserialize, Debug)]
+#[serde(untagged)]
+pub enum NodeMessage {
+    V1 {
+        #[serde(flatten)]
+        payload: Payload,
+    },
+    V2 {
+        id: NodeMessageId,
+        payload: Payload,
+    },
+}
+
+impl From<NodeMessage> for internal::NodeMessage {
+    fn from(msg: NodeMessage) -> Self {
+        match msg {
+            NodeMessage::V1 { payload } => internal::NodeMessage::V1 {
+                payload: payload.into(),
+            },
+            NodeMessage::V2 { id, payload } => internal::NodeMessage::V2 {
+                id,
+                payload: payload.into(),
+            },
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(tag = "msg")]
+pub enum Payload {
+    #[serde(rename = "system.connected")]
+    SystemConnected(SystemConnected),
+    #[serde(rename = "system.interval")]
+    SystemInterval(SystemInterval),
+    #[serde(rename = "block.import")]
+    BlockImport(Block),
+    #[serde(rename = "notify.finalized")]
+    NotifyFinalized(Finalized),
+    #[serde(rename = "txpool.import")]
+    TxPoolImport,
+    #[serde(rename = "afg.finalized")]
+    AfgFinalized(AfgFinalized),
+    #[serde(rename = "afg.received_precommit")]
+    AfgReceivedPrecommit(AfgReceived),
+    #[serde(rename = "afg.received_prevote")]
+    AfgReceivedPrevote(AfgReceived),
+    #[serde(rename = "afg.received_commit")]
+    AfgReceivedCommit(AfgReceived),
+    #[serde(rename = "afg.authority_set")]
+    AfgAuthoritySet(AfgAuthoritySet),
+    #[serde(rename = "afg.finalized_blocks_up_to")]
+    AfgFinalizedBlocksUpTo,
+    #[serde(rename = "aura.pre_sealed_block")]
+    AuraPreSealedBlock,
+    #[serde(rename = "prepared_block_for_proposing")]
+    PreparedBlockForProposing,
+}
+
+impl From<Payload> for internal::Payload {
+    fn from(msg: Payload) -> Self {
+        match msg {
+            Payload::SystemConnected(m) => internal::Payload::SystemConnected(m.into()),
+            Payload::SystemInterval(m) => internal::Payload::SystemInterval(m.into()),
+            Payload::BlockImport(m) => internal::Payload::BlockImport(m.into()),
+            Payload::NotifyFinalized(m) => internal::Payload::NotifyFinalized(m.into()),
+            Payload::TxPoolImport => internal::Payload::TxPoolImport,
+            Payload::AfgFinalized(m) => internal::Payload::AfgFinalized(m.into()),
+            Payload::AfgReceivedPrecommit(m) => internal::Payload::AfgReceivedPrecommit(m.into()),
+            Payload::AfgReceivedPrevote(m) => internal::Payload::AfgReceivedPrevote(m.into()),
+            Payload::AfgReceivedCommit(m) => internal::Payload::AfgReceivedCommit(m.into()),
+            Payload::AfgAuthoritySet(m) => internal::Payload::AfgAuthoritySet(m.into()),
+            Payload::AfgFinalizedBlocksUpTo => internal::Payload::AfgFinalizedBlocksUpTo,
+            Payload::AuraPreSealedBlock => internal::Payload::AuraPreSealedBlock,
+            Payload::PreparedBlockForProposing => internal::Payload::PreparedBlockForProposing,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct SystemConnected {
+    pub genesis_hash: Hash,
+    #[serde(flatten)]
+    pub node: NodeDetails,
+}
+
+impl From<SystemConnected> for internal::SystemConnected {
+    fn from(msg: SystemConnected) -> Self {
+        internal::SystemConnected {
+            genesis_hash: msg.genesis_hash.into(),
+            node: msg.node.into(),
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct SystemInterval {
+    pub peers: Option<u64>,
+    pub txcount: Option<u64>,
+    pub bandwidth_upload: Option<f64>,
+    pub bandwidth_download: Option<f64>,
+    pub finalized_height: Option<BlockNumber>,
+    pub finalized_hash: Option<Hash>,
+    #[serde(flatten)]
+    pub block: Option<Block>,
+    pub used_state_cache_size: Option<f32>,
+}
+
+impl From<SystemInterval> for internal::SystemInterval {
+    fn from(msg: SystemInterval) -> Self {
+        internal::SystemInterval {
+            peers: msg.peers,
+            txcount: msg.txcount,
+            bandwidth_upload: msg.bandwidth_upload,
+            bandwidth_download: msg.bandwidth_download,
+            finalized_height: msg.finalized_height,
+            finalized_hash: msg.finalized_hash.map(|h| h.into()),
+            block: msg.block.map(|b| b.into()),
+            used_state_cache_size: msg.used_state_cache_size,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct Finalized {
+    #[serde(rename = "best")]
+    pub hash: Hash,
+    pub height: Box<str>,
+}
+
+impl From<Finalized> for internal::Finalized {
+    fn from(msg: Finalized) -> Self {
+        internal::Finalized {
+            hash: msg.hash.into(),
+            height: msg.height,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct AfgAuthoritySet {
+    pub authority_id: Box<str>,
+    pub authorities: Box<str>,
+    pub authority_set_id: Box<str>,
+}
+
+impl From<AfgAuthoritySet> for internal::AfgAuthoritySet {
+    fn from(msg: AfgAuthoritySet) -> Self {
+        internal::AfgAuthoritySet {
+            authority_id: msg.authority_id,
+            authorities: msg.authorities,
+            authority_set_id: msg.authority_set_id,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct AfgFinalized {
+    pub finalized_hash: Hash,
+    pub finalized_number: Box<str>,
+}
+
+impl From<AfgFinalized> for internal::AfgFinalized {
+    fn from(msg: AfgFinalized) -> Self {
+        internal::AfgFinalized {
+            finalized_hash: msg.finalized_hash.into(),
+            finalized_number: msg.finalized_number,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct AfgReceived {
+    pub target_hash: Hash,
+    pub target_number: Box<str>,
+    pub voter: Option<Box<str>>,
+}
+
+impl From<AfgReceived> for internal::AfgReceived {
+    fn from(msg: AfgReceived) -> Self {
+        internal::AfgReceived {
+            target_hash: msg.target_hash.into(),
+            target_number: msg.target_number,
+            voter: msg.voter,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone, Copy)]
+pub struct Block {
+    #[serde(rename = "best")]
+    pub hash: Hash,
+    pub height: BlockNumber,
+}
+
+impl From<Block> for node_types::Block {
+    fn from(block: Block) -> Self {
+        node_types::Block {
+            hash: block.hash.into(),
+            height: block.height,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct NodeDetails {
+    pub chain: Box<str>,
+    pub name: Box<str>,
+    pub implementation: Box<str>,
+    pub version: Box<str>,
+    pub validator: Option<Box<str>>,
+    pub network_id: Option<Box<str>>,
+    pub startup_time: Option<Box<str>>,
+}
+
+impl From<NodeDetails> for node_types::NodeDetails {
+    fn from(details: NodeDetails) -> Self {
+        node_types::NodeDetails {
+            chain: details.chain,
+            name: details.name,
+            implementation: details.implementation,
+            version: details.version,
+            validator: details.validator,
+            network_id: details.network_id,
+            startup_time: details.startup_time,
+        }
+    }
+}
+
+type NodeMessageId = u64;
+type BlockNumber = u64;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn message_v1() {
+        let json = r#"{
+            "msg":"notify.finalized",
+            "level":"INFO",
+            "ts":"2021-01-13T12:38:25.410794650+01:00",
+            "best":"0x031c3521ca2f9c673812d692fc330b9a18e18a2781e3f9976992f861fd3ea0cb",
+            "height":"50"
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V1 { .. },
+            ),
+            "message did not match variant V1",
+        );
+    }
+
+    #[test]
+    fn message_v2() {
+        let json = r#"{
+            "id":1,
+            "ts":"2021-01-13T12:22:20.053527101+01:00",
+            "payload":{
+                "best":"0xcc41708573f2acaded9dd75e07dac2d4163d136ca35b3061c558d7a35a09dd8d",
+                "height":"209",
+                "msg":"notify.finalized"
+            }
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V2 { .. },
+            ),
+            "message did not match variant V2",
+        );
+    }
+
+    #[test]
+    fn message_v2_received_precommit() {
+        let json = r#"{
+            "id":1,
+            "ts":"2021-01-13T12:22:20.053527101+01:00",
+            "payload":{
+                "target_hash":"0xcc41708573f2acaded9dd75e07dac2d4163d136ca35b3061c558d7a35a09dd8d",
+                "target_number":"209",
+                "voter":"foo",
+                "msg":"afg.received_precommit"
+            }
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V2 {
+                    payload: Payload::AfgReceivedPrecommit(..),
+                    ..
+                },
+            ),
+            "message did not match the expected output",
+        );
+    }
+
+    #[test]
+    fn message_v2_tx_pool_import() {
+        // We should happily ignore any fields we don't care about.
+        let json = r#"{
+            "id":1,
+            "ts":"2021-01-13T12:22:20.053527101+01:00",
+            "payload":{
+                "foo":"Something",
+                "bar":123,
+                "wibble":"wobble",
+                "msg":"txpool.import"
+            }
+        }"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V2 {
+                    payload: Payload::TxPoolImport,
+                    ..
+                },
+            ),
+            "message did not match the expected output",
+        );
+    }
+}

backend/telemetry_shard/src/main.rs

@@ -0,0 +1,305 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[warn(missing_docs)]
+mod aggregator;
+mod blocked_addrs;
+mod connection;
+mod json_message;
+mod real_ip;
+
+use std::{collections::HashSet, net::IpAddr, time::Duration};
+
+use aggregator::{Aggregator, FromWebsocket};
+use blocked_addrs::BlockedAddrs;
+use common::byte_size::ByteSize;
+use common::http_utils;
+use common::node_message;
+use common::rolling_total::RollingTotalBuilder;
+use futures::{channel::mpsc, SinkExt, StreamExt};
+use http::Uri;
+use hyper::{Method, Response};
+use simple_logger::SimpleLogger;
+use structopt::StructOpt;
+
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
+const NAME: &str = "Substrate Telemetry Backend Shard";
+const ABOUT: &str = "This is the Telemetry Backend Shard that forwards the \
+                     data sent by Substrate/Polkadot nodes to the Backend Core";
+
+#[derive(StructOpt, Debug)]
+#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
+struct Opts {
+    /// This is the socket address that this shard is listening to. This is restricted to
+    /// localhost (127.0.0.1) by default and should be fine for most use cases. If
+    /// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
+    #[structopt(short = "l", long = "listen", default_value = "127.0.0.1:8001")]
+    socket: std::net::SocketAddr,
+    /// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
+    /// 'error' only logs errors and 'trace' logs everything.
+    #[structopt(long = "log", default_value = "info")]
+    log_level: log::LevelFilter,
+    /// Url to the Backend Core endpoint accepting shard connections
+    #[structopt(
+        short = "c",
+        long = "core",
+        default_value = "ws://127.0.0.1:8000/shard_submit/"
+    )]
+    core_url: Uri,
+    /// How many different nodes is a given connection to the /submit endpoint allowed to
+    /// tell us about before we ignore the rest?
+    ///
+    /// This is important because without a limit, a single connection could exhaust
+    /// RAM by suggesting that it accounts for billions of nodes.
+    #[structopt(long, default_value = "20")]
+    max_nodes_per_connection: usize,
+    /// What is the maximum number of bytes per second, on average, that a connection from a
+    /// node is allowed to send to a shard before it gets booted. This is averaged over a
+    /// rolling window of 10 seconds, and so spikes beyond this limit are allowed as long as
+    /// the average traffic in the last 10 seconds falls below this value.
+    ///
+    /// As a reference point, syncing a new Polkadot node leads to a maximum of about 25k of
+    /// traffic on average (at least initially).
+    #[structopt(long, default_value = "256k")]
+    max_node_data_per_second: ByteSize,
+    /// How many seconds is a "/feed" connection that violates the '--max-node-data-per-second'
+    /// value prevented from reconnecting to this shard for, in seconds.
+    #[structopt(long, default_value = "600")]
+    node_block_seconds: u64,
+    /// Number of worker threads to spawn. If "0" is given, use the number of CPUs available
+    /// on the machine. If no value is given, use an internal default that we have deemed sane.
+    #[structopt(long)]
+    worker_threads: Option<usize>,
+}
+
+fn main() {
+    let opts = Opts::from_args();
+
+    SimpleLogger::new()
+        .with_level(opts.log_level)
+        .init()
+        .expect("Must be able to start a logger");
+
+    log::info!("Starting Telemetry Shard version: {}", VERSION);
+
+    let worker_threads = match opts.worker_threads {
+        Some(0) => num_cpus::get(),
+        Some(n) => n,
+        // By default, use a max of 4 worker threads, as we don't
+        // expect to need a lot of parallelism in shards.
+        None => usize::min(num_cpus::get(), 4),
+    };
+
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .worker_threads(worker_threads)
+        .thread_name("telemetry_shard_worker")
+        .build()
+        .unwrap()
+        .block_on(async {
+            if let Err(e) = start_server(opts).await {
+                log::error!("Error starting server: {}", e);
+            }
+        });
+}
+
+/// Declare our routes and start the server.
+async fn start_server(opts: Opts) -> anyhow::Result<()> {
+    let block_list = BlockedAddrs::new(Duration::from_secs(opts.node_block_seconds));
+    let aggregator = Aggregator::spawn(opts.core_url).await?;
+    let socket_addr = opts.socket;
+    let max_nodes_per_connection = opts.max_nodes_per_connection;
+    let bytes_per_second = opts.max_node_data_per_second;
+
+    let server = http_utils::start_server(socket_addr, move |addr, req| {
+        let aggregator = aggregator.clone();
+        let block_list = block_list.clone();
+        async move {
+            match (req.method(), req.uri().path().trim_end_matches('/')) {
+                // Check that the server is up and running:
+                (&Method::GET, "/health") => Ok(Response::new("OK".into())),
+                // Nodes send messages here:
+                (&Method::GET, "/submit") => {
+                    let real_addr = real_ip::real_ip(addr, req.headers());
+
+                    if let Some(reason) = block_list.blocked_reason(&real_addr) {
+                        return Ok(Response::builder().status(403).body(reason.into()).unwrap());
+                    }
+
+                    Ok(http_utils::upgrade_to_websocket(
+                        req,
+                        move |ws_send, ws_recv| async move {
+                            log::info!("Opening /submit connection from {:?}", addr);
+                            let tx_to_aggregator = aggregator.subscribe_node();
+                            let (mut tx_to_aggregator, mut ws_send) =
+                                handle_node_websocket_connection(
+                                    real_addr,
+                                    ws_send,
+                                    ws_recv,
+                                    tx_to_aggregator,
+                                    max_nodes_per_connection,
+                                    bytes_per_second,
+                                    block_list,
+                                )
+                                .await;
+                            log::info!("Closing /submit connection from {:?}", addr);
+                            // Tell the aggregator that this connection has closed, so it can tidy up.
+                            let _ = tx_to_aggregator.send(FromWebsocket::Disconnected).await;
+                            let _ = ws_send.close().await;
+                        },
+                    ))
+                }
+                // 404 for anything else:
+                _ => Ok(Response::builder()
+                    .status(404)
+                    .body("Not found".into())
+                    .unwrap()),
+            }
+        }
+    });
+
+    server.await?;
+    Ok(())
+}
+
+/// This takes care of handling messages from an established socket connection.
+async fn handle_node_websocket_connection<S>(
+    real_addr: IpAddr,
+    ws_send: http_utils::WsSender,
+    mut ws_recv: http_utils::WsReceiver,
+    mut tx_to_aggregator: S,
+    max_nodes_per_connection: usize,
+    bytes_per_second: ByteSize,
+    block_list: BlockedAddrs,
+) -> (S, http_utils::WsSender)
+where
+    S: futures::Sink<FromWebsocket, Error = anyhow::Error> + Unpin + Send + 'static,
+{
+    // Limit the number of bytes based on a rolling total and the incoming bytes per second
+    // that has been configured via the CLI opts.
+    let bytes_per_second = bytes_per_second.num_bytes();
+    let mut rolling_total_bytes = RollingTotalBuilder::new()
+        .granularity(Duration::from_secs(1))
+        .window_size_multiple(10)
+        .start();
+
+    // Track all of the message IDs that we've seen so far. If we exceed the
+    // max_nodes_per_connection limit we ignore subsequent message IDs.
+    let mut message_ids_seen = HashSet::new();
+
+    // This could be a oneshot channel, but it's useful to be able to clone
+    // messages, and we can't clone oneshot channel senders.
+    let (close_connection_tx, mut close_connection_rx) = mpsc::channel(0);
+
+    // Tell the aggregator about this new connection, and give it a way to close this connection:
+    let init_msg = FromWebsocket::Initialize {
+        close_connection: close_connection_tx,
+    };
+    if let Err(e) = tx_to_aggregator.send(init_msg).await {
+        log::error!("Error sending message to aggregator: {}", e);
+        return (tx_to_aggregator, ws_send);
+    }
+
+    // Now we've "initialized", wait for messages from the node. Messages will
+    // either be `SystemConnected` type messages that inform us that a new set
+    // of messages with some message ID will be sent (a node could have more
+    // than one of these), or updates linked to a specific message_id.
+    loop {
+        let mut bytes = Vec::new();
+        tokio::select! {
+            // The close channel has fired, so end the loop. `ws_recv.receive_data` is
+            // *not* cancel safe, but since we're closing the connection we don't care.
+            _ = close_connection_rx.next() => {
+                log::info!("connection to {:?} being closed by aggregator", real_addr);
+                break
+            },
+            // A message was received; handle it:
+            msg_info = ws_recv.receive_data(&mut bytes) => {
+                // Handle the socket closing, or errors receiving the message.
+                if let Err(soketto::connection::Error::Closed) = msg_info {
+                    break;
+                }
+                if let Err(e) = msg_info {
+                    log::error!("Shutting down websocket connection: Failed to receive data: {}", e);
+                    break;
+                }
+
+                // Keep track of total bytes and bail if average over last 10 secs exceeds preference.
+                rolling_total_bytes.push(bytes.len());
+                let this_bytes_per_second = rolling_total_bytes.total() / 10;
+                if this_bytes_per_second > bytes_per_second {
+                    block_list.block_addr(real_addr, "Too much traffic");
+                    log::error!("Shutting down websocket connection: Too much traffic ({}bps averaged over last 10s)", this_bytes_per_second);
+                    break;
+                }
+
+                // Deserialize from JSON, warning in debug mode if deserialization fails:
+                let node_message: json_message::NodeMessage = match serde_json::from_slice(&bytes) {
+                    Ok(node_message) => node_message,
+                    #[cfg(debug)]
+                    Err(e) => {
+                        let bytes: &[u8] = bytes.get(..512).unwrap_or_else(|| &bytes);
+                        let msg_start = std::str::from_utf8(bytes).unwrap_or_else(|_| "INVALID UTF8");
+                        log::warn!("Failed to parse node message ({}): {}", msg_start, e);
+                        continue;
+                    },
+                    #[cfg(not(debug))]
+                    Err(_) => {
+                        continue;
+                    }
+                };
+
+                // Pull relevant details from the message:
+                let node_message: node_message::NodeMessage = node_message.into();
+                let message_id = node_message.id();
+                let payload = node_message.into_payload();
+
+                // Ignore messages from IDs that exceed our limit:
+                if message_ids_seen.contains(&message_id) {
+                    // continue on; we're happy
+                } else if message_ids_seen.len() >= max_nodes_per_connection {
+                    // ignore this message; it's not a "seen" ID and we've hit our limit.
+                    continue;
+                } else {
+                    // not seen ID, not hit limit; make note of new ID
+                    message_ids_seen.insert(message_id);
+                }
+
+                // Until the aggregator receives an `Add` message, which we can create once
+                // we see one of these SystemConnected ones, it will ignore messages with
+                // the corresponding message_id.
+                if let node_message::Payload::SystemConnected(info) = payload {
+                    let _ = tx_to_aggregator.send(FromWebsocket::Add {
+                        message_id,
+                        ip: real_addr,
+                        node: info.node,
+                        genesis_hash: info.genesis_hash,
+                    }).await;
+                }
+                // Anything that's not an "Add" is an Update. The aggregator will ignore
+                // updates against a message_id that hasn't first been Added, above.
+                else if let Err(e) = tx_to_aggregator.send(FromWebsocket::Update { message_id, payload } ).await {
+                    log::error!("Failed to send node message to aggregator: {}", e);
+                    continue;
+                }
+            }
+        }
+    }
+
+    // Return what we need to close the connection gracefully:
+    (tx_to_aggregator, ws_send)
+}

backend/telemetry_shard/src/real_ip.rs

@@ -0,0 +1,151 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::net::{IpAddr, SocketAddr};
+
+/**
+Extract the "real" IP address of the connection by looking at headers
+set by proxies (this is inspired by Actix Web's implementation of the feature).
+
+First, check for the standardised "Forwarded" header. This looks something like:
+
+"Forwarded: for=12.34.56.78;host=example.com;proto=https, for=23.45.67.89"
+
+Each proxy can append to this comma separated list of forwarded-details. We'll look for
+the first "for" address and try to decode that.
+
+If this doesn't yield a result, look for the non-standard but common X-Forwarded-For header,
+which contains a comma separated list of addresses; each proxy in the potential chain possibly
+appending one to the end. So, take the first of these if it exists.
+
+If still no luck, look for the X-Real-IP header, which we expect to contain a single IP address.
+
+If that _still_ doesn't work, fall back to the socket address of the connection.
+*/
+pub fn real_ip(addr: SocketAddr, headers: &hyper::HeaderMap) -> IpAddr {
+    let forwarded = headers.get("forwarded").and_then(header_as_str);
+    let forwarded_for = headers.get("x-forwarded-for").and_then(header_as_str);
+    let real_ip = headers.get("x-real-ip").and_then(header_as_str);
+    pick_best_ip_from_options(forwarded, forwarded_for, real_ip, addr)
+}
+
+fn header_as_str(value: &hyper::header::HeaderValue) -> Option<&str> {
+    std::str::from_utf8(value.as_bytes()).ok()
+}
+
+fn pick_best_ip_from_options(
+    // Forwarded header value (if present)
+    forwarded: Option<&str>,
+    // X-Forwarded-For header value (if present)
+    forwarded_for: Option<&str>,
+    // X-Real-IP header value (if present)
+    real_ip: Option<&str>,
+    // socket address (if known)
+    addr: SocketAddr,
+) -> IpAddr {
+    let realip = forwarded
+        .as_ref()
+        .and_then(|val| get_first_addr_from_forwarded_header(val))
+        .or_else(|| {
+            // fall back to X-Forwarded-For
+            forwarded_for
+                .as_ref()
+                .and_then(|val| get_first_addr_from_x_forwarded_for_header(val))
+        })
+        .or_else(|| {
+            // fall back to X-Real-IP
+            real_ip.as_ref().map(|val| val.trim())
+        })
+        .and_then(|ip| {
+            // Try parsing assuming it may have a port first,
+            // and then assuming it doesn't.
+            ip.parse::<SocketAddr>()
+                .map(|s| s.ip())
+                .or_else(|_| ip.parse::<IpAddr>())
+                .ok()
+        })
+        // Fall back to local IP address if the above fails
+        .unwrap_or(addr.ip());
+
+    realip
+}
+
+/// Follow <https://datatracker.ietf.org/doc/html/rfc7239> to decode the Forwarded header value.
+/// Roughly, proxies can add new sets of values by appending a comma to the existing list
+/// (so we have something like "values1, values2, values3" from proxy1, proxy2 and proxy3 for
+/// instance) and then the valeus themselves are ';' separated name=value pairs. The value in each
+/// pair may or may not be surrounded in double quotes.
+///
+/// Examples from the RFC:
+///
+/// ```text
+/// Forwarded: for="_gazonk"
+/// Forwarded: For="[2001:db8:cafe::17]:4711"
+/// Forwarded: for=192.0.2.60;proto=http;by=203.0.113.43
+/// Forwarded: for=192.0.2.43, for=198.51.100.17
+/// ```
+fn get_first_addr_from_forwarded_header(value: &str) -> Option<&str> {
+    let first_values = value.split(',').next()?;
+
+    for pair in first_values.split(';') {
+        let mut parts = pair.trim().splitn(2, '=');
+        let key = parts.next()?;
+        let value = parts.next()?;
+
+        if key.to_lowercase() == "for" {
+            // trim double quotes if they surround the value:
+            let value = if value.starts_with('"') && value.ends_with('"') {
+                &value[1..value.len() - 1]
+            } else {
+                value
+            };
+            return Some(value);
+        }
+    }
+
+    None
+}
+
+fn get_first_addr_from_x_forwarded_for_header(value: &str) -> Option<&str> {
+    value.split(",").map(|val| val.trim()).next()
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn get_addr_from_forwarded_rfc_examples() {
+        let examples = vec![
+            (r#"for="_gazonk""#, "_gazonk"),
+            (
+                r#"For="[2001:db8:cafe::17]:4711""#,
+                "[2001:db8:cafe::17]:4711",
+            ),
+            (r#"for=192.0.2.60;proto=http;by=203.0.113.43"#, "192.0.2.60"),
+            (r#"for=192.0.2.43, for=198.51.100.17"#, "192.0.2.43"),
+        ];
+
+        for (value, expected) in examples {
+            assert_eq!(
+                get_first_addr_from_forwarded_header(value),
+                Some(expected),
+                "Header value: {}",
+                value
+            );
+        }
+    }
+}

backend/test_utils/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "test_utils"
+version = "0.1.0"
+authors = ["James Wilson <james@jsdw.me>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.41"
+futures = "0.3.15"
+http = "0.2.4"
+log = "0.4.14"
+serde_json = "1.0.64"
+soketto = "0.6.0"
+thiserror = "1.0.25"
+tokio = { version = "1.7.1", features = ["full"] }
+tokio-util = { version = "0.6.7", features = ["full"] }
+common = { path = "../common" }
+time = { version = "0.3.0", features = ["formatting"] }

backend/test_utils/src/contains_matches.rs

@@ -0,0 +1,110 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/**
+This macro checks to see whether an iterable container contains each of the
+match items given, in the order that they are given in (but not necessarily
+contiguous, ie other items may be interspersed between the ones we're looking
+to match).
+
+Similar to `matches!`.
+
+```
+enum Item {
+    Foo { a: usize },
+    Bar(bool),
+    Wibble
+}
+
+use Item::*;
+
+let does_contain: bool = test_utils::contains_matches!(
+    vec![Foo { a: 2 }, Wibble, Bar(true), Foo { a: 100 }],
+    Foo { a: 2 } | Foo { a: 3 },
+    Bar(true),
+    Foo {..}
+);
+
+assert!(does_contain);
+```
+*/
+#[macro_export]
+macro_rules! contains_matches {
+    ($expression:expr, $( $( $pattern:pat )|+ $( if $guard:expr )? ),+ $(,)?) => {{
+        let mut items = $expression.into_iter();
+
+        // For each pattern we want to match, we consume items until
+        // we find the first match, and then break the loop and do the
+        // same again with the next pattern. If we run out of items, we
+        // set the validity to false and stop trying to match. Else, we
+        // match againse each of the patterns and return true.
+        let mut is_valid = true;
+        $(
+            while is_valid {
+                let item = match items.next() {
+                    Some(item) => item,
+                    None => {
+                        is_valid = false;
+                        break;
+                    }
+                };
+
+                match item {
+                    $( $pattern )|+ $( if $guard )? => break,
+                    _ => continue
+                }
+            }
+        )+
+
+        is_valid
+    }}
+}
+
+/**
+This macro checks to see whether an iterable container contains each of the
+match items given, in the order that they are given in (but not necessarily
+contiguous, ie other items may be interspersed between the ones we're looking
+to match).
+
+Panics if this is not the case.
+```
+enum Item {
+    Foo { a: usize },
+    Bar(bool),
+    Wibble
+}
+
+use Item::*;
+
+test_utils::assert_contains_matches!(
+    vec![Foo { a: 2 }, Wibble, Bar(true), Foo { a: 100 }],
+    Foo { a: 2 },
+    Bar(true),
+    Foo {..}
+);
+```
+*/
+#[macro_export]
+macro_rules! assert_contains_matches {
+    ($expression:expr, $( $( $pattern:pat )|+ $( if $guard:expr )? ),+ $(,)?) => {
+        let does_contain_matches = $crate::contains_matches!(
+            $expression,
+            $( $( $pattern )|+ $( if $guard )? ),+
+        );
+
+        assert!(does_contain_matches);
+    }
+}

backend/test_utils/src/fake_telemetry.rs

@@ -0,0 +1,198 @@
+use ::time::{format_description::well_known::Rfc3339, OffsetDateTime};
+use common::node_types::BlockHash;
+use serde_json::json;
+use std::future::Future;
+use std::time::Duration;
+use tokio::time::{self, MissedTickBehavior};
+
+/// This emits fake but realistic looking telemetry messages.
+/// Can be connected to a telemetry server to emit realistic messages.
+pub struct FakeTelemetry {
+    block_time: Duration,
+    node_name: String,
+    chain: String,
+    message_id: usize,
+}
+
+impl FakeTelemetry {
+    pub fn new(block_time: Duration, node_name: String, chain: String, message_id: usize) -> Self {
+        Self {
+            block_time,
+            node_name,
+            chain,
+            message_id,
+        }
+    }
+
+    /// Begin emitting messages from this node, calling the provided callback each
+    /// time a new message is emitted.
+    // Unused assignments allowed because macro seems to mess with knowledge of what's
+    // been read.
+    #[allow(unused_assignments)]
+    pub async fn start<Func, Fut, E>(self, mut on_message: Func) -> Result<(), E>
+    where
+        Func: Send + FnMut(Vec<u8>) -> Fut,
+        Fut: Future<Output = Result<(), E>>,
+        E: Into<anyhow::Error>,
+    {
+        let id = self.message_id;
+        let name = self.node_name;
+        let chain = self.chain;
+        let block_time = self.block_time;
+
+        // Our "state". These numbers can be hashed to give a block hash,
+        // and also represent the height of the chain so far. Increment each
+        // as needed.
+        let mut best_block_n: u64 = 0;
+        let mut finalized_block_n: u64 = 0;
+
+        // A helper to send JSON messages without consuming on_message:
+        macro_rules! send_msg {
+            ($($json:tt)+) => {{
+                let msg = json!($($json)+);
+                let bytes = serde_json::to_vec(&msg).unwrap();
+                on_message(bytes).await
+            }}
+        }
+
+        // Send system connected immediately
+        send_msg!({
+            "id":id,
+            "payload": {
+                "authority":true,
+                "chain":chain,
+                "config":"",
+                "genesis_hash":block_hash(best_block_n),
+                "implementation":"Substrate Node",
+                "msg":"system.connected",
+                "name":name,
+                "network_id":"12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp",
+                "startup_time":"1627986634759",
+                "version":"2.0.0-07a1af348-aarch64-macos"
+            },
+            "ts":now_iso()
+        })?;
+        best_block_n += 1;
+
+        // First block import immediately (height 1)
+        send_msg!({
+            "id":id,
+            "payload":{
+                "best":block_hash(best_block_n),
+                "height":best_block_n,
+                "msg":"block.import",
+                "origin":"Own"
+            },
+            "ts":now_iso()
+        })?;
+        best_block_n += 1;
+
+        let now = tokio::time::Instant::now();
+
+        // Configure our message intervals:
+        let mut new_block_every = time::interval_at(now + block_time, block_time);
+        new_block_every.set_missed_tick_behavior(MissedTickBehavior::Burst);
+
+        let mut system_interval_every =
+            time::interval_at(now + Duration::from_secs(2), block_time * 2);
+        new_block_every.set_missed_tick_behavior(MissedTickBehavior::Burst);
+
+        let mut finalised_every =
+            time::interval_at(now + Duration::from_secs(1) + block_time * 3, block_time);
+        new_block_every.set_missed_tick_behavior(MissedTickBehavior::Burst);
+
+        // Send messages every interval:
+        loop {
+            tokio::select! {
+                // Add a new block:
+                _ = new_block_every.tick() => {
+
+                    send_msg!({
+                        "id":id,
+                        "payload":{
+                            "hash":"0x918bf5125307b4ac1b2c67aa43ed38517617720ac96cbd5664d7a0f0aa32e1b1", // Don't think this matters
+                            "msg":"prepared_block_for_proposing",
+                            "number":best_block_n.to_string() // seems to be a string, not a number in the "real" JSON
+                        },
+                        "ts":now_iso()
+                    })?;
+                    send_msg!({
+                        "id":id,
+                        "payload":{
+                            "best":block_hash(best_block_n),
+                            "height":best_block_n,
+                            "msg":"block.import",
+                            "origin":"Own"
+                        },
+                        "ts":now_iso()
+                    })?;
+                    best_block_n += 1;
+
+                },
+                // Periodic updates on system state:
+                _ = system_interval_every.tick() => {
+
+                    send_msg!({
+                        "id":id,
+                        "payload":{
+                            "best":block_hash(best_block_n),
+                            "finalized_hash":block_hash(finalized_block_n),
+                            "finalized_height":finalized_block_n,
+                            "height":best_block_n,
+                            "msg":"system.interval",
+                            "txcount":0,
+                            "used_state_cache_size":870775
+                        },
+                        "ts":now_iso()
+                    })?;
+                    send_msg!({
+                        "id":id,
+                        "payload":{
+                            "bandwidth_download":0,
+                            "bandwidth_upload":0,
+                            "msg":"system.interval",
+                            "peers":0
+                        },
+                        "ts":now_iso()
+                    })?;
+
+                },
+                // Finalise a block:
+                _ = finalised_every.tick() => {
+
+                    send_msg!({
+                        "id":1,
+                        "payload":{
+                            "hash":block_hash(finalized_block_n),
+                            "msg":"afg.finalized_blocks_up_to",
+                            "number":finalized_block_n.to_string(), // string in "real" JSON.
+                        },
+                        "ts":now_iso()
+                    })?;
+                    send_msg!({
+                        "id":1,
+                        "payload":{
+                            "best":block_hash(finalized_block_n),
+                            "height":finalized_block_n.to_string(), // string in "real" JSON.
+                            "msg":"notify.finalized"
+                        },
+                        "ts":now_iso()
+                    })?;
+                    finalized_block_n += 1;
+
+                },
+            };
+        }
+    }
+}
+
+fn now_iso() -> String {
+    OffsetDateTime::now_utc().format(&Rfc3339).unwrap()
+}
+
+/// Spread the u64 across the resulting u256 hash so that it's
+/// more visible in the UI.
+fn block_hash(n: u64) -> BlockHash {
+    let a: [u8; 32] = unsafe { std::mem::transmute([n, n, n, n]) };
+    BlockHash::from(a)
+}

backend/test_utils/src/feed_message_de.rs

@@ -0,0 +1,386 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::Context;
+use common::node_types::{
+    BlockDetails, BlockHash, BlockNumber, NodeLocation, NodeStats, Timestamp,
+};
+use serde_json::value::RawValue;
+
+#[derive(Debug, PartialEq)]
+pub enum FeedMessage {
+    Version(usize),
+    BestBlock {
+        block_number: BlockNumber,
+        timestamp: Timestamp,
+        avg_block_time: Option<u64>,
+    },
+    BestFinalized {
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    AddedNode {
+        node_id: usize,
+        node: NodeDetails,
+        stats: NodeStats,
+        // io: NodeIO, // can't losslessly deserialize
+        // hardware: NodeHardware, // can't losslessly deserialize
+        block_details: BlockDetails,
+        location: Option<NodeLocation>,
+        startup_time: Option<Timestamp>,
+    },
+    RemovedNode {
+        node_id: usize,
+    },
+    LocatedNode {
+        node_id: usize,
+        lat: f32,
+        long: f32,
+        city: String,
+    },
+    ImportedBlock {
+        node_id: usize,
+        block_details: BlockDetails,
+    },
+    FinalizedBlock {
+        node_id: usize,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    NodeStatsUpdate {
+        node_id: usize,
+        stats: NodeStats,
+    },
+    Hardware {
+        node_id: usize,
+        // hardware: NodeHardware, // Can't losslessly deserialize
+    },
+    TimeSync {
+        time: Timestamp,
+    },
+    AddedChain {
+        name: String,
+        node_count: usize,
+    },
+    RemovedChain {
+        name: String,
+    },
+    SubscribedTo {
+        name: String,
+    },
+    UnsubscribedFrom {
+        name: String,
+    },
+    Pong {
+        msg: String,
+    },
+    AfgFinalized {
+        address: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    AfgReceivedPrevote {
+        address: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+        voter: Option<String>,
+    },
+    AfgReceivedPrecommit {
+        address: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+        voter: Option<String>,
+    },
+    AfgAuthoritySet {
+        // Not used currently; not sure what "address" params are:
+        a1: String,
+        a2: String,
+        a3: String,
+        block_number: BlockNumber,
+        block_hash: BlockHash,
+    },
+    StaleNode {
+        node_id: usize,
+    },
+    NodeIOUpdate {
+        node_id: usize,
+        // details: NodeIO, // can't losslessly deserialize
+    },
+    /// A "special" case when we don't know how to decode an action:
+    UnknownValue {
+        action: u8,
+        value: String,
+    },
+}
+
+#[derive(Debug, PartialEq)]
+pub struct NodeDetails {
+    pub name: String,
+    pub implementation: String,
+    pub version: String,
+    pub validator: Option<String>,
+    pub network_id: Option<String>,
+}
+
+impl FeedMessage {
+    /// Decode a slice of bytes into a vector of feed messages
+    pub fn from_bytes(bytes: &[u8]) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        let v: Vec<&RawValue> = serde_json::from_slice(bytes)?;
+
+        let mut feed_messages = vec![];
+        for raw_keyval in v.chunks(2) {
+            let raw_key = raw_keyval[0];
+            let raw_val = raw_keyval[1];
+            let action: u8 = serde_json::from_str(raw_key.get())?;
+            let msg = FeedMessage::decode(action, raw_val)
+                .with_context(|| format!("Failed to decode message with action {}", action))?;
+
+            feed_messages.push(msg);
+        }
+
+        Ok(feed_messages)
+    }
+
+    // Deserialize the feed message to a value based on the "action" key
+    fn decode(action: u8, raw_val: &RawValue) -> Result<FeedMessage, anyhow::Error> {
+        let feed_message = match action {
+            // Version:
+            0 => {
+                let version = serde_json::from_str(raw_val.get())?;
+                FeedMessage::Version(version)
+            }
+            // BestBlock
+            1 => {
+                let (block_number, timestamp, avg_block_time) =
+                    serde_json::from_str(raw_val.get())?;
+                FeedMessage::BestBlock {
+                    block_number,
+                    timestamp,
+                    avg_block_time,
+                }
+            }
+            // BestFinalized
+            2 => {
+                let (block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::BestFinalized {
+                    block_number,
+                    block_hash,
+                }
+            }
+            // AddNode
+            3 => {
+                let (
+                    node_id,
+                    (name, implementation, version, validator, network_id),
+                    stats,
+                    io,
+                    hardware,
+                    block_details,
+                    location,
+                    startup_time,
+                ) = serde_json::from_str(raw_val.get())?;
+
+                // Give these two types but don't use the results:
+                let (_, _): (&RawValue, &RawValue) = (io, hardware);
+
+                FeedMessage::AddedNode {
+                    node_id,
+                    node: NodeDetails {
+                        name,
+                        implementation,
+                        version,
+                        validator,
+                        network_id,
+                    },
+                    stats,
+                    block_details,
+                    location,
+                    startup_time,
+                }
+            }
+            // RemoveNode
+            4 => {
+                let node_id = serde_json::from_str(raw_val.get())?;
+                FeedMessage::RemovedNode { node_id }
+            }
+            // LocatedNode
+            5 => {
+                let (node_id, lat, long, city) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::LocatedNode {
+                    node_id,
+                    lat,
+                    long,
+                    city,
+                }
+            }
+            // ImportedBlock
+            6 => {
+                let (node_id, block_details) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::ImportedBlock {
+                    node_id,
+                    block_details,
+                }
+            }
+            // FinalizedBlock
+            7 => {
+                let (node_id, block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::FinalizedBlock {
+                    node_id,
+                    block_number,
+                    block_hash,
+                }
+            }
+            // NodeStatsUpdate
+            8 => {
+                let (node_id, stats) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::NodeStatsUpdate { node_id, stats }
+            }
+            // Hardware
+            9 => {
+                let (node_id, _hardware): (_, &RawValue) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::Hardware { node_id }
+            }
+            // TimeSync
+            10 => {
+                let time = serde_json::from_str(raw_val.get())?;
+                FeedMessage::TimeSync { time }
+            }
+            // AddedChain
+            11 => {
+                let (name, node_count) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::AddedChain { name, node_count }
+            }
+            // RemovedChain
+            12 => {
+                let name = serde_json::from_str(raw_val.get())?;
+                FeedMessage::RemovedChain { name }
+            }
+            // SubscribedTo
+            13 => {
+                let name = serde_json::from_str(raw_val.get())?;
+                FeedMessage::SubscribedTo { name }
+            }
+            // UnsubscribedFrom
+            14 => {
+                let name = serde_json::from_str(raw_val.get())?;
+                FeedMessage::UnsubscribedFrom { name }
+            }
+            // Pong
+            15 => {
+                let msg = serde_json::from_str(raw_val.get())?;
+                FeedMessage::Pong { msg }
+            }
+            // AfgFinalized
+            16 => {
+                let (address, block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgFinalized {
+                    address,
+                    block_number,
+                    block_hash,
+                }
+            }
+            // AfgReceivedPrevote
+            17 => {
+                let (address, block_number, block_hash, voter) =
+                    serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgReceivedPrevote {
+                    address,
+                    block_number,
+                    block_hash,
+                    voter,
+                }
+            }
+            // AfgReceivedPrecommit
+            18 => {
+                let (address, block_number, block_hash, voter) =
+                    serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgReceivedPrecommit {
+                    address,
+                    block_number,
+                    block_hash,
+                    voter,
+                }
+            }
+            // AfgAuthoritySet
+            19 => {
+                let (a1, a2, a3, block_number, block_hash) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::AfgAuthoritySet {
+                    a1,
+                    a2,
+                    a3,
+                    block_number,
+                    block_hash,
+                }
+            }
+            // StaleNode
+            20 => {
+                let node_id = serde_json::from_str(raw_val.get())?;
+                FeedMessage::StaleNode { node_id }
+            }
+            // NodeIOUpdate
+            21 => {
+                // ignore NodeIO for now:
+                let (node_id, _node_io): (_, &RawValue) = serde_json::from_str(raw_val.get())?;
+                FeedMessage::NodeIOUpdate { node_id }
+            }
+            // A catchall for messages we don't know/care about yet:
+            _ => {
+                let value = raw_val.to_string();
+                FeedMessage::UnknownValue { action, value }
+            }
+        };
+
+        Ok(feed_message)
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+
+    #[test]
+    fn decode_remove_node_msg() {
+        // "remove chain ''":
+        let msg = r#"[12,""]"#;
+
+        assert_eq!(
+            FeedMessage::from_bytes(msg.as_bytes()).unwrap(),
+            vec![FeedMessage::RemovedChain {
+                name: "".to_owned()
+            }]
+        );
+    }
+
+    #[test]
+    fn decode_remove_then_add_node_msg() {
+        // "remove chain '', then add chain 'Local Testnet' with 1 node":
+        let msg = r#"[12,"",11,["Local Testnet",1]]"#;
+
+        assert_eq!(
+            FeedMessage::from_bytes(msg.as_bytes()).unwrap(),
+            vec![
+                FeedMessage::RemovedChain {
+                    name: "".to_owned()
+                },
+                FeedMessage::AddedChain {
+                    name: "Local Testnet".to_owned(),
+                    node_count: 1
+                },
+            ]
+        );
+    }
+}

backend/test_utils/src/lib.rs

@@ -0,0 +1,33 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+/// Create/connect to a server consisting of shards and a core process that we can interact with.
+pub mod server;
+
+/// Test support for deserializing feed messages from the feed processes. This basically
+/// is the slightly-lossy inverse of the custom serialization we do to feed messages.
+pub mod feed_message_de;
+
+/// A couple of macros to make it easier to test for the presense of things (mainly, feed messages)
+/// in an iterable container.
+#[macro_use]
+pub mod contains_matches;
+
+/// Utilities to help with running tests from within this current workspace.
+pub mod workspace;
+
+/// A utility to generate fake telemetry messages at realistic intervals.
+pub mod fake_telemetry;

backend/test_utils/src/server/channels.rs

@@ -0,0 +1,300 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+    ops::{Deref, DerefMut},
+    time::Duration,
+};
+
+use crate::feed_message_de::FeedMessage;
+use common::ws_client;
+use futures::{Sink, SinkExt, Stream, StreamExt};
+
+/// Wrap a `ws_client::Sender` with convenient utility methods for shard connections
+pub struct ShardSender(ws_client::Sender);
+
+impl From<ws_client::Sender> for ShardSender {
+    fn from(c: ws_client::Sender) -> Self {
+        ShardSender(c)
+    }
+}
+
+impl Sink<ws_client::SentMessage> for ShardSender {
+    type Error = ws_client::SendError;
+    fn poll_ready(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_ready_unpin(cx)
+    }
+    fn start_send(
+        mut self: std::pin::Pin<&mut Self>,
+        item: ws_client::SentMessage,
+    ) -> Result<(), Self::Error> {
+        self.0.start_send_unpin(item)
+    }
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_flush_unpin(cx)
+    }
+    fn poll_close(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_close_unpin(cx)
+    }
+}
+
+impl ShardSender {
+    /// Send JSON as a binary websocket message
+    pub fn send_json_binary(
+        &mut self,
+        json: serde_json::Value,
+    ) -> Result<(), ws_client::SendError> {
+        let bytes = serde_json::to_vec(&json).expect("valid bytes");
+        self.unbounded_send(ws_client::SentMessage::Binary(bytes))
+    }
+    /// Send JSON as a textual websocket message
+    pub fn send_json_text(&mut self, json: serde_json::Value) -> Result<(), ws_client::SendError> {
+        let s = serde_json::to_string(&json).expect("valid string");
+        self.unbounded_send(ws_client::SentMessage::Text(s))
+    }
+}
+
+impl Deref for ShardSender {
+    type Target = ws_client::Sender;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for ShardSender {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+/// Wrap a `ws_client::Receiver` with convenient utility methods for shard connections
+pub struct ShardReceiver(ws_client::Receiver);
+
+impl From<ws_client::Receiver> for ShardReceiver {
+    fn from(c: ws_client::Receiver) -> Self {
+        ShardReceiver(c)
+    }
+}
+
+impl Stream for ShardReceiver {
+    type Item = Result<ws_client::RecvMessage, ws_client::RecvError>;
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        self.0.poll_next_unpin(cx)
+    }
+}
+
+impl Deref for ShardReceiver {
+    type Target = ws_client::Receiver;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for ShardReceiver {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+/// Wrap a `ws_client::Sender` with convenient utility methods for feed connections
+pub struct FeedSender(ws_client::Sender);
+
+impl From<ws_client::Sender> for FeedSender {
+    fn from(c: ws_client::Sender) -> Self {
+        FeedSender(c)
+    }
+}
+
+impl Sink<ws_client::SentMessage> for FeedSender {
+    type Error = ws_client::SendError;
+    fn poll_ready(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_ready_unpin(cx)
+    }
+    fn start_send(
+        mut self: std::pin::Pin<&mut Self>,
+        item: ws_client::SentMessage,
+    ) -> Result<(), Self::Error> {
+        self.0.start_send_unpin(item)
+    }
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_flush_unpin(cx)
+    }
+    fn poll_close(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.0.poll_close_unpin(cx)
+    }
+}
+
+impl Deref for FeedSender {
+    type Target = ws_client::Sender;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for FeedSender {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl FeedSender {
+    /// Send a command into the feed. A command consists of a string
+    /// "command" part, and another string "parameter" part.
+    pub fn send_command<S: AsRef<str>>(
+        &self,
+        command: S,
+        param: S,
+    ) -> Result<(), ws_client::SendError> {
+        self.unbounded_send(ws_client::SentMessage::Text(format!(
+            "{}:{}",
+            command.as_ref(),
+            param.as_ref()
+        )))
+    }
+}
+
+/// Wrap a `ws_client::Receiver` with convenient utility methods for feed connections
+pub struct FeedReceiver(ws_client::Receiver);
+
+impl From<ws_client::Receiver> for FeedReceiver {
+    fn from(c: ws_client::Receiver) -> Self {
+        FeedReceiver(c)
+    }
+}
+
+impl Stream for FeedReceiver {
+    type Item = Result<ws_client::RecvMessage, ws_client::RecvError>;
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        self.0.poll_next_unpin(cx).map_err(|e| e.into())
+    }
+}
+
+impl Deref for FeedReceiver {
+    type Target = ws_client::Receiver;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for FeedReceiver {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl FeedReceiver {
+    /// Wait for the next set of feed messages to arrive. Returns an error if the connection
+    /// is closed, or the messages that come back cannot be properly decoded.
+    ///
+    /// Prefer [`FeedReceiver::recv_feed_messages`]; tests should generally be
+    /// robust in assuming that messages may not all be delivered at once (unless we are
+    /// specifically testing which messages are buffered together).
+    pub async fn recv_feed_messages_once_timeout(
+        &mut self,
+        timeout: Duration,
+    ) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        let msg = match tokio::time::timeout(timeout, self.0.next()).await {
+            // Timeout elapsed; no messages back:
+            Err(_) => return Ok(Vec::new()),
+            // Something back; Complain if error no stream closed:
+            Ok(res) => res.ok_or_else(|| anyhow::anyhow!("Stream closed: no more messages"))??,
+        };
+
+        match msg {
+            ws_client::RecvMessage::Binary(data) => {
+                let messages = FeedMessage::from_bytes(&data)?;
+                Ok(messages)
+            }
+            ws_client::RecvMessage::Text(text) => {
+                let messages = FeedMessage::from_bytes(text.as_bytes())?;
+                Ok(messages)
+            }
+        }
+    }
+
+    /// Wait for the next set of feed messages to arrive.
+    /// See `recv_feed_messages_once_timeout`.
+    pub async fn recv_feed_messages_once(&mut self) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        // This will never practically end; use the `timeout` version explciitly if you want that.
+        self.recv_feed_messages_once_timeout(Duration::from_secs(u64::MAX))
+            .await
+    }
+
+    /// Wait for feed messages to be sent back, building up a list of output messages until
+    /// the channel goes quiet for a short while.
+    ///
+    /// If no new messages are received within the timeout given, bail with whatever we have so far.
+    /// This differs from `recv_feed_messages` and `recv_feed_messages_once`, which will block indefinitely
+    /// waiting for something to arrive
+    pub async fn recv_feed_messages_timeout(
+        &mut self,
+        timeout: Duration,
+    ) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        // Block as long as needed for messages to start coming in:
+        let mut feed_messages =
+            match tokio::time::timeout(timeout, self.recv_feed_messages_once()).await {
+                Ok(msgs) => msgs?,
+                Err(_) => return Ok(Vec::new()),
+            };
+
+        // Then, loop a little to make sure we catch any additional messages that are sent soon after:
+        loop {
+            match tokio::time::timeout(Duration::from_millis(250), self.recv_feed_messages_once())
+                .await
+            {
+                // Timeout elapsed; return the messages we have so far
+                Err(_) => {
+                    break Ok(feed_messages);
+                }
+                // Append messages that come back to our vec
+                Ok(Ok(mut msgs)) => {
+                    feed_messages.append(&mut msgs);
+                }
+                // Error came back receiving messages; return it
+                Ok(Err(e)) => break Err(e),
+            }
+        }
+    }
+
+    /// Wait for feed messages until nothing else arrives in a timely fashion.
+    /// See `recv_feed_messages_timeout`.
+    pub async fn recv_feed_messages(&mut self) -> Result<Vec<FeedMessage>, anyhow::Error> {
+        // This will never practically end; use the `timeout` version explciitly if you want that.
+        self.recv_feed_messages_timeout(Duration::from_secs(u64::MAX))
+            .await
+    }
+}

backend/test_utils/src/server/mod.rs

@@ -0,0 +1,21 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod server;
+mod utils;
+
+pub mod channels;
+pub use server::*;

backend/test_utils/src/server/server.rs

@@ -0,0 +1,511 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::{channels, utils};
+use common::ws_client;
+use common::{id_type, DenseMap};
+use std::ffi::OsString;
+use std::marker::PhantomData;
+use tokio::process::{self, Command as TokioCommand};
+
+id_type! {
+    /// The ID of a running process. Cannot be constructed externally.
+    pub struct ProcessId(usize);
+}
+
+pub enum StartOpts {
+    /// Start a single core process that is expected
+    /// to have both `/feed` and `/submit` endpoints
+    SingleProcess {
+        /// Command to run to start the process.
+        /// The `--listen` and `--log` arguments will be appended within and shouldn't be provided.
+        command: Command,
+        /// Log output from started processes to stderr?
+        log_output: bool,
+    },
+    /// Start a core process with a `/feed` andpoint as well as (optionally)
+    /// multiple shard processes with `/submit` endpoints.
+    ShardAndCore {
+        /// Command to run to start a shard.
+        /// The `--listen` and `--log` arguments will be appended within and shouldn't be provided.
+        shard_command: Command,
+        /// Command to run to start a telemetry core process.
+        /// The `--listen` and `--log` arguments will be appended within and shouldn't be provided.
+        core_command: Command,
+        /// Log output from started processes to stderr?
+        log_output: bool,
+    },
+    /// Connect to existing process(es).
+    ConnectToExisting {
+        /// Where are the processes that we can `/submit` things to?
+        /// Eg: `vec![127.0.0.1:12345, 127.0.0.1:9091]`
+        submit_hosts: Vec<String>,
+        /// Where is the process that we can subscribe to the `/feed` of?
+        /// Eg: `127.0.0.1:3000`
+        feed_host: String,
+        /// Log output from started processes to stderr?
+        log_output: bool,
+    },
+}
+
+/// This represents a telemetry server. It can be in different modes
+/// depending on how it was started, but the interface is similar in every case
+/// so that tests are somewhat compatible with multiple configurations.
+pub struct Server {
+    /// Should we log output from the processes we start?
+    log_output: bool,
+    /// Core process that we can connect to.
+    core: CoreProcess,
+    /// Things that vary based on the mode we are in.
+    mode: ServerMode,
+}
+pub enum ServerMode {
+    SingleProcessMode {
+        /// A virtual shard that we can hand out.
+        virtual_shard: ShardProcess,
+    },
+    ShardAndCoreMode {
+        /// Command to run to start a new shard.
+        shard_command: Command,
+        /// Shard processes that we can connect to.
+        shards: DenseMap<ProcessId, ShardProcess>,
+    },
+    ConnectToExistingMode {
+        /// The hosts that we can connect to to submit things.
+        submit_hosts: Vec<String>,
+        /// Which host do we use next (we'll cycle around them
+        /// as shards are "added").
+        next_submit_host_idx: usize,
+        /// Shard processes that we can connect to.
+        shards: DenseMap<ProcessId, ShardProcess>,
+    },
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Can't establsih connection: {0}")]
+    ConnectionError(#[from] ws_client::ConnectError),
+    #[error("Can't establsih connection: {0}")]
+    JoinError(#[from] tokio::task::JoinError),
+    #[error("Can't establsih connection: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("Could not obtain port for process as the line we waited for in log output didn't show up: {0}")]
+    ErrorObtainingPort(anyhow::Error),
+    #[error("Whoops; attempt to kill a process we didn't start (and so have no handle to)")]
+    CannotKillNoHandle,
+    #[error(
+        "Can't add a shard: command not provided, or we are not in charge of spawning processes"
+    )]
+    CannotAddShard,
+    #[error("The URI provided was invalid: {0}")]
+    InvalidUri(#[from] http::uri::InvalidUri),
+}
+
+impl Server {
+    pub fn get_core(&self) -> &CoreProcess {
+        &self.core
+    }
+
+    pub fn get_shard(&self, id: ProcessId) -> Option<&ShardProcess> {
+        match &self.mode {
+            ServerMode::SingleProcessMode { virtual_shard, .. } => Some(virtual_shard),
+            ServerMode::ShardAndCoreMode { shards, .. } => shards.get(id),
+            ServerMode::ConnectToExistingMode { shards, .. } => shards.get(id),
+        }
+    }
+
+    pub async fn kill_shard(&mut self, id: ProcessId) -> bool {
+        let shard = match &mut self.mode {
+            // Can't remove the pretend shard:
+            ServerMode::SingleProcessMode { .. } => return false,
+            ServerMode::ShardAndCoreMode { shards, .. } => shards.remove(id),
+            ServerMode::ConnectToExistingMode { shards, .. } => shards.remove(id),
+        };
+
+        let shard = match shard {
+            Some(shard) => shard,
+            None => return false,
+        };
+
+        // With this, killing will complete even if the promise returned is cancelled
+        // (it should regardless, but just to play it safe..)
+        let _ = tokio::spawn(async move {
+            let _ = shard.kill().await;
+        })
+        .await;
+
+        true
+    }
+
+    /// Kill everything and tidy up
+    pub async fn shutdown(self) {
+        // Spawn so we don't need to await cleanup if we don't care.
+        // Run all kill futs simultaneously.
+        let handle = tokio::spawn(async move {
+            let core = self.core;
+            let shards = match self.mode {
+                ServerMode::SingleProcessMode { .. } => DenseMap::new(),
+                ServerMode::ShardAndCoreMode { shards, .. } => shards,
+                ServerMode::ConnectToExistingMode { shards, .. } => shards,
+            };
+
+            let shard_kill_futs = shards.into_iter().map(|(_, s)| s.kill());
+            let _ = tokio::join!(futures::future::join_all(shard_kill_futs), core.kill());
+        });
+
+        // You can wait for cleanup but aren't obliged to:
+        let _ = handle.await;
+    }
+
+    /// Connect a new shard and return a process that you can interact with:
+    pub async fn add_shard(&mut self) -> Result<ProcessId, Error> {
+        match &mut self.mode {
+            // Always get back the same "virtual" shard; we're always just talking to the core anyway.
+            ServerMode::SingleProcessMode { virtual_shard, .. } => Ok(virtual_shard.id),
+            // We're connecting to an existing process. Find the next host we've been told about
+            // round-robin style and use that as our new virtual shard.
+            ServerMode::ConnectToExistingMode {
+                submit_hosts,
+                next_submit_host_idx,
+                shards,
+                ..
+            } => {
+                let host = match submit_hosts.get(*next_submit_host_idx % submit_hosts.len()) {
+                    Some(host) => host,
+                    None => return Err(Error::CannotAddShard),
+                };
+                *next_submit_host_idx += 1;
+
+                let pid = shards.add_with(|id| Process {
+                    id,
+                    host: format!("{}", host),
+                    handle: None,
+                    _channel_type: PhantomData,
+                });
+
+                Ok(pid)
+            }
+            // Start a new process and return that.
+            ServerMode::ShardAndCoreMode {
+                shard_command,
+                shards,
+            } => {
+                // Where is the URI we'll want to submit things to?
+                let core_shard_submit_uri = format!("http://{}/shard_submit", self.core.host);
+
+                let mut shard_cmd: TokioCommand = shard_command.clone().into();
+                shard_cmd
+                    .arg("--listen")
+                    .arg("127.0.0.1:0") // 0 to have a port picked by the kernel
+                    .arg("--log")
+                    .arg("info")
+                    .arg("--core")
+                    .arg(core_shard_submit_uri)
+                    .kill_on_drop(true)
+                    .stdout(std::process::Stdio::piped())
+                    .stdin(std::process::Stdio::piped());
+
+                let mut shard_process = shard_cmd.spawn()?;
+                let mut child_stdout = shard_process.stdout.take().expect("shard stdout");
+                let shard_port = utils::get_port(&mut child_stdout)
+                    .await
+                    .map_err(|e| Error::ErrorObtainingPort(e))?;
+
+                // Attempt to wait until we've received word that the shard is connected to the
+                // core before continuing. If we don't wait for this, the connection may happen
+                // after we've attempted to connect node sockets, and they would be booted and
+                // made to reconnect, which we don't want to deal with in general.
+                let _ = utils::wait_for_line_containing(
+                    &mut child_stdout,
+                    |s| s.contains("Connected to telemetry core"),
+                    std::time::Duration::from_secs(5),
+                )
+                .await;
+
+                // Since we're piping stdout from the child process, we need somewhere for it to go
+                // else the process will get stuck when it tries to produce output:
+                if self.log_output {
+                    utils::drain(child_stdout, tokio::io::stderr());
+                } else {
+                    utils::drain(child_stdout, tokio::io::sink());
+                }
+
+                let pid = shards.add_with(|id| Process {
+                    id,
+                    host: format!("127.0.0.1:{}", shard_port),
+                    handle: Some(shard_process),
+                    _channel_type: PhantomData,
+                });
+
+                Ok(pid)
+            }
+        }
+    }
+
+    /// Start a server.
+    pub async fn start(opts: StartOpts) -> Result<Server, Error> {
+        let server = match opts {
+            StartOpts::SingleProcess {
+                command,
+                log_output,
+            } => {
+                let core_process = Server::start_core(log_output, command).await?;
+                let virtual_shard_host = core_process.host.clone();
+                Server {
+                    log_output,
+                    core: core_process,
+                    mode: ServerMode::SingleProcessMode {
+                        virtual_shard: Process {
+                            id: ProcessId(0),
+                            host: virtual_shard_host,
+                            handle: None,
+                            _channel_type: PhantomData,
+                        },
+                    },
+                }
+            }
+            StartOpts::ShardAndCore {
+                core_command,
+                shard_command,
+                log_output,
+            } => {
+                let core_process = Server::start_core(log_output, core_command).await?;
+                Server {
+                    log_output,
+                    core: core_process,
+                    mode: ServerMode::ShardAndCoreMode {
+                        shard_command,
+                        shards: DenseMap::new(),
+                    },
+                }
+            }
+            StartOpts::ConnectToExisting {
+                feed_host,
+                submit_hosts,
+                log_output,
+            } => Server {
+                log_output,
+                core: Process {
+                    id: ProcessId(0),
+                    host: feed_host,
+                    handle: None,
+                    _channel_type: PhantomData,
+                },
+                mode: ServerMode::ConnectToExistingMode {
+                    submit_hosts,
+                    next_submit_host_idx: 0,
+                    shards: DenseMap::new(),
+                },
+            },
+        };
+
+        Ok(server)
+    }
+
+    /// Start up a core process and return it.
+    async fn start_core(log_output: bool, command: Command) -> Result<CoreProcess, Error> {
+        let mut tokio_core_cmd: TokioCommand = command.into();
+        let mut child = tokio_core_cmd
+            .arg("--listen")
+            .arg("127.0.0.1:0") // 0 to have a port picked by the kernel
+            .arg("--log")
+            .arg("info")
+            .kill_on_drop(true)
+            .stdout(std::process::Stdio::piped())
+            .stdin(std::process::Stdio::piped())
+            .spawn()?;
+
+        // Find out the port that this is running on
+        let mut child_stdout = child.stdout.take().expect("core stdout");
+        let core_port = utils::get_port(&mut child_stdout)
+            .await
+            .map_err(|e| Error::ErrorObtainingPort(e))?;
+
+        // Since we're piping stdout from the child process, we need somewhere for it to go
+        // else the process will get stuck when it tries to produce output:
+        if log_output {
+            utils::drain(child_stdout, tokio::io::stderr());
+        } else {
+            utils::drain(child_stdout, tokio::io::sink());
+        }
+
+        let core_process = Process {
+            id: ProcessId(0),
+            host: format!("127.0.0.1:{}", core_port),
+            handle: Some(child),
+            _channel_type: PhantomData,
+        };
+
+        Ok(core_process)
+    }
+}
+
+/// This represents a running process that we can connect to, which
+/// may be either a `telemetry_shard` or `telemetry_core`.
+pub struct Process<Channel> {
+    id: ProcessId,
+    /// Host that the process is running on (eg 127.0.0.1:8080).
+    host: String,
+    /// If we started the processes ourselves, we'll have a handle to
+    /// them which we can use to kill them. Else, we may not.
+    handle: Option<process::Child>,
+    /// The kind of the process (lets us add methods specific to shard/core).
+    _channel_type: PhantomData<Channel>,
+}
+
+/// A shard process with shard-specific methods.
+pub type ShardProcess = Process<(channels::ShardSender, channels::ShardReceiver)>;
+
+/// A core process with core-specific methods.
+pub type CoreProcess = Process<(channels::FeedSender, channels::FeedReceiver)>;
+
+impl<Channel> Process<Channel> {
+    /// Get the ID of this process
+    pub fn id(&self) -> ProcessId {
+        self.id
+    }
+
+    /// Get the host that this process is running on
+    pub fn host(&self) -> &str {
+        &self.host
+    }
+
+    /// Kill the process and wait for this to complete
+    /// Not public: Klling done via Server.
+    async fn kill(self) -> Result<(), Error> {
+        match self.handle {
+            Some(mut handle) => Ok(handle.kill().await?),
+            None => Err(Error::CannotKillNoHandle),
+        }
+    }
+}
+
+/// Establish a raw WebSocket connection (not cancel-safe)
+async fn connect_to_uri_raw(
+    uri: &http::Uri,
+) -> Result<(ws_client::RawSender, ws_client::RawReceiver), Error> {
+    ws_client::connect(uri)
+        .await
+        .map(|c| c.into_raw())
+        .map_err(|e| e.into())
+}
+
+impl<Send: From<ws_client::Sender>, Recv: From<ws_client::Receiver>> Process<(Send, Recv)> {
+    /// Establish a connection to the process
+    async fn connect_to_uri(uri: &http::Uri) -> Result<(Send, Recv), Error> {
+        ws_client::connect(uri)
+            .await
+            .map(|c| c.into_channels())
+            .map(|(s, r)| (s.into(), r.into()))
+            .map_err(|e| e.into())
+    }
+
+    /// Establish multiple connections to the process
+    async fn connect_multiple_to_uri(
+        uri: &http::Uri,
+        num_connections: usize,
+    ) -> Result<Vec<(Send, Recv)>, Error> {
+        utils::connect_multiple_to_uri(uri, num_connections)
+            .await
+            .map(|v| v.into_iter().map(|(s, r)| (s.into(), r.into())).collect())
+            .map_err(|e| e.into())
+    }
+}
+
+impl ShardProcess {
+    /// Establish a raw connection to the process
+    pub async fn connect_node_raw(
+        &self,
+    ) -> Result<(ws_client::RawSender, ws_client::RawReceiver), Error> {
+        let uri = format!("http://{}/submit", self.host).parse()?;
+        connect_to_uri_raw(&uri).await
+    }
+
+    /// Establish a connection to the process
+    pub async fn connect_node(
+        &self,
+    ) -> Result<(channels::ShardSender, channels::ShardReceiver), Error> {
+        let uri = format!("http://{}/submit", self.host).parse()?;
+        Process::connect_to_uri(&uri).await
+    }
+
+    /// Establish multiple connections to the process
+    pub async fn connect_multiple_nodes(
+        &self,
+        num_connections: usize,
+    ) -> Result<Vec<(channels::ShardSender, channels::ShardReceiver)>, Error> {
+        let uri = format!("http://{}/submit", self.host).parse()?;
+        Process::connect_multiple_to_uri(&uri, num_connections).await
+    }
+}
+
+impl CoreProcess {
+    /// Establish a raw connection to the process
+    pub async fn connect_feed_raw(
+        &self,
+    ) -> Result<(ws_client::RawSender, ws_client::RawReceiver), Error> {
+        let uri = format!("http://{}/feed", self.host).parse()?;
+        connect_to_uri_raw(&uri).await
+    }
+
+    /// Establish a connection to the process
+    pub async fn connect_feed(
+        &self,
+    ) -> Result<(channels::FeedSender, channels::FeedReceiver), Error> {
+        let uri = format!("http://{}/feed", self.host).parse()?;
+        Process::connect_to_uri(&uri).await
+    }
+
+    /// Establish multiple connections to the process
+    pub async fn connect_multiple_feeds(
+        &self,
+        num_connections: usize,
+    ) -> Result<Vec<(channels::FeedSender, channels::FeedReceiver)>, Error> {
+        let uri = format!("http://{}/feed", self.host).parse()?;
+        Process::connect_multiple_to_uri(&uri, num_connections).await
+    }
+}
+
+/// This defines a command to run. This exists because [`tokio::process::Command`]
+/// cannot be cloned, but we need to be able to clone our command to spawn multiple
+/// processes with it.
+#[derive(Clone, Debug)]
+pub struct Command {
+    command: OsString,
+    args: Vec<OsString>,
+}
+
+impl Command {
+    pub fn new<S: Into<OsString>>(command: S) -> Command {
+        Command {
+            command: command.into(),
+            args: Vec::new(),
+        }
+    }
+
+    pub fn arg<S: Into<OsString>>(mut self, arg: S) -> Command {
+        self.args.push(arg.into());
+        self
+    }
+}
+
+impl Into<TokioCommand> for Command {
+    fn into(self) -> TokioCommand {
+        let mut cmd = TokioCommand::new(self.command);
+        cmd.args(self.args);
+        cmd
+    }
+}

backend/test_utils/src/server/utils.rs

@@ -0,0 +1,109 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::{anyhow, Context};
+use common::ws_client;
+use tokio::io::BufReader;
+use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite};
+use tokio::time::Duration;
+
+/// Reads from the stdout of the shard/core process to extract the port that was assigned to it,
+/// with the side benefit that we'll wait for it to start listening before returning. We do this
+/// because we want to allow the kernel to assign ports and so don't specify a port as an arg.
+pub async fn get_port<R: AsyncRead + Unpin>(reader: R) -> Result<u16, anyhow::Error> {
+    // For the new service:
+    let new_expected_text = "listening on http://127.0.0.1:";
+    // For the older non-sharded actix based service:
+    let old_expected_text = "service on 127.0.0.1:";
+
+    let is_text = |s: &str| s.contains(new_expected_text) || s.contains(old_expected_text);
+    wait_for_line_containing(reader, is_text, Duration::from_secs(240))
+        .await
+        .and_then(|line| {
+            // The line must match one of our expected strings:
+            let (_, port_str) = line
+                .rsplit_once(new_expected_text)
+                .unwrap_or_else(|| line.rsplit_once(old_expected_text).unwrap());
+            // Grab the port after the string:
+            port_str
+                .trim()
+                .parse()
+                .with_context(|| format!("Could not parse output to port: {}", port_str))
+        })
+}
+
+/// Wait for a line of output containing the text given. Also provide a timeout,
+/// such that if we don't see a new line of output within the timeout we bail out
+/// and return an error.
+pub async fn wait_for_line_containing<R: AsyncRead + Unpin, F: Fn(&str) -> bool>(
+    reader: R,
+    is_match: F,
+    max_wait_between_lines: Duration,
+) -> Result<String, anyhow::Error> {
+    let reader = BufReader::new(reader);
+    let mut reader_lines = reader.lines();
+
+    loop {
+        let line = tokio::time::timeout(max_wait_between_lines, reader_lines.next_line()).await;
+
+        let line = match line {
+            // timeout expired; couldn't get port:
+            Err(_) => return Err(anyhow!("Timeout elapsed waiting for text match")),
+            // Something went wrong reading line; bail:
+            Ok(Err(e)) => return Err(anyhow!("Could not read line from stdout: {}", e)),
+            // No more output; process ended? bail:
+            Ok(Ok(None)) => {
+                return Err(anyhow!(
+                    "No more output from stdout; has the process ended?"
+                ))
+            }
+            // All OK, and a line is given back; phew!
+            Ok(Ok(Some(line))) => line,
+        };
+
+        if is_match(&line) {
+            return Ok(line);
+        }
+    }
+}
+
+/// Establish multiple connections to a URI and return them all.
+pub async fn connect_multiple_to_uri(
+    uri: &http::Uri,
+    num_connections: usize,
+) -> Result<Vec<(ws_client::Sender, ws_client::Receiver)>, ws_client::ConnectError> {
+    // Previous versions of this used future::join_all to concurrently establish all of the
+    // connections we want. However, trying to do that with more than say ~130 connections on
+    // MacOS led to hitting "Connection reset by peer" errors, so let's do it one-at-a-time.
+    // (Side note: on Ubuntu the concurrency seemed to be no issue up to at least 10k connections).
+    let mut sockets = vec![];
+    for _ in 0..num_connections {
+        sockets.push(ws_client::connect(uri).await?.into_channels());
+    }
+    Ok(sockets)
+}
+
+/// Drain output from a reader to stdout. After acquiring port details from spawned processes,
+/// they expect their stdout to be continue to be consumed, and so we do this here.
+pub fn drain<R, W>(mut reader: R, mut writer: W)
+where
+    R: AsyncRead + Unpin + Send + 'static,
+    W: AsyncWrite + Unpin + Send + 'static,
+{
+    tokio::spawn(async move {
+        let _ = tokio::io::copy(&mut reader, &mut writer).await;
+    });
+}

backend/test_utils/src/workspace/commands.rs

@@ -0,0 +1,63 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Commands that we can use when running `cargo test` style tests in this workspace
+//! that want to test the current code.
+use crate::server::Command;
+use std::path::PathBuf;
+
+/// Runs `cargo run` in the current workspace to start up a telemetry shard process.
+///
+/// Note: The CWD must be somewhere within this backend workspace for the command to work.
+pub fn cargo_run_telemetry_shard(release_mode: bool) -> Result<Command, std::io::Error> {
+    telemetry_command("telemetry_shard", release_mode)
+}
+
+/// Runs `cargo run` in the current workspace to start up a telemetry core process.
+///
+/// Note: The CWD must be somewhere within this backend workspace for the command to work.
+pub fn cargo_run_telemetry_core(release_mode: bool) -> Result<Command, std::io::Error> {
+    telemetry_command("telemetry_core", release_mode)
+}
+
+fn telemetry_command(bin: &'static str, release_mode: bool) -> Result<Command, std::io::Error> {
+    let mut workspace_dir = try_find_workspace_dir()?;
+    workspace_dir.push("Cargo.toml");
+
+    let mut cmd = Command::new("cargo").arg("run");
+
+    // Release mode?
+    if release_mode {
+        cmd = cmd.arg("--release");
+    }
+
+    cmd = cmd
+        .arg("--bin")
+        .arg(bin)
+        .arg("--manifest-path")
+        .arg(workspace_dir)
+        .arg("--");
+
+    Ok(cmd)
+}
+
+/// A _very_ naive way to find the workspace ("backend") directory
+/// from the current path (which is assumed to be inside it).
+fn try_find_workspace_dir() -> Result<PathBuf, std::io::Error> {
+    let mut dir = std::env::current_dir()?;
+    while !dir.ends_with("backend") && dir.pop() {}
+    Ok(dir)
+}

backend/test_utils/src/workspace/mod.rs

@@ -0,0 +1,20 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod commands;
+mod start_server;
+
+pub use start_server::*;

backend/test_utils/src/workspace/start_server.rs

@@ -0,0 +1,193 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::commands;
+use crate::server::{self, Command, Server};
+
+/// Options for the server
+pub struct ServerOpts {
+    pub release_mode: bool,
+    pub log_output: bool,
+}
+
+impl Default for ServerOpts {
+    fn default() -> Self {
+        Self {
+            release_mode: false,
+            log_output: false,
+        }
+    }
+}
+
+/// Additional options to pass to the core command.
+pub struct CoreOpts {
+    pub feed_timeout: Option<u64>,
+    pub worker_threads: Option<usize>,
+    pub num_aggregators: Option<usize>,
+}
+
+impl Default for CoreOpts {
+    fn default() -> Self {
+        Self {
+            feed_timeout: None,
+            worker_threads: None,
+            num_aggregators: None,
+        }
+    }
+}
+
+/// Additional options to pass to the shard command.
+pub struct ShardOpts {
+    pub max_nodes_per_connection: Option<usize>,
+    pub max_node_data_per_second: Option<usize>,
+    pub node_block_seconds: Option<u64>,
+    pub worker_threads: Option<usize>,
+}
+
+impl Default for ShardOpts {
+    fn default() -> Self {
+        Self {
+            max_nodes_per_connection: None,
+            max_node_data_per_second: None,
+            node_block_seconds: None,
+            worker_threads: None,
+        }
+    }
+}
+
+/// Start a telemetry server. We'll use `cargo run` by default, but you can also provide
+/// env vars to configure the binary that runs for the shard and core process. Either:
+///
+/// - `TELEMETRY_BIN` - path to the telemetry binary (which can function as shard _and_ core)
+///
+/// Or alternately neither/one/both of:
+///
+/// - `TELEMETRY_SHARD_BIN` - path to telemetry_shard binary
+/// - `TELEMETRY_CORE_BIN` - path to telemetry_core binary
+///
+/// (Whatever is not provided will be substituted with a `cargo run` variant instead)
+///
+/// Or alternately alternately, we can connect to a running instance by providing:
+///
+/// - `TELEMETRY_SUBMIT_HOSTS` - hosts (comma separated) to connect to for telemetry `/submit`s.
+/// - `TELEMETRY_FEED_HOST` - host to connect to for feeds (eg 127.0.0.1:3000)
+///
+pub async fn start_server(
+    server_opts: ServerOpts,
+    core_opts: CoreOpts,
+    shard_opts: ShardOpts,
+) -> Server {
+    // Start to a single process:
+    if let Ok(bin) = std::env::var("TELEMETRY_BIN") {
+        return Server::start(server::StartOpts::SingleProcess {
+            command: Command::new(bin),
+            log_output: server_opts.log_output,
+        })
+        .await
+        .unwrap();
+    }
+
+    // Connect to a running instance:
+    if let Ok(feed_host) = std::env::var("TELEMETRY_FEED_HOST") {
+        let feed_host = feed_host.trim().into();
+        let submit_hosts: Vec<_> = std::env::var("TELEMETRY_SUBMIT_HOSTS")
+            .map(|var| var.split(",").map(|var| var.trim().into()).collect())
+            .unwrap_or(Vec::new());
+        return Server::start(server::StartOpts::ConnectToExisting {
+            feed_host,
+            submit_hosts,
+            log_output: server_opts.log_output,
+        })
+        .await
+        .unwrap();
+    }
+
+    // Build the shard command
+    let mut shard_command = std::env::var("TELEMETRY_SHARD_BIN")
+        .map(|val| Command::new(val))
+        .unwrap_or_else(|_| {
+            commands::cargo_run_telemetry_shard(server_opts.release_mode)
+                .expect("must be in rust workspace to run shard command")
+        });
+
+    // Append additional opts to the shard command
+    if let Some(val) = shard_opts.max_nodes_per_connection {
+        shard_command = shard_command
+            .arg("--max-nodes-per-connection")
+            .arg(val.to_string());
+    }
+    if let Some(val) = shard_opts.max_node_data_per_second {
+        shard_command = shard_command
+            .arg("--max-node-data-per-second")
+            .arg(val.to_string());
+    }
+    if let Some(val) = shard_opts.node_block_seconds {
+        shard_command = shard_command
+            .arg("--node-block-seconds")
+            .arg(val.to_string());
+    }
+    if let Some(val) = shard_opts.worker_threads {
+        shard_command = shard_command.arg("--worker-threads").arg(val.to_string());
+    }
+
+    // Build the core command
+    let mut core_command = std::env::var("TELEMETRY_CORE_BIN")
+        .map(|val| Command::new(val))
+        .unwrap_or_else(|_| {
+            commands::cargo_run_telemetry_core(server_opts.release_mode)
+                .expect("must be in rust workspace to run core command")
+        });
+
+    // Append additional opts to the core command
+    if let Some(val) = core_opts.feed_timeout {
+        core_command = core_command.arg("--feed-timeout").arg(val.to_string());
+    }
+    if let Some(val) = core_opts.worker_threads {
+        core_command = core_command.arg("--worker-threads").arg(val.to_string());
+    }
+    if let Some(val) = core_opts.num_aggregators {
+        core_command = core_command.arg("--num-aggregators").arg(val.to_string());
+    }
+
+    // Start the server
+    Server::start(server::StartOpts::ShardAndCore {
+        shard_command,
+        core_command,
+        log_output: server_opts.log_output,
+    })
+    .await
+    .unwrap()
+}
+
+/// Start a telemetry core server in debug mode. see [`start_server`] for details.
+pub async fn start_server_debug() -> Server {
+    start_server(
+        ServerOpts::default(),
+        CoreOpts::default(),
+        ShardOpts::default(),
+    )
+    .await
+}
+
+/// Start a telemetry core server in release mode. see [`start_server`] for details.
+pub async fn start_server_release() -> Server {
+    start_server(
+        ServerOpts::default(),
+        CoreOpts::default(),
+        ShardOpts::default(),
+    )
+    .await
+}

docker-compose.yml

@@ -9,14 +9,34 @@ services:
     volumes:
       - /app/node_modules
       - ./packages:/app/packages
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx/default:/etc/nginx/sites-available/default
+      - ./env-config.js:/usr/share/nginx/html/env-config.js
     ports:
       - 3000:80
-  telemetry-backend:
+    expose:
+      - 3000
+  telemetry-backend-shard:
     build:
       dockerfile: Dockerfile
       context: ./backend/
-    environment:
-      - PORT=8000
+    command: [
+      'telemetry_shard',
+      '--listen', '0.0.0.0:8001',
+      '--core', 'http://telemetry-backend-core:8000/shard_submit'
+    ]
+    ports:
+      - 8001:8001
+    expose:
+      - 8001
+  telemetry-backend-core:
+    build:
+      dockerfile: Dockerfile
+      context: ./backend/
+    command: [
+      'telemetry_core',
+      '--listen', '0.0.0.0:8000'
+    ]
     ports:
       - 8000:8000
     expose:

frontend/Dockerfile

@@ -1,5 +1,5 @@
 #### BUILDER IMAGE  ####
-FROM node:12 as builder
+FROM docker.io/node:12 as builder
 LABEL maintainer="Chevdor <chevdor@gmail.com>"
 LABEL description="Polkadot Telemetry frontend builder image"
 
@@ -11,7 +11,7 @@ RUN yarn install && \
     yarn cache clean
 
 #### OUTPUT IMAGE ####
-FROM nginx:stable-alpine
+FROM docker.io/nginx:stable-alpine
 LABEL maintainer="Chevdor <chevdor@gmail.com>"
 LABEL description="Polkadot Telemetry frontend"
 
@@ -19,13 +19,14 @@ ENV SUBSTRATE_TELEMETRY_URL=
 
 WORKDIR /usr/share/nginx/html
 
-COPY --from=builder /opt/builder/env.sh /usr/bin/
-RUN apk add --no-cache bash; chmod +x /usr/bin/env.sh
+#COPY --from=builder /opt/builder/env.sh /usr/bin/
+#RUN apk add --no-cache bash; chmod +x /usr/bin/env.sh
 
-COPY --from=builder /opt/builder/nginx/nginx.conf /etc/nginx/nginx.conf
-COPY --from=builder /opt/builder/nginx/default /etc/nginx/sites-available/default
+#COPY --from=builder /opt/builder/nginx/nginx.conf /etc/nginx/nginx.conf
+#COPY --from=builder /opt/builder/nginx/default /etc/nginx/sites-available/default
 COPY --from=builder /opt/builder/build /usr/share/nginx/html
 
 EXPOSE 80
 
-CMD ["/bin/bash", "-c", "/usr/bin/env.sh && nginx -g \"daemon off;\""]
+#CMD ["/bin/bash", "-c", "/usr/bin/env.sh && nginx -g \"daemon off;\""]
+CMD [ "nginx", "-g" ,"daemon off;"]

frontend/declarations/index.d.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 declare module '@fnando/sparkline' {
   namespace sparkline {
     export interface Options {

frontend/images.d.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 declare module '*.svg'
 declare module '*.png'
 declare module '*.jpg'

frontend/src/AfgHandling.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import { Types } from './common';
 import { State, Update } from './state';
 import { ConsensusDetail } from './common/types';

frontend/src/App.css

@@ -1,3 +1,21 @@
+/*
+Source code for the Substrate Telemetry Server.
+Copyright (C) 2021 Parity Technologies (UK) Ltd.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
 .App {
   text-align: left;
   font-family: Roboto, Helvetica, Arial, sans-serif;

frontend/src/App.tsx

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import * as React from 'react';
 import { Types, SortedCollection, Maybe, Compare } from './common';
 import { AllChains, Chains, Chain, Ago, OfflineIndicator } from './components';

frontend/src/Connection.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import { VERSION, timestamp, FeedMessage, Types, Maybe, sleep } from './common';
 import { State, Update, Node, ChainData, PINNED_CHAINS } from './state';
 import { PersistentSet } from './persist';

frontend/src/common/SortedCollection.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import { Maybe, Opaque } from './helpers';
 
 export type Compare<T> = (a: T, b: T) => number;

frontend/src/common/feed.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import { Maybe } from './helpers';
 import { stringify, parse, Stringified } from './stringify';
 import {

frontend/src/common/helpers.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import { Milliseconds, Timestamp } from './types';
 
 /**

frontend/src/common/id.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import { Opaque } from './helpers';
 
 /**

frontend/src/common/index.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 export * from './helpers';
 export * from './id';
 export * from './stringify';

frontend/src/common/iterators.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 export function* map<T, U>(
   iter: IterableIterator<T>,
   fn: (item: T) => U

frontend/src/common/stringify.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 export abstract class Stringified<T> {
   public __PHANTOM__: T;
 }

frontend/src/common/types.ts

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import { Opaque, Maybe } from './helpers';
 import { Id } from './id';
 

frontend/src/components/Ago.tsx

@@ -1,3 +1,19 @@
+// Source code for the Substrate Telemetry Server.
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 import * as React from 'react';
 import './Tile.css';
 import { timestamp, Types } from '../common';

frontend/src/components/AllChains.css

@@ -1,3 +1,21 @@
+/*
+Source code for the Substrate Telemetry Server.
+Copyright (C) 2021 Parity Technologies (UK) Ltd.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
 .AllChains {
   position: fixed;
   z-index: 20;