rename shared to common to disambiguate from 'shard'

backend/common/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod node;
+pub mod shard;
+pub mod types;
+pub mod util;
+pub mod ws;

backend/common/src/node.rs

@@ -0,0 +1,218 @@
+use crate::types::{Block, BlockHash, BlockNumber, ConnId, NodeDetails};
+use crate::util::{Hash, NullAny};
+use actix::prelude::*;
+use serde::{Deserialize, Serialize};
+use serde::ser::Serializer;
+
+#[derive(Deserialize, Debug, Message)]
+#[rtype(result = "()")]
+#[serde(untagged)]
+pub enum NodeMessage {
+    V1 {
+        #[serde(flatten)]
+        payload: Payload,
+    },
+    V2 {
+        id: ConnId,
+        payload: Payload,
+    },
+}
+
+impl NodeMessage {
+    /// Returns the connection ID or 0 if there is no ID.
+    pub fn id(&self) -> ConnId {
+        match self {
+            NodeMessage::V1 { .. } => 0,
+            NodeMessage::V2 { id, .. } => *id,
+        }
+    }
+}
+
+impl From<NodeMessage> for Payload {
+    fn from(msg: NodeMessage) -> Payload {
+        match msg {
+            NodeMessage::V1 { payload, .. } | NodeMessage::V2 { payload, .. } => payload,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(tag = "msg")]
+pub enum Payload {
+    #[serde(rename = "system.connected")]
+    SystemConnected(SystemConnected),
+    #[serde(rename = "system.interval")]
+    SystemInterval(SystemInterval),
+    #[serde(rename = "block.import")]
+    BlockImport(Block),
+    #[serde(rename = "notify.finalized")]
+    NotifyFinalized(Finalized),
+    #[serde(rename = "txpool.import")]
+    TxPoolImport(NullAny),
+    // #[serde(rename = "afg.finalized")]
+    // AfgFinalized(AfgFinalized),
+    // #[serde(rename = "afg.received_precommit")]
+    // AfgReceivedPrecommit(AfgReceivedPrecommit),
+    // #[serde(rename = "afg.received_prevote")]
+    // AfgReceivedPrevote(AfgReceivedPrevote),
+    // #[serde(rename = "afg.received_commit")]
+    // AfgReceivedCommit(AfgReceivedCommit),
+    // #[serde(rename = "afg.authority_set")]
+    // AfgAuthoritySet(AfgAuthoritySet),
+    // #[serde(rename = "afg.finalized_blocks_up_to")]
+    // AfgFinalizedBlocksUpTo(NullAny),
+    // #[serde(rename = "aura.pre_sealed_block")]
+    // AuraPreSealedBlock(NullAny),
+    #[serde(rename = "prepared_block_for_proposing")]
+    PreparedBlockForProposing(NullAny),
+}
+
+impl Serialize for Payload {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        use Payload::*;
+
+        match self {
+            SystemConnected(val) => serializer.serialize_newtype_variant("Payload", 0, "system.connected", val),
+            SystemInterval(val) => serializer.serialize_newtype_variant("Payload", 1, "system.interval", val),
+            BlockImport(val) => serializer.serialize_newtype_variant("Payload", 3, "block.import", val),
+            NotifyFinalized(val) => serializer.serialize_newtype_variant("Payload", 4, "notify.finalized", val),
+            TxPoolImport(_) => serializer.serialize_unit_variant("Payload", 3, "txpool.import"),
+            PreparedBlockForProposing(_) => serializer.serialize_unit_variant("Payload", 4, "prepared_block_for_proposing"),
+            _ => unimplemented!()
+        }
+    }
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct SystemConnected {
+    pub genesis_hash: Hash,
+    #[serde(flatten)]
+    pub node: NodeDetails,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct SystemInterval {
+    pub peers: Option<u64>,
+    pub txcount: Option<u64>,
+    pub bandwidth_upload: Option<f64>,
+    pub bandwidth_download: Option<f64>,
+    pub finalized_height: Option<BlockNumber>,
+    pub finalized_hash: Option<BlockHash>,
+    #[serde(flatten)]
+    pub block: Option<Block>,
+    pub used_state_cache_size: Option<f32>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Finalized {
+    #[serde(rename = "best")]
+    pub hash: BlockHash,
+    pub height: Box<str>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct AfgAuthoritySet {
+    pub authority_id: Box<str>,
+    pub authorities: Box<str>,
+    pub authority_set_id: Box<str>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct AfgFinalized {
+    pub finalized_hash: BlockHash,
+    pub finalized_number: Box<str>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct AfgReceived {
+    pub target_hash: BlockHash,
+    pub target_number: Box<str>,
+    pub voter: Option<Box<str>>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct AfgReceivedPrecommit {
+    #[serde(flatten)]
+    pub received: AfgReceived,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct AfgReceivedPrevote {
+    #[serde(flatten)]
+    pub received: AfgReceived,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct AfgReceivedCommit {
+    #[serde(flatten)]
+    pub received: AfgReceived,
+}
+
+impl Payload {
+    pub fn best_block(&self) -> Option<&Block> {
+        match self {
+            Payload::BlockImport(block) => Some(block),
+            Payload::SystemInterval(SystemInterval { block, .. }) => block.as_ref(),
+            _ => None,
+        }
+    }
+
+    pub fn finalized_block(&self) -> Option<Block> {
+        match self {
+            Payload::SystemInterval(ref interval) => Some(Block {
+                hash: interval.finalized_hash?,
+                height: interval.finalized_height?,
+            }),
+            Payload::NotifyFinalized(ref finalized) => Some(Block {
+                hash: finalized.hash,
+                height: finalized.height.parse().ok()?,
+            }),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bincode::Options;
+
+    #[test]
+    fn message_v1() {
+        let json = r#"{"msg":"notify.finalized","level":"INFO","ts":"2021-01-13T12:38:25.410794650+01:00","best":"0x031c3521ca2f9c673812d692fc330b9a18e18a2781e3f9976992f861fd3ea0cb","height":"50"}"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V1 { .. },
+            ),
+            "message did not match variant V1",
+        );
+    }
+
+    #[test]
+    fn message_v2() {
+        let json = r#"{"id":1,"ts":"2021-01-13T12:22:20.053527101+01:00","payload":{"best":"0xcc41708573f2acaded9dd75e07dac2d4163d136ca35b3061c558d7a35a09dd8d","height":"209","msg":"notify.finalized"}}"#;
+        assert!(
+            matches!(
+                serde_json::from_str::<NodeMessage>(json).unwrap(),
+                NodeMessage::V2 { .. },
+            ),
+            "message did not match variant V2",
+        );
+    }
+
+    #[test]
+    fn bincode_block_zero() {
+        let raw = Block::zero();
+
+        let bytes = bincode::options().serialize(&raw).unwrap();
+
+        let deserialized: Block = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(raw.hash, deserialized.hash);
+        assert_eq!(raw.height, deserialized.height);
+    }
+}

backend/common/src/shard.rs

@@ -0,0 +1,38 @@
+use std::net::Ipv4Addr;
+
+use crate::ws::MuteReason;
+use crate::node::Payload;
+use crate::types::{NodeId, NodeDetails};
+use serde::{Deserialize, Serialize};
+
+/// Alias for the ID of the node connection
+pub type ShardConnId = u32;
+
+/// Message sent from the shard to the backend core
+#[derive(Deserialize, Serialize, Debug)]
+pub enum ShardMessage {
+    /// Get a connection id for a new node, passing IPv4
+    AddNode {
+    	ip: Option<Ipv4Addr>,
+    	node: NodeDetails,
+    	sid: ShardConnId,
+    },
+    /// Send a message payload for a given node
+    UpdateNode {
+        nid: NodeId,
+        payload: Payload,
+    },
+}
+
+/// Message sent form the backend core to the shard
+#[derive(Deserialize, Serialize, Debug)]
+pub enum BackendMessage {
+	Initialize {
+		sid: ShardConnId,
+		nid: NodeId,
+	},
+	Mute {
+		sid: ShardConnId,
+		reason: MuteReason,
+	},
+}

backend/common/src/types.rs

@@ -0,0 +1,164 @@
+use serde::ser::{SerializeTuple, Serializer};
+use serde::{Deserialize, Serialize};
+
+use crate::util::{now, MeanList};
+
+pub type NodeId = usize;
+pub type ConnId = u64;
+pub type BlockNumber = u64;
+pub type Timestamp = u64;
+pub type Address = Box<str>;
+pub use primitive_types::H256 as BlockHash;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct NodeDetails {
+    pub chain: Box<str>,
+    pub name: Box<str>,
+    pub implementation: Box<str>,
+    pub version: Box<str>,
+    pub validator: Option<Box<str>>,
+    pub network_id: Option<Box<str>>,
+    pub startup_time: Option<Box<str>>,
+}
+
+#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct NodeStats {
+    pub peers: u64,
+    pub txcount: u64,
+}
+
+#[derive(Default)]
+pub struct NodeIO {
+    pub used_state_cache_size: MeanList<f32>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone, Copy)]
+pub struct Block {
+    #[serde(rename = "best")]
+    pub hash: BlockHash,
+    pub height: BlockNumber,
+}
+
+impl Block {
+    pub fn zero() -> Self {
+        Block {
+            hash: BlockHash::from([0; 32]),
+            height: 0,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct BlockDetails {
+    pub block: Block,
+    pub block_time: u64,
+    pub block_timestamp: u64,
+    pub propagation_time: Option<u64>,
+}
+
+impl Default for BlockDetails {
+    fn default() -> Self {
+        BlockDetails {
+            block: Block::zero(),
+            block_timestamp: now(),
+            block_time: 0,
+            propagation_time: None,
+        }
+    }
+}
+
+#[derive(Default)]
+pub struct NodeHardware {
+    /// Upload uses means
+    pub upload: MeanList<f64>,
+    /// Download uses means
+    pub download: MeanList<f64>,
+    /// Stampchange uses means
+    pub chart_stamps: MeanList<f64>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct NodeLocation {
+    pub latitude: f32,
+    pub longitude: f32,
+    pub city: Box<str>,
+}
+
+// impl Serialize for NodeDetails {
+//     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+//     where
+//         S: Serializer,
+//     {
+//         let mut tup = serializer.serialize_tuple(6)?;
+//         tup.serialize_element(&self.name)?;
+//         tup.serialize_element(&self.implementation)?;
+//         tup.serialize_element(&self.version)?;
+//         tup.serialize_element(&self.validator)?;
+//         tup.serialize_element(&self.network_id)?;
+//         tup.end()
+//     }
+// }
+
+impl Serialize for NodeStats {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(2)?;
+        tup.serialize_element(&self.peers)?;
+        tup.serialize_element(&self.txcount)?;
+        tup.end()
+    }
+}
+
+impl Serialize for NodeIO {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(1)?;
+        tup.serialize_element(self.used_state_cache_size.slice())?;
+        tup.end()
+    }
+}
+
+impl Serialize for BlockDetails {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(5)?;
+        tup.serialize_element(&self.block.height)?;
+        tup.serialize_element(&self.block.hash)?;
+        tup.serialize_element(&self.block_time)?;
+        tup.serialize_element(&self.block_timestamp)?;
+        tup.serialize_element(&self.propagation_time)?;
+        tup.end()
+    }
+}
+
+impl Serialize for NodeLocation {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(3)?;
+        tup.serialize_element(&self.latitude)?;
+        tup.serialize_element(&self.longitude)?;
+        tup.serialize_element(&&*self.city)?;
+        tup.end()
+    }
+}
+
+impl Serialize for NodeHardware {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut tup = serializer.serialize_tuple(3)?;
+        tup.serialize_element(self.upload.slice())?;
+        tup.serialize_element(self.download.slice())?;
+        tup.serialize_element(self.chart_stamps.slice())?;
+        tup.end()
+    }
+}

backend/common/src/util.rs

@@ -0,0 +1,31 @@
+mod dense_map;
+mod hash;
+mod mean_list;
+mod null;
+mod num_stats;
+
+pub use dense_map::DenseMap;
+pub use hash::Hash;
+pub use mean_list::MeanList;
+pub use null::NullAny;
+pub use num_stats::NumStats;
+
+pub fn fnv<D: AsRef<[u8]>>(data: D) -> u64 {
+    use fnv::FnvHasher;
+    use std::hash::Hasher;
+
+    let mut hasher = FnvHasher::default();
+
+    hasher.write(data.as_ref());
+    hasher.finish()
+}
+
+/// Returns current unix time in ms (compatible with JS Date.now())
+pub fn now() -> u64 {
+    use std::time::SystemTime;
+
+    SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .expect("System time must be configured to be post Unix Epoch start; qed")
+        .as_millis() as u64
+}

backend/common/src/util/dense_map.rs

@@ -0,0 +1,80 @@
+pub type Id = usize;
+
+pub struct DenseMap<T> {
+    /// List of retired indexes that can be re-used
+    retired: Vec<Id>,
+    /// All items
+    items: Vec<Option<T>>,
+}
+
+impl<T> DenseMap<T> {
+    pub fn new() -> Self {
+        DenseMap {
+            retired: Vec::new(),
+            items: Vec::new(),
+        }
+    }
+
+    pub fn add(&mut self, item: T) -> Id {
+        self.add_with(|_| item)
+    }
+
+    pub fn add_with<F>(&mut self, f: F) -> Id
+    where
+        F: FnOnce(Id) -> T,
+    {
+        match self.retired.pop() {
+            Some(id) => {
+                self.items[id] = Some(f(id));
+                id
+            }
+            None => {
+                let id = self.items.len();
+                self.items.push(Some(f(id)));
+                id
+            }
+        }
+    }
+
+    pub fn get(&self, id: Id) -> Option<&T> {
+        self.items.get(id).and_then(|item| item.as_ref())
+    }
+
+    pub fn get_mut(&mut self, id: Id) -> Option<&mut T> {
+        self.items.get_mut(id).and_then(|item| item.as_mut())
+    }
+
+    pub fn remove(&mut self, id: Id) -> Option<T> {
+        let old = self.items.get_mut(id).and_then(|item| item.take());
+
+        if old.is_some() {
+            // something was actually removed, so lets add the id to
+            // the list of retired ids!
+            self.retired.push(id);
+        }
+
+        old
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (Id, &T)> + '_ {
+        self.items
+            .iter()
+            .enumerate()
+            .filter_map(|(id, item)| Some((id, item.as_ref()?)))
+    }
+
+    pub fn iter_mut(&mut self) -> impl Iterator<Item = (Id, &mut T)> + '_ {
+        self.items
+            .iter_mut()
+            .enumerate()
+            .filter_map(|(id, item)| Some((id, item.as_mut()?)))
+    }
+
+    pub fn len(&self) -> usize {
+        self.items.len() - self.retired.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+}

backend/common/src/util/hash.rs

@@ -0,0 +1,209 @@
+use std::fmt::{self, Debug, Display};
+use std::str::FromStr;
+
+use actix_web::error::ResponseError;
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer, Unexpected, Visitor, SeqAccess};
+
+const HASH_BYTES: usize = 32;
+
+/// Newtype wrapper for 32-byte hash values, implementing readable `Debug` and `serde::Deserialize`.
+// We could use primitive_types::H256 here, but opted for a custom type to avoid more dependencies.
+#[derive(Hash, PartialEq, Eq, Clone, Copy)]
+pub struct Hash([u8; HASH_BYTES]);
+
+struct HashVisitor;
+
+impl<'de> Visitor<'de> for HashVisitor {
+    type Value = Hash;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("byte array of length 32, or hexidecimal string of 32 bytes beginning with 0x")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        value
+            .parse()
+            .map_err(|_| de::Error::invalid_value(Unexpected::Str(value), &self))
+    }
+
+    fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.len() == HASH_BYTES {
+            let mut hash = [0; HASH_BYTES];
+
+            hash.copy_from_slice(value);
+
+            return Ok(Hash(hash));
+        }
+
+        Hash::from_ascii(value)
+            .map_err(|_| de::Error::invalid_value(Unexpected::Bytes(value), &self))
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: SeqAccess<'de>,
+    {
+        let mut hash = [0u8; HASH_BYTES];
+
+        for (i, byte) in hash.iter_mut().enumerate() {
+            match seq.next_element()? {
+                Some(b) => *byte = b,
+                None => return Err(de::Error::invalid_length(i, &"an array of 32 bytes"))
+            }
+        }
+
+        if seq.next_element::<u8>()?.is_some() {
+            return Err(de::Error::invalid_length(33, &"an array of 32 bytes"));
+        }
+
+        Ok(Hash(hash))
+    }
+}
+
+impl Hash {
+    pub fn from_ascii(value: &[u8]) -> Result<Self, HashParseError> {
+        if !value.starts_with(b"0x") {
+            return Err(HashParseError::InvalidPrefix);
+        }
+
+        let mut hash = [0; HASH_BYTES];
+
+        hex::decode_to_slice(&value[2..], &mut hash).map_err(HashParseError::HexError)?;
+
+        Ok(Hash(hash))
+    }
+}
+
+impl FromStr for Hash {
+    type Err = HashParseError;
+
+    fn from_str(value: &str) -> Result<Self, Self::Err> {
+        Hash::from_ascii(value.as_bytes())
+    }
+}
+
+impl<'de> Deserialize<'de> for Hash {
+    fn deserialize<D>(deserializer: D) -> Result<Hash, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_bytes(HashVisitor)
+    }
+}
+
+impl Serialize for Hash {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(&self.0)
+    }
+}
+
+impl Display for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str("0x")?;
+
+        let mut ascii = [0; HASH_BYTES * 2];
+
+        hex::encode_to_slice(self.0, &mut ascii)
+            .expect("Encoding 32 bytes into 64 bytes of ascii; qed");
+
+        f.write_str(std::str::from_utf8(&ascii).expect("ASCII hex encoded bytes canot fail; qed"))
+    }
+}
+
+impl Debug for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        Display::fmt(self, f)
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum HashParseError {
+    HexError(hex::FromHexError),
+    InvalidPrefix,
+}
+
+impl Display for HashParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        Debug::fmt(self, f)
+    }
+}
+
+impl ResponseError for HashParseError {}
+
+#[cfg(test)]
+mod tests {
+    use super::Hash;
+    use bincode::Options;
+
+    const DUMMY: Hash = {
+        let mut hash = [0; 32];
+        hash[0] = 0xDE;
+        hash[1] = 0xAD;
+        hash[2] = 0xBE;
+        hash[3] = 0xEF;
+        Hash(hash)
+    };
+
+    #[test]
+    fn deserialize_json_hash_str() {
+        let json = r#""0xdeadBEEF00000000000000000000000000000000000000000000000000000000""#;
+
+        let hash: Hash = serde_json::from_str(json).unwrap();
+
+        assert_eq!(hash, DUMMY);
+    }
+
+    #[test]
+    fn deserialize_json_array() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let hash: Hash = serde_json::from_str(json).unwrap();
+
+        assert_eq!(hash, DUMMY);
+    }
+
+
+    #[test]
+    fn deserialize_json_array_too_short() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let res = serde_json::from_str::<Hash>(json);
+
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn deserialize_json_array_too_long() {
+        let json = r#"[222,173,190,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]"#;
+
+        let res = serde_json::from_str::<Hash>(json);
+
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn bincode() {
+        let bytes = bincode::options().serialize(&DUMMY).unwrap();
+
+        let mut expected = [0; 33];
+
+        expected[0] = 32; // length
+        expected[1..].copy_from_slice(&DUMMY.0);
+
+        assert_eq!(bytes, &expected);
+
+        let deserialized: Hash = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(DUMMY, deserialized);
+    }
+}

backend/common/src/util/mean_list.rs

@@ -0,0 +1,79 @@
+use num_traits::{Float, Zero};
+use std::ops::AddAssign;
+
+pub struct MeanList<T>
+where
+    T: Float + AddAssign + Zero + From<u8>,
+{
+    period_sum: T,
+    period_count: u8,
+    mean_index: u8,
+    means: [T; 20],
+    ticks_per_mean: u8,
+}
+
+impl<T> Default for MeanList<T>
+where
+    T: Float + AddAssign + Zero + From<u8>,
+{
+    fn default() -> MeanList<T> {
+        MeanList {
+            period_sum: T::zero(),
+            period_count: 0,
+            mean_index: 0,
+            means: [T::zero(); 20],
+            ticks_per_mean: 1,
+        }
+    }
+}
+
+impl<T> MeanList<T>
+where
+    T: Float + AddAssign + Zero + From<u8>,
+{
+    pub fn slice(&self) -> &[T] {
+        &self.means[..usize::from(self.mean_index)]
+    }
+
+    pub fn push(&mut self, val: T) -> bool {
+        if self.mean_index == 20 && self.ticks_per_mean < 32 {
+            self.squash_means();
+        }
+
+        self.period_sum += val;
+        self.period_count += 1;
+
+        if self.period_count == self.ticks_per_mean {
+            self.push_mean();
+            true
+        } else {
+            false
+        }
+    }
+
+    fn push_mean(&mut self) {
+        let mean = self.period_sum / std::convert::From::from(self.period_count);
+
+        if self.mean_index == 20 && self.ticks_per_mean == 32 {
+            self.means.rotate_left(1);
+            self.means[19] = mean;
+        } else {
+            self.means[usize::from(self.mean_index)] = mean;
+            self.mean_index += 1;
+        }
+
+        self.period_sum = T::zero();
+        self.period_count = 0;
+    }
+
+    fn squash_means(&mut self) {
+        self.ticks_per_mean *= 2;
+        self.mean_index = 10;
+
+        for i in 0..10 {
+            let i2 = i * 2;
+
+            self.means[i] = (self.means[i2] + self.means[i2 + 1]) / std::convert::From::from(2)
+        }
+    }
+}

backend/common/src/util/null.rs

@@ -0,0 +1,136 @@
+use serde::de::{Deserialize, Deserializer, IgnoredAny};
+use serde::ser::{Serialize, Serializer};
+
+/// Alternative to `serde::de::IgnoreAny` that implements `Serialize`.
+/// Will serialize to `null` in JSON, or empty data in bincode.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct NullAny;
+
+impl<'de> Deserialize<'de> for NullAny {
+    #[inline]
+    fn deserialize<D>(deserializer: D) -> Result<NullAny, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        // `bincode` is going to throw an error here as it does not support `IgnoredAny`.
+        //
+        // When using `bincode` `NullAny` will always serialize to unit (aka no data), so
+        // this safely becomes a no-op.
+        let _ = deserializer.deserialize_ignored_any(IgnoredAny);
+
+        Ok(NullAny)
+    }
+}
+
+impl Serialize for NullAny {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_unit()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NullAny;
+    use bincode::Options;
+    use serde::{Deserialize, Serialize};
+
+    #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+    struct Dummy {
+        ignore: NullAny,
+    }
+
+    #[test]
+    fn deserialize_json_null() {
+        let dummy: Dummy = serde_json::from_str(r#"{"ignore":null}"#).unwrap();
+
+        assert_eq!(dummy, Dummy { ignore: NullAny });
+    }
+
+    #[test]
+    fn deserialize_json_struct() {
+        let dummy: Dummy = serde_json::from_str(r#"{"ignore":{"foo":"bar"}}"#).unwrap();
+
+        assert_eq!(dummy, Dummy { ignore: NullAny });
+    }
+
+    #[test]
+    fn deserialize_json_struct_invalid() {
+        let dummy = serde_json::from_str::<Dummy>(r#"{"ignore":{"foo":"bar"}"#);
+
+        assert!(dummy.is_err());
+    }
+
+    #[test]
+    fn deserialize_json_vec_any() {
+        let raw = [NullAny; 10];
+        let json = r#"[null,true,false,10,{},[],[null],{"foo":"bar"},[9,9,9],"ten"]"#;
+
+        let deserialized: Vec<NullAny> = serde_json::from_str(json).unwrap();
+
+        assert_eq!(&raw[..], &deserialized);
+    }
+
+    #[test]
+    fn serialize_json_null() {
+        let dummy = Dummy { ignore: NullAny };
+
+        let json = serde_json::to_string(&dummy).unwrap();
+
+        assert_eq!(json, r#"{"ignore":null}"#);
+    }
+
+    #[test]
+    fn bincode_vec() {
+        let raw = vec![NullAny; 10];
+
+        let bytes = bincode::options().serialize(&raw).unwrap();
+
+        assert_eq!(bytes, &[10u8]);
+
+        let deserialized: Vec<NullAny> = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(raw, deserialized);
+    }
+
+    #[test]
+    fn bincode_tuple() {
+        let raw = (NullAny, "Hello world".to_string());
+
+        let bytes = bincode::options().serialize(&raw).unwrap();
+
+        assert_eq!(bytes, b"\x0BHello world"); // 0B = 11 = length of string
+
+        let deserialized: (NullAny, String) = bincode::options().deserialize(&bytes).unwrap();
+
+        assert_eq!(raw, deserialized);
+    }
+
+    #[test]
+    fn json_vec() {
+        let raw = vec![NullAny; 10];
+
+        let json = serde_json::to_string(&raw).unwrap();
+
+        assert_eq!(json, "[null,null,null,null,null,null,null,null,null,null]");
+
+        let deserialized: Vec<NullAny> = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(raw, deserialized);
+    }
+
+    #[test]
+    fn json_tuple() {
+        let raw = (NullAny, "Hello world".to_string());
+
+        let json = serde_json::to_string(&raw).unwrap();
+
+        assert_eq!(json, r#"[null,"Hello world"]"#);
+
+        let deserialized: (NullAny, String) = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(raw, deserialized);
+    }
+}

backend/common/src/util/num_stats.rs

@@ -0,0 +1,104 @@
+use num_traits::{Bounded, NumOps, Zero};
+use std::convert::TryFrom;
+use std::iter::Sum;
+
+/// Keep track of last N numbers pushed onto internal stack.
+/// Provides means to get an average of said numbers.
+pub struct NumStats<T> {
+    stack: Box<[T]>,
+    index: usize,
+    sum: T,
+}
+
+impl<T: NumOps + Zero + Bounded + Copy + Sum + TryFrom<usize>> NumStats<T> {
+    pub fn new(size: usize) -> Self {
+        NumStats {
+            stack: vec![T::zero(); size].into_boxed_slice(),
+            index: 0,
+            sum: T::zero(),
+        }
+    }
+
+    pub fn push(&mut self, val: T) {
+        let slot = &mut self.stack[self.index % self.stack.len()];
+
+        self.sum = (self.sum + val) - *slot;
+
+        *slot = val;
+
+        self.index += 1;
+    }
+
+    pub fn average(&self) -> T {
+        let cap = std::cmp::min(self.index, self.stack.len());
+
+        if cap == 0 {
+            return T::zero();
+        }
+
+        let cap = T::try_from(cap).unwrap_or_else(|_| T::max_value());
+
+        self.sum / cap
+    }
+
+    pub fn reset(&mut self) {
+        self.index = 0;
+        self.sum = T::zero();
+
+        for val in self.stack.iter_mut() {
+            *val = T::zero();
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn calculates_correct_average() {
+        let mut stats: NumStats<u64> = NumStats::new(10);
+
+        stats.push(3);
+        stats.push(7);
+
+        assert_eq!(stats.average(), 5);
+    }
+
+    #[test]
+    fn calculates_correct_average_over_bounds() {
+        let mut stats: NumStats<u64> = NumStats::new(10);
+
+        stats.push(100);
+
+        for _ in 0..9 {
+            stats.push(0);
+        }
+
+        assert_eq!(stats.average(), 10);
+
+        stats.push(0);
+
+        assert_eq!(stats.average(), 0);
+    }
+
+    #[test]
+    fn resets_properly() {
+        let mut stats: NumStats<u64> = NumStats::new(10);
+
+        for _ in 0..10 {
+            stats.push(100);
+        }
+
+        assert_eq!(stats.average(), 100);
+
+        stats.reset();
+
+        assert_eq!(stats.average(), 0);
+
+        stats.push(7);
+        stats.push(3);
+
+        assert_eq!(stats.average(), 5);
+    }
+}

backend/common/src/ws.rs

@@ -0,0 +1,98 @@
+use actix_http::ws::Item;
+use actix_web_actors::ws::{self, CloseReason, CloseCode};
+use bytes::{Bytes, BytesMut};
+use serde::{Serialize, Deserialize};
+use actix::prelude::Message;
+
+/// Helper that will buffer continuation messages from actix
+/// until completion, capping at 10mb.
+#[derive(Default)]
+pub struct MultipartHandler {
+    buf: BytesMut,
+}
+
+/// Message to signal that a node should be muted for a reason that's
+/// cheap to transfer between Actors or over the wire for shards.
+#[derive(Serialize, Deserialize, Message, Clone, Copy, Debug)]
+#[rtype("()")]
+pub enum MuteReason {
+    /// Node was denied connection for any arbitrary reason,
+    /// and should not attempt to reconnect.
+    Denied,
+    /// Node was denied because the chain it belongs to is currently
+    /// at the limit of allowed nodes, and it may attempt to reconnect.
+    Overquota,
+}
+
+impl From<MuteReason> for CloseReason {
+    fn from(mute: MuteReason) -> CloseReason {
+        match mute {
+            MuteReason::Denied => CloseReason {
+                code: CloseCode::Abnormal,
+                description: Some("Denied".into()),
+            },
+            MuteReason::Overquota => CloseReason {
+                code: CloseCode::Again,
+                description: Some("Overquota".into()),
+            },
+        }
+    }
+}
+
+/// Continuation buffer limit, 10mb
+const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
+
+pub enum WsMessage {
+    Nop,
+    Ping(Bytes),
+    Data(Bytes),
+    Close(Option<CloseReason>),
+}
+
+impl MultipartHandler {
+    pub fn handle(&mut self, msg: ws::Message) -> WsMessage {
+        match msg {
+            ws::Message::Ping(msg) => WsMessage::Ping(msg),
+            ws::Message::Pong(_) => WsMessage::Nop,
+            ws::Message::Text(text) => WsMessage::Data(text.into_bytes()),
+            ws::Message::Binary(data) => WsMessage::Data(data),
+            ws::Message::Close(reason) => WsMessage::Close(reason),
+            ws::Message::Nop => WsMessage::Nop,
+            ws::Message::Continuation(cont) => match cont {
+                Item::FirstText(bytes) | Item::FirstBinary(bytes) => {
+                    self.start_frame(&bytes);
+                    WsMessage::Nop
+                }
+                Item::Continue(bytes) => {
+                    self.continue_frame(&bytes);
+                    WsMessage::Nop
+                }
+                Item::Last(bytes) => {
+                    self.continue_frame(&bytes);
+                    WsMessage::Data(self.finish_frame())
+                }
+            },
+        }
+    }
+
+    fn start_frame(&mut self, bytes: &[u8]) {
+        if !self.buf.is_empty() {
+            log::error!("Unused continuation buffer");
+            self.buf.clear();
+        }
+        self.continue_frame(bytes);
+    }
+
+    fn continue_frame(&mut self, bytes: &[u8]) {
+        if self.buf.len() + bytes.len() <= CONT_BUF_LIMIT {
+            self.buf.extend_from_slice(&bytes);
+        } else {
+            log::error!("Continuation buffer overflow");
+            self.buf = BytesMut::new();
+        }
+    }
+
+    fn finish_frame(&mut self) -> Bytes {
+        std::mem::replace(&mut self.buf, BytesMut::new()).freeze()
+    }
+}