Rework: Shard working, Telemetry Core needs logic filling in

backend/common/src/internal_messages.rs

@@ -0,0 +1,36 @@
+use std::net::IpAddr;
+
+use crate::node::Payload;
+use crate::types::{NodeDetails};
+use serde::{Deserialize, Serialize};
+
+/// The shard-local ID of a given node, where a single connection
+/// might send data on behalf of more than one chain.
+pub type LocalId = u64;
+
+/// A global ID assigned to messages from each different pair of ConnId+LocalId.
+pub type GlobalId = u64;
+
+/// Message sent from the shard to the backend core
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum FromShardAggregator {
+    /// Get information about a new node, passing IPv4
+    AddNode {
+    	ip: Option<IpAddr>,
+    	node: NodeDetails,
+    	local_id: LocalId,
+    },
+    /// Send a message payload to update details for a node
+    UpdateNode {
+        local_id: LocalId,
+        payload: Payload,
+    },
+}
+
+/// Message sent form the backend core to the shard
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub enum FromTelemetryCore {
+	Mute {
+		local_id: LocalId
+	}
+}

backend/common/src/json/hash.rs

@@ -1,7 +1,5 @@
 use std::fmt::{self, Debug, Display};
 use std::str::FromStr;
-
-use actix_web::error::ResponseError;
 use serde::ser::{Serialize, Serializer};
 use serde::de::{self, Deserialize, Deserializer, Unexpected, Visitor, SeqAccess};
 
@@ -140,18 +138,12 @@ impl Debug for Hash {
 
 #[derive(thiserror::Error, Debug)]
 pub enum HashParseError {
+    #[error("Error parsing string into hex: {0}")]
     HexError(hex::FromHexError),
+    #[error("Invalid hex prefix: expected '0x'")]
     InvalidPrefix,
 }
 
-impl Display for HashParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        Debug::fmt(self, f)
-    }
-}
-
-impl ResponseError for HashParseError {}
-
 #[cfg(test)]
 mod tests {
     use super::Hash;

backend/common/src/json/node_message.rs

@@ -26,7 +26,7 @@ pub enum NodeMessage {
         payload: Payload,
     },
     V2 {
-        id: ConnId,
+        id: NodeMessageId,
         payload: Payload,
     },
 }
@@ -127,7 +127,7 @@ pub struct NodeDetails {
     pub startup_time: Option<Box<str>>,
 }
 
-type ConnId = u64;
+type NodeMessageId = u64;
 type BlockNumber = u64;
 
 #[cfg(test)]

backend/common/src/lib.rs

@@ -1,6 +1,6 @@
 pub mod node;
-pub mod shard;
+pub mod internal_messages;
 pub mod types;
 pub mod util;
-pub mod ws;
-pub mod json;
+pub mod json;
+pub mod log_level;

backend/common/src/log_level.rs

@@ -0,0 +1,36 @@
+/// A log level that can be used as an expected argument,
+/// decoded from a string, and converted into a [`log::LevelFilter`]
+#[derive(Debug, PartialEq)]
+pub enum LogLevel {
+    Error,
+    Warn,
+    Info,
+    Debug,
+    Trace,
+}
+
+impl std::str::FromStr for LogLevel {
+    type Err = &'static str;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "error" => Ok(LogLevel::Error),
+            "warn"  => Ok(LogLevel::Warn),
+            "info"  => Ok(LogLevel::Info),
+            "debug" => Ok(LogLevel::Debug),
+            "trace" => Ok(LogLevel::Trace),
+            _       => Err("expected 'error', 'warn', 'info', 'debug' or 'trace'")
+        }
+    }
+}
+
+impl From<&LogLevel> for log::LevelFilter {
+    fn from(log_level: &LogLevel) -> Self {
+        match log_level {
+            LogLevel::Error => log::LevelFilter::Error,
+            LogLevel::Warn => log::LevelFilter::Warn,
+            LogLevel::Info => log::LevelFilter::Info,
+            LogLevel::Debug => log::LevelFilter::Debug,
+            LogLevel::Trace => log::LevelFilter::Trace,
+        }
+    }
+}

backend/common/src/node.rs

@@ -1,36 +1,41 @@
-use crate::types::{Block, BlockHash, BlockNumber, ConnId, NodeDetails};
+use crate::types::{Block, BlockHash, BlockNumber, NodeDetails};
 use crate::json;
-
-use actix::prelude::*;
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Debug, Message)]
-#[rtype(result = "()")]
+pub type NodeMessageId = u64;
+
+#[derive(Serialize, Deserialize, Debug)]
 pub enum NodeMessage {
     V1 {
         payload: Payload,
     },
     V2 {
-        id: ConnId,
+        id: NodeMessageId,
         payload: Payload,
     },
 }
 
 impl NodeMessage {
-    /// Returns the connection ID or 0 if there is no ID.
-    pub fn id(&self) -> ConnId {
+    /// Returns the ID associated with the node message, or 0
+    /// if the message has no ID.
+    pub fn id(&self) -> NodeMessageId {
         match self {
             NodeMessage::V1 { .. } => 0,
             NodeMessage::V2 { id, .. } => *id,
         }
     }
+    /// Return the payload associated with the message.
+    pub fn into_payload(self) -> Payload {
+        match self {
+            NodeMessage::V1 { payload, .. } |
+            NodeMessage::V2 { payload, .. } => payload,
+        }
+    }
 }
 
 impl From<NodeMessage> for Payload {
     fn from(msg: NodeMessage) -> Payload {
-        match msg {
-            NodeMessage::V1 { payload, .. } | NodeMessage::V2 { payload, .. } => payload,
-        }
+        msg.into_payload()
     }
 }
 
@@ -47,7 +52,7 @@ impl From<json::NodeMessage> for NodeMessage {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub enum Payload {
     SystemConnected(SystemConnected),
     SystemInterval(SystemInterval),
@@ -110,7 +115,7 @@ impl From<json::Payload> for Payload {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct SystemConnected {
     pub genesis_hash: BlockHash,
     pub node: NodeDetails,
@@ -125,7 +130,7 @@ impl From<json::SystemConnected> for SystemConnected {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct SystemInterval {
     pub peers: Option<u64>,
     pub txcount: Option<u64>,
@@ -152,7 +157,7 @@ impl From<json::SystemInterval> for SystemInterval {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Finalized {
     pub hash: BlockHash,
     pub height: Box<str>,
@@ -199,7 +204,7 @@ impl From<json::AfgReceived> for AfgReceived {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct AfgAuthoritySet {
     pub authority_id: Box<str>,
     pub authorities: Box<str>,

backend/common/src/shard.rs

@@ -1,38 +0,0 @@
-use std::net::Ipv4Addr;
-
-use crate::ws::MuteReason;
-use crate::node::Payload;
-use crate::types::{NodeId, NodeDetails};
-use serde::{Deserialize, Serialize};
-
-/// Alias for the ID of the node connection
-pub type ShardConnId = u32;
-
-/// Message sent from the shard to the backend core
-#[derive(Deserialize, Serialize, Debug)]
-pub enum ShardMessage {
-    /// Get a connection id for a new node, passing IPv4
-    AddNode {
-    	ip: Option<Ipv4Addr>,
-    	node: NodeDetails,
-    	sid: ShardConnId,
-    },
-    /// Send a message payload for a given node
-    UpdateNode {
-        nid: NodeId,
-        payload: Payload,
-    },
-}
-
-/// Message sent form the backend core to the shard
-#[derive(Deserialize, Serialize, Debug)]
-pub enum BackendMessage {
-	Initialize {
-		sid: ShardConnId,
-		nid: NodeId,
-	},
-	Mute {
-		sid: ShardConnId,
-		reason: MuteReason,
-	},
-}

backend/common/src/types.rs

@@ -5,7 +5,6 @@ use crate::util::{now, MeanList};
 use crate::json;
 
 pub type NodeId = usize;
-pub type ConnId = u64;
 pub type BlockNumber = u64;
 pub type Timestamp = u64;
 pub type Address = Box<str>;

backend/common/src/ws.rs

@@ -1,98 +0,0 @@
-use actix_http::ws::Item;
-use actix_web_actors::ws::{self, CloseReason, CloseCode};
-use bytes::{Bytes, BytesMut};
-use serde::{Serialize, Deserialize};
-use actix::prelude::Message;
-
-/// Helper that will buffer continuation messages from actix
-/// until completion, capping at 10mb.
-#[derive(Default)]
-pub struct MultipartHandler {
-    buf: BytesMut,
-}
-
-/// Message to signal that a node should be muted for a reason that's
-/// cheap to transfer between Actors or over the wire for shards.
-#[derive(Serialize, Deserialize, Message, Clone, Copy, Debug)]
-#[rtype("()")]
-pub enum MuteReason {
-    /// Node was denied connection for any arbitrary reason,
-    /// and should not attempt to reconnect.
-    Denied,
-    /// Node was denied because the chain it belongs to is currently
-    /// at the limit of allowed nodes, and it may attempt to reconnect.
-    Overquota,
-}
-
-impl From<MuteReason> for CloseReason {
-    fn from(mute: MuteReason) -> CloseReason {
-        match mute {
-            MuteReason::Denied => CloseReason {
-                code: CloseCode::Abnormal,
-                description: Some("Denied".into()),
-            },
-            MuteReason::Overquota => CloseReason {
-                code: CloseCode::Again,
-                description: Some("Overquota".into()),
-            },
-        }
-    }
-}
-
-/// Continuation buffer limit, 10mb
-const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
-
-pub enum WsMessage {
-    Nop,
-    Ping(Bytes),
-    Data(Bytes),
-    Close(Option<CloseReason>),
-}
-
-impl MultipartHandler {
-    pub fn handle(&mut self, msg: ws::Message) -> WsMessage {
-        match msg {
-            ws::Message::Ping(msg) => WsMessage::Ping(msg),
-            ws::Message::Pong(_) => WsMessage::Nop,
-            ws::Message::Text(text) => WsMessage::Data(text.into_bytes()),
-            ws::Message::Binary(data) => WsMessage::Data(data),
-            ws::Message::Close(reason) => WsMessage::Close(reason),
-            ws::Message::Nop => WsMessage::Nop,
-            ws::Message::Continuation(cont) => match cont {
-                Item::FirstText(bytes) | Item::FirstBinary(bytes) => {
-                    self.start_frame(&bytes);
-                    WsMessage::Nop
-                }
-                Item::Continue(bytes) => {
-                    self.continue_frame(&bytes);
-                    WsMessage::Nop
-                }
-                Item::Last(bytes) => {
-                    self.continue_frame(&bytes);
-                    WsMessage::Data(self.finish_frame())
-                }
-            },
-        }
-    }
-
-    fn start_frame(&mut self, bytes: &[u8]) {
-        if !self.buf.is_empty() {
-            log::error!("Unused continuation buffer");
-            self.buf.clear();
-        }
-        self.continue_frame(bytes);
-    }
-
-    fn continue_frame(&mut self, bytes: &[u8]) {
-        if self.buf.len() + bytes.len() <= CONT_BUF_LIMIT {
-            self.buf.extend_from_slice(&bytes);
-        } else {
-            log::error!("Continuation buffer overflow");
-            self.buf = BytesMut::new();
-        }
-    }
-
-    fn finish_frame(&mut self) -> Bytes {
-        std::mem::replace(&mut self.buf, BytesMut::new()).freeze()
-    }
-}