mirror of
https://github.com/pezkuwichain/pezkuwi-telemetry.git
synced 2026-05-30 11:11:08 +00:00
Rework: Shard working, Telemetry Core needs logic filling in
This commit is contained in:
James Wilson
Generated
+417
-1126
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -1,7 +1,7 @@
|
||||
[workspace]
|
||||
members = [
|
||||
"common",
|
||||
"core",
|
||||
"telemetry",
|
||||
"shard"
|
||||
]
|
||||
|
||||
|
||||
@@ -5,10 +5,6 @@ authors = ["Parity Technologies Ltd. <admin@parity.io>"]
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
actix = "0.11.1"
|
||||
actix-web = { version = "4.0.0-beta.4", default-features = false }
|
||||
actix-web-actors = "4.0.0-beta.3"
|
||||
actix-http = "3.0.0-beta.4"
|
||||
bytes = "1.0.1"
|
||||
fnv = "1.0.7"
|
||||
hex = "0.4.3"
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
use std::net::IpAddr;
|
||||
|
||||
use crate::node::Payload;
|
||||
use crate::types::{NodeDetails};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// The shard-local ID of a given node, where a single connection
|
||||
/// might send data on behalf of more than one chain.
|
||||
pub type LocalId = u64;
|
||||
|
||||
/// A global ID assigned to messages from each different pair of ConnId+LocalId.
|
||||
pub type GlobalId = u64;
|
||||
|
||||
/// Message sent from the shard to the backend core
|
||||
#[derive(Deserialize, Serialize, Debug, Clone)]
|
||||
pub enum FromShardAggregator {
|
||||
/// Get information about a new node, passing IPv4
|
||||
AddNode {
|
||||
ip: Option<IpAddr>,
|
||||
node: NodeDetails,
|
||||
local_id: LocalId,
|
||||
},
|
||||
/// Send a message payload to update details for a node
|
||||
UpdateNode {
|
||||
local_id: LocalId,
|
||||
payload: Payload,
|
||||
},
|
||||
}
|
||||
|
||||
/// Message sent form the backend core to the shard
|
||||
#[derive(Deserialize, Serialize, Debug, Clone)]
|
||||
pub enum FromTelemetryCore {
|
||||
Mute {
|
||||
local_id: LocalId
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,5 @@
|
||||
use std::fmt::{self, Debug, Display};
|
||||
use std::str::FromStr;
|
||||
|
||||
use actix_web::error::ResponseError;
|
||||
use serde::ser::{Serialize, Serializer};
|
||||
use serde::de::{self, Deserialize, Deserializer, Unexpected, Visitor, SeqAccess};
|
||||
|
||||
@@ -140,18 +138,12 @@ impl Debug for Hash {
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum HashParseError {
|
||||
#[error("Error parsing string into hex: {0}")]
|
||||
HexError(hex::FromHexError),
|
||||
#[error("Invalid hex prefix: expected '0x'")]
|
||||
InvalidPrefix,
|
||||
}
|
||||
|
||||
impl Display for HashParseError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
Debug::fmt(self, f)
|
||||
}
|
||||
}
|
||||
|
||||
impl ResponseError for HashParseError {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Hash;
|
||||
|
||||
@@ -26,7 +26,7 @@ pub enum NodeMessage {
|
||||
payload: Payload,
|
||||
},
|
||||
V2 {
|
||||
id: ConnId,
|
||||
id: NodeMessageId,
|
||||
payload: Payload,
|
||||
},
|
||||
}
|
||||
@@ -127,7 +127,7 @@ pub struct NodeDetails {
|
||||
pub startup_time: Option<Box<str>>,
|
||||
}
|
||||
|
||||
type ConnId = u64;
|
||||
type NodeMessageId = u64;
|
||||
type BlockNumber = u64;
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
pub mod node;
|
||||
pub mod shard;
|
||||
pub mod internal_messages;
|
||||
pub mod types;
|
||||
pub mod util;
|
||||
pub mod ws;
|
||||
pub mod json;
|
||||
pub mod log_level;
|
||||
@@ -0,0 +1,36 @@
|
||||
/// A log level that can be used as an expected argument,
|
||||
/// decoded from a string, and converted into a [`log::LevelFilter`]
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub enum LogLevel {
|
||||
Error,
|
||||
Warn,
|
||||
Info,
|
||||
Debug,
|
||||
Trace,
|
||||
}
|
||||
|
||||
impl std::str::FromStr for LogLevel {
|
||||
type Err = &'static str;
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"error" => Ok(LogLevel::Error),
|
||||
"warn" => Ok(LogLevel::Warn),
|
||||
"info" => Ok(LogLevel::Info),
|
||||
"debug" => Ok(LogLevel::Debug),
|
||||
"trace" => Ok(LogLevel::Trace),
|
||||
_ => Err("expected 'error', 'warn', 'info', 'debug' or 'trace'")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&LogLevel> for log::LevelFilter {
|
||||
fn from(log_level: &LogLevel) -> Self {
|
||||
match log_level {
|
||||
LogLevel::Error => log::LevelFilter::Error,
|
||||
LogLevel::Warn => log::LevelFilter::Warn,
|
||||
LogLevel::Info => log::LevelFilter::Info,
|
||||
LogLevel::Debug => log::LevelFilter::Debug,
|
||||
LogLevel::Trace => log::LevelFilter::Trace,
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
-16
@@ -1,36 +1,41 @@
|
||||
use crate::types::{Block, BlockHash, BlockNumber, ConnId, NodeDetails};
|
||||
use crate::types::{Block, BlockHash, BlockNumber, NodeDetails};
|
||||
use crate::json;
|
||||
|
||||
use actix::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub type NodeMessageId = u64;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub enum NodeMessage {
|
||||
V1 {
|
||||
payload: Payload,
|
||||
},
|
||||
V2 {
|
||||
id: ConnId,
|
||||
id: NodeMessageId,
|
||||
payload: Payload,
|
||||
},
|
||||
}
|
||||
|
||||
impl NodeMessage {
|
||||
/// Returns the connection ID or 0 if there is no ID.
|
||||
pub fn id(&self) -> ConnId {
|
||||
/// Returns the ID associated with the node message, or 0
|
||||
/// if the message has no ID.
|
||||
pub fn id(&self) -> NodeMessageId {
|
||||
match self {
|
||||
NodeMessage::V1 { .. } => 0,
|
||||
NodeMessage::V2 { id, .. } => *id,
|
||||
}
|
||||
}
|
||||
/// Return the payload associated with the message.
|
||||
pub fn into_payload(self) -> Payload {
|
||||
match self {
|
||||
NodeMessage::V1 { payload, .. } |
|
||||
NodeMessage::V2 { payload, .. } => payload,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<NodeMessage> for Payload {
|
||||
fn from(msg: NodeMessage) -> Payload {
|
||||
match msg {
|
||||
NodeMessage::V1 { payload, .. } | NodeMessage::V2 { payload, .. } => payload,
|
||||
}
|
||||
msg.into_payload()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +52,7 @@ impl From<json::NodeMessage> for NodeMessage {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub enum Payload {
|
||||
SystemConnected(SystemConnected),
|
||||
SystemInterval(SystemInterval),
|
||||
@@ -110,7 +115,7 @@ impl From<json::Payload> for Payload {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct SystemConnected {
|
||||
pub genesis_hash: BlockHash,
|
||||
pub node: NodeDetails,
|
||||
@@ -125,7 +130,7 @@ impl From<json::SystemConnected> for SystemConnected {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct SystemInterval {
|
||||
pub peers: Option<u64>,
|
||||
pub txcount: Option<u64>,
|
||||
@@ -152,7 +157,7 @@ impl From<json::SystemInterval> for SystemInterval {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct Finalized {
|
||||
pub hash: BlockHash,
|
||||
pub height: Box<str>,
|
||||
@@ -199,7 +204,7 @@ impl From<json::AfgReceived> for AfgReceived {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct AfgAuthoritySet {
|
||||
pub authority_id: Box<str>,
|
||||
pub authorities: Box<str>,
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
use std::net::Ipv4Addr;
|
||||
|
||||
use crate::ws::MuteReason;
|
||||
use crate::node::Payload;
|
||||
use crate::types::{NodeId, NodeDetails};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Alias for the ID of the node connection
|
||||
pub type ShardConnId = u32;
|
||||
|
||||
/// Message sent from the shard to the backend core
|
||||
#[derive(Deserialize, Serialize, Debug)]
|
||||
pub enum ShardMessage {
|
||||
/// Get a connection id for a new node, passing IPv4
|
||||
AddNode {
|
||||
ip: Option<Ipv4Addr>,
|
||||
node: NodeDetails,
|
||||
sid: ShardConnId,
|
||||
},
|
||||
/// Send a message payload for a given node
|
||||
UpdateNode {
|
||||
nid: NodeId,
|
||||
payload: Payload,
|
||||
},
|
||||
}
|
||||
|
||||
/// Message sent form the backend core to the shard
|
||||
#[derive(Deserialize, Serialize, Debug)]
|
||||
pub enum BackendMessage {
|
||||
Initialize {
|
||||
sid: ShardConnId,
|
||||
nid: NodeId,
|
||||
},
|
||||
Mute {
|
||||
sid: ShardConnId,
|
||||
reason: MuteReason,
|
||||
},
|
||||
}
|
||||
@@ -5,7 +5,6 @@ use crate::util::{now, MeanList};
|
||||
use crate::json;
|
||||
|
||||
pub type NodeId = usize;
|
||||
pub type ConnId = u64;
|
||||
pub type BlockNumber = u64;
|
||||
pub type Timestamp = u64;
|
||||
pub type Address = Box<str>;
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
use actix_http::ws::Item;
|
||||
use actix_web_actors::ws::{self, CloseReason, CloseCode};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use serde::{Serialize, Deserialize};
|
||||
use actix::prelude::Message;
|
||||
|
||||
/// Helper that will buffer continuation messages from actix
|
||||
/// until completion, capping at 10mb.
|
||||
#[derive(Default)]
|
||||
pub struct MultipartHandler {
|
||||
buf: BytesMut,
|
||||
}
|
||||
|
||||
/// Message to signal that a node should be muted for a reason that's
|
||||
/// cheap to transfer between Actors or over the wire for shards.
|
||||
#[derive(Serialize, Deserialize, Message, Clone, Copy, Debug)]
|
||||
#[rtype("()")]
|
||||
pub enum MuteReason {
|
||||
/// Node was denied connection for any arbitrary reason,
|
||||
/// and should not attempt to reconnect.
|
||||
Denied,
|
||||
/// Node was denied because the chain it belongs to is currently
|
||||
/// at the limit of allowed nodes, and it may attempt to reconnect.
|
||||
Overquota,
|
||||
}
|
||||
|
||||
impl From<MuteReason> for CloseReason {
|
||||
fn from(mute: MuteReason) -> CloseReason {
|
||||
match mute {
|
||||
MuteReason::Denied => CloseReason {
|
||||
code: CloseCode::Abnormal,
|
||||
description: Some("Denied".into()),
|
||||
},
|
||||
MuteReason::Overquota => CloseReason {
|
||||
code: CloseCode::Again,
|
||||
description: Some("Overquota".into()),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Continuation buffer limit, 10mb
|
||||
const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
|
||||
|
||||
pub enum WsMessage {
|
||||
Nop,
|
||||
Ping(Bytes),
|
||||
Data(Bytes),
|
||||
Close(Option<CloseReason>),
|
||||
}
|
||||
|
||||
impl MultipartHandler {
|
||||
pub fn handle(&mut self, msg: ws::Message) -> WsMessage {
|
||||
match msg {
|
||||
ws::Message::Ping(msg) => WsMessage::Ping(msg),
|
||||
ws::Message::Pong(_) => WsMessage::Nop,
|
||||
ws::Message::Text(text) => WsMessage::Data(text.into_bytes()),
|
||||
ws::Message::Binary(data) => WsMessage::Data(data),
|
||||
ws::Message::Close(reason) => WsMessage::Close(reason),
|
||||
ws::Message::Nop => WsMessage::Nop,
|
||||
ws::Message::Continuation(cont) => match cont {
|
||||
Item::FirstText(bytes) | Item::FirstBinary(bytes) => {
|
||||
self.start_frame(&bytes);
|
||||
WsMessage::Nop
|
||||
}
|
||||
Item::Continue(bytes) => {
|
||||
self.continue_frame(&bytes);
|
||||
WsMessage::Nop
|
||||
}
|
||||
Item::Last(bytes) => {
|
||||
self.continue_frame(&bytes);
|
||||
WsMessage::Data(self.finish_frame())
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn start_frame(&mut self, bytes: &[u8]) {
|
||||
if !self.buf.is_empty() {
|
||||
log::error!("Unused continuation buffer");
|
||||
self.buf.clear();
|
||||
}
|
||||
self.continue_frame(bytes);
|
||||
}
|
||||
|
||||
fn continue_frame(&mut self, bytes: &[u8]) {
|
||||
if self.buf.len() + bytes.len() <= CONT_BUF_LIMIT {
|
||||
self.buf.extend_from_slice(&bytes);
|
||||
} else {
|
||||
log::error!("Continuation buffer overflow");
|
||||
self.buf = BytesMut::new();
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_frame(&mut self) -> Bytes {
|
||||
std::mem::replace(&mut self.buf, BytesMut::new()).freeze()
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
[package]
|
||||
name = "telemetry"
|
||||
version = "0.3.0"
|
||||
authors = ["Parity Technologies Ltd. <admin@parity.io>"]
|
||||
edition = "2018"
|
||||
license = "GPL-3.0"
|
||||
|
||||
[dependencies]
|
||||
actix = "0.11.1"
|
||||
actix-web = { version = "4.0.0-beta.4", default-features = false }
|
||||
actix-web-actors = "4.0.0-beta.3"
|
||||
actix-http = "3.0.0-beta.4"
|
||||
bincode = "1.3.3"
|
||||
bytes = "1.0.1"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = { version = "1.0", features = ["raw_value"] }
|
||||
common = { path = "../common" }
|
||||
log = "0.4"
|
||||
simple_logger = "1.11.0"
|
||||
parking_lot = "0.11"
|
||||
reqwest = { version = "0.11.1", features = ["blocking", "json"] }
|
||||
rustc-hash = "1.1.0"
|
||||
clap = "3.0.0-beta.2"
|
||||
ctor = "0.1.20"
|
||||
@@ -1,395 +0,0 @@
|
||||
use actix::prelude::*;
|
||||
use ctor::ctor;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::shard::connector::ShardConnector;
|
||||
use crate::chain::{self, Chain, ChainId, Label};
|
||||
use crate::feed::connector::{Connected, FeedConnector, FeedId};
|
||||
use crate::feed::{self, FeedMessageSerializer};
|
||||
use common::ws::MuteReason;
|
||||
use common::shard::ShardConnId;
|
||||
use common::types::{ConnId, NodeDetails, BlockHash};
|
||||
use common::util::{DenseMap};
|
||||
|
||||
pub struct Aggregator {
|
||||
genesis_hashes: HashMap<BlockHash, ChainId>,
|
||||
labels: HashMap<Label, ChainId>,
|
||||
chains: DenseMap<ChainEntry>,
|
||||
feeds: DenseMap<Addr<FeedConnector>>,
|
||||
serializer: FeedMessageSerializer,
|
||||
/// Denylist for networks we do not want to allow connecting.
|
||||
denylist: HashSet<String>,
|
||||
}
|
||||
|
||||
pub struct ChainEntry {
|
||||
/// Address to the `Chain` agent
|
||||
addr: Addr<Chain>,
|
||||
/// Genesis [`Hash`] of the chain
|
||||
genesis_hash: BlockHash,
|
||||
/// String name of the chain
|
||||
label: Label,
|
||||
/// Node count
|
||||
nodes: usize,
|
||||
/// Maximum allowed nodes
|
||||
max_nodes: usize,
|
||||
}
|
||||
|
||||
#[ctor]
|
||||
/// Labels of chains we consider "first party". These chains allow any
|
||||
/// number of nodes to connect.
|
||||
static FIRST_PARTY_NETWORKS: HashSet<&'static str> = {
|
||||
let mut set = HashSet::new();
|
||||
set.insert("Polkadot");
|
||||
set.insert("Kusama");
|
||||
set.insert("Westend");
|
||||
set.insert("Rococo");
|
||||
set
|
||||
};
|
||||
|
||||
/// Max number of nodes allowed to connect to the telemetry server.
|
||||
const THIRD_PARTY_NETWORKS_MAX_NODES: usize = 500;
|
||||
|
||||
impl Aggregator {
|
||||
pub fn new(denylist: HashSet<String>) -> Self {
|
||||
Aggregator {
|
||||
genesis_hashes: HashMap::new(),
|
||||
labels: HashMap::new(),
|
||||
chains: DenseMap::new(),
|
||||
feeds: DenseMap::new(),
|
||||
serializer: FeedMessageSerializer::new(),
|
||||
denylist,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get an address to the chain actor by name. If the address is not found,
|
||||
/// or the address is disconnected (actor dropped), create a new one.
|
||||
pub fn lazy_chain(
|
||||
&mut self,
|
||||
genesis_hash: BlockHash,
|
||||
label: &str,
|
||||
ctx: &mut <Self as Actor>::Context,
|
||||
) -> ChainId {
|
||||
let cid = match self.genesis_hashes.get(&genesis_hash).copied() {
|
||||
Some(cid) => cid,
|
||||
None => {
|
||||
self.serializer.push(feed::AddedChain(&label, 1));
|
||||
|
||||
let addr = ctx.address();
|
||||
let max_nodes = max_nodes(label);
|
||||
let label: Label = label.into();
|
||||
let cid = self.chains.add_with(|cid| ChainEntry {
|
||||
addr: Chain::new(cid, addr, label.clone()).start(),
|
||||
genesis_hash,
|
||||
label: label.clone(),
|
||||
nodes: 1,
|
||||
max_nodes,
|
||||
});
|
||||
|
||||
self.labels.insert(label, cid);
|
||||
self.genesis_hashes.insert(genesis_hash, cid);
|
||||
|
||||
self.broadcast();
|
||||
|
||||
cid
|
||||
}
|
||||
};
|
||||
|
||||
cid
|
||||
}
|
||||
|
||||
fn get_chain(&mut self, label: &str) -> Option<&mut ChainEntry> {
|
||||
let chains = &mut self.chains;
|
||||
self.labels
|
||||
.get(label)
|
||||
.and_then(move |&cid| chains.get_mut(cid))
|
||||
}
|
||||
|
||||
fn broadcast(&mut self) {
|
||||
if let Some(msg) = self.serializer.finalize() {
|
||||
for (_, feed) in self.feeds.iter() {
|
||||
feed.do_send(msg.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Actor for Aggregator {
|
||||
type Context = Context<Self>;
|
||||
}
|
||||
|
||||
/// Message sent from the NodeConnector to the Aggregator upon getting all node details
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct AddNode {
|
||||
/// Details of the node being added to the aggregator
|
||||
pub node: NodeDetails,
|
||||
/// Genesis [`Hash`] of the chain the node is being added to.
|
||||
pub genesis_hash: BlockHash,
|
||||
/// Source from which this node is being added (Direct | Shard)
|
||||
pub source: NodeSource,
|
||||
}
|
||||
|
||||
pub enum NodeSource {
|
||||
Shard {
|
||||
/// `ShardConnId` that identifies the node connection within a shard.
|
||||
sid: ShardConnId,
|
||||
/// Address to the ShardConnector actor
|
||||
shard_connector: Addr<ShardConnector>,
|
||||
}
|
||||
}
|
||||
|
||||
/// Message sent from the Chain to the Aggregator when the Chain loses all nodes
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct DropChain(pub ChainId);
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct RenameChain(pub ChainId, pub Label);
|
||||
|
||||
/// Message sent from the FeedConnector to the Aggregator when subscribing to a new chain
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "bool")]
|
||||
pub struct Subscribe {
|
||||
pub chain: Label,
|
||||
pub feed: Addr<FeedConnector>,
|
||||
}
|
||||
|
||||
/// Message sent from the FeedConnector to the Aggregator consensus requested
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct SendFinality {
|
||||
pub chain: Label,
|
||||
pub fid: FeedId,
|
||||
}
|
||||
|
||||
/// Message sent from the FeedConnector to the Aggregator no more consensus required
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct NoMoreFinality {
|
||||
pub chain: Label,
|
||||
pub fid: FeedId,
|
||||
}
|
||||
|
||||
/// Message sent from the FeedConnector to the Aggregator when first connected
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Connect(pub Addr<FeedConnector>);
|
||||
|
||||
/// Message sent from the FeedConnector to the Aggregator when disconnecting
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Disconnect(pub FeedId);
|
||||
|
||||
/// Message sent from the Chain to the Aggergator when the node count on the chain changes
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct NodeCount(pub ChainId, pub usize);
|
||||
|
||||
/// Message sent to the Aggregator to get a health check
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "usize")]
|
||||
pub struct GetHealth;
|
||||
|
||||
impl NodeSource {
|
||||
pub fn mute(&self, reason: MuteReason) {
|
||||
match self {
|
||||
// TODO
|
||||
NodeSource::Shard { shard_connector, .. } => {
|
||||
// shard_connector.do_send(Mute { reason });
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<AddNode> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
|
||||
if self.denylist.contains(&*msg.node.chain) {
|
||||
log::warn!(target: "Aggregator::AddNode", "'{}' is on the denylist.", msg.node.chain);
|
||||
|
||||
msg.source.mute(MuteReason::Denied);
|
||||
return;
|
||||
}
|
||||
let AddNode {
|
||||
node,
|
||||
genesis_hash,
|
||||
source,
|
||||
// conn_id,
|
||||
// node_connector,
|
||||
} = msg;
|
||||
log::trace!(target: "Aggregator::AddNode", "New node connected. Chain '{}'", node.chain);
|
||||
|
||||
let cid = self.lazy_chain(genesis_hash, &node.chain, ctx);
|
||||
let chain = self
|
||||
.chains
|
||||
.get_mut(cid)
|
||||
.expect("Entry just created above; qed");
|
||||
if chain.nodes < chain.max_nodes {
|
||||
chain.addr.do_send(chain::AddNode {
|
||||
node,
|
||||
source,
|
||||
});
|
||||
} else {
|
||||
log::warn!(target: "Aggregator::AddNode", "Chain {} is over quota ({})", chain.label, chain.max_nodes);
|
||||
|
||||
source.mute(MuteReason::Overquota);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<DropChain> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: DropChain, _: &mut Self::Context) {
|
||||
let DropChain(cid) = msg;
|
||||
|
||||
if let Some(entry) = self.chains.remove(cid) {
|
||||
let label = &entry.label;
|
||||
self.genesis_hashes.remove(&entry.genesis_hash);
|
||||
self.labels.remove(label);
|
||||
self.serializer.push(feed::RemovedChain(label));
|
||||
log::info!("Dropped chain [{}] from the aggregator", label);
|
||||
self.broadcast();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<RenameChain> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: RenameChain, _: &mut Self::Context) {
|
||||
let RenameChain(cid, new) = msg;
|
||||
|
||||
if let Some(entry) = self.chains.get_mut(cid) {
|
||||
if entry.label == new {
|
||||
return;
|
||||
}
|
||||
|
||||
// Update UI
|
||||
self.serializer.push(feed::RemovedChain(&entry.label));
|
||||
self.serializer.push(feed::AddedChain(&new, entry.nodes));
|
||||
|
||||
// Update labels -> cid map
|
||||
self.labels.remove(&entry.label);
|
||||
self.labels.insert(new.clone(), cid);
|
||||
|
||||
// Update entry
|
||||
entry.label = new;
|
||||
|
||||
self.broadcast();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Subscribe> for Aggregator {
|
||||
type Result = bool;
|
||||
|
||||
fn handle(&mut self, msg: Subscribe, _: &mut Self::Context) -> bool {
|
||||
let Subscribe { chain, feed } = msg;
|
||||
|
||||
if let Some(chain) = self.get_chain(&chain) {
|
||||
chain.addr.do_send(chain::Subscribe(feed));
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<SendFinality> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: SendFinality, _: &mut Self::Context) {
|
||||
let SendFinality { chain, fid } = msg;
|
||||
if let Some(chain) = self.get_chain(&chain) {
|
||||
chain.addr.do_send(chain::SendFinality(fid));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<NoMoreFinality> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: NoMoreFinality, _: &mut Self::Context) {
|
||||
let NoMoreFinality { chain, fid } = msg;
|
||||
if let Some(chain) = self.get_chain(&chain) {
|
||||
chain.addr.do_send(chain::NoMoreFinality(fid));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Connect> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Connect, _: &mut Self::Context) {
|
||||
let Connect(connector) = msg;
|
||||
|
||||
let fid = self.feeds.add(connector.clone());
|
||||
|
||||
log::info!("Feed #{} connected", fid);
|
||||
|
||||
connector.do_send(Connected(fid));
|
||||
|
||||
self.serializer.push(feed::Version(31));
|
||||
|
||||
// TODO: keep track on number of nodes connected to each chain
|
||||
for (_, entry) in self.chains.iter() {
|
||||
self.serializer
|
||||
.push(feed::AddedChain(&entry.label, entry.nodes));
|
||||
}
|
||||
|
||||
if let Some(msg) = self.serializer.finalize() {
|
||||
connector.do_send(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Disconnect> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Disconnect, _: &mut Self::Context) {
|
||||
let Disconnect(fid) = msg;
|
||||
|
||||
log::info!("Feed #{} disconnected", fid);
|
||||
|
||||
self.feeds.remove(fid);
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<NodeCount> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: NodeCount, _: &mut Self::Context) {
|
||||
let NodeCount(cid, count) = msg;
|
||||
|
||||
if let Some(entry) = self.chains.get_mut(cid) {
|
||||
entry.nodes = count;
|
||||
|
||||
if count != 0 {
|
||||
self.serializer.push(feed::AddedChain(&entry.label, count));
|
||||
self.broadcast();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<GetHealth> for Aggregator {
|
||||
type Result = usize;
|
||||
|
||||
fn handle(&mut self, _: GetHealth, _: &mut Self::Context) -> Self::Result {
|
||||
self.chains.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// First party networks (Polkadot, Kusama etc) are allowed any number of nodes.
|
||||
/// Third party networks are allowed `THIRD_PARTY_NETWORKS_MAX_NODES` nodes and
|
||||
/// no more.
|
||||
fn max_nodes(label: &str) -> usize {
|
||||
if FIRST_PARTY_NETWORKS.contains(label) {
|
||||
usize::MAX
|
||||
} else {
|
||||
THIRD_PARTY_NETWORKS_MAX_NODES
|
||||
}
|
||||
}
|
||||
@@ -1,567 +0,0 @@
|
||||
use actix::prelude::*;
|
||||
use rustc_hash::FxHashMap;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::aggregator::{Aggregator, DropChain, NodeCount, NodeSource, RenameChain};
|
||||
use crate::feed::connector::{FeedConnector, FeedId, Subscribed, Unsubscribed};
|
||||
use crate::feed::{self, FeedMessageSerializer};
|
||||
use crate::node::Node;
|
||||
use common::types::{Block, NodeDetails, NodeId, NodeLocation, Timestamp};
|
||||
use common::util::{now, DenseMap, NumStats};
|
||||
use common::node::Payload;
|
||||
|
||||
const STALE_TIMEOUT: u64 = 2 * 60 * 1000; // 2 minutes
|
||||
|
||||
pub type ChainId = usize;
|
||||
pub type Label = Arc<str>;
|
||||
|
||||
pub struct Chain {
|
||||
cid: ChainId,
|
||||
/// Who to inform if the Chain drops itself
|
||||
aggregator: Addr<Aggregator>,
|
||||
/// Label of this chain, along with count of nodes that use this label
|
||||
label: (Label, usize),
|
||||
/// Dense mapping of NodeId -> Node
|
||||
nodes: DenseMap<Node>,
|
||||
/// Dense mapping of FeedId -> Addr<FeedConnector>,
|
||||
feeds: DenseMap<Addr<FeedConnector>>,
|
||||
/// Mapping of FeedId -> Addr<FeedConnector> for feeds requiring finality info,
|
||||
finality_feeds: FxHashMap<FeedId, Addr<FeedConnector>>,
|
||||
/// Best block
|
||||
best: Block,
|
||||
/// Finalized block
|
||||
finalized: Block,
|
||||
/// Block times history, stored so we can calculate averages
|
||||
block_times: NumStats<u64>,
|
||||
/// Calculated average block time
|
||||
average_block_time: Option<u64>,
|
||||
/// Message serializer
|
||||
serializer: FeedMessageSerializer,
|
||||
/// When the best block first arrived
|
||||
timestamp: Option<Timestamp>,
|
||||
/// Some nodes might manifest a different label, note them here
|
||||
labels: HashMap<Label, usize>,
|
||||
}
|
||||
|
||||
impl Chain {
|
||||
pub fn new(cid: ChainId, aggregator: Addr<Aggregator>, label: Label) -> Self {
|
||||
log::info!("[{}] Created", label);
|
||||
|
||||
Chain {
|
||||
cid,
|
||||
aggregator,
|
||||
label: (label, 0),
|
||||
nodes: DenseMap::new(),
|
||||
feeds: DenseMap::new(),
|
||||
finality_feeds: FxHashMap::default(),
|
||||
best: Block::zero(),
|
||||
finalized: Block::zero(),
|
||||
block_times: NumStats::new(50),
|
||||
average_block_time: None,
|
||||
serializer: FeedMessageSerializer::new(),
|
||||
timestamp: None,
|
||||
labels: HashMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn increment_label_count(&mut self, label: &str) {
|
||||
let count = match self.labels.get_mut(label) {
|
||||
Some(count) => {
|
||||
*count += 1;
|
||||
*count
|
||||
}
|
||||
None => {
|
||||
self.labels.insert(label.into(), 1);
|
||||
1
|
||||
}
|
||||
};
|
||||
|
||||
if &*self.label.0 == label {
|
||||
self.label.1 += 1;
|
||||
} else if count > self.label.1 {
|
||||
self.rename(label.into(), count);
|
||||
}
|
||||
}
|
||||
|
||||
fn decrement_label_count(&mut self, label: &str) {
|
||||
match self.labels.get_mut(label) {
|
||||
Some(count) => *count -= 1,
|
||||
None => return,
|
||||
};
|
||||
|
||||
if &*self.label.0 == label {
|
||||
self.label.1 -= 1;
|
||||
|
||||
for (label, &count) in self.labels.iter() {
|
||||
if count > self.label.1 {
|
||||
let label: Arc<_> = label.clone();
|
||||
self.rename(label, count);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rename(&mut self, label: Label, count: usize) {
|
||||
self.label = (label, count);
|
||||
|
||||
self.aggregator
|
||||
.do_send(RenameChain(self.cid, self.label.0.clone()));
|
||||
}
|
||||
|
||||
fn broadcast(&mut self) {
|
||||
if let Some(msg) = self.serializer.finalize() {
|
||||
for (_, feed) in self.feeds.iter() {
|
||||
feed.do_send(msg.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn broadcast_finality(&mut self) {
|
||||
if let Some(msg) = self.serializer.finalize() {
|
||||
for feed in self.finality_feeds.values() {
|
||||
feed.do_send(msg.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Triggered when the number of nodes in this chain has changed, Aggregator will
|
||||
/// propagate new counts to all connected feeds
|
||||
fn update_count(&self) {
|
||||
self.aggregator
|
||||
.do_send(NodeCount(self.cid, self.nodes.len()));
|
||||
}
|
||||
|
||||
/// Check if the chain is stale (has not received a new best block in a while).
|
||||
/// If so, find a new best block, ignoring any stale nodes and marking them as such.
|
||||
fn update_stale_nodes(&mut self, now: u64) {
|
||||
let threshold = now - STALE_TIMEOUT;
|
||||
let timestamp = match self.timestamp {
|
||||
Some(ts) => ts,
|
||||
None => return,
|
||||
};
|
||||
|
||||
if timestamp > threshold {
|
||||
// Timestamp is in range, nothing to do
|
||||
return;
|
||||
}
|
||||
|
||||
let mut best = Block::zero();
|
||||
let mut finalized = Block::zero();
|
||||
let mut timestamp = None;
|
||||
|
||||
for (nid, node) in self.nodes.iter_mut() {
|
||||
if !node.update_stale(threshold) {
|
||||
if node.best().height > best.height {
|
||||
best = *node.best();
|
||||
timestamp = Some(node.best_timestamp());
|
||||
}
|
||||
|
||||
if node.finalized().height > finalized.height {
|
||||
finalized = *node.finalized();
|
||||
}
|
||||
} else {
|
||||
self.serializer.push(feed::StaleNode(nid));
|
||||
}
|
||||
}
|
||||
|
||||
if self.best.height != 0 || self.finalized.height != 0 {
|
||||
self.best = best;
|
||||
self.finalized = finalized;
|
||||
self.block_times.reset();
|
||||
self.timestamp = timestamp;
|
||||
|
||||
self.serializer.push(feed::BestBlock(
|
||||
self.best.height,
|
||||
timestamp.unwrap_or(now),
|
||||
None,
|
||||
));
|
||||
self.serializer
|
||||
.push(feed::BestFinalized(finalized.height, finalized.hash));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Actor for Chain {
|
||||
type Context = Context<Self>;
|
||||
|
||||
fn stopped(&mut self, _: &mut Self::Context) {
|
||||
self.aggregator.do_send(DropChain(self.cid));
|
||||
|
||||
for (_, feed) in self.feeds.iter() {
|
||||
feed.do_send(Unsubscribed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Message sent from the Aggregator to the Chain when new Node is connected
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct AddNode {
|
||||
/// Details of the node being added to the aggregator
|
||||
pub node: NodeDetails,
|
||||
/// Source from which this node is being added (Direct | Shard)
|
||||
pub source: NodeSource,
|
||||
}
|
||||
|
||||
/// Message sent from the NodeConnector to the Chain when it receives new telemetry data
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct UpdateNode {
|
||||
pub nid: NodeId,
|
||||
pub payload: Payload,
|
||||
}
|
||||
|
||||
/// Message sent from the NodeConnector to the Chain when the connector disconnects
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct RemoveNode(pub NodeId);
|
||||
|
||||
/// Message sent from the Aggregator to the Chain when the connector wants to subscribe to that chain
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Subscribe(pub Addr<FeedConnector>);
|
||||
|
||||
/// Message sent from the FeedConnector before it subscribes to a new chain, or if it disconnects
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Unsubscribe(pub FeedId);
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct SendFinality(pub FeedId);
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct NoMoreFinality(pub FeedId);
|
||||
|
||||
/// Message sent from the NodeConnector to the Chain when it receives location data
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct LocateNode {
|
||||
pub nid: NodeId,
|
||||
pub location: Arc<NodeLocation>,
|
||||
}
|
||||
|
||||
impl NodeSource {
|
||||
pub fn init(self, nid: NodeId, chain: Addr<Chain>) -> bool {
|
||||
match self {
|
||||
NodeSource::Shard { sid, shard_connector } => {
|
||||
shard_connector
|
||||
.try_send(crate::shard::connector::Initialize {
|
||||
nid,
|
||||
sid,
|
||||
chain,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<AddNode> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
|
||||
let AddNode {
|
||||
node,
|
||||
source,
|
||||
} = msg;
|
||||
log::trace!(target: "Chain::AddNode", "New node connected. Chain '{}', node count goes from {} to {}", node.chain, self.nodes.len(), self.nodes.len() + 1);
|
||||
self.increment_label_count(&node.chain);
|
||||
|
||||
let nid = self.nodes.add(Node::new(node));
|
||||
let chain = ctx.address();
|
||||
|
||||
if source.init(nid, chain) {
|
||||
self.nodes.remove(nid);
|
||||
} else if let Some(node) = self.nodes.get(nid) {
|
||||
self.serializer.push(feed::AddedNode(nid, node));
|
||||
self.broadcast();
|
||||
}
|
||||
|
||||
self.update_count();
|
||||
}
|
||||
}
|
||||
|
||||
impl Chain {
|
||||
fn handle_block(&mut self, block: &Block, nid: NodeId) {
|
||||
let mut propagation_time = None;
|
||||
let now = now();
|
||||
let nodes_len = self.nodes.len();
|
||||
|
||||
self.update_stale_nodes(now);
|
||||
|
||||
let node = match self.nodes.get_mut(nid) {
|
||||
Some(node) => node,
|
||||
None => return,
|
||||
};
|
||||
|
||||
if node.update_block(*block) {
|
||||
if block.height > self.best.height {
|
||||
self.best = *block;
|
||||
log::debug!(
|
||||
"[{}] [nodes={}/feeds={}] new best block={}/{:?}",
|
||||
self.label.0,
|
||||
nodes_len,
|
||||
self.feeds.len(),
|
||||
self.best.height,
|
||||
self.best.hash,
|
||||
);
|
||||
if let Some(timestamp) = self.timestamp {
|
||||
self.block_times.push(now - timestamp);
|
||||
self.average_block_time = Some(self.block_times.average());
|
||||
}
|
||||
self.timestamp = Some(now);
|
||||
self.serializer.push(feed::BestBlock(
|
||||
self.best.height,
|
||||
now,
|
||||
self.average_block_time,
|
||||
));
|
||||
propagation_time = Some(0);
|
||||
} else if block.height == self.best.height {
|
||||
if let Some(timestamp) = self.timestamp {
|
||||
propagation_time = Some(now - timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(details) = node.update_details(now, propagation_time) {
|
||||
self.serializer.push(feed::ImportedBlock(nid, details));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<UpdateNode> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: UpdateNode, _: &mut Self::Context) {
|
||||
let UpdateNode { nid, payload } = msg;
|
||||
|
||||
if let Some(block) = payload.best_block() {
|
||||
self.handle_block(block, nid);
|
||||
}
|
||||
|
||||
if let Some(node) = self.nodes.get_mut(nid) {
|
||||
match payload {
|
||||
Payload::SystemInterval(ref interval) => {
|
||||
if node.update_hardware(interval) {
|
||||
self.serializer.push(feed::Hardware(nid, node.hardware()));
|
||||
}
|
||||
|
||||
if let Some(stats) = node.update_stats(interval) {
|
||||
self.serializer.push(feed::NodeStatsUpdate(nid, stats));
|
||||
}
|
||||
|
||||
if let Some(io) = node.update_io(interval) {
|
||||
self.serializer.push(feed::NodeIOUpdate(nid, io));
|
||||
}
|
||||
}
|
||||
// Payload::AfgAuthoritySet(authority) => {
|
||||
// node.set_validator_address(authority.authority_id.clone());
|
||||
// self.broadcast();
|
||||
// return;
|
||||
// }
|
||||
// Payload::AfgFinalized(finalized) => {
|
||||
// if let Ok(finalized_number) = finalized.finalized_number.parse::<BlockNumber>()
|
||||
// {
|
||||
// if let Some(addr) = node.details().validator.clone() {
|
||||
// self.serializer.push(feed::AfgFinalized(
|
||||
// addr,
|
||||
// finalized_number,
|
||||
// finalized.finalized_hash,
|
||||
// ));
|
||||
// self.broadcast_finality();
|
||||
// }
|
||||
// }
|
||||
// return;
|
||||
// }
|
||||
// Payload::AfgReceivedPrecommit(precommit) => {
|
||||
// if let Ok(finalized_number) =
|
||||
// precommit.received.target_number.parse::<BlockNumber>()
|
||||
// {
|
||||
// if let Some(addr) = node.details().validator.clone() {
|
||||
// let voter = precommit.received.voter.clone();
|
||||
// self.serializer.push(feed::AfgReceivedPrecommit(
|
||||
// addr,
|
||||
// finalized_number,
|
||||
// precommit.received.target_hash,
|
||||
// voter,
|
||||
// ));
|
||||
// self.broadcast_finality();
|
||||
// }
|
||||
// }
|
||||
// return;
|
||||
// }
|
||||
// Payload::AfgReceivedPrevote(prevote) => {
|
||||
// if let Ok(finalized_number) =
|
||||
// prevote.received.target_number.parse::<BlockNumber>()
|
||||
// {
|
||||
// if let Some(addr) = node.details().validator.clone() {
|
||||
// let voter = prevote.received.voter.clone();
|
||||
// self.serializer.push(feed::AfgReceivedPrevote(
|
||||
// addr,
|
||||
// finalized_number,
|
||||
// prevote.received.target_hash,
|
||||
// voter,
|
||||
// ));
|
||||
// self.broadcast_finality();
|
||||
// }
|
||||
// }
|
||||
// return;
|
||||
// }
|
||||
// Payload::AfgReceivedCommit(_) => {}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
if let Some(block) = payload.finalized_block() {
|
||||
if let Some(finalized) = node.update_finalized(block) {
|
||||
self.serializer.push(feed::FinalizedBlock(
|
||||
nid,
|
||||
finalized.height,
|
||||
finalized.hash,
|
||||
));
|
||||
|
||||
if finalized.height > self.finalized.height {
|
||||
self.finalized = *finalized;
|
||||
self.serializer
|
||||
.push(feed::BestFinalized(finalized.height, finalized.hash));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.broadcast();
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<LocateNode> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: LocateNode, _: &mut Self::Context) {
|
||||
let LocateNode { nid, location } = msg;
|
||||
|
||||
if let Some(node) = self.nodes.get_mut(nid) {
|
||||
self.serializer.push(feed::LocatedNode(
|
||||
nid,
|
||||
location.latitude,
|
||||
location.longitude,
|
||||
&location.city,
|
||||
));
|
||||
|
||||
node.update_location(location);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<RemoveNode> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: RemoveNode, ctx: &mut Self::Context) {
|
||||
let RemoveNode(nid) = msg;
|
||||
|
||||
if let Some(node) = self.nodes.remove(nid) {
|
||||
self.decrement_label_count(&node.details().chain);
|
||||
}
|
||||
|
||||
if self.nodes.is_empty() {
|
||||
log::info!("[{}] Lost all nodes, dropping...", self.label.0);
|
||||
ctx.stop();
|
||||
}
|
||||
|
||||
self.serializer.push(feed::RemovedNode(nid));
|
||||
self.broadcast();
|
||||
self.update_count();
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Subscribe> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Subscribe, ctx: &mut Self::Context) {
|
||||
let Subscribe(feed) = msg;
|
||||
let fid = self.feeds.add(feed.clone());
|
||||
|
||||
feed.do_send(Subscribed(fid, ctx.address().recipient()));
|
||||
|
||||
self.serializer.push(feed::SubscribedTo(&self.label.0));
|
||||
self.serializer.push(feed::TimeSync(now()));
|
||||
self.serializer.push(feed::BestBlock(
|
||||
self.best.height,
|
||||
self.timestamp.unwrap_or(0),
|
||||
self.average_block_time,
|
||||
));
|
||||
self.serializer.push(feed::BestFinalized(
|
||||
self.finalized.height,
|
||||
self.finalized.hash,
|
||||
));
|
||||
|
||||
for (idx, (nid, node)) in self.nodes.iter().enumerate() {
|
||||
// Send subscription confirmation and chain head before doing all the nodes,
|
||||
// and continue sending batches of 32 nodes a time over the wire subsequently
|
||||
if idx % 32 == 0 {
|
||||
if let Some(serialized) = self.serializer.finalize() {
|
||||
feed.do_send(serialized);
|
||||
}
|
||||
}
|
||||
|
||||
self.serializer.push(feed::AddedNode(nid, node));
|
||||
self.serializer.push(feed::FinalizedBlock(
|
||||
nid,
|
||||
node.finalized().height,
|
||||
node.finalized().hash,
|
||||
));
|
||||
if node.stale() {
|
||||
self.serializer.push(feed::StaleNode(nid));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(serialized) = self.serializer.finalize() {
|
||||
feed.do_send(serialized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<SendFinality> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: SendFinality, _ctx: &mut Self::Context) {
|
||||
let SendFinality(fid) = msg;
|
||||
if let Some(feed) = self.feeds.get(fid) {
|
||||
self.finality_feeds.insert(fid, feed.clone());
|
||||
}
|
||||
|
||||
// info!("Added new finality feed {}", fid);
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<NoMoreFinality> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: NoMoreFinality, _: &mut Self::Context) {
|
||||
let NoMoreFinality(fid) = msg;
|
||||
|
||||
// info!("Removed finality feed {}", fid);
|
||||
self.finality_feeds.remove(&fid);
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Unsubscribe> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Unsubscribe, _: &mut Self::Context) {
|
||||
let Unsubscribe(fid) = msg;
|
||||
|
||||
if let Some(feed) = self.feeds.get(fid) {
|
||||
self.serializer.push(feed::UnsubscribedFrom(&self.label.0));
|
||||
|
||||
if let Some(serialized) = self.serializer.finalize() {
|
||||
feed.do_send(serialized);
|
||||
}
|
||||
}
|
||||
|
||||
self.feeds.remove(fid);
|
||||
self.finality_feeds.remove(&fid);
|
||||
}
|
||||
}
|
||||
@@ -1,219 +0,0 @@
|
||||
use crate::aggregator::{Aggregator, Connect, Disconnect, NoMoreFinality, SendFinality, Subscribe};
|
||||
use crate::chain::Unsubscribe;
|
||||
use crate::feed::{FeedMessageSerializer, Pong};
|
||||
use actix::prelude::*;
|
||||
use actix_web_actors::ws;
|
||||
use bytes::Bytes;
|
||||
use common::util::fnv;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub type FeedId = usize;
|
||||
|
||||
/// How often heartbeat pings are sent
|
||||
const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
|
||||
/// How long before lack of client response causes a timeout
|
||||
const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
pub struct FeedConnector {
|
||||
/// FeedId that Aggregator holds of this actor
|
||||
fid_aggregator: FeedId,
|
||||
/// FeedId that Chain holds of this actor
|
||||
fid_chain: FeedId,
|
||||
/// Client must send ping at least once per 10 seconds (CLIENT_TIMEOUT),
|
||||
hb: Instant,
|
||||
/// Aggregator actor address
|
||||
aggregator: Addr<Aggregator>,
|
||||
/// Chain actor address
|
||||
chain: Option<Recipient<Unsubscribe>>,
|
||||
/// FNV hash of the chain label, optimization to avoid double-subscribing
|
||||
chain_label_hash: u64,
|
||||
/// Message serializer
|
||||
serializer: FeedMessageSerializer,
|
||||
}
|
||||
|
||||
impl Actor for FeedConnector {
|
||||
type Context = ws::WebsocketContext<Self>;
|
||||
|
||||
fn started(&mut self, ctx: &mut Self::Context) {
|
||||
self.heartbeat(ctx);
|
||||
self.aggregator.do_send(Connect(ctx.address()));
|
||||
}
|
||||
|
||||
fn stopped(&mut self, _: &mut Self::Context) {
|
||||
if let Some(chain) = self.chain.take() {
|
||||
let _ = chain.do_send(Unsubscribe(self.fid_chain));
|
||||
}
|
||||
|
||||
self.aggregator.do_send(Disconnect(self.fid_aggregator));
|
||||
}
|
||||
}
|
||||
|
||||
impl FeedConnector {
|
||||
pub fn new(aggregator: Addr<Aggregator>) -> Self {
|
||||
Self {
|
||||
// Garbage id, will be replaced by the Connected message
|
||||
fid_aggregator: !0,
|
||||
// Garbage id, will be replaced by the Subscribed message
|
||||
fid_chain: !0,
|
||||
hb: Instant::now(),
|
||||
aggregator,
|
||||
chain: None,
|
||||
chain_label_hash: 0,
|
||||
serializer: FeedMessageSerializer::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
|
||||
ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
|
||||
// check client heartbeats
|
||||
if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
|
||||
// stop actor
|
||||
ctx.stop();
|
||||
} else {
|
||||
ctx.ping(b"")
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_cmd(&mut self, cmd: &str, payload: &str, ctx: &mut <Self as Actor>::Context) {
|
||||
match cmd {
|
||||
"subscribe" => {
|
||||
// Hash the chain label the frontend wants to subscribe to.
|
||||
// If it's already subscribed to the same chain, nothing to do.
|
||||
match fnv(payload) {
|
||||
hash if hash == self.chain_label_hash => return,
|
||||
hash => self.chain_label_hash = hash,
|
||||
}
|
||||
|
||||
self.aggregator
|
||||
.send(Subscribe {
|
||||
chain: payload.into(),
|
||||
feed: ctx.address(),
|
||||
})
|
||||
.into_actor(self)
|
||||
.then(|res, actor, _| {
|
||||
match res {
|
||||
Ok(true) => (),
|
||||
// Chain not found, reset hash
|
||||
_ => actor.chain_label_hash = 0,
|
||||
}
|
||||
async {}.into_actor(actor)
|
||||
})
|
||||
.wait(ctx);
|
||||
}
|
||||
"send-finality" => {
|
||||
self.aggregator.do_send(SendFinality {
|
||||
chain: payload.into(),
|
||||
fid: self.fid_chain,
|
||||
});
|
||||
}
|
||||
"no-more-finality" => {
|
||||
self.aggregator.do_send(NoMoreFinality {
|
||||
chain: payload.into(),
|
||||
fid: self.fid_chain,
|
||||
});
|
||||
}
|
||||
"ping" => {
|
||||
self.serializer.push(Pong(payload));
|
||||
if let Some(serialized) = self.serializer.finalize() {
|
||||
ctx.binary(serialized.0);
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Message sent form Chain to the FeedConnector upon successful subscription
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Subscribed(pub FeedId, pub Recipient<Unsubscribe>);
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Unsubscribed;
|
||||
|
||||
/// Message sent from Aggregator to FeedConnector upon successful connection
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Connected(pub FeedId);
|
||||
|
||||
/// Message sent from either Aggregator or Chain to FeedConnector containing
|
||||
/// serialized message(s) for the frontend
|
||||
///
|
||||
/// Since Bytes is ARC'ed, this is cheap to clone
|
||||
#[derive(Message, Clone)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Serialized(pub Bytes);
|
||||
|
||||
impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for FeedConnector {
|
||||
fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
|
||||
match msg {
|
||||
Ok(ws::Message::Ping(msg)) => {
|
||||
self.hb = Instant::now();
|
||||
ctx.pong(&msg);
|
||||
}
|
||||
Ok(ws::Message::Pong(_)) => self.hb = Instant::now(),
|
||||
Ok(ws::Message::Text(text)) => {
|
||||
if let Some(idx) = text.find(':') {
|
||||
let cmd = &text[..idx];
|
||||
let payload = &text[idx + 1..];
|
||||
|
||||
log::info!("New FEED message: {}", cmd);
|
||||
|
||||
self.handle_cmd(cmd, payload, ctx);
|
||||
}
|
||||
}
|
||||
Ok(ws::Message::Close(_)) => ctx.stop(),
|
||||
Ok(_) => (),
|
||||
Err(error) => {
|
||||
log::error!("{:?}", error);
|
||||
ctx.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Subscribed> for FeedConnector {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Subscribed, _: &mut Self::Context) {
|
||||
let Subscribed(fid_chain, chain) = msg;
|
||||
|
||||
if let Some(current) = self.chain.take() {
|
||||
let _ = current.do_send(Unsubscribe(self.fid_chain));
|
||||
}
|
||||
|
||||
self.fid_chain = fid_chain;
|
||||
self.chain = Some(chain);
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Unsubscribed> for FeedConnector {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, _: Unsubscribed, _: &mut Self::Context) {
|
||||
self.chain = None;
|
||||
self.chain_label_hash = 0;
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Connected> for FeedConnector {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Connected, _: &mut Self::Context) {
|
||||
let Connected(fid_aggregator) = msg;
|
||||
|
||||
self.fid_aggregator = fid_aggregator;
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<Serialized> for FeedConnector {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Serialized, ctx: &mut Self::Context) {
|
||||
let Serialized(bytes) = msg;
|
||||
|
||||
ctx.binary(bytes);
|
||||
}
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
use std::net::Ipv4Addr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix::prelude::*;
|
||||
use parking_lot::RwLock;
|
||||
use rustc_hash::FxHashMap;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::chain::{Chain, LocateNode};
|
||||
use common::types::{NodeId, NodeLocation};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Locator {
|
||||
client: reqwest::blocking::Client,
|
||||
cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
|
||||
}
|
||||
|
||||
pub struct LocatorFactory {
|
||||
cache: Arc<RwLock<FxHashMap<Ipv4Addr, Option<Arc<NodeLocation>>>>>,
|
||||
}
|
||||
|
||||
impl LocatorFactory {
|
||||
pub fn new() -> Self {
|
||||
let mut cache = FxHashMap::default();
|
||||
|
||||
// Default entry for localhost
|
||||
cache.insert(
|
||||
Ipv4Addr::new(127, 0, 0, 1),
|
||||
Some(Arc::new(NodeLocation {
|
||||
latitude: 52.516_6667,
|
||||
longitude: 13.4,
|
||||
city: "Berlin".into(),
|
||||
})),
|
||||
);
|
||||
|
||||
LocatorFactory {
|
||||
cache: Arc::new(RwLock::new(cache)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create(&self) -> Locator {
|
||||
Locator {
|
||||
client: reqwest::blocking::Client::new(),
|
||||
cache: self.cache.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Actor for Locator {
|
||||
type Context = SyncContext<Self>;
|
||||
}
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct LocateRequest {
|
||||
pub ip: Ipv4Addr,
|
||||
pub nid: NodeId,
|
||||
pub chain: Addr<Chain>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct IPApiLocate {
|
||||
city: Box<str>,
|
||||
loc: Box<str>,
|
||||
}
|
||||
|
||||
impl IPApiLocate {
|
||||
fn into_node_location(self) -> Option<NodeLocation> {
|
||||
let IPApiLocate { city, loc } = self;
|
||||
|
||||
let mut loc = loc.split(',').map(|n| n.parse());
|
||||
|
||||
let latitude = loc.next()?.ok()?;
|
||||
let longitude = loc.next()?.ok()?;
|
||||
|
||||
// Guarantee that the iterator has been exhausted
|
||||
if loc.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(NodeLocation {
|
||||
latitude,
|
||||
longitude,
|
||||
city,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<LocateRequest> for Locator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: LocateRequest, _: &mut Self::Context) {
|
||||
let LocateRequest { ip, nid, chain } = msg;
|
||||
|
||||
if let Some(item) = self.cache.read().get(&ip) {
|
||||
if let Some(location) = item {
|
||||
return chain.do_send(LocateNode {
|
||||
nid,
|
||||
location: location.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
let location = match self.iplocate(ip) {
|
||||
Ok(location) => location,
|
||||
Err(err) => return log::debug!("GET error for ip location: {:?}", err),
|
||||
};
|
||||
|
||||
self.cache.write().insert(ip, location.clone());
|
||||
|
||||
if let Some(location) = location {
|
||||
chain.do_send(LocateNode { nid, location });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Locator {
|
||||
fn iplocate(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
|
||||
let location = self.iplocate_ipapi_co(ip)?;
|
||||
|
||||
match location {
|
||||
Some(location) => Ok(Some(location)),
|
||||
None => self.iplocate_ipinfo_io(ip),
|
||||
}
|
||||
}
|
||||
|
||||
fn iplocate_ipapi_co(&self, ip: Ipv4Addr) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
|
||||
let location = self
|
||||
.query(&format!("https://ipapi.co/{}/json", ip))?
|
||||
.map(Arc::new);
|
||||
|
||||
Ok(location)
|
||||
}
|
||||
|
||||
fn iplocate_ipinfo_io(
|
||||
&self,
|
||||
ip: Ipv4Addr,
|
||||
) -> Result<Option<Arc<NodeLocation>>, reqwest::Error> {
|
||||
let location = self
|
||||
.query(&format!("https://ipinfo.io/{}/json", ip))?
|
||||
.and_then(|loc: IPApiLocate| loc.into_node_location().map(Arc::new));
|
||||
|
||||
Ok(location)
|
||||
}
|
||||
|
||||
fn query<T>(&self, url: &str) -> Result<Option<T>, reqwest::Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
match self.client.get(url).send()?.json::<T>() {
|
||||
Ok(result) => Ok(Some(result)),
|
||||
Err(err) => {
|
||||
log::debug!("JSON error for ip location: {:?}", err);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ipapi_locate_to_node_location() {
|
||||
let ipapi = IPApiLocate {
|
||||
loc: "12.5,56.25".into(),
|
||||
city: "Foobar".into(),
|
||||
};
|
||||
|
||||
let location = ipapi.into_node_location().unwrap();
|
||||
|
||||
assert_eq!(location.latitude, 12.5);
|
||||
assert_eq!(location.longitude, 56.25);
|
||||
assert_eq!(&*location.city, "Foobar");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipapi_locate_to_node_location_too_many() {
|
||||
let ipapi = IPApiLocate {
|
||||
loc: "12.5,56.25,1.0".into(),
|
||||
city: "Foobar".into(),
|
||||
};
|
||||
|
||||
let location = ipapi.into_node_location();
|
||||
|
||||
assert!(location.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::iter::FromIterator;
|
||||
|
||||
use actix::prelude::*;
|
||||
use actix_http::ws::Codec;
|
||||
use actix_web::{get, middleware, web, App, Error, HttpRequest, HttpResponse, HttpServer};
|
||||
use actix_web_actors::ws;
|
||||
use clap::Clap;
|
||||
use simple_logger::SimpleLogger;
|
||||
|
||||
mod aggregator;
|
||||
mod chain;
|
||||
mod feed;
|
||||
mod location;
|
||||
mod node;
|
||||
mod shard;
|
||||
|
||||
use aggregator::{Aggregator, GetHealth};
|
||||
use feed::connector::FeedConnector;
|
||||
use location::{Locator, LocatorFactory};
|
||||
use shard::connector::ShardConnector;
|
||||
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
|
||||
const NAME: &str = "Substrate Telemetry Backend Core";
|
||||
const ABOUT: &str = "This is the Telemetry Backend Core that injects and provide the data sent by Substrate/Polkadot nodes";
|
||||
|
||||
#[derive(Clap, Debug)]
|
||||
#[clap(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
|
||||
struct Opts {
|
||||
#[clap(
|
||||
short = 'l',
|
||||
long = "listen",
|
||||
default_value = "127.0.0.1:8000",
|
||||
about = "This is the socket address Telemetry is listening to. This is restricted to localhost (127.0.0.1) by default and should be fine for most use cases. If you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'"
|
||||
)]
|
||||
socket: std::net::SocketAddr,
|
||||
#[clap(
|
||||
required = false,
|
||||
long = "denylist",
|
||||
about = "Space delimited list of chains that are not allowed to connect to telemetry. Case sensitive."
|
||||
)]
|
||||
denylist: Vec<String>,
|
||||
#[clap(
|
||||
arg_enum,
|
||||
required = false,
|
||||
long = "log",
|
||||
default_value = "info",
|
||||
about = "Log level."
|
||||
)]
|
||||
log_level: LogLevel,
|
||||
}
|
||||
|
||||
#[derive(Clap, Debug, PartialEq)]
|
||||
enum LogLevel {
|
||||
Error,
|
||||
Warn,
|
||||
Info,
|
||||
Debug,
|
||||
Trace,
|
||||
}
|
||||
|
||||
impl From<&LogLevel> for log::LevelFilter {
|
||||
fn from(log_level: &LogLevel) -> Self {
|
||||
match log_level {
|
||||
LogLevel::Error => log::LevelFilter::Error,
|
||||
LogLevel::Warn => log::LevelFilter::Warn,
|
||||
LogLevel::Info => log::LevelFilter::Info,
|
||||
LogLevel::Debug => log::LevelFilter::Debug,
|
||||
LogLevel::Trace => log::LevelFilter::Trace,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[get("/shard_submit/{chain_hash}")]
|
||||
async fn shard_route(
|
||||
req: HttpRequest,
|
||||
stream: web::Payload,
|
||||
aggregator: web::Data<Addr<Aggregator>>,
|
||||
locator: web::Data<Addr<Locator>>,
|
||||
path: web::Path<Box<str>>,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let hash_str = path.into_inner();
|
||||
let genesis_hash = hash_str.parse::<common::json::Hash>()?.into();
|
||||
|
||||
println!("Genesis hash {}", genesis_hash);
|
||||
|
||||
let mut res = ws::handshake(&req)?;
|
||||
|
||||
let aggregator = aggregator.get_ref().clone();
|
||||
let locator = locator.get_ref().clone().recipient();
|
||||
|
||||
Ok(res.streaming(ws::WebsocketContext::with_codec(
|
||||
ShardConnector::new(aggregator, locator, genesis_hash),
|
||||
stream,
|
||||
Codec::new().max_size(10 * 1024 * 1024), // 10mb frame limit
|
||||
)))
|
||||
}
|
||||
|
||||
/// Entry point for connecting feeds
|
||||
#[get("/feed")]
|
||||
async fn feed_route(
|
||||
req: HttpRequest,
|
||||
stream: web::Payload,
|
||||
aggregator: web::Data<Addr<Aggregator>>,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let aggregator = aggregator.get_ref().clone();
|
||||
|
||||
ws::start(
|
||||
FeedConnector::new(aggregator),
|
||||
&req,
|
||||
stream,
|
||||
)
|
||||
}
|
||||
|
||||
/// Entry point for health check monitoring bots
|
||||
#[get("/health")]
|
||||
async fn health(aggregator: web::Data<Addr<Aggregator>>) -> Result<HttpResponse, Error> {
|
||||
match aggregator.send(GetHealth).await {
|
||||
Ok(count) => {
|
||||
let body = format!("Connected chains: {}", count);
|
||||
|
||||
HttpResponse::Ok().body(body).await
|
||||
}
|
||||
Err(error) => {
|
||||
log::error!("Health check mailbox error: {:?}", error);
|
||||
|
||||
HttpResponse::InternalServerError().await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Telemetry entry point. Listening by default on 127.0.0.1:8000.
|
||||
/// This can be changed using the `PORT` and `BIND` ENV variables.
|
||||
#[actix_web::main]
|
||||
async fn main() -> std::io::Result<()> {
|
||||
let opts = Opts::parse();
|
||||
let log_level = &opts.log_level;
|
||||
SimpleLogger::new()
|
||||
.with_level(log_level.into())
|
||||
.init()
|
||||
.expect("Must be able to start a logger");
|
||||
|
||||
let denylist = HashSet::from_iter(opts.denylist);
|
||||
let aggregator = Aggregator::new(denylist).start();
|
||||
let factory = LocatorFactory::new();
|
||||
let locator = SyncArbiter::start(4, move || factory.create());
|
||||
log::info!("Starting Telemetry Core version: {}", env!("CARGO_PKG_VERSION"));
|
||||
HttpServer::new(move || {
|
||||
App::new()
|
||||
.wrap(middleware::NormalizePath::default())
|
||||
.data(aggregator.clone())
|
||||
.data(locator.clone())
|
||||
.service(feed_route)
|
||||
.service(shard_route)
|
||||
.service(health)
|
||||
})
|
||||
.bind(opts.socket)?
|
||||
.run()
|
||||
.await
|
||||
}
|
||||
@@ -1,192 +0,0 @@
|
||||
use std::time::{Duration, Instant};
|
||||
use std::collections::BTreeMap;
|
||||
use std::net::Ipv4Addr;
|
||||
|
||||
use crate::aggregator::{AddNode, Aggregator, NodeSource};
|
||||
use crate::chain::{Chain, RemoveNode, UpdateNode};
|
||||
use crate::location::LocateRequest;
|
||||
use actix::prelude::*;
|
||||
use actix_web_actors::ws::{self, CloseReason};
|
||||
use bincode::Options;
|
||||
use common::types::{BlockHash, NodeId};
|
||||
use common::ws::{MultipartHandler, WsMessage};
|
||||
use common::shard::{ShardMessage, ShardConnId, BackendMessage};
|
||||
|
||||
/// How often heartbeat pings are sent
|
||||
const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
|
||||
/// How long before lack of client response causes a timeout
|
||||
const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
pub struct ShardConnector {
|
||||
/// Client must send ping at least once every 60 seconds (CLIENT_TIMEOUT),
|
||||
hb: Instant,
|
||||
/// Aggregator actor address
|
||||
aggregator: Addr<Aggregator>,
|
||||
/// Genesis hash of the chain this connection will be submitting data for
|
||||
genesis_hash: BlockHash,
|
||||
/// Chain address to which this shard connector is delegating messages
|
||||
chain: Option<Addr<Chain>>,
|
||||
/// Transient mapping of `ShardConnId` to external IP address.
|
||||
ips: BTreeMap<ShardConnId, Ipv4Addr>,
|
||||
/// Mapping of `ShardConnId` to initialized `NodeId`s.
|
||||
nodes: BTreeMap<ShardConnId, NodeId>,
|
||||
/// Actix address of location services
|
||||
locator: Recipient<LocateRequest>,
|
||||
/// Container for handling continuation messages
|
||||
multipart: MultipartHandler,
|
||||
}
|
||||
|
||||
impl Actor for ShardConnector {
|
||||
type Context = ws::WebsocketContext<Self>;
|
||||
|
||||
fn started(&mut self, ctx: &mut Self::Context) {
|
||||
self.heartbeat(ctx);
|
||||
}
|
||||
|
||||
fn stopped(&mut self, _: &mut Self::Context) {
|
||||
if let Some(ref chain) = self.chain {
|
||||
for nid in self.nodes.values() {
|
||||
chain.do_send(RemoveNode(*nid))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ShardConnector {
|
||||
pub fn new(
|
||||
aggregator: Addr<Aggregator>,
|
||||
locator: Recipient<LocateRequest>,
|
||||
genesis_hash: BlockHash,
|
||||
) -> Self {
|
||||
Self {
|
||||
hb: Instant::now(),
|
||||
aggregator,
|
||||
genesis_hash,
|
||||
chain: None,
|
||||
ips: BTreeMap::new(),
|
||||
nodes: BTreeMap::new(),
|
||||
locator,
|
||||
multipart: MultipartHandler::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn shard_send(msg: BackendMessage, ctx: &mut <Self as Actor>::Context) {
|
||||
let bytes = bincode::options().serialize(&msg).expect("Must be able to serialize to vec; qed");
|
||||
|
||||
println!("Sending back {} bytes", bytes.len());
|
||||
|
||||
ctx.binary(bytes);
|
||||
}
|
||||
|
||||
fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
|
||||
ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
|
||||
// check client heartbeats
|
||||
if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
|
||||
// stop actor
|
||||
ctx.close(Some(CloseReason {
|
||||
code: ws::CloseCode::Abnormal,
|
||||
description: Some("Missed heartbeat".into()),
|
||||
}));
|
||||
ctx.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_message(&mut self, msg: ShardMessage, ctx: &mut <Self as Actor>::Context) {
|
||||
println!("{:?}", msg);
|
||||
|
||||
match msg {
|
||||
ShardMessage::AddNode { ip, node, sid } => {
|
||||
if let Some(ip) = ip {
|
||||
self.ips.insert(sid, ip);
|
||||
}
|
||||
|
||||
self.aggregator.do_send(AddNode {
|
||||
node,
|
||||
genesis_hash: self.genesis_hash,
|
||||
source: NodeSource::Shard {
|
||||
sid,
|
||||
shard_connector: ctx.address(),
|
||||
}
|
||||
});
|
||||
},
|
||||
ShardMessage::UpdateNode { nid, payload } => {
|
||||
if let Some(chain) = self.chain.as_ref() {
|
||||
chain.do_send(UpdateNode {
|
||||
nid,
|
||||
payload,
|
||||
});
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Initialize {
|
||||
pub nid: NodeId,
|
||||
pub sid: ShardConnId,
|
||||
pub chain: Addr<Chain>,
|
||||
}
|
||||
|
||||
impl Handler<Initialize> for ShardConnector {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Initialize, ctx: &mut Self::Context) {
|
||||
let Initialize {
|
||||
nid,
|
||||
sid,
|
||||
chain,
|
||||
} = msg;
|
||||
log::trace!(target: "ShardConnector::Initialize", "Initializing a node, nid={}, on conn_id={}", nid, 0);
|
||||
|
||||
if self.chain.is_none() {
|
||||
self.chain = Some(chain.clone());
|
||||
}
|
||||
|
||||
let be_msg = BackendMessage::Initialize { sid, nid };
|
||||
|
||||
Self::shard_send(be_msg, ctx);
|
||||
|
||||
// Acquire the node's physical location
|
||||
if let Some(ip) = self.ips.remove(&sid) {
|
||||
let _ = self.locator.do_send(LocateRequest { ip, nid, chain });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for ShardConnector {
|
||||
fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
|
||||
self.hb = Instant::now();
|
||||
|
||||
let data = match msg.map(|msg| self.multipart.handle(msg)) {
|
||||
Ok(WsMessage::Nop) => return,
|
||||
Ok(WsMessage::Ping(msg)) => {
|
||||
ctx.pong(&msg);
|
||||
return;
|
||||
}
|
||||
Ok(WsMessage::Data(data)) => data,
|
||||
Ok(WsMessage::Close(reason)) => {
|
||||
ctx.close(reason);
|
||||
ctx.stop();
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
log::error!("{:?}", error);
|
||||
ctx.stop();
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match bincode::options().deserialize(&data) {
|
||||
Ok(msg) => self.handle_message(msg, ctx),
|
||||
// #[cfg(debug)]
|
||||
Err(err) => {
|
||||
log::warn!("Failed to parse shard message: {}", err,)
|
||||
}
|
||||
// #[cfg(not(debug))]
|
||||
// Err(_) => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
pub mod connector;
|
||||
+13
-15
@@ -5,22 +5,20 @@ authors = ["Parity Technologies Ltd. <admin@parity.io>"]
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
actix = "0.11.1"
|
||||
actix-web = { version = "4.0.0-beta.4", default-features = false }
|
||||
actix-web-actors = "4.0.0-beta.3"
|
||||
actix-http = "3.0.0-beta.4"
|
||||
anyhow = "1.0.40"
|
||||
anyhow = "1.0.41"
|
||||
bincode = "1.3.3"
|
||||
bytes = "1.0.1"
|
||||
clap = "3.0.0-beta.2"
|
||||
log = "0.4"
|
||||
rustc-hash = "1.1.0"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = { version = "1.0", features = ["raw_value"] }
|
||||
primitive-types = { version = "0.9.0", features = ["serde"] }
|
||||
common = { path = "../common" }
|
||||
futures = "0.3.15"
|
||||
hex = "0.4.3"
|
||||
http = "0.2.4"
|
||||
log = "0.4.14"
|
||||
primitive-types = { version = "0.9.0", features = ["serde"] }
|
||||
serde = { version = "1.0.126", features = ["derive"] }
|
||||
serde_json = "1.0.64"
|
||||
simple_logger = "1.11.0"
|
||||
soketto = "0.4.2"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
soketto = "0.6.0"
|
||||
structopt = "0.3.21"
|
||||
thiserror = "1.0.25"
|
||||
tokio = { version = "1.7.0", features = ["full"] }
|
||||
tokio-util = { version = "0.6", features = ["compat"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
warp = "0.3.1"
|
||||
|
||||
+185
-211
@@ -1,239 +1,213 @@
|
||||
use std::net::Ipv4Addr;
|
||||
use std::fmt;
|
||||
// use std::sync::mpsc::{self, Sender};
|
||||
use common::{internal_messages::{self, LocalId}, node};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use futures::{channel::mpsc, future};
|
||||
use futures::{ Sink, SinkExt, StreamExt };
|
||||
use std::collections::{ HashMap, HashSet };
|
||||
use crate::connection::{ create_ws_connection, Message };
|
||||
|
||||
use actix::prelude::*;
|
||||
use actix_http::http::Uri;
|
||||
use bincode::Options;
|
||||
use rustc_hash::FxHashMap;
|
||||
use common::util::{DenseMap};
|
||||
use common::types::{ConnId, NodeDetails, NodeId, BlockHash};
|
||||
use common::node::Payload;
|
||||
use common::shard::{ShardConnId, ShardMessage, BackendMessage};
|
||||
use common::json;
|
||||
use soketto::handshake::{Client, ServerResponse};
|
||||
use crate::node::{NodeConnector, Initialize};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::mpsc::{self, UnboundedSender};
|
||||
use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
|
||||
/// A unique Id is assigned per websocket connection (or more accurately,
|
||||
/// per thing-that-subscribes-to-the-aggregator). That connection might send
|
||||
/// data on behalf of multiple chains, so this ID is local to the aggregator,
|
||||
/// and a unique ID is assigned per batch of data too ([`internal_messages::LocalId`]).
|
||||
type ConnId = u64;
|
||||
|
||||
type WsSender = soketto::Sender<Compat<TcpStream>>;
|
||||
type WsReceiver = soketto::Receiver<Compat<TcpStream>>;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Aggregator {
|
||||
url: Uri,
|
||||
chains: FxHashMap<BlockHash, UnboundedSender<ChainMessage>>,
|
||||
/// Incoming messages are either from websocket connections or
|
||||
/// from the telemetry core. This can be private since the only
|
||||
/// external messages are via subscriptions that take
|
||||
/// [`FromWebsocket`] instances.
|
||||
#[derive(Clone,Debug)]
|
||||
enum ToAggregator {
|
||||
DisconnectedFromTelemetryCore,
|
||||
ConnectedToTelemetryCore,
|
||||
FromWebsocket(ConnId, FromWebsocket),
|
||||
FromTelemetryCore(internal_messages::FromTelemetryCore)
|
||||
}
|
||||
|
||||
impl Actor for Aggregator {
|
||||
type Context = Context<Self>;
|
||||
/// An incoming socket connection can provide these messages.
|
||||
/// Until a node has been Added via [`FromWebsocket::Add`],
|
||||
/// messages from it will be ignored.
|
||||
#[derive(Clone,Debug)]
|
||||
pub enum FromWebsocket {
|
||||
/// Tell the aggregator about a new node.
|
||||
Add {
|
||||
message_id: node::NodeMessageId,
|
||||
ip: Option<std::net::IpAddr>,
|
||||
node: common::types::NodeDetails,
|
||||
/// When a message is sent back up this channel, we terminate
|
||||
/// the websocket connection and force the node to reconnect
|
||||
/// so that it sends its system info again incase the telemetry
|
||||
/// core has restarted.
|
||||
close_connection: mpsc::Sender<()>
|
||||
},
|
||||
/// Update/pass through details about a node.
|
||||
Update {
|
||||
message_id: node::NodeMessageId,
|
||||
payload: node::Payload
|
||||
}
|
||||
}
|
||||
|
||||
pub type FromAggregator = internal_messages::FromShardAggregator;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Aggregator(Arc<AggregatorInternal>);
|
||||
|
||||
struct AggregatorInternal {
|
||||
/// Nodes that connect are each assigned a unique connection ID. Nodes
|
||||
/// can send messages on behalf of more than one chain, and so this ID is
|
||||
/// only really used inside the Aggregator in conjunction with a per-message
|
||||
/// ID.
|
||||
conn_id: AtomicU64,
|
||||
/// Send messages to the aggregator from websockets via this. This is
|
||||
/// stored here so that anybody holding an `Aggregator` handle can
|
||||
/// make use of it.
|
||||
tx_to_aggregator: mpsc::Sender<ToAggregator>
|
||||
}
|
||||
|
||||
impl Aggregator {
|
||||
pub fn new(url: Uri) -> Self {
|
||||
Aggregator {
|
||||
url,
|
||||
chains: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Spawn a new Aggregator. This connects to the telemetry backend
|
||||
pub async fn spawn(telemetry_uri: http::Uri) -> anyhow::Result<Aggregator> {
|
||||
let (tx_to_aggregator, rx_from_external) = mpsc::channel(10);
|
||||
|
||||
pub struct Chain {
|
||||
/// Base URL of Backend Core
|
||||
url: Uri,
|
||||
/// Genesis hash of the chain, required to construct the URL to connect to the Backend Core
|
||||
genesis_hash: BlockHash,
|
||||
/// Dense mapping of SharedConnId -> Addr<NodeConnector> + multiplexing ConnId sent from the node.
|
||||
nodes: DenseMap<(Addr<NodeConnector>, ConnId)>,
|
||||
}
|
||||
|
||||
impl Chain {
|
||||
pub fn new(url: Uri, genesis_hash: BlockHash) -> Self {
|
||||
Chain {
|
||||
url,
|
||||
genesis_hash,
|
||||
nodes: DenseMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spawn(mut self) -> UnboundedSender<ChainMessage> {
|
||||
let (tx_ret, mut rx) = mpsc::unbounded_channel();
|
||||
|
||||
let tx = tx_ret.clone();
|
||||
|
||||
tokio::task::spawn(async move {
|
||||
let mut sender = match self.connect(tx.clone()).await {
|
||||
Ok(pair) => pair,
|
||||
Err(err) => {
|
||||
log::error!("Failed to connect to Backend Core: {:?}", err);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// tokio::task::spawn(async move {
|
||||
|
||||
// });
|
||||
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Some(ChainMessage::AddNode(msg)) => {
|
||||
println!("Add node {:?}", msg);
|
||||
|
||||
let AddNode { node, ip, conn_id, node_connector, .. } = msg;
|
||||
let sid = self.nodes.add((node_connector, conn_id)) as ShardConnId;
|
||||
|
||||
let bytes = bincode::options().serialize(&ShardMessage::AddNode {
|
||||
ip,
|
||||
node,
|
||||
sid,
|
||||
}).unwrap();
|
||||
|
||||
println!("Sending {} bytes", bytes.len());
|
||||
|
||||
let _ = sender.send_binary_mut(bytes).await;
|
||||
let _ = sender.flush().await;
|
||||
},
|
||||
Some(ChainMessage::UpdateNode(nid, payload)) => {
|
||||
let msg = ShardMessage::UpdateNode {
|
||||
nid,
|
||||
payload,
|
||||
};
|
||||
|
||||
let bytes = bincode::options().serialize(&msg).unwrap();
|
||||
|
||||
println!("Sending update: {} bytes", bytes.len());
|
||||
|
||||
let _ = sender.send_binary_mut(bytes).await;
|
||||
let _ = sender.flush().await;
|
||||
},
|
||||
Some(ChainMessage::Backend(BackendMessage::Initialize { sid, nid })) => {
|
||||
if let Some((addr, conn_id)) = self.nodes.get(sid as usize) {
|
||||
addr.do_send(Initialize {
|
||||
nid,
|
||||
conn_id: *conn_id,
|
||||
chain: tx.clone(),
|
||||
// Map responses from our connection into messages that will be sent to the aggregator:
|
||||
let tx_from_connection = tx_to_aggregator.clone().with(|msg| {
|
||||
future::ok::<_,mpsc::SendError>(match msg {
|
||||
Message::Connected => ToAggregator::ConnectedToTelemetryCore,
|
||||
Message::Disconnected => ToAggregator::DisconnectedFromTelemetryCore,
|
||||
Message::Data(data) => ToAggregator::FromTelemetryCore(data)
|
||||
})
|
||||
}
|
||||
},
|
||||
Some(ChainMessage::Backend(BackendMessage::Mute { sid, reason })) => {
|
||||
// TODO
|
||||
},
|
||||
None => (),
|
||||
}
|
||||
}
|
||||
// let mut client = Client::new(socket.compat(), host, &path);
|
||||
|
||||
// let (mut sender, mut receiver) = match client.handshake().await? {
|
||||
// ServerResponse::Accepted { .. } => client.into_builder().finish(),
|
||||
// ServerResponse::Redirect { status_code, location } => unimplemented!("follow location URL"),
|
||||
// ServerResponse::Rejected { status_code } => unimplemented!("handle failure")
|
||||
// };
|
||||
});
|
||||
|
||||
tx_ret
|
||||
// Establish a resiliant connection to the core (this retries as needed):
|
||||
let tx_to_telemetry_core = create_ws_connection(
|
||||
tx_from_connection,
|
||||
telemetry_uri
|
||||
).await;
|
||||
|
||||
// Handle any incoming messages in our handler loop:
|
||||
tokio::spawn(Aggregator::handle_messages(rx_from_external, tx_to_telemetry_core));
|
||||
|
||||
// Return a handle to our aggregator:
|
||||
Ok(Aggregator(Arc::new(AggregatorInternal {
|
||||
conn_id: AtomicU64::new(1),
|
||||
tx_to_aggregator,
|
||||
})))
|
||||
}
|
||||
|
||||
pub async fn connect(&self, tx: UnboundedSender<ChainMessage>) -> anyhow::Result<WsSender> {
|
||||
let host = self.url.host().unwrap_or("127.0.0.1");
|
||||
let port = self.url.port_u16().unwrap_or(8000);
|
||||
let json_hash: json::Hash = self.genesis_hash.into();
|
||||
let path = format!("{}{}", self.url.path(), json_hash);
|
||||
// This is spawned into a separate task and handles any messages coming
|
||||
// in to the aggregator. If nobody is tolding the tx side of the channel
|
||||
// any more, this task will gracefully end.
|
||||
async fn handle_messages(mut rx_from_external: mpsc::Receiver<ToAggregator>, mut tx_to_telemetry_core: mpsc::Sender<FromAggregator>) {
|
||||
use internal_messages::{ FromShardAggregator, FromTelemetryCore };
|
||||
|
||||
let socket = TcpStream::connect((host, port)).await?;
|
||||
let mut next_local_id: LocalId = 1;
|
||||
|
||||
socket.set_nodelay(true).unwrap();
|
||||
// Just as an optimisation, we can keep track of whether we're connected to the backend
|
||||
// or not, and ignore incoming messages while we aren't.
|
||||
let mut connected_to_telemetry_core = false;
|
||||
|
||||
let mut client = Client::new(socket.compat(), host, &path);
|
||||
// A list of close channels for the current connections. Send an empty tuple to
|
||||
// these to ask the connections to be closed.
|
||||
let mut close_connections: Vec<mpsc::Sender<()>> = vec![];
|
||||
|
||||
let (sender, receiver) = match client.handshake().await? {
|
||||
ServerResponse::Accepted { .. } => client.into_builder().finish(),
|
||||
ServerResponse::Redirect { status_code, .. } |
|
||||
ServerResponse::Rejected { status_code } => {
|
||||
return Err(anyhow::anyhow!("Failed to connect, status code: {}", status_code));
|
||||
// Maintain mappings from the connection ID and node message ID to the "local ID" which we
|
||||
// broadcast to the telemetry core.
|
||||
let mut to_local_id: HashMap<(ConnId, node::NodeMessageId), LocalId> = HashMap::new();
|
||||
let mut from_local_id: HashMap<LocalId, (ConnId, node::NodeMessageId)> = HashMap::new();
|
||||
|
||||
// Any messages coming from nodes that have been muted are ignored:
|
||||
let mut muted: HashSet<LocalId> = HashSet::new();
|
||||
|
||||
// Now, loop and receive messages to handle.
|
||||
while let Some(msg) = rx_from_external.next().await {
|
||||
match msg {
|
||||
ToAggregator::ConnectedToTelemetryCore => {
|
||||
// Take hold of the connection closers and run them all.
|
||||
let closers = close_connections;
|
||||
|
||||
for mut closer in closers {
|
||||
// if this fails, it probably means the connection has died already anyway.
|
||||
let _ = closer.send(());
|
||||
}
|
||||
|
||||
// We've told everything to disconnect. Now, reset our state:
|
||||
close_connections = vec![];
|
||||
to_local_id = HashMap::new();
|
||||
from_local_id = HashMap::new();
|
||||
muted = HashSet::new();
|
||||
connected_to_telemetry_core = true;
|
||||
log::info!("Connected to telemetry core");
|
||||
},
|
||||
ToAggregator::DisconnectedFromTelemetryCore => {
|
||||
connected_to_telemetry_core = false;
|
||||
log::info!("Disconnected from telemetry core");
|
||||
},
|
||||
ToAggregator::FromWebsocket(conn_id, FromWebsocket::Add { message_id, ip, node, close_connection }) => {
|
||||
// Keep the close_connection channel incase we need it:
|
||||
close_connections.push(close_connection);
|
||||
|
||||
// Don't bother doing anything else if we're disconnected, since we'll force the
|
||||
// ndoe to reconnect anyway when the backend does:
|
||||
if !connected_to_telemetry_core { continue }
|
||||
|
||||
// Generate a new "local ID" for messages from this connection:
|
||||
let local_id = next_local_id;
|
||||
next_local_id += 1;
|
||||
|
||||
// Store mapping to/from local_id to conn/message ID paid:
|
||||
to_local_id.insert((conn_id, message_id), local_id);
|
||||
from_local_id.insert(local_id, (conn_id, message_id));
|
||||
|
||||
// Send the message to the telemetry core with this local ID:
|
||||
let _ = tx_to_telemetry_core.send(FromShardAggregator::AddNode {
|
||||
ip,
|
||||
node,
|
||||
local_id
|
||||
}).await;
|
||||
},
|
||||
ToAggregator::FromWebsocket(conn_id, FromWebsocket::Update { message_id, payload }) => {
|
||||
// Ignore incoming messages if we're not connected to the backend:
|
||||
if !connected_to_telemetry_core { continue }
|
||||
|
||||
// Get the local ID, ignoring the message if none match:
|
||||
let local_id = match to_local_id.get(&(conn_id, message_id)) {
|
||||
Some(id) => *id,
|
||||
None => continue
|
||||
};
|
||||
|
||||
async fn read(tx: UnboundedSender<ChainMessage>, mut receiver: WsReceiver) -> anyhow::Result<()> {
|
||||
let mut data = Vec::with_capacity(128);
|
||||
// ignore the message if this node has been muted:
|
||||
if muted.contains(&local_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
loop {
|
||||
data.clear();
|
||||
// Send the message to the telemetry core with this local ID:
|
||||
let _ = tx_to_telemetry_core.send(FromShardAggregator::UpdateNode {
|
||||
local_id,
|
||||
payload
|
||||
}).await;
|
||||
},
|
||||
ToAggregator::FromTelemetryCore(FromTelemetryCore::Mute { local_id }) => {
|
||||
// Ignore incoming messages if we're not connected to the backend:
|
||||
if !connected_to_telemetry_core { continue }
|
||||
|
||||
receiver.receive_data(&mut data).await?;
|
||||
|
||||
println!("Received {} bytes from Backend Core", data.len());
|
||||
|
||||
match bincode::options().deserialize(&data) {
|
||||
Ok(msg) => tx.send(ChainMessage::Backend(msg))?,
|
||||
Err(err) => {
|
||||
log::error!("Failed to read message from Backend Core: {:?}", err);
|
||||
// Mute the local ID we've been told to:
|
||||
muted.insert(local_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
/// Return a sink that a node can send messages into to be handled by the aggregator.
|
||||
pub fn subscribe_node(&self) -> impl Sink<FromWebsocket, Error = anyhow::Error> + Unpin {
|
||||
// Assign a unique aggregator-local ID to each connection that subscribes, and pass
|
||||
// that along with every message to the aggregator loop:
|
||||
let conn_id: ConnId = self.0.conn_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let tx_to_aggregator = self.0.tx_to_aggregator.clone();
|
||||
|
||||
tokio::task::spawn(read(tx, receiver));
|
||||
|
||||
Ok(sender)
|
||||
}
|
||||
}
|
||||
|
||||
impl Actor for Chain {
|
||||
type Context = Context<Self>;
|
||||
}
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct AddNode {
|
||||
pub ip: Option<Ipv4Addr>,
|
||||
pub genesis_hash: BlockHash,
|
||||
pub node: NodeDetails,
|
||||
pub conn_id: ConnId,
|
||||
pub node_connector: Addr<NodeConnector>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ChainMessage {
|
||||
AddNode(AddNode),
|
||||
UpdateNode(NodeId, Payload),
|
||||
Backend(BackendMessage),
|
||||
}
|
||||
|
||||
impl fmt::Debug for AddNode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str("AddNode")
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<AddNode> for Aggregator {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
|
||||
let AddNode { genesis_hash, .. } = msg;
|
||||
|
||||
let url = &self.url;
|
||||
let chain = self
|
||||
.chains
|
||||
.entry(genesis_hash)
|
||||
.or_insert_with(move || Chain::new(url.clone(), genesis_hash).spawn());
|
||||
|
||||
if let Err(err) = chain.send(ChainMessage::AddNode(msg)) {
|
||||
let msg = err.0;
|
||||
log::error!("Failed to add node to chain, shutting down chain");
|
||||
self.chains.remove(&genesis_hash);
|
||||
// TODO: Send a message back to clean up node connections
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler<AddNode> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
|
||||
let AddNode { ip, node_connector, .. } = msg;
|
||||
|
||||
println!("Node connected to {}: {:?}", self.genesis_hash, ip);
|
||||
// Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
|
||||
// but pinning by boxing is the easy solution for now:
|
||||
Box::pin(tx_to_aggregator.with(move |msg| async move {
|
||||
Ok(ToAggregator::FromWebsocket(conn_id, msg))
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
use futures::channel::{ mpsc };
|
||||
use futures::{ Sink, SinkExt, StreamExt };
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_util::compat::{ TokioAsyncReadCompatExt };
|
||||
|
||||
#[derive(Clone,Debug)]
|
||||
pub enum Message<Out> {
|
||||
Connected,
|
||||
Disconnected,
|
||||
Data(Out)
|
||||
}
|
||||
|
||||
/// Connect to a websocket server, retrying the connection if we're disconnected.
|
||||
/// - Sends messages when disconnected, reconnected or data received from the connection.
|
||||
/// - Returns a channel that allows you to send messages to the connection.
|
||||
/// - Messages all encoded/decoded from bincode.
|
||||
pub async fn create_ws_connection<In, Out, S, E>(mut tx_to_external: S, telemetry_uri: http::Uri) -> mpsc::Sender<In>
|
||||
where
|
||||
S: Sink<Message<Out>, Error = E> + Unpin + Send + Clone + 'static,
|
||||
E: std::fmt::Debug + std::fmt::Display + Send + 'static,
|
||||
In: serde::Serialize + Send + 'static,
|
||||
Out: serde::de::DeserializeOwned + Send + 'static
|
||||
{
|
||||
// Set up a proxy channel to relay messages to the telemetry core, and return one end of it.
|
||||
// Once a connection to the backend is established, we pass messages along to it. If the connection
|
||||
// fails, we
|
||||
let (tx_to_connection_proxy, mut rx_from_external_proxy) = mpsc::channel(10);
|
||||
tokio::spawn(async move {
|
||||
let mut connected = false;
|
||||
|
||||
loop {
|
||||
// Throw away any pending messages from the incoming channel so that it
|
||||
// doesn't get blocked up while we're looping and waiting for a reconnection.
|
||||
while let Ok(Some(_)) = rx_from_external_proxy.try_next() {}
|
||||
|
||||
// The connection will pass messages back to this.
|
||||
let tx_from_connection = tx_to_external.clone();
|
||||
|
||||
// Attempt to reconnect.
|
||||
match create_ws_connection_no_retry(tx_from_connection, telemetry_uri.clone()).await {
|
||||
Ok(mut tx_to_connection) => {
|
||||
connected = true;
|
||||
|
||||
// Inform the handler loop that we've reconnected.
|
||||
tx_to_external.send(Message::Connected)
|
||||
.await
|
||||
.expect("must be able to send reconnect msg");
|
||||
|
||||
// Start forwarding messages on to the backend.
|
||||
while let Some(msg) = rx_from_external_proxy.next().await {
|
||||
if let Err(e) = tx_to_connection.send(msg).await {
|
||||
// Issue forwarding a message to the telemetry core?
|
||||
// Give up and try to reconnect on the next loop iteration.
|
||||
log::error!("Error sending message to websocker server (will reconnect): {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
// Issue connecting? Wait and try again on the next loop iteration.
|
||||
log::error!("Error connecting to websocker server (will reconnect): {}", e);
|
||||
}
|
||||
};
|
||||
|
||||
// Tell the aggregator that we're disconnected so that, if we like, we can discard
|
||||
// messages without doing any futher processing on them.
|
||||
if connected {
|
||||
connected = false;
|
||||
let _ = tx_to_external.send(Message::Disconnected).await;
|
||||
}
|
||||
|
||||
// Wait a little before trying to reconnect.
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
}
|
||||
});
|
||||
|
||||
tx_to_connection_proxy
|
||||
}
|
||||
|
||||
/// This spawns a connection to a websocket server, serializing/deserialziing
|
||||
/// from bincode as messages are sent or received.
|
||||
async fn create_ws_connection_no_retry<In, Out, S, E>(mut tx_to_external: S, telemetry_uri: http::Uri) -> anyhow::Result<mpsc::Sender<In>>
|
||||
where
|
||||
S: Sink<Message<Out>, Error = E> + Unpin + Send + 'static,
|
||||
E: std::fmt::Debug + std::fmt::Display,
|
||||
In: serde::Serialize + Send + 'static,
|
||||
Out: serde::de::DeserializeOwned + Send + 'static
|
||||
{
|
||||
use soketto::handshake::{Client, ServerResponse};
|
||||
use bincode::Options;
|
||||
|
||||
let host = telemetry_uri.host().unwrap_or("127.0.0.1");
|
||||
let port = telemetry_uri.port_u16().unwrap_or(8000);
|
||||
let path = telemetry_uri.path();
|
||||
|
||||
let socket = TcpStream::connect((host, port)).await?;
|
||||
socket.set_nodelay(true).unwrap();
|
||||
|
||||
// Open a websocket connection with the relemetry core:
|
||||
let mut client = Client::new(socket.compat(), host, &path);
|
||||
let (mut ws_to_connection, mut ws_from_connection) = match client.handshake().await? {
|
||||
ServerResponse::Accepted { .. } => client.into_builder().finish(),
|
||||
ServerResponse::Redirect { status_code, .. } |
|
||||
ServerResponse::Rejected { status_code } => {
|
||||
return Err(anyhow::anyhow!("Failed to connect to {}{}, status code: {}", host, path, status_code));
|
||||
}
|
||||
};
|
||||
|
||||
// This task reads data sent from the telemetry core and
|
||||
// forwards it on to our aggregator loop:
|
||||
tokio::spawn(async move {
|
||||
let mut data = Vec::with_capacity(128);
|
||||
loop {
|
||||
// Clear the buffer and wait for the next message to arrive:
|
||||
data.clear();
|
||||
if let Err(e) = ws_from_connection.receive_data(&mut data).await {
|
||||
// Couldn't receive data may mean all senders are gone, so log
|
||||
// the error and shut this down:
|
||||
log::error!("Shutting down websocket connection: Failed to receive data: {}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
// Attempt to deserialize, and send to our handler loop:
|
||||
match bincode::options().deserialize(&data) {
|
||||
Ok(msg) => {
|
||||
if let Err(e) = tx_to_external.send(Message::Data(msg)).await {
|
||||
// Failure to send to our loop likely means it's hit an
|
||||
// issue and shut down, so bail on this loop as well:
|
||||
log::error!("Shutting down websocket connection: Failed to send data out: {}", e);
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
// Log the error but otherwise ignore it and keep running:
|
||||
log::warn!("Failed to decode message from Backend Core: {:?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// This task receives messages from the aggregator,
|
||||
// encodes them and sends them to the telemetry core:
|
||||
let (tx_to_connection, mut rx_from_aggregator) = mpsc::channel(10);
|
||||
tokio::spawn(async move {
|
||||
while let Some(msg) = rx_from_aggregator.next().await {
|
||||
let bytes = bincode::options()
|
||||
.serialize(&msg)
|
||||
.expect("must be able to serialize msg");
|
||||
|
||||
// Any errors sending the message leads to this task ending, which should cascade to
|
||||
// the entire connection being ended.
|
||||
if let Err(e) = ws_to_connection.send_binary_mut(bytes).await {
|
||||
log::error!("Shutting down websocket connection: Failed to send data in: {}", e);
|
||||
return;
|
||||
}
|
||||
if let Err(e) = ws_to_connection.flush().await {
|
||||
log::error!("Shutting down websocket connection: Failed to flush data: {}", e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// We return a channel that you can send messages down in order to have
|
||||
// them sent to the telemetry core:
|
||||
Ok(tx_to_connection)
|
||||
}
|
||||
+173
-91
@@ -1,125 +1,207 @@
|
||||
use std::net::Ipv4Addr;
|
||||
|
||||
use actix::prelude::*;
|
||||
use actix_http::ws::Codec;
|
||||
use actix_http::http::Uri;
|
||||
use actix_web::{get, middleware, web, App, Error, HttpRequest, HttpResponse, HttpServer};
|
||||
use actix_web_actors::ws;
|
||||
use clap::Clap;
|
||||
use simple_logger::SimpleLogger;
|
||||
|
||||
mod aggregator;
|
||||
mod node;
|
||||
mod connection;
|
||||
|
||||
use aggregator::Aggregator;
|
||||
use node::NodeConnector;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use structopt::StructOpt;
|
||||
use http::Uri;
|
||||
use simple_logger::SimpleLogger;
|
||||
use futures::{StreamExt, SinkExt, channel::mpsc};
|
||||
use warp::Filter;
|
||||
use warp::filters::ws;
|
||||
use common::{json, node, log_level::LogLevel};
|
||||
use aggregator::{ Aggregator, FromWebsocket };
|
||||
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
|
||||
const NAME: &str = "Substrate Telemetry Backend Shard";
|
||||
const ABOUT: &str = "This is the Telemetry Backend Shard that forwards the data sent by Substrate/Polkadot nodes to the Backend Core";
|
||||
const ABOUT: &str = "This is the Telemetry Backend Shard that forwards the \
|
||||
data sent by Substrate/Polkadot nodes to the Backend Core";
|
||||
|
||||
#[derive(Clap, Debug)]
|
||||
#[clap(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
|
||||
#[derive(StructOpt, Debug)]
|
||||
#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
|
||||
struct Opts {
|
||||
#[clap(
|
||||
short = 'l',
|
||||
/// This is the socket address that this shard is listening to. This is restricted to
|
||||
/// localhost (127.0.0.1) by default and should be fine for most use cases. If
|
||||
/// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
|
||||
#[structopt(
|
||||
short = "l",
|
||||
long = "listen",
|
||||
default_value = "127.0.0.1:8001",
|
||||
about = "This is the socket address Telemetry is listening to. This is restricted to localhost (127.0.0.1) by default and should be fine for most use cases. If you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'"
|
||||
)]
|
||||
socket: std::net::SocketAddr,
|
||||
#[clap(
|
||||
arg_enum,
|
||||
/// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
|
||||
/// 'error' only logs errors and 'trace' logs everything.
|
||||
#[structopt(
|
||||
required = false,
|
||||
long = "log",
|
||||
default_value = "info",
|
||||
about = "Log level."
|
||||
)]
|
||||
log_level: LogLevel,
|
||||
#[clap(
|
||||
short = 'c',
|
||||
/// Url to the Backend Core endpoint accepting shard connections
|
||||
#[structopt(
|
||||
short = "c",
|
||||
long = "core",
|
||||
default_value = "ws://127.0.0.1:8000/shard_submit/",
|
||||
about = "Url to the Backend Core endpoint accepting shard connections"
|
||||
)]
|
||||
core_url: Uri,
|
||||
}
|
||||
|
||||
#[derive(Clap, Debug, PartialEq)]
|
||||
enum LogLevel {
|
||||
Error,
|
||||
Warn,
|
||||
Info,
|
||||
Debug,
|
||||
Trace,
|
||||
}
|
||||
|
||||
impl From<&LogLevel> for log::LevelFilter {
|
||||
fn from(log_level: &LogLevel) -> Self {
|
||||
match log_level {
|
||||
LogLevel::Error => log::LevelFilter::Error,
|
||||
LogLevel::Warn => log::LevelFilter::Warn,
|
||||
LogLevel::Info => log::LevelFilter::Info,
|
||||
LogLevel::Debug => log::LevelFilter::Debug,
|
||||
LogLevel::Trace => log::LevelFilter::Trace,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Entry point for connecting nodes
|
||||
#[get("/submit")]
|
||||
async fn node_route(
|
||||
req: HttpRequest,
|
||||
stream: web::Payload,
|
||||
aggregator: web::Data<Addr<Aggregator>>,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let ip = req
|
||||
.connection_info()
|
||||
.realip_remote_addr()
|
||||
.and_then(|mut addr| {
|
||||
if let Some(port_idx) = addr.find(':') {
|
||||
addr = &addr[..port_idx];
|
||||
}
|
||||
addr.parse::<Ipv4Addr>().ok()
|
||||
});
|
||||
|
||||
let mut res = ws::handshake(&req)?;
|
||||
let aggregator = aggregator.get_ref().clone();
|
||||
|
||||
Ok(res.streaming(ws::WebsocketContext::with_codec(
|
||||
NodeConnector::new(aggregator, ip),
|
||||
stream,
|
||||
Codec::new().max_size(10 * 1024 * 1024), // 10mb frame limit
|
||||
)))
|
||||
}
|
||||
|
||||
/// Telemetry entry point. Listening by default on 127.0.0.1:8000.
|
||||
/// This can be changed using the `PORT` and `BIND` ENV variables.
|
||||
#[actix_web::main]
|
||||
async fn main() -> std::io::Result<()> {
|
||||
let opts = Opts::parse();
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let opts = Opts::from_args();
|
||||
let log_level = &opts.log_level;
|
||||
|
||||
SimpleLogger::new()
|
||||
.with_level(log_level.into())
|
||||
.init()
|
||||
.expect("Must be able to start a logger");
|
||||
|
||||
println!("URL? {:?} {:?}", opts.core_url.host(), opts.core_url.port_u16());
|
||||
|
||||
let aggregator = Aggregator::new(opts.core_url).start();
|
||||
|
||||
log::info!(
|
||||
"Starting Telemetry Shard version: {}",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
VERSION
|
||||
);
|
||||
HttpServer::new(move || {
|
||||
App::new()
|
||||
.wrap(middleware::NormalizePath::default())
|
||||
.data(aggregator.clone())
|
||||
.service(node_route)
|
||||
})
|
||||
.bind(opts.socket)?
|
||||
.run()
|
||||
.await
|
||||
|
||||
if let Err(e) = start_server(opts).await {
|
||||
log::error!("Error starting server: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
/// Declare our routes and start the server.
|
||||
async fn start_server(opts: Opts) -> anyhow::Result<()> {
|
||||
|
||||
let aggregator = Aggregator::spawn(opts.core_url).await?;
|
||||
|
||||
// Handle requests to /health by returning OK.
|
||||
let health_route =
|
||||
warp::path("health")
|
||||
.map(|| "OK");
|
||||
|
||||
// Handle websocket requests to /submit.
|
||||
let ws_route =
|
||||
warp::path("submit")
|
||||
.and(warp::ws())
|
||||
.and(warp::filters::addr::remote())
|
||||
.map(move |ws: ws::Ws, addr: Option<SocketAddr>| {
|
||||
// Send messages from the websocket connection to this sink
|
||||
// to have them pass to the aggregator.
|
||||
let tx_to_aggregator = aggregator.subscribe_node();
|
||||
log::info!("Opening /submit connection from {:?}", addr);
|
||||
ws.on_upgrade(move |websocket| async move {
|
||||
handle_websocket_connection(websocket, tx_to_aggregator, addr).await;
|
||||
log::info!("Closing /submit connection from {:?}", addr);
|
||||
})
|
||||
});
|
||||
|
||||
// Merge the routes and start our server:
|
||||
let routes = ws_route.or(health_route);
|
||||
warp::serve(routes).run(opts.socket).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// This takes care of handling messages from an established socket connection.
|
||||
async fn handle_websocket_connection<S>(websocket: ws::WebSocket, mut tx_to_aggregator: S, addr: Option<SocketAddr>)
|
||||
where S: futures::Sink<FromWebsocket, Error = anyhow::Error> + Unpin
|
||||
{
|
||||
let mut websocket = websocket.fuse();
|
||||
|
||||
// This could be a oneshot channel, but it's useful to be able to clone
|
||||
// messages, and we can't clone oneshot channel senders.
|
||||
let (close_connection_tx, mut close_connection_rx) = mpsc::channel(0);
|
||||
|
||||
// First, we wait until we receive a SystemConnected message.
|
||||
// Until this turns up, we ignore other messages. We could buffer
|
||||
// a few quite easily if we liked.
|
||||
while let Some(msg) = websocket.next().await {
|
||||
let node_message = match deserialize_ws_message(msg) {
|
||||
Ok(Some(msg)) => msg,
|
||||
Ok(None) => continue,
|
||||
Err(e) => { log::error!("{}", e); break }
|
||||
};
|
||||
|
||||
let message_id = node_message.id();
|
||||
let payload = node_message.into_payload();
|
||||
|
||||
if let node::Payload::SystemConnected(info) = payload {
|
||||
let _ = tx_to_aggregator.send(FromWebsocket::Add {
|
||||
message_id,
|
||||
ip: addr.map(|a| a.ip()),
|
||||
node: info.node,
|
||||
close_connection: close_connection_tx,
|
||||
}).await;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Now, the node has been added, so we forward messages along as updates.
|
||||
// We keep an eye on the close_connection channel; if that resolves, then
|
||||
// end this loop and let the connection close gracefully.
|
||||
loop {
|
||||
futures::select_biased! {
|
||||
// The close channel has fired, so end the loop:
|
||||
_ = close_connection_rx.next() => {
|
||||
break
|
||||
},
|
||||
// A message was received; handle it:
|
||||
msg = websocket.next() => {
|
||||
let msg = match msg {
|
||||
Some(msg) => msg,
|
||||
None => break
|
||||
};
|
||||
|
||||
let node_message = match deserialize_ws_message(msg) {
|
||||
Ok(Some(msg)) => msg,
|
||||
Ok(None) => continue,
|
||||
Err(e) => { log::error!("{}", e); break }
|
||||
};
|
||||
|
||||
let message_id = node_message.id();
|
||||
let payload = node_message.into_payload();
|
||||
|
||||
if let Err(e) = tx_to_aggregator.send(FromWebsocket::Update { message_id, payload } ).await {
|
||||
log::error!("Failed to send node message to aggregator: {}", e);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loops ended; attempt to close the connection gracefully:
|
||||
let _ = websocket.close().await;
|
||||
}
|
||||
|
||||
/// Deserialize an incoming websocket message, returning an error if something
|
||||
/// fatal went wrong, [`Some`] message if all went well, and [`None`] if a non-fatal
|
||||
/// issue was encountered and the message should simply be ignored.
|
||||
fn deserialize_ws_message(msg: Result<ws::Message, warp::Error>) -> anyhow::Result<Option<node::NodeMessage>> {
|
||||
// If we see any errors, log them and end our loop:
|
||||
let msg = match msg {
|
||||
Err(e) => {
|
||||
return Err(anyhow::anyhow!("Error in node websocket connection: {}", e));
|
||||
},
|
||||
Ok(msg) => msg
|
||||
};
|
||||
|
||||
// If the message isn't something we want to handle, just ignore it.
|
||||
// This includes system messages like "pings" and such, so don't log anything.
|
||||
if !msg.is_binary() && !msg.is_text() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Deserialize from JSON, warning if deserialization fails:
|
||||
let bytes = msg.as_bytes();
|
||||
let node_message: json::NodeMessage = match serde_json::from_slice(bytes) {
|
||||
Ok(node_message) => node_message,
|
||||
Err(_e) => {
|
||||
// let bytes: &[u8] = bytes.get(..512).unwrap_or_else(|| &bytes);
|
||||
// let msg_start = std::str::from_utf8(bytes).unwrap_or_else(|_| "INVALID UTF8");
|
||||
// log::warn!("Failed to parse node message ({}): {}", msg_start, e);
|
||||
return Ok(None)
|
||||
}
|
||||
};
|
||||
|
||||
// Pull relevant details from the message:
|
||||
let node_message: node::NodeMessage = node_message.into();
|
||||
Ok(Some(node_message))
|
||||
}
|
||||
@@ -1,203 +0,0 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::aggregator::{AddNode, Aggregator, ChainMessage};
|
||||
// use crate::chain::{Chain, RemoveNode, UpdateNode};
|
||||
use actix::prelude::*;
|
||||
use actix_web_actors::ws::{self, CloseReason};
|
||||
use common::node::{NodeMessage, Payload};
|
||||
use common::types::{ConnId, NodeId};
|
||||
use common::json;
|
||||
use common::ws::{MultipartHandler, WsMessage};
|
||||
use tokio::sync::mpsc::UnboundedSender;
|
||||
|
||||
/// How often heartbeat pings are sent
|
||||
const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20);
|
||||
/// How long before lack of client response causes a timeout
|
||||
const CLIENT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
pub struct NodeConnector {
|
||||
/// Multiplexing connections by id
|
||||
multiplex: BTreeMap<ConnId, ConnMultiplex>,
|
||||
/// Client must send ping at least once every 60 seconds (CLIENT_TIMEOUT),
|
||||
hb: Instant,
|
||||
/// Aggregator actor address
|
||||
aggregator: Addr<Aggregator>,
|
||||
/// IP address of the node this connector is responsible for
|
||||
ip: Option<Ipv4Addr>,
|
||||
/// Helper for handling continuation messages
|
||||
multipart: MultipartHandler,
|
||||
}
|
||||
|
||||
enum ConnMultiplex {
|
||||
Connected {
|
||||
/// Id of the node this multiplex connector is responsible for handling
|
||||
nid: NodeId,
|
||||
/// Chain address to which this multiplex connector is delegating messages
|
||||
chain: UnboundedSender<ChainMessage>,
|
||||
},
|
||||
Waiting {
|
||||
/// Backlog of messages to be sent once we get a recipient handle to the chain
|
||||
backlog: Vec<Payload>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for ConnMultiplex {
|
||||
fn default() -> Self {
|
||||
ConnMultiplex::Waiting {
|
||||
backlog: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Actor for NodeConnector {
|
||||
type Context = ws::WebsocketContext<Self>;
|
||||
|
||||
fn started(&mut self, ctx: &mut Self::Context) {
|
||||
self.heartbeat(ctx);
|
||||
}
|
||||
|
||||
fn stopped(&mut self, _: &mut Self::Context) {
|
||||
// for mx in self.multiplex.values() {
|
||||
// if let ConnMultiplex::Connected { chain, nid } = mx {
|
||||
// chain.do_send(RemoveNode(*nid));
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
impl NodeConnector {
|
||||
pub fn new(aggregator: Addr<Aggregator>, ip: Option<Ipv4Addr>) -> Self {
|
||||
Self {
|
||||
multiplex: BTreeMap::new(),
|
||||
hb: Instant::now(),
|
||||
aggregator,
|
||||
ip,
|
||||
multipart: MultipartHandler::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn heartbeat(&self, ctx: &mut <Self as Actor>::Context) {
|
||||
ctx.run_interval(HEARTBEAT_INTERVAL, |act, ctx| {
|
||||
// check client heartbeats
|
||||
if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
|
||||
// stop actor
|
||||
ctx.close(Some(CloseReason {
|
||||
code: ws::CloseCode::Abnormal,
|
||||
description: Some("Missed heartbeat".into()),
|
||||
}));
|
||||
ctx.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_message(
|
||||
&mut self,
|
||||
msg: json::NodeMessage,
|
||||
ctx: &mut <Self as Actor>::Context,
|
||||
) {
|
||||
let msg: NodeMessage = msg.into();
|
||||
let conn_id = msg.id();
|
||||
let payload = msg.into();
|
||||
|
||||
match self.multiplex.entry(conn_id).or_default() {
|
||||
ConnMultiplex::Connected { nid, chain } => {
|
||||
// TODO: error handle
|
||||
let _ = chain.send(ChainMessage::UpdateNode(*nid, payload));
|
||||
}
|
||||
ConnMultiplex::Waiting { backlog } => {
|
||||
if let Payload::SystemConnected(connected) = payload {
|
||||
println!("Node connected {:?}", connected.node);
|
||||
self.aggregator.do_send(AddNode {
|
||||
genesis_hash: connected.genesis_hash,
|
||||
ip: self.ip,
|
||||
node: connected.node,
|
||||
conn_id,
|
||||
node_connector: ctx.address(),
|
||||
});
|
||||
} else {
|
||||
if backlog.len() >= 10 {
|
||||
backlog.remove(0);
|
||||
}
|
||||
|
||||
backlog.push(payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Initialize {
|
||||
pub nid: NodeId,
|
||||
pub conn_id: ConnId,
|
||||
pub chain: UnboundedSender<ChainMessage>,
|
||||
}
|
||||
|
||||
impl Handler<Initialize> for NodeConnector {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: Initialize, _: &mut Self::Context) {
|
||||
let Initialize {
|
||||
nid,
|
||||
conn_id,
|
||||
chain,
|
||||
} = msg;
|
||||
log::trace!(target: "NodeConnector::Initialize", "Initializing a node, nid={}, on conn_id={}", nid, conn_id);
|
||||
let mx = self.multiplex.entry(conn_id).or_default();
|
||||
|
||||
if let ConnMultiplex::Waiting { backlog } = mx {
|
||||
for payload in backlog.drain(..) {
|
||||
// TODO: error handle.
|
||||
let _ = chain.send(ChainMessage::UpdateNode(nid, payload));
|
||||
}
|
||||
|
||||
*mx = ConnMultiplex::Connected {
|
||||
nid,
|
||||
chain,
|
||||
};
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for NodeConnector {
|
||||
fn handle(&mut self, msg: Result<ws::Message, ws::ProtocolError>, ctx: &mut Self::Context) {
|
||||
self.hb = Instant::now();
|
||||
|
||||
let data = match msg.map(|msg| self.multipart.handle(msg)) {
|
||||
Ok(WsMessage::Nop) => return,
|
||||
Ok(WsMessage::Ping(msg)) => {
|
||||
ctx.pong(&msg);
|
||||
return;
|
||||
}
|
||||
Ok(WsMessage::Data(data)) => data,
|
||||
Ok(WsMessage::Close(reason)) => {
|
||||
ctx.close(reason);
|
||||
ctx.stop();
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
log::error!("{:?}", error);
|
||||
ctx.stop();
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match serde_json::from_slice(&data) {
|
||||
Ok(msg) => self.handle_message(msg, ctx),
|
||||
#[cfg(debug)]
|
||||
Err(err) => {
|
||||
let data: &[u8] = data.get(..512).unwrap_or_else(|| &data);
|
||||
log::warn!(
|
||||
"Failed to parse node message: {} {}",
|
||||
err,
|
||||
std::str::from_utf8(data).unwrap_or_else(|_| "INVALID UTF8")
|
||||
)
|
||||
}
|
||||
#[cfg(not(debug))]
|
||||
Err(_) => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "telemetry"
|
||||
version = "0.1.0"
|
||||
authors = ["Parity Technologies Ltd. <admin@parity.io>"]
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.41"
|
||||
bincode = "1.3.3"
|
||||
common = { path = "../common" }
|
||||
futures = "0.3.15"
|
||||
hex = "0.4.3"
|
||||
http = "0.2.4"
|
||||
log = "0.4.14"
|
||||
primitive-types = { version = "0.9.0", features = ["serde"] }
|
||||
serde = { version = "1.0.126", features = ["derive"] }
|
||||
serde_json = "1.0.64"
|
||||
simple_logger = "1.11.0"
|
||||
soketto = "0.6.0"
|
||||
structopt = "0.3.21"
|
||||
thiserror = "1.0.25"
|
||||
tokio = { version = "1.7.0", features = ["full"] }
|
||||
tokio-util = { version = "0.6", features = ["compat"] }
|
||||
warp = "0.3.1"
|
||||
@@ -0,0 +1,202 @@
|
||||
use common::{internal_messages::{self, LocalId}, node};
|
||||
use std::{str::FromStr, sync::Arc};
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use futures::channel::{ mpsc, oneshot };
|
||||
use futures::{ Sink, SinkExt, StreamExt };
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_util::compat::{ TokioAsyncReadCompatExt };
|
||||
use std::collections::{ HashMap, HashSet };
|
||||
|
||||
/// A unique Id is assigned per websocket connection (or more accurately,
|
||||
/// per feed socket and per shard socket). This can be combined with the
|
||||
/// [`LocalId`] of messages to give us a global ID.
|
||||
type ConnId = u64;
|
||||
|
||||
/// Incoming messages come via subscriptions, and end up looking like this.
|
||||
#[derive(Debug)]
|
||||
enum ToAggregator {
|
||||
FromShardWebsocket(ConnId, FromShardWebsocket),
|
||||
FromFeedWebsocket(ConnId, FromFeedWebsocket),
|
||||
}
|
||||
|
||||
/// An incoming shard connection can send these messages to the aggregator.
|
||||
#[derive(Debug)]
|
||||
pub enum FromShardWebsocket {
|
||||
/// When the socket is opened, it'll send this first
|
||||
/// so that we have a way to communicate back to it.
|
||||
Initialize {
|
||||
channel: mpsc::Sender<ToShardWebsocket>,
|
||||
},
|
||||
/// Tell the aggregator about a new node.
|
||||
Add {
|
||||
local_id: LocalId,
|
||||
ip: Option<std::net::IpAddr>,
|
||||
node: common::types::NodeDetails,
|
||||
},
|
||||
/// Update/pass through details about a node.
|
||||
Update {
|
||||
local_id: LocalId,
|
||||
payload: node::Payload
|
||||
}
|
||||
}
|
||||
|
||||
/// The aggregator can these messages back to a shard connection.
|
||||
#[derive(Debug)]
|
||||
pub enum ToShardWebsocket {
|
||||
/// Mute messages to the core by passing the shard-local ID of them.
|
||||
Mute {
|
||||
local_id: LocalId
|
||||
}
|
||||
}
|
||||
|
||||
/// An incoming feed connection can send these messages to the aggregator.
|
||||
#[derive(Debug)]
|
||||
pub enum FromFeedWebsocket {
|
||||
/// When the socket is opened, it'll send this first
|
||||
/// so that we have a way to communicate back to it.
|
||||
Initialize {
|
||||
channel: mpsc::Sender<ToFeedWebsocket>,
|
||||
},
|
||||
/// The feed can subscribe to a chain to receive
|
||||
/// messages relating to it.
|
||||
Subscribe {
|
||||
chain: Box<str>
|
||||
},
|
||||
/// The feed wants finality info for the chain, too.
|
||||
SendFinality {
|
||||
chain: Box<str>
|
||||
},
|
||||
/// The feed doesn't want any more finality info for the chain.
|
||||
NoMoreFinality {
|
||||
chain: Box<str>
|
||||
},
|
||||
/// An explicit ping message.
|
||||
Ping {
|
||||
chain: Box<str>
|
||||
}
|
||||
}
|
||||
|
||||
// The frontend sends text based commands; parse them into these messages:
|
||||
impl FromStr for FromFeedWebsocket {
|
||||
type Err = anyhow::Error;
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let (cmd, chain) = match s.find(':') {
|
||||
Some(idx) => (&s[..idx], s[idx+1..].into()),
|
||||
None => return Err(anyhow::anyhow!("Expecting format `CMD:CHAIN_NAME`"))
|
||||
};
|
||||
match cmd {
|
||||
"ping" => Ok(FromFeedWebsocket::Ping { chain }),
|
||||
"subscribe" => Ok(FromFeedWebsocket::Subscribe { chain }),
|
||||
"send-finality" => Ok(FromFeedWebsocket::SendFinality { chain }),
|
||||
"no-more-finality" => Ok(FromFeedWebsocket::NoMoreFinality { chain }),
|
||||
_ => return Err(anyhow::anyhow!("Command {} not recognised", cmd))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The aggregator can these messages back to a feed connection.
|
||||
#[derive(Debug)]
|
||||
pub enum ToFeedWebsocket {
|
||||
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Aggregator(Arc<AggregatorInternal>);
|
||||
|
||||
struct AggregatorInternal {
|
||||
/// Shards that connect are each assigned a unique connection ID.
|
||||
/// This helps us know who to send messages back to (especially in
|
||||
/// conjunction with the [`LocalId`] that messages will come with).
|
||||
shard_conn_id: AtomicU64,
|
||||
/// Feeds that connect have their own unique connection ID, too.
|
||||
feed_conn_id: AtomicU64,
|
||||
/// Send messages in to the aggregator from the outside via this. This is
|
||||
/// stored here so that anybody holding an `Aggregator` handle can
|
||||
/// make use of it.
|
||||
tx_to_aggregator: mpsc::Sender<ToAggregator>
|
||||
}
|
||||
|
||||
impl Aggregator {
|
||||
/// Spawn a new Aggregator. This connects to the telemetry backend
|
||||
pub async fn spawn(denylist: Vec<String>) -> anyhow::Result<Aggregator> {
|
||||
let (tx_to_aggregator, rx_from_external) = mpsc::channel(10);
|
||||
|
||||
// Handle any incoming messages in our handler loop:
|
||||
tokio::spawn(Aggregator::handle_messages(rx_from_external, denylist));
|
||||
|
||||
// Return a handle to our aggregator:
|
||||
Ok(Aggregator(Arc::new(AggregatorInternal {
|
||||
shard_conn_id: AtomicU64::new(1),
|
||||
feed_conn_id: AtomicU64::new(1),
|
||||
tx_to_aggregator,
|
||||
})))
|
||||
}
|
||||
|
||||
// This is spawned into a separate task and handles any messages coming
|
||||
// in to the aggregator. If nobody is tolding the tx side of the channel
|
||||
// any more, this task will gracefully end.
|
||||
async fn handle_messages(mut rx_from_external: mpsc::Receiver<ToAggregator>, denylist: Vec<String>) {
|
||||
|
||||
// Temporary: if we drop channels to shards, they will be booted:
|
||||
let mut to_shards = vec![];
|
||||
|
||||
// Now, loop and receive messages to handle.
|
||||
while let Some(msg) = rx_from_external.next().await {
|
||||
match msg {
|
||||
ToAggregator::FromFeedWebsocket(feed_conn_id, FromFeedWebsocket::Initialize { channel }) => {
|
||||
|
||||
},
|
||||
ToAggregator::FromFeedWebsocket(feed_conn_id, FromFeedWebsocket::Ping { chain }) => {
|
||||
|
||||
},
|
||||
ToAggregator::FromFeedWebsocket(feed_conn_id, FromFeedWebsocket::Subscribe { chain }) => {
|
||||
|
||||
},
|
||||
ToAggregator::FromFeedWebsocket(feed_conn_id, FromFeedWebsocket::SendFinality { chain }) => {
|
||||
|
||||
},
|
||||
ToAggregator::FromFeedWebsocket(feed_conn_id, FromFeedWebsocket::NoMoreFinality { chain }) => {
|
||||
|
||||
},
|
||||
ToAggregator::FromShardWebsocket(shard_conn_id, FromShardWebsocket::Initialize { channel }) => {
|
||||
to_shards.push(channel);
|
||||
},
|
||||
ToAggregator::FromShardWebsocket(shard_conn_id, FromShardWebsocket::Add { local_id, ip, node }) => {
|
||||
|
||||
},
|
||||
ToAggregator::FromShardWebsocket(shard_conn_id, FromShardWebsocket::Update { local_id, payload }) => {
|
||||
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a sink that a shard can send messages into to be handled by the aggregator.
|
||||
pub fn subscribe_shard(&self) -> impl Sink<FromShardWebsocket, Error = anyhow::Error> + Unpin {
|
||||
// Assign a unique aggregator-local ID to each connection that subscribes, and pass
|
||||
// that along with every message to the aggregator loop:
|
||||
let shard_conn_id: ConnId = self.0.shard_conn_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let tx_to_aggregator = self.0.tx_to_aggregator.clone();
|
||||
|
||||
// Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
|
||||
// but pinning by boxing is the easy solution for now:
|
||||
Box::pin(tx_to_aggregator.with(move |msg| async move {
|
||||
Ok(ToAggregator::FromShardWebsocket(shard_conn_id, msg))
|
||||
}))
|
||||
}
|
||||
|
||||
/// Return a sink that a feed can send messages into to be handled by the aggregator.
|
||||
pub fn subscribe_feed(&self) -> impl Sink<FromFeedWebsocket, Error = anyhow::Error> + Unpin {
|
||||
// Assign a unique aggregator-local ID to each connection that subscribes, and pass
|
||||
// that along with every message to the aggregator loop:
|
||||
let feed_conn_id: ConnId = self.0.feed_conn_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let tx_to_aggregator = self.0.tx_to_aggregator.clone();
|
||||
|
||||
// Calling `send` on this Sink requires Unpin. There may be a nicer way than this,
|
||||
// but pinning by boxing is the easy solution for now:
|
||||
Box::pin(tx_to_aggregator.with(move |msg| async move {
|
||||
Ok(ToAggregator::FromFeedWebsocket(feed_conn_id, msg))
|
||||
}))
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
//! This module provides the messages that will be
|
||||
//! sent to subscribing feeds.
|
||||
|
||||
use serde::ser::{SerializeTuple, Serializer};
|
||||
use serde::Serialize;
|
||||
use std::mem;
|
||||
@@ -9,10 +12,6 @@ use common::types::{
|
||||
Timestamp, NodeDetails,
|
||||
};
|
||||
|
||||
pub mod connector;
|
||||
|
||||
use connector::Serialized;
|
||||
|
||||
pub trait FeedMessage {
|
||||
const ACTION: u8;
|
||||
}
|
||||
@@ -66,16 +65,16 @@ impl FeedMessageSerializer {
|
||||
let _ = to_writer(&mut self.buffer, value);
|
||||
}
|
||||
|
||||
pub fn finalize(&mut self) -> Option<Serialized> {
|
||||
pub fn finalize(&mut self) -> Option<Vec<u8>> {
|
||||
if self.buffer.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
self.buffer.push(b']');
|
||||
|
||||
let bytes = mem::replace(&mut self.buffer, Vec::with_capacity(BUFCAP)).into();
|
||||
let bytes = mem::replace(&mut self.buffer, Vec::with_capacity(BUFCAP));
|
||||
|
||||
Some(Serialized(bytes))
|
||||
Some(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,28 +89,28 @@ macro_rules! actions {
|
||||
}
|
||||
|
||||
actions! {
|
||||
0x00: Version,
|
||||
0x01: BestBlock,
|
||||
0x02: BestFinalized,
|
||||
0x03: AddedNode<'_>,
|
||||
0x04: RemovedNode,
|
||||
0x05: LocatedNode<'_>,
|
||||
0x06: ImportedBlock<'_>,
|
||||
0x07: FinalizedBlock,
|
||||
0x08: NodeStatsUpdate<'_>,
|
||||
0x09: Hardware<'_>,
|
||||
0x0A: TimeSync,
|
||||
0x0B: AddedChain<'_>,
|
||||
0x0C: RemovedChain<'_>,
|
||||
0x0D: SubscribedTo<'_>,
|
||||
0x0E: UnsubscribedFrom<'_>,
|
||||
0x0F: Pong<'_>,
|
||||
0x10: AfgFinalized,
|
||||
0x11: AfgReceivedPrevote,
|
||||
0x12: AfgReceivedPrecommit,
|
||||
0x13: AfgAuthoritySet,
|
||||
0x14: StaleNode,
|
||||
0x15: NodeIOUpdate<'_>,
|
||||
0: Version,
|
||||
1: BestBlock,
|
||||
2: BestFinalized,
|
||||
3: AddedNode<'_>,
|
||||
4: RemovedNode,
|
||||
5: LocatedNode<'_>,
|
||||
6: ImportedBlock<'_>,
|
||||
7: FinalizedBlock,
|
||||
8: NodeStatsUpdate<'_>,
|
||||
9: Hardware<'_>,
|
||||
10: TimeSync,
|
||||
11: AddedChain<'_>,
|
||||
12: RemovedChain<'_>,
|
||||
13: SubscribedTo<'_>,
|
||||
14: UnsubscribedFrom<'_>,
|
||||
15: Pong<'_>,
|
||||
16: AfgFinalized,
|
||||
17: AfgReceivedPrevote,
|
||||
18: AfgReceivedPrecommit,
|
||||
19: AfgAuthoritySet,
|
||||
20: StaleNode,
|
||||
21: NodeIOUpdate<'_>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -0,0 +1,285 @@
|
||||
mod aggregator;
|
||||
mod feed_message;
|
||||
mod node;
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::str::FromStr;
|
||||
|
||||
use bincode::Options;
|
||||
use common::internal_messages;
|
||||
use structopt::StructOpt;
|
||||
use simple_logger::SimpleLogger;
|
||||
use futures::{StreamExt, SinkExt, channel::mpsc};
|
||||
use warp::Filter;
|
||||
use warp::filters::ws;
|
||||
use common::{log_level::LogLevel};
|
||||
use aggregator::{ Aggregator, FromFeedWebsocket, ToFeedWebsocket, FromShardWebsocket, ToShardWebsocket };
|
||||
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
const AUTHORS: &str = env!("CARGO_PKG_AUTHORS");
|
||||
const NAME: &str = "Substrate Telemetry Backend Core";
|
||||
const ABOUT: &str = "This is the Telemetry Backend Core that receives telemetry messages \
|
||||
from Substrate/Polkadot nodes and provides the data to a subsribed feed";
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
#[structopt(name = NAME, version = VERSION, author = AUTHORS, about = ABOUT)]
|
||||
struct Opts {
|
||||
/// This is the socket address that Telemetryis listening to. This is restricted to
|
||||
/// localhost (127.0.0.1) by default and should be fine for most use cases. If
|
||||
/// you are using Telemetry in a container, you likely want to set this to '0.0.0.0:8000'
|
||||
#[structopt(
|
||||
short = "l",
|
||||
long = "listen",
|
||||
default_value = "127.0.0.1:8000",
|
||||
)]
|
||||
socket: std::net::SocketAddr,
|
||||
/// The desired log level; one of 'error', 'warn', 'info', 'debug' or 'trace', where
|
||||
/// 'error' only logs errors and 'trace' logs everything.
|
||||
#[structopt(
|
||||
required = false,
|
||||
long = "log",
|
||||
default_value = "info",
|
||||
about = "Log level."
|
||||
)]
|
||||
log_level: LogLevel,
|
||||
/// Space delimited list of the names of chains that are not allowed to connect to
|
||||
/// telemetry. Case sensitive.
|
||||
#[structopt(
|
||||
required = false,
|
||||
long = "denylist",
|
||||
)]
|
||||
denylist: Vec<String>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let opts = Opts::from_args();
|
||||
let log_level = &opts.log_level;
|
||||
|
||||
SimpleLogger::new()
|
||||
.with_level(log_level.into())
|
||||
.init()
|
||||
.expect("Must be able to start a logger");
|
||||
|
||||
log::info!(
|
||||
"Starting Telemetry Core version: {}",
|
||||
VERSION
|
||||
);
|
||||
|
||||
if let Err(e) = start_server(opts).await {
|
||||
log::error!("Error starting server: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
/// Declare our routes and start the server.
|
||||
async fn start_server(opts: Opts) -> anyhow::Result<()> {
|
||||
|
||||
let shard_aggregator = Aggregator::spawn(opts.denylist).await?;
|
||||
let feed_aggregator = shard_aggregator.clone();
|
||||
|
||||
// Handle requests to /health by returning OK.
|
||||
let health_route =
|
||||
warp::path("health")
|
||||
.map(|| "OK");
|
||||
|
||||
// Handle websocket requests from shards.
|
||||
let ws_shard_submit_route =
|
||||
warp::path("shard_submit")
|
||||
.and(warp::ws())
|
||||
.and(warp::filters::addr::remote())
|
||||
.map(move |ws: ws::Ws, addr: Option<SocketAddr>| {
|
||||
let tx_to_aggregator = shard_aggregator.subscribe_shard();
|
||||
log::info!("Opening /shard_submit connection from {:?}", addr);
|
||||
ws.on_upgrade(move |websocket| async move {
|
||||
let websocket = handle_shard_websocket_connection(websocket, tx_to_aggregator).await;
|
||||
log::info!("Closing /shard_submit connection from {:?}", addr);
|
||||
let _ = websocket.close().await;
|
||||
})
|
||||
});
|
||||
|
||||
// Handle websocket requests from frontends.
|
||||
let ws_feed_route =
|
||||
warp::path("feed")
|
||||
.and(warp::ws())
|
||||
.and(warp::filters::addr::remote())
|
||||
.map(move |ws: ws::Ws, addr: Option<SocketAddr>| {
|
||||
let tx_to_aggregator = feed_aggregator.subscribe_feed();
|
||||
log::info!("Opening /feed connection from {:?}", addr);
|
||||
ws.on_upgrade(move |websocket| async move {
|
||||
let websocket = handle_feed_websocket_connection(websocket, tx_to_aggregator).await;
|
||||
log::info!("Closing /feed connection from {:?}", addr);
|
||||
let _ = websocket.close().await;
|
||||
})
|
||||
});
|
||||
|
||||
// Merge the routes and start our server:
|
||||
let routes = ws_shard_submit_route
|
||||
.or(ws_feed_route)
|
||||
.or(health_route);
|
||||
warp::serve(routes).run(opts.socket).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// This handles messages coming to/from a shard connection
|
||||
async fn handle_shard_websocket_connection<S>(mut websocket: ws::WebSocket, mut tx_to_aggregator: S) -> ws::WebSocket
|
||||
where S: futures::Sink<FromShardWebsocket, Error = anyhow::Error> + Unpin
|
||||
{
|
||||
let (tx_to_shard_conn, mut rx_from_aggregator) = mpsc::channel(10);
|
||||
|
||||
// Tell the aggregator about this new connection, and give it a way to send messages to us:
|
||||
let init_msg = FromShardWebsocket::Initialize {
|
||||
channel: tx_to_shard_conn
|
||||
};
|
||||
if let Err(e) = tx_to_aggregator.send(init_msg).await {
|
||||
log::error!("Error sending message to aggregator: {}", e);
|
||||
return websocket;
|
||||
}
|
||||
|
||||
// Loop, handling new messages from the shard or from the aggregator:
|
||||
loop {
|
||||
tokio::select! {
|
||||
// AGGREGATOR -> SHARD
|
||||
msg = rx_from_aggregator.next() => {
|
||||
// End the loop when connection from aggregator ends:
|
||||
let msg = match msg {
|
||||
Some(msg) => msg,
|
||||
None => break
|
||||
};
|
||||
|
||||
let internal_msg = match msg {
|
||||
ToShardWebsocket::Mute { local_id } => {
|
||||
internal_messages::FromTelemetryCore::Mute { local_id }
|
||||
}
|
||||
};
|
||||
|
||||
let bytes = bincode::options()
|
||||
.serialize(&internal_msg)
|
||||
.expect("message to shard should serialize");
|
||||
|
||||
if let Err(e) = websocket.send(ws::Message::binary(bytes)).await {
|
||||
log::error!("Error sending message to shard; booting it: {}", e);
|
||||
break
|
||||
}
|
||||
}
|
||||
// SHARD -> AGGREGATOR
|
||||
msg = websocket.next() => {
|
||||
// End the loop when connection from shard ends:
|
||||
let msg = match msg {
|
||||
Some(msg) => msg,
|
||||
None => break
|
||||
};
|
||||
|
||||
let msg = match msg {
|
||||
Err(e) => {
|
||||
log::error!("Error receiving message from shard; booting it: {}", e);
|
||||
break;
|
||||
},
|
||||
Ok(msg) => msg
|
||||
};
|
||||
|
||||
// If the message isn't something we want to handle, just ignore it.
|
||||
// This includes system messages like "pings" and such, so don't log anything.
|
||||
if !msg.is_binary() && !msg.is_text() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let bytes = msg.as_bytes();
|
||||
let msg: internal_messages::FromShardAggregator = match bincode::options().deserialize(bytes) {
|
||||
Ok(msg) => msg,
|
||||
Err(e) => {
|
||||
log::error!("Failed to deserialize message from shard; booting it: {}", e);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
// Convert and send to the aggregator:
|
||||
let aggregator_msg = match msg {
|
||||
internal_messages::FromShardAggregator::AddNode { ip, node, local_id } => {
|
||||
FromShardWebsocket::Add { ip, node, local_id }
|
||||
},
|
||||
internal_messages::FromShardAggregator::UpdateNode { payload, local_id } => {
|
||||
FromShardWebsocket::Update { local_id, payload }
|
||||
},
|
||||
};
|
||||
if let Err(e) = tx_to_aggregator.send(aggregator_msg).await {
|
||||
log::error!("Failed to send message to aggregator; closing shard: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loop ended; give socket back to parent:
|
||||
websocket
|
||||
}
|
||||
|
||||
/// This handles messages coming from a feed connection
|
||||
async fn handle_feed_websocket_connection<S>(mut websocket: ws::WebSocket, mut tx_to_aggregator: S) -> ws::WebSocket
|
||||
where S: futures::Sink<FromFeedWebsocket, Error = anyhow::Error> + Unpin
|
||||
{
|
||||
let (tx_to_feed_conn, mut rx_from_aggregator) = mpsc::channel(10);
|
||||
|
||||
// Tell the aggregator about this new connection, and give it a way to send messages to us:
|
||||
let init_msg = FromFeedWebsocket::Initialize {
|
||||
channel: tx_to_feed_conn
|
||||
};
|
||||
if let Err(e) = tx_to_aggregator.send(init_msg).await {
|
||||
log::error!("Error sending message to aggregator: {}", e);
|
||||
return websocket;
|
||||
}
|
||||
|
||||
// Loop, handling new messages from the shard or from the aggregator:
|
||||
loop {
|
||||
tokio::select! {
|
||||
// AGGREGATOR -> FRONTEND
|
||||
msg = rx_from_aggregator.next() => {
|
||||
// End the loop when connection from aggregator ends:
|
||||
let msg = match msg {
|
||||
Some(msg) => msg,
|
||||
None => break
|
||||
};
|
||||
|
||||
println!("TODO: encode message and send down feed websocket: {:?}", msg);
|
||||
}
|
||||
// FRONTEND -> AGGREGATOR
|
||||
msg = websocket.next() => {
|
||||
// End the loop when connection from feed ends:
|
||||
let msg = match msg {
|
||||
Some(msg) => msg,
|
||||
None => break
|
||||
};
|
||||
|
||||
// If we see any errors, log them and end our loop:
|
||||
let msg = match msg {
|
||||
Err(e) => {
|
||||
log::error!("Error in node websocket connection: {}", e);
|
||||
break;
|
||||
},
|
||||
Ok(msg) => msg
|
||||
};
|
||||
|
||||
// We ignore all but text messages from the frontend:
|
||||
let text = match msg.to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => continue
|
||||
};
|
||||
|
||||
// Parse the message into a command we understand and send it to the aggregator:
|
||||
let cmd = match FromFeedWebsocket::from_str(text) {
|
||||
Ok(cmd) => cmd,
|
||||
Err(e) => {
|
||||
log::warn!("Ignoring invalid command '{}' from the frontend: {}", text, e);
|
||||
continue
|
||||
}
|
||||
};
|
||||
if let Err(e) = tx_to_aggregator.send(cmd).await {
|
||||
log::error!("Failed to send message to aggregator; closing feed: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loop ended; give socket back to parent:
|
||||
websocket
|
||||
}
|
||||
Reference in New Issue
Block a user