mirror of
https://github.com/pezkuwichain/pezkuwi-telemetry.git
synced 2026-04-22 02:08:04 +00:00
Merge branch 'master' into dp-add-changelog
This commit is contained in:
David Palm
@@ -3,6 +3,7 @@ name: Backend CI
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- '.github/workflows/**'
|
||||
- 'backend/**'
|
||||
- '!frontend/**'
|
||||
|
||||
@@ -22,14 +23,15 @@ jobs:
|
||||
- name: Build release and call executable
|
||||
working-directory: ./backend
|
||||
run: cargo run --release -- --help
|
||||
- name: Build and Push template image
|
||||
uses: docker/build-push-action@v1 # https://github.com/docker/build-push-action
|
||||
- name: Login to Dockerhub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
path: ./backend/
|
||||
dockerfile: ./backend/Dockerfile
|
||||
repository: parity/substrate-telemetry-backend
|
||||
- name: Build and Push template image
|
||||
uses: docker/build-push-action@v2 # https://github.com/docker/build-push-action
|
||||
with:
|
||||
context: './backend'
|
||||
push: ${{ startsWith(github.ref, 'refs/tags/') }}
|
||||
tags: latest
|
||||
add_git_labels: true
|
||||
tags: parity/substrate-telemetry-backend:latest
|
||||
# add_git_labels: true
|
||||
|
||||
@@ -6,6 +6,7 @@ name: Frontend CI
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- '.github/workflows/**'
|
||||
- 'frontend/**'
|
||||
- '!backend/**'
|
||||
|
||||
@@ -36,15 +37,16 @@ jobs:
|
||||
- name: Build
|
||||
working-directory: ./frontend
|
||||
run: yarn build
|
||||
- name: Build and Push template image
|
||||
uses: docker/build-push-action@v1 # https://github.com/docker/build-push-action
|
||||
if: matrix.node-version == '12.x'
|
||||
- name: Login to Dockerhub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
path: ./frontend/
|
||||
dockerfile: ./frontend/Dockerfile
|
||||
repository: parity/substrate-telemetry-frontend
|
||||
- name: Build and Push template image
|
||||
uses: docker/build-push-action@v2 # https://github.com/docker/build-push-action
|
||||
if: matrix.node-version == '12.x'
|
||||
with:
|
||||
context: './frontend'
|
||||
push: ${{ startsWith(github.ref, 'refs/tags/') }}
|
||||
tags: latest
|
||||
add_git_labels: true
|
||||
tags: parity/substrate-telemetry-frontend:latest
|
||||
# add_git_labels: true
|
||||
|
||||
Generated
+15
-76
@@ -56,14 +56,11 @@ dependencies = [
|
||||
"ahash",
|
||||
"base64",
|
||||
"bitflags",
|
||||
"brotli2",
|
||||
"bytes",
|
||||
"bytestring",
|
||||
"cfg-if 1.0.0",
|
||||
"cookie",
|
||||
"derive_more",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
@@ -256,12 +253,6 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.7.2"
|
||||
@@ -376,26 +367,6 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli-sys"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli2"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e"
|
||||
dependencies = [
|
||||
"brotli-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.4.0"
|
||||
@@ -519,17 +490,6 @@ version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2"
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"time 0.2.26",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.1"
|
||||
@@ -552,15 +512,6 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.0"
|
||||
@@ -588,6 +539,16 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
|
||||
|
||||
[[package]]
|
||||
name = "ctor"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.11"
|
||||
@@ -641,18 +602,6 @@ dependencies = [
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"crc32fast",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
@@ -1017,16 +966,6 @@ version = "0.3.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
|
||||
dependencies = [
|
||||
"adler",
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.7.11"
|
||||
@@ -1303,9 +1242,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.7"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
|
||||
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -1726,9 +1665,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.48"
|
||||
version = "1.0.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
|
||||
checksum = "f3a1d708c221c5a612956ef9f75b37e454e88d1f7b899fbd3a18d4252012d663"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1752,8 +1691,8 @@ dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"clap",
|
||||
"ctor",
|
||||
"fnv",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"num-traits",
|
||||
"parking_lot",
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ license = "GPL-3.0"
|
||||
|
||||
[dependencies]
|
||||
actix = "0.11.1"
|
||||
actix-web = "4.0.0-beta.4"
|
||||
actix-web = { version = "4.0.0-beta.4", default-features = false }
|
||||
actix-web-actors = "4.0.0-beta.3"
|
||||
actix-http = "3.0.0-beta.4"
|
||||
bytes = "1.0.1"
|
||||
@@ -23,7 +23,7 @@ parking_lot = "0.11"
|
||||
reqwest = { version = "0.11.1", features = ["blocking", "json"] }
|
||||
rustc-hash = "1.1.0"
|
||||
clap = "3.0.0-beta.2"
|
||||
lazy_static = "1"
|
||||
ctor = "0.1.20"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
|
||||
+6
-9
@@ -1,26 +1,23 @@
|
||||
#### BUILDER IMAGE -- quite big one ####
|
||||
FROM paritytech/musl-ci-linux as builder
|
||||
LABEL maintainer="Daniel Maricic daniel@woss.io"
|
||||
LABEL description="Polkadot Telemetry backend builder image"
|
||||
FROM paritytech/ci-linux:production as builder
|
||||
|
||||
ARG PROFILE=release
|
||||
WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN cargo build --${PROFILE} --bins --target x86_64-unknown-linux-musl
|
||||
|
||||
RUN cargo build --${PROFILE} --bins
|
||||
|
||||
# MAIN IMAGE FOR PEOPLE TO PULL --- small one#
|
||||
FROM scratch
|
||||
LABEL maintainer="Daniel Maricic daniel@woss.io"
|
||||
FROM debian:buster-slim
|
||||
LABEL maintainer="Parity Technologies"
|
||||
LABEL description="Polkadot Telemetry backend, static build"
|
||||
|
||||
ARG PROFILE=release
|
||||
WORKDIR /usr/local/bin
|
||||
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=builder /app/target/x86_64-unknown-linux-musl/$PROFILE/telemetry /usr/local/bin
|
||||
COPY --from=builder /app/target/$PROFILE/telemetry /usr/local/bin
|
||||
RUN apt-get -y update && apt-get -y install openssl && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
|
||||
+25
-19
@@ -1,8 +1,9 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use actix::prelude::*;
|
||||
use lazy_static::lazy_static;
|
||||
use actix_web_actors::ws::{CloseReason, CloseCode};
|
||||
use ctor::ctor;
|
||||
|
||||
use crate::node::connector::Initialize;
|
||||
use crate::node::connector::{Mute, NodeConnector};
|
||||
use crate::feed::connector::{FeedConnector, Connected, FeedId};
|
||||
use crate::util::DenseMap;
|
||||
use crate::feed::{self, FeedMessageSerializer};
|
||||
@@ -29,18 +30,18 @@ pub struct ChainEntry {
|
||||
max_nodes: usize,
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
/// Labels of chains we consider "first party". These chains are allowed any
|
||||
/// number of nodes to connect.
|
||||
static ref FIRST_PARTY_NETWORKS: HashSet<&'static str> = {
|
||||
let mut set = HashSet::new();
|
||||
set.insert("Polkadot");
|
||||
set.insert("Kusama");
|
||||
set.insert("Westend");
|
||||
set.insert("Rococo");
|
||||
set
|
||||
};
|
||||
}
|
||||
#[ctor]
|
||||
/// Labels of chains we consider "first party". These chains allow any
|
||||
/// number of nodes to connect.
|
||||
static FIRST_PARTY_NETWORKS: HashSet<&'static str> = {
|
||||
let mut set = HashSet::new();
|
||||
set.insert("Polkadot");
|
||||
set.insert("Kusama");
|
||||
set.insert("Westend");
|
||||
set.insert("Rococo");
|
||||
set
|
||||
};
|
||||
|
||||
/// Max number of nodes allowed to connect to the telemetry server.
|
||||
const THIRD_PARTY_NETWORKS_MAX_NODES: usize = 500;
|
||||
|
||||
@@ -129,8 +130,8 @@ pub struct AddNode {
|
||||
pub node: NodeDetails,
|
||||
/// Connection id used by the node connector for multiplexing parachains
|
||||
pub conn_id: ConnId,
|
||||
/// Recipient for the initialization message
|
||||
pub rec: Recipient<Initialize>,
|
||||
/// Address of the NodeConnector actor
|
||||
pub node_connector: Addr<NodeConnector>,
|
||||
}
|
||||
|
||||
/// Message sent from the Chain to the Aggregator when the Chain loses all nodes
|
||||
@@ -196,10 +197,13 @@ impl Handler<AddNode> for Aggregator {
|
||||
|
||||
fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
|
||||
if self.denylist.contains(&*msg.node.chain) {
|
||||
log::debug!(target: "Aggregator::AddNode", "'{}' is on the denylist.", msg.node.chain);
|
||||
log::warn!(target: "Aggregator::AddNode", "'{}' is on the denylist.", msg.node.chain);
|
||||
let AddNode { node_connector, .. } = msg;
|
||||
let reason = CloseReason{ code: CloseCode::Abnormal, description: Some("Denied".into()) };
|
||||
node_connector.do_send(Mute { reason });
|
||||
return;
|
||||
}
|
||||
let AddNode { node, conn_id, rec } = msg;
|
||||
let AddNode { node, conn_id, node_connector } = msg;
|
||||
log::trace!(target: "Aggregator::AddNode", "New node connected. Chain '{}'", node.chain);
|
||||
|
||||
let cid = self.lazy_chain(&node.chain, ctx);
|
||||
@@ -208,10 +212,12 @@ impl Handler<AddNode> for Aggregator {
|
||||
chain.addr.do_send(chain::AddNode {
|
||||
node,
|
||||
conn_id,
|
||||
rec,
|
||||
node_connector,
|
||||
});
|
||||
} else {
|
||||
log::warn!(target: "Aggregator::AddNode", "Chain {} is over quota ({})", chain.label, chain.max_nodes);
|
||||
let reason = CloseReason{ code: CloseCode::Again, description: Some("Overquota".into()) };
|
||||
node_connector.do_send(Mute { reason });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use bytes::Bytes;
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use crate::aggregator::{Aggregator, DropChain, RenameChain, NodeCount};
|
||||
use crate::node::{Node, connector::Initialize, message::{NodeMessage, Payload}};
|
||||
use crate::node::{Node, connector::{Initialize, NodeConnector}, message::{NodeMessage, Payload}};
|
||||
use crate::feed::connector::{FeedId, FeedConnector, Subscribed, Unsubscribed};
|
||||
use crate::feed::{self, FeedMessageSerializer};
|
||||
use crate::util::{DenseMap, NumStats, now};
|
||||
@@ -196,8 +196,8 @@ pub struct AddNode {
|
||||
pub node: NodeDetails,
|
||||
/// Connection id used by the node connector for multiplexing parachains
|
||||
pub conn_id: ConnId,
|
||||
/// Recipient for the initialization message
|
||||
pub rec: Recipient<Initialize>,
|
||||
/// Address of the NodeConnector actor to which we send [`Initialize`] or [`Mute`] messages.
|
||||
pub node_connector: Addr<NodeConnector>,
|
||||
}
|
||||
|
||||
/// Message sent from the NodeConnector to the Chain when it receives new telemetry data
|
||||
@@ -250,14 +250,14 @@ impl Handler<AddNode> for Chain {
|
||||
type Result = ();
|
||||
|
||||
fn handle(&mut self, msg: AddNode, ctx: &mut Self::Context) {
|
||||
let AddNode { node, conn_id, rec } = msg;
|
||||
let AddNode { node, conn_id, node_connector } = msg;
|
||||
log::trace!(target: "Chain::AddNode", "New node connected. Chain '{}', node count goes from {} to {}", node.chain, self.nodes.len(), self.nodes.len() + 1);
|
||||
self.increment_label_count(&node.chain);
|
||||
|
||||
let nid = self.nodes.add(Node::new(node));
|
||||
let chain = ctx.address();
|
||||
|
||||
if rec.do_send(Initialize { nid, conn_id, chain }).is_err() {
|
||||
if node_connector.try_send(Initialize { nid, conn_id, chain }).is_err() {
|
||||
self.nodes.remove(nid);
|
||||
} else if let Some(node) = self.nodes.get(nid) {
|
||||
self.serializer.push(feed::AddedNode(nid, node));
|
||||
@@ -284,7 +284,7 @@ impl Chain {
|
||||
if node.update_block(*block) {
|
||||
if block.height > self.best.height {
|
||||
self.best = *block;
|
||||
log::info!(
|
||||
log::debug!(
|
||||
"[{}] [nodes={}/feeds={}] new best block={}/{:?}",
|
||||
self.label.0,
|
||||
nodes_len,
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::mem;
|
||||
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use actix::prelude::*;
|
||||
use actix_web_actors::ws;
|
||||
use actix_web_actors::ws::{self, CloseReason};
|
||||
use actix_http::ws::Item;
|
||||
use crate::aggregator::{Aggregator, AddNode};
|
||||
use crate::chain::{Chain, UpdateNode, RemoveNode};
|
||||
@@ -24,7 +24,7 @@ const CONT_BUF_LIMIT: usize = 10 * 1024 * 1024;
|
||||
pub struct NodeConnector {
|
||||
/// Multiplexing connections by id
|
||||
multiplex: BTreeMap<ConnId, ConnMultiplex>,
|
||||
/// Client must send ping at least once per 10 seconds (CLIENT_TIMEOUT),
|
||||
/// Client must send ping at least once every 60 seconds (CLIENT_TIMEOUT),
|
||||
hb: Instant,
|
||||
/// Aggregator actor address
|
||||
aggregator: Addr<Aggregator>,
|
||||
@@ -90,6 +90,7 @@ impl NodeConnector {
|
||||
// check client heartbeats
|
||||
if Instant::now().duration_since(act.hb) > CLIENT_TIMEOUT {
|
||||
// stop actor
|
||||
ctx.close(Some(CloseReason { code: ws::CloseCode::Abnormal, description: Some("Missed heartbeat".into())}));
|
||||
ctx.stop();
|
||||
}
|
||||
});
|
||||
@@ -109,16 +110,14 @@ impl NodeConnector {
|
||||
ConnMultiplex::Waiting { backlog } => {
|
||||
if let Payload::SystemConnected(connected) = msg.payload() {
|
||||
let mut node = connected.node.clone();
|
||||
let rec = ctx.address().recipient();
|
||||
|
||||
// FIXME: Use genesis hash instead of names to avoid this mess
|
||||
match &*node.chain {
|
||||
"Kusama CC3" => node.chain = "Kusama".into(),
|
||||
"Polkadot CC1" => node.chain = "Polkadot".into(),
|
||||
_ => (),
|
||||
_ => ()
|
||||
}
|
||||
|
||||
self.aggregator.do_send(AddNode { node, conn_id, rec });
|
||||
self.aggregator.do_send(AddNode { node, conn_id, node_connector: ctx.address() });
|
||||
} else {
|
||||
if backlog.len() >= 10 {
|
||||
backlog.remove(0);
|
||||
@@ -152,6 +151,23 @@ impl NodeConnector {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Mute {
|
||||
pub reason: CloseReason,
|
||||
}
|
||||
|
||||
impl Handler<Mute> for NodeConnector {
|
||||
type Result = ();
|
||||
fn handle(&mut self, msg: Mute, ctx: &mut Self::Context) {
|
||||
let Mute { reason } = msg;
|
||||
log::debug!(target: "NodeConnector::Mute", "Muting a node. Reason: {:?}", reason.description);
|
||||
|
||||
ctx.close(Some(reason));
|
||||
ctx.stop();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Message)]
|
||||
#[rtype(result = "()")]
|
||||
pub struct Initialize {
|
||||
@@ -198,7 +214,8 @@ impl StreamHandler<Result<ws::Message, ws::ProtocolError>> for NodeConnector {
|
||||
Ok(ws::Message::Pong(_)) => return,
|
||||
Ok(ws::Message::Text(text)) => text.into_bytes(),
|
||||
Ok(ws::Message::Binary(data)) => data,
|
||||
Ok(ws::Message::Close(_)) => {
|
||||
Ok(ws::Message::Close(reason)) => {
|
||||
ctx.close(reason);
|
||||
ctx.stop();
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user