Merge commit '392447f5c8f986ded2559a78457f4cd87942f393' into update-bridges-subtree-r/w

This commit is contained in:
antonio-dropulic
2021-12-01 09:46:14 +01:00
321 changed files with 28385 additions and 10466 deletions
+6
View File
@@ -7,6 +7,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
[dependencies]
ansi_term = "0.12"
anyhow = "1.0"
async-std = "1.6.5"
async-trait = "0.1.40"
backoff = "0.2"
@@ -19,6 +20,11 @@ num-traits = "0.2"
serde_json = "1.0"
sysinfo = "0.15"
time = "0.2"
thiserror = "1.0.26"
# Bridge dependencies
bp-runtime = { path = "../../primitives/runtime" }
# Substrate dependencies
@@ -0,0 +1,46 @@
// Copyright 2019-2021 Parity Technologies (UK) Ltd.
// This file is part of Parity Bridges Common.
// Parity Bridges Common is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity Bridges Common is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity Bridges Common. If not, see <http://www.gnu.org/licenses/>.
use std::net::AddrParseError;
use thiserror::Error;
/// Result type used by relay utilities.
pub type Result<T> = std::result::Result<T, Error>;
/// Relay utilities errors.
#[derive(Error, Debug)]
pub enum Error {
/// Failed to request a float value from HTTP service.
#[error("Failed to fetch token price from remote server: {0}")]
FetchTokenPrice(#[source] anyhow::Error),
/// Failed to parse the response from HTTP service.
#[error("Failed to parse HTTP service response: {0:?}. Response: {1:?}")]
ParseHttp(serde_json::Error, String),
/// Failed to select response value from the Json response.
#[error("Failed to select value from response: {0:?}. Response: {1:?}")]
SelectResponseValue(jsonpath_lib::JsonPathError, String),
/// Failed to parse float value from the selected value.
#[error(
"Failed to parse float value {0:?} from response. It is assumed to be positive and normal"
)]
ParseFloat(f64),
/// Couldn't found value in the JSON response.
#[error("Missing required value from response: {0:?}")]
MissingResponseValue(String),
/// Invalid host address was used for exposing Prometheus metrics.
#[error("Invalid host {0} is used to expose Prometheus metrics: {1}")]
ExposingMetricsInvalidHost(String, AddrParseError),
/// Prometheus error.
#[error("{0}")]
Prometheus(#[from] substrate_prometheus_endpoint::prometheus::Error),
}
@@ -62,14 +62,7 @@ pub fn initialize_logger(with_timestamp: bool) {
let log_level = color_level(record.level());
let log_target = color_target(record.target());
writeln!(
buf,
"{}{} {} {}",
loop_name_prefix(),
log_level,
log_target,
record.args(),
)
writeln!(buf, "{}{} {} {}", loop_name_prefix(), log_level, log_target, record.args(),)
});
}
@@ -81,12 +74,14 @@ pub(crate) fn initialize_loop(loop_name: String) {
LOOP_NAME.with(|g_loop_name| *g_loop_name.borrow_mut() = loop_name);
}
/// Returns loop name prefix to use in logs. The prefix is initialized with the `initialize_loop` call.
/// Returns loop name prefix to use in logs. The prefix is initialized with the `initialize_loop`
/// call.
fn loop_name_prefix() -> String {
// try_with to avoid panic outside of async-std task context
LOOP_NAME
.try_with(|loop_name| {
// using borrow is ok here, because loop is only initialized once (=> borrow_mut will only be called once)
// using borrow is ok here, because loop is only initialized once (=> borrow_mut will
// only be called once)
let loop_name = loop_name.borrow();
if loop_name.is_empty() {
String::new()
+16 -20
View File
@@ -16,11 +16,14 @@
//! Utilities used by different relays.
pub use bp_runtime::HeaderId;
pub use error::Error;
pub use relay_loop::{relay_loop, relay_metrics};
use backoff::{backoff::Backoff, ExponentialBackoff};
use futures::future::FutureExt;
use std::time::Duration;
use thiserror::Error;
/// Max delay after connection-unrelated error happened before we'll try the
/// same request again.
@@ -29,6 +32,7 @@ pub const MAX_BACKOFF_INTERVAL: Duration = Duration::from_secs(60);
/// reconnection again.
pub const CONNECTION_ERROR_DELAY: Duration = Duration::from_secs(10);
pub mod error;
pub mod initialize;
pub mod metrics;
pub mod relay_loop;
@@ -100,10 +104,6 @@ macro_rules! bail_on_arg_error {
};
}
/// Ethereum header Id.
#[derive(Debug, Default, Clone, Copy, Eq, Hash, PartialEq)]
pub struct HeaderId<Hash, Number>(pub Number, pub Hash);
/// Error type that can signal connection errors.
pub trait MaybeConnectionError {
/// Returns true if error (maybe) represents connection error.
@@ -111,11 +111,13 @@ pub trait MaybeConnectionError {
}
/// Stringified error that may be either connection-related or not.
#[derive(Debug)]
#[derive(Error, Debug)]
pub enum StringifiedMaybeConnectionError {
/// The error is connection-related error.
#[error("{0}")]
Connection(String),
/// The error is connection-unrelated error.
#[error("{0}")]
NonConnection(String),
}
@@ -139,15 +141,6 @@ impl MaybeConnectionError for StringifiedMaybeConnectionError {
}
}
impl ToString for StringifiedMaybeConnectionError {
fn to_string(&self) -> String {
match *self {
StringifiedMaybeConnectionError::Connection(ref err) => err.clone(),
StringifiedMaybeConnectionError::NonConnection(ref err) => err.clone(),
}
}
}
/// Exponential backoff for connection-unrelated errors retries.
pub fn retry_backoff() -> ExponentialBackoff {
ExponentialBackoff {
@@ -168,12 +161,12 @@ pub fn format_ids<Id: std::fmt::Debug>(mut ids: impl ExactSizeIterator<Item = Id
let id0 = ids.next().expect(NTH_PROOF);
let id1 = ids.next().expect(NTH_PROOF);
format!("[{:?}, {:?}]", id0, id1)
}
},
len => {
let id0 = ids.next().expect(NTH_PROOF);
let id_last = ids.last().expect(NTH_PROOF);
format!("{}:[{:?} ... {:?}]", len, id0, id_last)
}
},
}
}
@@ -220,7 +213,10 @@ impl ProcessFutureResult {
/// Returns Ok(true) if future has succeeded.
/// Returns Ok(false) if future has failed with non-connection error.
/// Returns Err if future is `ConnectionFailed`.
pub fn fail_if_connection_error(self, failed_client: FailedClient) -> Result<bool, FailedClient> {
pub fn fail_if_connection_error(
self,
failed_client: FailedClient,
) -> Result<bool, FailedClient> {
match self {
ProcessFutureResult::Success => Ok(true),
ProcessFutureResult::Failed => Ok(false),
@@ -247,7 +243,7 @@ where
on_success(result);
retry_backoff.reset();
ProcessFutureResult::Success
}
},
Err(error) if error.is_connection_error() => {
log::error!(
target: "bridge",
@@ -259,7 +255,7 @@ where
retry_backoff.reset();
go_offline_future.set(go_offline(CONNECTION_ERROR_DELAY).fuse());
ProcessFutureResult::ConnectionFailed
}
},
Err(error) => {
let retry_delay = retry_backoff.next_backoff().unwrap_or(CONNECTION_ERROR_DELAY);
log::error!(
@@ -272,6 +268,6 @@ where
go_offline_future.set(go_offline(retry_delay).fuse());
ProcessFutureResult::Failed
}
},
}
}
+14 -18
View File
@@ -21,12 +21,16 @@ pub use substrate_prometheus_endpoint::{
register, Counter, CounterVec, Gauge, GaugeVec, Opts, PrometheusError, Registry, F64, U64,
};
use async_std::sync::{Arc, RwLock};
use async_trait::async_trait;
use std::{fmt::Debug, time::Duration};
mod float_json_value;
mod global;
/// Shared reference to `f64` value that is updated by the metric.
pub type F64SharedRef = Arc<RwLock<Option<f64>>>;
/// Unparsed address that needs to be used to expose Prometheus metrics.
#[derive(Debug, Clone)]
pub struct MetricsAddress {
@@ -78,21 +82,14 @@ pub trait StandaloneMetrics: Metrics {
impl Default for MetricsAddress {
fn default() -> Self {
MetricsAddress {
host: "127.0.0.1".into(),
port: 9616,
}
MetricsAddress { host: "127.0.0.1".into(), port: 9616 }
}
}
impl MetricsParams {
/// Creates metrics params so that metrics are not exposed.
pub fn disabled() -> Self {
MetricsParams {
address: None,
registry: None,
metrics_prefix: None,
}
MetricsParams { address: None, registry: None, metrics_prefix: None }
}
/// Do not expose metrics.
@@ -110,11 +107,7 @@ impl MetricsParams {
impl From<Option<MetricsAddress>> for MetricsParams {
fn from(address: Option<MetricsAddress>) -> Self {
MetricsParams {
address,
registry: None,
metrics_prefix: None,
}
MetricsParams { address, registry: None, metrics_prefix: None }
}
}
@@ -130,7 +123,10 @@ pub fn metric_name(prefix: Option<&str>, name: &str) -> String {
/// Set value of gauge metric.
///
/// If value is `Ok(None)` or `Err(_)`, metric would have default value.
pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(gauge: &Gauge<V>, value: Result<Option<T>, E>) {
pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(
gauge: &Gauge<V>,
value: Result<Option<T>, E>,
) {
gauge.set(match value {
Ok(Some(value)) => {
log::trace!(
@@ -140,7 +136,7 @@ pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(gauge: &G
value,
);
value
}
},
Ok(None) => {
log::warn!(
target: "bridge-metrics",
@@ -148,7 +144,7 @@ pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(gauge: &G
gauge.desc().first().map(|d| &d.fq_name),
);
Default::default()
}
},
Err(error) => {
log::warn!(
target: "bridge-metrics",
@@ -157,6 +153,6 @@ pub fn set_gauge_value<T: Default + Debug, V: Atomic<T = T>, E: Debug>(gauge: &G
error,
);
Default::default()
}
},
})
}
@@ -14,8 +14,15 @@
// You should have received a copy of the GNU General Public License
// along with Parity Bridges Common. If not, see <http://www.gnu.org/licenses/>.
use crate::metrics::{metric_name, register, Gauge, PrometheusError, Registry, StandaloneMetrics, F64};
use crate::{
error::{self, Error},
metrics::{
metric_name, register, F64SharedRef, Gauge, PrometheusError, Registry, StandaloneMetrics,
F64,
},
};
use async_std::sync::{Arc, RwLock};
use async_trait::async_trait;
use std::time::Duration;
@@ -23,11 +30,15 @@ use std::time::Duration;
const UPDATE_INTERVAL: Duration = Duration::from_secs(60);
/// Metric that represents float value received from HTTP service as float gauge.
///
/// The float value returned by the service is assumed to be normal (`f64::is_normal`
/// should return `true`) and strictly positive.
#[derive(Debug, Clone)]
pub struct FloatJsonValueMetric {
url: String,
json_path: String,
metric: Gauge<F64>,
shared_value_ref: F64SharedRef,
}
impl FloatJsonValueMetric {
@@ -40,34 +51,32 @@ impl FloatJsonValueMetric {
name: String,
help: String,
) -> Result<Self, PrometheusError> {
let shared_value_ref = Arc::new(RwLock::new(None));
Ok(FloatJsonValueMetric {
url,
json_path,
metric: register(Gauge::new(metric_name(prefix, &name), help)?, registry)?,
shared_value_ref,
})
}
/// Read value from HTTP service.
async fn read_value(&self) -> Result<f64, String> {
/// Get shared reference to metric value.
pub fn shared_value_ref(&self) -> F64SharedRef {
self.shared_value_ref.clone()
}
/// Request value from HTTP service.
async fn request_value(&self) -> anyhow::Result<String> {
use isahc::{AsyncReadResponseExt, HttpClient, Request};
fn map_isahc_err(err: impl std::fmt::Display) -> String {
format!("Failed to fetch token price from remote server: {}", err)
}
let request = Request::get(&self.url)
.header("Accept", "application/json")
.body(())
.map_err(map_isahc_err)?;
let raw_response = HttpClient::new()
.map_err(map_isahc_err)?
.send_async(request)
.await
.map_err(map_isahc_err)?
.text()
.await
.map_err(map_isahc_err)?;
let request = Request::get(&self.url).header("Accept", "application/json").body(())?;
let raw_response = HttpClient::new()?.send_async(request).await?.text().await?;
Ok(raw_response)
}
/// Read value from HTTP service.
async fn read_value(&self) -> error::Result<f64> {
let raw_response = self.request_value().await.map_err(Error::FetchTokenPrice)?;
parse_service_response(&self.json_path, &raw_response)
}
}
@@ -79,30 +88,28 @@ impl StandaloneMetrics for FloatJsonValueMetric {
}
async fn update(&self) {
crate::metrics::set_gauge_value(&self.metric, self.read_value().await.map(Some));
let value = self.read_value().await;
let maybe_ok = value.as_ref().ok().copied();
crate::metrics::set_gauge_value(&self.metric, value.map(Some));
*self.shared_value_ref.write().await = maybe_ok;
}
}
/// Parse HTTP service response.
fn parse_service_response(json_path: &str, response: &str) -> Result<f64, String> {
let json = serde_json::from_str(response).map_err(|err| {
format!(
"Failed to parse HTTP service response: {:?}. Response: {:?}",
err, response,
)
})?;
fn parse_service_response(json_path: &str, response: &str) -> error::Result<f64> {
let json =
serde_json::from_str(response).map_err(|err| Error::ParseHttp(err, response.to_owned()))?;
let mut selector = jsonpath_lib::selector(&json);
let maybe_selected_value = selector(json_path).map_err(|err| {
format!(
"Failed to select value from response: {:?}. Response: {:?}",
err, response,
)
})?;
let maybe_selected_value =
selector(json_path).map_err(|err| Error::SelectResponseValue(err, response.to_owned()))?;
let selected_value = maybe_selected_value
.first()
.and_then(|v| v.as_f64())
.ok_or_else(|| format!("Missing required value from response: {:?}", response,))?;
.ok_or_else(|| Error::MissingResponseValue(response.to_owned()))?;
if !selected_value.is_normal() || selected_value < 0.0 {
return Err(Error::ParseFloat(selected_value))
}
Ok(selected_value)
}
@@ -118,4 +125,19 @@ mod tests {
Ok(433.05),
);
}
#[test]
fn parse_service_response_rejects_negative_numbers() {
assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":-433.05}}"#).is_err());
}
#[test]
fn parse_service_response_rejects_zero_numbers() {
assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":0.0}}"#).is_err());
}
#[test]
fn parse_service_response_rejects_nan() {
assert!(parse_service_response("$.kusama.usd", r#"{"kusama":{"usd":NaN}}"#).is_err());
}
}
@@ -17,7 +17,8 @@
//! Global system-wide Prometheus metrics exposed by relays.
use crate::metrics::{
metric_name, register, Gauge, GaugeVec, Opts, PrometheusError, Registry, StandaloneMetrics, F64, U64,
metric_name, register, Gauge, GaugeVec, Opts, PrometheusError, Registry, StandaloneMetrics,
F64, U64,
};
use async_std::sync::{Arc, Mutex};
@@ -50,7 +51,10 @@ impl GlobalMetrics {
registry,
)?,
process_cpu_usage_percentage: register(
Gauge::new(metric_name(prefix, "process_cpu_usage_percentage"), "Process CPU usage")?,
Gauge::new(
metric_name(prefix, "process_cpu_usage_percentage"),
"Process CPU usage",
)?,
registry,
)?,
process_memory_usage_bytes: register(
@@ -92,16 +96,19 @@ impl StandaloneMetrics for GlobalMetrics {
memory_usage,
);
self.process_cpu_usage_percentage
.set(if cpu_usage.is_finite() { cpu_usage } else { 0f64 });
self.process_cpu_usage_percentage.set(if cpu_usage.is_finite() {
cpu_usage
} else {
0f64
});
self.process_memory_usage_bytes.set(memory_usage);
}
},
_ => {
log::warn!(
target: "bridge-metrics",
"Failed to refresh process information. Metrics may show obsolete values",
);
}
},
}
}
+50 -39
View File
@@ -14,8 +14,11 @@
// You should have received a copy of the GNU General Public License
// along with Parity Bridges Common. If not, see <http://www.gnu.org/licenses/>.
use crate::metrics::{Metrics, MetricsAddress, MetricsParams, PrometheusError, StandaloneMetrics};
use crate::{FailedClient, MaybeConnectionError};
use crate::{
error::Error,
metrics::{Metrics, MetricsAddress, MetricsParams, PrometheusError, StandaloneMetrics},
FailedClient, MaybeConnectionError,
};
use async_trait::async_trait;
use std::{fmt::Debug, future::Future, net::SocketAddr, time::Duration};
@@ -27,24 +30,29 @@ pub const RECONNECT_DELAY: Duration = Duration::from_secs(10);
/// Basic blockchain client from relay perspective.
#[async_trait]
pub trait Client: 'static + Clone + Send + Sync {
/// Type of error this clients returns.
/// Type of error these clients returns.
type Error: 'static + Debug + MaybeConnectionError + Send + Sync;
/// Try to reconnect to source node.
async fn reconnect(&mut self) -> Result<(), Self::Error>;
}
/// Returns generic loop that may be customized and started.
pub fn relay_loop<SC, TC>(source_client: SC, target_client: TC) -> Loop<SC, TC, ()> {
Loop {
reconnect_delay: RECONNECT_DELAY,
source_client,
target_client,
loop_metric: None,
#[async_trait]
impl Client for () {
type Error = crate::StringifiedMaybeConnectionError;
async fn reconnect(&mut self) -> Result<(), Self::Error> {
Ok(())
}
}
/// Returns generic relay loop metrics that may be customized and used in one or several relay loops.
/// Returns generic loop that may be customized and started.
pub fn relay_loop<SC, TC>(source_client: SC, target_client: TC) -> Loop<SC, TC, ()> {
Loop { reconnect_delay: RECONNECT_DELAY, source_client, target_client, loop_metric: None }
}
/// Returns generic relay loop metrics that may be customized and used in one or several relay
/// loops.
pub fn relay_metrics(prefix: Option<String>, params: MetricsParams) -> LoopMetrics<(), (), ()> {
LoopMetrics {
relay_loop: Loop {
@@ -85,7 +93,11 @@ impl<SC, TC, LM> Loop<SC, TC, LM> {
}
/// Start building loop metrics using given prefix.
pub fn with_metrics(self, prefix: Option<String>, params: MetricsParams) -> LoopMetrics<SC, TC, ()> {
pub fn with_metrics(
self,
prefix: Option<String>,
params: MetricsParams,
) -> LoopMetrics<SC, TC, ()> {
LoopMetrics {
relay_loop: Loop {
reconnect_delay: self.reconnect_delay,
@@ -102,10 +114,10 @@ impl<SC, TC, LM> Loop<SC, TC, LM> {
/// Run relay loop.
///
/// This function represents an outer loop, which in turn calls provided `run_loop` function to do
/// actual job. When `run_loop` returns, this outer loop reconnects to failed client (source,
/// This function represents an outer loop, which in turn calls provided `run_loop` function to
/// do actual job. When `run_loop` returns, this outer loop reconnects to failed client (source,
/// target or both) and calls `run_loop` again.
pub async fn run<R, F>(mut self, loop_name: String, run_loop: R) -> Result<(), String>
pub async fn run<R, F>(mut self, loop_name: String, run_loop: R) -> Result<(), Error>
where
R: 'static + Send + Fn(SC, TC, Option<LM>) -> F,
F: 'static + Send + Future<Output = Result<(), FailedClient>>,
@@ -118,20 +130,20 @@ impl<SC, TC, LM> Loop<SC, TC, LM> {
loop {
let loop_metric = self.loop_metric.clone();
let future_result = run_loop(self.source_client.clone(), self.target_client.clone(), loop_metric);
let future_result =
run_loop(self.source_client.clone(), self.target_client.clone(), loop_metric);
let result = future_result.await;
match result {
Ok(()) => break,
Err(failed_client) => {
Err(failed_client) =>
reconnect_failed_client(
failed_client,
self.reconnect_delay,
&mut self.source_client,
&mut self.target_client,
)
.await
}
.await,
}
log::debug!(target: "bridge", "Restarting relay loop");
@@ -151,8 +163,8 @@ impl<SC, TC, LM> LoopMetrics<SC, TC, LM> {
pub fn loop_metric<NewLM: Metrics>(
self,
create_metric: impl FnOnce(&Registry, Option<&str>) -> Result<NewLM, PrometheusError>,
) -> Result<LoopMetrics<SC, TC, NewLM>, String> {
let loop_metric = create_metric(&self.registry, self.metrics_prefix.as_deref()).map_err(|e| e.to_string())?;
) -> Result<LoopMetrics<SC, TC, NewLM>, Error> {
let loop_metric = create_metric(&self.registry, self.metrics_prefix.as_deref())?;
Ok(LoopMetrics {
relay_loop: self.relay_loop,
@@ -167,13 +179,13 @@ impl<SC, TC, LM> LoopMetrics<SC, TC, LM> {
pub fn standalone_metric<M: StandaloneMetrics>(
self,
create_metric: impl FnOnce(&Registry, Option<&str>) -> Result<M, PrometheusError>,
) -> Result<Self, String> {
// since standalone metrics are updating themselves, we may just ignore the fact that the same
// standalone metric is exposed by several loops && only spawn single metric
) -> Result<Self, Error> {
// since standalone metrics are updating themselves, we may just ignore the fact that the
// same standalone metric is exposed by several loops && only spawn single metric
match create_metric(&self.registry, self.metrics_prefix.as_deref()) {
Ok(standalone_metrics) => standalone_metrics.spawn(),
Err(PrometheusError::AlreadyReg) => (),
Err(e) => return Err(e.to_string()),
Err(e) => return Err(e.into()),
}
Ok(self)
@@ -191,15 +203,13 @@ impl<SC, TC, LM> LoopMetrics<SC, TC, LM> {
/// Expose metrics using address passed at creation.
///
/// If passed `address` is `None`, metrics are not exposed.
pub async fn expose(self) -> Result<Loop<SC, TC, LM>, String> {
pub async fn expose(self) -> Result<Loop<SC, TC, LM>, Error> {
if let Some(address) = self.address {
let socket_addr = SocketAddr::new(
address.host.parse().map_err(|err| {
format!(
"Invalid host {} is used to expose Prometheus metrics: {}",
address.host, err,
)
})?,
address
.host
.parse()
.map_err(|err| Error::ExposingMetricsInvalidHost(address.host.clone(), err))?,
address.port,
);
@@ -242,8 +252,8 @@ pub async fn reconnect_failed_client(
reconnect_delay.as_secs(),
error,
);
continue;
}
continue
},
}
}
if failed_client == FailedClient::Both || failed_client == FailedClient::Target {
@@ -256,12 +266,12 @@ pub async fn reconnect_failed_client(
reconnect_delay.as_secs(),
error,
);
continue;
}
continue
},
}
}
break;
break
}
}
@@ -270,8 +280,9 @@ fn create_metrics_registry(prefix: Option<String>) -> Registry {
match prefix {
Some(prefix) => {
assert!(!prefix.is_empty(), "Metrics prefix can not be empty");
Registry::new_custom(Some(prefix), None).expect("only fails if prefix is empty; prefix is not empty; qed")
}
Registry::new_custom(Some(prefix), None)
.expect("only fails if prefix is empty; prefix is not empty; qed")
},
None => Registry::new(),
}
}