polkadot-node-subsystems: ChainApiBackend added + polkadot-debug image version fixed (#2411)

The out-dated version (bad tag) of [polkadot
image](https://github.com/paritytech/polkadot-sdk/blob/ede4a362622dfa69eb167eaa876246b1289f4b41/.gitlab/pipeline/zombienet/cumulus.yml#L31)
([docker
info](https://hub.docker.com/layers/paritypr/polkadot-debug/master/images/sha256:adb1658052cf671b50c90d5cece5c7a131efa1a95978249bd5cb85a5ad654f7a?context=explore))
was used. This PR fixes this.

See also:
https://github.com/paritytech/polkadot-sdk/pull/2411#issuecomment-1822632724

Also adds an abstraction that allows asynchronous backends to be passed to `ChainApiSubsystem`
---------

Co-authored-by: Sebastian Kunert <skunert49@gmail.com>
This commit is contained in:
Michal Kucharczyk
2023-11-22 19:24:11 +01:00
committed by GitHub
parent 0956357b60
commit 49874882cf
12 changed files with 213 additions and 55 deletions
+1 -1
View File
@@ -28,7 +28,7 @@
- job: build-push-image-test-parachain
artifacts: true
variables:
POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master"
POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:${DOCKER_IMAGES_VERSION}"
GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet/tests"
LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/cumulus/zombienet/tests"
COL_IMAGE: "docker.io/paritypr/test-parachain:${DOCKER_IMAGES_VERSION}"
Generated
+6
View File
@@ -4183,6 +4183,7 @@ dependencies = [
"polkadot-core-primitives",
"polkadot-network-bridge",
"polkadot-node-collation-generation",
"polkadot-node-core-chain-api",
"polkadot-node-core-prospective-parachains",
"polkadot-node-core-runtime-api",
"polkadot-node-network-protocol",
@@ -4190,16 +4191,19 @@ dependencies = [
"polkadot-overseer",
"polkadot-primitives",
"sc-authority-discovery",
"sc-client-api",
"sc-network",
"sc-network-common",
"sc-service",
"sc-tracing",
"sc-utils",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-consensus-babe",
"sp-runtime",
"substrate-prometheus-endpoint",
"tokio",
"tracing",
]
@@ -12443,6 +12447,7 @@ dependencies = [
"polkadot-node-primitives",
"polkadot-node-subsystem",
"polkadot-node-subsystem-test-helpers",
"polkadot-node-subsystem-types",
"polkadot-primitives",
"sc-client-api",
"sc-consensus-babe",
@@ -12851,6 +12856,7 @@ dependencies = [
"smallvec",
"sp-api",
"sp-authority-discovery",
"sp-blockchain",
"sp-consensus-babe",
"substrate-prometheus-endpoint",
"thiserror",
@@ -19,6 +19,7 @@ polkadot-collator-protocol = { path = "../../../polkadot/node/network/collator-p
polkadot-network-bridge = { path = "../../../polkadot/node/network/bridge" }
polkadot-node-collation-generation = { path = "../../../polkadot/node/collation-generation" }
polkadot-node-core-runtime-api = { path = "../../../polkadot/node/core/runtime-api" }
polkadot-node-core-chain-api = { path = "../../../polkadot/node/core/chain-api" }
polkadot-node-core-prospective-parachains = { path = "../../../polkadot/node/core/prospective-parachains" }
# substrate deps
@@ -26,6 +27,7 @@ sc-authority-discovery = { path = "../../../substrate/client/authority-discovery
sc-network = { path = "../../../substrate/client/network" }
sc-network-common = { path = "../../../substrate/client/network/common" }
sc-service = { path = "../../../substrate/client/service" }
sc-client-api = { path = "../../../substrate/client/api" }
substrate-prometheus-endpoint = { path = "../../../substrate/utils/prometheus" }
sc-tracing = { path = "../../../substrate/client/tracing" }
sc-utils = { path = "../../../substrate/client/utils" }
@@ -33,6 +35,8 @@ sp-api = { path = "../../../substrate/primitives/api" }
sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" }
sp-consensus = { path = "../../../substrate/primitives/consensus/common" }
sp-runtime = { path = "../../../substrate/primitives/runtime" }
sp-blockchain = { path = "../../../substrate/primitives/blockchain" }
tokio = { version = "1.32.0", features = ["macros"] }
# cumulus deps
cumulus-relay-chain-interface = { path = "../relay-chain-interface" }
@@ -20,14 +20,16 @@ use cumulus_relay_chain_interface::{RelayChainError, RelayChainResult};
use cumulus_relay_chain_rpc_interface::RelayChainRpcClient;
use futures::{Stream, StreamExt};
use polkadot_core_primitives::{Block, BlockNumber, Hash, Header};
use polkadot_overseer::RuntimeApiSubsystemClient;
use polkadot_overseer::{ChainApiBackend, RuntimeApiSubsystemClient};
use polkadot_primitives::{
async_backing::{AsyncBackingParams, BackingState},
slashing,
vstaging::NodeFeatures,
};
use sc_authority_discovery::{AuthorityDiscovery, Error as AuthorityDiscoveryError};
use sp_api::{ApiError, RuntimeApiInfo};
use sc_client_api::AuxStore;
use sp_api::{ApiError, BlockT, HeaderT, NumberFor, RuntimeApiInfo};
use sp_blockchain::Info;
#[derive(Clone)]
pub struct BlockChainRpcClient {
@@ -54,6 +56,64 @@ impl BlockChainRpcClient {
}
}
#[async_trait::async_trait]
impl ChainApiBackend for BlockChainRpcClient {
async fn header(
&self,
hash: <Block as BlockT>::Hash,
) -> sp_blockchain::Result<Option<<Block as BlockT>::Header>> {
Ok(self.rpc_client.chain_get_header(Some(hash)).await?)
}
async fn info(&self) -> sp_blockchain::Result<Info<Block>> {
let (best_header_opt, genesis_hash, finalized_head) = futures::try_join!(
self.rpc_client.chain_get_header(None),
self.rpc_client.chain_get_head(Some(0)),
self.rpc_client.chain_get_finalized_head()
)?;
let best_header = best_header_opt.ok_or_else(|| {
RelayChainError::GenericError(
"Unable to retrieve best header from relay chain.".to_string(),
)
})?;
let finalized_header =
self.rpc_client.chain_get_header(Some(finalized_head)).await?.ok_or_else(|| {
RelayChainError::GenericError(
"Unable to retrieve finalized header from relay chain.".to_string(),
)
})?;
Ok(Info {
best_hash: best_header.hash(),
best_number: best_header.number,
genesis_hash,
finalized_hash: finalized_head,
finalized_number: finalized_header.number,
finalized_state: Some((finalized_header.hash(), finalized_header.number)),
number_leaves: 1,
block_gap: None,
})
}
async fn number(
&self,
hash: <Block as BlockT>::Hash,
) -> sp_blockchain::Result<Option<<<Block as BlockT>::Header as HeaderT>::Number>> {
Ok(self
.rpc_client
.chain_get_header(Some(hash))
.await?
.map(|maybe_header| maybe_header.number))
}
async fn hash(
&self,
number: NumberFor<Block>,
) -> sp_blockchain::Result<Option<<Block as BlockT>::Hash>> {
Ok(self.rpc_client.chain_get_block_hash(number.into()).await?)
}
}
#[async_trait::async_trait]
impl RuntimeApiSubsystemClient for BlockChainRpcClient {
async fn validators(
@@ -403,3 +463,25 @@ impl BlockChainRpcClient {
Ok(self.rpc_client.get_finalized_heads_stream()?.boxed())
}
}
// Implementation required by ChainApiSubsystem
// but never called in our case.
impl AuxStore for BlockChainRpcClient {
fn insert_aux<
'a,
'b: 'a,
'c: 'a,
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
>(
&self,
_insert: I,
_delete: D,
) -> sp_blockchain::Result<()> {
unimplemented!("Not supported on the RPC collator")
}
fn get_aux(&self, _key: &[u8]) -> sp_blockchain::Result<Option<Vec<u8>>> {
unimplemented!("Not supported on the RPC collator")
}
}
@@ -24,6 +24,7 @@ use polkadot_network_bridge::{
NetworkBridgeTx as NetworkBridgeTxSubsystem,
};
use polkadot_node_collation_generation::CollationGenerationSubsystem;
use polkadot_node_core_chain_api::ChainApiSubsystem;
use polkadot_node_core_prospective_parachains::ProspectiveParachainsSubsystem;
use polkadot_node_core_runtime_api::RuntimeApiSubsystem;
use polkadot_node_network_protocol::{
@@ -112,7 +113,7 @@ fn build_overseer(
.candidate_backing(DummySubsystem)
.candidate_validation(DummySubsystem)
.pvf_checker(DummySubsystem)
.chain_api(DummySubsystem)
.chain_api(ChainApiSubsystem::new(runtime_client.clone(), Metrics::register(registry)?))
.collation_generation(CollationGenerationSubsystem::new(Metrics::register(registry)?))
.collator_protocol({
let side = ProtocolSide::Collator {
+3 -2
View File
@@ -9,10 +9,9 @@ description = "The Chain API subsystem provides access to chain related utility
[dependencies]
futures = "0.3.21"
gum = { package = "tracing-gum", path = "../../gum" }
sp-blockchain = { path = "../../../../substrate/primitives/blockchain" }
polkadot-primitives = { path = "../../../primitives" }
polkadot-node-metrics = { path = "../../metrics" }
polkadot-node-subsystem = { path = "../../subsystem" }
polkadot-node-subsystem-types = { path = "../../subsystem-types" }
sc-client-api = { path = "../../../../substrate/client/api" }
sc-consensus-babe = { path = "../../../../substrate/client/consensus/babe" }
@@ -21,5 +20,7 @@ futures = { version = "0.3.21", features = ["thread-pool"] }
maplit = "1.0.2"
parity-scale-codec = "3.6.1"
polkadot-node-primitives = { path = "../../primitives" }
polkadot-primitives = { path = "../../../primitives" }
polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
sp-core = { path = "../../../../substrate/primitives/core" }
sp-blockchain = { path = "../../../../substrate/primitives/blockchain" }
+42 -35
View File
@@ -35,13 +35,13 @@ use std::sync::Arc;
use futures::prelude::*;
use sc_client_api::AuxStore;
use sp_blockchain::HeaderBackend;
use futures::stream::StreamExt;
use polkadot_node_subsystem::{
messages::ChainApiMessage, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem,
SubsystemError, SubsystemResult,
};
use polkadot_primitives::Block;
use polkadot_node_subsystem_types::ChainApiBackend;
mod metrics;
use self::metrics::Metrics;
@@ -67,7 +67,7 @@ impl<Client> ChainApiSubsystem<Client> {
#[overseer::subsystem(ChainApi, error = SubsystemError, prefix = self::overseer)]
impl<Client, Context> ChainApiSubsystem<Client>
where
Client: HeaderBackend<Block> + AuxStore + 'static,
Client: ChainApiBackend + AuxStore + 'static,
{
fn start(self, ctx: Context) -> SpawnedSubsystem {
let future = run::<Client, Context>(ctx, self)
@@ -83,7 +83,7 @@ async fn run<Client, Context>(
subsystem: ChainApiSubsystem<Client>,
) -> SubsystemResult<()>
where
Client: HeaderBackend<Block> + AuxStore,
Client: ChainApiBackend + AuxStore,
{
loop {
match ctx.recv().await? {
@@ -93,13 +93,15 @@ where
FromOrchestra::Communication { msg } => match msg {
ChainApiMessage::BlockNumber(hash, response_channel) => {
let _timer = subsystem.metrics.time_block_number();
let result = subsystem.client.number(hash).map_err(|e| e.to_string().into());
let result =
subsystem.client.number(hash).await.map_err(|e| e.to_string().into());
subsystem.metrics.on_request(result.is_ok());
let _ = response_channel.send(result);
},
ChainApiMessage::BlockHeader(hash, response_channel) => {
let _timer = subsystem.metrics.time_block_header();
let result = subsystem.client.header(hash).map_err(|e| e.to_string().into());
let result =
subsystem.client.header(hash).await.map_err(|e| e.to_string().into());
subsystem.metrics.on_request(result.is_ok());
let _ = response_channel.send(result);
},
@@ -113,46 +115,51 @@ where
ChainApiMessage::FinalizedBlockHash(number, response_channel) => {
let _timer = subsystem.metrics.time_finalized_block_hash();
// Note: we don't verify it's finalized
let result = subsystem.client.hash(number).map_err(|e| e.to_string().into());
let result =
subsystem.client.hash(number).await.map_err(|e| e.to_string().into());
subsystem.metrics.on_request(result.is_ok());
let _ = response_channel.send(result);
},
ChainApiMessage::FinalizedBlockNumber(response_channel) => {
let _timer = subsystem.metrics.time_finalized_block_number();
let result = subsystem.client.info().finalized_number;
// always succeeds
subsystem.metrics.on_request(true);
let _ = response_channel.send(Ok(result));
let result = subsystem
.client
.info()
.await
.map_err(|e| e.to_string().into())
.map(|info| info.finalized_number);
subsystem.metrics.on_request(result.is_ok());
let _ = response_channel.send(result);
},
ChainApiMessage::Ancestors { hash, k, response_channel } => {
let _timer = subsystem.metrics.time_ancestors();
gum::trace!(target: LOG_TARGET, hash=%hash, k=k, "ChainApiMessage::Ancestors");
let mut hash = hash;
let next_parent_stream = futures::stream::unfold(
(hash, subsystem.client.clone()),
|(hash, client)| async move {
let maybe_header = client.header(hash).await;
match maybe_header {
// propagate the error
Err(e) => {
let e = e.to_string().into();
Some((Err(e), (hash, client)))
},
// fewer than `k` ancestors are available
Ok(None) => None,
Ok(Some(header)) => {
// stop at the genesis header.
if header.number == 0 {
None
} else {
Some((Ok(header.parent_hash), (header.parent_hash, client)))
}
},
}
},
);
let next_parent = core::iter::from_fn(|| {
let maybe_header = subsystem.client.header(hash);
match maybe_header {
// propagate the error
Err(e) => {
let e = e.to_string().into();
Some(Err(e))
},
// fewer than `k` ancestors are available
Ok(None) => None,
Ok(Some(header)) => {
// stop at the genesis header.
if header.number == 0 {
None
} else {
hash = header.parent_hash;
Some(Ok(hash))
}
},
}
});
let result = next_parent.take(k).collect::<Result<Vec<_>, _>>();
let result = next_parent_stream.take(k).try_collect().await;
subsystem.metrics.on_request(result.is_ok());
let _ = response_channel.send(result);
},
+12 -9
View File
@@ -22,7 +22,8 @@ use std::collections::BTreeMap;
use polkadot_node_primitives::BlockWeight;
use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle};
use polkadot_primitives::{BlockNumber, Hash, Header};
use polkadot_node_subsystem_types::ChainApiBackend;
use polkadot_primitives::{Block, BlockNumber, Hash, Header};
use sp_blockchain::Info as BlockInfo;
use sp_core::testing::TaskExecutor;
@@ -110,7 +111,7 @@ fn last_key_value<K: Clone, V: Clone>(map: &BTreeMap<K, V>) -> (K, V) {
map.iter().last().map(|(k, v)| (k.clone(), v.clone())).unwrap()
}
impl HeaderBackend<Block> for TestClient {
impl sp_blockchain::HeaderBackend<Block> for TestClient {
fn info(&self) -> BlockInfo<Block> {
let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap();
let (best_hash, best_number) = last_key_value(&self.blocks);
@@ -191,8 +192,8 @@ fn request_block_number() {
async move {
let zero = Hash::zero();
let test_cases = [
(TWO, client.number(TWO).unwrap()),
(zero, client.number(zero).unwrap()), // not here
(TWO, client.number(TWO).await.unwrap()),
(zero, client.number(zero).await.unwrap()), // not here
];
for (hash, expected) in &test_cases {
let (tx, rx) = oneshot::channel();
@@ -217,8 +218,10 @@ fn request_block_header() {
test_harness(|client, mut sender| {
async move {
const NOT_HERE: Hash = Hash::repeat_byte(0x5);
let test_cases =
[(TWO, client.header(TWO).unwrap()), (NOT_HERE, client.header(NOT_HERE).unwrap())];
let test_cases = [
(TWO, client.header(TWO).await.unwrap()),
(NOT_HERE, client.header(NOT_HERE).await.unwrap()),
];
for (hash, expected) in &test_cases {
let (tx, rx) = oneshot::channel();
@@ -270,8 +273,8 @@ fn request_finalized_hash() {
test_harness(|client, mut sender| {
async move {
let test_cases = [
(1, client.hash(1).unwrap()), // not here
(2, client.hash(2).unwrap()),
(1, client.hash(1).await.unwrap()), // not here
(2, client.hash(2).await.unwrap()),
];
for (number, expected) in &test_cases {
let (tx, rx) = oneshot::channel();
@@ -297,7 +300,7 @@ fn request_last_finalized_number() {
async move {
let (tx, rx) = oneshot::channel();
let expected = client.info().finalized_number;
let expected = client.info().await.unwrap().finalized_number;
sender
.send(FromOrchestra::Communication {
msg: ChainApiMessage::FinalizedBlockNumber(tx),
+2 -2
View File
@@ -87,8 +87,8 @@ use polkadot_node_subsystem_types::messages::{
pub use polkadot_node_subsystem_types::{
errors::{SubsystemError, SubsystemResult},
jaeger, ActivatedLeaf, ActiveLeavesUpdate, OverseerSignal, RuntimeApiSubsystemClient,
UnpinHandle,
jaeger, ActivatedLeaf, ActiveLeavesUpdate, ChainApiBackend, OverseerSignal,
RuntimeApiSubsystemClient, UnpinHandle,
};
pub mod metrics;
+1
View File
@@ -17,6 +17,7 @@ polkadot-node-jaeger = { path = "../jaeger" }
orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] }
sc-network = { path = "../../../substrate/client/network" }
sp-api = { path = "../../../substrate/primitives/api" }
sp-blockchain = { path = "../../../substrate/primitives/blockchain" }
sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" }
sp-authority-discovery = { path = "../../../substrate/primitives/authority-discovery" }
sc-client-api = { path = "../../../substrate/client/api" }
+1 -1
View File
@@ -40,7 +40,7 @@ pub mod errors;
pub mod messages;
mod runtime_client;
pub use runtime_client::{DefaultSubsystemClient, RuntimeApiSubsystemClient};
pub use runtime_client::{ChainApiBackend, DefaultSubsystemClient, RuntimeApiSubsystemClient};
pub use jaeger::*;
pub use polkadot_node_jaeger as jaeger;
@@ -18,17 +18,70 @@ use async_trait::async_trait;
use polkadot_primitives::{
async_backing, runtime_api::ParachainHost, slashing, vstaging, Block, BlockNumber,
CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState,
DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id, InboundDownwardMessage,
DisputeState, ExecutorParams, GroupRotationInfo, Hash, Header, Id, InboundDownwardMessage,
InboundHrmpMessage, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement,
ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash,
ValidatorId, ValidatorIndex, ValidatorSignature,
};
use sc_client_api::HeaderBackend;
use sc_transaction_pool_api::OffchainTransactionPoolFactory;
use sp_api::{ApiError, ApiExt, ProvideRuntimeApi};
use sp_api::{ApiError, ApiExt, HeaderT, NumberFor, ProvideRuntimeApi};
use sp_authority_discovery::AuthorityDiscoveryApi;
use sp_blockchain::Info;
use sp_consensus_babe::{BabeApi, Epoch};
use std::{collections::BTreeMap, sync::Arc};
/// Offers header utilities.
///
/// This is a async wrapper trait for ['HeaderBackend'] to be used with the
/// `ChainApiSubsystem`.
// This trait was introduced to suit the needs of collators. Depending on their operating mode, they
// might not have a client of the relay chain that can supply a synchronous HeaderBackend
// implementation.
#[async_trait]
pub trait ChainApiBackend: Send + Sync {
/// Get block header. Returns `None` if block is not found.
async fn header(&self, hash: Hash) -> sp_blockchain::Result<Option<Header>>;
/// Get blockchain info.
async fn info(&self) -> sp_blockchain::Result<Info<Block>>;
/// Get block number by hash. Returns `None` if the header is not in the chain.
async fn number(
&self,
hash: Hash,
) -> sp_blockchain::Result<Option<<Header as HeaderT>::Number>>;
/// Get block hash by number. Returns `None` if the header is not in the chain.
async fn hash(&self, number: NumberFor<Block>) -> sp_blockchain::Result<Option<Hash>>;
}
#[async_trait]
impl<T> ChainApiBackend for T
where
T: HeaderBackend<Block>,
{
/// Get block header. Returns `None` if block is not found.
async fn header(&self, hash: Hash) -> sp_blockchain::Result<Option<Header>> {
HeaderBackend::header(self, hash)
}
/// Get blockchain info.
async fn info(&self) -> sp_blockchain::Result<Info<Block>> {
Ok(HeaderBackend::info(self))
}
/// Get block number by hash. Returns `None` if the header is not in the chain.
async fn number(
&self,
hash: Hash,
) -> sp_blockchain::Result<Option<<Header as HeaderT>::Number>> {
HeaderBackend::number(self, hash)
}
/// Get block hash by number. Returns `None` if the header is not in the chain.
async fn hash(&self, number: NumberFor<Block>) -> sp_blockchain::Result<Option<Hash>> {
HeaderBackend::hash(self, number)
}
}
/// Exposes all runtime calls that are used by the runtime API subsystem.
#[async_trait]
pub trait RuntimeApiSubsystemClient {