dispute-coordinator: past session dispute slashing (#6811)

* runtime/vstaging: unapplied_slashes runtime API

* runtime/vstaging: key_ownership_proof runtime API

* runtime/ParachainHost: submit_report_dispute_lost

* fix key_ownership_proof API

* runtime: submit_report_dispute_lost runtime API

* nits

* Update node/subsystem-types/src/messages.rs

Co-authored-by: Marcin S. <marcin@bytedude.com>

* revert unrelated fmt changes

* dispute-coordinator: past session dispute slashing

* encapsule runtime api call for submitting report

* prettify: extract into a function

* do not exit on runtime api error

* fix tests

* try initial zombienet test

* try something

* fix a typo

* try cumulus-based collator

* fix clippy

* build polkadot-debug images with fast-runtime enabled

* wip

* runtime/inclusion: fix availability_threshold

* fix wip

* fix wip II

* revert native provider

* propagate tx submission error

* DEBUG: sync=trace

* print key ownership proof len

* panic repro

* log validator index in panic message

* post merge fixes

* replace debug assertion with a log

* fix compilation

* Let's log the dispatch info in validate block.

* fix double encoding

* Revert "Let's log the dispatch info in validate block."

This reverts commit a70fbc51b464d7f4355dbada5e16cd83cf71eab4.

* Revert "Let's log the dispatch info in validate block."

This reverts commit a70fbc51b464d7f4355dbada5e16cd83cf71eab4.

* fix compilation

* update to latest zombienet and fix test

* lower finality lag to 11

* bump zombienet again

* add a workaround, but still does not work

* Update .gitlab-ci.yml

bump zombienet.

* add a comment and search logs on all nodes

---------

Co-authored-by: Marcin S. <marcin@bytedude.com>
Co-authored-by: Bastian Köcher <info@kchr.de>
Co-authored-by: Javier Viola <javier@parity.io>
This commit is contained in:
ordian
2023-06-05 18:21:42 +02:00
committed by GitHub
parent f2fe05a757
commit 01a19b45e3
12 changed files with 407 additions and 46 deletions
@@ -39,9 +39,11 @@ use polkadot_node_subsystem::{
},
overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, OverseerSignal,
};
use polkadot_node_subsystem_util::runtime::RuntimeInfo;
use polkadot_node_subsystem_util::runtime::{
key_ownership_proof, submit_report_dispute_lost, RuntimeInfo,
};
use polkadot_primitives::{
BlockNumber, CandidateHash, CandidateReceipt, CompactStatement, DisputeStatement,
vstaging, BlockNumber, CandidateHash, CandidateReceipt, CompactStatement, DisputeStatement,
DisputeStatementSet, Hash, ScrapedOnChainVotes, SessionIndex, ValidDisputeStatementKind,
ValidatorId, ValidatorIndex,
};
@@ -52,6 +54,7 @@ use crate::{
import::{CandidateEnvironment, CandidateVoteState},
is_potential_spam,
metrics::Metrics,
scraping::ScrapedUpdates,
status::{get_active_with_status, Clock},
DisputeCoordinatorSubsystem, LOG_TARGET,
};
@@ -348,27 +351,167 @@ impl Initialized {
},
}
let ScrapedUpdates { unapplied_slashes, on_chain_votes, .. } = scraped_updates;
self.process_unapplied_slashes(ctx, new_leaf.hash, unapplied_slashes).await;
gum::trace!(
target: LOG_TARGET,
timestamp = now,
"Will process {} onchain votes",
scraped_updates.on_chain_votes.len()
on_chain_votes.len()
);
self.process_chain_import_backlog(
ctx,
overlay_db,
scraped_updates.on_chain_votes,
now,
new_leaf.hash,
)
.await;
self.process_chain_import_backlog(ctx, overlay_db, on_chain_votes, now, new_leaf.hash)
.await;
}
gum::trace!(target: LOG_TARGET, timestamp = now, "Done processing ActiveLeavesUpdate");
Ok(())
}
/// For each unapplied (past-session) slash, report an unsigned extrinsic
/// to the runtime.
async fn process_unapplied_slashes<Context>(
&mut self,
ctx: &mut Context,
relay_parent: Hash,
unapplied_slashes: Vec<(SessionIndex, CandidateHash, vstaging::slashing::PendingSlashes)>,
) {
for (session_index, candidate_hash, pending) in unapplied_slashes {
gum::info!(
target: LOG_TARGET,
?session_index,
?candidate_hash,
n_slashes = pending.keys.len(),
"Processing unapplied validator slashes",
);
let inclusions = self.scraper.get_blocks_including_candidate(&candidate_hash);
if inclusions.is_empty() {
gum::info!(
target: LOG_TARGET,
"Couldn't find inclusion parent for an unapplied slash",
);
return
}
// Find the first inclusion parent that we can use
// to generate key ownership proof on.
// We use inclusion parents because of the proper session index.
let mut key_ownership_proofs = Vec::new();
let mut dispute_proofs = Vec::new();
for (_height, inclusion_parent) in inclusions {
for (validator_index, validator_id) in pending.keys.iter() {
let res =
key_ownership_proof(ctx.sender(), inclusion_parent, validator_id.clone())
.await;
match res {
Ok(Some(key_ownership_proof)) => {
key_ownership_proofs.push(key_ownership_proof);
let time_slot = vstaging::slashing::DisputesTimeSlot::new(
session_index,
candidate_hash,
);
let dispute_proof = vstaging::slashing::DisputeProof {
time_slot,
kind: pending.kind,
validator_index: *validator_index,
validator_id: validator_id.clone(),
};
dispute_proofs.push(dispute_proof);
},
Ok(None) => {},
Err(error) => {
gum::debug!(
target: LOG_TARGET,
?error,
?session_index,
?candidate_hash,
?validator_id,
"Could not generate key ownership proof",
);
},
}
}
if !key_ownership_proofs.is_empty() {
// If we found a parent that we can use, stop searching.
// If one key ownership was resolved successfully, all of them should be.
debug_assert_eq!(key_ownership_proofs.len(), pending.keys.len());
break
}
}
let expected_keys = pending.keys.len();
let resolved_keys = key_ownership_proofs.len();
if resolved_keys < expected_keys {
gum::warn!(
target: LOG_TARGET,
?session_index,
?candidate_hash,
"Could not generate key ownership proofs for {} keys",
expected_keys - resolved_keys,
);
}
debug_assert_eq!(resolved_keys, dispute_proofs.len());
for (key_ownership_proof, dispute_proof) in
key_ownership_proofs.into_iter().zip(dispute_proofs.into_iter())
{
let validator_id = dispute_proof.validator_id.clone();
gum::info!(
target: LOG_TARGET,
?session_index,
?candidate_hash,
key_ownership_proof_len = key_ownership_proof.len(),
"Trying to submit a slashing report",
);
let res = submit_report_dispute_lost(
ctx.sender(),
relay_parent,
dispute_proof,
key_ownership_proof,
)
.await;
match res {
Err(error) => {
gum::warn!(
target: LOG_TARGET,
?error,
?session_index,
?candidate_hash,
"Error reporting pending slash",
);
},
Ok(Some(())) => {
gum::info!(
target: LOG_TARGET,
?session_index,
?candidate_hash,
?validator_id,
"Successfully reported pending slash",
);
},
Ok(None) => {
gum::debug!(
target: LOG_TARGET,
?session_index,
?candidate_hash,
?validator_id,
"Duplicate pending slash report",
);
},
}
}
}
}
/// Process one batch of our `chain_import_backlog`.
///
/// `new_votes` will be appended beforehand.
@@ -475,10 +618,11 @@ impl Initialized {
validator_public.clone(),
validator_signature.clone(),
).is_ok(),
"Scraped backing votes had invalid signature! candidate: {:?}, session: {:?}, validator_public: {:?}",
"Scraped backing votes had invalid signature! candidate: {:?}, session: {:?}, validator_public: {:?}, validator_index: {}",
candidate_hash,
session,
validator_public,
validator_index.0,
);
let signed_dispute_statement =
SignedDisputeStatement::new_unchecked_from_trusted_source(
@@ -27,9 +27,12 @@ use polkadot_node_subsystem::{
messages::ChainApiMessage, overseer, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError,
SubsystemSender,
};
use polkadot_node_subsystem_util::runtime::{get_candidate_events, get_on_chain_votes};
use polkadot_node_subsystem_util::runtime::{
get_candidate_events, get_on_chain_votes, get_unapplied_slashes,
};
use polkadot_primitives::{
BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, Hash, ScrapedOnChainVotes,
vstaging, BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, Hash,
ScrapedOnChainVotes, SessionIndex,
};
use crate::{
@@ -64,11 +67,16 @@ const LRU_OBSERVED_BLOCKS_CAPACITY: NonZeroUsize = match NonZeroUsize::new(20) {
pub struct ScrapedUpdates {
pub on_chain_votes: Vec<ScrapedOnChainVotes>,
pub included_receipts: Vec<CandidateReceipt>,
pub unapplied_slashes: Vec<(SessionIndex, CandidateHash, vstaging::slashing::PendingSlashes)>,
}
impl ScrapedUpdates {
pub fn new() -> Self {
Self { on_chain_votes: Vec::new(), included_receipts: Vec::new() }
Self {
on_chain_votes: Vec::new(),
included_receipts: Vec::new(),
unapplied_slashes: Vec::new(),
}
}
}
@@ -120,7 +128,7 @@ impl Inclusions {
.retain(|_, blocks_including| blocks_including.keys().len() > 0);
}
pub fn get(&mut self, candidate: &CandidateHash) -> Vec<(BlockNumber, Hash)> {
pub fn get(&self, candidate: &CandidateHash) -> Vec<(BlockNumber, Hash)> {
let mut inclusions_as_vec: Vec<(BlockNumber, Hash)> = Vec::new();
if let Some(blocks_including) = self.inclusions_inner.get(candidate) {
for (height, blocks_at_height) in blocks_including.iter() {
@@ -256,6 +264,22 @@ impl ChainScraper {
}
}
// for unapplied slashes, we only look at the latest activated hash,
// it should accumulate them all
match get_unapplied_slashes(sender, activated.hash).await {
Ok(unapplied_slashes) => {
scraped_updates.unapplied_slashes = unapplied_slashes;
},
Err(error) => {
gum::debug!(
target: LOG_TARGET,
block_hash = ?activated.hash,
?error,
"Error fetching unapplied slashes.",
);
},
}
self.last_observed_blocks.put(activated.hash, ());
Ok(scraped_updates)
@@ -403,7 +427,7 @@ impl ChainScraper {
}
pub fn get_blocks_including_candidate(
&mut self,
&self,
candidate: &CandidateHash,
) -> Vec<(BlockNumber, Hash)> {
self.inclusions.get(candidate)
@@ -81,6 +81,7 @@ impl TestState {
)
.await;
assert_chain_vote_request(&mut ctx_handle, &chain).await;
assert_unapplied_slashes_request(&mut ctx_handle, &chain).await;
};
let (scraper, _) = join(ChainScraper::new(ctx.sender(), leaf.clone()), overseer_fut)
@@ -242,6 +243,18 @@ async fn assert_chain_vote_request(virtual_overseer: &mut VirtualOverseer, _chai
);
}
async fn assert_unapplied_slashes_request(virtual_overseer: &mut VirtualOverseer, _chain: &[Hash]) {
assert_matches!(
overseer_recv(virtual_overseer).await,
AllMessages::RuntimeApi(RuntimeApiMessage::Request(
_hash,
RuntimeApiRequest::UnappliedSlashes(tx),
)) => {
tx.send(Ok(Vec::new())).unwrap();
}
);
}
async fn assert_finalized_block_number_request(
virtual_overseer: &mut VirtualOverseer,
response: BlockNumber,
@@ -287,6 +300,7 @@ async fn overseer_process_active_leaves_update<F>(
assert_candidate_events_request(virtual_overseer, chain, event_generator.clone()).await;
assert_chain_vote_request(virtual_overseer, chain).await;
}
assert_unapplied_slashes_request(virtual_overseer, chain).await;
}
#[test]
@@ -385,6 +385,12 @@ impl TestState {
})))
.unwrap();
},
AllMessages::RuntimeApi(RuntimeApiMessage::Request(
_new_leaf,
RuntimeApiRequest::UnappliedSlashes(tx),
)) => {
tx.send(Ok(Vec::new())).unwrap();
},
AllMessages::ChainApi(ChainApiMessage::Ancestors { hash, k, response_channel }) => {
let target_header = self
.headers
+6 -3
View File
@@ -42,8 +42,8 @@ use futures::channel::{mpsc, oneshot};
use parity_scale_codec::Encode;
use polkadot_primitives::{
AuthorityDiscoveryId, CandidateEvent, CommittedCandidateReceipt, CoreState, EncodeAs,
GroupIndex, GroupRotationInfo, Hash, Id as ParaId, OccupiedCoreAssumption,
vstaging, AuthorityDiscoveryId, CandidateEvent, CandidateHash, CommittedCandidateReceipt,
CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, Id as ParaId, OccupiedCoreAssumption,
PersistedValidationData, ScrapedOnChainVotes, SessionIndex, SessionInfo, Signed,
SigningContext, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex,
ValidatorSignature,
@@ -211,7 +211,10 @@ specialize_requests! {
fn request_validation_code_hash(para_id: ParaId, assumption: OccupiedCoreAssumption)
-> Option<ValidationCodeHash>; ValidationCodeHash;
fn request_on_chain_votes() -> Option<ScrapedOnChainVotes>; FetchOnChainVotes;
fn request_session_executor_params(session_index: SessionIndex) -> Option<ExecutorParams>; SessionExecutorParams;
fn request_session_executor_params(session_index: SessionIndex) -> Option<ExecutorParams>;SessionExecutorParams;
fn request_unapplied_slashes() -> Vec<(SessionIndex, CandidateHash, vstaging::slashing::PendingSlashes)>; UnappliedSlashes;
fn request_key_ownership_proof(validator_id: ValidatorId) -> Option<vstaging::slashing::OpaqueKeyOwnershipProof>; KeyOwnershipProof;
fn request_submit_report_dispute_lost(dp: vstaging::slashing::DisputeProof, okop: vstaging::slashing::OpaqueKeyOwnershipProof) -> Option<()>; SubmitReportDisputeLost;
}
/// Requests executor parameters from the runtime effective at given relay-parent. First obtains
@@ -27,14 +27,16 @@ use sp_keystore::{Keystore, KeystorePtr};
use polkadot_node_subsystem::{messages::RuntimeApiMessage, overseer, SubsystemSender};
use polkadot_primitives::{
CandidateEvent, CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, IndexedVec,
OccupiedCore, ScrapedOnChainVotes, SessionIndex, SessionInfo, Signed, SigningContext,
UncheckedSigned, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex,
vstaging, CandidateEvent, CandidateHash, CoreState, EncodeAs, GroupIndex, GroupRotationInfo,
Hash, IndexedVec, OccupiedCore, ScrapedOnChainVotes, SessionIndex, SessionInfo, Signed,
SigningContext, UncheckedSigned, ValidationCode, ValidationCodeHash, ValidatorId,
ValidatorIndex,
};
use crate::{
request_availability_cores, request_candidate_events, request_on_chain_votes,
request_session_index_for_child, request_session_info, request_validation_code_by_hash,
request_availability_cores, request_candidate_events, request_key_ownership_proof,
request_on_chain_votes, request_session_index_for_child, request_session_info,
request_submit_report_dispute_lost, request_unapplied_slashes, request_validation_code_by_hash,
request_validator_groups,
};
@@ -343,3 +345,51 @@ where
recv_runtime(request_validation_code_by_hash(relay_parent, validation_code_hash, sender).await)
.await
}
/// Fetch a list of `PendingSlashes` from the runtime.
pub async fn get_unapplied_slashes<Sender>(
sender: &mut Sender,
relay_parent: Hash,
) -> Result<Vec<(SessionIndex, CandidateHash, vstaging::slashing::PendingSlashes)>>
where
Sender: SubsystemSender<RuntimeApiMessage>,
{
recv_runtime(request_unapplied_slashes(relay_parent, sender).await).await
}
/// Generate validator key ownership proof.
///
/// Note: The choice of `relay_parent` is important here, it needs to match
/// the desired session index of the validator set in question.
pub async fn key_ownership_proof<Sender>(
sender: &mut Sender,
relay_parent: Hash,
validator_id: ValidatorId,
) -> Result<Option<vstaging::slashing::OpaqueKeyOwnershipProof>>
where
Sender: SubsystemSender<RuntimeApiMessage>,
{
recv_runtime(request_key_ownership_proof(relay_parent, validator_id, sender).await).await
}
/// Submit a past-session dispute slashing report.
pub async fn submit_report_dispute_lost<Sender>(
sender: &mut Sender,
relay_parent: Hash,
dispute_proof: vstaging::slashing::DisputeProof,
key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof,
) -> Result<Option<()>>
where
Sender: SubsystemSender<RuntimeApiMessage>,
{
recv_runtime(
request_submit_report_dispute_lost(
relay_parent,
dispute_proof,
key_ownership_proof,
sender,
)
.await,
)
.await
}
@@ -96,4 +96,9 @@ impl OpaqueKeyOwnershipProof {
pub fn decode<T: Decode>(self) -> Option<T> {
Decode::decode(&mut &self.0[..]).ok()
}
/// Length of the encoded proof.
pub fn len(&self) -> usize {
self.0.len()
}
}
@@ -462,16 +462,13 @@ pub mod pallet {
) -> DispatchResultWithPostInfo {
ensure_none(origin)?;
let validator_set_count = key_owner_proof.validator_count() as ValidatorSetCount;
// check the membership proof to extract the offender's id
let key = (primitives::PARACHAIN_KEY_TYPE_ID, dispute_proof.validator_id.clone());
let offender = T::KeyOwnerProofSystem::check_proof(key, key_owner_proof)
.ok_or(Error::<T>::InvalidKeyOwnershipProof)?;
let session_index = dispute_proof.time_slot.session_index;
let validator_set_count = crate::session_info::Pallet::<T>::session_info(session_index)
.ok_or(Error::<T>::InvalidSessionIndex)?
.discovery_keys
.len() as ValidatorSetCount;
// check that there is a pending slash for the given
// validator index and candidate hash
@@ -705,22 +702,26 @@ where
};
match SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into()) {
Ok(()) => log::info!(
target: LOG_TARGET,
"Submitted dispute slashing report, session({}), index({}), kind({:?})",
session_index,
validator_index,
kind,
),
Err(()) => log::error!(
target: LOG_TARGET,
"Error submitting dispute slashing report, session({}), index({}), kind({:?})",
session_index,
validator_index,
kind,
),
Ok(()) => {
log::info!(
target: LOG_TARGET,
"Submitted dispute slashing report, session({}), index({}), kind({:?})",
session_index,
validator_index,
kind,
);
Ok(())
},
Err(()) => {
log::error!(
target: LOG_TARGET,
"Error submitting dispute slashing report, session({}), index({}), kind({:?})",
session_index,
validator_index,
kind,
);
Err(sp_runtime::DispatchError::Other(""))
},
}
Ok(())
}
}
@@ -21,7 +21,7 @@ build-linux-stable:
# Ensure we run the UI tests.
RUN_UI_TESTS: 1
script:
- time cargo build --locked --profile testnet --features pyroscope --verbose --bin polkadot
- time cargo build --locked --profile testnet --features pyroscope,fast-runtime --verbose --bin polkadot
# pack artifacts
- mkdir -p ./artifacts
- VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
@@ -124,6 +124,38 @@ zombienet-tests-parachains-disputes-garbage-candidate:
tags:
- zombienet-polkadot-integration-test
zombienet-tests-parachains-disputes-past-session:
stage: zombienet
image: "${ZOMBIENET_IMAGE}"
extends:
- .kubernetes-env
- .zombienet-refs
needs:
- job: publish-polkadot-debug-image
- job: publish-test-collators-image
- job: publish-malus-image
variables:
GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional"
before_script:
- echo "Zombie-net Tests Config"
- echo "${ZOMBIENET_IMAGE_NAME}"
- echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
- echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
- echo "${GH_DIR}"
- export DEBUG=zombie,zombie::network-node
- export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
- export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
- export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
script:
- /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
--github-remote-dir="${GH_DIR}"
--test="0004-parachains-disputes-past-session.zndsl"
allow_failure: true
retry: 2
tags:
- zombienet-polkadot-integration-test
zombienet-test-parachains-upgrade-smoke-test:
stage: zombienet
image: "${ZOMBIENET_IMAGE}"
@@ -0,0 +1,45 @@
[settings]
timeout = 1000
bootnode = true
[relaychain.genesis.runtime.configuration.config]
max_validators_per_core = 1
needed_approvals = 3
group_rotation_frequency = 4
[relaychain]
default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
chain = "westend-local" # using westend-local to enable slashing
default_command = "polkadot"
[relaychain.default_resources]
limits = { memory = "4G", cpu = "2" }
requests = { memory = "2G", cpu = "1" }
[[relaychain.nodes]]
name = "alice"
invulnerable = true # it will go offline, we don't want to disable it
args = ["-lparachain=debug,runtime=debug"]
[[relaychain.node_groups]]
name = "honest-validator"
count = 2
args = ["-lruntime=debug,sync=trace"]
[[relaychain.node_groups]]
image = "{{MALUS_IMAGE}}"
name = "malus-validator"
command = "malus suggest-garbage-candidate"
args = ["-lMALUS=trace"]
count = 1
[[parachains]]
id = 1000
cumulus_based = true
[parachains.collator]
name = "collator"
command = "polkadot-parachain"
image = "docker.io/parity/polkadot-parachain:latest"
# image = "{{COL_IMAGE}}"
args = ["-lparachain=debug"]
@@ -0,0 +1,37 @@
Description: Past-session dispute slashing
Network: ./0004-parachains-disputes-past-session.toml
Creds: config
alice: reports node_roles is 4
# pause alice so that disputes don't conclude
alice: pause
# Ensure parachain is registered.
honest-validator-0: parachain 1000 is registered within 100 seconds
# Ensure parachain made progress.
honest-validator-0: parachain 1000 block height is at least 1 within 300 seconds
# There should be disputes initiated
honest-validator-0: reports parachain_candidate_disputes_total is at least 2 within 200 seconds
# Stop issuing disputes
malus-validator-0: pause
# wait for the next session
sleep 120 seconds
# But should not resolve
honest-validator-0: reports block height minus finalised block is at least 10 within 100 seconds
# Now resume alice
alice: resume
# Disputes should start concluding now
honest-validator-0: reports parachain_candidate_dispute_concluded{validity="invalid"} is at least 1 within 200 seconds
# Disputes should always end as "invalid"
honest-validator-0: reports parachain_candidate_dispute_concluded{validity="valid"} is 0
# Check an unsigned extrinsic is submitted
honest-validator: log line contains "Successfully reported pending slash" within 180 seconds