mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-05-30 11:41:02 +00:00
Companion for Store voters in unsorted bags to get good stake iteration properties (#3413)
* add voter bags generation script and generated values for all runtimes * VoterBagThresholds for test-runtime * semicolon * turns out the header is called file_header.txt * perform benchmarks to get real numbers * add note about when to re-run this script * Suggestions for #3413 (Store voters in unsorted bags companion) (#3488) Co-authored-by: Andronik Ordian <write@reusable.software> Co-authored-by: parity-processbot <> Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com> Co-authored-by: radupopa2010 <radupopa2010@yahoo.com> Co-authored-by: CI system <> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com> Co-authored-by: thiolliere <gui.thiolliere@gmail.com> Co-authored-by: André Silva <andrerfosilva@gmail.com> Co-authored-by: Keith Yeung <kungfukeith11@gmail.com> Co-authored-by: Bernhard Schuster <bernhard@ahoi.io> Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: Robert Habermeier <rphmeier@gmail.com> Co-authored-by: Robert Klotzner <eskimor@users.noreply.github.com> Co-authored-by: Alexander Theißen <alex.theissen@me.com> Co-authored-by: Lldenaurois <ljdenaurois@gmail.com> Co-authored-by: Qinxuan Chen <koushiro.cqx@gmail.com> Co-authored-by: Seun Lanlege <seun@parity.io> Co-authored-by: Denis Pisarev <denis.pisarev@parity.io> Co-authored-by: Andreas Doerr <adoerr@users.noreply.github.com> Co-authored-by: Chevdor <chevdor@users.noreply.github.com> Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com> Co-authored-by: Denis Tsai <denistsai@aetheras.io> Co-authored-by: Pierre Besson <pierre.besson@parity.io> Co-authored-by: kianenigma <kian@parity.io> Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com> Co-authored-by: Xiliang Chen <xlchen1291@gmail.com> Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com> Co-authored-by: ferrell-code <automatedcharlesferrell@gmail.com> Co-authored-by: Shaun W <spxwang@gmail.com> * Try resolve Cargo.lock conflicts * Add temp VoterSnapshotPerBlock value to compile * Improve voter_bags test with nominator/validator breakdown * Remove miner.log * Update README * Module doc comments * Build working * remote-ext-tests very WIP * Remove system module query * formatting and log target * clean up * Save * Fix issues with polkadot runtime after mergin * Save * Some updates to runtimes to make sure things compile * save * udpate weights * Use UseNominatorsAndUpdateBagsList * make remote-ext tests a bin * Adjust remote-ext test to use pub; update some comments * Use list_bags_get * Try fix cargo.lock * Gate kusama feat code * Carg.lock .. again * point bags-list deps to master * Update voter-bags CLI to take total issuance and min balance as args * Switch polkadot runtime to UseNominatorMap * Some tweaks to the remote-test * Delete some stuff for polkadot; comments * Various fixes * Add comment for VoterSnapshotPerBlock * Remove bags-list import from polakdot cargo * use benchmarking feature for pallet-bags-list * Add custom migration for westend and kusama for bags-list * :facepalm * Fix kusama runtime * Small clean up * Be more alphabetical ordered * Use 22_500 for VoterSnapshotPerBlock * point back to master * remove unused type * SortedListProvider for staking runtime * Remove polkadot for remote-ext bags test * Revert "Remove polkadot for remote-ext bags test" This reverts commit 3608c466002063761cd669880a00b2e05c1ff7d1. * Remove polkadot for remote-ext bags test * update Substrate Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> Co-authored-by: Andronik Ordian <write@reusable.software> Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com> Co-authored-by: radupopa2010 <radupopa2010@yahoo.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com> Co-authored-by: thiolliere <gui.thiolliere@gmail.com> Co-authored-by: André Silva <andrerfosilva@gmail.com> Co-authored-by: Keith Yeung <kungfukeith11@gmail.com> Co-authored-by: Bernhard Schuster <bernhard@ahoi.io> Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: Robert Habermeier <rphmeier@gmail.com> Co-authored-by: Robert Klotzner <eskimor@users.noreply.github.com> Co-authored-by: Alexander Theißen <alex.theissen@me.com> Co-authored-by: Lldenaurois <ljdenaurois@gmail.com> Co-authored-by: Qinxuan Chen <koushiro.cqx@gmail.com> Co-authored-by: Seun Lanlege <seun@parity.io> Co-authored-by: Denis Pisarev <denis.pisarev@parity.io> Co-authored-by: Andreas Doerr <adoerr@users.noreply.github.com> Co-authored-by: Chevdor <chevdor@users.noreply.github.com> Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com> Co-authored-by: Denis Tsai <denistsai@aetheras.io> Co-authored-by: Pierre Besson <pierre.besson@parity.io> Co-authored-by: kianenigma <kian@parity.io> Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com> Co-authored-by: Xiliang Chen <xlchen1291@gmail.com> Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com> Co-authored-by: ferrell-code <automatedcharlesferrell@gmail.com> Co-authored-by: Shaun W <spxwang@gmail.com> Co-authored-by: parity-processbot <>
This commit is contained in:
Peter Goodspeed-Niklaus
committed by
GitHub
GitHub
parent
608ad0f6bf
commit
9a6f43ab8f
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "remote-ext-tests-bags-list"
|
||||
version = "0.9.9"
|
||||
authors = ["Parity Technologies <admin@parity.io>"]
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
polkadot-runtime = { version = "0.9.8", path = "../../../runtime/polkadot" }
|
||||
kusama-runtime = { version = "0.9.8", path = "../../../runtime/kusama" }
|
||||
|
||||
pallet-staking = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
pallet-election-provider-multi-phase = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
pallet-bags-list = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
frame-election-provider-support = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
sp-storage = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
sp-std = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
frame-support = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
remote-externalities = { git = "https://github.com/paritytech/substrate", branch = "master" }
|
||||
tokio = { version = "1", features = ["macros"] }
|
||||
log = { version = "0.4.14" }
|
||||
structopt = {version = "0.3.23" }
|
||||
@@ -0,0 +1,3 @@
|
||||
# Remote Extension Tests For Pallet Bags List
|
||||
|
||||
Integration tests that use state from live chains via remote externalities.
|
||||
@@ -0,0 +1,59 @@
|
||||
// Copyright 2021 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Polkadot.
|
||||
|
||||
// Polkadot is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Polkadot is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Remote tests.
|
||||
|
||||
use structopt::StructOpt;
|
||||
|
||||
mod voter_bags;
|
||||
|
||||
#[derive(StructOpt)]
|
||||
enum Runtime {
|
||||
Kusama,
|
||||
}
|
||||
|
||||
impl std::str::FromStr for Runtime {
|
||||
type Err = &'static str;
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"kusama" => Ok(Runtime::Kusama),
|
||||
_ => Err("wrong Runtime: can be 'polkadot' or 'kusama'."),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(StructOpt)]
|
||||
struct Cli {
|
||||
#[structopt(long, default_value = "wss://rpc.kusama.io")]
|
||||
uri: String,
|
||||
#[structopt(long, short, default_value = "kusama")]
|
||||
runtime: Runtime,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let options = Cli::from_args();
|
||||
match options.runtime {
|
||||
Runtime::Kusama => {
|
||||
use kusama_runtime::{constants::currency::UNITS, Block, Runtime};
|
||||
voter_bags::test_voter_bags_migration::<Runtime, Block>(
|
||||
UNITS as u64,
|
||||
options.uri.clone(),
|
||||
)
|
||||
.await;
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright 2021 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Polkadot.
|
||||
|
||||
// Polkadot is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Polkadot is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Generic remote tests for the voter bags module.
|
||||
|
||||
use frame_election_provider_support::SortedListProvider;
|
||||
use frame_support::traits::Get;
|
||||
use pallet_election_provider_multi_phase as EPM;
|
||||
use pallet_staking::{BalanceOf, MinNominatorBond, Nominators};
|
||||
use remote_externalities::{Builder, Mode, OnlineConfig};
|
||||
use sp_runtime::traits::Block as BlockT;
|
||||
use sp_std::convert::TryInto;
|
||||
use sp_storage::well_known_keys;
|
||||
|
||||
const LOG_TARGET: &'static str = "remote-ext-tests::bags-list";
|
||||
|
||||
/// Test voter bags migration. `currency_unit` is the number of planks per the
|
||||
/// the runtimes `UNITS` (i.e. number of decimal places per DOT, KSM etc)
|
||||
pub(crate) async fn test_voter_bags_migration<
|
||||
Runtime: pallet_staking::Config + pallet_bags_list::Config + EPM::Config,
|
||||
Block: BlockT,
|
||||
>(
|
||||
currency_unit: u64,
|
||||
ws_url: String,
|
||||
) {
|
||||
sp_tracing::try_init_simple();
|
||||
|
||||
let mut ext = Builder::<Block>::new()
|
||||
.mode(Mode::Online(OnlineConfig {
|
||||
transport: ws_url.to_string().into(),
|
||||
modules: vec!["Staking".to_string()],
|
||||
at: None,
|
||||
state_snapshot: None,
|
||||
}))
|
||||
.inject_hashed_key(well_known_keys::CODE)
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
ext.execute_with(|| {
|
||||
// set the ss58 prefix so addresses printed below are human friendly.
|
||||
sp_core::crypto::set_default_ss58_version(Runtime::SS58Prefix::get().try_into().unwrap());
|
||||
|
||||
// get the nominator & validator count prior to migrating; these should be invariant.
|
||||
let pre_migrate_nominator_count = <Nominators<Runtime>>::iter().count() as u32;
|
||||
log::info!(target: LOG_TARGET, "Nominator count: {}", pre_migrate_nominator_count);
|
||||
|
||||
// run the actual migration,
|
||||
let moved = <Runtime as pallet_staking::Config>::SortedListProvider::regenerate(
|
||||
pallet_staking::Nominators::<Runtime>::iter().map(|(n, _)| n),
|
||||
pallet_staking::Pallet::<Runtime>::weight_of_fn(),
|
||||
);
|
||||
log::info!(target: LOG_TARGET, "Moved {} nominators", moved);
|
||||
|
||||
let voter_list_len =
|
||||
<Runtime as pallet_staking::Config>::SortedListProvider::iter().count() as u32;
|
||||
let voter_list_count = <Runtime as pallet_staking::Config>::SortedListProvider::count();
|
||||
// and confirm it is equal to the length of the `VoterList`.
|
||||
assert_eq!(pre_migrate_nominator_count, voter_list_len);
|
||||
assert_eq!(pre_migrate_nominator_count, voter_list_count);
|
||||
|
||||
let min_nominator_bond = <MinNominatorBond<Runtime>>::get();
|
||||
log::info!(target: LOG_TARGET, "min nominator bond is {:?}", min_nominator_bond);
|
||||
|
||||
// go through every bag to track the total number of voters within bags
|
||||
// and log some info about how voters are distributed within the bags.
|
||||
let mut seen_in_bags = 0;
|
||||
for vote_weight_thresh in <Runtime as pallet_bags_list::Config>::BagThresholds::get() {
|
||||
// threshold in terms of UNITS (e.g. KSM, DOT etc)
|
||||
let vote_weight_thresh_as_unit = *vote_weight_thresh as f64 / currency_unit as f64;
|
||||
let pretty_thresh = format!("Threshold: {}.", vote_weight_thresh_as_unit);
|
||||
|
||||
let bag = match pallet_bags_list::Pallet::<Runtime>::list_bags_get(*vote_weight_thresh)
|
||||
{
|
||||
Some(bag) => bag,
|
||||
None => {
|
||||
log::info!(target: LOG_TARGET, "{} NO VOTERS.", pretty_thresh);
|
||||
continue
|
||||
},
|
||||
};
|
||||
|
||||
let voters_in_bag = bag.std_iter().count() as u32;
|
||||
|
||||
// if this bag is below the min nominator bond print out all the members
|
||||
let vote_weight_as_balance: BalanceOf<Runtime> =
|
||||
(*vote_weight_thresh).try_into().map_err(|_| "should not fail").unwrap();
|
||||
if vote_weight_as_balance <= min_nominator_bond {
|
||||
for id in bag.std_iter().map(|node| node.std_id().clone()) {
|
||||
log::trace!(
|
||||
target: LOG_TARGET,
|
||||
"{} Account found below min bond: {:?}.",
|
||||
pretty_thresh,
|
||||
id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// update our overall counter
|
||||
seen_in_bags += voters_in_bag;
|
||||
|
||||
// percentage of all nominators
|
||||
let percent_of_voters = percent(voters_in_bag, voter_list_count);
|
||||
|
||||
log::info!(
|
||||
target: LOG_TARGET,
|
||||
"{} Nominators: {} [%{:.3}]",
|
||||
pretty_thresh,
|
||||
voters_in_bag,
|
||||
percent_of_voters,
|
||||
);
|
||||
}
|
||||
|
||||
if seen_in_bags != voter_list_count {
|
||||
log::error!(
|
||||
target: LOG_TARGET,
|
||||
"bags list population ({}) not on par whoever is voter_list ({})",
|
||||
seen_in_bags,
|
||||
voter_list_count,
|
||||
)
|
||||
}
|
||||
|
||||
// now let's test the process of a snapshot being created..
|
||||
EPM::Pallet::<Runtime>::create_snapshot().unwrap();
|
||||
|
||||
log::info!(
|
||||
target: LOG_TARGET,
|
||||
"a snapshot has been created using the new runtime and data, with metadata {:?}",
|
||||
EPM::Pallet::<Runtime>::snapshot_metadata(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
fn percent(portion: u32, total: u32) -> f64 {
|
||||
(portion as f64 / total as f64) * 100f64
|
||||
}
|
||||
Reference in New Issue
Block a user