Offchain Phragmén BREAKING. (#4517)

* Initial skeleton for offchain phragmen

* Basic compact encoding decoding for results

* add compact files

* Bring back Self::ensure_storage_upgraded();

* Make staking use compact stuff.

* First seemingly working version of reduce, full of todos

* Everything phragmen related works again.

* Signing made easier, still issues.

* Signing from offchain compile fine 😎

* make compact work with staked asssignment

* Evaluation basics are in place.

* Move reduce into crate. Document stuff

* move reduce into no_std

* Add files

* Remove other std deps. Runtime compiles

* Seemingly it is al stable; cycle implemented but not integrated.

* Add fuzzing code.

* Cleanup reduce a bit more.

* a metric ton of tests for staking; wip 🔨

* Implement a lot more of the tests.

* wip getting the unsigned stuff to work

* A bit gleanup for unsigned debug

* Clean and finalize compact code.

* Document reduce.

* Still problems with signing

* We officaly duct taped the transaction submission stuff. 🤓

* Deadlock with keys again

* Runtime builds

* Unsigned test works 🙌

* Some cleanups

* Make all the tests compile and stuff

* Minor cleanup

* fix more merge stuff

* Most tests work again.

* a very nasty bug in reduce

* Fix all integrations

* Fix more todos

* Revamp everything and everything

* Remove bogus test

* Some review grumbles.

* Some fixes

* Fix doc test

* loop for submission

* Fix cli, keyring etc.

* some cleanup

* Fix staking tests again

* fix per-things; bring patches from benchmarking

* better score prediction

* Add fuzzer, more patches.

* Some fixes

* More docs

* Remove unused generics

* Remove max-nominator footgun

* Better fuzzer

* Disable it 

* Bump.

* Another round of self-review

* Refactor a lot

* More major fixes in perThing

* Add new fuzz file

* Update lock

* fix fuzzing code.

* Fix nominator retain test

* Add slashing check

* Update frame/staking/src/tests.rs

Co-Authored-By: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>

* Some formatting nits

* Review comments.

* Fix cargo file

* Almost all tests work again

* Update frame/staking/src/tests.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Fix review comments

* More review stuff

* Some nits

* Fix new staking / session / babe relation

* Update primitives/phragmen/src/lib.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Update primitives/phragmen/src/lib.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Update primitives/phragmen/compact/src/lib.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Some doc updates to slashing

* Fix derive

* Remove imports

* Remove unimplemented tests

* nits

* Remove dbg

* Better fuzzing params

* Remove unused pref map

* Deferred Slashing/Offence for offchain Phragmen  (#5151)

* Some boilerplate

* Add test

* One more test

* Review comments

* Fix build

* review comments

* fix more

* fix build

* Some cleanups and self-reviews

* More minor self reviews

* Final nits

* Some merge fixes.

* opt comment

* Fix build

* Fix build again.

* Update frame/staking/fuzz/fuzz_targets/submit_solution.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

* Update frame/staking/src/slashing.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

* Update frame/staking/src/offchain_election.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

* Fix review comments

* fix test

* === 🔑 Revamp without staking key.

* final round of changes.

* Fix cargo-deny

* Update frame/staking/src/lib.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

Co-authored-by: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>
Co-authored-by: thiolliere <gui.thiolliere@gmail.com>
Co-authored-by: Gavin Wood <gavin@parity.io>
This commit is contained in:
Kian Paimani
2020-03-26 15:37:40 +01:00
committed by GitHub
parent 2a67e6c437
commit 970c5f94f2
64 changed files with 11953 additions and 892 deletions
@@ -0,0 +1,210 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Code generation for the ratio assignment type.
use crate::field_name_for;
use proc_macro2::{TokenStream as TokenStream2};
use syn::{GenericArgument};
use quote::quote;
fn from_impl(count: usize) -> TokenStream2 {
let from_impl_single = {
let name = field_name_for(1);
quote!(1 => compact.#name.push(
(
index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?,
index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?,
)
),)
};
let from_impl_double = {
let name = field_name_for(2);
quote!(2 => compact.#name.push(
(
index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?,
(
index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?,
distribution[0].1,
),
index_of_target(&distribution[1].0).ok_or(_phragmen::Error::CompactInvalidIndex)?,
)
),)
};
let from_impl_rest = (3..=count).map(|c| {
let inner = (0..c-1).map(|i|
quote!((index_of_target(&distribution[#i].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, distribution[#i].1),)
).collect::<TokenStream2>();
let field_name = field_name_for(c);
let last_index = c - 1;
let last = quote!(index_of_target(&distribution[#last_index].0).ok_or(_phragmen::Error::CompactInvalidIndex)?);
quote!(
#c => compact.#field_name.push((index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, [#inner], #last)),
)
}).collect::<TokenStream2>();
quote!(
#from_impl_single
#from_impl_double
#from_impl_rest
)
}
fn into_impl(count: usize) -> TokenStream2 {
let into_impl_single = {
let name = field_name_for(1);
quote!(
for (voter_index, target_index) in self.#name {
assignments.push(_phragmen::Assignment {
who: voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?,
distribution: vec![
(target_at(target_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, Accuracy::one())
],
})
}
)
};
let into_impl_double = {
let name = field_name_for(2);
quote!(
for (voter_index, (t1_idx, p1), t2_idx) in self.#name {
if p1 >= Accuracy::one() {
return Err(_phragmen::Error::CompactStakeOverflow);
}
// defensive only. Since Percent doesn't have `Sub`.
let p2 = _phragmen::sp_runtime::traits::Saturating::saturating_sub(
Accuracy::one(),
p1,
);
assignments.push( _phragmen::Assignment {
who: voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?,
distribution: vec![
(target_at(t1_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, p1),
(target_at(t2_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, p2),
]
});
}
)
};
let into_impl_rest = (3..=count).map(|c| {
let name = field_name_for(c);
quote!(
for (voter_index, inners, t_last_idx) in self.#name {
let mut sum = Accuracy::zero();
let mut inners_parsed = inners
.iter()
.map(|(ref t_idx, p)| {
sum = _phragmen::sp_runtime::traits::Saturating::saturating_add(sum, *p);
let target = target_at(*t_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?;
Ok((target, *p))
})
.collect::<Result<Vec<(A, Accuracy)>, _phragmen::Error>>()?;
if sum >= Accuracy::one() {
return Err(_phragmen::Error::CompactStakeOverflow);
}
// defensive only. Since Percent doesn't have `Sub`.
let p_last = _phragmen::sp_runtime::traits::Saturating::saturating_sub(
Accuracy::one(),
sum,
);
inners_parsed.push((target_at(t_last_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, p_last));
assignments.push(_phragmen::Assignment {
who: voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?,
distribution: inners_parsed,
});
}
)
}).collect::<TokenStream2>();
quote!(
#into_impl_single
#into_impl_double
#into_impl_rest
)
}
pub(crate) fn assignment(
ident: syn::Ident,
voter_type: GenericArgument,
target_type: GenericArgument,
count: usize,
) -> TokenStream2 {
let from_impl = from_impl(count);
let into_impl = into_impl(count);
quote!(
impl<
#voter_type: _phragmen::codec::Codec + Default + Copy,
#target_type: _phragmen::codec::Codec + Default + Copy,
Accuracy:
_phragmen::codec::Codec + Default + Clone + _phragmen::sp_runtime::PerThing +
PartialOrd,
>
#ident<#voter_type, #target_type, Accuracy>
{
pub fn from_assignment<FV, FT, A>(
assignments: Vec<_phragmen::Assignment<A, Accuracy>>,
index_of_voter: FV,
index_of_target: FT,
) -> Result<Self, _phragmen::Error>
where
for<'r> FV: Fn(&'r A) -> Option<#voter_type>,
for<'r> FT: Fn(&'r A) -> Option<#target_type>,
A: _phragmen::IdentifierT,
{
let mut compact: #ident<
#voter_type,
#target_type,
Accuracy,
> = Default::default();
for _phragmen::Assignment { who, distribution } in assignments {
match distribution.len() {
0 => continue,
#from_impl
_ => {
return Err(_phragmen::Error::CompactTargetOverflow);
}
}
};
Ok(compact)
}
pub fn into_assignment<A: _phragmen::IdentifierT>(
self,
voter_at: impl Fn(#voter_type) -> Option<A>,
target_at: impl Fn(#target_type) -> Option<A>,
) -> Result<Vec<_phragmen::Assignment<A, Accuracy>>, _phragmen::Error> {
let mut assignments: Vec<_phragmen::Assignment<A, Accuracy>> = Default::default();
#into_impl
Ok(assignments)
}
}
)
}
@@ -0,0 +1,219 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Proc macro for phragmen compact assignment.
use proc_macro::TokenStream;
use proc_macro2::{TokenStream as TokenStream2, Span, Ident};
use proc_macro_crate::crate_name;
use quote::quote;
use syn::{GenericArgument, Type, parse::{Parse, ParseStream, Result}};
mod assignment;
mod staked;
// prefix used for struct fields in compact.
const PREFIX: &'static str = "votes";
/// Generates a struct to store the phragmen assignments in a compact way. The struct can only store
/// distributions up to the given input count. The given count must be greater than 2.
///
/// ```ignore
/// // generate a struct with nominator and edge weight u128, with maximum supported
/// // edge per voter of 16.
/// generate_compact_solution_type(pub TestCompact, 16)
/// ```
///
/// This generates:
///
/// ```ignore
/// pub struct TestCompact<V, T, W> {
/// votes1: Vec<(V, T)>,
/// votes2: Vec<(V, (T, W), T)>,
/// votes3: Vec<(V, [(T, W); 2usize], T)>,
/// votes4: Vec<(V, [(T, W); 3usize], T)>,
/// votes5: Vec<(V, [(T, W); 4usize], T)>,
/// votes6: Vec<(V, [(T, W); 5usize], T)>,
/// votes7: Vec<(V, [(T, W); 6usize], T)>,
/// votes8: Vec<(V, [(T, W); 7usize], T)>,
/// votes9: Vec<(V, [(T, W); 8usize], T)>,
/// votes10: Vec<(V, [(T, W); 9usize], T)>,
/// votes11: Vec<(V, [(T, W); 10usize], T)>,
/// votes12: Vec<(V, [(T, W); 11usize], T)>,
/// votes13: Vec<(V, [(T, W); 12usize], T)>,
/// votes14: Vec<(V, [(T, W); 13usize], T)>,
/// votes15: Vec<(V, [(T, W); 14usize], T)>,
/// votes16: Vec<(V, [(T, W); 15usize], T)>,
/// }
/// ```
///
/// The generic arguments are:
/// - `V`: identifier/index for voter (nominator) types.
/// - `T` identifier/index for candidate (validator) types.
/// - `W` weight type.
///
/// Some conversion implementations are provided by default if
/// - `W` is u128, or
/// - `W` is anything that implements `PerThing` (such as `Perbill`)
///
/// The ideas behind the structure are as follows:
///
/// - For single distribution, no weight is stored. The weight is known to be 100%.
/// - For all the rest, the weight if the last distribution is omitted. This value can be computed
/// from the rest.
///
#[proc_macro]
pub fn generate_compact_solution_type(item: TokenStream) -> TokenStream {
let CompactSolutionDef {
vis,
ident,
count,
} = syn::parse_macro_input!(item as CompactSolutionDef);
let voter_type = GenericArgument::Type(Type::Verbatim(quote!(V)));
let target_type = GenericArgument::Type(Type::Verbatim(quote!(T)));
let weight_type = GenericArgument::Type(Type::Verbatim(quote!(W)));
let imports = imports().unwrap_or_else(|e| e.to_compile_error());
let compact_def = struct_def(
vis,
ident.clone(),
count,
voter_type.clone(),
target_type.clone(),
weight_type,
).unwrap_or_else(|e| e.to_compile_error());
let assignment_impls = assignment::assignment(
ident.clone(),
voter_type.clone(),
target_type.clone(),
count,
);
let staked_impls = staked::staked(
ident,
voter_type,
target_type,
count,
);
quote!(
#imports
#compact_def
#assignment_impls
#staked_impls
).into()
}
fn struct_def(
vis: syn::Visibility,
ident: syn::Ident,
count: usize,
voter_type: GenericArgument,
target_type: GenericArgument,
weight_type: GenericArgument,
) -> Result<TokenStream2> {
if count <= 2 {
Err(syn::Error::new(
Span::call_site(),
"cannot build compact solution struct with capacity less than 2."
))?
}
let singles = {
let name = field_name_for(1);
quote!(#name: Vec<(#voter_type, #target_type)>,)
};
let doubles = {
let name = field_name_for(2);
quote!(#name: Vec<(#voter_type, (#target_type, #weight_type), #target_type)>,)
};
let rest = (3..=count).map(|c| {
let field_name = field_name_for(c);
let array_len = c - 1;
quote!(
#field_name: Vec<(
#voter_type,
[(#target_type, #weight_type); #array_len],
#target_type
)>,
)
}).collect::<TokenStream2>();
Ok(quote! (
/// A struct to encode a Phragmen assignment in a compact way.
#[derive(
Default,
PartialEq,
Eq,
Clone,
_phragmen::sp_runtime::RuntimeDebug,
_phragmen::codec::Encode,
_phragmen::codec::Decode,
)]
#vis struct #ident<#voter_type, #target_type, #weight_type> {
// _marker: sp_std::marker::PhantomData<A>,
#singles
#doubles
#rest
}
impl<#voter_type, #target_type, #weight_type> _phragmen::VotingLimit
for #ident<#voter_type, #target_type, #weight_type>
{
const LIMIT: usize = #count;
}
))
}
fn imports() -> Result<TokenStream2> {
let sp_phragmen_imports = match crate_name("sp-phragmen") {
Ok(sp_phragmen) => {
let ident = syn::Ident::new(&sp_phragmen, Span::call_site());
quote!( extern crate #ident as _phragmen; )
}
Err(e) => return Err(syn::Error::new(Span::call_site(), &e)),
};
Ok(quote!(
#sp_phragmen_imports
))
}
struct CompactSolutionDef {
vis: syn::Visibility,
ident: syn::Ident,
count: usize,
}
impl Parse for CompactSolutionDef {
fn parse(input: ParseStream) -> syn::Result<Self> {
let vis: syn::Visibility = input.parse()?;
let ident: syn::Ident = input.parse()?;
let _ = <syn::Token![,]>::parse(input)?;
let count_literal: syn::LitInt = input.parse()?;
let count = count_literal.base10_parse::<usize>()?;
Ok(Self { vis, ident, count } )
}
}
fn field_name_for(n: usize) -> Ident {
Ident::new(&format!("{}{}", PREFIX, n), Span::call_site())
}
@@ -0,0 +1,208 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Code generation for the staked assignment type.
use crate::field_name_for;
use proc_macro2::{TokenStream as TokenStream2};
use syn::{GenericArgument};
use quote::quote;
fn from_impl(count: usize) -> TokenStream2 {
let from_impl_single = {
let name = field_name_for(1);
quote!(1 => compact.#name.push(
(
index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?,
index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?,
)
),)
};
let from_impl_double = {
let name = field_name_for(2);
quote!(2 => compact.#name.push(
(
index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?,
(
index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?,
distribution[0].1,
),
index_of_target(&distribution[1].0).ok_or(_phragmen::Error::CompactInvalidIndex)?,
)
),)
};
let from_impl_rest = (3..=count).map(|c| {
let inner = (0..c-1).map(|i|
quote!((index_of_target(&distribution[#i].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, distribution[#i].1),)
).collect::<TokenStream2>();
let field_name = field_name_for(c);
let last_index = c - 1;
let last = quote!(index_of_target(&distribution[#last_index].0).ok_or(_phragmen::Error::CompactInvalidIndex)?);
quote!(
#c => compact.#field_name.push((index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, [#inner], #last)),
)
}).collect::<TokenStream2>();
quote!(
#from_impl_single
#from_impl_double
#from_impl_rest
)
}
fn into_impl(count: usize) -> TokenStream2 {
let into_impl_single = {
let name = field_name_for(1);
quote!(
for (voter_index, target_index) in self.#name {
let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?;
let all_stake = max_of(&who);
assignments.push(_phragmen::StakedAssignment {
who,
distribution: vec![(target_at(target_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, all_stake)],
})
}
)
};
let into_impl_double = {
let name = field_name_for(2);
quote!(
for (voter_index, (t1_idx, w1), t2_idx) in self.#name {
let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?;
let all_stake = max_of(&who);
if w1 >= all_stake {
return Err(_phragmen::Error::CompactStakeOverflow);
}
// w2 is ensured to be positive.
let w2 = all_stake - w1;
assignments.push( _phragmen::StakedAssignment {
who,
distribution: vec![
(target_at(t1_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, w1),
(target_at(t2_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, w2),
]
});
}
)
};
let into_impl_rest = (3..=count).map(|c| {
let name = field_name_for(c);
quote!(
for (voter_index, inners, t_last_idx) in self.#name {
let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?;
let mut sum = u128::min_value();
let all_stake = max_of(&who);
let mut inners_parsed = inners
.iter()
.map(|(ref t_idx, w)| {
sum = sum.saturating_add(*w);
let target = target_at(*t_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?;
Ok((target, *w))
}).collect::<Result<Vec<(A, u128)>, _phragmen::Error>>()?;
if sum >= all_stake {
return Err(_phragmen::Error::CompactStakeOverflow);
}
// w_last is proved to be positive.
let w_last = all_stake - sum;
inners_parsed.push((target_at(t_last_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, w_last));
assignments.push(_phragmen::StakedAssignment {
who,
distribution: inners_parsed,
});
}
)
}).collect::<TokenStream2>();
quote!(
#into_impl_single
#into_impl_double
#into_impl_rest
)
}
pub(crate) fn staked(
ident: syn::Ident,
voter_type: GenericArgument,
target_type: GenericArgument,
count: usize,
) -> TokenStream2 {
let from_impl = from_impl(count);
let into_impl = into_impl(count);
quote!(
impl<
#voter_type: _phragmen::codec::Codec + Default + Copy,
#target_type: _phragmen::codec::Codec + Default + Copy,
>
#ident<#voter_type, #target_type, u128>
{
/// Generate self from a vector of `StakedAssignment`.
pub fn from_staked<FV, FT, A>(
assignments: Vec<_phragmen::StakedAssignment<A>>,
index_of_voter: FV,
index_of_target: FT,
) -> Result<Self, _phragmen::Error>
where
for<'r> FV: Fn(&'r A) -> Option<#voter_type>,
for<'r> FT: Fn(&'r A) -> Option<#target_type>,
A: _phragmen::IdentifierT
{
let mut compact: #ident<#voter_type, #target_type, u128> = Default::default();
for _phragmen::StakedAssignment { who, distribution } in assignments {
match distribution.len() {
0 => continue,
#from_impl
_ => {
return Err(_phragmen::Error::CompactTargetOverflow);
}
}
};
Ok(compact)
}
/// Convert self into `StakedAssignment`. The given function should return the total
/// weight of a voter. It is used to subtract the sum of all the encoded weights to
/// infer the last one.
pub fn into_staked<FM, A>(
self,
max_of: FM,
voter_at: impl Fn(#voter_type) -> Option<A>,
target_at: impl Fn(#target_type) -> Option<A>,
)
-> Result<Vec<_phragmen::StakedAssignment<A>>, _phragmen::Error>
where
for<'r> FM: Fn(&'r A) -> u128,
A: _phragmen::IdentifierT,
{
let mut assignments: Vec<_phragmen::StakedAssignment<A>> = Default::default();
#into_impl
Ok(assignments)
}
}
)
}