Split Indices module from Balances (#1404)

* Indices module

* Remove indices stuff from balances

* Rejob node, move Lookup into system.

* Fix up some modules.

* Fix democracy tests

* Fix staking tests

* Fix more tests

* Final test fixes

* Bump runtime versions

* Assets uses compact dispatchers

* Contracts module uses indexed addressing

* Democracy has more compact encoding

* Example now demonstrates compact eencoding

* Sudo uses indexed address

* Upgrade key also uses indexed lookups

* Assets more compact types.

* Fix test

* Rebuild runtime, whitespace

* Remove TOODs

* Remove TODOs

* Add a couple of tests back to balances.

* Update lib.rs

* Update lib.rs

substrate/srml/indices/src/address.rs

@@ -0,0 +1,109 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Address type that is union of index and id for an account.
+
+#[cfg(feature = "std")]
+use std::fmt;
+use super::{Member, Decode, Encode, As, Input, Output};
+
+/// A vetted and verified extrinsic from the external world.
+#[derive(PartialEq, Eq, Clone)]
+#[cfg_attr(feature = "std", derive(Debug, Hash))]
+pub enum Address<AccountId, AccountIndex> where
+	AccountId: Member,
+	AccountIndex: Member,
+{
+	/// It's an account ID (pubkey).
+	Id(AccountId),
+	/// It's an account index.
+	Index(AccountIndex),
+}
+
+#[cfg(feature = "std")]
+impl<AccountId, AccountIndex> fmt::Display for Address<AccountId, AccountIndex> where
+	AccountId: Member,
+	AccountIndex: Member,
+{
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(f, "{:?}", self)
+	}
+}
+
+impl<AccountId, AccountIndex> From<AccountId> for Address<AccountId, AccountIndex> where
+	AccountId: Member,
+	AccountIndex: Member,
+{
+	fn from(a: AccountId) -> Self {
+		Address::Id(a)
+	}
+}
+
+fn need_more_than<T: PartialOrd>(a: T, b: T) -> Option<T> {
+	if a < b { Some(a) } else { None }
+}
+
+impl<AccountId, AccountIndex> Decode for Address<AccountId, AccountIndex> where
+	AccountId: Member + Decode,
+	AccountIndex: Member + Decode + PartialOrd<AccountIndex> + Ord + As<u32> + As<u16> + As<u8> + Copy,
+{
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Some(match input.read_byte()? {
+			x @ 0x00...0xef => Address::Index(As::sa(x)),
+			0xfc => Address::Index(As::sa(need_more_than(0xef, u16::decode(input)?)?)),
+			0xfd => Address::Index(As::sa(need_more_than(0xffff, u32::decode(input)?)?)),
+			0xfe => Address::Index(need_more_than(As::sa(0xffffffffu32), Decode::decode(input)?)?),
+			0xff => Address::Id(Decode::decode(input)?),
+			_ => return None,
+		})
+	}
+}
+
+impl<AccountId, AccountIndex> Encode for Address<AccountId, AccountIndex> where
+	AccountId: Member + Encode,
+	AccountIndex: Member + Encode + PartialOrd<AccountIndex> + Ord + As<u32> + As<u16> + As<u8> + Copy,
+{
+	fn encode_to<T: Output>(&self, dest: &mut T) {
+		match *self {
+			Address::Id(ref i) => {
+				dest.push_byte(255);
+				dest.push(i);
+			}
+			Address::Index(i) if i > As::sa(0xffffffffu32) => {
+				dest.push_byte(254);
+				dest.push(&i);
+			}
+			Address::Index(i) if i > As::sa(0xffffu32) => {
+				dest.push_byte(253);
+				dest.push(&As::<u32>::as_(i));
+			}
+			Address::Index(i) if i >= As::sa(0xf0u32) => {
+				dest.push_byte(252);
+				dest.push(&As::<u16>::as_(i));
+			}
+			Address::Index(i) => dest.push_byte(As::<u8>::as_(i)),
+		}
+	}
+}
+
+impl<AccountId, AccountIndex> Default for Address<AccountId, AccountIndex> where
+	AccountId: Member + Default,
+	AccountIndex: Member,
+{
+	fn default() -> Self {
+		Address::Id(Default::default())
+	}
+}

substrate/srml/indices/src/lib.rs

@@ -0,0 +1,222 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Balances: Handles setting and retrieval of free balance,
+//! retrieving total balance, reserve and unreserve balance,
+//! repatriating a reserved balance to a beneficiary account that exists,
+//! transfering a balance between accounts (when not reserved),
+//! slashing an account balance, account removal, rewards,
+//! lookup of an index to reclaim an account (when not balance not reserved),
+//! increasing total stake.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[macro_use]
+extern crate srml_support as runtime_support;
+
+extern crate sr_std as rstd;
+
+#[macro_use]
+extern crate parity_codec_derive;
+
+extern crate parity_codec as codec;
+extern crate sr_primitives as primitives;
+extern crate srml_system as system;
+
+#[cfg(test)]
+#[macro_use]
+extern crate ref_thread_local;
+#[cfg(test)]
+extern crate sr_io as runtime_io;
+#[cfg(test)]
+extern crate substrate_primitives;
+
+use rstd::{prelude::*, result, marker::PhantomData};
+use codec::{Encode, Decode, Codec, Input, Output};
+use runtime_support::{StorageValue, StorageMap, Parameter};
+use primitives::traits::{One, SimpleArithmetic, As, StaticLookup, Member};
+use address::Address as RawAddress;
+use system::{IsDeadAccount, OnNewAccount};
+
+mod mock;
+
+pub mod address;
+mod tests;
+
+/// Number of account IDs stored per enum set.
+const ENUM_SET_SIZE: usize = 64;
+
+pub type Address<T> = RawAddress<<T as system::Trait>::AccountId, <T as Trait>::AccountIndex>;
+
+/// Turn an Id into an Index, or None for the purpose of getting
+/// a hint at a possibly desired index.
+pub trait ResolveHint<AccountId: Encode, AccountIndex: As<usize>> {
+	/// Turn an Id into an Index, or None for the purpose of getting
+	/// a hint at a possibly desired index.
+	fn resolve_hint(who: &AccountId) -> Option<AccountIndex>;
+}
+
+/// Simple encode-based resolve hint implemenntation.
+pub struct SimpleResolveHint<AccountId, AccountIndex>(PhantomData<(AccountId, AccountIndex)>);
+impl<AccountId: Encode, AccountIndex: As<usize>> ResolveHint<AccountId, AccountIndex> for SimpleResolveHint<AccountId, AccountIndex> {
+	fn resolve_hint(who: &AccountId) -> Option<AccountIndex> {
+		Some(AccountIndex::sa(who.using_encoded(|e| e[0] as usize + e[1] as usize * 256)))
+	}
+}
+
+/// The module's config trait.
+pub trait Trait: system::Trait {
+	/// Type used for storing an account's index; implies the maximum number of accounts the system
+	/// can hold.
+	type AccountIndex: Parameter + Member + Codec + Default + SimpleArithmetic + As<u8> + As<u16> + As<u32> + As<u64> + As<usize> + Copy;
+
+	/// Whether an account is dead or not.
+	type IsDeadAccount: IsDeadAccount<Self::AccountId>;
+
+	/// How to turn an id into an index.
+	type ResolveHint: ResolveHint<Self::AccountId, Self::AccountIndex>;
+
+	/// The overarching event type.
+	type Event: From<Event<Self>> + Into<<Self as system::Trait>::Event>;
+}
+
+decl_module! {
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+		fn deposit_event<T>() = default;
+	}
+}
+
+decl_event!(
+	pub enum Event<T> where
+		<T as system::Trait>::AccountId,
+		<T as Trait>::AccountIndex
+	{
+		/// A new account was created.
+		NewAccountIndex(AccountId, AccountIndex),
+	}
+);
+
+decl_storage! {
+	trait Store for Module<T: Trait> as Indices {
+		/// The next free enumeration set.
+		pub NextEnumSet get(next_enum_set) build(|config: &GenesisConfig<T>| {
+			T::AccountIndex::sa(config.ids.len() / ENUM_SET_SIZE)
+		}): T::AccountIndex;
+
+		/// The enumeration sets.
+		pub EnumSet get(enum_set): map T::AccountIndex => Vec<T::AccountId>;
+	}
+	add_extra_genesis {
+		config(ids): Vec<T::AccountId>;
+		build(|storage: &mut primitives::StorageMap, _: &mut primitives::ChildrenStorageMap, config: &GenesisConfig<T>| {
+			for i in 0..(config.ids.len() + ENUM_SET_SIZE - 1) / ENUM_SET_SIZE {
+				storage.insert(GenesisConfig::<T>::hash(&<EnumSet<T>>::key_for(T::AccountIndex::sa(i))).to_vec(),
+					config.ids[i * ENUM_SET_SIZE..config.ids.len().min((i + 1) * ENUM_SET_SIZE)].to_owned().encode());
+			}
+		});
+	}
+}
+
+impl<T: Trait> Module<T> {
+	// PUBLIC IMMUTABLES
+
+	/// Lookup an T::AccountIndex to get an Id, if there's one there.
+	pub fn lookup_index(index: T::AccountIndex) -> Option<T::AccountId> {
+		let enum_set_size = Self::enum_set_size();
+		let set = Self::enum_set(index / enum_set_size);
+		let i: usize = (index % enum_set_size).as_();
+		set.get(i).map(|x| x.clone())
+	}
+
+	/// `true` if the account `index` is ready for reclaim.
+	pub fn can_reclaim(try_index: T::AccountIndex) -> bool {
+		let enum_set_size = Self::enum_set_size();
+		let try_set = Self::enum_set(try_index / enum_set_size);
+		let i = (try_index % enum_set_size).as_();
+		i < try_set.len() && T::IsDeadAccount::is_dead_account(&try_set[i])
+	}
+
+	/// Lookup an address to get an Id, if there's one there.
+	pub fn lookup_address(a: address::Address<T::AccountId, T::AccountIndex>) -> Option<T::AccountId> {
+		match a {
+			address::Address::Id(i) => Some(i),
+			address::Address::Index(i) => Self::lookup_index(i),
+		}
+	}
+
+	// PUBLIC MUTABLES (DANGEROUS)
+
+	fn enum_set_size() -> T::AccountIndex {
+		T::AccountIndex::sa(ENUM_SET_SIZE)
+	}
+}
+
+impl<T: Trait> OnNewAccount<T::AccountId> for Module<T> {
+	fn on_new_account(who: &T::AccountId) {
+		let enum_set_size = Self::enum_set_size();
+		let next_set_index = Self::next_enum_set();
+
+		if let Some(try_index) = T::ResolveHint::resolve_hint(who) {
+			// then check to see if this account id identifies a dead account index.
+			let set_index = try_index / enum_set_size;
+			let mut try_set = Self::enum_set(set_index);
+			let item_index = (try_index % enum_set_size).as_();
+			if item_index < try_set.len() {
+				if T::IsDeadAccount::is_dead_account(&try_set[item_index]) {
+					// yup - this index refers to a dead account. can be reused.
+					try_set[item_index] = who.clone();
+					<EnumSet<T>>::insert(set_index, try_set);
+
+					return
+				}
+			}
+		}
+
+		// insert normally as a back up
+		let mut set_index = next_set_index;
+		// defensive only: this loop should never iterate since we keep NextEnumSet up to date later.
+		let mut set = loop {
+			let set = Self::enum_set(set_index);
+			if set.len() < ENUM_SET_SIZE {
+				break set;
+			}
+			set_index += One::one();
+		};
+
+		let index = T::AccountIndex::sa(set_index.as_() * ENUM_SET_SIZE + set.len());
+
+		// update set.
+		set.push(who.clone());
+
+		// keep NextEnumSet up to date
+		if set.len() == ENUM_SET_SIZE {
+			<NextEnumSet<T>>::put(set_index + One::one());
+		}
+
+		// write set.
+		<EnumSet<T>>::insert(set_index, set);
+
+		Self::deposit_event(RawEvent::NewAccountIndex(who.clone(), index));
+	}
+}
+
+impl<T: Trait> StaticLookup for Module<T> {
+	type Source = address::Address<T::AccountId, T::AccountIndex>;
+	type Target = T::AccountId;
+	fn lookup(a: Self::Source) -> result::Result<Self::Target, &'static str> {
+		Self::lookup_address(a).ok_or("invalid account index")
+	}
+}

substrate/srml/indices/src/mock.rs

@@ -0,0 +1,102 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test utilities
+
+#![cfg(test)]
+
+use std::collections::HashSet;
+use ref_thread_local::RefThreadLocal;
+use primitives::BuildStorage;
+use primitives::testing::{Digest, DigestItem, Header};
+use substrate_primitives::{H256, Blake2Hasher};
+use runtime_io;
+use {GenesisConfig, Module, Trait, system};
+use super::{IsDeadAccount, OnNewAccount, ResolveHint};
+
+impl_outer_origin!{
+	pub enum Origin for Runtime {}
+}
+
+ref_thread_local! {
+	static managed ALIVE: HashSet<u64> = HashSet::new();
+}
+
+pub fn make_account(who: u64) {
+	ALIVE.borrow_mut().insert(who);
+	Indices::on_new_account(&who);
+}
+
+pub fn kill_account(who: u64) {
+	ALIVE.borrow_mut().remove(&who);
+}
+
+pub struct TestIsDeadAccount {}
+impl IsDeadAccount<u64> for TestIsDeadAccount {
+	fn is_dead_account(who: &u64) -> bool {
+		!ALIVE.borrow_mut().contains(who)
+	}
+}
+
+pub struct TestResolveHint;
+impl ResolveHint<u64, u64> for TestResolveHint {
+	fn resolve_hint(who: &u64) -> Option<u64> {
+		if *who < 256 {
+			None
+		} else {
+			Some(*who - 256)
+		}
+	}
+}
+
+// Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Runtime;
+impl system::Trait for Runtime {
+	type Origin = Origin;
+	type Index = u64;
+	type BlockNumber = u64;
+	type Hash = H256;
+	type Hashing = ::primitives::traits::BlakeTwo256;
+	type Digest = Digest;
+	type AccountId = u64;
+	type Lookup = Indices;
+	type Header = Header;
+	type Event = ();
+	type Log = DigestItem;
+}
+impl Trait for Runtime {
+	type AccountIndex = u64;
+	type IsDeadAccount = TestIsDeadAccount;
+	type ResolveHint = TestResolveHint;
+	type Event = ();
+}
+
+pub fn new_test_ext() -> runtime_io::TestExternalities<Blake2Hasher> {
+	{
+		let mut h = ALIVE.borrow_mut();
+		h.clear();
+		for i in 1..5 { h.insert(i); }
+	}
+
+	let mut t = system::GenesisConfig::<Runtime>::default().build_storage().unwrap().0;
+	t.extend(GenesisConfig::<Runtime> {
+		ids: vec![1, 2, 3, 4]
+	}.build_storage().unwrap().0);
+	t.into()
+}
+
+pub type Indices = Module<Runtime>;

substrate/srml/indices/src/tests.rs

@@ -0,0 +1,80 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for the module.
+
+#![cfg(test)]
+
+use super::*;
+use mock::{Indices, new_test_ext, make_account, kill_account, TestIsDeadAccount};
+use runtime_io::with_externalities;
+
+#[test]
+fn indexing_lookup_should_work() {
+	with_externalities(
+		&mut new_test_ext(),
+		|| {
+			assert_eq!(Indices::lookup_index(0), Some(1));
+			assert_eq!(Indices::lookup_index(1), Some(2));
+			assert_eq!(Indices::lookup_index(2), Some(3));
+			assert_eq!(Indices::lookup_index(3), Some(4));
+			assert_eq!(Indices::lookup_index(4), None);
+		},
+	);
+}
+
+#[test]
+fn default_indexing_on_new_accounts_should_work() {
+	with_externalities(
+		&mut new_test_ext(),
+		|| {
+			assert_eq!(Indices::lookup_index(4), None);
+			make_account(5);
+			assert_eq!(Indices::lookup_index(4), Some(5));
+		},
+	);
+}
+
+#[test]
+fn reclaim_indexing_on_new_accounts_should_work() {
+	with_externalities(
+		&mut new_test_ext(),
+		|| {
+			assert_eq!(Indices::lookup_index(1), Some(2));
+			assert_eq!(Indices::lookup_index(4), None);
+
+			kill_account(2);					// index 1 no longer locked to id 2
+
+			make_account(1 + 256);				// id 257 takes index 1.
+			assert_eq!(Indices::lookup_index(1), Some(257));
+		},
+	);
+}
+
+#[test]
+fn alive_account_should_prevent_reclaim() {
+	with_externalities(
+		&mut new_test_ext(),
+		|| {
+			assert!(!TestIsDeadAccount::is_dead_account(&2));
+			assert_eq!(Indices::lookup_index(1), Some(2));
+			assert_eq!(Indices::lookup_index(4), None);
+
+			make_account(1 + 256);				// id 257 takes index 1.
+			assert_eq!(Indices::lookup_index(4), Some(257));
+		},
+	);
+}