implement bitfield distribution subsystem (#1368)

* feat bitfield distribution

* feat bitfield distribution part 2

* pair programming with rustc & cargo

* lets go

* move bitfield-distribution to the node/network folder

* shape shifting

* lunchtime

* ignore the two fn recursion for now

* step by step

* triplesteps

* bandaid commit

* unordered futures magic

* chore

* reword markdown

* clarify

* lacks abortable processing impl details

* slimify

* fix: warnings and avoid ctx.clone() improve comments

* review comments

* fix details

* make sure outgoing messages are tracked

* fix name

* fix subsystem

* partial test impl

* relax context bounds

* test

* X

* X

* initial test

* fix relay_message not tracked when origin is self

* fix/guide: grammar

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* work around missing Eq+PartialEq

* fix: add missing message to provisioner

* unify per_job to job_data

* fix/review: part one

* fix/review: more grumbles

* fix/review: track incoming messages per peer

* fix/review: extract fn, avoid nested matches

* fix/review: more tests, simplify test

* fix/review: extend tests to cover more cases

* chore/rename: Tracker -> ProtocolState

* chore check and comment rewording

* feat test: invalid peer message

* remove ignored test cases and unused macros

* fix master merge fallout + warnings

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

polkadot/Cargo.lock

@@ -219,6 +219,35 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
 
+[[package]]
+name = "async-channel"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee81ba99bee79f3c8ae114ae4baa7eaa326f63447cf2ec65e4393618b63f8770"
+dependencies = [
+ "concurrent-queue",
+ "event-listener",
+ "futures-core",
+]
+
+[[package]]
+name = "async-io"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a0fc2017a5cca12763bb5636092a7786b52789c23c5838a392db2eb99963fd3"
+dependencies = [
+ "cfg-if",
+ "concurrent-queue",
+ "futures-lite",
+ "libc",
+ "once_cell",
+ "parking",
+ "socket2",
+ "vec-arena",
+ "wepoll-sys-stjepang",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "async-std"
 version = "1.6.2"
@@ -239,7 +268,7 @@ dependencies = [
  "pin-project-lite",
  "pin-utils",
  "slab",
- "smol",
+ "smol 0.1.18",
  "wasm-bindgen-futures",
 ]
 
@@ -272,6 +301,12 @@ dependencies = [
  "syn 1.0.33",
 ]
 
+[[package]]
+name = "atomic-waker"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a"
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -469,12 +504,13 @@ dependencies = [
 
 [[package]]
 name = "blocking"
-version = "0.4.6"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d17efb70ce4421e351d61aafd90c16a20fb5bfe339fcdc32a86816280e62ce0"
+checksum = "d2468ff7bf85066b4a3678fede6fe66db31846d753ff0adfbfab2c6a6e81612b"
 dependencies = [
- "futures-channel",
+ "async-channel",
- "futures-util",
+ "atomic-waker",
+ "futures-lite",
  "once_cell",
  "parking",
  "waker-fn",
@@ -1155,6 +1191,12 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "event-listener"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "699d84875f1b72b4da017e6b0f77dfa88c0137f089958a88974d15938cbc2976"
+
 [[package]]
 name = "exit-future"
 version = "0.2.0"
@@ -1593,6 +1635,19 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789"
 
+[[package]]
+name = "futures-lite"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af0bbcb0ec905ef6ee23fab499119b5da2362b8697d66e08d1ef01a8c0d438e2"
+dependencies = [
+ "fastrand",
+ "futures-core",
+ "futures-io",
+ "memchr",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "futures-macro"
 version = "0.3.5"
@@ -2988,6 +3043,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "maplit"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
+
 [[package]]
 name = "matches"
 version = "0.1.8"
@@ -3198,6 +3259,17 @@ dependencies = [
  "unsigned-varint 0.4.0",
 ]
 
+[[package]]
+name = "multitask"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c09c35271e7dcdb5f709779111f2c8e8ab8e06c1b587c1c6a9e179d865aaa5b4"
+dependencies = [
+ "async-task",
+ "concurrent-queue",
+ "fastrand",
+]
+
 [[package]]
 name = "nalgebra"
 version = "0.18.1"
@@ -4102,9 +4174,9 @@ checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865"
 
 [[package]]
 name = "parking"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1efcee3c6d23b94012e240525f131c6abaa9e5eeb8f211002d93beec3b7be350"
+checksum = "50d4a6da31f8144a32532fe38fe8fb439a6842e0ec633f0037f0144c14e7f907"
 
 [[package]]
 name = "parking_lot"
@@ -4277,6 +4349,31 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "polkadot-availability-bitfield-distribution"
+version = "0.1.0"
+dependencies = [
+ "assert_matches",
+ "bitvec",
+ "env_logger",
+ "futures 0.3.5",
+ "futures-timer 3.0.2",
+ "log 0.4.8",
+ "maplit",
+ "parity-scale-codec",
+ "parking_lot 0.10.2",
+ "polkadot-network",
+ "polkadot-network-bridge",
+ "polkadot-node-primitives",
+ "polkadot-node-subsystem",
+ "polkadot-primitives",
+ "sc-network",
+ "smol 0.2.0",
+ "smol-timeout",
+ "sp-core",
+ "streamunordered",
+]
+
 [[package]]
 name = "polkadot-availability-store"
 version = "0.8.19"
@@ -7147,6 +7244,29 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "smol"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "346a94824d48ed7c5fc7247f3cbbf0317bdfe15fc39d08f9262609cccce61254"
+dependencies = [
+ "async-io",
+ "blocking",
+ "multitask",
+ "num_cpus",
+ "once_cell",
+]
+
+[[package]]
+name = "smol-timeout"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "024818c1f00b80e8171ddcfcee33860134293aa3aced60c9cbd7a5a2d41db392"
+dependencies = [
+ "pin-project",
+ "smol 0.1.18",
+]
+
 [[package]]
 name = "snow"
 version = "0.7.1"
@@ -8929,6 +9049,12 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c"
 
+[[package]]
+name = "vec-arena"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17dfb54bf57c9043f4616cb03dab30eff012cc26631b797d8354b916708db919"
+
 [[package]]
 name = "vec_map"
 version = "0.8.2"

polkadot/Cargo.toml

@@ -47,6 +47,7 @@ members = [
 	"node/network/bridge",
 	"node/network/pov-distribution",
 	"node/network/statement-distribution",
+	"node/network/bitfield-distribution",
 	"node/overseer",
 	"node/primitives",
 	"node/service",

polkadot/node/network/bitfield-distribution/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "polkadot-availability-bitfield-distribution"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+futures = "0.3.5"
+futures-timer = "3.0.2"
+log = "0.4.8"
+streamunordered = "0.5.1"
+codec = { package="parity-scale-codec", version = "1.3.0" }
+node-primitives = { package = "polkadot-node-primitives", path = "../../primitives" }
+polkadot-primitives = { path = "../../../primitives" }
+polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" }
+polkadot-network-bridge = { path = "../../network/bridge" }
+polkadot-network = { path = "../../../network" }
+sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
+
+[dev-dependencies]
+polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem", features = [ "test-helpers" ] }
+bitvec = { version = "0.17.4", default-features = false, features = ["alloc"] }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+parking_lot = "0.10.0"
+maplit = "1.0.2"
+smol = "0.2.0"
+smol-timeout = "0.1.0"
+env_logger = "0.7.1"
+assert_matches = "1.3.0"

polkadot/node/subsystem/src/messages.rs

@@ -34,7 +34,6 @@ use polkadot_primitives::v1::{
 use polkadot_node_primitives::{
 	MisbehaviorReport, SignedFullStatement, View, ProtocolId, ValidationResult,
 };
-
 use std::sync::Arc;
 
 pub use sc_network::{ObservedRole, ReputationChange, PeerId};

polkadot/roadmap/implementers-guide/src/node/availability/bitfield-distribution.md

@@ -6,16 +6,30 @@ Validators vote on the availability of a backed candidate by issuing signed bitf
 
 `ProtocolId`: `b"bitd"`
 
-Input: [`BitfieldDistributionMessage`](../../types/overseer-protocol.md#bitfield-distribution-message)
+Input:
+[`BitfieldDistributionMessage`](../../types/overseer-protocol.md#bitfield-distribution-message) which are gossiped to all peers, no matter if validator or not.
+
 Output:
 
-- `NetworkBridge::RegisterEventProducer(ProtocolId)`
+- `NetworkBridge::RegisterEventProducer(ProtocolId)` in order to register ourself as an event provider for the protocol.
-- `NetworkBridge::SendMessage([PeerId], ProtocolId, Bytes)`
+- `NetworkBridge::SendMessage([PeerId], ProtocolId, Bytes)` gossip a verified incoming bitfield on to interested subsystems within this validator node.
-- `NetworkBridge::ReportPeer(PeerId, cost_or_benefit)`
+- `NetworkBridge::ReportPeer(PeerId, cost_or_benefit)` improve or penalize the reputation of peers based on the messages that are received relative to the current view.
-- `BlockAuthorshipProvisioning::Bitfield(relay_parent, SignedAvailabilityBitfield)`
+- `ProvisionerMessage::ProvisionableData(ProvisionableData::Bitfield(relay_parent, SignedAvailabilityBitfield))` pass
+  on the bitfield to the other submodules via the overseer.
 
 ## Functionality
 
-This is implemented as a gossip system. Register a [network bridge](../utility/network-bridge.md) event producer on startup and track peer connection, view change, and disconnection events. Only accept bitfields relevant to our current view and only distribute bitfields to other peers when relevant to their most recent view. Check bitfield signatures in this subsystem and accept and distribute only one bitfield per validator.
+This is implemented as a gossip system. Register a [network bridge](../utility/network-bridge.md) event producer on startup.
+
+It is necessary to track peer connection, view change, and disconnection events, in order to maintain an index of which peers are interested in which relay parent bitfields.
+
+
+Before gossiping incoming bitfields, they must be checked to be signed by one of the validators
+of the validator set relevant to the current relay parent.
+Only accept bitfields relevant to our current view and only distribute bitfields to other peers when relevant to their most recent view.
+Accept and distribute only one bitfield per validator.
+
 
 When receiving a bitfield either from the network or from a `DistributeBitfield` message, forward it along to the block authorship (provisioning) subsystem for potential inclusion in a block.
+
+Peers connecting after a set of valid bitfield gossip messages was received, those messages must be cached and sent upon connection of new peers or re-connecting peers.