Statement store (#13701)

* WIP Statement store

* Sync with networking changes in master

* WIP statement pallet

* Statement validation

* pallet tests

* Validation queue

* Store maintenance

* Basic statement refactoring + tests + docs

* Store metrics

* Store tests

* Store maintenance test

* cargo fmt

* Build fix

* OCW Api

* Offchain worker

* Enable host functions

* fmt

* Minor tweaks

* Fixed a warning

* Removed tracing

* Manual expiration

* Reworked constraint management

* Updated pallet constraint calculation

* Added small test

* Added remove function to the APIs

* Copy-paste spec into readme

* Comments

* Made the store optional

* Removed network protocol controller

* fmt

* Clippy fixes

* fmt

* fmt

* More clippy fixes

* More clippy fixes

* More clippy fixes

* Update client/statement-store/README.md

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <git@kchr.de>

* Removed sstore from node-template

* Sort out data path

* Added offline check

* Removed dispatch_statement

* Renamed into_generic

* Fixed commit placement

* Use HashSet for tracking peers/statements

* fmt

* Use ExtendedHostFunctions

* Fixed benches

* Tweaks

* Apply suggestions from code review

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* Fixed priority mixup

* Rename

* newtypes for priorities

* Added MAX_TOPICS

* Fixed key filtering logic

* Remove empty entrie

* Removed prefix from signing

* More documentation

* fmt

* Moved store setup from sc-service to node

* Handle maintenance task in sc-statement-store

* Use statement iterator

* Renamed runtime API mod

* fmt

* Remove dump_encoded

* fmt

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <git@kchr.de>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <git@kchr.de>

* Fixed build after applying review suggestions

* License exceptions

* fmt

* Store options

* Moved pallet consts to config trait

* Removed global priority

* Validate fields when decoding

* Limit validation channel size

* Made a comment into module doc

* Removed submit_encoded

---------

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Bastian Köcher <git@kchr.de>

substrate/Cargo.lock

@@ -3783,6 +3783,7 @@ dependencies = [
  "pallet-staking-reward-curve",
  "pallet-staking-runtime-api",
  "pallet-state-trie-migration",
+ "pallet-statement",
  "pallet-sudo",
  "pallet-timestamp",
  "pallet-tips",
@@ -3808,6 +3809,7 @@ dependencies = [
  "sp-runtime",
  "sp-session",
  "sp-staking",
+ "sp-statement-store",
  "sp-std",
  "sp-transaction-pool",
  "sp-version",
@@ -5120,10 +5122,12 @@ dependencies = [
  "sc-keystore",
  "sc-network",
  "sc-network-common",
+ "sc-network-statement",
  "sc-network-sync",
  "sc-rpc",
  "sc-service",
  "sc-service-test",
+ "sc-statement-store",
  "sc-storage-monitor",
  "sc-sync-state-rpc",
  "sc-sysinfo",
@@ -5192,6 +5196,7 @@ dependencies = [
  "sp-keystore",
  "sp-runtime",
  "sp-state-machine",
+ "sp-statement-store",
  "sp-tracing",
  "sp-trie",
  "wat",
@@ -5251,6 +5256,7 @@ dependencies = [
  "sp-consensus-babe",
  "sp-keystore",
  "sp-runtime",
+ "sp-statement-store",
  "substrate-frame-rpc-system",
  "substrate-state-trie-migration-rpc",
 ]
@@ -5288,6 +5294,7 @@ dependencies = [
  "sc-rpc",
  "sc-rpc-api",
  "sc-service",
+ "sc-statement-store",
  "sc-telemetry",
  "sc-transaction-pool",
  "sc-transaction-pool-api",
@@ -6990,6 +6997,24 @@ dependencies = [
  "zstd 0.12.3+zstd.1.5.2",
 ]
 
+[[package]]
+name = "pallet-statement"
+version = "4.0.0-dev"
+dependencies = [
+ "frame-support",
+ "frame-system",
+ "log",
+ "pallet-balances",
+ "parity-scale-codec",
+ "scale-info",
+ "sp-api",
+ "sp-core",
+ "sp-io",
+ "sp-runtime",
+ "sp-statement-store",
+ "sp-std",
+]
+
 [[package]]
 name = "pallet-sudo"
 version = "4.0.0-dev"
@@ -8618,6 +8643,7 @@ dependencies = [
  "sp-keystore",
  "sp-runtime",
  "sp-state-machine",
+ "sp-statement-store",
  "sp-storage",
  "sp-test-primitives",
  "substrate-prometheus-endpoint",
@@ -9293,6 +9319,26 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "sc-network-statement"
+version = "0.10.0-dev"
+dependencies = [
+ "array-bytes",
+ "async-channel",
+ "futures",
+ "libp2p",
+ "log",
+ "parity-scale-codec",
+ "pin-project",
+ "sc-network",
+ "sc-network-common",
+ "sc-peerset",
+ "sp-consensus",
+ "sp-runtime",
+ "sp-statement-store",
+ "substrate-prometheus-endpoint",
+]
+
 [[package]]
 name = "sc-network-sync"
 version = "0.10.0-dev"
@@ -9474,6 +9520,7 @@ dependencies = [
  "sp-rpc",
  "sp-runtime",
  "sp-session",
+ "sp-statement-store",
  "sp-version",
  "substrate-test-runtime-client",
  "tokio",
@@ -9672,6 +9719,30 @@ dependencies = [
  "sp-core",
 ]
 
+[[package]]
+name = "sc-statement-store"
+version = "4.0.0-dev"
+dependencies = [
+ "async-trait",
+ "env_logger 0.9.3",
+ "futures",
+ "futures-timer",
+ "log",
+ "parity-db",
+ "parity-scale-codec",
+ "parking_lot 0.12.1",
+ "sc-client-api",
+ "sp-api",
+ "sp-blockchain",
+ "sp-core",
+ "sp-runtime",
+ "sp-statement-store",
+ "sp-tracing",
+ "substrate-prometheus-endpoint",
+ "tempfile",
+ "tokio",
+]
+
 [[package]]
 name = "sc-storage-monitor"
 version = "0.1.0"
@@ -10980,6 +11051,23 @@ dependencies = [
  "trie-db",
 ]
 
+[[package]]
+name = "sp-statement-store"
+version = "4.0.0-dev"
+dependencies = [
+ "log",
+ "parity-scale-codec",
+ "scale-info",
+ "sp-api",
+ "sp-application-crypto",
+ "sp-core",
+ "sp-externalities",
+ "sp-runtime",
+ "sp-runtime-interface",
+ "sp-std",
+ "thiserror",
+]
+
 [[package]]
 name = "sp-std"
 version = "5.0.0"