pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-12 20:31:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Request based availability distribution (#2423)

Browse Source 
* WIP

* availability distribution, still very wip.

Work on the requesting side of things.

* Some docs on what I intend to do.

* Checkpoint of session cache implementation

as I will likely replace it with something smarter.

* More work, mostly on cache

and getting things to type check.

* Only derive MallocSizeOf and Debug for std.

* availability-distribution: Cache feature complete.

* Sketch out logic in `FetchTask` for actual fetching.

- Compile fixes.
- Cleanup.

* Format cleanup.

* More format fixes.

* Almost feature complete `fetch_task`.

Missing:

- Check for cancel
- Actual querying of peer ids.

* Finish FetchTask so far.

* Directly use AuthorityDiscoveryId in protocol and cache.

* Resolve `AuthorityDiscoveryId` on sending requests.

* Rework fetch_task

- also make it impossible to check the wrong chunk index.
- Export needed function in validator_discovery.

* From<u32> implementation for `ValidatorIndex`.

* Fixes and more integration work.

* Make session cache proper lru cache.

* Use proper lru cache.

* Requester finished.

* ProtocolState -> Requester

Also make sure to not fetch our own chunk.

* Cleanup + fixes.

* Remove unused functions

- FetchTask::is_finished
- SessionCache::fetch_session_info

* availability-distribution responding side.

* Cleanup + Fixes.

* More fixes.

* More fixes.

adder-collator is running!

* Some docs.

* Docs.

* Fix reporting of bad guys.

* Fix tests

* Make all tests compile.

* Fix test.

* Cleanup + get rid of some warnings.

* state -> requester

* Mostly doc fixes.

* Fix test suite.

* Get rid of now redundant message types.

* WIP

* Rob's review remarks.

* Fix test suite.

* core.relay_parent -> leaf for session request.

* Style fix.

* Decrease request timeout.

* Cleanup obsolete errors.

* Metrics + don't fail on non fatal errors.

* requester.rs -> requester/mod.rs

* Panic on invalid BadValidator report.

* Fix indentation.

* Use typed default timeout constant.

* Make channel size 0, as each sender gets one slot anyways.

* Fix incorrect metrics initialization.

* Fix build after merge.

* More fixes.

* Hopefully valid metrics names.

* Better metrics names.

* Some tests that already work.

* Slightly better docs.

* Some more tests.

* Fix network bridge test.
This commit is contained in:
Robert Klotzner
2021-02-26 18:58:07 +01:00
committed by GitHub
parent 241b1f12a7
commit 48409e5548
45 changed files with 2037 additions and 1523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/node/core/av-store/src/lib.rs
+5 -5
View File
@@ -968,7 +968,7 @@ fn process_message(
AvailabilityStoreMessage::QueryChunkAvailability(candidate, validator_index, tx) => {
let a = load_meta(&subsystem.db, &candidate)?
.map_or(false, |m|
*m.chunks_stored.get(validator_index as usize).as_deref().unwrap_or(&false)
*m.chunks_stored.get(validator_index.0 as usize).as_deref().unwrap_or(&false)
);
let _ = tx.send(a);
}
@@ -1034,10 +1034,10 @@ fn store_chunk(
None => return Ok(false), // we weren't informed of this candidate by import events.
};
match meta.chunks_stored.get(chunk.index as usize).map(|b| *b) {
match meta.chunks_stored.get(chunk.index.0 as usize).map(|b| *b) {
Some(true) => return Ok(true), // already stored.
Some(false) => {
meta.chunks_stored.set(chunk.index as usize, true);
meta.chunks_stored.set(chunk.index.0 as usize, true);
write_chunk(&mut tx, &candidate_hash, chunk.index, &chunk);
write_meta(&mut tx, &candidate_hash, &meta);
@@ -1090,7 +1090,7 @@ fn store_available_data(
.map(|(index, (chunk, proof))| ErasureChunk {
chunk: chunk.clone(),
proof,
index: index as u32,
index: ValidatorIndex(index as u32),
});
for chunk in erasure_chunks {
@@ -1142,7 +1142,7 @@ fn prune_all(db: &Arc<dyn KeyValueDB>, clock: &dyn Clock) -> Result<(), Error> {
// delete chunks.
for (i, b) in meta.chunks_stored.iter().enumerate() {
if *b {
delete_chunk(&mut tx, &candidate_hash, i as _);
delete_chunk(&mut tx, &candidate_hash, ValidatorIndex(i as _));
}
}
polkadot/node/core/av-store/src/tests.rs
+13 -13
View File
@@ -260,7 +260,7 @@ fn runtime_api_error_does_not_stop_the_subsystem() {
// but that's fine, we're still alive
let (tx, rx) = oneshot::channel();
let candidate_hash = CandidateHash(Hash::repeat_byte(33));
let validator_index = 5;
let validator_index = ValidatorIndex(5);
let query_chunk = AvailabilityStoreMessage::QueryChunk(
candidate_hash,
validator_index,
@@ -281,7 +281,7 @@ fn store_chunk_works() {
let TestHarness { mut virtual_overseer } = test_harness;
let relay_parent = Hash::repeat_byte(32);
let candidate_hash = CandidateHash(Hash::repeat_byte(33));
let validator_index = 5;
let validator_index = ValidatorIndex(5);
let n_validators = 10;
let chunk = ErasureChunk {
@@ -333,7 +333,7 @@ fn store_chunk_does_nothing_if_no_entry_already() {
let TestHarness { mut virtual_overseer } = test_harness;
let relay_parent = Hash::repeat_byte(32);
let candidate_hash = CandidateHash(Hash::repeat_byte(33));
let validator_index = 5;
let validator_index = ValidatorIndex(5);
let chunk = ErasureChunk {
chunk: vec![1, 2, 3],
@@ -372,7 +372,7 @@ fn query_chunk_checks_meta() {
test_harness(TestState::default(), store.clone(), |test_harness| async move {
let TestHarness { mut virtual_overseer } = test_harness;
let candidate_hash = CandidateHash(Hash::repeat_byte(33));
let validator_index = 5;
let validator_index = ValidatorIndex(5);
let n_validators = 10;
// Ensure an entry already exists. In reality this would come from watching
@@ -382,7 +382,7 @@ fn query_chunk_checks_meta() {
data_available: false,
chunks_stored: {
let mut v = bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators];
v.set(validator_index as usize, true);
v.set(validator_index.0 as usize, true);
v
},
state: State::Unavailable(BETimestamp(0)),
@@ -402,7 +402,7 @@ fn query_chunk_checks_meta() {
let (tx, rx) = oneshot::channel();
let query_chunk = AvailabilityStoreMessage::QueryChunkAvailability(
candidate_hash,
validator_index + 1,
ValidatorIndex(validator_index.0 + 1),
tx,
);
@@ -418,7 +418,7 @@ fn store_block_works() {
test_harness(test_state.clone(), store.clone(), |test_harness| async move {
let TestHarness { mut virtual_overseer } = test_harness;
let candidate_hash = CandidateHash(Hash::repeat_byte(1));
let validator_index = 5;
let validator_index = ValidatorIndex(5);
let n_validators = 10;
let pov = PoV {
@@ -455,7 +455,7 @@ fn store_block_works() {
let branch = branches.nth(5).unwrap();
let expected_chunk = ErasureChunk {
chunk: branch.1.to_vec(),
index: 5,
index: ValidatorIndex(5),
proof: branch.0,
};
@@ -497,10 +497,10 @@ fn store_pov_and_query_chunk_works() {
assert_eq!(rx.await.unwrap(), Ok(()));
for validator_index in 0..n_validators {
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, validator_index).await.unwrap();
for i in 0..n_validators {
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, ValidatorIndex(i as _)).await.unwrap();
assert_eq!(chunk.chunk, chunks_expected[validator_index as usize]);
assert_eq!(chunk.chunk, chunks_expected[i as usize]);
}
});
}
@@ -842,7 +842,7 @@ async fn query_available_data(
async fn query_chunk(
virtual_overseer: &mut test_helpers::TestSubsystemContextHandle<AvailabilityStoreMessage>,
candidate_hash: CandidateHash,
index: u32,
index: ValidatorIndex,
) -> Option<ErasureChunk> {
let (tx, rx) = oneshot::channel();
@@ -859,7 +859,7 @@ async fn query_all_chunks(
expect_present: bool,
) -> bool {
for i in 0..n_validators {
if query_chunk(virtual_overseer, candidate_hash, i).await.is_some() != expect_present {
if query_chunk(virtual_overseer, candidate_hash, ValidatorIndex(i)).await.is_some() != expect_present {
return false
}
}