mirror of
https://github.com/pezkuwichain/pezkuwi-fellows.git
synced 2026-05-30 08:11:01 +00:00
bkchr
453 lines
30 KiB
HTML
453 lines
30 KiB
HTML
|
|
<!DOCTYPE HTML>
|
|
<html lang="en" class="polkadot" dir="ltr">
|
|
<head>
|
|
<!-- Book generated using mdBook -->
|
|
<meta charset="UTF-8">
|
|
<title>RFC-0139: Faster Erasure Coding - Polkadot Fellowship RFCs</title>
|
|
|
|
|
|
<!-- Custom HTML head -->
|
|
|
|
<meta name="description" content="An online book of RFCs approved or proposed within the Polkadot Fellowship.">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="theme-color" content="#ffffff">
|
|
|
|
<link rel="icon" href="../favicon.svg">
|
|
<link rel="shortcut icon" href="../favicon.png">
|
|
<link rel="stylesheet" href="../css/variables.css">
|
|
<link rel="stylesheet" href="../css/general.css">
|
|
<link rel="stylesheet" href="../css/chrome.css">
|
|
<link rel="stylesheet" href="../css/print.css" media="print">
|
|
|
|
<!-- Fonts -->
|
|
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
|
<link rel="stylesheet" href="../fonts/fonts.css">
|
|
|
|
<!-- Highlight.js Stylesheets -->
|
|
<link rel="stylesheet" href="../highlight.css">
|
|
<link rel="stylesheet" href="../tomorrow-night.css">
|
|
<link rel="stylesheet" href="../ayu-highlight.css">
|
|
|
|
<!-- Custom theme stylesheets -->
|
|
<link rel="stylesheet" href="../theme/polkadot.css">
|
|
|
|
</head>
|
|
<body class="sidebar-visible no-js">
|
|
<div id="body-container">
|
|
<!-- Provide site root to javascript -->
|
|
<script>
|
|
var path_to_root = "../";
|
|
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "polkadot" : "polkadot";
|
|
</script>
|
|
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
|
<script>
|
|
try {
|
|
var theme = localStorage.getItem('mdbook-theme');
|
|
var sidebar = localStorage.getItem('mdbook-sidebar');
|
|
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
|
}
|
|
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
|
}
|
|
} catch (e) { }
|
|
</script>
|
|
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
|
<script>
|
|
var theme;
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
|
var html = document.querySelector('html');
|
|
html.classList.remove('polkadot')
|
|
html.classList.add(theme);
|
|
var body = document.querySelector('body');
|
|
body.classList.remove('no-js')
|
|
body.classList.add('js');
|
|
</script>
|
|
|
|
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
|
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
|
<script>
|
|
var body = document.querySelector('body');
|
|
var sidebar = null;
|
|
var sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
|
if (document.body.clientWidth >= 1080) {
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
|
sidebar = sidebar || 'visible';
|
|
} else {
|
|
sidebar = 'hidden';
|
|
}
|
|
sidebar_toggle.checked = sidebar === 'visible';
|
|
body.classList.remove('sidebar-visible');
|
|
body.classList.add("sidebar-" + sidebar);
|
|
</script>
|
|
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
|
<div class="sidebar-scrollbox">
|
|
<ol class="chapter"><li class="chapter-item expanded affix "><a href="../introduction.html">Introduction</a></li><li class="spacer"></li><li class="chapter-item expanded affix "><li class="part-title">Newly Proposed</li><li class="chapter-item expanded "><a href="../new/0139-faster-erasure-coding.html" class="active">RFC-0139: Faster Erasure Coding</a></li><li class="spacer"></li><li class="chapter-item expanded affix "><li class="part-title">Proposed</li><li class="chapter-item expanded "><a href="../proposed/0112-compress-state-response-message-in-state-sync.html">RFC-0112: Compress the State Response Message in State Sync</a></li><li class="chapter-item expanded "><a href="../proposed/0123-pending-code-as-storage-location-for-runtime-upgrades.html">RFC-0123: Introduce :pending_code as intermediate storage key for the runtime code</a></li><li class="chapter-item expanded "><a href="../proposed/0126-introduce-pvq.html">RFC-0126: Introduce PVQ (PolkaVM Query)</a></li><li class="chapter-item expanded "><a href="../proposed/0135-compressed-blob-prefixes.html">RFC-0135: Compressed Blob Prefixes</a></li><li class="chapter-item expanded "><a href="../proposed/0138-invulnerable-collator-election.html">RFC-0138: Election mechanism for invulnerable collators on system chains</a></li><li class="spacer"></li><li class="chapter-item expanded affix "><li class="part-title">Approved</li><li class="chapter-item expanded "><a href="../approved/0001-agile-coretime.html">RFC-1: Agile Coretime</a></li><li class="chapter-item expanded "><a href="../approved/0005-coretime-interface.html">RFC-5: Coretime Interface</a></li><li class="chapter-item expanded "><a href="../approved/0007-system-collator-selection.html">RFC-0007: System Collator Selection</a></li><li class="chapter-item expanded "><a href="../approved/0008-parachain-bootnodes-dht.html">RFC-0008: Store parachain bootnodes in relay chain DHT</a></li><li class="chapter-item expanded "><a href="../approved/0009-improved-net-light-client-requests.html">RFC-0009: Improved light client requests networking protocol</a></li><li class="chapter-item expanded "><a href="../approved/0010-burn-coretime-revenue.html">RFC-0010: Burn Coretime Revenue</a></li><li class="chapter-item expanded "><a href="../approved/0012-process-for-adding-new-collectives.html">RFC-0012: Process for Adding New System Collectives</a></li><li class="chapter-item expanded "><a href="../approved/0013-prepare-blockbuilder-and-core-runtime-apis-for-mbms.html">RFC-0013: Prepare Core runtime API for MBMs</a></li><li class="chapter-item expanded "><a href="../approved/0014-improve-locking-mechanism-for-parachains.html">RFC-0014: Improve locking mechanism for parachains</a></li><li class="chapter-item expanded "><a href="../approved/0022-adopt-encointer-runtime.html">RFC-0022: Adopt Encointer Runtime</a></li><li class="chapter-item expanded "><a href="../approved/0026-sassafras-consensus.html">RFC-0026: Sassafras Consensus Protocol</a></li><li class="chapter-item expanded "><a href="../approved/0032-minimal-relay.html">RFC-0032: Minimal Relay</a></li><li class="chapter-item expanded "><a href="../approved/0042-extrinsics-state-version.html">RFC-0042: Add System version that replaces StateVersion on RuntimeVersion</a></li><li class="chapter-item expanded "><a href="../approved/0043-storage-proof-size-hostfunction.html">RFC-0043: Introduce storage_proof_size Host Function for Improved Parachain Block Utilization</a></li><li class="chapter-item expanded "><a href="../approved/0045-nft-deposits-asset-hub.html">RFC-0045: Lowering NFT Deposits on Asset Hub</a></li><li class="chapter-item expanded "><a href="../approved/0047-assignment-of-availability-chunks.html">RFC-0047: Assignment of availability chunks to validators</a></li><li class="chapter-item expanded "><a href="../approved/0048-session-keys-runtime-api.html">RFC-0048: Generate ownership proof for SessionKeys</a></li><li class="chapter-item expanded "><a href="../approved/0050-fellowship-salaries.html">RFC-0050: Fellowship Salaries</a></li><li class="chapter-item expanded "><a href="../approved/0056-one-transaction-per-notification.html">RFC-0056: Enforce only one transaction per notification</a></li><li class="chapter-item expanded "><a href="../approved/0059-nodes-capabilities-discovery.html">RFC-0059: Add a discovery mechanism for nodes based on their capabilities</a></li><li class="chapter-item expanded "><a href="../approved/0078-merkleized-metadata.html">RFC-0078: Merkleized Metadata</a></li><li class="chapter-item expanded "><a href="../approved/0084-general-transaction-extrinsic-format.html">RFC-0084: General transactions in extrinsic format</a></li><li class="chapter-item expanded "><a href="../approved/0091-dht-record-creation-time.html">RFC-0091: DHT Authority discovery record creation time</a></li><li class="chapter-item expanded "><a href="../approved/0097-unbonding_queue.html">RFC-0097: Unbonding Queue</a></li><li class="chapter-item expanded "><a href="../approved/0099-transaction-extension-version.html">RFC-0099: Introduce a transaction extension version</a></li><li class="chapter-item expanded "><a href="../approved/0100-xcm-multi-type-asset-transfer.html">RFC-0100: New XCM instruction: InitiateAssetsTransfer</a></li><li class="chapter-item expanded "><a href="../approved/0101-xcm-transact-remove-max-weight-param.html">RFC-0101: XCM Transact remove require_weight_at_most parameter</a></li><li class="chapter-item expanded "><a href="../approved/0103-introduce-core-index-commitment.html">RFC-0103: Introduce a CoreIndex commitment and a SessionIndex field in candidate receipts</a></li><li class="chapter-item expanded "><a href="../approved/0105-xcm-improved-fee-mechanism.html">RFC-0105: XCM improved fee mechanism</a></li><li class="chapter-item expanded "><a href="../approved/0107-xcm-execution-hints.html">RFC-0107: XCM Execution hints</a></li><li class="chapter-item expanded "><a href="../approved/0108-xcm-remove-testnet-ids.html">RFC-0108: Remove XCM testnet NetworkIds</a></li><li class="chapter-item expanded "><a href="../approved/0122-alias-origin-on-asset-transfers.html">RFC-0122: Asset transfers can alias XCM origin on destination to original origin</a></li><li class="chapter-item expanded "><a href="../approved/0125-xcm-asset-metadata.html">RFC-0125: XCM Asset Metadata</a></li><li class="spacer"></li><li class="chapter-item expanded affix "><li class="part-title">Stale</li><li class="chapter-item expanded "><a href="../stale/0000-rewards.html">RFC-0000: Validator Rewards</a></li><li class="chapter-item expanded "><a href="../stale/0004-remove-unnecessary-allocator-usage.html">RFC-0004: Remove the host-side runtime memory allocator</a></li><li class="chapter-item expanded "><a href="../stale/0006-dynamic-pricing-for-bulk-coretime-sales.html">RFC-0006: Dynamic Pricing for Bulk Coretime Sales</a></li><li class="chapter-item expanded "><a href="../stale/0015-market-design-revisit.html">RFC-0015: Market Design Revisit</a></li><li class="chapter-item expanded "><a href="../stale/0034-xcm-absolute-location-account-derivation.html">RFC-34: XCM Absolute Location Account Derivation</a></li><li class="chapter-item expanded "><a href="../stale/0035-conviction-voting-delegation-modifications.html"> RFC-0035: Conviction Voting Delegation Modifications</a></li><li class="chapter-item expanded "><a href="../stale/0044-rent-based-registration.html">RFC-0044: Rent based registration model</a></li><li class="chapter-item expanded "><a href="../stale/0054-remove-heap-pages.html">RFC-0054: Remove the concept of "heap pages" from the client</a></li><li class="chapter-item expanded "><a href="../stale/0070-x-track-kusamanetwork.html">RFC-0070: X Track for @kusamanetwork</a></li><li class="chapter-item expanded "><a href="../stale/0073-referedum-deposit-track.html">RFC-0073: Decision Deposit Referendum Track</a></li><li class="chapter-item expanded "><a href="../stale/0074-stateful-multisig-pallet.html">RFC-0074: Stateful Multisig Pallet</a></li><li class="chapter-item expanded "><a href="../stale/0077-increase-max-length-of-identity-pgp-fingerprint-value.html">RFC-0077: Increase maximum length of identity PGP fingerprint values from 20 bytes</a></li><li class="chapter-item expanded "><a href="../stale/0088-broker-pallet-slashable-deposit-purchaser-reputation-reserved-cores.html">RFC-0088: Add slashable locked deposit, purchaser reputation, and reserved cores for on-chain identities to broker pallet</a></li><li class="chapter-item expanded "><a href="../stale/00xx-secondary-marketplace-for-regions.html">RFC-0001: Secondary Market for Regions</a></li><li class="chapter-item expanded "><a href="../stale/00xx-smart-contracts-coretime-chain.html">RFC-0002: Smart Contracts on the Coretime Chain</a></li><li class="chapter-item expanded "><a href="../stale/0102-offchain-parachain-runtime-upgrades.html">RFC-0000: Feature Name Here</a></li><li class="chapter-item expanded "><a href="../stale/0106-xcm-remove-fees-mode.html">RFC-0106: Remove XCM fees mode</a></li><li class="chapter-item expanded "><a href="../stale/0111-pure-proxy-replication.html">RFC-0111: Pure Proxy Replication</a></li><li class="chapter-item expanded "><a href="../stale/0114-secp256r1-hostfunction.html">RFC-0114: Introduce secp256r1_ecdsa_verify_prehashed Host Function to verify NIST-P256 elliptic curve signatures</a></li><li class="chapter-item expanded "><a href="../stale/0117-unbrick-collective.html">RFC-0117: The Unbrick Collective</a></li><li class="chapter-item expanded "><a href="../stale/0120-referenda-confirmation-by-candle-mechanism.html">RFC-0120: Referenda Confirmation by Candle Mechanism</a></li><li class="chapter-item expanded "><a href="../stale/0121-iterable-referenda-tracks.html">RFC-0121: Iterable Referenda Tracks</a></li><li class="chapter-item expanded "><a href="../stale/0124-extrinsic-version-5.html">RFC-0124: Extrinsic version 5</a></li><li class="chapter-item expanded "><a href="../stale/RFC-114 Adjust Tipper Track Confirmation Periods.html">RFC-114: Adjust Tipper Track Confirmation Periods</a></li><li class="chapter-item expanded "><a href="../stale/TODO-stale-nomination-reward-curve.html">RFC-TODO: Stale Nomination Reward Curve</a></li></ol>
|
|
</div>
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
|
|
</nav>
|
|
|
|
<!-- Track and set sidebar scroll position -->
|
|
<script>
|
|
var sidebarScrollbox = document.querySelector('#sidebar .sidebar-scrollbox');
|
|
sidebarScrollbox.addEventListener('click', function(e) {
|
|
if (e.target.tagName === 'A') {
|
|
sessionStorage.setItem('sidebar-scroll', sidebarScrollbox.scrollTop);
|
|
}
|
|
}, { passive: true });
|
|
var sidebarScrollTop = sessionStorage.getItem('sidebar-scroll');
|
|
sessionStorage.removeItem('sidebar-scroll');
|
|
if (sidebarScrollTop) {
|
|
// preserve sidebar scroll position when navigating via links within sidebar
|
|
sidebarScrollbox.scrollTop = sidebarScrollTop;
|
|
} else {
|
|
// scroll sidebar to current active section when navigating via "next/previous chapter" buttons
|
|
var activeSection = document.querySelector('#sidebar .active');
|
|
if (activeSection) {
|
|
activeSection.scrollIntoView({ block: 'center' });
|
|
}
|
|
}
|
|
</script>
|
|
|
|
<div id="page-wrapper" class="page-wrapper">
|
|
|
|
<div class="page">
|
|
<div id="menu-bar-hover-placeholder"></div>
|
|
<div id="menu-bar" class="menu-bar sticky">
|
|
<div class="left-buttons">
|
|
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
|
<i class="fa fa-bars"></i>
|
|
</label>
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
|
<i class="fa fa-paint-brush"></i>
|
|
</button>
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
|
<li role="none"><button role="menuitem" class="theme" id="polkadot">Polkadot</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
|
</ul>
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
|
|
<i class="fa fa-search"></i>
|
|
</button>
|
|
</div>
|
|
|
|
<h1 class="menu-title">Polkadot Fellowship RFCs</h1>
|
|
|
|
<div class="right-buttons">
|
|
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
|
<i id="print-button" class="fa fa-print"></i>
|
|
</a>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div id="search-wrapper" class="hidden">
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
|
</form>
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
|
<ul id="searchresults">
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
|
<script>
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
|
});
|
|
</script>
|
|
|
|
<div id="content" class="content">
|
|
<main>
|
|
<p><a href="https://github.com/polkadot-fellows/RFCs/pull/139">(source)</a></p>
|
|
<p><strong>Table of Contents</strong></p>
|
|
<ul>
|
|
<li><a href="#rfc-0139-faster-erasure-coding">RFC-0139: Faster Erasure Coding</a>
|
|
<ul>
|
|
<li><a href="#summary">Summary</a></li>
|
|
<li><a href="#motivation">Motivation</a></li>
|
|
<li><a href="#stakeholders">Stakeholders</a></li>
|
|
<li><a href="#explanation">Explanation</a></li>
|
|
<li><a href="#drawbacks">Drawbacks</a></li>
|
|
<li><a href="#testing-security-and-privacy">Testing, Security, and Privacy</a></li>
|
|
<li><a href="#performance-and-compatibility">Performance and Compatibility</a>
|
|
<ul>
|
|
<li><a href="#performance">Performance</a></li>
|
|
<li><a href="#compatibility">Compatibility</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a href="#prior-art-and-references">Prior Art and References</a></li>
|
|
<li><a href="#unresolved-questions">Unresolved Questions</a></li>
|
|
<li><a href="#future-directions-and-related-material">Future Directions and Related Material</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<h1 id="rfc-0139-faster-erasure-coding"><a class="header" href="#rfc-0139-faster-erasure-coding">RFC-0139: Faster Erasure Coding</a></h1>
|
|
<div class="table-wrapper"><table><thead><tr><th></th><th></th></tr></thead><tbody>
|
|
<tr><td><strong>Start Date</strong></td><td>7 March 2025</td></tr>
|
|
<tr><td><strong>Description</strong></td><td>Faster algorithm for Data Availability Layer</td></tr>
|
|
<tr><td><strong>Authors</strong></td><td>ordian</td></tr>
|
|
</tbody></table>
|
|
</div>
|
|
<h2 id="summary"><a class="header" href="#summary">Summary</a></h2>
|
|
<p>This RFC proposes changes to the erasure coding algorithm and the way the erasure root is computed on Polkadot to make both processes faster.</p>
|
|
<h2 id="motivation"><a class="header" href="#motivation">Motivation</a></h2>
|
|
<p>The Data Availability (DA) Layer provided by Polkadot serves as a foundational layer for
|
|
shared security, currently allowing Approval Checkers and Collators to download
|
|
the Proofs-of-Validity (PoV) for security and liveness purposes respectively.
|
|
As the number of parachains and PoV sizes grow, it is increasingly important
|
|
for the DA to be as performant as possible.</p>
|
|
<p><a href="https://github.com/polkadot-fellows/RFCs/blob/main/text/0047-assignment-of-availability-chunks.md">RFC-47</a>
|
|
proposed a way to enable systematic chunk recovery for Polkadot's DA, improving
|
|
the efficiency/reducing the CPU overhead. However, systematic recovery can only
|
|
work with an almost ideal networking scenario where everyone is connected to the
|
|
corresponding third of validators, and as such, we need to ensure the system will
|
|
sustain the load in the worst-case scenario. On top of that, enabling it
|
|
requires making a breaking change to the protocol (including the collator node
|
|
side).</p>
|
|
<p>We propose bundling another breaking change to the protocol along with RFC-47
|
|
to speed up erasure coding, which constitutes the CPU bottleneck of DA.</p>
|
|
<h2 id="stakeholders"><a class="header" href="#stakeholders">Stakeholders</a></h2>
|
|
<ul>
|
|
<li>Infrastructure providers (people who run validator/collator nodes)
|
|
will need to upgrade their client version in time</li>
|
|
</ul>
|
|
<h2 id="explanation"><a class="header" href="#explanation">Explanation</a></h2>
|
|
<p>In particular, two changes are being proposed:</p>
|
|
<ol>
|
|
<li>Switch the erasure coding algorithm to the one described in the Graypaper,
|
|
Appendix H. SIMD implementations of this algorithm are available in:</li>
|
|
</ol>
|
|
<ul>
|
|
<li><a href="https://github.com/AndersTrier/reed-solomon-simd">Rust</a>,</li>
|
|
<li><a href="https://github.com/catid/leopard">C++</a> and</li>
|
|
<li><a href="https://github.com/celestiaorg/go-leopard">Go</a>.</li>
|
|
</ul>
|
|
<ol start="2">
|
|
<li>For computing the erasure root, switch from Merkle Patricia Trie to a Binary
|
|
Merkle Tree.</li>
|
|
</ol>
|
|
<p>Here is a reference implementation for that:</p>
|
|
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>use blake2b_simd::{blake2b as hash_fn, Hash, State as Hasher};
|
|
|
|
/// Yields all erasure chunks as an iterator.
|
|
pub struct MerklizedChunks {
|
|
root: ErasureRoot,
|
|
data: VecDeque<Vec<u8>>,
|
|
// This is a Binary Merkle Tree,
|
|
// where each level is a vector of hashes starting from leaves.
|
|
// \`\`\`
|
|
// 0 -> [c, d, e, Hash::zero()]
|
|
// 1 -> [a = hash(c, d), b = hash(e, Hash::zero())]
|
|
// 2 -> hash(a, b)
|
|
// \`\`\`
|
|
// Levels are guaranteed to have a power of 2 elements.
|
|
// Leaves might be padded with `Hash::zero()`.
|
|
tree: Vec<Vec<Hash>>,
|
|
// Used by the iterator implementation.
|
|
current_index: u16,
|
|
}
|
|
|
|
type ErasureRoot = Hash;
|
|
pub struct Proof(BoundedVec<Hash, ConstU32<16>>);
|
|
|
|
/// A chunk of erasure-encoded block data.
|
|
pub struct ErasureChunk {
|
|
/// The erasure-encoded chunk of data belonging to the candidate block.
|
|
pub chunk: Vec<u8>,
|
|
/// The index of this erasure-encoded chunk of data.
|
|
pub index: u16,
|
|
/// Proof for this chunk against an erasure root.
|
|
pub proof: Proof,
|
|
}
|
|
|
|
impl Iterator for MerklizedChunks {
|
|
type Item = ErasureChunk;
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
let chunk = self.data.pop_front()?;
|
|
let d = self.tree.len() - 1;
|
|
let idx = self.current_index.0;
|
|
let mut index = idx as usize;
|
|
let mut path = Vec::with_capacity(d);
|
|
for i in 0..d {
|
|
let layer = &self.tree[i];
|
|
if index % 2 == 0 {
|
|
path.push(layer[index + 1]);
|
|
} else {
|
|
path.push(layer[index - 1]);
|
|
}
|
|
index /= 2;
|
|
}
|
|
self.current_index += 1;
|
|
Some(ErasureChunk {
|
|
chunk,
|
|
proof: Proof::try_from(path).expect("the path is limited by tree depth; qed"),
|
|
index: idx,
|
|
})
|
|
}
|
|
}
|
|
|
|
impl MerklizedChunks {
|
|
/// Compute `MerklizedChunks` from a list of erasure chunks.
|
|
pub fn compute(chunks: Vec<Vec<u8>>) -> Self {
|
|
let mut hashes: Vec<Hash> = chunks
|
|
.iter()
|
|
.map(|chunk| {
|
|
let hash = hash_fn(chunk);
|
|
Hash::from(hash)
|
|
})
|
|
.collect();
|
|
hashes.resize(chunks.len().next_power_of_two(), Hash::default());
|
|
|
|
let depth = hashes.len().ilog2() as usize + 1;
|
|
let mut tree = vec![Vec::new(); depth];
|
|
tree[0] = hashes;
|
|
|
|
// Build the tree bottom-up.
|
|
(1..depth).for_each(|lvl| {
|
|
let len = 2usize.pow((depth - 1 - lvl) as u32);
|
|
tree[lvl].resize(len, Hash::default());
|
|
|
|
// NOTE: This can be parallelized.
|
|
(0..len).for_each(|i| {
|
|
let prev = &tree[lvl - 1];
|
|
|
|
let hash = combine(prev[2 * i], prev[2 * i + 1]);
|
|
|
|
tree[lvl][i] = hash;
|
|
});
|
|
});
|
|
|
|
assert!(tree[tree.len() - 1].len() == 1, "root must be a single hash");
|
|
|
|
Self {
|
|
root: ErasureRoot::from(tree[tree.len() - 1][0]),
|
|
data: chunks.into(),
|
|
tree,
|
|
current_index: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
fn combine(left: Hash, right: Hash) -> Hash {
|
|
let mut hasher = Hasher::new();
|
|
|
|
hasher.update(left.0.as_slice());
|
|
hasher.update(right.0.as_slice());
|
|
|
|
hasher.finalize().into()
|
|
}
|
|
|
|
impl ErasureChunk {
|
|
/// Verify the proof of the chunk against the erasure root and index.
|
|
pub fn verify(&self, root: &ErasureRoot) -> bool {
|
|
let leaf_hash = Hash::from(hash_fn(&self.chunk));
|
|
let bits = Bitfield(self.index.0);
|
|
|
|
let root_hash = self.proof.0.iter().fold((leaf_hash, 0), |(acc, i), hash| {
|
|
let (a, b) = if bits.get_bit(i) { (*hash, acc) } else { (acc, *hash) };
|
|
(combine(a, b), i + 1)
|
|
});
|
|
|
|
// check the index doesn't contain more bits than the proof length
|
|
let index_bits = 16 - self.index.0.leading_zeros() as usize;
|
|
index_bits <= self.proof.0.len() && root_hash.0 == root.0
|
|
}
|
|
}
|
|
|
|
struct Bitfield(u16);
|
|
|
|
impl Bitfield {
|
|
/// Get the bit at the given index.
|
|
pub fn get_bit(&self, i: usize) -> bool {
|
|
self.0 & (1u16 << i) != 0
|
|
}
|
|
}
|
|
<span class="boring">}</span></code></pre></pre>
|
|
<h2 id="drawbacks"><a class="header" href="#drawbacks">Drawbacks</a></h2>
|
|
<p>Bundling breaking changes with RFC 47 might reset the progress of updating collators. However, the omni node initiative can alleviate this problem.</p>
|
|
<h2 id="testing-security-and-privacy"><a class="header" href="#testing-security-and-privacy">Testing, Security, and Privacy</a></h2>
|
|
<p>Some testing needs to be done to ensure binary compatibility across implementations in multiple languages.</p>
|
|
<h2 id="performance-and-compatibility"><a class="header" href="#performance-and-compatibility">Performance and Compatibility</a></h2>
|
|
<h3 id="performance"><a class="header" href="#performance">Performance</a></h3>
|
|
<p>According to <a href="https://gist.github.com/ordian/0af2822e20bf905d53410a48dc122fd0">these benchmarks</a>, a proper SIMD implementation of Reed-Solomon is 3-4x faster in encoding and up to 9x faster in full decoding.</p>
|
|
<h3 id="compatibility"><a class="header" href="#compatibility">Compatibility</a></h3>
|
|
<p>This is a breaking change that can be coordinated in the same way as done in RFC 47.</p>
|
|
<h2 id="prior-art-and-references"><a class="header" href="#prior-art-and-references">Prior Art and References</a></h2>
|
|
<p>JAM is utilizing the same optimizations as described in the Graypaper.</p>
|
|
<h2 id="unresolved-questions"><a class="header" href="#unresolved-questions">Unresolved Questions</a></h2>
|
|
<p>None.</p>
|
|
<h2 id="future-directions-and-related-material"><a class="header" href="#future-directions-and-related-material">Future Directions and Related Material</a></h2>
|
|
<p>In the future, ZK proofs could be used to avoid the need to re-encode the data to verify that
|
|
the encoding was done correctly.</p>
|
|
|
|
</main>
|
|
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
|
<!-- Mobile navigation buttons -->
|
|
<a rel="prev" href="../introduction.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../proposed/0112-compress-state-response-message-in-state-sync.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
<div style="clear: both"></div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
|
<a rel="prev" href="../introduction.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next prefetch" href="../proposed/0112-compress-state-response-message-in-state-sync.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
window.playground_copyable = true;
|
|
</script>
|
|
|
|
|
|
<script src="../elasticlunr.min.js"></script>
|
|
<script src="../mark.min.js"></script>
|
|
<script src="../searcher.js"></script>
|
|
|
|
<script src="../clipboard.min.js"></script>
|
|
<script src="../highlight.js"></script>
|
|
<script src="../book.js"></script>
|
|
|
|
<!-- Custom JS scripts -->
|
|
|
|
|
|
</div>
|
|
</body>
|
|
</html>
|