Storage chain fixes + guide (#9504)

* Transaction storage guide and fixes

* Apply suggestions from code review

Co-authored-by: Bruno Škvorc <bruno@skvorc.me>

* Update frame/transaction-storage/README.md

Co-authored-by: Bruno Škvorc <bruno@skvorc.me>

* Extended example

Co-authored-by: Bruno Škvorc <bruno@skvorc.me>

substrate/Cargo.lock

@@ -4243,6 +4243,7 @@ dependencies = [
  "sp-runtime",
  "sp-timestamp",
  "sp-transaction-pool",
+ "sp-transaction-storage-proof",
  "sp-trie",
  "structopt",
  "substrate-browser-utils",

substrate/bin/node/cli/Cargo.toml

@@ -57,6 +57,7 @@ sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" }
 sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" }
 sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" }
 sp-transaction-pool = { version = "4.0.0-dev", path = "../../../primitives/transaction-pool" }
+sp-transaction-storage-proof = { version = "4.0.0-dev", path = "../../../primitives/transaction-storage-proof" }
 
 # client dependencies
 sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" }

substrate/bin/node/cli/src/service.rs

@@ -322,7 +322,13 @@ pub fn new_full_base(
 							slot_duration,
 						);
 
-					Ok((timestamp, slot, uncles))
+					let storage_proof =
+						sp_transaction_storage_proof::registration::new_data_provider(
+							&*client_clone,
+							&parent,
+						)?;
+
+					Ok((timestamp, slot, uncles, storage_proof))
 				}
 			},
 			force_authoring,

substrate/client/network/src/block_request_handler.rs

@@ -333,7 +333,10 @@ impl<B: BlockT> BlockRequestHandler<B> {
 							target: LOG_TARGET,
 							"Missing indexed block data for block request."
 						);
-						break
+						// If the indexed body is missing we still continue returning headers.
+						// Ideally `None` should distinguish a missing body from the empty body,
+						// but the current protobuf based protocol does not allow it.
+						Vec::new()
 					},
 				}
 			} else {

substrate/frame/transaction-storage/README.md

@@ -1,8 +1,82 @@
 # Transaction Storage Pallet
 
 Indexes transactions and manages storage proofs.
-# Transaction Storage Pallet
 
-Indexes transactions and manages storage proofs.
+Allows storing arbitrary data on the chain. Data is automatically removed after `StoragePeriod` blocks, unless the storage is renewed.
+Validators must submit proof of storing a random chunk of data for block `N - StoragePeriod` when producing block `N`.
+
+# Running a chain
+
+The following describes how to set up a new storage chain.
+
+Start with generating a chain spec.
+
+```bash
+cargo run --release -- build-spec --chain=local > sc_init.json
+```
+
+Edit the json chain spec file to customise the chain. The storage chain genesis params are configured in the `transactionStorage` section.
+Note that `storagePeriod` is specified in blocks and changing it also requires code changes at the moment.
+
+Build a raw spec from the init spec.
+
+```bash
+cargo run --release build-spec --chain=sc_init.json --raw > sc.json
+```
+
+Run a few validator nodes.
+
+```bash
+cargo run --release -- --chain=sc.json -d /tmp/alice --storage-chain --keep-blocks=100800 --ipfs-server --validator --alice
+cargo run --release -- --chain=sc.json -d /tmp/bob --storage-chain --keep-blocks=100800 --ipfs-server --validator --bob
+```
+
+`--storage-chain` enables transaction indexing.
+`--keep-blocks=100800` enables block pruning. The value here should be greater or equal than the storage period.
+`--ipfs-server` enables serving stored content over IPFS.
+
+Once the network is started, any other joining nodes need to sync with `--sync=fast`. Regular sync will fail because block pruning removes old blocks. The chain does not keep full block history.
+
+```bash
+cargo run --release -- --chain=sc.json -d /tmp/charlie --storage-chain --keep-blocks=100800 --ipfs-server --validator --charlie --sync=fast
+```
+
+# Making transactions
+
+To store data use the `transactionStorage.store` extrinsic. And IPFS CID can be generated from the Blake2-256 hash of the data.
+
+```js
+const util_crypto = require('@polkadot/util-crypto');
+const keyring_api = require('@polkadot/keyring');
+const polkadot_api = require('@polkadot/api');
+const fs = require('fs');
+const multihash = require('multihashes');
+const CID = require('cids')
+
+const wsProvider = new polkadot_api.WsProvider();
+const api = await polkadot_api.ApiPromise.create({ provider: wsProvider });
+
+const keyring = new keyring_api.Keyring({ type: "sr25519" });
+const alice = keyring.addFromUri("//Alice");
+
+const file = fs.readFileSync('cute_kitten.jpeg');
+const hash = util_crypto.blake2AsU8a(file)
+const encoded_hash = multihash.encode(hash, 'blake2b-256');
+
+const cid = new CID(1, 'blake2b-256', encoded_hash)
+console.log(cid.toString());
+
+const txHash = await api.tx.transactionStorage.store('0x' + file.toString('hex')).signAndSend(alice);
+```
+Data can be queried over IPFS
+
+```bash
+ipfs swarm connect <substrate peer address>
+ipfs block get /ipfs/<CID> > kitten.jpeg
+```
+
+To renew data and prevent it from being disposed after the storage period, use `transactionStorage.renew(block, index)`
+where `block` is the block number of the previous store or renew transction, and index is the index of that transaction in the block.
+
 
 License: Apache-2.0

substrate/primitives/transaction-storage-proof/src/lib.rs

@@ -164,8 +164,9 @@ pub mod registration {
 		}
 
 		let proof = match client.block_indexed_body(number)? {
-			Some(transactions) => Some(build_proof(parent.as_ref(), transactions)?),
+			Some(transactions) if !transactions.is_empty() =>
-			None => {
+				Some(build_proof(parent.as_ref(), transactions)?),
+			Some(_) | None => {
 				// Nothing was indexed in that block.
 				None
 			},