Persistent Local Storage for offchain workers. (#2894)

* WiP.

* Implement offchain storage APIs.

* Change compare_and_set to return bool.

* Add offchain http test.

* Fix tests.

* Bump spec version.

* Fix warnings and test.

* Fix compilation.

* Remove unused code.

* Introduce Local (fork-aware) and Persistent storage.

* Fix borked merge.

* Prevent warning on depreacated client.backend

* Fix long lines.

* Clean up dependencies.

* Update core/primitives/src/offchain.rs

Co-Authored-By: André Silva <andre.beat@gmail.com>

* Update core/primitives/src/offchain.rs

Co-Authored-By: André Silva <andre.beat@gmail.com>

substrate/core/client/src/backend.rs

@@ -139,6 +139,8 @@ pub trait Backend<Block, H>: AuxStore + Send + Sync where
 	type State: StateBackend<H>;
 	/// Changes trie storage.
 	type ChangesTrieStorage: PrunableStateChangesTrieStorage<Block, H>;
+	/// Offchain workers local storage.
+	type OffchainStorage: OffchainStorage;
 
 	/// Begin a new block insertion transaction with given parent block id.
 	/// When constructing the genesis, this is called with all-zero hash.
@@ -156,6 +158,8 @@ pub trait Backend<Block, H>: AuxStore + Send + Sync where
 	fn used_state_cache_size(&self) -> Option<usize>;
 	/// Returns reference to changes trie storage.
 	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;
+	/// Returns a handle to offchain storage.
+	fn offchain_storage(&self) -> Option<Self::OffchainStorage>;
 	/// Returns true if state for given block is available.
 	fn have_state_at(&self, hash: &Block::Hash, _number: NumberFor<Block>) -> bool {
 		self.state_at(BlockId::Hash(hash.clone())).is_ok()
@@ -194,6 +198,26 @@ pub trait Backend<Block, H>: AuxStore + Send + Sync where
 	fn get_import_lock(&self) -> &Mutex<()>;
 }
 
+/// Offchain workers local storage.
+pub trait OffchainStorage: Clone + Send + Sync {
+	/// Persist a value in storage under given key and prefix.
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]);
+
+	/// Retrieve a value from storage under given key and prefix.
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Replace the value in storage if given old_value matches the current one.
+	///
+	/// Returns `true` if the value has been set and false otherwise.
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+		old_value: &[u8],
+		new_value: &[u8],
+	) -> bool;
+}
+
 /// Changes trie storage that supports pruning.
 pub trait PrunableStateChangesTrieStorage<Block: BlockT, H: Hasher>:
 	StateChangesTrieStorage<H, NumberFor<Block>>

substrate/core/client/src/in_mem.rs

@@ -593,6 +593,7 @@ where
 	type Blockchain = Blockchain<Block>;
 	type State = InMemory<H>;
 	type ChangesTrieStorage = ChangesTrieStorage<Block, H>;
+	type OffchainStorage = OffchainStorage;
 
 	fn begin_operation(&self) -> error::Result<Self::BlockImportOperation> {
 		let old_state = self.state_at(BlockId::Hash(Default::default()))?;
@@ -669,6 +670,10 @@ where
 		Some(&self.changes_trie_storage)
 	}
 
+	fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
+		None
+	}
+
 	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State> {
 		match block {
 			BlockId::Hash(h) if h == Default::default() => {
@@ -768,8 +773,46 @@ pub fn check_genesis_storage(top: &StorageOverlay, children: &ChildrenStorageOve
 	Ok(())
 }
 
+/// In-memory storage for offchain workers.
+#[derive(Debug, Clone, Default)]
+pub struct OffchainStorage {
+	storage: HashMap<Vec<u8>, Vec<u8>>,
+}
+
+impl backend::OffchainStorage for OffchainStorage {
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		let key = prefix.iter().chain(key).cloned().collect();
+		self.storage.insert(key, value.to_vec());
+	}
+
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		self.storage.get(&key).cloned()
+	}
+
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+		old_value: &[u8],
+		new_value: &[u8],
+	) -> bool {
+		let key = prefix.iter().chain(key).cloned().collect();
+
+		let mut set = false;
+		self.storage.entry(key).and_modify(|val| {
+			if val.as_slice() == old_value {
+				*val = new_value.to_vec();
+				set = true;
+			}
+		});
+		set
+	}
+}
+
 #[cfg(test)]
 mod tests {
+	use super::*;
 	use std::sync::Arc;
 	use test_client;
 	use primitives::Blake2Hasher;
@@ -789,4 +832,22 @@ mod tests {
 
 		test_client::trait_tests::test_blockchain_query_by_number_gets_canonical(backend);
 	}
+
+	#[test]
+	fn in_memory_offchain_storage() {
+		use crate::backend::OffchainStorage as _;
+
+		let mut storage = OffchainStorage::default();
+		assert_eq!(storage.get(b"A", b"B"), None);
+		assert_eq!(storage.get(b"B", b"A"), None);
+
+		storage.set(b"A", b"B", b"C");
+		assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec()));
+		assert_eq!(storage.get(b"B", b"A"), None);
+
+		storage.compare_and_set(b"A", b"B", b"X", b"D");
+		assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec()));
+		storage.compare_and_set(b"A", b"B", b"C", b"D");
+		assert_eq!(storage.get(b"A", b"B"), Some(b"D".to_vec()));
+	}
 }

substrate/core/client/src/light/backend.rs

@@ -118,6 +118,7 @@ impl<S, F, Block, H> ClientBackend<Block, H> for Backend<S, F, H> where
 	type Blockchain = Blockchain<S, F>;
 	type State = OnDemandOrGenesisState<Block, S, F, H>;
 	type ChangesTrieStorage = in_mem::ChangesTrieStorage<Block, H>;
+	type OffchainStorage = in_mem::OffchainStorage;
 
 	fn begin_operation(&self) -> ClientResult<Self::BlockImportOperation> {
 		Ok(ImportOperation {
@@ -195,6 +196,10 @@ impl<S, F, Block, H> ClientBackend<Block, H> for Backend<S, F, H> where
 		None
 	}
 
+	fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
+		None
+	}
+
 	fn state_at(&self, block: BlockId<Block>) -> ClientResult<Self::State> {
 		let block_number = self.blockchain.expect_block_number_from_id(&block)?;