Persistent Local Storage for offchain workers. (#2894)

* WiP.

* Implement offchain storage APIs.

* Change compare_and_set to return bool.

* Add offchain http test.

* Fix tests.

* Bump spec version.

* Fix warnings and test.

* Fix compilation.

* Remove unused code.

* Introduce Local (fork-aware) and Persistent storage.

* Fix borked merge.

* Prevent warning on depreacated client.backend

* Fix long lines.

* Clean up dependencies.

* Update core/primitives/src/offchain.rs

Co-Authored-By: André Silva <andre.beat@gmail.com>

* Update core/primitives/src/offchain.rs

Co-Authored-By: André Silva <andre.beat@gmail.com>

substrate/core/client/db/src/lib.rs

@@ -24,7 +24,10 @@
 //!
 //! Finality implies canonicality but not vice-versa.
 
+#![warn(missing_docs)]
+
 pub mod light;
+pub mod offchain;
 
 mod cache;
 mod storage_cache;
@@ -77,6 +80,10 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10);
 /// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
 pub type DbState = state_machine::TrieBackend<Arc<dyn state_machine::Storage<Blake2Hasher>>, Blake2Hasher>;
 
+/// A reference tracking state.
+///
+/// It makes sure that the hash we are using stays pinned in storage
+/// until this structure is dropped.
 pub struct RefTrackingState<Block: BlockT> {
 	state: DbState,
 	storage: Arc<StorageDb<Block>>,
@@ -202,7 +209,7 @@ pub fn new_client<E, S, Block, RA>(
 	Ok(client::Client::new(backend, executor, genesis_storage, execution_strategies)?)
 }
 
-mod columns {
+pub(crate) mod columns {
 	pub const META: Option<u32> = crate::utils::COLUMN_META;
 	pub const STATE: Option<u32> = Some(1);
 	pub const STATE_META: Option<u32> = Some(2);
@@ -213,6 +220,8 @@ mod columns {
 	pub const JUSTIFICATION: Option<u32> = Some(6);
 	pub const CHANGES_TRIE: Option<u32> = Some(7);
 	pub const AUX: Option<u32> = Some(8);
+	/// Offchain workers local storage
+	pub const OFFCHAIN: Option<u32> = Some(9);
 }
 
 struct PendingBlock<Block: BlockT> {
@@ -531,6 +540,7 @@ impl state_machine::Storage<Blake2Hasher> for DbGenesisStorage {
 	}
 }
 
+/// A database wrapper for changes tries.
 pub struct DbChangesTrieStorage<Block: BlockT> {
 	db: Arc<dyn KeyValueDB>,
 	meta: Arc<RwLock<Meta<NumberFor<Block>, Block::Hash>>>,
@@ -675,6 +685,7 @@ where
 /// Otherwise, trie nodes are kept only from some recent blocks.
 pub struct Backend<Block: BlockT> {
 	storage: Arc<StorageDb<Block>>,
+	offchain_storage: offchain::LocalStorage,
 	changes_tries_storage: DbChangesTrieStorage<Block>,
 	/// None<*> means that the value hasn't been cached yet. Some(*) means that the value (either None or
 	/// Some(*)) has been cached and is valid.
@@ -689,31 +700,29 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 	/// Create a new instance of database backend.
 	///
 	/// The pruning window is how old a block must be before the state is pruned.
-	pub fn new(config: DatabaseSettings, canonicalization_delay: u64) -> Result<Self, client::error::Error> {
+	pub fn new(config: DatabaseSettings, canonicalization_delay: u64) -> client::error::Result<Self> {
 		Self::new_inner(config, canonicalization_delay)
 	}
 
-	#[cfg(feature = "kvdb-rocksdb")]
 	fn new_inner(config: DatabaseSettings, canonicalization_delay: u64) -> Result<Self, client::error::Error> {
+		#[cfg(feature = "kvdb-rocksdb")]
 		let db = crate::utils::open_database(&config, columns::META, "full")?;
-		Backend::from_kvdb(db as Arc<_>, canonicalization_delay, &config)
-	}
-
-	#[cfg(not(feature = "kvdb-rocksdb"))]
-	fn new_inner(config: DatabaseSettings, canonicalization_delay: u64) -> Result<Self, client::error::Error> {
-		log::warn!("Running without the RocksDB feature. The database will NOT be saved.");
-		let db = Arc::new(kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
-		Backend::from_kvdb(db as Arc<_>, canonicalization_delay, &config)
+		#[cfg(not(feature = "kvdb-rocksdb"))]
+		let db = {
+			log::warn!("Running without the RocksDB feature. The database will NOT be saved.");
+			Arc::new(kvdb_memorydb::create(crate::utils::NUM_COLUMNS))
+		};
+		Self::from_kvdb(db as Arc<_>, canonicalization_delay, &config)
 	}
 
+	/// Create new memory-backed client backend for tests.
 	#[cfg(any(test, feature = "test-helpers"))]
 	pub fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self {
-		use utils::NUM_COLUMNS;
-
-		let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS));
+		let db = Arc::new(kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
 		Self::new_test_db(keep_blocks, canonicalization_delay, db as Arc<_>)
 	}
 
+	/// Creates a client backend with test settings.
 	#[cfg(any(test, feature = "test-helpers"))]
 	pub fn new_test_db(keep_blocks: u32, canonicalization_delay: u64, db: Arc<dyn KeyValueDB>) -> Self {
 
@@ -724,7 +733,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 			path: Default::default(),
 			pruning: PruningMode::keep_blocks(keep_blocks),
 		};
-		Backend::from_kvdb(
+		Self::from_kvdb(
 			db,
 			canonicalization_delay,
 			&db_setting,
@@ -745,6 +754,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 			db: db.clone(),
 			state_db,
 		};
+		let offchain_storage = offchain::LocalStorage::new(db.clone());
 		let changes_tries_storage = DbChangesTrieStorage {
 			db,
 			meta,
@@ -754,6 +764,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 
 		Ok(Backend {
 			storage: Arc::new(storage_db),
+			offchain_storage,
 			changes_tries_storage,
 			changes_trie_config: Mutex::new(None),
 			blockchain,
@@ -1232,6 +1243,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 	type Blockchain = BlockchainDb<Block>;
 	type State = CachingState<Blake2Hasher, RefTrackingState<Block>, Block>;
 	type ChangesTrieStorage = DbChangesTrieStorage<Block>;
+	type OffchainStorage = offchain::LocalStorage;
 
 	fn begin_operation(&self) -> Result<Self::BlockImportOperation, client::error::Error> {
 		let old_state = self.state_at(BlockId::Hash(Default::default()))?;
@@ -1305,6 +1317,10 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 		Some(&self.changes_tries_storage)
 	}
 
+	fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
+		Some(self.offchain_storage.clone())
+	}
+
 	fn revert(&self, n: NumberFor<Block>) -> Result<NumberFor<Block>, client::error::Error> {
 		let mut best = self.blockchain.info().best_number;
 		let finalized = self.blockchain.info().finalized_number;

substrate/core/client/db/src/light.rs

@@ -84,6 +84,7 @@ impl<Block> LightStorage<Block>
 		Self::from_kvdb(db as Arc<_>)
 	}
 
+	/// Create new memory-backed `LightStorage` for tests.
 	#[cfg(any(test, feature = "test-helpers"))]
 	pub fn new_test() -> Self {
 		use utils::NUM_COLUMNS;

substrate/core/client/db/src/offchain.rs

@@ -0,0 +1,137 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! RocksDB-based offchain workers local storage.
+
+use std::{
+	collections::HashMap,
+	sync::Arc,
+};
+
+use crate::columns;
+use kvdb::KeyValueDB;
+use parking_lot::Mutex;
+
+/// Offchain local storage
+#[derive(Clone)]
+pub struct LocalStorage {
+	db: Arc<KeyValueDB>,
+	locks: Arc<Mutex<HashMap<Vec<u8>, Arc<Mutex<()>>>>>,
+}
+
+impl std::fmt::Debug for LocalStorage {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		fmt.debug_struct("LocalStorage")
+			.finish()
+	}
+}
+
+impl LocalStorage {
+	/// Create new offchain storage for tests (backed by memorydb)
+	#[cfg(any(test, feature = "test-helpers"))]
+	pub fn new_test() -> Self {
+		let db = Arc::new(::kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
+		Self::new(db as _)
+	}
+
+	/// Create offchain local storage with given `KeyValueDB` backend.
+	pub fn new(db: Arc<KeyValueDB>) -> Self {
+		Self {
+			db,
+			locks: Default::default(),
+		}
+	}
+}
+
+impl client::backend::OffchainStorage for LocalStorage {
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		let mut tx = self.db.transaction();
+		tx.put(columns::OFFCHAIN, &key, value);
+
+		if let Err(e) = self.db.write(tx) {
+			log::warn!("Error writing to the offchain DB: {:?}", e);
+		}
+	}
+
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		self.db.get(columns::OFFCHAIN, &key)
+			.ok()
+			.and_then(|x| x)
+			.map(|v| v.to_vec())
+	}
+
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		item_key: &[u8],
+		old_value: &[u8],
+		new_value: &[u8],
+	) -> bool {
+		let key: Vec<u8> = prefix.iter().chain(item_key).cloned().collect();
+		let key_lock = {
+			let mut locks = self.locks.lock();
+			locks.entry(key.clone()).or_default().clone()
+		};
+
+		let is_set;
+		{
+			let _key_guard = key_lock.lock();
+			is_set = self.db.get(columns::OFFCHAIN, &key)
+				.ok()
+				.and_then(|x| x)
+				.map(|v| &*v == old_value)
+				.unwrap_or(true);
+
+			if is_set {
+				self.set(prefix, item_key, new_value)
+			}
+		}
+
+		// clean the lock map if we're the only entry
+		let mut locks = self.locks.lock();
+		{
+			drop(key_lock);
+			let key_lock = locks.get_mut(&key);
+			if let Some(_) = key_lock.and_then(Arc::get_mut) {
+				locks.remove(&key);
+			}
+		}
+		is_set
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use client::backend::OffchainStorage;
+
+	#[test]
+	fn should_compare_and_set_and_clear_the_locks_map() {
+		let mut storage = LocalStorage::new_test();
+		let prefix = b"prefix";
+		let key = b"key";
+		let value = b"value";
+
+		storage.set(prefix, key, value);
+		assert_eq!(storage.get(prefix, key), Some(value.to_vec()));
+
+		assert_eq!(storage.compare_and_set(prefix, key, value, b"asd"), true);
+		assert_eq!(storage.get(prefix, key), Some(b"asd".to_vec()));
+		assert!(storage.locks.lock().is_empty(), "Locks map should be empty!");
+	}
+}

substrate/core/client/db/src/utils.rs

@@ -17,7 +17,9 @@
 //! Db-based backend utility structures and functions, used by both
 //! full and light storages.
 
-use std::{io, convert::TryInto, sync::Arc};
+#[cfg(feature = "kvdb-rocksdb")]
+use std::sync::Arc;
+use std::{io, convert::TryInto};
 
 use kvdb::{KeyValueDB, DBTransaction};
 #[cfg(feature = "kvdb-rocksdb")]
@@ -32,11 +34,12 @@ use runtime_primitives::traits::{
 	Block as BlockT, Header as HeaderT, Zero, UniqueSaturatedFrom,
 	UniqueSaturatedInto, CheckedConversion
 };
+#[cfg(feature = "kvdb-rocksdb")]
 use crate::DatabaseSettings;
 
 /// Number of columns in the db. Must be the same for both full && light dbs.
 /// Otherwise RocksDb will fail to open database && check its type.
-pub const NUM_COLUMNS: u32 = 9;
+pub const NUM_COLUMNS: u32 = 10;
 /// Meta column. The set of keys in the column is shared by full && light storages.
 pub const COLUMN_META: Option<u32> = Some(0);