Offchain execution extensions (#4145)

* Pass Extensions instead of individual objects.

* Move TransactionPool to a separate ExternalitiesExtension.

* Fix compilation.?

* Clean up.

* Refactor testing utilities.

* Add docs, fix tests.

* Fix doctest.

* Fix formatting and add some logs.

* Add some docs.

* Remove unused files.

substrate/primitives/core/src/offchain/mod.rs

@@ -0,0 +1,738 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Offchain workers types
+
+use codec::{Encode, Decode};
+use rstd::{prelude::{Vec, Box}, convert::TryFrom};
+use crate::RuntimeDebug;
+use runtime_interface::pass_by::{PassByCodec, PassByInner, PassByEnum};
+
+pub use crate::crypto::KeyTypeId;
+
+#[cfg(feature = "std")]
+pub mod storage;
+#[cfg(feature = "std")]
+pub mod testing;
+
+/// Offchain workers local storage.
+pub trait OffchainStorage: Clone + Send + Sync {
+	/// Persist a value in storage under given key and prefix.
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]);
+
+	/// Retrieve a value from storage under given key and prefix.
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Replace the value in storage if given old_value matches the current one.
+	///
+	/// Returns `true` if the value has been set and false otherwise.
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool;
+}
+
+/// A type of supported crypto.
+#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, PassByEnum)]
+#[repr(C)]
+pub enum StorageKind {
+	/// Persistent storage is non-revertible and not fork-aware. It means that any value
+	/// set by the offchain worker triggered at block `N(hash1)` is persisted even
+	/// if that block is reverted as non-canonical and is available for the worker
+	/// that is re-run at block `N(hash2)`.
+	/// This storage can be used by offchain workers to handle forks
+	/// and coordinate offchain workers running on different forks.
+	PERSISTENT = 1,
+	/// Local storage is revertible and fork-aware. It means that any value
+	/// set by the offchain worker triggered at block `N(hash1)` is reverted
+	/// if that block is reverted as non-canonical and is NOT available for the worker
+	/// that is re-run at block `N(hash2)`.
+	LOCAL = 2,
+}
+
+impl TryFrom<u32> for StorageKind {
+	type Error = ();
+
+	fn try_from(kind: u32) -> Result<Self, Self::Error> {
+		match kind {
+			e if e == u32::from(StorageKind::PERSISTENT as u8) => Ok(StorageKind::PERSISTENT),
+			e if e == u32::from(StorageKind::LOCAL as u8) => Ok(StorageKind::LOCAL),
+			_ => Err(()),
+		}
+	}
+}
+
+impl From<StorageKind> for u32 {
+	fn from(c: StorageKind) -> Self {
+		c as u8 as u32
+	}
+}
+
+/// Opaque type for offchain http requests.
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, Encode, Decode, PassByInner)]
+#[cfg_attr(feature = "std", derive(Hash))]
+pub struct HttpRequestId(pub u16);
+
+impl From<HttpRequestId> for u32 {
+	fn from(c: HttpRequestId) -> Self {
+		c.0 as u32
+	}
+}
+
+/// An error enum returned by some http methods.
+#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByEnum)]
+#[repr(C)]
+pub enum HttpError {
+	/// The requested action couldn't been completed within a deadline.
+	DeadlineReached = 1,
+	/// There was an IO Error while processing the request.
+	IoError = 2,
+	/// The ID of the request is invalid in this context.
+	Invalid = 3,
+}
+
+impl TryFrom<u32> for HttpError {
+	type Error = ();
+
+	fn try_from(error: u32) -> Result<Self, Self::Error> {
+		match error {
+			e if e == HttpError::DeadlineReached as u8 as u32 => Ok(HttpError::DeadlineReached),
+			e if e == HttpError::IoError as u8 as u32 => Ok(HttpError::IoError),
+			e if e == HttpError::Invalid as u8 as u32 => Ok(HttpError::Invalid),
+			_ => Err(())
+		}
+	}
+}
+
+impl From<HttpError> for u32 {
+	fn from(c: HttpError) -> Self {
+		c as u8 as u32
+	}
+}
+
+/// Status of the HTTP request
+#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByCodec)]
+pub enum HttpRequestStatus {
+	/// Deadline was reached while we waited for this request to finish.
+	///
+	/// Note the deadline is controlled by the calling part, it not necessarily
+	/// means that the request has timed out.
+	DeadlineReached,
+	/// An error has occured during the request, for example a timeout or the
+	/// remote has closed our socket.
+	///
+	/// The request is now considered destroyed. To retry the request you need
+	/// to construct it again.
+	IoError,
+	/// The passed ID is invalid in this context.
+	Invalid,
+	/// The request has finished with given status code.
+	Finished(u16),
+}
+
+impl From<HttpRequestStatus> for u32 {
+	fn from(status: HttpRequestStatus) -> Self {
+		match status {
+			HttpRequestStatus::Invalid => 0,
+			HttpRequestStatus::DeadlineReached => 10,
+			HttpRequestStatus::IoError => 20,
+			HttpRequestStatus::Finished(code) => u32::from(code),
+		}
+	}
+}
+
+impl TryFrom<u32> for HttpRequestStatus {
+	type Error = ();
+
+	fn try_from(status: u32) -> Result<Self, Self::Error> {
+		match status {
+			0 => Ok(HttpRequestStatus::Invalid),
+			10 => Ok(HttpRequestStatus::DeadlineReached),
+			20 => Ok(HttpRequestStatus::IoError),
+			100..=999 => u16::try_from(status).map(HttpRequestStatus::Finished).map_err(|_| ()),
+			_ => Err(()),
+		}
+	}
+}
+
+/// A blob to hold information about the local node's network state
+/// without committing to its format.
+#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByCodec)]
+pub struct OpaqueNetworkState {
+	/// PeerId of the local node.
+	pub peer_id: OpaquePeerId,
+	/// List of addresses the node knows it can be reached as.
+	pub external_addresses: Vec<OpaqueMultiaddr>,
+}
+
+/// Simple blob to hold a `PeerId` without committing to its format.
+#[derive(Default, Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
+pub struct OpaquePeerId(pub Vec<u8>);
+
+impl OpaquePeerId {
+	/// Create new `OpaquePeerId`
+	pub fn new(vec: Vec<u8>) -> Self {
+		OpaquePeerId(vec)
+	}
+}
+
+/// Simple blob to hold a `Multiaddr` without committing to its format.
+#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
+pub struct OpaqueMultiaddr(pub Vec<u8>);
+
+impl OpaqueMultiaddr {
+	/// Create new `OpaqueMultiaddr`
+	pub fn new(vec: Vec<u8>) -> Self {
+		OpaqueMultiaddr(vec)
+	}
+}
+
+/// Opaque timestamp type
+#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)]
+pub struct Timestamp(u64);
+
+/// Duration type
+#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)]
+pub struct Duration(u64);
+
+impl Duration {
+	/// Create new duration representing given number of milliseconds.
+	pub fn from_millis(millis: u64) -> Self {
+		Duration(millis)
+	}
+
+	/// Returns number of milliseconds this Duration represents.
+	pub fn millis(&self) -> u64 {
+		self.0
+	}
+}
+
+impl Timestamp {
+	/// Creates new `Timestamp` given unix timestamp in miliseconds.
+	pub fn from_unix_millis(millis: u64) -> Self {
+		Timestamp(millis)
+	}
+
+	/// Increase the timestamp by given `Duration`.
+	pub fn add(&self, duration: Duration) -> Timestamp {
+		Timestamp(self.0.saturating_add(duration.0))
+	}
+
+	/// Decrease the timestamp by given `Duration`
+	pub fn sub(&self, duration: Duration) -> Timestamp {
+		Timestamp(self.0.saturating_sub(duration.0))
+	}
+
+	/// Returns a saturated difference (Duration) between two Timestamps.
+	pub fn diff(&self, other: &Self) -> Duration {
+		Duration(self.0.saturating_sub(other.0))
+	}
+
+	/// Return number of milliseconds since UNIX epoch.
+	pub fn unix_millis(&self) -> u64 {
+		self.0
+	}
+}
+
+/// Execution context extra capabilities.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[repr(u8)]
+pub enum Capability {
+	/// Access to transaction pool.
+	TransactionPool = 1,
+	/// External http calls.
+	Http = 2,
+	/// Keystore access.
+	Keystore = 4,
+	/// Randomness source.
+	Randomness = 8,
+	/// Access to opaque network state.
+	NetworkState = 16,
+	/// Access to offchain worker DB (read only).
+	OffchainWorkerDbRead = 32,
+	/// Access to offchain worker DB (writes).
+	OffchainWorkerDbWrite = 64,
+}
+
+/// A set of capabilities
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub struct Capabilities(u8);
+
+impl Capabilities {
+	/// Return an object representing an empty set of capabilities.
+	pub fn none() -> Self {
+		Self(0)
+	}
+
+	/// Return an object representing all capabilities enabled.
+	pub fn all() -> Self {
+		Self(u8::max_value())
+	}
+
+	/// Return capabilities for rich offchain calls.
+	///
+	/// Those calls should be allowed to sign and submit transactions
+	/// and access offchain workers database (but read only!).
+	pub fn rich_offchain_call() -> Self {
+		[
+			Capability::TransactionPool,
+			Capability::Keystore,
+			Capability::OffchainWorkerDbRead,
+		][..].into()
+	}
+
+	/// Check if particular capability is enabled.
+	pub fn has(&self, capability: Capability) -> bool {
+		self.0 & capability as u8 != 0
+	}
+
+	/// Check if this capability object represents all capabilities.
+	pub fn has_all(&self) -> bool {
+		self == &Capabilities::all()
+	}
+}
+
+impl<'a> From<&'a [Capability]> for Capabilities {
+	fn from(list: &'a [Capability]) -> Self {
+		Capabilities(list.iter().fold(0_u8, |a, b| a | *b as u8))
+	}
+}
+
+/// An extended externalities for offchain workers.
+pub trait Externalities: Send {
+	/// Returns if the local node is a potential validator.
+	///
+	/// Even if this function returns `true`, it does not mean that any keys are configured
+	/// and that the validator is registered in the chain.
+	fn is_validator(&self) -> bool;
+
+	/// Returns information about the local node's network state.
+	fn network_state(&self) -> Result<OpaqueNetworkState, ()>;
+
+	/// Returns current UNIX timestamp (in millis)
+	fn timestamp(&mut self) -> Timestamp;
+
+	/// Pause the execution until `deadline` is reached.
+	fn sleep_until(&mut self, deadline: Timestamp);
+
+	/// Returns a random seed.
+	///
+	/// This is a trully random non deterministic seed generated by host environment.
+	/// Obviously fine in the off-chain worker context.
+	fn random_seed(&mut self) -> [u8; 32];
+
+	/// Sets a value in the local storage.
+	///
+	/// Note this storage is not part of the consensus, it's only accessible by
+	/// offchain worker tasks running on the same machine. It IS persisted between runs.
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]);
+
+	/// Sets a value in the local storage if it matches current value.
+	///
+	/// Since multiple offchain workers may be running concurrently, to prevent
+	/// data races use CAS to coordinate between them.
+	///
+	/// Returns `true` if the value has been set, `false` otherwise.
+	///
+	/// Note this storage is not part of the consensus, it's only accessible by
+	/// offchain worker tasks running on the same machine. It IS persisted between runs.
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool;
+
+	/// Gets a value from the local storage.
+	///
+	/// If the value does not exist in the storage `None` will be returned.
+	/// Note this storage is not part of the consensus, it's only accessible by
+	/// offchain worker tasks running on the same machine. It IS persisted between runs.
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Initiates a http request given HTTP verb and the URL.
+	///
+	/// Meta is a future-reserved field containing additional, parity-scale-codec encoded parameters.
+	/// Returns the id of newly started request.
+	///
+	/// Returns an error if:
+	/// - No new request identifier could be allocated.
+	/// - The method or URI contain invalid characters.
+	///
+	fn http_request_start(
+		&mut self,
+		method: &str,
+		uri: &str,
+		meta: &[u8]
+	) -> Result<HttpRequestId, ()>;
+
+	/// Append header to the request.
+	///
+	/// Calling this function multiple times with the same header name continues appending new
+	/// headers. In other words, headers are never replaced.
+	///
+	/// Returns an error if:
+	/// - The request identifier is invalid.
+	/// - You have called `http_request_write_body` on that request.
+	/// - The name or value contain invalid characters.
+	///
+	/// An error doesn't poison the request, and you can continue as if the call had never been
+	/// made.
+	///
+	fn http_request_add_header(
+		&mut self,
+		request_id: HttpRequestId,
+		name: &str,
+		value: &str
+	) -> Result<(), ()>;
+
+	/// Write a chunk of request body.
+	///
+	/// Calling this function with a non-empty slice may or may not start the
+	/// HTTP request. Calling this function with an empty chunks finalizes the
+	/// request and always starts it. It is no longer valid to write more data
+	/// afterwards.
+	/// Passing `None` as deadline blocks forever.
+	///
+	/// Returns an error if:
+	/// - The request identifier is invalid.
+	/// - `http_response_wait` has already been called on this request.
+	/// - The deadline is reached.
+	/// - An I/O error has happened, for example the remote has closed our
+	///   request. The request is then considered invalid.
+	///
+	fn http_request_write_body(
+		&mut self,
+		request_id: HttpRequestId,
+		chunk: &[u8],
+		deadline: Option<Timestamp>
+	) -> Result<(), HttpError>;
+
+	/// Block and wait for the responses for given requests.
+	///
+	/// Returns a vector of request statuses (the len is the same as ids).
+	/// Note that if deadline is not provided the method will block indefinitely,
+	/// otherwise unready responses will produce `DeadlineReached` status.
+	///
+	/// If a response returns an `IoError`, it is then considered destroyed.
+	/// Its id is then invalid.
+	///
+	/// Passing `None` as deadline blocks forever.
+	fn http_response_wait(
+		&mut self,
+		ids: &[HttpRequestId],
+		deadline: Option<Timestamp>
+	) -> Vec<HttpRequestStatus>;
+
+	/// Read all response headers.
+	///
+	/// Returns a vector of pairs `(HeaderKey, HeaderValue)`.
+	///
+	/// Dispatches the request if it hasn't been done yet. It is no longer
+	/// valid to modify the headers or write data to the request.
+	///
+	/// Returns an empty list if the identifier is unknown/invalid, hasn't
+	/// received a response, or has finished.
+	fn http_response_headers(
+		&mut self,
+		request_id: HttpRequestId
+	) -> Vec<(Vec<u8>, Vec<u8>)>;
+
+	/// Read a chunk of body response to given buffer.
+	///
+	/// Dispatches the request if it hasn't been done yet. It is no longer
+	/// valid to modify the headers or write data to the request.
+	///
+	/// Returns the number of bytes written or an error in case a deadline
+	/// is reached or server closed the connection.
+	/// Passing `None` as a deadline blocks forever.
+	///
+	/// If `Ok(0)` or `Err(IoError)` is returned, the request is considered
+	/// destroyed. Doing another read or getting the response's headers, for
+	/// example, is then invalid.
+	///
+	/// Returns an error if:
+	/// - The request identifier is invalid.
+	/// - The deadline is reached.
+	/// - An I/O error has happened, for example the remote has closed our
+	///   request. The request is then considered invalid.
+	///
+	fn http_response_read_body(
+		&mut self,
+		request_id: HttpRequestId,
+		buffer: &mut [u8],
+		deadline: Option<Timestamp>
+	) -> Result<usize, HttpError>;
+
+}
+impl<T: Externalities + ?Sized> Externalities for Box<T> {
+	fn is_validator(&self) -> bool {
+		(& **self).is_validator()
+	}
+
+	fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
+		(& **self).network_state()
+	}
+
+	fn timestamp(&mut self) -> Timestamp {
+		(&mut **self).timestamp()
+	}
+
+	fn sleep_until(&mut self, deadline: Timestamp) {
+		(&mut **self).sleep_until(deadline)
+	}
+
+	fn random_seed(&mut self) -> [u8; 32] {
+		(&mut **self).random_seed()
+	}
+
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
+		(&mut **self).local_storage_set(kind, key, value)
+	}
+
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		(&mut **self).local_storage_compare_and_set(kind, key, old_value, new_value)
+	}
+
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
+		(&mut **self).local_storage_get(kind, key)
+	}
+
+	fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<HttpRequestId, ()> {
+		(&mut **self).http_request_start(method, uri, meta)
+	}
+
+	fn http_request_add_header(&mut self, request_id: HttpRequestId, name: &str, value: &str) -> Result<(), ()> {
+		(&mut **self).http_request_add_header(request_id, name, value)
+	}
+
+	fn http_request_write_body(
+		&mut self,
+		request_id: HttpRequestId,
+		chunk: &[u8],
+		deadline: Option<Timestamp>
+	) -> Result<(), HttpError> {
+		(&mut **self).http_request_write_body(request_id, chunk, deadline)
+	}
+
+	fn http_response_wait(&mut self, ids: &[HttpRequestId], deadline: Option<Timestamp>) -> Vec<HttpRequestStatus> {
+		(&mut **self).http_response_wait(ids, deadline)
+	}
+
+	fn http_response_headers(&mut self, request_id: HttpRequestId) -> Vec<(Vec<u8>, Vec<u8>)> {
+		(&mut **self).http_response_headers(request_id)
+	}
+
+	fn http_response_read_body(
+		&mut self,
+		request_id: HttpRequestId,
+		buffer: &mut [u8],
+		deadline: Option<Timestamp>
+	) -> Result<usize, HttpError> {
+		(&mut **self).http_response_read_body(request_id, buffer, deadline)
+	}
+}
+/// An `OffchainExternalities` implementation with limited capabilities.
+pub struct LimitedExternalities<T> {
+	capabilities: Capabilities,
+	externalities: T,
+}
+
+impl<T> LimitedExternalities<T> {
+	/// Create new externalities limited to given `capabilities`.
+	pub fn new(capabilities: Capabilities, externalities: T) -> Self {
+		Self {
+			capabilities,
+			externalities,
+		}
+	}
+
+	/// Check if given capability is allowed.
+	///
+	/// Panics in case it is not.
+	fn check(&self, capability: Capability, name: &'static str) {
+		if !self.capabilities.has(capability) {
+			panic!("Accessing a forbidden API: {}. No: {:?} capability.", name, capability);
+		}
+	}
+}
+
+impl<T: Externalities> Externalities for LimitedExternalities<T> {
+	fn is_validator(&self) -> bool {
+		self.check(Capability::Keystore, "is_validator");
+		self.externalities.is_validator()
+	}
+
+	fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
+		self.check(Capability::NetworkState, "network_state");
+		self.externalities.network_state()
+	}
+
+	fn timestamp(&mut self) -> Timestamp {
+		self.check(Capability::Http, "timestamp");
+		self.externalities.timestamp()
+	}
+
+	fn sleep_until(&mut self, deadline: Timestamp) {
+		self.check(Capability::Http, "sleep_until");
+		self.externalities.sleep_until(deadline)
+	}
+
+	fn random_seed(&mut self) -> [u8; 32] {
+		self.check(Capability::Randomness, "random_seed");
+		self.externalities.random_seed()
+	}
+
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
+		self.check(Capability::OffchainWorkerDbWrite, "local_storage_set");
+		self.externalities.local_storage_set(kind, key, value)
+	}
+
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		self.check(Capability::OffchainWorkerDbWrite, "local_storage_compare_and_set");
+		self.externalities.local_storage_compare_and_set(kind, key, old_value, new_value)
+	}
+
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
+		self.check(Capability::OffchainWorkerDbRead, "local_storage_get");
+		self.externalities.local_storage_get(kind, key)
+	}
+
+	fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<HttpRequestId, ()> {
+		self.check(Capability::Http, "http_request_start");
+		self.externalities.http_request_start(method, uri, meta)
+	}
+
+	fn http_request_add_header(&mut self, request_id: HttpRequestId, name: &str, value: &str) -> Result<(), ()> {
+		self.check(Capability::Http, "http_request_add_header");
+		self.externalities.http_request_add_header(request_id, name, value)
+	}
+
+	fn http_request_write_body(
+		&mut self,
+		request_id: HttpRequestId,
+		chunk: &[u8],
+		deadline: Option<Timestamp>
+	) -> Result<(), HttpError> {
+		self.check(Capability::Http, "http_request_write_body");
+		self.externalities.http_request_write_body(request_id, chunk, deadline)
+	}
+
+	fn http_response_wait(&mut self, ids: &[HttpRequestId], deadline: Option<Timestamp>) -> Vec<HttpRequestStatus> {
+		self.check(Capability::Http, "http_response_wait");
+		self.externalities.http_response_wait(ids, deadline)
+	}
+
+	fn http_response_headers(&mut self, request_id: HttpRequestId) -> Vec<(Vec<u8>, Vec<u8>)> {
+		self.check(Capability::Http, "http_response_headers");
+		self.externalities.http_response_headers(request_id)
+	}
+
+	fn http_response_read_body(
+		&mut self,
+		request_id: HttpRequestId,
+		buffer: &mut [u8],
+		deadline: Option<Timestamp>
+	) -> Result<usize, HttpError> {
+		self.check(Capability::Http, "http_response_read_body");
+		self.externalities.http_response_read_body(request_id, buffer, deadline)
+	}
+}
+
+#[cfg(feature = "std")]
+externalities::decl_extension! {
+	/// The offchain extension that will be registered at the Substrate externalities.
+	pub struct OffchainExt(Box<dyn Externalities>);
+}
+
+#[cfg(feature = "std")]
+impl OffchainExt {
+	/// Create a new instance of `Self`.
+	pub fn new<O: Externalities + 'static>(offchain: O) -> Self {
+		Self(Box::new(offchain))
+	}
+}
+
+/// Abstraction over transaction pool.
+///
+/// This trait is currently used within the `ExternalitiesExtension`
+/// to provide offchain calls with access to the transaction pool without
+/// tight coupling with any pool implementation.
+#[cfg(feature = "std")]
+pub trait TransactionPool {
+	/// Submit transaction.
+	///
+	/// The transaction will end up in the pool and be propagated to others.
+	fn submit_transaction(&mut self, extrinsic: Vec<u8>) -> Result<(), ()>;
+}
+
+#[cfg(feature = "std")]
+externalities::decl_extension! {
+	/// An externalities extension to submit transactions to the pool.
+	pub struct TransactionPoolExt(Box<dyn TransactionPool + Send>);
+}
+
+#[cfg(feature = "std")]
+impl TransactionPoolExt {
+	/// Create a new instance of `TransactionPoolExt`.
+	pub fn new<O: TransactionPool + Send + 'static>(pool: O) -> Self {
+		Self(Box::new(pool))
+	}
+}
+
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn timestamp_ops() {
+		let t = Timestamp(5);
+		assert_eq!(t.add(Duration::from_millis(10)), Timestamp(15));
+		assert_eq!(t.sub(Duration::from_millis(10)), Timestamp(0));
+		assert_eq!(t.diff(&Timestamp(3)), Duration(2));
+	}
+
+	#[test]
+	fn capabilities() {
+		let none = Capabilities::none();
+		let all = Capabilities::all();
+		let some = Capabilities::from(&[Capability::Keystore, Capability::Randomness][..]);
+
+		assert!(!none.has(Capability::Keystore));
+		assert!(all.has(Capability::Keystore));
+		assert!(some.has(Capability::Keystore));
+		assert!(!none.has(Capability::TransactionPool));
+		assert!(all.has(Capability::TransactionPool));
+		assert!(!some.has(Capability::TransactionPool));
+	}
+}

substrate/primitives/core/src/offchain/storage.rs

@@ -0,0 +1,60 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! In-memory implementation of offchain workers database.
+
+use std::collections::hash_map::{HashMap, Entry};
+use crate::offchain::OffchainStorage;
+
+/// In-memory storage for offchain workers.
+#[derive(Debug, Clone, Default)]
+pub struct InMemOffchainStorage {
+	storage: HashMap<Vec<u8>, Vec<u8>>,
+}
+
+impl OffchainStorage for InMemOffchainStorage {
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		let key = prefix.iter().chain(key).cloned().collect();
+		self.storage.insert(key, value.to_vec());
+	}
+
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		self.storage.get(&key).cloned()
+	}
+
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		let key = prefix.iter().chain(key).cloned().collect();
+
+		match self.storage.entry(key) {
+			Entry::Vacant(entry) => if old_value.is_none() {
+				entry.insert(new_value.to_vec());
+				true
+			} else { false },
+			Entry::Occupied(ref mut entry) if Some(entry.get().as_slice()) == old_value => {
+				entry.insert(new_value.to_vec());
+				true
+			},
+			_ => false,
+		}
+	}
+}

substrate/primitives/core/src/offchain/testing.rs

@@ -0,0 +1,339 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utilities for offchain calls testing.
+//!
+//! Namely all ExecutionExtensions that allow mocking
+//! the extra APIs.
+
+use std::{
+	collections::BTreeMap,
+	sync::Arc,
+};
+use crate::offchain::{
+	self,
+	storage::InMemOffchainStorage,
+	HttpError,
+	HttpRequestId as RequestId,
+	HttpRequestStatus as RequestStatus,
+	Timestamp,
+	StorageKind,
+	OpaqueNetworkState,
+	TransactionPool,
+	OffchainStorage,
+};
+use parking_lot::RwLock;
+
+/// Pending request.
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct PendingRequest {
+	/// HTTP method
+	pub method: String,
+	/// URI
+	pub uri: String,
+	/// Encoded Metadata
+	pub meta: Vec<u8>,
+	/// Request headers
+	pub headers: Vec<(String, String)>,
+	/// Request body
+	pub body: Vec<u8>,
+	/// Has the request been sent already.
+	pub sent: bool,
+	/// Response body
+	pub response: Option<Vec<u8>>,
+	/// Number of bytes already read from the response body.
+	pub read: usize,
+	/// Response headers
+	pub response_headers: Vec<(String, String)>,
+}
+
+/// Internal state of the externalities.
+///
+/// This can be used in tests to respond or assert stuff about interactions.
+#[derive(Debug, Default)]
+pub struct OffchainState {
+	/// A list of pending requests.
+	pub requests: BTreeMap<RequestId, PendingRequest>,
+	expected_requests: BTreeMap<RequestId, PendingRequest>,
+	/// Persistent local storage
+	pub persistent_storage: InMemOffchainStorage,
+	/// Local storage
+	pub local_storage: InMemOffchainStorage,
+}
+
+impl OffchainState {
+	/// Asserts that pending request has been submitted and fills it's response.
+	pub fn fulfill_pending_request(
+		&mut self,
+		id: u16,
+		expected: PendingRequest,
+		response: impl Into<Vec<u8>>,
+		response_headers: impl IntoIterator<Item=(String, String)>,
+	) {
+		match self.requests.get_mut(&RequestId(id)) {
+			None => {
+				panic!("Missing pending request: {:?}.\n\nAll: {:?}", id, self.requests);
+			}
+			Some(req) => {
+				assert_eq!(
+					*req,
+					expected,
+				);
+				req.response = Some(response.into());
+				req.response_headers = response_headers.into_iter().collect();
+			}
+		}
+	}
+
+	fn fulfill_expected(&mut self, id: u16) {
+		if let Some(mut req) = self.expected_requests.remove(&RequestId(id)) {
+			let response = req.response.take().expect("Response checked while added.");
+			let headers = std::mem::replace(&mut req.response_headers, vec![]);
+			self.fulfill_pending_request(id, req, response, headers);
+		}
+	}
+
+	/// Add expected HTTP request.
+	///
+	/// This method can be used to initialize expected HTTP requests and their responses
+	/// before running the actual code that utilizes them (for instance before calling into runtime).
+	/// Expected request has to be fulfilled before this struct is dropped,
+	/// the `response` and `response_headers` fields will be used to return results to the callers.
+	pub fn expect_request(&mut self, id: u16, expected: PendingRequest) {
+		if expected.response.is_none() {
+			panic!("Expected request needs to have a response.");
+		}
+		self.expected_requests.insert(RequestId(id), expected);
+	}
+}
+
+impl Drop for OffchainState {
+	fn drop(&mut self) {
+		// If we panic! while we are already in a panic, the test dies with an illegal instruction.
+		if !self.expected_requests.is_empty() && !std::thread::panicking() {
+			panic!("Unfulfilled expected requests: {:?}", self.expected_requests);
+		}
+	}
+}
+
+/// Implementation of offchain externalities used for tests.
+#[derive(Clone, Default, Debug)]
+pub struct TestOffchainExt(pub Arc<RwLock<OffchainState>>);
+
+impl TestOffchainExt {
+	/// Create new `TestOffchainExt` and a reference to the internal state.
+	pub fn new() -> (Self, Arc<RwLock<OffchainState>>) {
+		let ext = Self::default();
+		let state = ext.0.clone();
+		(ext, state)
+	}
+}
+
+impl offchain::Externalities for TestOffchainExt {
+	fn is_validator(&self) -> bool {
+		unimplemented!("not needed in tests so far")
+	}
+
+	fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
+		Ok(OpaqueNetworkState {
+			peer_id: Default::default(),
+			external_addresses: vec![],
+		})
+	}
+
+	fn timestamp(&mut self) -> Timestamp {
+		unimplemented!("not needed in tests so far")
+	}
+
+	fn sleep_until(&mut self, _deadline: Timestamp) {
+		unimplemented!("not needed in tests so far")
+	}
+
+	fn random_seed(&mut self) -> [u8; 32] {
+		unimplemented!("not needed in tests so far")
+	}
+
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
+		let mut state = self.0.write();
+		match kind {
+			StorageKind::LOCAL => &mut state.local_storage,
+			StorageKind::PERSISTENT => &mut state.persistent_storage,
+		}.set(b"", key, value);
+	}
+
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8]
+	) -> bool {
+		let mut state = self.0.write();
+		match kind {
+			StorageKind::LOCAL => &mut state.local_storage,
+			StorageKind::PERSISTENT => &mut state.persistent_storage,
+		}.compare_and_set(b"", key, old_value, new_value)
+	}
+
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
+		let state = self.0.read();
+		match kind {
+			StorageKind::LOCAL => &state.local_storage,
+			StorageKind::PERSISTENT => &state.persistent_storage,
+		}.get(b"", key)
+	}
+
+	fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<RequestId, ()> {
+		let mut state = self.0.write();
+		let id = RequestId(state.requests.len() as u16);
+		state.requests.insert(id.clone(), PendingRequest {
+			method: method.into(),
+			uri: uri.into(),
+			meta: meta.into(),
+			..Default::default()
+		});
+		Ok(id)
+	}
+
+	fn http_request_add_header(
+		&mut self,
+		request_id: RequestId,
+		name: &str,
+		value: &str,
+	) -> Result<(), ()> {
+		let mut state = self.0.write();
+		if let Some(req) = state.requests.get_mut(&request_id) {
+			req.headers.push((name.into(), value.into()));
+			Ok(())
+		} else {
+			Err(())
+		}
+	}
+
+	fn http_request_write_body(
+		&mut self,
+		request_id: RequestId,
+		chunk: &[u8],
+		_deadline: Option<Timestamp>
+	) -> Result<(), HttpError> {
+		let mut state = self.0.write();
+
+		let sent = {
+			let req = state.requests.get_mut(&request_id).ok_or(HttpError::IoError)?;
+			req.body.extend(chunk);
+			if chunk.is_empty() {
+				req.sent = true;
+			}
+			req.sent
+		};
+
+		if sent {
+			state.fulfill_expected(request_id.0);
+		}
+
+		Ok(())
+	}
+
+	fn http_response_wait(
+		&mut self,
+		ids: &[RequestId],
+		_deadline: Option<Timestamp>,
+	) -> Vec<RequestStatus> {
+		let state = self.0.read();
+
+		ids.iter().map(|id| match state.requests.get(id) {
+			Some(req) if req.response.is_none() =>
+				panic!("No `response` provided for request with id: {:?}", id),
+			None => RequestStatus::Invalid,
+			_ => RequestStatus::Finished(200),
+		}).collect()
+	}
+
+	fn http_response_headers(&mut self, request_id: RequestId) -> Vec<(Vec<u8>, Vec<u8>)> {
+		let state = self.0.read();
+		if let Some(req) = state.requests.get(&request_id) {
+			req.response_headers
+				.clone()
+				.into_iter()
+				.map(|(k, v)| (k.into_bytes(), v.into_bytes()))
+				.collect()
+		} else {
+			Default::default()
+		}
+	}
+
+	fn http_response_read_body(
+		&mut self,
+		request_id: RequestId,
+		buffer: &mut [u8],
+		_deadline: Option<Timestamp>
+	) -> Result<usize, HttpError> {
+		let mut state = self.0.write();
+		if let Some(req) = state.requests.get_mut(&request_id) {
+			let response = req.response
+				.as_mut()
+				.expect(&format!("No response provided for request: {:?}", request_id));
+
+			if req.read >= response.len() {
+				// Remove the pending request as per spec.
+				state.requests.remove(&request_id);
+				Ok(0)
+			} else {
+				let read = std::cmp::min(buffer.len(), response[req.read..].len());
+				buffer[0..read].copy_from_slice(&response[req.read..read]);
+				req.read += read;
+				Ok(read)
+			}
+		} else {
+			Err(HttpError::IoError)
+		}
+	}
+}
+
+/// The internal state of the fake transaction pool.
+#[derive(Default)]
+pub struct PoolState {
+	/// A vector of transactions submitted from the runtime.
+	pub transactions: Vec<Vec<u8>>,
+}
+
+/// Implementation of transaction pool used for test.
+///
+/// Note that this implementation does not verify correctness
+/// of sent extrinsics. It's meant to be used in contexts
+/// where an actual runtime is not known.
+///
+/// It's advised to write integration tests that include the
+/// actual transaction pool to make sure the produced
+/// transactions are valid.
+#[derive(Default)]
+pub struct TestTransactionPoolExt(Arc<RwLock<PoolState>>);
+
+impl TestTransactionPoolExt {
+	/// Create new `TestTransactionPoolExt` and a reference to the internal state.
+	pub fn new() -> (Self, Arc<RwLock<PoolState>>) {
+		let ext = Self::default();
+		let state = ext.0.clone();
+		(ext, state)
+	}
+}
+
+impl TransactionPool for TestTransactionPoolExt {
+	fn submit_transaction(&mut self, extrinsic: Vec<u8>) -> Result<(), ()> {
+		self.0.write().transactions.push(extrinsic);
+		Ok(())
+	}
+}