Offchain Workers: Example Pallet (#4989)

* Example of offchain worker pallet.

* Fix compilation issues.

* Use serde_json to parse JSON.

* Add some basic tests.

* Working on docs.

* Fix compilation

* Finish docs for signed.

* Work on unsigned send.

* Add some tests and missing docs.

* Add example of StorageValueRef

* Add weight.

* Extra \n

* Fix im-online test.

* Bump runtime.

* Fix tests.

* Apply suggestions from code review

Co-Authored-By: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>
Co-Authored-By: Gavin Wood <gavin@parity.io>

* Address review comments.

Co-authored-by: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>
Co-authored-by: Gavin Wood <github@gavwood.com>

substrate/Cargo.lock

@@ -3722,6 +3722,12 @@ version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
 
+[[package]]
+name = "nohash-hasher"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "721a2bf1c26159ebf17e0a980bc4ce61f4b2fec5ec3b42d42fddd7a84a9e538f"
+
 [[package]]
 name = "nohash-hasher"
 version = "0.2.0"
@@ -4149,6 +4155,21 @@ dependencies = [
  "sp-std",
 ]
 
+[[package]]
+name = "pallet-example-offchain-worker"
+version = "2.0.0"
+dependencies = [
+ "frame-support",
+ "frame-system",
+ "parity-scale-codec",
+ "serde",
+ "serde_json",
+ "sp-core",
+ "sp-io",
+ "sp-runtime",
+ "sp-std",
+]
+
 [[package]]
 name = "pallet-finality-tracker"
 version = "2.0.0"
@@ -6215,7 +6236,7 @@ dependencies = [
  "linked_hash_set",
  "log 0.4.8",
  "lru 0.4.3",
- "nohash-hasher",
+ "nohash-hasher 0.2.0",
  "parity-scale-codec",
  "parking_lot 0.10.0",
  "prost",
@@ -9168,14 +9189,14 @@ checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57"
 
 [[package]]
 name = "yamux"
-version = "0.4.3"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d73295bc9d9acf89dd9336b3b5f5b57731ee72b587857dd4312721a0196b48e5"
+checksum = "902f4cee32c401c211b6b69f4a3f6f4cf3515644db5bd822cf685a7dbd6201f9"
 dependencies = [
  "bytes 0.5.4",
  "futures 0.3.4",
  "log 0.4.8",
- "nohash-hasher",
+ "nohash-hasher 0.1.3",
  "parking_lot 0.10.0",
  "rand 0.7.3",
  "thiserror",

substrate/Cargo.toml

@@ -71,6 +71,7 @@ members = [
 	"frame/elections",
 	"frame/evm",
 	"frame/example",
+	"frame/example-offchain-worker",
 	"frame/executive",
 	"frame/finality-tracker",
 	"frame/generic-asset",

substrate/bin/node/runtime/src/lib.rs

@@ -82,7 +82,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	// implementation changes and behavior does not, then leave spec_version as
 	// is and increment impl_version.
 	spec_version: 223,
-	impl_version: 0,
+	impl_version: 1,
 	apis: RUNTIME_API_VERSIONS,
 };
 

substrate/frame/example-offchain-worker/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "pallet-example-offchain-worker"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false }
+frame-support = { version = "2.0.0", default-features = false, path = "../support" }
+frame-system = { version = "2.0.0", default-features = false, path = "../system" }
+serde = { version = "1.0.101", optional = true }
+sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" }
+sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" }
+sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" }
+sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" }
+serde_json = { version = "1.0.46", default-features = false, features = ["alloc"] }
+
+[features]
+default = ["std"]
+std = [
+	"codec/std",
+	"frame-support/std",
+	"frame-system/std",
+	"serde",
+	"sp-core/std",
+	"sp-io/std",
+	"sp-runtime/std",
+	"sp-std/std",
+]

substrate/frame/example-offchain-worker/src/lib.rs

@@ -0,0 +1,548 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! # Offchain Worker Example Module
+//!
+//! The Offchain Worker Example: A simple pallet demonstrating
+//! concepts, APIs and structures common to most offchain workers.
+//!
+//! Run `cargo doc --package pallet-example-offchain-worker --open` to view this module's
+//! documentation.
+//!
+//! - \[`pallet_example_offchain_worker::Trait`](./trait.Trait.html)
+//! - \[`Call`](./enum.Call.html)
+//! - \[`Module`](./struct.Module.html)
+//!
+//!
+//! \## Overview
+//!
+//! In this example we are going to build a very simplistic, naive and definitely NOT
+//! production-ready oracle for BTC/USD price.
+//! Offchain Worker (OCW) will be triggered after every block, fetch the current price
+//! and prepare either signed or unsigned transaction to feed the result back on chain.
+//! The on-chain logic will simply aggregate the results and store last `64` values to compute
+//! the average price.
+//! Additional logic in OCW is put in place to prevent spamming the network with both signed
+//! and unsigned transactions, and custom `UnsignedValidator` makes sure that there is only
+//! one unsigned transaction floating in the network.
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use frame_support::{
+	debug,
+	dispatch::DispatchResult, decl_module, decl_storage, decl_event,
+	traits::Get,
+	weights::SimpleDispatchInfo,
+};
+use frame_system::{self as system, ensure_signed, ensure_none, offchain};
+use serde_json as json;
+use sp_core::crypto::KeyTypeId;
+use sp_runtime::{
+	offchain::{http, Duration, storage::StorageValueRef},
+	traits::Zero,
+	transaction_validity::{InvalidTransaction, ValidTransaction, TransactionValidity},
+};
+
+#[cfg(test)]
+mod tests;
+
+/// Defines application identifier for crypto keys of this module.
+///
+/// Every module that deals with signatures needs to declare its unique identifier for
+/// its crypto keys.
+/// When offchain worker is signing transactions it's going to request keys of type
+/// `KeyTypeId` from the keystore and use the ones it finds to sign the transaction.
+/// The keys can be inserted manually via RPC (see `author_insertKey`).
+pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"btc!");
+
+/// Based on the above `KeyTypeId` we need to generate a pallet-specific crypto type wrappers.
+/// We can use from supported crypto kinds (`sr25519`, `ed25519` and `ecdsa`) and augment
+/// the types with this pallet-specific identifier.
+pub mod crypto {
+	use super::KEY_TYPE;
+	use sp_runtime::app_crypto::{app_crypto, sr25519};
+	app_crypto!(sr25519, KEY_TYPE);
+}
+
+/// This pallet's configuration trait
+pub trait Trait: frame_system::Trait {
+	/// The type to sign and submit transactions.
+	type SubmitSignedTransaction:
+		offchain::SubmitSignedTransaction<Self, <Self as Trait>::Call>;
+	/// The type to submit unsigned transactions.
+	type SubmitUnsignedTransaction:
+		offchain::SubmitUnsignedTransaction<Self, <Self as Trait>::Call>;
+
+	/// The overarching event type.
+	type Event: From<Event<Self>> + Into<<Self as frame_system::Trait>::Event>;
+	/// The overarching dispatch call type.
+	type Call: From<Call<Self>>;
+
+	// Configuration parameters
+
+	/// A grace period after we send transaction.
+	///
+	/// To avoid sending too many transactions, we only attempt to send one
+	/// every `GRACE_PERIOD` blocks. We use Local Storage to coordinate
+	/// sending between distinct runs of this offchain worker.
+	type GracePeriod: Get<Self::BlockNumber>;
+
+	/// Number of blocks of cooldown after unsigned transaction is included.
+	///
+	/// This ensures that we only accept unsigned transactions once, every `UnsignedInterval` blocks.
+	type UnsignedInterval: Get<Self::BlockNumber>;
+}
+
+decl_storage! {
+	trait Store for Module<T: Trait> as Example {
+		/// A vector of recently submitted prices.
+		///
+		/// This is used to calculate average price, should have bounded size.
+		Prices get(fn prices): Vec<u32>;
+		/// Defines the block when next unsigned transaction will be accepted.
+		///
+		/// To prevent spam of unsigned (and unpayed!) transactions on the network,
+		/// we only allow one transaction every `T::UnsignedInterval` blocks.
+		/// This storage entry defines when new transaction is going to be accepted.
+		NextUnsignedAt get(fn next_unsigned_at): T::BlockNumber;
+	}
+}
+
+decl_event!(
+	/// Events generated by the module.
+	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId {
+		/// Event generated when new price is accepted to contribute to the average.
+		NewPrice(u32, AccountId),
+	}
+);
+
+decl_module! {
+	/// A public part of the pallet.
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+		fn deposit_event() = default;
+
+		/// Submit new price to the list.
+		///
+		/// This method is a public function of the module and can be called from within
+		/// a transaction. It appends given `price` to current list of prices.
+		/// In our example the `offchain worker` will create, sign & submit a transaction that
+		/// calls this function passing the price.
+		///
+		/// The transaction needs to be signed (see `ensure_signed`) check, so that the caller
+		/// pays a fee to execute it.
+		/// This makes sure that it's not easy (or rather cheap) to attack the chain by submitting
+		/// excesive transactions, but note that it doesn't ensure the price oracle is actually
+		/// working and receives (and provides) meaningful data.
+		/// This example is not focused on correctness of the oracle itself, but rather its
+		/// purpose is to showcase offchain worker capabilities.
+		#[weight = SimpleDispatchInfo::FixedNormal(10_000)]
+		pub fn submit_price(origin, price: u32) -> DispatchResult {
+			// Retrieve sender of the transaction.
+			let who = ensure_signed(origin)?;
+			// Add the price to the on-chain list.
+			Self::add_price(who, price);
+			Ok(())
+		}
+
+		/// Submit new price to the list via unsigned transaction.
+		///
+		/// Works exactly like the `submit_price` function, but since we allow sending the
+		/// transaction without a signature, and hence without paying any fees,
+		/// we need a way to make sure that only some transactions are accepted.
+		/// This function can be called only once every `T::UnsignedInterval` blocks.
+		/// Transactions that call that function are de-duplicated on the pool level
+		/// via `validate_unsigned` implementation and also are rendered invalid if
+		/// the function has already been called in current "session".
+		///
+		/// It's important to specify `weight` for unsigned calls as well, because even though
+		/// they don't charge fees, we still don't want a single block to contain unlimited
+		/// number of such transactions.
+		///
+		/// This example is not focused on correctness of the oracle itself, but rather its
+		/// purpose is to showcase offchain worker capabilities.
+		#[weight = SimpleDispatchInfo::FixedNormal(10_000)]
+		pub fn submit_price_unsigned(origin, _block_number: T::BlockNumber, price: u32)
+			-> DispatchResult
+		{
+			// This ensures that the function can only be called via unsigned transaction.
+			ensure_none(origin)?;
+			// Add the price to the on-chain list, but mark it as coming from an empty address.
+			Self::add_price(Default::default(), price);
+			// now increment the block number at which we expect next unsigned transaction.
+			let current_block = <system::Module<T>>::block_number();
+			<NextUnsignedAt<T>>::put(current_block + T::UnsignedInterval::get());
+			Ok(())
+		}
+
+		/// Offchain Worker entry point.
+		///
+		/// By implementing `fn offchain_worker` within `decl_module!` you declare a new offchain
+		/// worker.
+		/// This function will be called when the node is fully synced and a new best block is
+		/// succesfuly imported.
+		/// Note that it's not guaranteed for offchain workers to run on EVERY block, there might
+		/// be cases where some blocks are skipped, or for some the worker runs twice (re-orgs),
+		/// so the code should be able to handle that.
+		/// You can use `Local Storage` API to coordinate runs of the worker.
+		fn offchain_worker(block_number: T::BlockNumber) {
+			// It's a good idea to add logs to your offchain workers.
+			// Using the `frame_support::debug` module you have access to the same API exposed by
+			// the `log` crate.
+			// Note that having logs compiled to WASM may cause the size of the blob to increase
+			// significantly. You can use `RuntimeDebug` custom derive to hide details of the types
+			// in WASM or use `debug::native` namespace to produce logs only when the worker is
+			// running natively.
+			debug::native::info!("Hello World from offchain workers!");
+
+			// Since off-chain workers are just part of the runtime code, they have direct access
+			// to the storage and other included pallets.
+			//
+			// We can easily import `frame_system` and retrieve a block hash of the parent block.
+			let parent_hash = <system::Module<T>>::block_hash(block_number - 1.into());
+			debug::debug!("Current block: {:?} (parent hash: {:?})", block_number, parent_hash);
+
+			// It's a good practice to keep `fn offchain_worker()` function minimal, and move most
+			// of the code to separate `impl` block.
+			// Here we call a helper function to calculate current average price.
+			// This function reads storage entries of the current state.
+			let average: Option<u32> = Self::average_price();
+			debug::debug!("Current price: {:?}", average);
+
+			// For this example we are going to send both signed and unsigned transactions
+			// depending on the block number.
+			// Usually it's enough to choose one or the other.
+			let should_send = Self::choose_transaction_type(block_number);
+			let res = match should_send {
+				TransactionType::Signed => Self::fetch_price_and_send_signed(),
+				TransactionType::Unsigned => Self::fetch_price_and_send_unsigned(block_number),
+				TransactionType::None => Ok(()),
+			};
+			if let Err(e) = res {
+				debug::error!("Error: {}", e);
+			}
+		}
+	}
+}
+
+enum TransactionType {
+	Signed,
+	Unsigned,
+	None,
+}
+
+/// Most of the functions are moved outside of the `decl_module!` macro.
+///
+/// This greatly helps with error messages, as the ones inside the macro
+/// can sometimes be hard to debug.
+impl<T: Trait> Module<T> {
+	/// Chooses which transaction type to send.
+	///
+	/// This function serves mostly to showcase `StorageValue` helper
+	/// and local storage usage.
+	///
+	/// Returns a type of transaction that should be produced in current run.
+	fn choose_transaction_type(block_number: T::BlockNumber) -> TransactionType {
+		/// A friendlier name for the error that is going to be returned in case we are in the grace
+		/// period.
+		const RECENTLY_SENT: () = ();
+
+		// Start off by creating a reference to Local Storage value.
+		// Since the local storage is common for all offchain workers, it's a good practice
+		// to prepend your entry with the module name.
+		let val = StorageValueRef::persistent(b"example_ocw::last_send");
+		// The Local Storage is persisted and shared between runs of the offchain workers,
+		// and offchain workers may run concurrently. We can use the `mutate` function, to
+		// write a storage entry in an atomic fashion. Under the hood it uses `compare_and_set`
+		// low-level method of local storage API, which means that only one worker
+		// will be able to "acquire a lock" and send a transaction if multiple workers
+		// happen to be executed concurrently.
+		let res = val.mutate(|last_send: Option<Option<T::BlockNumber>>| {
+			// We match on the value decoded from the storage. The first `Option`
+			// indicates if the value was present in the storage at all,
+			// the second (inner) `Option` indicates if the value was succesfuly
+			// decoded to expected type (`T::BlockNumber` in our case).
+			match last_send {
+				// If we already have a value in storage and the block number is recent enough
+				// we avoid sending another transaction at this time.
+				Some(Some(block)) if block + T::GracePeriod::get() < block_number => {
+					Err(RECENTLY_SENT)
+				},
+				// In every other case we attempt to acquire the lock and send a transaction.
+				_ => Ok(block_number)
+			}
+		});
+
+		// The result of `mutate` call will give us a nested `Result` type.
+		// The first one matches the return of the closure passed to `mutate`, i.e.
+		// if we return `Err` from the closure, we get an `Err` here.
+		// In case we return `Ok`, here we will have another (inner) `Result` that indicates
+		// if the value has been set to the storage correctly - i.e. if it wasn't
+		// written to in the meantime.
+		match res {
+			// The value has been set correctly, which means we can safely send a transaction now.
+			Ok(Ok(block_number)) => {
+				// Depending if the block is even or odd we will send a `Signed` or `Unsigned`
+				// transaction.
+				// Note that this logic doesn't really guarantee that the transactions will be sent
+				// in an alternating fashion (i.e. fairly distributed). Depending on the execution
+				// order and lock acquisition, we may end up for instance sending two `Signed`
+				// transactions in a row. If a strict order is desired, it's better to use
+				// the storage entry for that. (for instance store both block number and a flag
+				// indicating the type of next transaction to send).
+				let send_signed = block_number % 2.into() == Zero::zero();
+				if send_signed {
+					TransactionType::Signed
+				} else {
+					TransactionType::Unsigned
+				}
+			},
+			// We are in the grace period, we should not send a transaction this time.
+			Err(RECENTLY_SENT) => TransactionType::None,
+			// We wanted to send a transaction, but failed to write the block number (acquire a
+			// lock). This indicates that another offchain worker that was running concurrently
+			// most likely executed the same logic and succeeded at writing to storage.
+			// Thus we don't really want to send the transaction, knowing that the other run
+			// already did.
+			Ok(Err(_)) => TransactionType::None,
+		}
+	}
+
+	/// A helper function to fetch the price and send signed transaction.
+	fn fetch_price_and_send_signed() -> Result<(), String> {
+		use system::offchain::SubmitSignedTransaction;
+		// Firstly we check if there are any accounts in the local keystore that are capable of
+		// signing the transaction.
+		// If not it doesn't even make sense to make external HTTP requests, since we won't be able
+		// to put the results back on-chain.
+		if !T::SubmitSignedTransaction::can_sign() {
+			return Err(
+				"No local accounts available. Consider adding one via `author_insertKey` RPC."
+			)?
+		}
+
+		// Make an external HTTP request to fetch the current price.
+		// Note this call will block until response is received.
+		let price = Self::fetch_price().map_err(|e| format!("{:?}", e))?;
+
+		// Received price is wrapped into a call to `submit_price` public function of this pallet.
+		// This means that the transaction, when executed, will simply call that function passing
+		// `price` as an argument.
+		let call = Call::submit_price(price);
+
+		// Using `SubmitSignedTransaction` associated type we create and submit a transaction
+		// representing the call, we've just created.
+		// Submit signed will return a vector of results for all accounts that were found in the
+		// local keystore with expected `KEY_TYPE`.
+		let results = T::SubmitSignedTransaction::submit_signed(call);
+		for (acc, res) in &results {
+			match res {
+				Ok(()) => debug::info!("[{:?}] Submitted price of {} cents", acc, price),
+				Err(e) => debug::error!("[{:?}] Failed to submit transaction: {:?}", acc, e),
+			}
+		}
+
+		Ok(())
+	}
+
+	/// A helper function to fetch the price and send unsigned transaction.
+	fn fetch_price_and_send_unsigned(block_number: T::BlockNumber) -> Result<(), String> {
+		use system::offchain::SubmitUnsignedTransaction;
+		// Make sure we don't fetch the price if unsigned transaction is going to be rejected
+		// anyway.
+		let next_unsigned_at = <NextUnsignedAt<T>>::get();
+		if next_unsigned_at > block_number {
+			return Err(
+				format!("Too early to send unsigned transaction. Next at: {:?}", next_unsigned_at)
+			)?
+		}
+
+		// Make an external HTTP request to fetch the current price.
+		// Note this call will block until response is received.
+		let price = Self::fetch_price().map_err(|e| format!("{:?}", e))?;
+
+		// Received price is wrapped into a call to `submit_price_unsigned` public function of this
+		// pallet. This means that the transaction, when executed, will simply call that function
+		// passing `price` as an argument.
+		let call = Call::submit_price_unsigned(block_number, price);
+
+		// Now let's create an unsigned transaction out of this call and submit it to the pool.
+		// By default unsigned transactions are disallowed, so we need to whitelist this case
+		// by writing `UnsignedValidator`. Note that it's EXTREMELY important to carefuly
+		// implement unsigned validation logic, as any mistakes can lead to opening DoS or spam
+		// attack vectors. See validation logic docs for more details.
+		T::SubmitUnsignedTransaction::submit_unsigned(call)
+			.map_err(|()| "Unable to submit unsigned transaction.".into())
+
+	}
+
+	/// Fetch current price and return the result in cents.
+	fn fetch_price() -> Result<u32, http::Error> {
+		// We want to keep the offchain worker execution time reasonable, so we set a hard-coded
+		// deadline to 2s to complete the external call.
+		// You can also wait idefinitely for the response, however you may still get a timeout
+		// coming from the host machine.
+		let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(2_000));
+		// Initiate an external HTTP GET request.
+		// This is using high-level wrappers from `sp_runtime`, for the low-level calls that
+		// you can find in `sp_io`. The API is trying to be similar to `reqwest`, but
+		// since we are running in a custom WASM execution environment we can't simply
+		// import the library here.
+		let request = http::Request::get(
+			"https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD"
+		);
+		// We set the deadline for sending of the request, note that awaiting response can
+		// have a separate deadline. Next we send the request, before that it's also possible
+		// to alter request headers or stream body content in case of non-GET requests.
+		let pending = request
+			.deadline(deadline)
+			.send()
+			.map_err(|_| http::Error::IoError)?;
+
+		// The request is already being processed by the host, we are free to do anything
+		// else in the worker (we can send multiple concurrent requests too).
+		// At some point however we probably want to check the response though,
+		// so we can block current thread and wait for it to finish.
+		// Note that since the request is being driven by the host, we don't have to wait
+		// for the request to have it complete, we will just not read the response.
+		let response = pending.try_wait(deadline)
+			.map_err(|_| http::Error::DeadlineReached)??;
+		// Let's check the status code before we proceed to reading the response.
+		if response.code != 200 {
+			debug::warn!("Unexpected status code: {}", response.code);
+			return Err(http::Error::Unknown);
+		}
+
+		// Next we want to fully read the response body and collect it to a vector of bytes.
+		// Note that the return object allows you to read the body in chunks as well
+		// with a way to control the deadline.
+		let body = response.body().collect::<Vec<u8>>();
+		// Next we parse the response using `serde_json`. Even though it's possible to use
+		// `serde_derive` and deserialize to a struct it's not recommended due to blob size
+		// overhead introduced by such code. Deserializing to `json::Value` is much more
+		// lightweight and should be preferred, especially if we only care about a small number
+		// of properties from the response.
+		let val: Result<json::Value, _> = json::from_slice(&body);
+		// Let's parse the price as float value. Note that you should avoid using floats in the
+		// runtime, it's fine to do that in the offchain worker, but we do convert it to an integer
+		// before submitting on-chain.
+		let price = val.ok().and_then(|v| v.get("USD").and_then(|v| v.as_f64()));
+		let price = match price {
+			Some(pricef) => Ok((pricef * 100.) as u32),
+			None => {
+				let s = core::str::from_utf8(&body);
+				debug::warn!("Unable to extract price from the response: {:?}", s);
+				Err(http::Error::Unknown)
+			}
+		}?;
+
+		debug::warn!("Got price: {} cents", price);
+
+		Ok(price)
+	}
+
+	/// Add new price to the list.
+	fn add_price(who: T::AccountId, price: u32) {
+		debug::info!("Adding to the average: {}", price);
+		Prices::mutate(|prices| {
+			const MAX_LEN: usize = 64;
+
+			if prices.len() < MAX_LEN {
+				prices.push(price);
+			} else {
+				prices[price as usize % MAX_LEN] = price;
+			}
+		});
+
+		let average = Self::average_price()
+			.expect("The average is not empty, because it was just mutated; qed");
+		debug::info!("Current average price is: {}", average);
+		// here we are raising the NewPrice event
+		Self::deposit_event(RawEvent::NewPrice(price, who));
+	}
+
+	/// Calculate current average price.
+	fn average_price() -> Option<u32> {
+		let prices = Prices::get();
+		if prices.is_empty() {
+			None
+		} else {
+			Some(prices.iter().fold(0_u32, |a, b| a.saturating_add(*b)) / prices.len() as u32)
+		}
+	}
+}
+
+#[allow(deprecated)] // ValidateUnsigned
+impl<T: Trait> frame_support::unsigned::ValidateUnsigned for Module<T> {
+	type Call = Call<T>;
+
+	/// Validate unsigned call to this module.
+	///
+	/// By default unsigned transactions are disallowed, but implementing the validator
+	/// here we make sure that some particular calls (the ones produced by offchain worker)
+	/// are being whitelisted and marked as valid.
+	fn validate_unsigned(call: &Self::Call) -> TransactionValidity {
+		// Firstly let's check that we call the right function.
+		if let Call::submit_price_unsigned(block_number, new_price) = call {
+			// Now let's check if the transaction has any chance to succeed.
+			let next_unsigned_at = <NextUnsignedAt<T>>::get();
+			if &next_unsigned_at > block_number {
+				return InvalidTransaction::Stale.into();
+			}
+			// Let's make sure to reject transactions from the future.
+			let current_block = <system::Module<T>>::block_number();
+			if &current_block < block_number {
+				return InvalidTransaction::Future.into();
+			}
+
+			// We prioritize transactions that are more far away from current average.
+			//
+			// Note this doesn't make much sense when building an actual oracle, but this example
+			// is here mostly to show off offchain workers capabilities, not about building an
+			// oracle.
+			let avg_price = Self::average_price()
+				.map(|price| if &price > new_price { price - new_price } else { new_price - price })
+				.unwrap_or(0);
+
+			Ok(ValidTransaction {
+				// We set base priority to 2**20 to make sure it's included before any other
+				// transactions in the pool. Next we tweak the priority depending on how much
+				// it differs from the current average. (the more it differs the more priority it
+				// has).
+				priority: (1 << 20) + avg_price as u64,
+				// This transaction does not require anything else to go before into the pool.
+				// In theory we could require `previous_unsigned_at` transaction to go first,
+				// but it's not necessary in our case.
+				requires: vec![],
+				// We set the `provides` tag to be the same as `next_unsigned_at`. This makes
+				// sure only one transaction produced after `next_unsigned_at` will ever
+				// get to the transaction pool and will end up in the block.
+				// We can still have multiple transactions compete for the same "spot",
+				// and the one with higher priority will replace other one in the pool.
+				provides: vec![codec::Encode::encode(&(KEY_TYPE.0, next_unsigned_at))],
+				// The transaction is only valid for next 5 blocks. After that it's
+				// going to be revalidated by the pool.
+				longevity: 5,
+				// It's fine to propagate that transaction to other peers, which means it can be
+				// created even by nodes that don't produce blocks.
+				// Note that sometimes it's better to keep it for yourself (if you are the block
+				// producer), since for instance in some schemes others may copy your solution and
+				// claim a reward.
+				propagate: true,
+			})
+		} else {
+			InvalidTransaction::Call.into()
+		}
+	}
+}

substrate/frame/example-offchain-worker/src/tests.rs

@@ -0,0 +1,210 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::*;
+
+use codec::Decode;
+use frame_support::{
+	assert_ok, impl_outer_origin, parameter_types,
+	weights::{GetDispatchInfo, Weight},
+};
+use sp_core::{
+	H256,
+	offchain::{OffchainExt, TransactionPoolExt, testing},
+	testing::KeyStore,
+	traits::KeystoreExt,
+};
+use sp_runtime::{
+	Perbill, RuntimeAppPublic,
+	testing::{Header, TestXt},
+	traits::{BlakeTwo256, IdentityLookup, Extrinsic as ExtrinsicsT},
+};
+
+impl_outer_origin! {
+	pub enum Origin for Test  where system = frame_system {}
+}
+
+// For testing the module, we construct most of a mock runtime. This means
+// first constructing a configuration type (`Test`) which `impl`s each of the
+// configuration traits of modules we want to use.
+#[derive(Clone, Eq, PartialEq)]
+pub struct Test;
+parameter_types! {
+	pub const BlockHashCount: u64 = 250;
+	pub const MaximumBlockWeight: Weight = 1024;
+	pub const MaximumBlockLength: u32 = 2 * 1024;
+	pub const AvailableBlockRatio: Perbill = Perbill::one();
+}
+impl frame_system::Trait for Test {
+	type Origin = Origin;
+	type Index = u64;
+	type BlockNumber = u64;
+	type Hash = H256;
+	type Call = ();
+	type Hashing = BlakeTwo256;
+	type AccountId = sp_core::sr25519::Public;
+	type Lookup = IdentityLookup<Self::AccountId>;
+	type Header = Header;
+	type Event = ();
+	type BlockHashCount = BlockHashCount;
+	type MaximumBlockWeight = MaximumBlockWeight;
+	type MaximumBlockLength = MaximumBlockLength;
+	type AvailableBlockRatio = AvailableBlockRatio;
+	type Version = ();
+	type ModuleToIndex = ();
+	type OnReapAccount = ();
+	type OnNewAccount = ();
+	type AccountData = ();
+}
+
+type Extrinsic = TestXt<Call<Test>, ()>;
+type SubmitTransaction = frame_system::offchain::TransactionSubmitter<
+	crypto::Public,
+	Test,
+	Extrinsic
+>;
+
+impl frame_system::offchain::CreateTransaction<Test, Extrinsic> for Test {
+	type Public = sp_core::sr25519::Public;
+	type Signature = sp_core::sr25519::Signature;
+
+	fn create_transaction<F: frame_system::offchain::Signer<Self::Public, Self::Signature>>(
+		call: <Extrinsic as ExtrinsicsT>::Call,
+		_public: Self::Public,
+		_account: <Test as frame_system::Trait>::AccountId,
+		nonce: <Test as frame_system::Trait>::Index,
+	) -> Option<(<Extrinsic as ExtrinsicsT>::Call, <Extrinsic as ExtrinsicsT>::SignaturePayload)> {
+		Some((call, (nonce, ())))
+	}
+}
+
+parameter_types! {
+	pub const GracePeriod: u64 = 5;
+	pub const UnsignedInterval: u64 = 128;
+}
+
+impl Trait for Test {
+	type Event = ();
+	type Call = Call<Test>;
+	type SubmitSignedTransaction = SubmitTransaction;
+	type SubmitUnsignedTransaction = SubmitTransaction;
+	type GracePeriod = GracePeriod;
+	type UnsignedInterval = UnsignedInterval;
+}
+
+type Example = Module<Test>;
+
+#[test]
+fn it_aggregates_the_price() {
+	sp_io::TestExternalities::default().execute_with(|| {
+		assert_eq!(Example::average_price(), None);
+
+		assert_ok!(Example::submit_price(Origin::signed(Default::default()), 27));
+		assert_eq!(Example::average_price(), Some(27));
+
+		assert_ok!(Example::submit_price(Origin::signed(Default::default()), 43));
+		assert_eq!(Example::average_price(), Some(35));
+	});
+}
+
+#[test]
+fn should_make_http_call_and_parse_result() {
+	let (offchain, state) = testing::TestOffchainExt::new();
+	let mut t = sp_io::TestExternalities::default();
+	t.register_extension(OffchainExt::new(offchain));
+
+	price_oracle_response(&mut state.write());
+
+	t.execute_with(|| {
+		// when
+		let price = Example::fetch_price().unwrap();
+		// then
+		assert_eq!(price, 15522);
+	});
+}
+
+#[test]
+fn should_submit_signed_transaction_on_chain() {
+	const PHRASE: &str = "news slush supreme milk chapter athlete soap sausage put clutch what kitten";
+
+	let (offchain, offchain_state) = testing::TestOffchainExt::new();
+	let (pool, pool_state) = testing::TestTransactionPoolExt::new();
+	let keystore = KeyStore::new();
+	keystore.write().sr25519_generate_new(
+		crate::crypto::Public::ID,
+		Some(&format!("{}/hunter1", PHRASE))
+	).unwrap();
+
+
+	let mut t = sp_io::TestExternalities::default();
+	t.register_extension(OffchainExt::new(offchain));
+	t.register_extension(TransactionPoolExt::new(pool));
+	t.register_extension(KeystoreExt(keystore));
+
+	price_oracle_response(&mut offchain_state.write());
+
+	t.execute_with(|| {
+		// when
+		Example::fetch_price_and_send_signed().unwrap();
+		// then
+		let tx = pool_state.write().transactions.pop().unwrap();
+		assert!(pool_state.read().transactions.is_empty());
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		assert_eq!(tx.signature.unwrap().0, 0);
+		assert_eq!(tx.call, Call::submit_price(15522));
+	});
+}
+
+#[test]
+fn should_submit_unsigned_transaction_on_chain() {
+	let (offchain, offchain_state) = testing::TestOffchainExt::new();
+	let (pool, pool_state) = testing::TestTransactionPoolExt::new();
+	let mut t = sp_io::TestExternalities::default();
+	t.register_extension(OffchainExt::new(offchain));
+	t.register_extension(TransactionPoolExt::new(pool));
+
+	price_oracle_response(&mut offchain_state.write());
+
+	t.execute_with(|| {
+		// when
+		Example::fetch_price_and_send_unsigned(1).unwrap();
+		// then
+		let tx = pool_state.write().transactions.pop().unwrap();
+		assert!(pool_state.read().transactions.is_empty());
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		assert_eq!(tx.signature, None);
+		assert_eq!(tx.call, Call::submit_price_unsigned(1, 15522));
+	});
+}
+
+#[test]
+fn weights_work() {
+	// must have a default weight.
+	let default_call = <Call<Test>>::submit_price(10);
+	let info = default_call.get_dispatch_info();
+	// aka. `let info = <Call<Test> as GetDispatchInfo>::get_dispatch_info(&default_call);`
+	assert_eq!(info.weight, 10_000);
+}
+
+fn price_oracle_response(state: &mut testing::OffchainState) {
+	state.expect_request(0, testing::PendingRequest {
+		method: "GET".into(),
+		uri: "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD".into(),
+		response: Some(br#"{"USD": 155.23}"#.to_vec()),
+		sent: true,
+		..Default::default()
+	});
+}

substrate/frame/executive/src/lib.rs

@@ -357,6 +357,11 @@ where
 			&digests,
 			frame_system::InitKind::Inspection,
 		);
+
+		// Initialize logger, so the log messages are visible
+		// also when running WASM.
+		frame_support::debug::RuntimeLogger::init();
+
 		<AllModules as OffchainWorker<System::BlockNumber>>::offchain_worker(
 			// to maintain backward compatibility we call module offchain workers
 			// with parent block number.

substrate/frame/im-online/src/lib.rs

@@ -341,8 +341,6 @@ decl_module! {
 
 		// Runs after every block.
 		fn offchain_worker(now: T::BlockNumber) {
-			debug::RuntimeLogger::init();
-
 			// Only send messages if we are a potential validator.
 			if sp_io::offchain::is_validator() {
 				for res in Self::send_heartbeats(now).into_iter().flatten() {

substrate/frame/im-online/src/tests.rs

@@ -192,6 +192,8 @@ fn late_heartbeat_should_fail() {
 
 #[test]
 fn should_generate_heartbeats() {
+	use sp_runtime::traits::OffchainWorker;
+
 	let mut ext = new_test_ext();
 	let (offchain, _state) = TestOffchainExt::new();
 	let (pool, state) = TestTransactionPoolExt::new();
@@ -202,6 +204,7 @@ fn should_generate_heartbeats() {
 		// given
 		let block = 1;
 		System::set_block_number(block);
+		UintAuthorityId::set_all_keys(vec![0, 1, 2]);
 		// buffer new validators
 		Session::rotate_session();
 		// enact the change and buffer another one
@@ -209,17 +212,13 @@ fn should_generate_heartbeats() {
 		Session::rotate_session();
 
 		// when
-		UintAuthorityId::set_all_keys(vec![0, 1, 2]);
+		ImOnline::offchain_worker(block);
-		ImOnline::send_heartbeats(2)
-			.unwrap()
-			// make sure to consume the iterator and check there are no errors.
-			.collect::<Result<Vec<_>, _>>().unwrap();
-
 
 		// then
 		let transaction = state.write().transactions.pop().unwrap();
-		// All validators have `0` as their session key, so we generate 3 transactions.
+		// All validators have `0` as their session key, so we generate 2 transactions.
 		assert_eq!(state.read().transactions.len(), 2);
+
 		// check stuff about the transaction.
 		let ex: Extrinsic = Decode::decode(&mut &*transaction).unwrap();
 		let heartbeat = match ex.call {
@@ -228,7 +227,7 @@ fn should_generate_heartbeats() {
 		};
 
 		assert_eq!(heartbeat, Heartbeat {
-			block_number: 2,
+			block_number: block,
 			network_state: sp_io::offchain::network_state().unwrap(),
 			session_index: 2,
 			authority_index: 2,

substrate/primitives/core/src/offchain/testing.rs

@@ -72,6 +72,8 @@ pub struct OffchainState {
 	pub persistent_storage: InMemOffchainStorage,
 	/// Local storage
 	pub local_storage: InMemOffchainStorage,
+	/// Current timestamp (unix millis)
+	pub timestamp: u64,
 }
 
 impl OffchainState {
@@ -144,7 +146,7 @@ impl TestOffchainExt {
 
 impl offchain::Externalities for TestOffchainExt {
 	fn is_validator(&self) -> bool {
-		unimplemented!("not needed in tests so far")
+		true
 	}
 
 	fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
@@ -155,7 +157,7 @@ impl offchain::Externalities for TestOffchainExt {
 	}
 
 	fn timestamp(&mut self) -> Timestamp {
-		unimplemented!("not needed in tests so far")
+		Timestamp::from_unix_millis(self.0.read().timestamp)
 	}
 
 	fn sleep_until(&mut self, _deadline: Timestamp) {

substrate/primitives/runtime/src/offchain/mod.rs

@@ -18,3 +18,5 @@
 
 pub mod http;
 pub mod storage;
+
+pub use sp_core::offchain::*;