Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates
This commit is contained in:
Benjamin Kampmann
2019-11-14 21:51:17 +01:00
committed by Bastian Köcher
parent becc3b0a4f
commit 60e5011c72
809 changed files with 7801 additions and 6464 deletions
+484
View File
@@ -0,0 +1,484 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use std::{
str::FromStr,
sync::Arc,
convert::TryFrom,
thread::sleep,
};
use client_api::OffchainStorage;
use futures::{StreamExt as _, Future, FutureExt as _, future, channel::mpsc};
use log::{info, debug, warn, error};
use network::{PeerId, Multiaddr, NetworkStateInfo};
use codec::{Encode, Decode};
use primitives::offchain::{
Externalities as OffchainExt, HttpRequestId, Timestamp, HttpRequestStatus, HttpError,
OpaqueNetworkState, OpaquePeerId, OpaqueMultiaddr, StorageKind,
};
pub use offchain_primitives::STORAGE_PREFIX;
use sr_primitives::{generic::BlockId, traits::{self, Extrinsic}};
use transaction_pool::txpool::{Pool, ChainApi};
#[cfg(not(target_os = "unknown"))]
mod http;
#[cfg(target_os = "unknown")]
use http_dummy as http;
#[cfg(target_os = "unknown")]
mod http_dummy;
mod timestamp;
/// A message between the offchain extension and the processing thread.
enum ExtMessage {
SubmitExtrinsic(Vec<u8>),
}
/// Asynchronous offchain API.
///
/// NOTE this is done to prevent recursive calls into the runtime (which are not supported currently).
pub(crate) struct Api<Storage, Block: traits::Block> {
sender: mpsc::UnboundedSender<ExtMessage>,
db: Storage,
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
_at: BlockId<Block>,
/// Is this node a potential validator?
is_validator: bool,
/// Everything HTTP-related is handled by a different struct.
http: http::HttpApi,
}
fn unavailable_yet<R: Default>(name: &str) -> R {
error!(
"The {:?} API is not available for offchain workers yet. Follow \
https://github.com/paritytech/substrate/issues/1458 for details", name
);
Default::default()
}
const LOCAL_DB: &str = "LOCAL (fork-aware) DB";
impl<Storage, Block> OffchainExt for Api<Storage, Block>
where
Storage: OffchainStorage,
Block: traits::Block,
{
fn is_validator(&self) -> bool {
self.is_validator
}
fn submit_transaction(&mut self, ext: Vec<u8>) -> Result<(), ()> {
self.sender
.unbounded_send(ExtMessage::SubmitExtrinsic(ext))
.map(|_| ())
.map_err(|_| ())
}
fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
let external_addresses = self.network_state.external_addresses();
let state = NetworkState::new(
self.network_state.peer_id(),
external_addresses,
);
Ok(OpaqueNetworkState::from(state))
}
fn timestamp(&mut self) -> Timestamp {
timestamp::now()
}
fn sleep_until(&mut self, deadline: Timestamp) {
sleep(timestamp::timestamp_from_now(deadline));
}
fn random_seed(&mut self) -> [u8; 32] {
rand::random()
}
fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
match kind {
StorageKind::PERSISTENT => self.db.set(STORAGE_PREFIX, key, value),
StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
}
}
fn local_storage_compare_and_set(
&mut self,
kind: StorageKind,
key: &[u8],
old_value: Option<&[u8]>,
new_value: &[u8],
) -> bool {
match kind {
StorageKind::PERSISTENT => {
self.db.compare_and_set(STORAGE_PREFIX, key, old_value, new_value)
},
StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
}
}
fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
match kind {
StorageKind::PERSISTENT => self.db.get(STORAGE_PREFIX, key),
StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
}
}
fn http_request_start(
&mut self,
method: &str,
uri: &str,
_meta: &[u8]
) -> Result<HttpRequestId, ()> {
self.http.request_start(method, uri)
}
fn http_request_add_header(
&mut self,
request_id: HttpRequestId,
name: &str,
value: &str
) -> Result<(), ()> {
self.http.request_add_header(request_id, name, value)
}
fn http_request_write_body(
&mut self,
request_id: HttpRequestId,
chunk: &[u8],
deadline: Option<Timestamp>
) -> Result<(), HttpError> {
self.http.request_write_body(request_id, chunk, deadline)
}
fn http_response_wait(
&mut self,
ids: &[HttpRequestId],
deadline: Option<Timestamp>
) -> Vec<HttpRequestStatus> {
self.http.response_wait(ids, deadline)
}
fn http_response_headers(
&mut self,
request_id: HttpRequestId
) -> Vec<(Vec<u8>, Vec<u8>)> {
self.http.response_headers(request_id)
}
fn http_response_read_body(
&mut self,
request_id: HttpRequestId,
buffer: &mut [u8],
deadline: Option<Timestamp>
) -> Result<usize, HttpError> {
self.http.response_read_body(request_id, buffer, deadline)
}
}
/// Information about the local node's network state.
#[derive(Clone, Eq, PartialEq, Debug)]
pub struct NetworkState {
peer_id: PeerId,
external_addresses: Vec<Multiaddr>,
}
impl NetworkState {
fn new(peer_id: PeerId, external_addresses: Vec<Multiaddr>) -> Self {
NetworkState {
peer_id,
external_addresses,
}
}
}
impl From<NetworkState> for OpaqueNetworkState {
fn from(state: NetworkState) -> OpaqueNetworkState {
let enc = Encode::encode(&state.peer_id.into_bytes());
let peer_id = OpaquePeerId::new(enc);
let external_addresses: Vec<OpaqueMultiaddr> = state
.external_addresses
.iter()
.map(|multiaddr| {
let e = Encode::encode(&multiaddr.to_string());
OpaqueMultiaddr::new(e)
})
.collect();
OpaqueNetworkState {
peer_id,
external_addresses,
}
}
}
impl TryFrom<OpaqueNetworkState> for NetworkState {
type Error = ();
fn try_from(state: OpaqueNetworkState) -> Result<Self, Self::Error> {
let inner_vec = state.peer_id.0;
let bytes: Vec<u8> = Decode::decode(&mut &inner_vec[..]).map_err(|_| ())?;
let peer_id = PeerId::from_bytes(bytes).map_err(|_| ())?;
let external_addresses: Result<Vec<Multiaddr>, Self::Error> = state.external_addresses
.iter()
.map(|enc_multiaddr| -> Result<Multiaddr, Self::Error> {
let inner_vec = &enc_multiaddr.0;
let bytes = <Vec<u8>>::decode(&mut &inner_vec[..]).map_err(|_| ())?;
let multiaddr_str = String::from_utf8(bytes).map_err(|_| ())?;
let multiaddr = Multiaddr::from_str(&multiaddr_str).map_err(|_| ())?;
Ok(multiaddr)
})
.collect();
let external_addresses = external_addresses?;
Ok(NetworkState {
peer_id,
external_addresses,
})
}
}
/// Offchain extensions implementation API
///
/// This is the asynchronous processing part of the API.
pub(crate) struct AsyncApi<A: ChainApi> {
receiver: Option<mpsc::UnboundedReceiver<ExtMessage>>,
transaction_pool: Arc<Pool<A>>,
at: BlockId<A::Block>,
/// Everything HTTP-related is handled by a different struct.
http: Option<http::HttpWorker>,
}
impl<A: ChainApi> AsyncApi<A> {
/// Creates new Offchain extensions API implementation an the asynchronous processing part.
pub fn new<S: OffchainStorage>(
transaction_pool: Arc<Pool<A>>,
db: S,
at: BlockId<A::Block>,
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
is_validator: bool,
) -> (Api<S, A::Block>, AsyncApi<A>) {
let (sender, rx) = mpsc::unbounded();
let (http_api, http_worker) = http::http();
let api = Api {
sender,
db,
network_state,
_at: at,
is_validator,
http: http_api,
};
let async_api = AsyncApi {
receiver: Some(rx),
transaction_pool,
at,
http: Some(http_worker),
};
(api, async_api)
}
/// Run a processing task for the API
pub fn process(mut self) -> impl Future<Output = ()> {
let receiver = self.receiver.take().expect("Take invoked only once.");
let http = self.http.take().expect("Take invoked only once.");
let extrinsics = receiver.for_each(move |msg| {
match msg {
ExtMessage::SubmitExtrinsic(ext) => self.submit_extrinsic(ext),
}
});
future::join(extrinsics, http)
.map(|((), ())| ())
}
fn submit_extrinsic(&mut self, ext: Vec<u8>) -> impl Future<Output = ()> {
let xt = match <A::Block as traits::Block>::Extrinsic::decode(&mut &*ext) {
Ok(xt) => xt,
Err(e) => {
warn!("Unable to decode extrinsic: {:?}: {}", ext, e.what());
return future::Either::Left(future::ready(()))
},
};
info!("Submitting transaction to the pool: {:?} (isSigned: {:?})", xt, xt.is_signed());
future::Either::Right(self.transaction_pool
.submit_one(&self.at, xt.clone())
.map(|result| match result {
Ok(hash) => { debug!("[{:?}] Offchain transaction added to the pool.", hash); },
Err(e) => { warn!("Couldn't submit offchain transaction: {:?}", e); },
}))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::{convert::{TryFrom, TryInto}, time::SystemTime};
use sr_primitives::traits::Zero;
use client_db::offchain::LocalStorage;
use network::PeerId;
use test_client::runtime::Block;
struct MockNetworkStateInfo();
impl NetworkStateInfo for MockNetworkStateInfo {
fn external_addresses(&self) -> Vec<Multiaddr> {
Vec::new()
}
fn peer_id(&self) -> PeerId {
PeerId::random()
}
}
fn offchain_api() -> (Api<LocalStorage, Block>, AsyncApi<impl ChainApi>) {
let _ = env_logger::try_init();
let db = LocalStorage::new_test();
let client = Arc::new(test_client::new());
let pool = Arc::new(
Pool::new(Default::default(), transaction_pool::FullChainApi::new(client.clone()))
);
let mock = Arc::new(MockNetworkStateInfo());
AsyncApi::new(
pool,
db,
BlockId::Number(Zero::zero()),
mock,
false,
)
}
#[test]
fn should_get_timestamp() {
let mut api = offchain_api().0;
// Get timestamp from std.
let now = SystemTime::now();
let d: u64 = now.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_millis().try_into().unwrap();
// Get timestamp from offchain api.
let timestamp = api.timestamp();
// Compare.
assert!(timestamp.unix_millis() > 0);
assert_eq!(timestamp.unix_millis(), d);
}
#[test]
fn should_sleep() {
let mut api = offchain_api().0;
// Arrange.
let now = api.timestamp();
let delta = primitives::offchain::Duration::from_millis(100);
let deadline = now.add(delta);
// Act.
api.sleep_until(deadline);
let new_now = api.timestamp();
// Assert.
// The diff could be more than the sleep duration.
assert!(new_now.unix_millis() - 100 >= now.unix_millis());
}
#[test]
fn should_set_and_get_local_storage() {
// given
let kind = StorageKind::PERSISTENT;
let mut api = offchain_api().0;
let key = b"test";
// when
assert_eq!(api.local_storage_get(kind, key), None);
api.local_storage_set(kind, key, b"value");
// then
assert_eq!(api.local_storage_get(kind, key), Some(b"value".to_vec()));
}
#[test]
fn should_compare_and_set_local_storage() {
// given
let kind = StorageKind::PERSISTENT;
let mut api = offchain_api().0;
let key = b"test";
api.local_storage_set(kind, key, b"value");
// when
assert_eq!(api.local_storage_compare_and_set(kind, key, Some(b"val"), b"xxx"), false);
assert_eq!(api.local_storage_get(kind, key), Some(b"value".to_vec()));
// when
assert_eq!(api.local_storage_compare_and_set(kind, key, Some(b"value"), b"xxx"), true);
assert_eq!(api.local_storage_get(kind, key), Some(b"xxx".to_vec()));
}
#[test]
fn should_compare_and_set_local_storage_with_none() {
// given
let kind = StorageKind::PERSISTENT;
let mut api = offchain_api().0;
let key = b"test";
// when
let res = api.local_storage_compare_and_set(kind, key, None, b"value");
// then
assert_eq!(res, true);
assert_eq!(api.local_storage_get(kind, key), Some(b"value".to_vec()));
}
#[test]
fn should_convert_network_states() {
// given
let state = NetworkState::new(
PeerId::random(),
vec![
Multiaddr::try_from("/ip4/127.0.0.1/tcp/1234".to_string()).unwrap(),
Multiaddr::try_from("/ip6/2601:9:4f81:9700:803e:ca65:66e8:c21").unwrap(),
],
);
// when
let opaque_state = OpaqueNetworkState::from(state.clone());
let converted_back_state = NetworkState::try_from(opaque_state).unwrap();
// then
assert_eq!(state, converted_back_state);
}
#[test]
fn should_get_random_seed() {
// given
let mut api = offchain_api().0;
let seed = api.random_seed();
// then
assert_ne!(seed, [0; 32]);
}
}
+975
View File
@@ -0,0 +1,975 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! This module is composed of two structs: [`HttpApi`] and [`HttpWorker`]. Calling the [`http`]
//! function returns a pair of [`HttpApi`] and [`HttpWorker`] that share some state.
//!
//! The [`HttpApi`] is (indirectly) passed to the runtime when calling an offchain worker, while
//! the [`HttpWorker`] must be processed in the background. The [`HttpApi`] mimicks the API of the
//! HTTP-related methods available to offchain workers.
//!
//! The reason for this design is driven by the fact that HTTP requests should continue running
//! (i.e.: the socket should continue being processed) in the background even if the runtime isn't
//! actively calling any function.
use crate::api::timestamp;
use bytes::Buf as _;
use fnv::FnvHashMap;
use futures::{prelude::*, channel::mpsc, compat::Compat01As03};
use log::error;
use primitives::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError};
use std::{fmt, io::Read as _, mem, pin::Pin, task::Context, task::Poll};
/// Creates a pair of [`HttpApi`] and [`HttpWorker`].
pub fn http() -> (HttpApi, HttpWorker) {
let (to_worker, from_api) = mpsc::unbounded();
let (to_api, from_worker) = mpsc::unbounded();
let api = HttpApi {
to_worker,
from_worker: from_worker.fuse(),
// We start with a random ID for the first HTTP request, to prevent mischievous people from
// writing runtime code with hardcoded IDs.
next_id: HttpRequestId(rand::random::<u16>() % 2000),
requests: FnvHashMap::default(),
};
let engine = HttpWorker {
to_api,
from_api,
http_client: hyper::Client::builder().build(hyper_rustls::HttpsConnector::new(1)),
requests: Vec::new(),
};
(api, engine)
}
/// Provides HTTP capabilities.
///
/// Since this struct is a helper for offchain workers, its API is mimicking the API provided
/// to offchain workers.
pub struct HttpApi {
/// Used to sends messages to the worker.
to_worker: mpsc::UnboundedSender<ApiToWorker>,
/// Used to receive messages from the worker.
/// We use a `Fuse` in order to have an extra protection against panicking.
from_worker: stream::Fuse<mpsc::UnboundedReceiver<WorkerToApi>>,
/// Id to assign to the next HTTP request that is started.
next_id: HttpRequestId,
/// List of HTTP requests in preparation or in progress.
requests: FnvHashMap<HttpRequestId, HttpApiRequest>,
}
/// One active request within `HttpApi`.
enum HttpApiRequest {
/// The request object is being constructed locally and not started yet.
NotDispatched(hyper::Request<hyper::Body>, hyper::body::Sender),
/// The request has been dispatched and we're in the process of sending out the body (if the
/// field is `Some`) or waiting for a response (if the field is `None`).
Dispatched(Option<hyper::body::Sender>),
/// Received a response.
Response(HttpApiRequestRp),
/// A request has been dispatched but the worker notified us of an error. We report this
/// failure to the user as an `IoError` and remove the request from the list as soon as
/// possible.
Fail(hyper::Error),
}
/// A request within `HttpApi` that has received a response.
struct HttpApiRequestRp {
/// We might still be writing the request's body when the response comes.
/// This field allows to continue writing that body.
sending_body: Option<hyper::body::Sender>,
/// Status code of the response.
status_code: hyper::StatusCode,
/// Headers of the response.
headers: hyper::HeaderMap,
/// Body of the response, as a channel of `Chunk` objects.
/// While the code is designed to drop the `Receiver` once it ends, we wrap it within a
/// `Fuse` in order to be extra precautious about panics.
/// Elements extracted from the channel are first put into `current_read_chunk`.
/// If the channel produces an error, then that is translated into an `IoError` and the request
/// is removed from the list.
body: stream::Fuse<mpsc::Receiver<Result<hyper::Chunk, hyper::Error>>>,
/// Chunk that has been extracted from the channel and that is currently being read.
/// Reading data from the response should read from this field in priority.
current_read_chunk: Option<bytes::Reader<hyper::Chunk>>,
}
impl HttpApi {
/// Mimicks the corresponding method in the offchain API.
pub fn request_start(
&mut self,
method: &str,
uri: &str
) -> Result<HttpRequestId, ()> {
// Start by building the prototype of the request.
// We do this first so that we don't touch anything in `self` if building the prototype
// fails.
let (body_sender, body) = hyper::Body::channel();
let mut request = hyper::Request::new(body);
*request.method_mut() = hyper::Method::from_bytes(method.as_bytes()).map_err(|_| ())?;
*request.uri_mut() = hyper::Uri::from_shared(From::from(uri)).map_err(|_| ())?;
let new_id = self.next_id;
debug_assert!(!self.requests.contains_key(&new_id));
match self.next_id.0.checked_add(1) {
Some(new_id) => self.next_id.0 = new_id,
None => {
error!("Overflow in offchain worker HTTP request ID assignment");
return Err(());
}
};
self.requests.insert(new_id, HttpApiRequest::NotDispatched(request, body_sender));
Ok(new_id)
}
/// Mimicks the corresponding method in the offchain API.
pub fn request_add_header(
&mut self,
request_id: HttpRequestId,
name: &str,
value: &str
) -> Result<(), ()> {
let request = match self.requests.get_mut(&request_id) {
Some(&mut HttpApiRequest::NotDispatched(ref mut rq, _)) => rq,
_ => return Err(())
};
let name = hyper::header::HeaderName::from_bytes(name.as_bytes()).map_err(|_| ())?;
let value = hyper::header::HeaderValue::from_str(value).map_err(|_| ())?;
// Note that we're always appending headers and never replacing old values.
// We assume here that the user knows what they're doing.
request.headers_mut().append(name, value);
Ok(())
}
/// Mimicks the corresponding method in the offchain API.
pub fn request_write_body(
&mut self,
request_id: HttpRequestId,
chunk: &[u8],
deadline: Option<Timestamp>
) -> Result<(), HttpError> {
// Extract the request from the list.
// Don't forget to add it back if necessary when returning.
let mut request = match self.requests.remove(&request_id) {
None => return Err(HttpError::Invalid),
Some(r) => r,
};
let mut deadline = timestamp::deadline_to_future(deadline);
// Closure that writes data to a sender, taking the deadline into account. Can return `Ok`
// (if the body has been written), or `DeadlineReached`, or `IoError`.
// If `IoError` is returned, don't forget to remove the request from the list.
let mut poll_sender = move |sender: &mut hyper::body::Sender| -> Result<(), HttpError> {
let mut when_ready = future::maybe_done(Compat01As03::new(
futures01::future::poll_fn(|| sender.poll_ready())
));
futures::executor::block_on(future::select(&mut when_ready, &mut deadline));
match when_ready {
future::MaybeDone::Done(Ok(())) => {}
future::MaybeDone::Done(Err(_)) => return Err(HttpError::IoError),
future::MaybeDone::Future(_) |
future::MaybeDone::Gone => {
debug_assert!(if let future::MaybeDone::Done(_) = deadline { true } else { false });
return Err(HttpError::DeadlineReached)
}
};
match sender.send_data(hyper::Chunk::from(chunk.to_owned())) {
Ok(()) => Ok(()),
Err(_chunk) => {
error!("HTTP sender refused data despite being ready");
Err(HttpError::IoError)
},
}
};
loop {
request = match request {
HttpApiRequest::NotDispatched(request, sender) => {
// If the request is not dispatched yet, dispatch it and loop again.
let _ = self.to_worker.unbounded_send(ApiToWorker::Dispatch {
id: request_id,
request
});
HttpApiRequest::Dispatched(Some(sender))
}
HttpApiRequest::Dispatched(Some(mut sender)) =>
if !chunk.is_empty() {
match poll_sender(&mut sender) {
Err(HttpError::IoError) => return Err(HttpError::IoError),
other => {
self.requests.insert(
request_id,
HttpApiRequest::Dispatched(Some(sender))
);
return other
}
}
} else {
// Writing an empty body is a hint that we should stop writing. Dropping
// the sender.
self.requests.insert(request_id, HttpApiRequest::Dispatched(None));
return Ok(())
}
HttpApiRequest::Response(mut response @ HttpApiRequestRp { sending_body: Some(_), .. }) =>
if !chunk.is_empty() {
match poll_sender(response.sending_body.as_mut()
.expect("Can only enter this match branch if Some; qed")) {
Err(HttpError::IoError) => return Err(HttpError::IoError),
other => {
self.requests.insert(request_id, HttpApiRequest::Response(response));
return other
}
}
} else {
// Writing an empty body is a hint that we should stop writing. Dropping
// the sender.
self.requests.insert(request_id, HttpApiRequest::Response(HttpApiRequestRp {
sending_body: None,
..response
}));
return Ok(())
}
HttpApiRequest::Fail(_) =>
// If the request has already failed, return without putting back the request
// in the list.
return Err(HttpError::IoError),
v @ HttpApiRequest::Dispatched(None) |
v @ HttpApiRequest::Response(HttpApiRequestRp { sending_body: None, .. }) => {
// We have already finished sending this body.
self.requests.insert(request_id, v);
return Err(HttpError::Invalid)
}
}
}
}
/// Mimicks the corresponding method in the offchain API.
pub fn response_wait(
&mut self,
ids: &[HttpRequestId],
deadline: Option<Timestamp>
) -> Vec<HttpRequestStatus> {
// First of all, dispatch all the non-dispatched requests and drop all senders so that the
// user can't write anymore data.
for id in ids {
match self.requests.get_mut(id) {
Some(HttpApiRequest::NotDispatched(_, _)) => {}
Some(HttpApiRequest::Dispatched(sending_body)) |
Some(HttpApiRequest::Response(HttpApiRequestRp { sending_body, .. })) => {
let _ = sending_body.take();
continue
}
_ => continue
};
let (request, _sender) = match self.requests.remove(id) {
Some(HttpApiRequest::NotDispatched(rq, s)) => (rq, s),
_ => unreachable!("we checked for NotDispatched above; qed")
};
let _ = self.to_worker.unbounded_send(ApiToWorker::Dispatch {
id: *id,
request
});
// We also destroy the sender in order to forbid writing more data.
self.requests.insert(*id, HttpApiRequest::Dispatched(None));
}
let mut deadline = timestamp::deadline_to_future(deadline);
loop {
// Within that loop, first try to see if we have all the elements for a response.
// This includes the situation where the deadline is reached.
{
let mut output = Vec::with_capacity(ids.len());
let mut must_wait_more = false;
for id in ids {
output.push(match self.requests.get_mut(id) {
None => HttpRequestStatus::Invalid,
Some(HttpApiRequest::NotDispatched(_, _)) =>
unreachable!("we replaced all the NotDispatched with Dispatched earlier; qed"),
Some(HttpApiRequest::Dispatched(_)) => {
must_wait_more = true;
HttpRequestStatus::DeadlineReached
},
Some(HttpApiRequest::Fail(_)) => HttpRequestStatus::IoError,
Some(HttpApiRequest::Response(HttpApiRequestRp { status_code, .. })) =>
HttpRequestStatus::Finished(status_code.as_u16()),
});
}
debug_assert_eq!(output.len(), ids.len());
// Are we ready to call `return`?
let is_done = if let future::MaybeDone::Done(_) = deadline {
true
} else if !must_wait_more {
true
} else {
false
};
if is_done {
// Requests in "fail" mode are purged before returning.
debug_assert_eq!(output.len(), ids.len());
for n in (0..ids.len()).rev() {
if let HttpRequestStatus::IoError = output[n] {
self.requests.remove(&ids[n]);
}
}
return output
}
}
// Grab next message from the worker. We call `continue` if deadline is reached so that
// we loop back and `return`.
let next_message = {
let mut next_msg = future::maybe_done(self.from_worker.next());
futures::executor::block_on(future::select(&mut next_msg, &mut deadline));
if let future::MaybeDone::Done(msg) = next_msg {
msg
} else {
debug_assert!(if let future::MaybeDone::Done(_) = deadline { true } else { false });
continue
}
};
// Update internal state based on received message.
match next_message {
Some(WorkerToApi::Response { id, status_code, headers, body }) =>
match self.requests.remove(&id) {
Some(HttpApiRequest::Dispatched(sending_body)) => {
self.requests.insert(id, HttpApiRequest::Response(HttpApiRequestRp {
sending_body,
status_code,
headers,
body: body.fuse(),
current_read_chunk: None,
}));
}
None => {} // can happen if we detected an IO error when sending the body
_ => error!("State mismatch between the API and worker"),
}
Some(WorkerToApi::Fail { id, error }) =>
match self.requests.remove(&id) {
Some(HttpApiRequest::Dispatched(_)) => {
self.requests.insert(id, HttpApiRequest::Fail(error));
}
None => {} // can happen if we detected an IO error when sending the body
_ => error!("State mismatch between the API and worker"),
}
None => {
error!("Worker has crashed");
return ids.iter().map(|_| HttpRequestStatus::IoError).collect()
}
}
}
}
/// Mimicks the corresponding method in the offchain API.
pub fn response_headers(
&mut self,
request_id: HttpRequestId
) -> Vec<(Vec<u8>, Vec<u8>)> {
// Do an implicit non-blocking wait on the request.
let _ = self.response_wait(&[request_id], Some(timestamp::now()));
let headers = match self.requests.get(&request_id) {
Some(HttpApiRequest::Response(HttpApiRequestRp { headers, .. })) => headers,
_ => return Vec::new()
};
headers
.iter()
.map(|(name, value)| (name.as_str().as_bytes().to_owned(), value.as_bytes().to_owned()))
.collect()
}
/// Mimicks the corresponding method in the offchain API.
pub fn response_read_body(
&mut self,
request_id: HttpRequestId,
buffer: &mut [u8],
deadline: Option<Timestamp>
) -> Result<usize, HttpError> {
// Do an implicit wait on the request.
let _ = self.response_wait(&[request_id], deadline);
// Remove the request from the list and handle situations where the request is invalid or
// in the wrong state.
let mut response = match self.requests.remove(&request_id) {
Some(HttpApiRequest::Response(r)) => r,
// Because we called `response_wait` above, we know that the deadline has been reached
// and we still haven't received a response.
Some(rq @ HttpApiRequest::Dispatched(_)) => {
self.requests.insert(request_id, rq);
return Err(HttpError::DeadlineReached)
},
// The request has failed.
Some(HttpApiRequest::Fail { .. }) =>
return Err(HttpError::IoError),
// Request hasn't been dispatched yet; reading the body is invalid.
Some(rq @ HttpApiRequest::NotDispatched(_, _)) => {
self.requests.insert(request_id, rq);
return Err(HttpError::Invalid)
}
None => return Err(HttpError::Invalid)
};
// Convert the deadline into a `Future` that resolves when the deadline is reached.
let mut deadline = timestamp::deadline_to_future(deadline);
loop {
// First read from `current_read_chunk`.
if let Some(mut current_read_chunk) = response.current_read_chunk.take() {
match current_read_chunk.read(buffer) {
Ok(0) => {}
Ok(n) => {
self.requests.insert(request_id, HttpApiRequest::Response(HttpApiRequestRp {
current_read_chunk: Some(current_read_chunk),
.. response
}));
return Ok(n)
},
Err(err) => {
// This code should never be reached unless there's a logic error somewhere.
error!("Failed to read from current read chunk: {:?}", err);
return Err(HttpError::IoError)
}
}
}
// If we reach here, that means the `current_read_chunk` is empty and needs to be
// filled with a new chunk from `body`. We block on either the next body or the
// deadline.
let mut next_body = future::maybe_done(response.body.next());
futures::executor::block_on(future::select(&mut next_body, &mut deadline));
if let future::MaybeDone::Done(next_body) = next_body {
match next_body {
Some(Ok(chunk)) => response.current_read_chunk = Some(chunk.reader()),
Some(Err(_)) => return Err(HttpError::IoError),
None => return Ok(0), // eof
}
}
if let future::MaybeDone::Done(_) = deadline {
self.requests.insert(request_id, HttpApiRequest::Response(response));
return Err(HttpError::DeadlineReached)
}
}
}
}
impl fmt::Debug for HttpApi {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_list()
.entries(self.requests.iter())
.finish()
}
}
impl fmt::Debug for HttpApiRequest {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
HttpApiRequest::NotDispatched(_, _) =>
f.debug_tuple("HttpApiRequest::NotDispatched").finish(),
HttpApiRequest::Dispatched(_) =>
f.debug_tuple("HttpApiRequest::Dispatched").finish(),
HttpApiRequest::Response(HttpApiRequestRp { status_code, headers, .. }) =>
f.debug_tuple("HttpApiRequest::Response").field(status_code).field(headers).finish(),
HttpApiRequest::Fail(err) =>
f.debug_tuple("HttpApiRequest::Fail").field(err).finish(),
}
}
}
/// Message send from the API to the worker.
enum ApiToWorker {
/// Dispatches a new HTTP request.
Dispatch {
/// ID to send back when the response comes back.
id: HttpRequestId,
/// Request to start executing.
request: hyper::Request<hyper::Body>,
}
}
/// Message send from the API to the worker.
enum WorkerToApi {
/// A request has succeeded.
Response {
/// The ID that was passed to the worker.
id: HttpRequestId,
/// Status code of the response.
status_code: hyper::StatusCode,
/// Headers of the response.
headers: hyper::HeaderMap,
/// Body of the response, as a channel of `Chunk` objects.
/// We send the body back through a channel instead of returning the hyper `Body` object
/// because we don't want the `HttpApi` to have to drive the reading.
/// Instead, reading an item from the channel will notify the worker task, which will push
/// the next item.
/// Can also be used to send an error, in case an error happend on the HTTP socket. After
/// an error is sent, the channel will close.
body: mpsc::Receiver<Result<hyper::Chunk, hyper::Error>>,
},
/// A request has failed because of an error. The request is then no longer valid.
Fail {
/// The ID that was passed to the worker.
id: HttpRequestId,
/// Error that happened.
error: hyper::Error,
},
}
/// Must be continuously polled for the [`HttpApi`] to properly work.
pub struct HttpWorker {
/// Used to sends messages to the `HttpApi`.
to_api: mpsc::UnboundedSender<WorkerToApi>,
/// Used to receive messages from the `HttpApi`.
from_api: mpsc::UnboundedReceiver<ApiToWorker>,
/// The engine that runs HTTP requests.
http_client: hyper::Client<hyper_rustls::HttpsConnector<hyper::client::HttpConnector>, hyper::Body>,
/// HTTP requests that are being worked on by the engine.
requests: Vec<(HttpRequestId, HttpWorkerRequest)>,
}
/// HTTP request being processed by the worker.
enum HttpWorkerRequest {
/// Request has been dispatched and is waiting for a response from the Internet.
Dispatched(Compat01As03<hyper::client::ResponseFuture>),
/// Progressively reading the body of the response and sending it to the channel.
ReadBody {
/// Body to read `Chunk`s from. Only used if the channel is ready to accept data.
body: Compat01As03<hyper::Body>,
/// Channel to the [`HttpApi`] where we send the chunks to.
tx: mpsc::Sender<Result<hyper::Chunk, hyper::Error>>,
},
}
impl Future for HttpWorker {
type Output = ();
fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
// Reminder: this is continuously run in the background.
// We use a `me` variable because the compiler isn't smart enough to allow borrowing
// multiple fields at once through a `Deref`.
let me = &mut *self;
// We remove each element from `requests` one by one and add them back only if necessary.
for n in (0..me.requests.len()).rev() {
let (id, request) = me.requests.swap_remove(n);
match request {
HttpWorkerRequest::Dispatched(mut future) => {
// Check for an HTTP response from the Internet.
let mut response = match Future::poll(Pin::new(&mut future), cx) {
Poll::Pending => {
me.requests.push((id, HttpWorkerRequest::Dispatched(future)));
continue
},
Poll::Ready(Ok(response)) => response,
Poll::Ready(Err(err)) => {
let _ = me.to_api.unbounded_send(WorkerToApi::Fail {
id,
error: err,
});
continue; // don't insert the request back
}
};
// We received a response! Decompose it into its parts.
let status_code = response.status();
let headers = mem::replace(response.headers_mut(), hyper::HeaderMap::new());
let body = Compat01As03::new(response.into_body());
let (body_tx, body_rx) = mpsc::channel(3);
let _ = me.to_api.unbounded_send(WorkerToApi::Response {
id,
status_code,
headers,
body: body_rx,
});
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx: body_tx }));
cx.waker().wake_by_ref(); // reschedule in order to poll the new future
continue
}
HttpWorkerRequest::ReadBody { mut body, mut tx } => {
// Before reading from the HTTP response, check that `tx` is ready to accept
// a new chunk.
match tx.poll_ready(cx) {
Poll::Ready(Ok(())) => {}
Poll::Ready(Err(_)) => continue, // don't insert the request back
Poll::Pending => {
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx }));
continue
}
}
// `tx` is ready. Read a chunk from the socket and send it to the channel.
match Stream::poll_next(Pin::new(&mut body), cx) {
Poll::Ready(Some(Ok(chunk))) => {
let _ = tx.start_send(Ok(chunk));
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx }));
cx.waker().wake_by_ref(); // reschedule in order to continue reading
}
Poll::Ready(Some(Err(err))) => {
let _ = tx.start_send(Err(err));
// don't insert the request back
},
Poll::Ready(None) => {} // EOF; don't insert the request back
Poll::Pending => {
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx }));
},
}
}
}
}
// Check for messages coming from the [`HttpApi`].
match Stream::poll_next(Pin::new(&mut me.from_api), cx) {
Poll::Pending => {},
Poll::Ready(None) => return Poll::Ready(()), // stops the worker
Poll::Ready(Some(ApiToWorker::Dispatch { id, request })) => {
let future = Compat01As03::new(me.http_client.request(request));
debug_assert!(me.requests.iter().all(|(i, _)| *i != id));
me.requests.push((id, HttpWorkerRequest::Dispatched(future)));
cx.waker().wake_by_ref(); // reschedule the task to poll the request
}
}
Poll::Pending
}
}
impl fmt::Debug for HttpWorker {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_list()
.entries(self.requests.iter())
.finish()
}
}
impl fmt::Debug for HttpWorkerRequest {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
HttpWorkerRequest::Dispatched(_) =>
f.debug_tuple("HttpWorkerRequest::Dispatched").finish(),
HttpWorkerRequest::ReadBody { .. } =>
f.debug_tuple("HttpWorkerRequest::Response").finish(),
}
}
}
#[cfg(test)]
mod tests {
use crate::api::timestamp;
use super::http;
use futures::prelude::*;
use futures01::Future as _;
use primitives::offchain::{HttpError, HttpRequestId, HttpRequestStatus, Duration};
// Returns an `HttpApi` whose worker is ran in the background, and a `SocketAddr` to an HTTP
// server that runs in the background as well.
macro_rules! build_api_server {
() => {{
let (api, worker) = http();
// Note: we have to use tokio because hyper still uses old futures.
std::thread::spawn(move || {
tokio::run(futures::compat::Compat::new(worker.map(|()| Ok::<(), ()>(()))))
});
let (addr_tx, addr_rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap())
.serve(|| {
hyper::service::service_fn_ok(move |_: hyper::Request<hyper::Body>| {
hyper::Response::new(hyper::Body::from("Hello World!"))
})
});
let _ = addr_tx.send(server.local_addr());
hyper::rt::run(server.map_err(|e| panic!("{:?}", e)));
});
(api, addr_rx.recv().unwrap())
}};
}
#[test]
fn basic_localhost() {
let deadline = timestamp::now().add(Duration::from_millis(10_000));
// Performs an HTTP query to a background HTTP server.
let (mut api, addr) = build_api_server!();
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_write_body(id, &[], Some(deadline)).unwrap();
match api.response_wait(&[id], Some(deadline))[0] {
HttpRequestStatus::Finished(200) => {},
v => panic!("Connecting to localhost failed: {:?}", v)
}
let headers = api.response_headers(id);
assert!(headers.iter().any(|(h, _)| h.eq_ignore_ascii_case(b"Date")));
let mut buf = vec![0; 2048];
let n = api.response_read_body(id, &mut buf, Some(deadline)).unwrap();
assert_eq!(&buf[..n], b"Hello World!");
}
#[test]
fn request_start_invalid_call() {
let (mut api, addr) = build_api_server!();
match api.request_start("\0", &format!("http://{}", addr)) {
Err(()) => {}
Ok(_) => panic!()
};
match api.request_start("GET", "http://\0localhost") {
Err(()) => {}
Ok(_) => panic!()
};
}
#[test]
fn request_add_header_invalid_call() {
let (mut api, addr) = build_api_server!();
match api.request_add_header(HttpRequestId(0xdead), "Foo", "bar") {
Err(()) => {}
Ok(_) => panic!()
};
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
match api.request_add_header(id, "\0", "bar") {
Err(()) => {}
Ok(_) => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
match api.request_add_header(id, "Foo", "\0") {
Err(()) => {}
Ok(_) => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_add_header(id, "Foo", "Bar").unwrap();
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
match api.request_add_header(id, "Foo2", "Bar") {
Err(()) => {}
Ok(_) => panic!()
};
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
api.response_headers(id);
match api.request_add_header(id, "Foo2", "Bar") {
Err(()) => {}
Ok(_) => panic!()
};
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
api.response_read_body(id, &mut [], None).unwrap();
match api.request_add_header(id, "Foo2", "Bar") {
Err(()) => {}
Ok(_) => panic!()
};
}
#[test]
fn request_write_body_invalid_call() {
let (mut api, addr) = build_api_server!();
match api.request_write_body(HttpRequestId(0xdead), &[1, 2, 3], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
match api.request_write_body(HttpRequestId(0xdead), &[], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
api.request_write_body(id, &[], None).unwrap();
match api.request_write_body(id, &[], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
api.request_write_body(id, &[], None).unwrap();
match api.request_write_body(id, &[1, 2, 3, 4], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
api.response_wait(&[id], None);
match api.request_write_body(id, &[], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
api.response_wait(&[id], None);
match api.request_write_body(id, &[1, 2, 3, 4], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.response_headers(id);
match api.request_write_body(id, &[1, 2, 3, 4], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
api.response_headers(id);
match api.request_write_body(id, &[], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.response_read_body(id, &mut [], None).unwrap();
match api.request_write_body(id, &[1, 2, 3, 4], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.response_read_body(id, &mut [], None).unwrap();
match api.request_write_body(id, &[], None) {
Err(HttpError::Invalid) => {}
_ => panic!()
};
}
#[test]
fn response_headers_invalid_call() {
let (mut api, addr) = build_api_server!();
assert!(api.response_headers(HttpRequestId(0xdead)).is_empty());
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
assert!(api.response_headers(id).is_empty());
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_write_body(id, &[], None).unwrap();
while api.response_headers(id).is_empty() {
std::thread::sleep(std::time::Duration::from_millis(100));
}
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
api.response_wait(&[id], None);
assert!(!api.response_headers(id).is_empty());
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
let mut buf = [0; 128];
while api.response_read_body(id, &mut buf, None).unwrap() != 0 {}
assert!(api.response_headers(id).is_empty());
}
#[test]
fn response_header_invalid_call() {
let (mut api, addr) = build_api_server!();
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
assert!(api.response_headers(id).is_empty());
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
api.request_add_header(id, "Foo", "Bar").unwrap();
assert!(api.response_headers(id).is_empty());
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
api.request_add_header(id, "Foo", "Bar").unwrap();
api.request_write_body(id, &[], None).unwrap();
// Note: this test actually sends out the request, and is supposed to test a situation
// where we haven't received any response yet. This test can theoretically fail if the
// HTTP response comes back faster than the kernel schedules our thread, but that is highly
// unlikely.
assert!(api.response_headers(id).is_empty());
}
#[test]
fn response_read_body_invalid_call() {
let (mut api, addr) = build_api_server!();
let mut buf = [0; 512];
match api.response_read_body(HttpRequestId(0xdead), &mut buf, None) {
Err(HttpError::Invalid) => {}
_ => panic!()
}
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
while api.response_read_body(id, &mut buf, None).unwrap() != 0 {}
match api.response_read_body(id, &mut buf, None) {
Err(HttpError::Invalid) => {}
_ => panic!()
}
}
#[test]
fn fuzzing() {
// Uses the API in random ways to try to trigger panicks.
// Doesn't test some paths, such as waiting for multiple requests. Also doesn't test what
// happens if the server force-closes our socket.
let (mut api, addr) = build_api_server!();
for _ in 0..50 {
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
for _ in 0..250 {
match rand::random::<u8>() % 6 {
0 => { let _ = api.request_add_header(id, "Foo", "Bar"); }
1 => { let _ = api.request_write_body(id, &[1, 2, 3, 4], None); }
2 => { let _ = api.request_write_body(id, &[], None); }
3 => { let _ = api.response_wait(&[id], None); }
4 => { let _ = api.response_headers(id); }
5 => {
let mut buf = [0; 512];
let _ = api.response_read_body(id, &mut buf, None);
}
6 ..= 255 => unreachable!()
}
}
}
}
}
@@ -0,0 +1,109 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Contains the same API as the `http` module, except that everything returns an error.
use primitives::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError};
use std::{future::Future, pin::Pin, task::Context, task::Poll};
/// Creates a pair of [`HttpApi`] and [`HttpWorker`].
pub fn http() -> (HttpApi, HttpWorker) {
(HttpApi, HttpWorker)
}
/// Dummy implementation of HTTP capabilities.
#[derive(Debug)]
pub struct HttpApi;
/// Dummy implementation of HTTP capabilities.
#[derive(Debug)]
pub struct HttpWorker;
impl HttpApi {
/// Mimicks the corresponding method in the offchain API.
pub fn request_start(
&mut self,
_: &str,
_: &str
) -> Result<HttpRequestId, ()> {
/// Because this always returns an error, none of the other methods should ever be called.
Err(())
}
/// Mimicks the corresponding method in the offchain API.
pub fn request_add_header(
&mut self,
_: HttpRequestId,
_: &str,
_: &str
) -> Result<(), ()> {
unreachable!("Creating a request always fails, thus this function will \
never be called; qed")
}
/// Mimicks the corresponding method in the offchain API.
pub fn request_write_body(
&mut self,
_: HttpRequestId,
_: &[u8],
_: Option<Timestamp>
) -> Result<(), HttpError> {
unreachable!("Creating a request always fails, thus this function will \
never be called; qed")
}
/// Mimicks the corresponding method in the offchain API.
pub fn response_wait(
&mut self,
requests: &[HttpRequestId],
_: Option<Timestamp>
) -> Vec<HttpRequestStatus> {
if requests.is_empty() {
Vec::new()
} else {
unreachable!("Creating a request always fails, thus the list of requests should \
always be empty; qed")
}
}
/// Mimicks the corresponding method in the offchain API.
pub fn response_headers(
&mut self,
_: HttpRequestId
) -> Vec<(Vec<u8>, Vec<u8>)> {
unreachable!("Creating a request always fails, thus this function will \
never be called; qed")
}
/// Mimicks the corresponding method in the offchain API.
pub fn response_read_body(
&mut self,
_: HttpRequestId,
_: &mut [u8],
_: Option<Timestamp>
) -> Result<usize, HttpError> {
unreachable!("Creating a request always fails, thus this function will \
never be called; qed")
}
}
impl Future for HttpWorker {
type Output = ();
fn poll(self: Pin<&mut Self>, _: &mut Context) -> Poll<Self::Output> {
Poll::Ready(())
}
}
@@ -0,0 +1,62 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Helper methods dedicated to timestamps.
use primitives::offchain::Timestamp;
use std::convert::TryInto;
use std::time::{SystemTime, Duration};
/// Returns the current time as a `Timestamp`.
pub fn now() -> Timestamp {
let now = SystemTime::now();
let epoch_duration = now.duration_since(SystemTime::UNIX_EPOCH);
match epoch_duration {
Err(_) => {
// Current time is earlier than UNIX_EPOCH.
Timestamp::from_unix_millis(0)
},
Ok(d) => {
let duration = d.as_millis();
// Assuming overflow won't happen for a few hundred years.
Timestamp::from_unix_millis(duration.try_into()
.expect("epoch milliseconds won't overflow u64 for hundreds of years; qed"))
}
}
}
/// Returns how a `Timestamp` compares to "now".
///
/// In other words, returns `timestamp - now()`.
pub fn timestamp_from_now(timestamp: Timestamp) -> Duration {
Duration::from_millis(timestamp.diff(&now()).millis())
}
/// Converts the deadline into a `Future` that resolves when the deadline is reached.
///
/// If `None`, returns a never-ending `Future`.
pub fn deadline_to_future(
deadline: Option<Timestamp>,
) -> futures::future::MaybeDone<impl futures::Future> {
use futures::future;
future::maybe_done(match deadline {
Some(deadline) => future::Either::Left(
futures_timer::Delay::new(timestamp_from_now(deadline))
),
None => future::Either::Right(future::pending())
})
}
+186
View File
@@ -0,0 +1,186 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Substrate offchain workers.
//!
//! The offchain workers is a special function of the runtime that
//! gets executed after block is imported. During execution
//! it's able to asynchronously submit extrinsics that will either
//! be propagated to other nodes added to the next block
//! produced by the node as unsigned transactions.
//!
//! Offchain workers can be used for computation-heavy tasks
//! that are not feasible for execution during regular block processing.
//! It can either be tasks that no consensus is required for,
//! or some form of consensus over the data can be built on-chain
//! for instance via:
//! 1. Challenge period for incorrect computations
//! 2. Majority voting for results
//! 3. etc
#![warn(missing_docs)]
use std::{fmt, marker::PhantomData, sync::Arc};
use parking_lot::Mutex;
use threadpool::ThreadPool;
use sr_api::ApiExt;
use futures::future::Future;
use log::{debug, warn};
use network::NetworkStateInfo;
use primitives::{offchain, ExecutionContext};
use sr_primitives::{generic::BlockId, traits::{self, ProvideRuntimeApi}};
use transaction_pool::txpool::{Pool, ChainApi};
use client_api::{OffchainStorage};
mod api;
pub mod testing;
pub use offchain_primitives::{OffchainWorkerApi, STORAGE_PREFIX};
/// An offchain workers manager.
pub struct OffchainWorkers<Client, Storage, Block: traits::Block> {
client: Arc<Client>,
db: Storage,
_block: PhantomData<Block>,
thread_pool: Mutex<ThreadPool>,
}
impl<Client, Storage, Block: traits::Block> OffchainWorkers<Client, Storage, Block> {
/// Creates new `OffchainWorkers`.
pub fn new(client: Arc<Client>, db: Storage) -> Self {
Self {
client,
db,
_block: PhantomData,
thread_pool: Mutex::new(ThreadPool::new(num_cpus::get())),
}
}
}
impl<Client, Storage, Block: traits::Block> fmt::Debug for OffchainWorkers<
Client,
Storage,
Block,
> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_tuple("OffchainWorkers").finish()
}
}
impl<Client, Storage, Block> OffchainWorkers<
Client,
Storage,
Block,
> where
Block: traits::Block,
Client: ProvideRuntimeApi + Send + Sync + 'static,
Client::Api: OffchainWorkerApi<Block>,
Storage: OffchainStorage + 'static,
{
/// Start the offchain workers after given block.
#[must_use]
pub fn on_block_imported<A>(
&self,
number: &<Block::Header as traits::Header>::Number,
pool: &Arc<Pool<A>>,
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
is_validator: bool,
) -> impl Future<Output = ()> where A: ChainApi<Block=Block> + 'static {
let runtime = self.client.runtime_api();
let at = BlockId::number(*number);
let has_api = runtime.has_api::<dyn OffchainWorkerApi<Block, Error = ()>>(&at);
debug!("Checking offchain workers at {:?}: {:?}", at, has_api);
if has_api.unwrap_or(false) {
let (api, runner) = api::AsyncApi::new(
pool.clone(),
self.db.clone(),
at.clone(),
network_state.clone(),
is_validator,
);
debug!("Spawning offchain workers at {:?}", at);
let number = *number;
let client = self.client.clone();
self.spawn_worker(move || {
let runtime = client.runtime_api();
let api = Box::new(api);
debug!("Running offchain workers at {:?}", at);
let run = runtime.offchain_worker_with_context(
&at,
ExecutionContext::OffchainCall(Some((api, offchain::Capabilities::all()))),
number,
);
if let Err(e) = run {
log::error!("Error running offchain workers at {:?}: {:?}", at, e);
}
});
futures::future::Either::Left(runner.process())
} else {
futures::future::Either::Right(futures::future::ready(()))
}
}
/// Spawns a new offchain worker.
///
/// We spawn offchain workers for each block in a separate thread,
/// since they can run for a significant amount of time
/// in a blocking fashion and we don't want to block the runtime.
///
/// Note that we should avoid that if we switch to future-based runtime in the future,
/// alternatively:
fn spawn_worker(&self, f: impl FnOnce() -> () + Send + 'static) {
self.thread_pool.lock().execute(f);
}
}
#[cfg(test)]
mod tests {
use super::*;
use network::{Multiaddr, PeerId};
struct MockNetworkStateInfo();
impl NetworkStateInfo for MockNetworkStateInfo {
fn external_addresses(&self) -> Vec<Multiaddr> {
Vec::new()
}
fn peer_id(&self) -> PeerId {
PeerId::random()
}
}
#[test]
fn should_call_into_runtime_and_produce_extrinsic() {
// given
let _ = env_logger::try_init();
let client = Arc::new(test_client::new());
let pool = Arc::new(Pool::new(Default::default(), transaction_pool::FullChainApi::new(client.clone())));
let db = client_db::offchain::LocalStorage::new_test();
let network_state = Arc::new(MockNetworkStateInfo());
// when
let offchain = OffchainWorkers::new(client, db);
futures::executor::block_on(offchain.on_block_imported(&0u64, &pool, network_state, false));
// then
assert_eq!(pool.status().ready, 1);
assert_eq!(pool.ready().next().unwrap().is_propagateable(), false);
}
}
+307
View File
@@ -0,0 +1,307 @@
// Copyright 2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Offchain Externalities implementation for tests.
use std::{
collections::BTreeMap,
sync::Arc,
};
use client_api::{OffchainStorage, InMemOffchainStorage};
use parking_lot::RwLock;
use primitives::offchain::{
self,
HttpError,
HttpRequestId as RequestId,
HttpRequestStatus as RequestStatus,
Timestamp,
StorageKind,
OpaqueNetworkState,
};
/// Pending request.
#[derive(Debug, Default, PartialEq, Eq)]
pub struct PendingRequest {
/// HTTP method
pub method: String,
/// URI
pub uri: String,
/// Encoded Metadata
pub meta: Vec<u8>,
/// Request headers
pub headers: Vec<(String, String)>,
/// Request body
pub body: Vec<u8>,
/// Has the request been sent already.
pub sent: bool,
/// Response body
pub response: Option<Vec<u8>>,
/// Number of bytes already read from the response body.
pub read: usize,
/// Response headers
pub response_headers: Vec<(String, String)>,
}
/// Internal state of the externalities.
///
/// This can be used in tests to respond or assert stuff about interactions.
#[derive(Debug, Default)]
pub struct State {
/// A list of pending requests.
pub requests: BTreeMap<RequestId, PendingRequest>,
expected_requests: BTreeMap<RequestId, PendingRequest>,
/// Persistent local storage
pub persistent_storage: InMemOffchainStorage,
/// Local storage
pub local_storage: InMemOffchainStorage,
/// A vector of transactions submitted from the runtime.
pub transactions: Vec<Vec<u8>>,
}
impl State {
/// Asserts that pending request has been submitted and fills it's response.
pub fn fulfill_pending_request(
&mut self,
id: u16,
expected: PendingRequest,
response: impl Into<Vec<u8>>,
response_headers: impl IntoIterator<Item=(String, String)>,
) {
match self.requests.get_mut(&RequestId(id)) {
None => {
panic!("Missing pending request: {:?}.\n\nAll: {:?}", id, self.requests);
}
Some(req) => {
assert_eq!(
*req,
expected,
);
req.response = Some(response.into());
req.response_headers = response_headers.into_iter().collect();
}
}
}
fn fulfill_expected(&mut self, id: u16) {
if let Some(mut req) = self.expected_requests.remove(&RequestId(id)) {
let response = req.response.take().expect("Response checked while added.");
let headers = std::mem::replace(&mut req.response_headers, vec![]);
self.fulfill_pending_request(id, req, response, headers);
}
}
/// Add expected HTTP request.
///
/// This method can be used to initialize expected HTTP requests and their responses
/// before running the actual code that utilizes them (for instance before calling into runtime).
/// Expected request has to be fulfilled before this struct is dropped,
/// the `response` and `response_headers` fields will be used to return results to the callers.
pub fn expect_request(&mut self, id: u16, expected: PendingRequest) {
if expected.response.is_none() {
panic!("Expected request needs to have a response.");
}
self.expected_requests.insert(RequestId(id), expected);
}
}
impl Drop for State {
fn drop(&mut self) {
// If we panic! while we are already in a panic, the test dies with an illegal instruction.
if !self.expected_requests.is_empty() && !std::thread::panicking() {
panic!("Unfulfilled expected requests: {:?}", self.expected_requests);
}
}
}
/// Implementation of offchain externalities used for tests.
#[derive(Clone, Default, Debug)]
pub struct TestOffchainExt(pub Arc<RwLock<State>>);
impl TestOffchainExt {
/// Create new `TestOffchainExt` and a reference to the internal state.
pub fn new() -> (Self, Arc<RwLock<State>>) {
let ext = Self::default();
let state = ext.0.clone();
(ext, state)
}
}
impl offchain::Externalities for TestOffchainExt {
fn is_validator(&self) -> bool {
unimplemented!("not needed in tests so far")
}
fn submit_transaction(&mut self, ex: Vec<u8>) -> Result<(), ()> {
let mut state = self.0.write();
state.transactions.push(ex);
Ok(())
}
fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
Ok(OpaqueNetworkState {
peer_id: Default::default(),
external_addresses: vec![],
})
}
fn timestamp(&mut self) -> Timestamp {
unimplemented!("not needed in tests so far")
}
fn sleep_until(&mut self, _deadline: Timestamp) {
unimplemented!("not needed in tests so far")
}
fn random_seed(&mut self) -> [u8; 32] {
unimplemented!("not needed in tests so far")
}
fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
let mut state = self.0.write();
match kind {
StorageKind::LOCAL => &mut state.local_storage,
StorageKind::PERSISTENT => &mut state.persistent_storage,
}.set(b"", key, value);
}
fn local_storage_compare_and_set(
&mut self,
kind: StorageKind,
key: &[u8],
old_value: Option<&[u8]>,
new_value: &[u8]
) -> bool {
let mut state = self.0.write();
match kind {
StorageKind::LOCAL => &mut state.local_storage,
StorageKind::PERSISTENT => &mut state.persistent_storage,
}.compare_and_set(b"", key, old_value, new_value)
}
fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
let state = self.0.read();
match kind {
StorageKind::LOCAL => &state.local_storage,
StorageKind::PERSISTENT => &state.persistent_storage,
}.get(b"", key)
}
fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<RequestId, ()> {
let mut state = self.0.write();
let id = RequestId(state.requests.len() as u16);
state.requests.insert(id.clone(), PendingRequest {
method: method.into(),
uri: uri.into(),
meta: meta.into(),
..Default::default()
});
Ok(id)
}
fn http_request_add_header(
&mut self,
request_id: RequestId,
name: &str,
value: &str,
) -> Result<(), ()> {
let mut state = self.0.write();
if let Some(req) = state.requests.get_mut(&request_id) {
req.headers.push((name.into(), value.into()));
Ok(())
} else {
Err(())
}
}
fn http_request_write_body(
&mut self,
request_id: RequestId,
chunk: &[u8],
_deadline: Option<Timestamp>
) -> Result<(), HttpError> {
let mut state = self.0.write();
let sent = {
let req = state.requests.get_mut(&request_id).ok_or(HttpError::IoError)?;
req.body.extend(chunk);
if chunk.is_empty() {
req.sent = true;
}
req.sent
};
if sent {
state.fulfill_expected(request_id.0);
}
Ok(())
}
fn http_response_wait(
&mut self,
ids: &[RequestId],
_deadline: Option<Timestamp>,
) -> Vec<RequestStatus> {
let state = self.0.read();
ids.iter().map(|id| match state.requests.get(id) {
Some(req) if req.response.is_none() =>
panic!("No `response` provided for request with id: {:?}", id),
None => RequestStatus::Invalid,
_ => RequestStatus::Finished(200),
}).collect()
}
fn http_response_headers(&mut self, request_id: RequestId) -> Vec<(Vec<u8>, Vec<u8>)> {
let state = self.0.read();
if let Some(req) = state.requests.get(&request_id) {
req.response_headers
.clone()
.into_iter()
.map(|(k, v)| (k.into_bytes(), v.into_bytes()))
.collect()
} else {
Default::default()
}
}
fn http_response_read_body(
&mut self,
request_id: RequestId,
buffer: &mut [u8],
_deadline: Option<Timestamp>
) -> Result<usize, HttpError> {
let mut state = self.0.write();
if let Some(req) = state.requests.get_mut(&request_id) {
let response = req.response
.as_mut()
.expect(&format!("No response provided for request: {:?}", request_id));
if req.read >= response.len() {
// Remove the pending request as per spec.
state.requests.remove(&request_id);
Ok(0)
} else {
let read = std::cmp::min(buffer.len(), response[req.read..].len());
buffer[0..read].copy_from_slice(&response[req.read..read]);
req.read += read;
Ok(read)
}
} else {
Err(HttpError::IoError)
}
}
}