mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-06-14 15:41:02 +00:00
Reorganising the repository - external renames and moves (#4074)
* Adding first rough ouline of the repository structure * Remove old CI stuff * add title * formatting fixes * move node-exits job's script to scripts dir * Move docs into subdir * move to bin * move maintainence scripts, configs and helpers into its own dir * add .local to ignore * move core->client * start up 'test' area * move test client * move test runtime * make test move compile * Add dependencies rule enforcement. * Fix indexing. * Update docs to reflect latest changes * Moving /srml->/paint * update docs * move client/sr-* -> primitives/ * clean old readme * remove old broken code in rhd * update lock * Step 1. * starting to untangle client * Fix after merge. * start splitting out client interfaces * move children and blockchain interfaces * Move trie and state-machine to primitives. * Fix WASM builds. * fixing broken imports * more interface moves * move backend and light to interfaces * move CallExecutor * move cli off client * moving around more interfaces * re-add consensus crates into the mix * fix subkey path * relieve client from executor * starting to pull out client from grandpa * move is_decendent_of out of client * grandpa still depends on client directly * lemme tests pass * rename srml->paint * Make it compile. * rename interfaces->client-api * Move keyring to primitives. * fixup libp2p dep * fix broken use * allow dependency enforcement to fail * move fork-tree * Moving wasm-builder * make env * move build-script-utils * fixup broken crate depdencies and names * fix imports for authority discovery * fix typo * update cargo.lock * fixing imports * Fix paths and add missing crates * re-add missing crates
This commit is contained in:
committed by
Bastian Köcher
parent
becc3b0a4f
commit
60e5011c72
@@ -0,0 +1,484 @@
|
||||
// Copyright 2019 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use std::{
|
||||
str::FromStr,
|
||||
sync::Arc,
|
||||
convert::TryFrom,
|
||||
thread::sleep,
|
||||
};
|
||||
|
||||
use client_api::OffchainStorage;
|
||||
use futures::{StreamExt as _, Future, FutureExt as _, future, channel::mpsc};
|
||||
use log::{info, debug, warn, error};
|
||||
use network::{PeerId, Multiaddr, NetworkStateInfo};
|
||||
use codec::{Encode, Decode};
|
||||
use primitives::offchain::{
|
||||
Externalities as OffchainExt, HttpRequestId, Timestamp, HttpRequestStatus, HttpError,
|
||||
OpaqueNetworkState, OpaquePeerId, OpaqueMultiaddr, StorageKind,
|
||||
};
|
||||
pub use offchain_primitives::STORAGE_PREFIX;
|
||||
use sr_primitives::{generic::BlockId, traits::{self, Extrinsic}};
|
||||
use transaction_pool::txpool::{Pool, ChainApi};
|
||||
|
||||
#[cfg(not(target_os = "unknown"))]
|
||||
mod http;
|
||||
|
||||
#[cfg(target_os = "unknown")]
|
||||
use http_dummy as http;
|
||||
#[cfg(target_os = "unknown")]
|
||||
mod http_dummy;
|
||||
|
||||
mod timestamp;
|
||||
|
||||
/// A message between the offchain extension and the processing thread.
|
||||
enum ExtMessage {
|
||||
SubmitExtrinsic(Vec<u8>),
|
||||
}
|
||||
|
||||
/// Asynchronous offchain API.
|
||||
///
|
||||
/// NOTE this is done to prevent recursive calls into the runtime (which are not supported currently).
|
||||
pub(crate) struct Api<Storage, Block: traits::Block> {
|
||||
sender: mpsc::UnboundedSender<ExtMessage>,
|
||||
db: Storage,
|
||||
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
|
||||
_at: BlockId<Block>,
|
||||
/// Is this node a potential validator?
|
||||
is_validator: bool,
|
||||
/// Everything HTTP-related is handled by a different struct.
|
||||
http: http::HttpApi,
|
||||
}
|
||||
|
||||
fn unavailable_yet<R: Default>(name: &str) -> R {
|
||||
error!(
|
||||
"The {:?} API is not available for offchain workers yet. Follow \
|
||||
https://github.com/paritytech/substrate/issues/1458 for details", name
|
||||
);
|
||||
Default::default()
|
||||
}
|
||||
|
||||
const LOCAL_DB: &str = "LOCAL (fork-aware) DB";
|
||||
|
||||
impl<Storage, Block> OffchainExt for Api<Storage, Block>
|
||||
where
|
||||
Storage: OffchainStorage,
|
||||
Block: traits::Block,
|
||||
{
|
||||
fn is_validator(&self) -> bool {
|
||||
self.is_validator
|
||||
}
|
||||
|
||||
fn submit_transaction(&mut self, ext: Vec<u8>) -> Result<(), ()> {
|
||||
self.sender
|
||||
.unbounded_send(ExtMessage::SubmitExtrinsic(ext))
|
||||
.map(|_| ())
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
|
||||
let external_addresses = self.network_state.external_addresses();
|
||||
|
||||
let state = NetworkState::new(
|
||||
self.network_state.peer_id(),
|
||||
external_addresses,
|
||||
);
|
||||
Ok(OpaqueNetworkState::from(state))
|
||||
}
|
||||
|
||||
fn timestamp(&mut self) -> Timestamp {
|
||||
timestamp::now()
|
||||
}
|
||||
|
||||
fn sleep_until(&mut self, deadline: Timestamp) {
|
||||
sleep(timestamp::timestamp_from_now(deadline));
|
||||
}
|
||||
|
||||
fn random_seed(&mut self) -> [u8; 32] {
|
||||
rand::random()
|
||||
}
|
||||
|
||||
fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
|
||||
match kind {
|
||||
StorageKind::PERSISTENT => self.db.set(STORAGE_PREFIX, key, value),
|
||||
StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
|
||||
}
|
||||
}
|
||||
|
||||
fn local_storage_compare_and_set(
|
||||
&mut self,
|
||||
kind: StorageKind,
|
||||
key: &[u8],
|
||||
old_value: Option<&[u8]>,
|
||||
new_value: &[u8],
|
||||
) -> bool {
|
||||
match kind {
|
||||
StorageKind::PERSISTENT => {
|
||||
self.db.compare_and_set(STORAGE_PREFIX, key, old_value, new_value)
|
||||
},
|
||||
StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
|
||||
}
|
||||
}
|
||||
|
||||
fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
|
||||
match kind {
|
||||
StorageKind::PERSISTENT => self.db.get(STORAGE_PREFIX, key),
|
||||
StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
|
||||
}
|
||||
}
|
||||
|
||||
fn http_request_start(
|
||||
&mut self,
|
||||
method: &str,
|
||||
uri: &str,
|
||||
_meta: &[u8]
|
||||
) -> Result<HttpRequestId, ()> {
|
||||
self.http.request_start(method, uri)
|
||||
}
|
||||
|
||||
fn http_request_add_header(
|
||||
&mut self,
|
||||
request_id: HttpRequestId,
|
||||
name: &str,
|
||||
value: &str
|
||||
) -> Result<(), ()> {
|
||||
self.http.request_add_header(request_id, name, value)
|
||||
}
|
||||
|
||||
fn http_request_write_body(
|
||||
&mut self,
|
||||
request_id: HttpRequestId,
|
||||
chunk: &[u8],
|
||||
deadline: Option<Timestamp>
|
||||
) -> Result<(), HttpError> {
|
||||
self.http.request_write_body(request_id, chunk, deadline)
|
||||
}
|
||||
|
||||
fn http_response_wait(
|
||||
&mut self,
|
||||
ids: &[HttpRequestId],
|
||||
deadline: Option<Timestamp>
|
||||
) -> Vec<HttpRequestStatus> {
|
||||
self.http.response_wait(ids, deadline)
|
||||
}
|
||||
|
||||
fn http_response_headers(
|
||||
&mut self,
|
||||
request_id: HttpRequestId
|
||||
) -> Vec<(Vec<u8>, Vec<u8>)> {
|
||||
self.http.response_headers(request_id)
|
||||
}
|
||||
|
||||
fn http_response_read_body(
|
||||
&mut self,
|
||||
request_id: HttpRequestId,
|
||||
buffer: &mut [u8],
|
||||
deadline: Option<Timestamp>
|
||||
) -> Result<usize, HttpError> {
|
||||
self.http.response_read_body(request_id, buffer, deadline)
|
||||
}
|
||||
}
|
||||
|
||||
/// Information about the local node's network state.
|
||||
#[derive(Clone, Eq, PartialEq, Debug)]
|
||||
pub struct NetworkState {
|
||||
peer_id: PeerId,
|
||||
external_addresses: Vec<Multiaddr>,
|
||||
}
|
||||
|
||||
impl NetworkState {
|
||||
fn new(peer_id: PeerId, external_addresses: Vec<Multiaddr>) -> Self {
|
||||
NetworkState {
|
||||
peer_id,
|
||||
external_addresses,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<NetworkState> for OpaqueNetworkState {
|
||||
fn from(state: NetworkState) -> OpaqueNetworkState {
|
||||
let enc = Encode::encode(&state.peer_id.into_bytes());
|
||||
let peer_id = OpaquePeerId::new(enc);
|
||||
|
||||
let external_addresses: Vec<OpaqueMultiaddr> = state
|
||||
.external_addresses
|
||||
.iter()
|
||||
.map(|multiaddr| {
|
||||
let e = Encode::encode(&multiaddr.to_string());
|
||||
OpaqueMultiaddr::new(e)
|
||||
})
|
||||
.collect();
|
||||
|
||||
OpaqueNetworkState {
|
||||
peer_id,
|
||||
external_addresses,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<OpaqueNetworkState> for NetworkState {
|
||||
type Error = ();
|
||||
|
||||
fn try_from(state: OpaqueNetworkState) -> Result<Self, Self::Error> {
|
||||
let inner_vec = state.peer_id.0;
|
||||
|
||||
let bytes: Vec<u8> = Decode::decode(&mut &inner_vec[..]).map_err(|_| ())?;
|
||||
let peer_id = PeerId::from_bytes(bytes).map_err(|_| ())?;
|
||||
|
||||
let external_addresses: Result<Vec<Multiaddr>, Self::Error> = state.external_addresses
|
||||
.iter()
|
||||
.map(|enc_multiaddr| -> Result<Multiaddr, Self::Error> {
|
||||
let inner_vec = &enc_multiaddr.0;
|
||||
let bytes = <Vec<u8>>::decode(&mut &inner_vec[..]).map_err(|_| ())?;
|
||||
let multiaddr_str = String::from_utf8(bytes).map_err(|_| ())?;
|
||||
let multiaddr = Multiaddr::from_str(&multiaddr_str).map_err(|_| ())?;
|
||||
Ok(multiaddr)
|
||||
})
|
||||
.collect();
|
||||
let external_addresses = external_addresses?;
|
||||
|
||||
Ok(NetworkState {
|
||||
peer_id,
|
||||
external_addresses,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Offchain extensions implementation API
|
||||
///
|
||||
/// This is the asynchronous processing part of the API.
|
||||
pub(crate) struct AsyncApi<A: ChainApi> {
|
||||
receiver: Option<mpsc::UnboundedReceiver<ExtMessage>>,
|
||||
transaction_pool: Arc<Pool<A>>,
|
||||
at: BlockId<A::Block>,
|
||||
/// Everything HTTP-related is handled by a different struct.
|
||||
http: Option<http::HttpWorker>,
|
||||
}
|
||||
|
||||
impl<A: ChainApi> AsyncApi<A> {
|
||||
/// Creates new Offchain extensions API implementation an the asynchronous processing part.
|
||||
pub fn new<S: OffchainStorage>(
|
||||
transaction_pool: Arc<Pool<A>>,
|
||||
db: S,
|
||||
at: BlockId<A::Block>,
|
||||
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
|
||||
is_validator: bool,
|
||||
) -> (Api<S, A::Block>, AsyncApi<A>) {
|
||||
let (sender, rx) = mpsc::unbounded();
|
||||
|
||||
let (http_api, http_worker) = http::http();
|
||||
|
||||
let api = Api {
|
||||
sender,
|
||||
db,
|
||||
network_state,
|
||||
_at: at,
|
||||
is_validator,
|
||||
http: http_api,
|
||||
};
|
||||
|
||||
let async_api = AsyncApi {
|
||||
receiver: Some(rx),
|
||||
transaction_pool,
|
||||
at,
|
||||
http: Some(http_worker),
|
||||
};
|
||||
|
||||
(api, async_api)
|
||||
}
|
||||
|
||||
/// Run a processing task for the API
|
||||
pub fn process(mut self) -> impl Future<Output = ()> {
|
||||
let receiver = self.receiver.take().expect("Take invoked only once.");
|
||||
let http = self.http.take().expect("Take invoked only once.");
|
||||
|
||||
let extrinsics = receiver.for_each(move |msg| {
|
||||
match msg {
|
||||
ExtMessage::SubmitExtrinsic(ext) => self.submit_extrinsic(ext),
|
||||
}
|
||||
});
|
||||
|
||||
future::join(extrinsics, http)
|
||||
.map(|((), ())| ())
|
||||
}
|
||||
|
||||
fn submit_extrinsic(&mut self, ext: Vec<u8>) -> impl Future<Output = ()> {
|
||||
let xt = match <A::Block as traits::Block>::Extrinsic::decode(&mut &*ext) {
|
||||
Ok(xt) => xt,
|
||||
Err(e) => {
|
||||
warn!("Unable to decode extrinsic: {:?}: {}", ext, e.what());
|
||||
return future::Either::Left(future::ready(()))
|
||||
},
|
||||
};
|
||||
|
||||
info!("Submitting transaction to the pool: {:?} (isSigned: {:?})", xt, xt.is_signed());
|
||||
future::Either::Right(self.transaction_pool
|
||||
.submit_one(&self.at, xt.clone())
|
||||
.map(|result| match result {
|
||||
Ok(hash) => { debug!("[{:?}] Offchain transaction added to the pool.", hash); },
|
||||
Err(e) => { warn!("Couldn't submit offchain transaction: {:?}", e); },
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::{convert::{TryFrom, TryInto}, time::SystemTime};
|
||||
use sr_primitives::traits::Zero;
|
||||
use client_db::offchain::LocalStorage;
|
||||
use network::PeerId;
|
||||
use test_client::runtime::Block;
|
||||
|
||||
struct MockNetworkStateInfo();
|
||||
|
||||
impl NetworkStateInfo for MockNetworkStateInfo {
|
||||
fn external_addresses(&self) -> Vec<Multiaddr> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn peer_id(&self) -> PeerId {
|
||||
PeerId::random()
|
||||
}
|
||||
}
|
||||
|
||||
fn offchain_api() -> (Api<LocalStorage, Block>, AsyncApi<impl ChainApi>) {
|
||||
let _ = env_logger::try_init();
|
||||
let db = LocalStorage::new_test();
|
||||
let client = Arc::new(test_client::new());
|
||||
let pool = Arc::new(
|
||||
Pool::new(Default::default(), transaction_pool::FullChainApi::new(client.clone()))
|
||||
);
|
||||
|
||||
let mock = Arc::new(MockNetworkStateInfo());
|
||||
AsyncApi::new(
|
||||
pool,
|
||||
db,
|
||||
BlockId::Number(Zero::zero()),
|
||||
mock,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_get_timestamp() {
|
||||
let mut api = offchain_api().0;
|
||||
|
||||
// Get timestamp from std.
|
||||
let now = SystemTime::now();
|
||||
let d: u64 = now.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_millis().try_into().unwrap();
|
||||
|
||||
// Get timestamp from offchain api.
|
||||
let timestamp = api.timestamp();
|
||||
|
||||
// Compare.
|
||||
assert!(timestamp.unix_millis() > 0);
|
||||
assert_eq!(timestamp.unix_millis(), d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_sleep() {
|
||||
let mut api = offchain_api().0;
|
||||
|
||||
// Arrange.
|
||||
let now = api.timestamp();
|
||||
let delta = primitives::offchain::Duration::from_millis(100);
|
||||
let deadline = now.add(delta);
|
||||
|
||||
// Act.
|
||||
api.sleep_until(deadline);
|
||||
let new_now = api.timestamp();
|
||||
|
||||
// Assert.
|
||||
// The diff could be more than the sleep duration.
|
||||
assert!(new_now.unix_millis() - 100 >= now.unix_millis());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_set_and_get_local_storage() {
|
||||
// given
|
||||
let kind = StorageKind::PERSISTENT;
|
||||
let mut api = offchain_api().0;
|
||||
let key = b"test";
|
||||
|
||||
// when
|
||||
assert_eq!(api.local_storage_get(kind, key), None);
|
||||
api.local_storage_set(kind, key, b"value");
|
||||
|
||||
// then
|
||||
assert_eq!(api.local_storage_get(kind, key), Some(b"value".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_compare_and_set_local_storage() {
|
||||
// given
|
||||
let kind = StorageKind::PERSISTENT;
|
||||
let mut api = offchain_api().0;
|
||||
let key = b"test";
|
||||
api.local_storage_set(kind, key, b"value");
|
||||
|
||||
// when
|
||||
assert_eq!(api.local_storage_compare_and_set(kind, key, Some(b"val"), b"xxx"), false);
|
||||
assert_eq!(api.local_storage_get(kind, key), Some(b"value".to_vec()));
|
||||
|
||||
// when
|
||||
assert_eq!(api.local_storage_compare_and_set(kind, key, Some(b"value"), b"xxx"), true);
|
||||
assert_eq!(api.local_storage_get(kind, key), Some(b"xxx".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_compare_and_set_local_storage_with_none() {
|
||||
// given
|
||||
let kind = StorageKind::PERSISTENT;
|
||||
let mut api = offchain_api().0;
|
||||
let key = b"test";
|
||||
|
||||
// when
|
||||
let res = api.local_storage_compare_and_set(kind, key, None, b"value");
|
||||
|
||||
// then
|
||||
assert_eq!(res, true);
|
||||
assert_eq!(api.local_storage_get(kind, key), Some(b"value".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_convert_network_states() {
|
||||
// given
|
||||
let state = NetworkState::new(
|
||||
PeerId::random(),
|
||||
vec![
|
||||
Multiaddr::try_from("/ip4/127.0.0.1/tcp/1234".to_string()).unwrap(),
|
||||
Multiaddr::try_from("/ip6/2601:9:4f81:9700:803e:ca65:66e8:c21").unwrap(),
|
||||
],
|
||||
);
|
||||
|
||||
// when
|
||||
let opaque_state = OpaqueNetworkState::from(state.clone());
|
||||
let converted_back_state = NetworkState::try_from(opaque_state).unwrap();
|
||||
|
||||
// then
|
||||
assert_eq!(state, converted_back_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_get_random_seed() {
|
||||
// given
|
||||
let mut api = offchain_api().0;
|
||||
let seed = api.random_seed();
|
||||
// then
|
||||
assert_ne!(seed, [0; 32]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,975 @@
|
||||
// Copyright 2019 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! This module is composed of two structs: [`HttpApi`] and [`HttpWorker`]. Calling the [`http`]
|
||||
//! function returns a pair of [`HttpApi`] and [`HttpWorker`] that share some state.
|
||||
//!
|
||||
//! The [`HttpApi`] is (indirectly) passed to the runtime when calling an offchain worker, while
|
||||
//! the [`HttpWorker`] must be processed in the background. The [`HttpApi`] mimicks the API of the
|
||||
//! HTTP-related methods available to offchain workers.
|
||||
//!
|
||||
//! The reason for this design is driven by the fact that HTTP requests should continue running
|
||||
//! (i.e.: the socket should continue being processed) in the background even if the runtime isn't
|
||||
//! actively calling any function.
|
||||
|
||||
use crate::api::timestamp;
|
||||
use bytes::Buf as _;
|
||||
use fnv::FnvHashMap;
|
||||
use futures::{prelude::*, channel::mpsc, compat::Compat01As03};
|
||||
use log::error;
|
||||
use primitives::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError};
|
||||
use std::{fmt, io::Read as _, mem, pin::Pin, task::Context, task::Poll};
|
||||
|
||||
/// Creates a pair of [`HttpApi`] and [`HttpWorker`].
|
||||
pub fn http() -> (HttpApi, HttpWorker) {
|
||||
let (to_worker, from_api) = mpsc::unbounded();
|
||||
let (to_api, from_worker) = mpsc::unbounded();
|
||||
|
||||
let api = HttpApi {
|
||||
to_worker,
|
||||
from_worker: from_worker.fuse(),
|
||||
// We start with a random ID for the first HTTP request, to prevent mischievous people from
|
||||
// writing runtime code with hardcoded IDs.
|
||||
next_id: HttpRequestId(rand::random::<u16>() % 2000),
|
||||
requests: FnvHashMap::default(),
|
||||
};
|
||||
|
||||
let engine = HttpWorker {
|
||||
to_api,
|
||||
from_api,
|
||||
http_client: hyper::Client::builder().build(hyper_rustls::HttpsConnector::new(1)),
|
||||
requests: Vec::new(),
|
||||
};
|
||||
|
||||
(api, engine)
|
||||
}
|
||||
|
||||
/// Provides HTTP capabilities.
|
||||
///
|
||||
/// Since this struct is a helper for offchain workers, its API is mimicking the API provided
|
||||
/// to offchain workers.
|
||||
pub struct HttpApi {
|
||||
/// Used to sends messages to the worker.
|
||||
to_worker: mpsc::UnboundedSender<ApiToWorker>,
|
||||
/// Used to receive messages from the worker.
|
||||
/// We use a `Fuse` in order to have an extra protection against panicking.
|
||||
from_worker: stream::Fuse<mpsc::UnboundedReceiver<WorkerToApi>>,
|
||||
/// Id to assign to the next HTTP request that is started.
|
||||
next_id: HttpRequestId,
|
||||
/// List of HTTP requests in preparation or in progress.
|
||||
requests: FnvHashMap<HttpRequestId, HttpApiRequest>,
|
||||
}
|
||||
|
||||
/// One active request within `HttpApi`.
|
||||
enum HttpApiRequest {
|
||||
/// The request object is being constructed locally and not started yet.
|
||||
NotDispatched(hyper::Request<hyper::Body>, hyper::body::Sender),
|
||||
/// The request has been dispatched and we're in the process of sending out the body (if the
|
||||
/// field is `Some`) or waiting for a response (if the field is `None`).
|
||||
Dispatched(Option<hyper::body::Sender>),
|
||||
/// Received a response.
|
||||
Response(HttpApiRequestRp),
|
||||
/// A request has been dispatched but the worker notified us of an error. We report this
|
||||
/// failure to the user as an `IoError` and remove the request from the list as soon as
|
||||
/// possible.
|
||||
Fail(hyper::Error),
|
||||
}
|
||||
|
||||
/// A request within `HttpApi` that has received a response.
|
||||
struct HttpApiRequestRp {
|
||||
/// We might still be writing the request's body when the response comes.
|
||||
/// This field allows to continue writing that body.
|
||||
sending_body: Option<hyper::body::Sender>,
|
||||
/// Status code of the response.
|
||||
status_code: hyper::StatusCode,
|
||||
/// Headers of the response.
|
||||
headers: hyper::HeaderMap,
|
||||
/// Body of the response, as a channel of `Chunk` objects.
|
||||
/// While the code is designed to drop the `Receiver` once it ends, we wrap it within a
|
||||
/// `Fuse` in order to be extra precautious about panics.
|
||||
/// Elements extracted from the channel are first put into `current_read_chunk`.
|
||||
/// If the channel produces an error, then that is translated into an `IoError` and the request
|
||||
/// is removed from the list.
|
||||
body: stream::Fuse<mpsc::Receiver<Result<hyper::Chunk, hyper::Error>>>,
|
||||
/// Chunk that has been extracted from the channel and that is currently being read.
|
||||
/// Reading data from the response should read from this field in priority.
|
||||
current_read_chunk: Option<bytes::Reader<hyper::Chunk>>,
|
||||
}
|
||||
|
||||
impl HttpApi {
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn request_start(
|
||||
&mut self,
|
||||
method: &str,
|
||||
uri: &str
|
||||
) -> Result<HttpRequestId, ()> {
|
||||
// Start by building the prototype of the request.
|
||||
// We do this first so that we don't touch anything in `self` if building the prototype
|
||||
// fails.
|
||||
let (body_sender, body) = hyper::Body::channel();
|
||||
let mut request = hyper::Request::new(body);
|
||||
*request.method_mut() = hyper::Method::from_bytes(method.as_bytes()).map_err(|_| ())?;
|
||||
*request.uri_mut() = hyper::Uri::from_shared(From::from(uri)).map_err(|_| ())?;
|
||||
|
||||
let new_id = self.next_id;
|
||||
debug_assert!(!self.requests.contains_key(&new_id));
|
||||
match self.next_id.0.checked_add(1) {
|
||||
Some(new_id) => self.next_id.0 = new_id,
|
||||
None => {
|
||||
error!("Overflow in offchain worker HTTP request ID assignment");
|
||||
return Err(());
|
||||
}
|
||||
};
|
||||
self.requests.insert(new_id, HttpApiRequest::NotDispatched(request, body_sender));
|
||||
|
||||
Ok(new_id)
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn request_add_header(
|
||||
&mut self,
|
||||
request_id: HttpRequestId,
|
||||
name: &str,
|
||||
value: &str
|
||||
) -> Result<(), ()> {
|
||||
let request = match self.requests.get_mut(&request_id) {
|
||||
Some(&mut HttpApiRequest::NotDispatched(ref mut rq, _)) => rq,
|
||||
_ => return Err(())
|
||||
};
|
||||
|
||||
let name = hyper::header::HeaderName::from_bytes(name.as_bytes()).map_err(|_| ())?;
|
||||
let value = hyper::header::HeaderValue::from_str(value).map_err(|_| ())?;
|
||||
// Note that we're always appending headers and never replacing old values.
|
||||
// We assume here that the user knows what they're doing.
|
||||
request.headers_mut().append(name, value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn request_write_body(
|
||||
&mut self,
|
||||
request_id: HttpRequestId,
|
||||
chunk: &[u8],
|
||||
deadline: Option<Timestamp>
|
||||
) -> Result<(), HttpError> {
|
||||
// Extract the request from the list.
|
||||
// Don't forget to add it back if necessary when returning.
|
||||
let mut request = match self.requests.remove(&request_id) {
|
||||
None => return Err(HttpError::Invalid),
|
||||
Some(r) => r,
|
||||
};
|
||||
|
||||
let mut deadline = timestamp::deadline_to_future(deadline);
|
||||
// Closure that writes data to a sender, taking the deadline into account. Can return `Ok`
|
||||
// (if the body has been written), or `DeadlineReached`, or `IoError`.
|
||||
// If `IoError` is returned, don't forget to remove the request from the list.
|
||||
let mut poll_sender = move |sender: &mut hyper::body::Sender| -> Result<(), HttpError> {
|
||||
let mut when_ready = future::maybe_done(Compat01As03::new(
|
||||
futures01::future::poll_fn(|| sender.poll_ready())
|
||||
));
|
||||
futures::executor::block_on(future::select(&mut when_ready, &mut deadline));
|
||||
match when_ready {
|
||||
future::MaybeDone::Done(Ok(())) => {}
|
||||
future::MaybeDone::Done(Err(_)) => return Err(HttpError::IoError),
|
||||
future::MaybeDone::Future(_) |
|
||||
future::MaybeDone::Gone => {
|
||||
debug_assert!(if let future::MaybeDone::Done(_) = deadline { true } else { false });
|
||||
return Err(HttpError::DeadlineReached)
|
||||
}
|
||||
};
|
||||
|
||||
match sender.send_data(hyper::Chunk::from(chunk.to_owned())) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(_chunk) => {
|
||||
error!("HTTP sender refused data despite being ready");
|
||||
Err(HttpError::IoError)
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
request = match request {
|
||||
HttpApiRequest::NotDispatched(request, sender) => {
|
||||
// If the request is not dispatched yet, dispatch it and loop again.
|
||||
let _ = self.to_worker.unbounded_send(ApiToWorker::Dispatch {
|
||||
id: request_id,
|
||||
request
|
||||
});
|
||||
HttpApiRequest::Dispatched(Some(sender))
|
||||
}
|
||||
|
||||
HttpApiRequest::Dispatched(Some(mut sender)) =>
|
||||
if !chunk.is_empty() {
|
||||
match poll_sender(&mut sender) {
|
||||
Err(HttpError::IoError) => return Err(HttpError::IoError),
|
||||
other => {
|
||||
self.requests.insert(
|
||||
request_id,
|
||||
HttpApiRequest::Dispatched(Some(sender))
|
||||
);
|
||||
return other
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Writing an empty body is a hint that we should stop writing. Dropping
|
||||
// the sender.
|
||||
self.requests.insert(request_id, HttpApiRequest::Dispatched(None));
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
HttpApiRequest::Response(mut response @ HttpApiRequestRp { sending_body: Some(_), .. }) =>
|
||||
if !chunk.is_empty() {
|
||||
match poll_sender(response.sending_body.as_mut()
|
||||
.expect("Can only enter this match branch if Some; qed")) {
|
||||
Err(HttpError::IoError) => return Err(HttpError::IoError),
|
||||
other => {
|
||||
self.requests.insert(request_id, HttpApiRequest::Response(response));
|
||||
return other
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
// Writing an empty body is a hint that we should stop writing. Dropping
|
||||
// the sender.
|
||||
self.requests.insert(request_id, HttpApiRequest::Response(HttpApiRequestRp {
|
||||
sending_body: None,
|
||||
..response
|
||||
}));
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
HttpApiRequest::Fail(_) =>
|
||||
// If the request has already failed, return without putting back the request
|
||||
// in the list.
|
||||
return Err(HttpError::IoError),
|
||||
|
||||
v @ HttpApiRequest::Dispatched(None) |
|
||||
v @ HttpApiRequest::Response(HttpApiRequestRp { sending_body: None, .. }) => {
|
||||
// We have already finished sending this body.
|
||||
self.requests.insert(request_id, v);
|
||||
return Err(HttpError::Invalid)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn response_wait(
|
||||
&mut self,
|
||||
ids: &[HttpRequestId],
|
||||
deadline: Option<Timestamp>
|
||||
) -> Vec<HttpRequestStatus> {
|
||||
// First of all, dispatch all the non-dispatched requests and drop all senders so that the
|
||||
// user can't write anymore data.
|
||||
for id in ids {
|
||||
match self.requests.get_mut(id) {
|
||||
Some(HttpApiRequest::NotDispatched(_, _)) => {}
|
||||
Some(HttpApiRequest::Dispatched(sending_body)) |
|
||||
Some(HttpApiRequest::Response(HttpApiRequestRp { sending_body, .. })) => {
|
||||
let _ = sending_body.take();
|
||||
continue
|
||||
}
|
||||
_ => continue
|
||||
};
|
||||
|
||||
let (request, _sender) = match self.requests.remove(id) {
|
||||
Some(HttpApiRequest::NotDispatched(rq, s)) => (rq, s),
|
||||
_ => unreachable!("we checked for NotDispatched above; qed")
|
||||
};
|
||||
|
||||
let _ = self.to_worker.unbounded_send(ApiToWorker::Dispatch {
|
||||
id: *id,
|
||||
request
|
||||
});
|
||||
|
||||
// We also destroy the sender in order to forbid writing more data.
|
||||
self.requests.insert(*id, HttpApiRequest::Dispatched(None));
|
||||
}
|
||||
|
||||
let mut deadline = timestamp::deadline_to_future(deadline);
|
||||
|
||||
loop {
|
||||
// Within that loop, first try to see if we have all the elements for a response.
|
||||
// This includes the situation where the deadline is reached.
|
||||
{
|
||||
let mut output = Vec::with_capacity(ids.len());
|
||||
let mut must_wait_more = false;
|
||||
for id in ids {
|
||||
output.push(match self.requests.get_mut(id) {
|
||||
None => HttpRequestStatus::Invalid,
|
||||
Some(HttpApiRequest::NotDispatched(_, _)) =>
|
||||
unreachable!("we replaced all the NotDispatched with Dispatched earlier; qed"),
|
||||
Some(HttpApiRequest::Dispatched(_)) => {
|
||||
must_wait_more = true;
|
||||
HttpRequestStatus::DeadlineReached
|
||||
},
|
||||
Some(HttpApiRequest::Fail(_)) => HttpRequestStatus::IoError,
|
||||
Some(HttpApiRequest::Response(HttpApiRequestRp { status_code, .. })) =>
|
||||
HttpRequestStatus::Finished(status_code.as_u16()),
|
||||
});
|
||||
}
|
||||
debug_assert_eq!(output.len(), ids.len());
|
||||
|
||||
// Are we ready to call `return`?
|
||||
let is_done = if let future::MaybeDone::Done(_) = deadline {
|
||||
true
|
||||
} else if !must_wait_more {
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
if is_done {
|
||||
// Requests in "fail" mode are purged before returning.
|
||||
debug_assert_eq!(output.len(), ids.len());
|
||||
for n in (0..ids.len()).rev() {
|
||||
if let HttpRequestStatus::IoError = output[n] {
|
||||
self.requests.remove(&ids[n]);
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
}
|
||||
|
||||
// Grab next message from the worker. We call `continue` if deadline is reached so that
|
||||
// we loop back and `return`.
|
||||
let next_message = {
|
||||
let mut next_msg = future::maybe_done(self.from_worker.next());
|
||||
futures::executor::block_on(future::select(&mut next_msg, &mut deadline));
|
||||
if let future::MaybeDone::Done(msg) = next_msg {
|
||||
msg
|
||||
} else {
|
||||
debug_assert!(if let future::MaybeDone::Done(_) = deadline { true } else { false });
|
||||
continue
|
||||
}
|
||||
};
|
||||
|
||||
// Update internal state based on received message.
|
||||
match next_message {
|
||||
Some(WorkerToApi::Response { id, status_code, headers, body }) =>
|
||||
match self.requests.remove(&id) {
|
||||
Some(HttpApiRequest::Dispatched(sending_body)) => {
|
||||
self.requests.insert(id, HttpApiRequest::Response(HttpApiRequestRp {
|
||||
sending_body,
|
||||
status_code,
|
||||
headers,
|
||||
body: body.fuse(),
|
||||
current_read_chunk: None,
|
||||
}));
|
||||
}
|
||||
None => {} // can happen if we detected an IO error when sending the body
|
||||
_ => error!("State mismatch between the API and worker"),
|
||||
}
|
||||
|
||||
Some(WorkerToApi::Fail { id, error }) =>
|
||||
match self.requests.remove(&id) {
|
||||
Some(HttpApiRequest::Dispatched(_)) => {
|
||||
self.requests.insert(id, HttpApiRequest::Fail(error));
|
||||
}
|
||||
None => {} // can happen if we detected an IO error when sending the body
|
||||
_ => error!("State mismatch between the API and worker"),
|
||||
}
|
||||
|
||||
None => {
|
||||
error!("Worker has crashed");
|
||||
return ids.iter().map(|_| HttpRequestStatus::IoError).collect()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn response_headers(
|
||||
&mut self,
|
||||
request_id: HttpRequestId
|
||||
) -> Vec<(Vec<u8>, Vec<u8>)> {
|
||||
// Do an implicit non-blocking wait on the request.
|
||||
let _ = self.response_wait(&[request_id], Some(timestamp::now()));
|
||||
|
||||
let headers = match self.requests.get(&request_id) {
|
||||
Some(HttpApiRequest::Response(HttpApiRequestRp { headers, .. })) => headers,
|
||||
_ => return Vec::new()
|
||||
};
|
||||
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str().as_bytes().to_owned(), value.as_bytes().to_owned()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn response_read_body(
|
||||
&mut self,
|
||||
request_id: HttpRequestId,
|
||||
buffer: &mut [u8],
|
||||
deadline: Option<Timestamp>
|
||||
) -> Result<usize, HttpError> {
|
||||
// Do an implicit wait on the request.
|
||||
let _ = self.response_wait(&[request_id], deadline);
|
||||
|
||||
// Remove the request from the list and handle situations where the request is invalid or
|
||||
// in the wrong state.
|
||||
let mut response = match self.requests.remove(&request_id) {
|
||||
Some(HttpApiRequest::Response(r)) => r,
|
||||
// Because we called `response_wait` above, we know that the deadline has been reached
|
||||
// and we still haven't received a response.
|
||||
Some(rq @ HttpApiRequest::Dispatched(_)) => {
|
||||
self.requests.insert(request_id, rq);
|
||||
return Err(HttpError::DeadlineReached)
|
||||
},
|
||||
// The request has failed.
|
||||
Some(HttpApiRequest::Fail { .. }) =>
|
||||
return Err(HttpError::IoError),
|
||||
// Request hasn't been dispatched yet; reading the body is invalid.
|
||||
Some(rq @ HttpApiRequest::NotDispatched(_, _)) => {
|
||||
self.requests.insert(request_id, rq);
|
||||
return Err(HttpError::Invalid)
|
||||
}
|
||||
None => return Err(HttpError::Invalid)
|
||||
};
|
||||
|
||||
// Convert the deadline into a `Future` that resolves when the deadline is reached.
|
||||
let mut deadline = timestamp::deadline_to_future(deadline);
|
||||
|
||||
loop {
|
||||
// First read from `current_read_chunk`.
|
||||
if let Some(mut current_read_chunk) = response.current_read_chunk.take() {
|
||||
match current_read_chunk.read(buffer) {
|
||||
Ok(0) => {}
|
||||
Ok(n) => {
|
||||
self.requests.insert(request_id, HttpApiRequest::Response(HttpApiRequestRp {
|
||||
current_read_chunk: Some(current_read_chunk),
|
||||
.. response
|
||||
}));
|
||||
return Ok(n)
|
||||
},
|
||||
Err(err) => {
|
||||
// This code should never be reached unless there's a logic error somewhere.
|
||||
error!("Failed to read from current read chunk: {:?}", err);
|
||||
return Err(HttpError::IoError)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we reach here, that means the `current_read_chunk` is empty and needs to be
|
||||
// filled with a new chunk from `body`. We block on either the next body or the
|
||||
// deadline.
|
||||
let mut next_body = future::maybe_done(response.body.next());
|
||||
futures::executor::block_on(future::select(&mut next_body, &mut deadline));
|
||||
|
||||
if let future::MaybeDone::Done(next_body) = next_body {
|
||||
match next_body {
|
||||
Some(Ok(chunk)) => response.current_read_chunk = Some(chunk.reader()),
|
||||
Some(Err(_)) => return Err(HttpError::IoError),
|
||||
None => return Ok(0), // eof
|
||||
}
|
||||
}
|
||||
|
||||
if let future::MaybeDone::Done(_) = deadline {
|
||||
self.requests.insert(request_id, HttpApiRequest::Response(response));
|
||||
return Err(HttpError::DeadlineReached)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for HttpApi {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.debug_list()
|
||||
.entries(self.requests.iter())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for HttpApiRequest {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match self {
|
||||
HttpApiRequest::NotDispatched(_, _) =>
|
||||
f.debug_tuple("HttpApiRequest::NotDispatched").finish(),
|
||||
HttpApiRequest::Dispatched(_) =>
|
||||
f.debug_tuple("HttpApiRequest::Dispatched").finish(),
|
||||
HttpApiRequest::Response(HttpApiRequestRp { status_code, headers, .. }) =>
|
||||
f.debug_tuple("HttpApiRequest::Response").field(status_code).field(headers).finish(),
|
||||
HttpApiRequest::Fail(err) =>
|
||||
f.debug_tuple("HttpApiRequest::Fail").field(err).finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Message send from the API to the worker.
|
||||
enum ApiToWorker {
|
||||
/// Dispatches a new HTTP request.
|
||||
Dispatch {
|
||||
/// ID to send back when the response comes back.
|
||||
id: HttpRequestId,
|
||||
/// Request to start executing.
|
||||
request: hyper::Request<hyper::Body>,
|
||||
}
|
||||
}
|
||||
|
||||
/// Message send from the API to the worker.
|
||||
enum WorkerToApi {
|
||||
/// A request has succeeded.
|
||||
Response {
|
||||
/// The ID that was passed to the worker.
|
||||
id: HttpRequestId,
|
||||
/// Status code of the response.
|
||||
status_code: hyper::StatusCode,
|
||||
/// Headers of the response.
|
||||
headers: hyper::HeaderMap,
|
||||
/// Body of the response, as a channel of `Chunk` objects.
|
||||
/// We send the body back through a channel instead of returning the hyper `Body` object
|
||||
/// because we don't want the `HttpApi` to have to drive the reading.
|
||||
/// Instead, reading an item from the channel will notify the worker task, which will push
|
||||
/// the next item.
|
||||
/// Can also be used to send an error, in case an error happend on the HTTP socket. After
|
||||
/// an error is sent, the channel will close.
|
||||
body: mpsc::Receiver<Result<hyper::Chunk, hyper::Error>>,
|
||||
},
|
||||
/// A request has failed because of an error. The request is then no longer valid.
|
||||
Fail {
|
||||
/// The ID that was passed to the worker.
|
||||
id: HttpRequestId,
|
||||
/// Error that happened.
|
||||
error: hyper::Error,
|
||||
},
|
||||
}
|
||||
|
||||
/// Must be continuously polled for the [`HttpApi`] to properly work.
|
||||
pub struct HttpWorker {
|
||||
/// Used to sends messages to the `HttpApi`.
|
||||
to_api: mpsc::UnboundedSender<WorkerToApi>,
|
||||
/// Used to receive messages from the `HttpApi`.
|
||||
from_api: mpsc::UnboundedReceiver<ApiToWorker>,
|
||||
/// The engine that runs HTTP requests.
|
||||
http_client: hyper::Client<hyper_rustls::HttpsConnector<hyper::client::HttpConnector>, hyper::Body>,
|
||||
/// HTTP requests that are being worked on by the engine.
|
||||
requests: Vec<(HttpRequestId, HttpWorkerRequest)>,
|
||||
}
|
||||
|
||||
/// HTTP request being processed by the worker.
|
||||
enum HttpWorkerRequest {
|
||||
/// Request has been dispatched and is waiting for a response from the Internet.
|
||||
Dispatched(Compat01As03<hyper::client::ResponseFuture>),
|
||||
/// Progressively reading the body of the response and sending it to the channel.
|
||||
ReadBody {
|
||||
/// Body to read `Chunk`s from. Only used if the channel is ready to accept data.
|
||||
body: Compat01As03<hyper::Body>,
|
||||
/// Channel to the [`HttpApi`] where we send the chunks to.
|
||||
tx: mpsc::Sender<Result<hyper::Chunk, hyper::Error>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Future for HttpWorker {
|
||||
type Output = ();
|
||||
|
||||
fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
|
||||
// Reminder: this is continuously run in the background.
|
||||
|
||||
// We use a `me` variable because the compiler isn't smart enough to allow borrowing
|
||||
// multiple fields at once through a `Deref`.
|
||||
let me = &mut *self;
|
||||
|
||||
// We remove each element from `requests` one by one and add them back only if necessary.
|
||||
for n in (0..me.requests.len()).rev() {
|
||||
let (id, request) = me.requests.swap_remove(n);
|
||||
match request {
|
||||
HttpWorkerRequest::Dispatched(mut future) => {
|
||||
// Check for an HTTP response from the Internet.
|
||||
let mut response = match Future::poll(Pin::new(&mut future), cx) {
|
||||
Poll::Pending => {
|
||||
me.requests.push((id, HttpWorkerRequest::Dispatched(future)));
|
||||
continue
|
||||
},
|
||||
Poll::Ready(Ok(response)) => response,
|
||||
Poll::Ready(Err(err)) => {
|
||||
let _ = me.to_api.unbounded_send(WorkerToApi::Fail {
|
||||
id,
|
||||
error: err,
|
||||
});
|
||||
continue; // don't insert the request back
|
||||
}
|
||||
};
|
||||
|
||||
// We received a response! Decompose it into its parts.
|
||||
let status_code = response.status();
|
||||
let headers = mem::replace(response.headers_mut(), hyper::HeaderMap::new());
|
||||
let body = Compat01As03::new(response.into_body());
|
||||
|
||||
let (body_tx, body_rx) = mpsc::channel(3);
|
||||
let _ = me.to_api.unbounded_send(WorkerToApi::Response {
|
||||
id,
|
||||
status_code,
|
||||
headers,
|
||||
body: body_rx,
|
||||
});
|
||||
|
||||
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx: body_tx }));
|
||||
cx.waker().wake_by_ref(); // reschedule in order to poll the new future
|
||||
continue
|
||||
}
|
||||
|
||||
HttpWorkerRequest::ReadBody { mut body, mut tx } => {
|
||||
// Before reading from the HTTP response, check that `tx` is ready to accept
|
||||
// a new chunk.
|
||||
match tx.poll_ready(cx) {
|
||||
Poll::Ready(Ok(())) => {}
|
||||
Poll::Ready(Err(_)) => continue, // don't insert the request back
|
||||
Poll::Pending => {
|
||||
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx }));
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// `tx` is ready. Read a chunk from the socket and send it to the channel.
|
||||
match Stream::poll_next(Pin::new(&mut body), cx) {
|
||||
Poll::Ready(Some(Ok(chunk))) => {
|
||||
let _ = tx.start_send(Ok(chunk));
|
||||
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx }));
|
||||
cx.waker().wake_by_ref(); // reschedule in order to continue reading
|
||||
}
|
||||
Poll::Ready(Some(Err(err))) => {
|
||||
let _ = tx.start_send(Err(err));
|
||||
// don't insert the request back
|
||||
},
|
||||
Poll::Ready(None) => {} // EOF; don't insert the request back
|
||||
Poll::Pending => {
|
||||
me.requests.push((id, HttpWorkerRequest::ReadBody { body, tx }));
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for messages coming from the [`HttpApi`].
|
||||
match Stream::poll_next(Pin::new(&mut me.from_api), cx) {
|
||||
Poll::Pending => {},
|
||||
Poll::Ready(None) => return Poll::Ready(()), // stops the worker
|
||||
Poll::Ready(Some(ApiToWorker::Dispatch { id, request })) => {
|
||||
let future = Compat01As03::new(me.http_client.request(request));
|
||||
debug_assert!(me.requests.iter().all(|(i, _)| *i != id));
|
||||
me.requests.push((id, HttpWorkerRequest::Dispatched(future)));
|
||||
cx.waker().wake_by_ref(); // reschedule the task to poll the request
|
||||
}
|
||||
}
|
||||
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for HttpWorker {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.debug_list()
|
||||
.entries(self.requests.iter())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for HttpWorkerRequest {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match self {
|
||||
HttpWorkerRequest::Dispatched(_) =>
|
||||
f.debug_tuple("HttpWorkerRequest::Dispatched").finish(),
|
||||
HttpWorkerRequest::ReadBody { .. } =>
|
||||
f.debug_tuple("HttpWorkerRequest::Response").finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::api::timestamp;
|
||||
use super::http;
|
||||
use futures::prelude::*;
|
||||
use futures01::Future as _;
|
||||
use primitives::offchain::{HttpError, HttpRequestId, HttpRequestStatus, Duration};
|
||||
|
||||
// Returns an `HttpApi` whose worker is ran in the background, and a `SocketAddr` to an HTTP
|
||||
// server that runs in the background as well.
|
||||
macro_rules! build_api_server {
|
||||
() => {{
|
||||
let (api, worker) = http();
|
||||
// Note: we have to use tokio because hyper still uses old futures.
|
||||
std::thread::spawn(move || {
|
||||
tokio::run(futures::compat::Compat::new(worker.map(|()| Ok::<(), ()>(()))))
|
||||
});
|
||||
let (addr_tx, addr_rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap())
|
||||
.serve(|| {
|
||||
hyper::service::service_fn_ok(move |_: hyper::Request<hyper::Body>| {
|
||||
hyper::Response::new(hyper::Body::from("Hello World!"))
|
||||
})
|
||||
});
|
||||
let _ = addr_tx.send(server.local_addr());
|
||||
hyper::rt::run(server.map_err(|e| panic!("{:?}", e)));
|
||||
});
|
||||
(api, addr_rx.recv().unwrap())
|
||||
}};
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn basic_localhost() {
|
||||
let deadline = timestamp::now().add(Duration::from_millis(10_000));
|
||||
|
||||
// Performs an HTTP query to a background HTTP server.
|
||||
|
||||
let (mut api, addr) = build_api_server!();
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_write_body(id, &[], Some(deadline)).unwrap();
|
||||
|
||||
match api.response_wait(&[id], Some(deadline))[0] {
|
||||
HttpRequestStatus::Finished(200) => {},
|
||||
v => panic!("Connecting to localhost failed: {:?}", v)
|
||||
}
|
||||
|
||||
let headers = api.response_headers(id);
|
||||
assert!(headers.iter().any(|(h, _)| h.eq_ignore_ascii_case(b"Date")));
|
||||
|
||||
let mut buf = vec![0; 2048];
|
||||
let n = api.response_read_body(id, &mut buf, Some(deadline)).unwrap();
|
||||
assert_eq!(&buf[..n], b"Hello World!");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_start_invalid_call() {
|
||||
let (mut api, addr) = build_api_server!();
|
||||
|
||||
match api.request_start("\0", &format!("http://{}", addr)) {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
|
||||
match api.request_start("GET", "http://\0localhost") {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_add_header_invalid_call() {
|
||||
let (mut api, addr) = build_api_server!();
|
||||
|
||||
match api.request_add_header(HttpRequestId(0xdead), "Foo", "bar") {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
match api.request_add_header(id, "\0", "bar") {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
match api.request_add_header(id, "Foo", "\0") {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_add_header(id, "Foo", "Bar").unwrap();
|
||||
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
|
||||
match api.request_add_header(id, "Foo2", "Bar") {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
api.response_headers(id);
|
||||
match api.request_add_header(id, "Foo2", "Bar") {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
api.response_read_body(id, &mut [], None).unwrap();
|
||||
match api.request_add_header(id, "Foo2", "Bar") {
|
||||
Err(()) => {}
|
||||
Ok(_) => panic!()
|
||||
};
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_write_body_invalid_call() {
|
||||
let (mut api, addr) = build_api_server!();
|
||||
|
||||
match api.request_write_body(HttpRequestId(0xdead), &[1, 2, 3], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
match api.request_write_body(HttpRequestId(0xdead), &[], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
|
||||
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
|
||||
api.request_write_body(id, &[], None).unwrap();
|
||||
match api.request_write_body(id, &[], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
|
||||
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
|
||||
api.request_write_body(id, &[], None).unwrap();
|
||||
match api.request_write_body(id, &[1, 2, 3, 4], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
|
||||
api.response_wait(&[id], None);
|
||||
match api.request_write_body(id, &[], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_write_body(id, &[1, 2, 3, 4], None).unwrap();
|
||||
api.response_wait(&[id], None);
|
||||
match api.request_write_body(id, &[1, 2, 3, 4], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.response_headers(id);
|
||||
match api.request_write_body(id, &[1, 2, 3, 4], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
api.response_headers(id);
|
||||
match api.request_write_body(id, &[], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.response_read_body(id, &mut [], None).unwrap();
|
||||
match api.request_write_body(id, &[1, 2, 3, 4], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.response_read_body(id, &mut [], None).unwrap();
|
||||
match api.request_write_body(id, &[], None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
};
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_headers_invalid_call() {
|
||||
let (mut api, addr) = build_api_server!();
|
||||
assert!(api.response_headers(HttpRequestId(0xdead)).is_empty());
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
assert!(api.response_headers(id).is_empty());
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_write_body(id, &[], None).unwrap();
|
||||
while api.response_headers(id).is_empty() {
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
api.response_wait(&[id], None);
|
||||
assert!(!api.response_headers(id).is_empty());
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
let mut buf = [0; 128];
|
||||
while api.response_read_body(id, &mut buf, None).unwrap() != 0 {}
|
||||
assert!(api.response_headers(id).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_header_invalid_call() {
|
||||
let (mut api, addr) = build_api_server!();
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
assert!(api.response_headers(id).is_empty());
|
||||
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
api.request_add_header(id, "Foo", "Bar").unwrap();
|
||||
assert!(api.response_headers(id).is_empty());
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
api.request_add_header(id, "Foo", "Bar").unwrap();
|
||||
api.request_write_body(id, &[], None).unwrap();
|
||||
// Note: this test actually sends out the request, and is supposed to test a situation
|
||||
// where we haven't received any response yet. This test can theoretically fail if the
|
||||
// HTTP response comes back faster than the kernel schedules our thread, but that is highly
|
||||
// unlikely.
|
||||
assert!(api.response_headers(id).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_read_body_invalid_call() {
|
||||
let (mut api, addr) = build_api_server!();
|
||||
let mut buf = [0; 512];
|
||||
|
||||
match api.response_read_body(HttpRequestId(0xdead), &mut buf, None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
}
|
||||
|
||||
let id = api.request_start("GET", &format!("http://{}", addr)).unwrap();
|
||||
while api.response_read_body(id, &mut buf, None).unwrap() != 0 {}
|
||||
match api.response_read_body(id, &mut buf, None) {
|
||||
Err(HttpError::Invalid) => {}
|
||||
_ => panic!()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzzing() {
|
||||
// Uses the API in random ways to try to trigger panicks.
|
||||
// Doesn't test some paths, such as waiting for multiple requests. Also doesn't test what
|
||||
// happens if the server force-closes our socket.
|
||||
|
||||
let (mut api, addr) = build_api_server!();
|
||||
|
||||
for _ in 0..50 {
|
||||
let id = api.request_start("POST", &format!("http://{}", addr)).unwrap();
|
||||
|
||||
for _ in 0..250 {
|
||||
match rand::random::<u8>() % 6 {
|
||||
0 => { let _ = api.request_add_header(id, "Foo", "Bar"); }
|
||||
1 => { let _ = api.request_write_body(id, &[1, 2, 3, 4], None); }
|
||||
2 => { let _ = api.request_write_body(id, &[], None); }
|
||||
3 => { let _ = api.response_wait(&[id], None); }
|
||||
4 => { let _ = api.response_headers(id); }
|
||||
5 => {
|
||||
let mut buf = [0; 512];
|
||||
let _ = api.response_read_body(id, &mut buf, None);
|
||||
}
|
||||
6 ..= 255 => unreachable!()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2019 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Contains the same API as the `http` module, except that everything returns an error.
|
||||
|
||||
use primitives::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError};
|
||||
use std::{future::Future, pin::Pin, task::Context, task::Poll};
|
||||
|
||||
/// Creates a pair of [`HttpApi`] and [`HttpWorker`].
|
||||
pub fn http() -> (HttpApi, HttpWorker) {
|
||||
(HttpApi, HttpWorker)
|
||||
}
|
||||
|
||||
/// Dummy implementation of HTTP capabilities.
|
||||
#[derive(Debug)]
|
||||
pub struct HttpApi;
|
||||
|
||||
/// Dummy implementation of HTTP capabilities.
|
||||
#[derive(Debug)]
|
||||
pub struct HttpWorker;
|
||||
|
||||
impl HttpApi {
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn request_start(
|
||||
&mut self,
|
||||
_: &str,
|
||||
_: &str
|
||||
) -> Result<HttpRequestId, ()> {
|
||||
/// Because this always returns an error, none of the other methods should ever be called.
|
||||
Err(())
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn request_add_header(
|
||||
&mut self,
|
||||
_: HttpRequestId,
|
||||
_: &str,
|
||||
_: &str
|
||||
) -> Result<(), ()> {
|
||||
unreachable!("Creating a request always fails, thus this function will \
|
||||
never be called; qed")
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn request_write_body(
|
||||
&mut self,
|
||||
_: HttpRequestId,
|
||||
_: &[u8],
|
||||
_: Option<Timestamp>
|
||||
) -> Result<(), HttpError> {
|
||||
unreachable!("Creating a request always fails, thus this function will \
|
||||
never be called; qed")
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn response_wait(
|
||||
&mut self,
|
||||
requests: &[HttpRequestId],
|
||||
_: Option<Timestamp>
|
||||
) -> Vec<HttpRequestStatus> {
|
||||
if requests.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
unreachable!("Creating a request always fails, thus the list of requests should \
|
||||
always be empty; qed")
|
||||
}
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn response_headers(
|
||||
&mut self,
|
||||
_: HttpRequestId
|
||||
) -> Vec<(Vec<u8>, Vec<u8>)> {
|
||||
unreachable!("Creating a request always fails, thus this function will \
|
||||
never be called; qed")
|
||||
}
|
||||
|
||||
/// Mimicks the corresponding method in the offchain API.
|
||||
pub fn response_read_body(
|
||||
&mut self,
|
||||
_: HttpRequestId,
|
||||
_: &mut [u8],
|
||||
_: Option<Timestamp>
|
||||
) -> Result<usize, HttpError> {
|
||||
unreachable!("Creating a request always fails, thus this function will \
|
||||
never be called; qed")
|
||||
}
|
||||
}
|
||||
|
||||
impl Future for HttpWorker {
|
||||
type Output = ();
|
||||
|
||||
fn poll(self: Pin<&mut Self>, _: &mut Context) -> Poll<Self::Output> {
|
||||
Poll::Ready(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2019 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Helper methods dedicated to timestamps.
|
||||
|
||||
use primitives::offchain::Timestamp;
|
||||
use std::convert::TryInto;
|
||||
use std::time::{SystemTime, Duration};
|
||||
|
||||
/// Returns the current time as a `Timestamp`.
|
||||
pub fn now() -> Timestamp {
|
||||
let now = SystemTime::now();
|
||||
let epoch_duration = now.duration_since(SystemTime::UNIX_EPOCH);
|
||||
match epoch_duration {
|
||||
Err(_) => {
|
||||
// Current time is earlier than UNIX_EPOCH.
|
||||
Timestamp::from_unix_millis(0)
|
||||
},
|
||||
Ok(d) => {
|
||||
let duration = d.as_millis();
|
||||
// Assuming overflow won't happen for a few hundred years.
|
||||
Timestamp::from_unix_millis(duration.try_into()
|
||||
.expect("epoch milliseconds won't overflow u64 for hundreds of years; qed"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns how a `Timestamp` compares to "now".
|
||||
///
|
||||
/// In other words, returns `timestamp - now()`.
|
||||
pub fn timestamp_from_now(timestamp: Timestamp) -> Duration {
|
||||
Duration::from_millis(timestamp.diff(&now()).millis())
|
||||
}
|
||||
|
||||
/// Converts the deadline into a `Future` that resolves when the deadline is reached.
|
||||
///
|
||||
/// If `None`, returns a never-ending `Future`.
|
||||
pub fn deadline_to_future(
|
||||
deadline: Option<Timestamp>,
|
||||
) -> futures::future::MaybeDone<impl futures::Future> {
|
||||
use futures::future;
|
||||
|
||||
future::maybe_done(match deadline {
|
||||
Some(deadline) => future::Either::Left(
|
||||
futures_timer::Delay::new(timestamp_from_now(deadline))
|
||||
),
|
||||
None => future::Either::Right(future::pending())
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
// Copyright 2019 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Substrate offchain workers.
|
||||
//!
|
||||
//! The offchain workers is a special function of the runtime that
|
||||
//! gets executed after block is imported. During execution
|
||||
//! it's able to asynchronously submit extrinsics that will either
|
||||
//! be propagated to other nodes added to the next block
|
||||
//! produced by the node as unsigned transactions.
|
||||
//!
|
||||
//! Offchain workers can be used for computation-heavy tasks
|
||||
//! that are not feasible for execution during regular block processing.
|
||||
//! It can either be tasks that no consensus is required for,
|
||||
//! or some form of consensus over the data can be built on-chain
|
||||
//! for instance via:
|
||||
//! 1. Challenge period for incorrect computations
|
||||
//! 2. Majority voting for results
|
||||
//! 3. etc
|
||||
|
||||
#![warn(missing_docs)]
|
||||
|
||||
use std::{fmt, marker::PhantomData, sync::Arc};
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use threadpool::ThreadPool;
|
||||
use sr_api::ApiExt;
|
||||
use futures::future::Future;
|
||||
use log::{debug, warn};
|
||||
use network::NetworkStateInfo;
|
||||
use primitives::{offchain, ExecutionContext};
|
||||
use sr_primitives::{generic::BlockId, traits::{self, ProvideRuntimeApi}};
|
||||
use transaction_pool::txpool::{Pool, ChainApi};
|
||||
use client_api::{OffchainStorage};
|
||||
|
||||
mod api;
|
||||
|
||||
pub mod testing;
|
||||
|
||||
pub use offchain_primitives::{OffchainWorkerApi, STORAGE_PREFIX};
|
||||
|
||||
/// An offchain workers manager.
|
||||
pub struct OffchainWorkers<Client, Storage, Block: traits::Block> {
|
||||
client: Arc<Client>,
|
||||
db: Storage,
|
||||
_block: PhantomData<Block>,
|
||||
thread_pool: Mutex<ThreadPool>,
|
||||
}
|
||||
|
||||
impl<Client, Storage, Block: traits::Block> OffchainWorkers<Client, Storage, Block> {
|
||||
/// Creates new `OffchainWorkers`.
|
||||
pub fn new(client: Arc<Client>, db: Storage) -> Self {
|
||||
Self {
|
||||
client,
|
||||
db,
|
||||
_block: PhantomData,
|
||||
thread_pool: Mutex::new(ThreadPool::new(num_cpus::get())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<Client, Storage, Block: traits::Block> fmt::Debug for OffchainWorkers<
|
||||
Client,
|
||||
Storage,
|
||||
Block,
|
||||
> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.debug_tuple("OffchainWorkers").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<Client, Storage, Block> OffchainWorkers<
|
||||
Client,
|
||||
Storage,
|
||||
Block,
|
||||
> where
|
||||
Block: traits::Block,
|
||||
Client: ProvideRuntimeApi + Send + Sync + 'static,
|
||||
Client::Api: OffchainWorkerApi<Block>,
|
||||
Storage: OffchainStorage + 'static,
|
||||
{
|
||||
/// Start the offchain workers after given block.
|
||||
#[must_use]
|
||||
pub fn on_block_imported<A>(
|
||||
&self,
|
||||
number: &<Block::Header as traits::Header>::Number,
|
||||
pool: &Arc<Pool<A>>,
|
||||
network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
|
||||
is_validator: bool,
|
||||
) -> impl Future<Output = ()> where A: ChainApi<Block=Block> + 'static {
|
||||
let runtime = self.client.runtime_api();
|
||||
let at = BlockId::number(*number);
|
||||
let has_api = runtime.has_api::<dyn OffchainWorkerApi<Block, Error = ()>>(&at);
|
||||
debug!("Checking offchain workers at {:?}: {:?}", at, has_api);
|
||||
|
||||
if has_api.unwrap_or(false) {
|
||||
let (api, runner) = api::AsyncApi::new(
|
||||
pool.clone(),
|
||||
self.db.clone(),
|
||||
at.clone(),
|
||||
network_state.clone(),
|
||||
is_validator,
|
||||
);
|
||||
debug!("Spawning offchain workers at {:?}", at);
|
||||
let number = *number;
|
||||
let client = self.client.clone();
|
||||
self.spawn_worker(move || {
|
||||
let runtime = client.runtime_api();
|
||||
let api = Box::new(api);
|
||||
debug!("Running offchain workers at {:?}", at);
|
||||
let run = runtime.offchain_worker_with_context(
|
||||
&at,
|
||||
ExecutionContext::OffchainCall(Some((api, offchain::Capabilities::all()))),
|
||||
number,
|
||||
);
|
||||
if let Err(e) = run {
|
||||
log::error!("Error running offchain workers at {:?}: {:?}", at, e);
|
||||
}
|
||||
});
|
||||
futures::future::Either::Left(runner.process())
|
||||
} else {
|
||||
futures::future::Either::Right(futures::future::ready(()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns a new offchain worker.
|
||||
///
|
||||
/// We spawn offchain workers for each block in a separate thread,
|
||||
/// since they can run for a significant amount of time
|
||||
/// in a blocking fashion and we don't want to block the runtime.
|
||||
///
|
||||
/// Note that we should avoid that if we switch to future-based runtime in the future,
|
||||
/// alternatively:
|
||||
fn spawn_worker(&self, f: impl FnOnce() -> () + Send + 'static) {
|
||||
self.thread_pool.lock().execute(f);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use network::{Multiaddr, PeerId};
|
||||
|
||||
struct MockNetworkStateInfo();
|
||||
|
||||
impl NetworkStateInfo for MockNetworkStateInfo {
|
||||
fn external_addresses(&self) -> Vec<Multiaddr> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn peer_id(&self) -> PeerId {
|
||||
PeerId::random()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_call_into_runtime_and_produce_extrinsic() {
|
||||
// given
|
||||
let _ = env_logger::try_init();
|
||||
let client = Arc::new(test_client::new());
|
||||
let pool = Arc::new(Pool::new(Default::default(), transaction_pool::FullChainApi::new(client.clone())));
|
||||
let db = client_db::offchain::LocalStorage::new_test();
|
||||
let network_state = Arc::new(MockNetworkStateInfo());
|
||||
|
||||
// when
|
||||
let offchain = OffchainWorkers::new(client, db);
|
||||
futures::executor::block_on(offchain.on_block_imported(&0u64, &pool, network_state, false));
|
||||
|
||||
// then
|
||||
assert_eq!(pool.status().ready, 1);
|
||||
assert_eq!(pool.ready().next().unwrap().is_propagateable(), false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
// Copyright 2019 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Offchain Externalities implementation for tests.
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::Arc,
|
||||
};
|
||||
use client_api::{OffchainStorage, InMemOffchainStorage};
|
||||
use parking_lot::RwLock;
|
||||
use primitives::offchain::{
|
||||
self,
|
||||
HttpError,
|
||||
HttpRequestId as RequestId,
|
||||
HttpRequestStatus as RequestStatus,
|
||||
Timestamp,
|
||||
StorageKind,
|
||||
OpaqueNetworkState,
|
||||
};
|
||||
|
||||
/// Pending request.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct PendingRequest {
|
||||
/// HTTP method
|
||||
pub method: String,
|
||||
/// URI
|
||||
pub uri: String,
|
||||
/// Encoded Metadata
|
||||
pub meta: Vec<u8>,
|
||||
/// Request headers
|
||||
pub headers: Vec<(String, String)>,
|
||||
/// Request body
|
||||
pub body: Vec<u8>,
|
||||
/// Has the request been sent already.
|
||||
pub sent: bool,
|
||||
/// Response body
|
||||
pub response: Option<Vec<u8>>,
|
||||
/// Number of bytes already read from the response body.
|
||||
pub read: usize,
|
||||
/// Response headers
|
||||
pub response_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Internal state of the externalities.
|
||||
///
|
||||
/// This can be used in tests to respond or assert stuff about interactions.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct State {
|
||||
/// A list of pending requests.
|
||||
pub requests: BTreeMap<RequestId, PendingRequest>,
|
||||
expected_requests: BTreeMap<RequestId, PendingRequest>,
|
||||
/// Persistent local storage
|
||||
pub persistent_storage: InMemOffchainStorage,
|
||||
/// Local storage
|
||||
pub local_storage: InMemOffchainStorage,
|
||||
/// A vector of transactions submitted from the runtime.
|
||||
pub transactions: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl State {
|
||||
/// Asserts that pending request has been submitted and fills it's response.
|
||||
pub fn fulfill_pending_request(
|
||||
&mut self,
|
||||
id: u16,
|
||||
expected: PendingRequest,
|
||||
response: impl Into<Vec<u8>>,
|
||||
response_headers: impl IntoIterator<Item=(String, String)>,
|
||||
) {
|
||||
match self.requests.get_mut(&RequestId(id)) {
|
||||
None => {
|
||||
panic!("Missing pending request: {:?}.\n\nAll: {:?}", id, self.requests);
|
||||
}
|
||||
Some(req) => {
|
||||
assert_eq!(
|
||||
*req,
|
||||
expected,
|
||||
);
|
||||
req.response = Some(response.into());
|
||||
req.response_headers = response_headers.into_iter().collect();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fulfill_expected(&mut self, id: u16) {
|
||||
if let Some(mut req) = self.expected_requests.remove(&RequestId(id)) {
|
||||
let response = req.response.take().expect("Response checked while added.");
|
||||
let headers = std::mem::replace(&mut req.response_headers, vec![]);
|
||||
self.fulfill_pending_request(id, req, response, headers);
|
||||
}
|
||||
}
|
||||
|
||||
/// Add expected HTTP request.
|
||||
///
|
||||
/// This method can be used to initialize expected HTTP requests and their responses
|
||||
/// before running the actual code that utilizes them (for instance before calling into runtime).
|
||||
/// Expected request has to be fulfilled before this struct is dropped,
|
||||
/// the `response` and `response_headers` fields will be used to return results to the callers.
|
||||
pub fn expect_request(&mut self, id: u16, expected: PendingRequest) {
|
||||
if expected.response.is_none() {
|
||||
panic!("Expected request needs to have a response.");
|
||||
}
|
||||
self.expected_requests.insert(RequestId(id), expected);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for State {
|
||||
fn drop(&mut self) {
|
||||
// If we panic! while we are already in a panic, the test dies with an illegal instruction.
|
||||
if !self.expected_requests.is_empty() && !std::thread::panicking() {
|
||||
panic!("Unfulfilled expected requests: {:?}", self.expected_requests);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Implementation of offchain externalities used for tests.
|
||||
#[derive(Clone, Default, Debug)]
|
||||
pub struct TestOffchainExt(pub Arc<RwLock<State>>);
|
||||
|
||||
impl TestOffchainExt {
|
||||
/// Create new `TestOffchainExt` and a reference to the internal state.
|
||||
pub fn new() -> (Self, Arc<RwLock<State>>) {
|
||||
let ext = Self::default();
|
||||
let state = ext.0.clone();
|
||||
(ext, state)
|
||||
}
|
||||
}
|
||||
|
||||
impl offchain::Externalities for TestOffchainExt {
|
||||
fn is_validator(&self) -> bool {
|
||||
unimplemented!("not needed in tests so far")
|
||||
}
|
||||
|
||||
fn submit_transaction(&mut self, ex: Vec<u8>) -> Result<(), ()> {
|
||||
let mut state = self.0.write();
|
||||
state.transactions.push(ex);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
|
||||
Ok(OpaqueNetworkState {
|
||||
peer_id: Default::default(),
|
||||
external_addresses: vec![],
|
||||
})
|
||||
}
|
||||
|
||||
fn timestamp(&mut self) -> Timestamp {
|
||||
unimplemented!("not needed in tests so far")
|
||||
}
|
||||
|
||||
fn sleep_until(&mut self, _deadline: Timestamp) {
|
||||
unimplemented!("not needed in tests so far")
|
||||
}
|
||||
|
||||
fn random_seed(&mut self) -> [u8; 32] {
|
||||
unimplemented!("not needed in tests so far")
|
||||
}
|
||||
|
||||
fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
|
||||
let mut state = self.0.write();
|
||||
match kind {
|
||||
StorageKind::LOCAL => &mut state.local_storage,
|
||||
StorageKind::PERSISTENT => &mut state.persistent_storage,
|
||||
}.set(b"", key, value);
|
||||
}
|
||||
|
||||
fn local_storage_compare_and_set(
|
||||
&mut self,
|
||||
kind: StorageKind,
|
||||
key: &[u8],
|
||||
old_value: Option<&[u8]>,
|
||||
new_value: &[u8]
|
||||
) -> bool {
|
||||
let mut state = self.0.write();
|
||||
match kind {
|
||||
StorageKind::LOCAL => &mut state.local_storage,
|
||||
StorageKind::PERSISTENT => &mut state.persistent_storage,
|
||||
}.compare_and_set(b"", key, old_value, new_value)
|
||||
}
|
||||
|
||||
fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
|
||||
let state = self.0.read();
|
||||
match kind {
|
||||
StorageKind::LOCAL => &state.local_storage,
|
||||
StorageKind::PERSISTENT => &state.persistent_storage,
|
||||
}.get(b"", key)
|
||||
}
|
||||
|
||||
fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<RequestId, ()> {
|
||||
let mut state = self.0.write();
|
||||
let id = RequestId(state.requests.len() as u16);
|
||||
state.requests.insert(id.clone(), PendingRequest {
|
||||
method: method.into(),
|
||||
uri: uri.into(),
|
||||
meta: meta.into(),
|
||||
..Default::default()
|
||||
});
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
fn http_request_add_header(
|
||||
&mut self,
|
||||
request_id: RequestId,
|
||||
name: &str,
|
||||
value: &str,
|
||||
) -> Result<(), ()> {
|
||||
let mut state = self.0.write();
|
||||
if let Some(req) = state.requests.get_mut(&request_id) {
|
||||
req.headers.push((name.into(), value.into()));
|
||||
Ok(())
|
||||
} else {
|
||||
Err(())
|
||||
}
|
||||
}
|
||||
|
||||
fn http_request_write_body(
|
||||
&mut self,
|
||||
request_id: RequestId,
|
||||
chunk: &[u8],
|
||||
_deadline: Option<Timestamp>
|
||||
) -> Result<(), HttpError> {
|
||||
let mut state = self.0.write();
|
||||
|
||||
let sent = {
|
||||
let req = state.requests.get_mut(&request_id).ok_or(HttpError::IoError)?;
|
||||
req.body.extend(chunk);
|
||||
if chunk.is_empty() {
|
||||
req.sent = true;
|
||||
}
|
||||
req.sent
|
||||
};
|
||||
|
||||
if sent {
|
||||
state.fulfill_expected(request_id.0);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn http_response_wait(
|
||||
&mut self,
|
||||
ids: &[RequestId],
|
||||
_deadline: Option<Timestamp>,
|
||||
) -> Vec<RequestStatus> {
|
||||
let state = self.0.read();
|
||||
|
||||
ids.iter().map(|id| match state.requests.get(id) {
|
||||
Some(req) if req.response.is_none() =>
|
||||
panic!("No `response` provided for request with id: {:?}", id),
|
||||
None => RequestStatus::Invalid,
|
||||
_ => RequestStatus::Finished(200),
|
||||
}).collect()
|
||||
}
|
||||
|
||||
fn http_response_headers(&mut self, request_id: RequestId) -> Vec<(Vec<u8>, Vec<u8>)> {
|
||||
let state = self.0.read();
|
||||
if let Some(req) = state.requests.get(&request_id) {
|
||||
req.response_headers
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k.into_bytes(), v.into_bytes()))
|
||||
.collect()
|
||||
} else {
|
||||
Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn http_response_read_body(
|
||||
&mut self,
|
||||
request_id: RequestId,
|
||||
buffer: &mut [u8],
|
||||
_deadline: Option<Timestamp>
|
||||
) -> Result<usize, HttpError> {
|
||||
let mut state = self.0.write();
|
||||
if let Some(req) = state.requests.get_mut(&request_id) {
|
||||
let response = req.response
|
||||
.as_mut()
|
||||
.expect(&format!("No response provided for request: {:?}", request_id));
|
||||
|
||||
if req.read >= response.len() {
|
||||
// Remove the pending request as per spec.
|
||||
state.requests.remove(&request_id);
|
||||
Ok(0)
|
||||
} else {
|
||||
let read = std::cmp::min(buffer.len(), response[req.read..].len());
|
||||
buffer[0..read].copy_from_slice(&response[req.read..read]);
|
||||
req.read += read;
|
||||
Ok(read)
|
||||
}
|
||||
} else {
|
||||
Err(HttpError::IoError)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user