Integrate litep2p into Polkadot SDK (#2944)

[litep2p](https://github.com/altonen/litep2p) is a libp2p-compatible P2P
networking library. It supports all of the features of `rust-libp2p`
that are currently being utilized by Polkadot SDK.

Compared to `rust-libp2p`, `litep2p` has a quite different architecture
which is why the new `litep2p` network backend is only able to use a
little of the existing code in `sc-network`. The design has been mainly
influenced by how we'd wish to structure our networking-related code in
Polkadot SDK: independent higher-levels protocols directly communicating
with the network over links that support bidirectional backpressure. A
good example would be `NotificationHandle`/`RequestResponseHandle`
abstractions which allow, e.g., `SyncingEngine` to directly communicate
with peers to announce/request blocks.

I've tried running `polkadot --network-backend litep2p` with a few
different peer configurations and there is a noticeable reduction in
networking CPU usage. For high load (`--out-peers 200`), networking CPU
usage goes down from ~110% to ~30% (80 pp) and for normal load
(`--out-peers 40`), the usage goes down from ~55% to ~18% (37 pp).

These should not be taken as final numbers because:

a) there are still some low-hanging optimization fruits, such as
enabling [receive window
auto-tuning](https://github.com/libp2p/rust-yamux/pull/176), integrating
`Peerset` more closely with `litep2p` or improving memory usage of the
WebSocket transport
b) fixing bugs/instabilities that incorrectly cause `litep2p` to do less
work will increase the networking CPU usage
c) verification in a more diverse set of tests/conditions is needed

Nevertheless, these numbers should give an early estimate for CPU usage
of the new networking backend.

This PR consists of three separate changes:
* introduce a generic `PeerId` (wrapper around `Multihash`) so that we
don't have use `NetworkService::PeerId` in every part of the code that
uses a `PeerId`
* introduce `NetworkBackend` trait, implement it for the libp2p network
stack and make Polkadot SDK generic over `NetworkBackend`
  * implement `NetworkBackend` for litep2p

The new library should be considered experimental which is why
`rust-libp2p` will remain as the default option for the time being. This
PR currently depends on the master branch of `litep2p` but I'll cut a
new release for the library once all review comments have been
addresses.

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
Co-authored-by: Alexandru Vasile <alexandru.vasile@parity.io>
This commit is contained in:
Aaro Altonen
2024-04-08 19:44:13 +03:00
committed by GitHub
parent 9543d31474
commit 80616f6d03
181 changed files with 11055 additions and 1862 deletions
@@ -41,19 +41,22 @@
//! Even though this does not guarantee that `ProtocolController` and `Notifications` have the same
//! view of the peers' states at any given moment, the eventual consistency is maintained.
use crate::{
peer_store::{PeerStoreProvider, ProtocolHandle as ProtocolHandleT},
PeerId,
};
use futures::{channel::oneshot, future::Either, FutureExt, StreamExt};
use libp2p::PeerId;
use log::{debug, error, trace, warn};
use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
use sp_arithmetic::traits::SaturatedConversion;
use std::{
collections::{HashMap, HashSet},
sync::Arc,
time::{Duration, Instant},
};
use wasm_timer::Delay;
use crate::peer_store::PeerStoreProvider;
/// Log target for this file.
pub const LOG_TARGET: &str = "peerset";
@@ -230,6 +233,12 @@ impl ProtocolHandle {
}
}
impl ProtocolHandleT for ProtocolHandle {
fn disconnect_peer(&self, peer_id: sc_network_types::PeerId) {
let _ = self.actions_tx.unbounded_send(Action::DisconnectPeer(peer_id.into()));
}
}
/// Direction of a connection
#[derive(Clone, Copy, Debug)]
enum Direction {
@@ -289,7 +298,7 @@ pub struct ProtocolController {
to_notifications: TracingUnboundedSender<Message>,
/// `PeerStore` handle for checking peer reputation values and getting connection candidates
/// with highest reputation.
peer_store: Box<dyn PeerStoreProvider>,
peer_store: Arc<dyn PeerStoreProvider>,
}
impl ProtocolController {
@@ -298,12 +307,12 @@ impl ProtocolController {
set_id: SetId,
config: ProtoSetConfig,
to_notifications: TracingUnboundedSender<Message>,
peer_store: Box<dyn PeerStoreProvider>,
peer_store: Arc<dyn PeerStoreProvider>,
) -> (ProtocolHandle, ProtocolController) {
let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000);
let (events_tx, events_rx) = tracing_unbounded("mpsc_notifications_protocol", 10_000);
let handle = ProtocolHandle { actions_tx, events_tx };
peer_store.register_protocol(handle.clone());
peer_store.register_protocol(Arc::new(handle.clone()));
let reserved_nodes =
config.reserved_nodes.iter().map(|p| (*p, PeerState::NotConnected)).collect();
let controller = ProtocolController {
@@ -445,12 +454,12 @@ impl ProtocolController {
/// Report peer disconnect event to `PeerStore` for it to update peer's reputation accordingly.
/// Should only be called if the remote node disconnected us, not the other way around.
fn report_disconnect(&mut self, peer_id: PeerId) {
self.peer_store.report_disconnect(peer_id);
self.peer_store.report_disconnect(peer_id.into());
}
/// Ask `Peerset` if the peer has a reputation value not sufficient for connection with it.
fn is_banned(&self, peer_id: &PeerId) -> bool {
self.peer_store.is_banned(peer_id)
self.peer_store.is_banned(&peer_id.into())
}
/// Add the peer to the set of reserved peers. [`ProtocolController`] will try to always
@@ -665,7 +674,7 @@ impl ProtocolController {
self.accept_connection(peer_id, incoming_index);
},
PeerState::NotConnected =>
if self.peer_store.is_banned(&peer_id) {
if self.peer_store.is_banned(&peer_id.into()) {
self.reject_connection(peer_id, incoming_index);
} else {
*state = PeerState::Connected(Direction::Inbound);
@@ -778,7 +787,7 @@ impl ProtocolController {
self.reserved_nodes
.iter_mut()
.filter_map(|(peer_id, state)| {
(!state.is_connected() && !self.peer_store.is_banned(peer_id)).then(|| {
(!state.is_connected() && !self.peer_store.is_banned(&peer_id.into())).then(|| {
*state = PeerState::Connected(Direction::Outbound);
peer_id
})
@@ -803,8 +812,11 @@ impl ProtocolController {
let ignored = self
.reserved_nodes
.keys()
.collect::<HashSet<&PeerId>>()
.union(&self.nodes.keys().collect::<HashSet<&PeerId>>())
.map(From::from)
.collect::<HashSet<sc_network_types::PeerId>>()
.union(
&self.nodes.keys().map(From::from).collect::<HashSet<sc_network_types::PeerId>>(),
)
.cloned()
.collect();
@@ -813,16 +825,17 @@ impl ProtocolController {
.outgoing_candidates(available_slots, ignored)
.into_iter()
.filter_map(|peer_id| {
(!self.reserved_nodes.contains_key(&peer_id) && !self.nodes.contains_key(&peer_id))
.then_some(peer_id)
.or_else(|| {
error!(
target: LOG_TARGET,
"`PeerStore` returned a node we asked to ignore: {peer_id}.",
);
debug_assert!(false, "`PeerStore` returned a node we asked to ignore.");
None
})
(!self.reserved_nodes.contains_key(&peer_id.into()) &&
!self.nodes.contains_key(&peer_id.into()))
.then_some(peer_id)
.or_else(|| {
error!(
target: LOG_TARGET,
"`PeerStore` returned a node we asked to ignore: {peer_id}.",
);
debug_assert!(false, "`PeerStore` returned a node we asked to ignore.");
None
})
})
.collect::<Vec<_>>();
@@ -836,8 +849,8 @@ impl ProtocolController {
candidates.into_iter().take(available_slots).for_each(|peer_id| {
self.num_out += 1;
self.nodes.insert(peer_id, Direction::Outbound);
self.start_connection(peer_id);
self.nodes.insert(peer_id.into(), Direction::Outbound);
self.start_connection(peer_id.into());
})
}
}
@@ -845,8 +858,10 @@ impl ProtocolController {
#[cfg(test)]
mod tests {
use super::*;
use crate::{peer_store::PeerStoreProvider, ReputationChange};
use libp2p::PeerId;
use crate::{
peer_store::{PeerStoreProvider, ProtocolHandle as ProtocolHandleT},
PeerId, ReputationChange,
};
use sc_network_common::role::ObservedRole;
use sc_utils::mpsc::{tracing_unbounded, TryRecvError};
use std::collections::HashSet;
@@ -856,14 +871,16 @@ mod tests {
pub PeerStoreHandle {}
impl PeerStoreProvider for PeerStoreHandle {
fn is_banned(&self, peer_id: &PeerId) -> bool;
fn register_protocol(&self, protocol_handle: ProtocolHandle);
fn report_disconnect(&mut self, peer_id: PeerId);
fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole);
fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange);
fn peer_reputation(&self, peer_id: &PeerId) -> i32;
fn peer_role(&self, peer_id: &PeerId) -> Option<ObservedRole>;
fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec<PeerId>;
fn is_banned(&self, peer_id: &sc_network_types::PeerId) -> bool;
fn register_protocol(&self, protocol_handle: Arc<dyn ProtocolHandleT>);
fn report_disconnect(&self, peer_id: sc_network_types::PeerId);
fn set_peer_role(&self, peer_id: &sc_network_types::PeerId, role: ObservedRole);
fn report_peer(&self, peer_id: sc_network_types::PeerId, change: ReputationChange);
fn peer_reputation(&self, peer_id: &sc_network_types::PeerId) -> i32;
fn peer_role(&self, peer_id: &sc_network_types::PeerId) -> Option<ObservedRole>;
fn outgoing_candidates(&self, count: usize, ignored: HashSet<sc_network_types::PeerId>) -> Vec<sc_network_types::PeerId>;
fn num_known_peers(&self) -> usize;
fn add_known_peer(&self, peer_id: sc_network_types::PeerId);
}
}
@@ -887,7 +904,7 @@ mod tests {
peer_store.expect_report_disconnect().times(2).return_const(());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Add second reserved node at runtime (this currently calls `alloc_slots` internally).
controller.on_add_reserved_peer(reserved2);
@@ -948,7 +965,7 @@ mod tests {
peer_store.expect_is_banned().times(6).return_const(true);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Add second reserved node at runtime (this currently calls `alloc_slots` internally).
controller.on_add_reserved_peer(reserved2);
@@ -1000,7 +1017,7 @@ mod tests {
peer_store.expect_report_disconnect().times(2).return_const(());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Add second reserved node at runtime (this calls `alloc_slots` internally).
controller.on_add_reserved_peer(reserved2);
@@ -1042,7 +1059,7 @@ mod tests {
fn nodes_supplied_by_peer_store_are_connected() {
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let candidates = vec![peer1, peer2];
let candidates = vec![peer1.into(), peer2.into()];
let config = ProtoSetConfig {
in_peers: 0,
@@ -1058,7 +1075,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Initiate connections.
controller.alloc_slots();
@@ -1092,7 +1109,7 @@ mod tests {
let reserved2 = PeerId::random();
let regular1 = PeerId::random();
let regular2 = PeerId::random();
let outgoing_candidates = vec![regular1, regular2];
let outgoing_candidates = vec![regular1.into(), regular2.into()];
let reserved_nodes = [reserved1, reserved2].iter().cloned().collect();
let config =
@@ -1105,7 +1122,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Initiate connections.
controller.alloc_slots();
@@ -1128,8 +1145,8 @@ mod tests {
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let peer3 = PeerId::random();
let candidates1 = vec![peer1, peer2];
let candidates2 = vec![peer3];
let candidates1 = vec![peer1.into(), peer2.into()];
let candidates2 = vec![peer3.into()];
let config = ProtoSetConfig {
in_peers: 0,
@@ -1147,7 +1164,7 @@ mod tests {
peer_store.expect_report_disconnect().times(2).return_const(());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Initiate connections.
controller.alloc_slots();
@@ -1214,7 +1231,7 @@ mod tests {
peer_store.expect_register_protocol().once().return_const(());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Initiate connections.
controller.alloc_slots();
@@ -1240,7 +1257,7 @@ mod tests {
peer_store.expect_register_protocol().once().return_const(());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
let peer = PeerId::random();
let incoming_index = IncomingIndex(1);
@@ -1262,7 +1279,7 @@ mod tests {
fn disabling_reserved_only_mode_allows_to_connect_to_peers() {
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let candidates = vec![peer1, peer2];
let candidates = vec![peer1.into(), peer2.into()];
let config = ProtoSetConfig {
in_peers: 0,
@@ -1278,7 +1295,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Initiate connections.
controller.alloc_slots();
@@ -1309,7 +1326,7 @@ mod tests {
let reserved2 = PeerId::random();
let regular1 = PeerId::random();
let regular2 = PeerId::random();
let outgoing_candidates = vec![regular1];
let outgoing_candidates = vec![regular1.into()];
let config = ProtoSetConfig {
in_peers: 10,
@@ -1325,7 +1342,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
assert_eq!(controller.num_out, 0);
assert_eq!(controller.num_in, 0);
@@ -1383,7 +1400,7 @@ mod tests {
peer_store.expect_register_protocol().once().return_const(());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
assert_eq!(controller.reserved_nodes.len(), 2);
assert_eq!(controller.nodes.len(), 0);
assert_eq!(controller.num_out, 0);
@@ -1416,7 +1433,7 @@ mod tests {
peer_store.expect_is_banned().times(2).return_const(false);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Initiate connections.
controller.alloc_slots();
@@ -1460,10 +1477,13 @@ mod tests {
let mut peer_store = MockPeerStoreHandle::new();
peer_store.expect_register_protocol().once().return_const(());
peer_store.expect_is_banned().times(2).return_const(false);
peer_store.expect_outgoing_candidates().once().return_const(Vec::new());
peer_store
.expect_outgoing_candidates()
.once()
.return_const(Vec::<sc_network_types::PeerId>::new());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Connect `peer1` as inbound, `peer2` as outbound.
controller.on_incoming_connection(peer1, IncomingIndex(1));
@@ -1493,7 +1513,7 @@ mod tests {
fn regular_nodes_stop_occupying_slots_when_become_reserved() {
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let outgoing_candidates = vec![peer1];
let outgoing_candidates = vec![peer1.into()];
let config = ProtoSetConfig {
in_peers: 10,
@@ -1509,7 +1529,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Connect `peer1` as outbound & `peer2` as inbound.
controller.alloc_slots();
@@ -1535,7 +1555,7 @@ mod tests {
fn disconnecting_regular_peers_work() {
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let outgoing_candidates = vec![peer1];
let outgoing_candidates = vec![peer1.into()];
let config = ProtoSetConfig {
in_peers: 10,
@@ -1551,7 +1571,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Connect `peer1` as outbound & `peer2` as inbound.
controller.alloc_slots();
@@ -1610,7 +1630,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(Vec::new());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Connect `reserved1` as inbound & `reserved2` as outbound.
controller.on_incoming_connection(reserved1, IncomingIndex(1));
@@ -1650,7 +1670,7 @@ mod tests {
fn dropping_regular_peers_work() {
let peer1 = PeerId::random();
let peer2 = PeerId::random();
let outgoing_candidates = vec![peer1];
let outgoing_candidates = vec![peer1.into()];
let config = ProtoSetConfig {
in_peers: 10,
@@ -1667,7 +1687,7 @@ mod tests {
peer_store.expect_report_disconnect().times(2).return_const(());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Connect `peer1` as outbound & `peer2` as inbound.
controller.alloc_slots();
@@ -1718,7 +1738,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(Vec::new());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Connect `reserved1` as inbound & `reserved2` as outbound.
controller.on_incoming_connection(reserved1, IncomingIndex(1));
@@ -1762,7 +1782,7 @@ mod tests {
fn incoming_request_for_connected_regular_node_switches_it_to_inbound() {
let regular1 = PeerId::random();
let regular2 = PeerId::random();
let outgoing_candidates = vec![regular1];
let outgoing_candidates = vec![regular1.into()];
let config = ProtoSetConfig {
in_peers: 10,
@@ -1778,7 +1798,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
assert_eq!(controller.num_out, 0);
assert_eq!(controller.num_in, 0);
@@ -1814,7 +1834,7 @@ mod tests {
fn incoming_request_for_connected_node_is_rejected_if_its_banned() {
let regular1 = PeerId::random();
let regular2 = PeerId::random();
let outgoing_candidates = vec![regular1];
let outgoing_candidates = vec![regular1.into()];
let config = ProtoSetConfig {
in_peers: 10,
@@ -1831,7 +1851,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
assert_eq!(controller.num_out, 0);
assert_eq!(controller.num_in, 0);
@@ -1867,7 +1887,7 @@ mod tests {
fn incoming_request_for_connected_node_is_rejected_if_no_slots_available() {
let regular1 = PeerId::random();
let regular2 = PeerId::random();
let outgoing_candidates = vec![regular1];
let outgoing_candidates = vec![regular1.into()];
let config = ProtoSetConfig {
in_peers: 1,
@@ -1883,7 +1903,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
assert_eq!(controller.num_out, 0);
assert_eq!(controller.num_in, 0);
@@ -1935,7 +1955,7 @@ mod tests {
peer_store.expect_is_banned().once().return_const(false);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Connect `peer1` as inbound.
controller.on_incoming_connection(peer1, IncomingIndex(1));
@@ -1965,7 +1985,7 @@ mod tests {
peer_store.expect_is_banned().once().return_const(true);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
// Incoming request.
controller.on_incoming_connection(peer1, IncomingIndex(1));
@@ -1990,7 +2010,7 @@ mod tests {
peer_store.expect_is_banned().once().return_const(true);
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
assert!(controller.reserved_nodes.contains_key(&reserved1));
// Incoming request.
@@ -2017,7 +2037,7 @@ mod tests {
peer_store.expect_outgoing_candidates().once().return_const(Vec::new());
let (_handle, mut controller) =
ProtocolController::new(SetId::from(0), config, tx, Box::new(peer_store));
ProtocolController::new(SetId::from(0), config, tx, Arc::new(peer_store));
assert!(matches!(controller.reserved_nodes.get(&reserved1), Some(PeerState::NotConnected)));
// Initiate connections