Sulkta-OSS/pallas
3
0
Fork 0
Code Issues 1 Pull requests 4 Projects Releases Packages Wiki Activity Actions

refactor: Merge multiplexer & miniprotocols into single crate (#244)

Browse source
This commit is contained in:
Santiago Carmuega 2023-04-11 00:51:38 +02:00 • committed by GitHub
parent b63d2052cb
commit d5f0c2fd80
80 changed files with 1694 additions and 3002 deletions
Download patch file Download diff file Expand all files Collapse all files

6
pallas-upstream/Cargo.toml
View file

@ -11,18 +11,16 @@ readme = "README.md"
authors = ["Santiago Carmuega <santiago@carmuega.me>"]
[dependencies]
async-trait = "0.1.68"
byteorder = "1.4.3"
gasket = { git = "https://github.com/construkts/gasket-rs" }
# gasket = { path = "../../../construkts/gasket-rs" }
hex = "0.4.3"
mio = { version = "0.8.6", features = ["net", "os-poll"] }
# gasket = { version = "0.1.0", path = "../../../construkts/gasket-rs" }
pallas-codec = { version = "0.18.0", path = "../pallas-codec" }
pallas-crypto = { version = "0.18.0", path = "../pallas-crypto" }
pallas-miniprotocols = { version = "0.18.0", path = "../pallas-miniprotocols" }
pallas-multiplexer = { version = "0.18.0", path = "../pallas-multiplexer" }
pallas-network = { version = "0.18.0", path = "../pallas-network" }
pallas-traverse = { version = "0.18.0", path = "../pallas-traverse" }
rayon = "1.7.0"
serde = { version = "1.0.154", features = ["derive"] }
thiserror = "1.0.31"
tokio = { version = "1", features = ["net", "macros", "io-util"] }

126
pallas-upstream/src/api.rs
View file

@ -1,126 +0,0 @@
pub use crate::framework::{BlockFetchEvent, Cursor, DownstreamPort, Intersection};
pub mod n2n {
use crate::{blockfetch, chainsync, framework::*, plexer};
use gasket::{
messaging::{SendAdapter, SendPort},
runtime::Tether,
};
pub struct Runtime {
pub plexer_tether: Tether,
pub chainsync_tether: Tether,
pub blockfetch_tether: Tether,
}
pub struct Bootstrapper<A, C>
where
A: SendAdapter<BlockFetchEvent>,
C: Cursor,
{
cursor: C,
peer_address: String,
network_magic: u64,
output: super::DownstreamPort<A>,
}
impl<A, C> Bootstrapper<A, C>
where
A: SendAdapter<BlockFetchEvent> + 'static,
C: Cursor + 'static,
{
pub fn new(cursor: C, peer_address: String, network_magic: u64) -> Self {
Bootstrapper {
cursor,
peer_address,
network_magic,
output: Default::default(),
}
}
pub fn connect_output(&mut self, adapter: A) {
self.output.connect(adapter);
}
pub fn spawn(self) -> Result<Runtime, Error> {
/*
TODO: this is how we envision the setup of complex pipelines leveraging Rust macros:
pipeline!(
plexer = plexer::Worker::new(xx),
chainsync = chainsync::Worker::new(yy),
blockfetch = blockfetch::Worker::new(yy),
reducer = reducer::Worker::new(yy),
plexer.demux2 => chainsync.demux2,
plexer.demux3 => blockfetch.demux3,
chainsync.mux2 + blockfetch.mux3 => plexer.mux,
chainsync.downstream => blockfetch.upstream,
blockfetch.downstream => reducer.upstream,
);
The above snippet would replace the rest of the code in this function, which is just a more verbose, manual way of saying the same thing.
*/
let mut mux_input = MuxInputPort::default();
let mut demux2_out = DemuxOutputPort::default();
let mut demux2_in = DemuxInputPort::default();
gasket::messaging::tokio::connect_ports(&mut demux2_out, &mut demux2_in, 1000);
let mut demux3_out = DemuxOutputPort::default();
let mut demux3_in = DemuxInputPort::default();
gasket::messaging::tokio::connect_ports(&mut demux3_out, &mut demux3_in, 1000);
let mut mux2_out = MuxOutputPort::default();
let mut mux3_out = MuxOutputPort::default();
gasket::messaging::tokio::funnel_ports(
vec![&mut mux2_out, &mut mux3_out],
&mut mux_input,
1000,
);
let mut chainsync_downstream = chainsync::DownstreamPort::default();
let mut blockfetch_upstream = blockfetch::UpstreamPort::default();
gasket::messaging::tokio::connect_ports(
&mut chainsync_downstream,
&mut blockfetch_upstream,
100,
);
let plexer_tether = gasket::runtime::spawn_stage(
plexer::Worker::new(
self.peer_address,
self.network_magic,
mux_input,
Some(demux2_out),
Some(demux3_out),
),
gasket::runtime::Policy::default(),
Some("plexer"),
);
let channel2 = ProtocolChannel(2, mux2_out, demux2_in);
let chainsync_tether = gasket::runtime::spawn_stage(
chainsync::Worker::new(self.cursor, channel2, chainsync_downstream),
gasket::runtime::Policy::default(),
Some("chainsync"),
);
let channel3 = ProtocolChannel(3, mux3_out, demux3_in);
let blockfetch_tether = gasket::runtime::spawn_stage(
blockfetch::Worker::new(channel3, blockfetch_upstream, self.output),
gasket::runtime::Policy::default(),
Some("blockfetch"),
);
Ok(Runtime {
plexer_tether,
chainsync_tether,
blockfetch_tether,
})
}
}
}

106
pallas-upstream/src/blockfetch.rs
View file

@ -1,106 +0,0 @@
use gasket::messaging::SendAdapter;
use gasket::runtime::WorkSchedule;
use tracing::{error, info, instrument};
use pallas_crypto::hash::Hash;
use pallas_miniprotocols::blockfetch;
use pallas_miniprotocols::Point;
use crate::framework::*;
pub type UpstreamPort = gasket::messaging::tokio::InputPort<ChainSyncEvent>;
pub type OuroborosClient = blockfetch::Client<ProtocolChannel>;
pub struct Worker<T>
where
T: Send + Sync,
{
client: OuroborosClient,
upstream: UpstreamPort,
downstream: DownstreamPort<T>,
block_count: gasket::metrics::Counter,
}
impl<T> Worker<T>
where
T: Send + Sync,
{
pub fn new(
plexer: ProtocolChannel,
upstream: UpstreamPort,
downstream: DownstreamPort<T>,
) -> Self {
let client = OuroborosClient::new(plexer);
Self {
client,
upstream,
downstream,
block_count: Default::default(),
}
}
#[instrument(skip(self), fields(slot, %hash))]
async fn fetch_block(
&mut self,
slot: u64,
hash: &Hash<32>,
) -> Result<Vec<u8>, gasket::error::Error> {
info!("fetching block");
match self
.client
.fetch_single(Point::Specific(slot, hash.to_vec()))
.await
{
Ok(x) => {
info!("block fetch succeeded");
Ok(x)
}
Err(blockfetch::Error::ChannelError(x)) => {
error!("plexer channel error: {}", x);
Err(gasket::error::Error::RetryableError)
}
Err(x) => {
error!("unrecoverable block fetch error: {}", x);
Err(gasket::error::Error::WorkPanic)
}
}
}
}
impl<A> gasket::runtime::Worker for Worker<A>
where
A: SendAdapter<BlockFetchEvent>,
{
fn metrics(&self) -> gasket::metrics::Registry {
gasket::metrics::Builder::new()
.with_counter("fetched_blocks", &self.block_count)
.build()
}
type WorkUnit = ChainSyncEvent;
async fn schedule(&mut self) -> gasket::runtime::ScheduleResult<Self::WorkUnit> {
let msg = self.upstream.recv().await?;
info!("scheduling block betch");
Ok(WorkSchedule::Unit(msg.payload))
}
async fn execute(&mut self, unit: &Self::WorkUnit) -> Result<(), gasket::error::Error> {
let output = match unit {
ChainSyncEvent::RollForward(s, h) => {
let body = self.fetch_block(*s, h).await?;
self.block_count.inc(1);
BlockFetchEvent::RollForward(*s, h.clone(), body)
}
ChainSyncEvent::Rollback(x) => BlockFetchEvent::Rollback(x.clone()),
};
self.downstream.send(output.into()).await?;
Ok(())
}
}

176
pallas-upstream/src/chainsync.rs
View file

@ -1,176 +0,0 @@
use gasket::error::AsWorkError;
use tracing::{debug, info};
use pallas_miniprotocols::chainsync::{HeaderContent, NextResponse, Tip};
use pallas_miniprotocols::{chainsync, Point};
use pallas_traverse::MultiEraHeader;
use crate::framework::*;
fn to_traverse(header: &chainsync::HeaderContent) -> Result<MultiEraHeader<'_>, Error> {
let out = match header.byron_prefix {
Some((subtag, _)) => MultiEraHeader::decode(header.variant, Some(subtag), &header.cbor),
None => MultiEraHeader::decode(header.variant, None, &header.cbor),
};
out.map_err(Error::parse)
}
pub type DownstreamPort = gasket::messaging::tokio::OutputPort<ChainSyncEvent>;
pub type OuroborosClient = chainsync::N2NClient<ProtocolChannel>;
pub struct Worker<C>
where
C: Cursor,
{
chain_cursor: C,
client: OuroborosClient,
downstream: DownstreamPort,
block_count: gasket::metrics::Counter,
chain_tip: gasket::metrics::Gauge,
}
impl<C> Worker<C>
where
C: Cursor,
{
pub fn new(chain_cursor: C, plexer: ProtocolChannel, downstream: DownstreamPort) -> Self {
let client = OuroborosClient::new(plexer);
Self {
chain_cursor,
client,
downstream,
block_count: Default::default(),
chain_tip: Default::default(),
}
}
fn notify_tip(&self, tip: Tip) {
self.chain_tip.set(tip.0.slot_or_default() as i64);
}
async fn intersect(&mut self) -> Result<(), gasket::error::Error> {
let value = self.chain_cursor.intersection();
let intersect = match value {
Intersection::Origin => {
info!("intersecting origin");
self.client.intersect_origin().await.or_restart()?.into()
}
Intersection::Tip => {
info!("intersecting tip");
self.client.intersect_tip().await.or_restart()?.into()
}
Intersection::Breadcrumbs(points) => {
info!("intersecting breadcrumbs");
let (point, tip) = self
.client
.find_intersect(Vec::from(points))
.await
.or_restart()?;
self.notify_tip(tip);
point
}
};
info!(?intersect, "intersected");
Ok(())
}
async fn process_next(
&mut self,
next: NextResponse<HeaderContent>,
) -> Result<(), gasket::error::Error> {
match next {
chainsync::NextResponse::RollForward(header, tip) => {
let header = to_traverse(&header).or_panic()?;
debug!(slot = header.slot(), hash = %header.hash(), "chain sync roll forward");
self.downstream
.send(ChainSyncEvent::RollForward(header.slot(), header.hash()).into())
.await?;
self.notify_tip(tip);
Ok(())
}
chainsync::NextResponse::RollBackward(point, tip) => {
match &point {
Point::Origin => debug!("rollback to origin"),
Point::Specific(slot, _) => debug!(slot, "rollback"),
};
self.downstream
.send(ChainSyncEvent::Rollback(point).into())
.await?;
self.notify_tip(tip);
Ok(())
}
chainsync::NextResponse::Await => {
info!("chain-sync reached the tip of the chain");
Ok(())
}
}
}
async fn request_next(&mut self) -> Result<(), gasket::error::Error> {
info!("requesting next block");
let next = self.client.request_next().await.or_restart()?;
self.process_next(next).await
}
async fn await_next(&mut self) -> Result<(), gasket::error::Error> {
info!("awaiting next block (blocking)");
let next = self.client.recv_while_must_reply().await.or_restart()?;
self.process_next(next).await
}
}
pub enum WorkUnit {
Intersect,
RequestNext,
AwaitNext,
}
impl<C> gasket::runtime::Worker for Worker<C>
where
C: Cursor + Sync + Send,
{
type WorkUnit = WorkUnit;
fn metrics(&self) -> gasket::metrics::Registry {
gasket::metrics::Builder::new()
.with_counter("received_blocks", &self.block_count)
.with_gauge("chain_tip", &self.chain_tip)
.build()
}
async fn bootstrap(&mut self) -> gasket::runtime::ScheduleResult<Self::WorkUnit> {
Ok(gasket::runtime::WorkSchedule::Unit(WorkUnit::Intersect))
}
async fn schedule(&mut self) -> gasket::runtime::ScheduleResult<Self::WorkUnit> {
match self.client.has_agency() {
true => Ok(gasket::runtime::WorkSchedule::Unit(WorkUnit::RequestNext)),
false => Ok(gasket::runtime::WorkSchedule::Unit(WorkUnit::AwaitNext)),
}
}
async fn execute(&mut self, unit: &Self::WorkUnit) -> Result<(), gasket::error::Error> {
match unit {
WorkUnit::Intersect => self.intersect().await?,
WorkUnit::RequestNext => self.request_next().await?,
WorkUnit::AwaitNext => self.await_next().await?,
};
Ok(())
}
}

111
pallas-upstream/src/framework.rs
View file

@ -1,8 +1,5 @@
use pallas_crypto::hash::Hash;
use pallas_miniprotocols::Point;
use pallas_multiplexer as multiplexer;
use thiserror::Error;
use tracing::{error, trace};
use pallas_network::miniprotocols::Point;
pub type BlockSlot = u64;
pub type BlockHash = Hash<32>;
@ -20,112 +17,10 @@ pub trait Cursor: Send + Sync {
}
#[derive(Debug, Clone)]
pub enum ChainSyncEvent {
RollForward(BlockSlot, BlockHash),
Rollback(Point),
}
#[derive(Debug, Clone)]
pub enum BlockFetchEvent {
pub enum UpstreamEvent {
RollForward(BlockSlot, BlockHash, RawBlock),
Rollback(Point),
}
// ports used by plexer
pub type MuxOutputPort = gasket::messaging::tokio::OutputPort<(u16, multiplexer::Payload)>;
pub type DemuxInputPort = gasket::messaging::tokio::InputPort<multiplexer::Payload>;
// ports used by mini-protocols
pub type MuxInputPort = gasket::messaging::tokio::InputPort<(u16, multiplexer::Payload)>;
pub type DemuxOutputPort = gasket::messaging::tokio::OutputPort<multiplexer::Payload>;
// final output port
pub type DownstreamPort<A> = gasket::messaging::OutputPort<A, BlockFetchEvent>;
pub struct ProtocolChannel(pub u16, pub MuxOutputPort, pub DemuxInputPort);
impl multiplexer::agents::Channel for ProtocolChannel {
async fn enqueue_chunk(
&mut self,
payload: multiplexer::Payload,
) -> Result<(), multiplexer::agents::ChannelError> {
trace!(
protocol = self.0,
payload = hex::encode(&payload),
"enqueing"
);
let res = self
.1
.send(gasket::messaging::Message::from((self.0, payload)))
.await;
match res {
Ok(_) => Ok(()),
Err(error) => {
error!(?error, "enqueue chunk failed");
Err(multiplexer::agents::ChannelError::NotConnected(None))
}
}
}
async fn dequeue_chunk(
&mut self,
) -> Result<multiplexer::Payload, multiplexer::agents::ChannelError> {
let res = self.2.recv().await;
match res {
Ok(msg) => Ok(msg.payload),
Err(error) => {
error!(?error, "dequeue chunk failed");
Err(multiplexer::agents::ChannelError::NotConnected(None))
}
}
}
}
#[derive(Error, Debug)]
pub enum Error {
#[error("{0}")]
Client(String),
#[error("{0}")]
Parse(String),
#[error("{0}")]
Server(String),
#[error("{0}")]
Message(String),
#[error("{0}")]
Custom(String),
}
impl Error {
pub fn client(error: impl ToString) -> Error {
Error::Client(error.to_string())
}
pub fn parse(error: impl ToString) -> Error {
Error::Parse(error.to_string())
}
pub fn server(error: impl ToString) -> Error {
Error::Server(error.to_string())
}
pub fn message(error: impl ToString) -> Error {
Error::Message(error.to_string())
}
pub fn custom(error: impl Into<Box<dyn std::error::Error>>) -> Error {
Error::Custom(format!("{}", error.into()))
}
}
impl From<Box<dyn std::error::Error>> for Error {
fn from(err: Box<dyn std::error::Error>) -> Self {
Error::custom(err)
}
}
pub type DownstreamPort<A> = gasket::messaging::OutputPort<A, UpstreamEvent>;

12
pallas-upstream/src/lib.rs
View file

@ -1,10 +1,8 @@
#![feature(async_fn_in_trait)]
pub(crate) mod blockfetch;
pub(crate) mod chainsync;
pub(crate) mod framework;
pub(crate) mod plexer;
pub(crate) mod worker;
mod api;
pub use crate::framework::{Cursor, DownstreamPort, Intersection, UpstreamEvent};
pub use api::*;
pub mod n2n {
pub use crate::worker::Worker;
}

436
pallas-upstream/src/plexer.rs
View file

@ -1,436 +0,0 @@
use std::future::ready;
use byteorder::{ByteOrder, NetworkEndian};
use gasket::error::AsWorkError;
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf, ReadHalf, WriteHalf};
use tokio::net::{TcpStream, ToSocketAddrs};
use tokio::select;
use tokio::time::Instant;
use tracing::{debug, error, info, trace, warn};
use pallas_miniprotocols::handshake;
use crate::framework::*;
const HEADER_LEN: usize = 8;
pub type Timestamp = u32;
pub type Payload = Vec<u8>;
pub type Protocol = u16;
/// A `Header` struct represents an Ouroboros segment header.
///
/// # Examples
///
/// Converting a `Header` to bytes:
///
/// ```
/// use byteorder::{BigEndian, ByteOrder};
/// use pallas_upstream::plexer::Header;
///
/// let header = Header {
/// protocol: 0x01,
/// timestamp: 1619804871,
/// payload_len: 42,
/// };
///
/// let header_bytes: [u8; 8] = header.into();
/// assert_eq!(header_bytes, [97, 75, 168, 15, 128, 1, 0, 42]);
/// ```
///
/// Converting bytes to a `Header`:
///
/// ```
/// use byteorder::{BigEndian, ByteOrder};
/// use pallas_upstream::plexer::Header;
///
/// let bytes = [97, 75, 168, 15, 128, 1, 0, 42];
/// let header: Header = (&bytes[..]).into();
///
/// assert_eq!(header.protocol, 0x01);
/// assert_eq!(header.timestamp, 1619804871);
/// assert_eq!(header.payload_len, 42);
/// ```
#[derive(Debug)]
pub struct Header {
pub protocol: Protocol,
pub timestamp: Timestamp,
pub payload_len: u16,
}
impl From<&[u8]> for Header {
fn from(value: &[u8]) -> Self {
let timestamp = NetworkEndian::read_u32(&value[0..4]);
let protocol = NetworkEndian::read_u16(&value[4..6]) ^ 0x8000;
let payload_len = NetworkEndian::read_u16(&value[6..8]);
Self {
timestamp,
protocol,
payload_len,
}
}
}
impl From<Header> for [u8; 8] {
fn from(value: Header) -> Self {
let mut out = [0u8; 8];
NetworkEndian::write_u32(&mut out[0..4], value.timestamp);
NetworkEndian::write_u16(&mut out[4..6], value.protocol);
NetworkEndian::write_u16(&mut out[6..8], value.payload_len);
out
}
}
pub struct Segment {
pub header: Header,
pub payload: Payload,
}
use tokio::io::{AsyncReadExt, AsyncWriteExt};
struct AsyncBearer(OwnedReadHalf, OwnedWriteHalf, Instant);
impl AsyncBearer {
async fn connect_tcp(addr: impl ToSocketAddrs) -> Result<Self, std::io::Error> {
let stream = TcpStream::connect(addr).await?;
let (read, write) = stream.into_split();
Ok(Self(read, write, Instant::now()))
}
}
impl AsyncBearer {
async fn readable(&self) -> tokio::io::Result<()> {
self.0.readable().await
}
/// Peek the available data in search for a frame header
async fn peek_header(&mut self) -> tokio::io::Result<Option<Header>> {
let mut buf = [0u8; HEADER_LEN];
let len = self.0.peek(&mut buf).await?;
if len < HEADER_LEN {
return Ok(None);
}
Ok(Some(Header::from(buf.as_slice())))
}
async fn has_payload(&mut self, payload_len: usize) -> tokio::io::Result<bool> {
let segment_size = HEADER_LEN + payload_len;
let mut buf = vec![0u8; segment_size];
let available = self.0.peek(&mut buf).await?;
return Ok(available >= segment_size);
}
/// Peeks the bearer to see if a full segment is available to be read
async fn has_segment(&mut self) -> std::io::Result<bool> {
let header = match self.peek_header().await? {
Some(x) => x,
None => return Ok(false),
};
self.has_payload(header.payload_len as usize).await
}
/// Reads a full segment from the bearer while consuming the bytes
///
/// This function is NOT "cancel safe", meaning that it shouldn't be used
/// inside the context of a select!. Only call this function once you're
/// sure that you can await until all the required bytes are available.
async fn read_segment(&mut self) -> tokio::io::Result<(Protocol, Payload)> {
let mut buf = [0u8; HEADER_LEN];
self.0.read_exact(&mut buf).await?;
let header = Header::from(buf.as_slice());
// TODO: assert any business invariants regarding timestamp from the other party
let mut payload = vec![0u8; header.payload_len as usize];
self.0.read_exact(&mut payload).await?;
Ok((header.protocol, payload))
}
async fn write_segment(&mut self, protocol: u16, payload: &[u8]) -> Result<(), std::io::Error> {
let header = Header {
protocol,
timestamp: self.2.elapsed().as_micros() as u32,
payload_len: payload.len() as u16,
};
let buf: [u8; 8] = header.into();
self.1.write_all(&buf).await?;
self.1.write_all(&payload).await?;
Ok(())
}
}
pub struct AsyncAgentChannel(
Protocol,
tokio::sync::mpsc::Sender<(Protocol, Payload)>,
tokio::sync::broadcast::Receiver<(Protocol, Payload)>,
);
impl pallas_multiplexer::agents::Channel for AsyncAgentChannel {
async fn enqueue_chunk(
&mut self,
chunk: pallas_multiplexer::Payload,
) -> Result<(), pallas_multiplexer::agents::ChannelError> {
let res = self.1.send((self.0, chunk)).await;
res.map_err(|err| pallas_multiplexer::agents::ChannelError::NotConnected(Some(err.0 .1)))
}
async fn dequeue_chunk(
&mut self,
) -> Result<pallas_multiplexer::Payload, pallas_multiplexer::agents::ChannelError> {
loop {
let (protocol, payload) = self
.2
.recv()
.await
.map_err(|err| pallas_multiplexer::agents::ChannelError::NotConnected(None))?;
if protocol == self.0 {
break Ok(payload);
}
}
}
}
pub type AsyncIngress = (
tokio::sync::mpsc::Sender<(Protocol, Payload)>,
tokio::sync::mpsc::Receiver<(Protocol, Payload)>,
);
pub type AsyncEgress = (
tokio::sync::broadcast::Sender<(Protocol, Payload)>,
tokio::sync::broadcast::Receiver<(Protocol, Payload)>,
);
struct AsyncPlexer {
bearer: AsyncBearer,
ingress: AsyncIngress,
egress: AsyncEgress,
}
impl AsyncPlexer {
pub fn new(bearer: AsyncBearer) -> Self {
Self {
bearer,
ingress: tokio::sync::mpsc::channel(100), // TODO: define buffer
egress: tokio::sync::broadcast::channel(100),
}
}
async fn mux(&mut self, msg: (Protocol, Payload)) -> tokio::io::Result<()> {
self.bearer.write_segment(msg.0, &msg.1).await?;
Ok(())
}
async fn demux(&mut self) -> tokio::io::Result<()> {
let (protocol, payload) = self.bearer.read_segment().await?;
self.egress.0.send((protocol, payload)).unwrap();
Ok(())
}
pub fn subscribe(&mut self, protocol: Protocol) -> AsyncAgentChannel {
let agent_tx = self.ingress.0.clone();
let agent_rx = self.egress.0.subscribe();
AsyncAgentChannel(protocol, agent_tx, agent_rx)
}
pub async fn run(&mut self) -> tokio::io::Result<()> {
loop {
select! {
Ok(_) = self.bearer.readable() => {
if let Ok(true) = self.bearer.has_segment().await {
trace!("demux selected");
self.demux().await?
}
},
Some(x) = self.ingress.1.recv() => {
trace!("mux selected");
self.mux(x).await?
},
}
}
}
}
impl From<AsyncBearer> for AsyncPlexer {
fn from(value: AsyncBearer) -> Self {
Self::new(value)
}
}
impl From<AsyncPlexer> for AsyncBearer {
fn from(value: AsyncPlexer) -> Self {
value.bearer
}
}
async fn handshake(
plexer: &mut AsyncPlexer,
network_magic: u64,
) -> Result<(), gasket::error::Error> {
info!("executing handshake");
let channel0 = plexer.subscribe(0);
let versions = handshake::n2n::VersionTable::v7_and_above(network_magic);
let mut client = handshake::Client::new(channel0);
//let p = tokio::spawn(plexer.run());
//let output = client.handshake(versions).or_restart()?;
let output = select! {
x = client.handshake(versions) => x.or_restart()?,
x = plexer.run() => {
match x.or_restart() {
Err(x) => return Err(x),
_ => unreachable!(),
};
},
};
debug!("handshake output: {:?}", output);
//p.abort();
match output {
handshake::Confirmation::Accepted(version, _) => {
info!(version, "connected to upstream peer");
Ok(())
}
_ => {
error!("couldn't agree on handshake version");
Err(gasket::error::Error::WorkPanic)
}
}
}
pub struct Worker {
peer_address: String,
network_magic: u64,
bearer: Option<AsyncBearer>,
mux_input: MuxInputPort,
channel2_out: Option<DemuxOutputPort>,
channel3_out: Option<DemuxOutputPort>,
ops_count: gasket::metrics::Counter,
}
impl Worker {
pub fn new(
peer_address: String,
network_magic: u64,
mux_input: MuxInputPort,
channel2_out: Option<DemuxOutputPort>,
channel3_out: Option<DemuxOutputPort>,
) -> Self {
Self {
peer_address,
network_magic,
channel2_out,
channel3_out,
mux_input,
bearer: None,
ops_count: Default::default(),
}
}
}
pub enum WorkUnit {
Connect,
Mux((u16, Vec<u8>)),
Demux,
}
impl gasket::runtime::Worker for Worker {
type WorkUnit = WorkUnit;
fn metrics(&self) -> gasket::metrics::Registry {
// TODO: define networking metrics (bytes in / out, etc)
gasket::metrics::Builder::new()
.with_counter("ops_count", &self.ops_count)
.build()
}
async fn bootstrap(&mut self) -> gasket::runtime::ScheduleResult<Self::WorkUnit> {
Ok(gasket::runtime::WorkSchedule::Unit(WorkUnit::Connect))
}
async fn schedule(&mut self) -> gasket::runtime::ScheduleResult<Self::WorkUnit> {
let bearer = self.bearer.as_mut().unwrap();
trace!("selecting");
select! {
Ok(msg) = self.mux_input.recv() => { Ok(gasket::runtime::WorkSchedule::Unit(WorkUnit::Mux(msg.payload))) }
Ok(true) = bearer.has_segment() => Ok(gasket::runtime::WorkSchedule::Unit(WorkUnit::Demux)),
_ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => Ok(gasket::runtime::WorkSchedule::Idle),
}
}
async fn execute(&mut self, unit: &Self::WorkUnit) -> Result<(), gasket::error::Error> {
match unit {
WorkUnit::Connect => {
debug!("connecting");
let bearer = AsyncBearer::connect_tcp(&self.peer_address)
.await
.or_retry()?;
let mut plexer = bearer.into();
handshake(&mut plexer, self.network_magic).await?;
self.bearer = Some(plexer.into());
}
WorkUnit::Mux(x) => {
trace!("muxing");
self.bearer
.as_mut()
.unwrap()
.write_segment(x.0, &x.1)
.await
.or_restart()?;
}
WorkUnit::Demux => {
trace!("demuxing");
let (protocol, payload) = self
.bearer
.as_mut()
.unwrap()
.read_segment()
.await
.or_restart()?;
match protocol {
2 => {
if let Some(channel) = &mut self.channel2_out {
channel.send(payload.into()).await?;
trace!("sent protocol 2 msg");
}
}
3 => {
if let Some(channel) = &mut self.channel3_out {
channel.send(payload.into()).await?;
trace!("sent protocol 3 msg");
}
}
x => warn!("trying to demux unexpected protocol {x}"),
}
}
};
Ok(())
}
}

197
pallas-upstream/src/worker.rs Normal file
View file

@ -0,0 +1,197 @@
use gasket::error::AsWorkError;
use tracing::{debug, info};
use pallas_network::facades::PeerClient;
use pallas_network::miniprotocols::chainsync::{self, HeaderContent, NextResponse, Tip};
use pallas_network::miniprotocols::Point;
use pallas_traverse::MultiEraHeader;
use crate::framework::*;
fn to_traverse(header: &HeaderContent) -> Result<MultiEraHeader<'_>, gasket::error::Error> {
let out = match header.byron_prefix {
Some((subtag, _)) => MultiEraHeader::decode(header.variant, Some(subtag), &header.cbor),
None => MultiEraHeader::decode(header.variant, None, &header.cbor),
};
out.or_panic()
}
pub type DownstreamPort = gasket::messaging::tokio::OutputPort<UpstreamEvent>;
pub struct Worker<C>
where
C: Cursor,
{
peer_address: String,
network_magic: u64,
chain_cursor: C,
peer_session: Option<PeerClient>,
downstream: DownstreamPort,
block_count: gasket::metrics::Counter,
chain_tip: gasket::metrics::Gauge,
}
impl<C> Worker<C>
where
C: Cursor,
{
pub fn new(
peer_address: String,
network_magic: u64,
chain_cursor: C,
downstream: DownstreamPort,
) -> Self {
Self {
peer_address,
network_magic,
chain_cursor,
downstream,
peer_session: None,
block_count: Default::default(),
chain_tip: Default::default(),
}
}
fn notify_tip(&self, tip: &Tip) {
self.chain_tip.set(tip.0.slot_or_default() as i64);
}
async fn intersect(&mut self) -> Result<(), gasket::error::Error> {
let value = self.chain_cursor.intersection();
let chainsync = self.peer_session.as_mut().unwrap().chainsync();
let intersect = match value {
Intersection::Origin => {
info!("intersecting origin");
chainsync.intersect_origin().await.or_restart()?.into()
}
Intersection::Tip => {
info!("intersecting tip");
chainsync.intersect_tip().await.or_restart()?.into()
}
Intersection::Breadcrumbs(points) => {
info!("intersecting breadcrumbs");
let (point, tip) = chainsync.find_intersect(points).await.or_restart()?;
self.notify_tip(&tip);
point
}
};
info!(?intersect, "intersected");
Ok(())
}
async fn process_next(
&mut self,
next: &NextResponse<HeaderContent>,
) -> Result<(), gasket::error::Error> {
match next {
NextResponse::RollForward(header, tip) => {
let header = to_traverse(header).or_panic()?;
let slot = header.slot();
let hash = header.hash();
debug!(slot, %hash, "chain sync roll forward");
let block = self
.peer_session
.as_mut()
.unwrap()
.blockfetch()
.fetch_single(pallas_network::miniprotocols::Point::Specific(
slot,
hash.to_vec(),
))
.await
.or_retry()?;
self.downstream
.send(UpstreamEvent::RollForward(slot, hash, block).into())
.await?;
self.notify_tip(tip);
Ok(())
}
chainsync::NextResponse::RollBackward(point, tip) => {
match &point {
Point::Origin => debug!("rollback to origin"),
Point::Specific(slot, _) => debug!(slot, "rollback"),
};
self.downstream
.send(UpstreamEvent::Rollback(point.clone()).into())
.await?;
self.notify_tip(tip);
Ok(())
}
chainsync::NextResponse::Await => {
info!("chain-sync reached the tip of the chain");
Ok(())
}
}
}
}
#[async_trait::async_trait]
impl<C> gasket::runtime::Worker for Worker<C>
where
C: Cursor + Sync + Send,
{
type WorkUnit = NextResponse<HeaderContent>;
fn metrics(&self) -> gasket::metrics::Registry {
gasket::metrics::Builder::new()
.with_counter("received_blocks", &self.block_count)
.with_gauge("chain_tip", &self.chain_tip)
.build()
}
async fn bootstrap(&mut self) -> Result<(), gasket::error::Error> {
debug!("connecting");
let peer = PeerClient::connect(&self.peer_address, self.network_magic)
.await
.or_restart()?;
self.peer_session = Some(peer);
self.intersect().await?;
Ok(())
}
async fn teardown(&mut self) -> Result<(), gasket::error::Error> {
self.peer_session.as_mut().unwrap().abort();
Ok(())
}
async fn schedule(&mut self) -> gasket::runtime::ScheduleResult<Self::WorkUnit> {
let client = self.peer_session.as_mut().unwrap().chainsync();
let next = match client.has_agency() {
true => {
info!("requesting next block");
client.request_next().await.or_restart()?
}
false => {
info!("awaiting next block (blocking)");
client.recv_while_must_reply().await.or_restart()?
}
};
Ok(gasket::runtime::WorkSchedule::Unit(next))
}
async fn execute(&mut self, unit: &Self::WorkUnit) -> Result<(), gasket::error::Error> {
self.process_next(unit).await
}
}

54
pallas-upstream/tests/integration.rs
View file

@ -1,24 +1,21 @@
#![feature(async_fn_in_trait)]
use std::time::Duration;
use gasket::{
messaging::{
tokio::{InputPort, OutputPort},
RecvPort, SendPort,
},
runtime::{ScheduleResult, WorkSchedule, Worker},
runtime::{WorkSchedule, Worker},
};
use pallas_miniprotocols::Point;
use pallas_upstream::{BlockFetchEvent, Cursor};
use tracing::{error, info};
use pallas_upstream::{Cursor, UpstreamEvent};
use tracing::error;
struct Witness {
input: InputPort<pallas_upstream::BlockFetchEvent>,
input: InputPort<UpstreamEvent>,
}
#[async_trait::async_trait]
impl Worker for Witness {
type WorkUnit = BlockFetchEvent;
type WorkUnit = UpstreamEvent;
fn metrics(&self) -> gasket::metrics::Registry {
gasket::metrics::Registry::new()
@ -30,7 +27,7 @@ impl Worker for Witness {
Ok(WorkSchedule::Unit(msg.payload))
}
async fn execute(&mut self, unit: &Self::WorkUnit) -> Result<(), gasket::error::Error> {
async fn execute(&mut self, _: &Self::WorkUnit) -> Result<(), gasket::error::Error> {
error!("witnessing block event");
Ok(())
@ -46,6 +43,7 @@ impl Cursor for StaticCursor {
}
#[test]
#[ignore]
fn test_mainnet_upstream() {
tracing::subscriber::set_global_default(
tracing_subscriber::FmtSubscriber::builder()
@ -54,34 +52,28 @@ fn test_mainnet_upstream() {
)
.unwrap();
let mut b = pallas_upstream::n2n::Bootstrapper::new(
StaticCursor,
"relays-new.cardano-mainnet.iohk.io:3001".into(),
764824073,
);
let (send, receive) = gasket::messaging::tokio::channel(200);
// let mut f = Faker {
// output: Default::default(),
// };
let mut output_port = OutputPort::default();
output_port.connect(send);
//f.output.connect(send);
let upstream = pallas_upstream::n2n::Worker::new(
"relays-new.cardano-mainnet.iohk.io:3001".into(),
764824073,
StaticCursor,
output_port,
);
b.connect_output(send);
let b = b.spawn().unwrap();
let mut w = Witness {
let mut witness = Witness {
input: Default::default(),
};
w.input.connect(receive);
witness.input.connect(receive);
//let f = gasket::runtime::spawn_stage(f, Default::default(), Some("faker"));
let w = gasket::runtime::spawn_stage(w, Default::default(), Some("witness"));
let upstream = gasket::runtime::spawn_stage(upstream, Default::default(), Some("upstream"));
let witness = gasket::runtime::spawn_stage(witness, Default::default(), Some("witness"));
let d = gasket::daemon::Daemon(vec![w]);
let daemon = gasket::daemon::Daemon(vec![upstream, witness]);
d.block();
daemon.block();
}