add tracing to listener

This commit is contained in:
ranfdev
2024-11-21 16:49:15 +01:00
parent dd5c334171
commit eac70af6a0
3 changed files with 120 additions and 124 deletions

View File

@ -1,13 +1,13 @@
macro_rules! send_command {
($self:expr, $command:expr) => {{
let (resp_tx, rx) = oneshot::channel();
let (resp_tx, resp_rx) = oneshot::channel();
use anyhow::Context;
$self
.command_tx
.send($command(resp_tx))
.await
.context("Actor mailbox error")?;
rx.await.context("Actor response error")?
resp_rx.await.context("Actor response error")?
}};
}

View File

@ -10,7 +10,7 @@ use tokio::{
sync::{mpsc, oneshot},
};
use tokio_stream::wrappers::LinesStream;
use tracing::{debug, error, info};
use tracing::{debug, error, info, warn, Instrument, Span};
use crate::credentials::Credentials;
use crate::http_client::HttpClient;
@ -106,30 +106,34 @@ pub struct ListenerActor {
impl ListenerActor {
pub async fn run_loop(mut self) {
let span = tracing::info_span!("listener_loop", topic = %self.config.topic);
async {
let mut commands_rx = self.commands_rx.take().unwrap();
loop {
select! {
_ = self.run_supervised_loop() => {
// the supervised loop cannot fail. If it finished, don't restart.
info!("supervised loop ended");
break;
},
cmd = commands_rx.recv() => {
match cmd {
Some(ListenerCommand::Restart) => {
info!("Received restart command");
info!("restarting listener");
continue;
}
Some(ListenerCommand::Shutdown) => {
info!("Received shutdown command");
info!("shutting down listener");
break;
}
Some(ListenerCommand::GetState(tx)) => {
info!("Received get state command");
debug!("getting listener state");
let state = self.state.clone();
let _ = tx.send(state);
if tx.send(state).is_err() {
warn!("failed to send state - receiver dropped");
}
}
None => {
error!("Channel closed for ListenerActor");
error!("command channel closed");
break;
}
}
@ -137,6 +141,9 @@ impl ListenerActor {
}
}
}
.instrument(span)
.await;
}
async fn set_state(&mut self, state: ConnectionState) {
self.state = state.clone();
@ -146,7 +153,8 @@ impl ListenerActor {
.unwrap();
}
async fn run_supervised_loop(&mut self) {
dbg!("supervised");
let span = tracing::info_span!("supervised_loop");
async {
let retrier = || {
crate::retry::WaitExponentialRandom::builder()
.min(Duration::from_secs(1))
@ -161,25 +169,36 @@ impl ListenerActor {
let uptime = std::time::Instant::now().duration_since(start_time);
// Reset retry delay to minimum if uptime was decent enough
if uptime > Duration::from_secs(60 * 4) {
debug!("resetting retry delay due to sufficient uptime");
retry = retrier();
}
error!(error = ?e);
error!(error = ?e, "connection error");
self.set_state(ConnectionState::Reconnecting {
retry_count: retry.count(),
delay: retry.next_delay(),
error: Some(Arc::new(e)),
})
.await;
info!(delay = ?retry.next_delay(), "restarting");
info!(delay = ?retry.next_delay(), "waiting before reconnect attempt");
retry.wait().await;
} else {
break;
}
}
}
.instrument(span)
.await;
}
async fn recv_and_forward_loop(&mut self) -> anyhow::Result<()> {
let span = tracing::info_span!("receive_loop",
endpoint = %self.config.endpoint,
topic = %self.config.topic,
since = %self.config.since
);
async {
let creds = self.config.credentials.get(&self.config.endpoint);
debug!("creating request");
let req = topic_request(
&self.config.http_client,
&self.config.endpoint,
@ -188,6 +207,8 @@ impl ListenerActor {
creds.as_ref().map(|x| x.username.as_str()),
creds.as_ref().map(|x| x.password.as_str()),
);
debug!("executing request");
let res = self.config.http_client.execute(req?).await?;
let res = res.error_for_status()?;
let reader = tokio_util::io::StreamReader::new(
@ -198,6 +219,7 @@ impl ListenerActor {
tokio::pin!(stream);
self.set_state(ConnectionState::Connected).await;
info!("connection established");
info!(topic = %&self.config.topic, "listening");
while let Some(msg) = stream.next().await {
@ -212,23 +234,26 @@ impl ListenerActor {
match event {
ServerEvent::Message(msg) => {
debug!("message event");
debug!(id = %msg.id, "forwarding message");
self.event_tx
.send(ListenerEvent::Message(msg))
.await
.unwrap();
}
ServerEvent::KeepAlive { .. } => {
debug!("keepalive event");
ServerEvent::KeepAlive { id, .. } => {
debug!(id = %id, "received keepalive");
}
ServerEvent::Open { .. } => {
debug!("open event");
ServerEvent::Open { id, .. } => {
debug!(id = %id, "received open event");
}
}
}
Ok(())
}
.instrument(span)
.await
}
}
// Reliable listener implementation
@ -323,13 +348,6 @@ mod tests {
ListenerEvent::ConnectionStateChanged(ConnectionState::Connected { .. }),
]
));
// assert!(matches!(
// listener,
// ListenerEvent::Error { .. },
// ListenerEvent::Disconnected { .. },
// ListenerEvent::Connected { .. },
// ));
});
local_set.await;
}
@ -372,26 +390,4 @@ mod tests {
});
local_set.await;
}
#[tokio::test]
async fn integration_connects_sends_receives_simple() {
let local_set = LocalSet::new();
local_set.spawn_local(async {
let http_client = HttpClient::new(reqwest::Client::new());
let credentials = Credentials::new_nullable(vec![]).await.unwrap();
let config = ListenerConfig {
http_client,
credentials,
endpoint: "http://localhost:8000".to_string(),
topic: "test".to_string(),
since: 0,
};
let listener = ListenerHandle::new(config.clone());
// assert_event_matches!(listener, ListenerEvent::Connected { .. },);
});
local_set.await;
}
}

View File

@ -394,6 +394,6 @@ impl NullNetworkMonitor {
impl NetworkMonitorProxy for NullNetworkMonitor {
fn listen(&self) -> Pin<Box<dyn Stream<Item = ()>>> {
todo!()
Box::pin(futures::stream::empty())
}
}