feat: add usage snapshot event and enhance compaction logic

- Introduced `UsageSnapshot` event to track token usage during conversation runs.
- Updated `RunEngine` to emit usage snapshots, providing better visibility into token consumption.
- Refactored compaction logic to utilize a new `compaction_estimate` function for improved token budget management.
- Added tests to validate timeout constants for blob synchronization and ensure correct behavior of usage tracking during compaction.
This commit is contained in:
leokun
2026-09-01 20:04:35 +08:00
parent e768980dad
commit 76417e005b
6 changed files with 81 additions and 27 deletions
+8
View File
@@ -399,6 +399,14 @@ impl ConversationOutput {
.unwrap_or_else(|_| serde_json::json!({})) .unwrap_or_else(|_| serde_json::json!({}))
}; };
} }
RunEvent::UsageSnapshot(usage) => {
if !self.context.compacting {
if let Some(output_tokens) = usage.output_tokens {
self.handle.emit(&events::token_delta(output_tokens))?;
}
context_tokens = usage.context_input_tokens;
}
}
RunEvent::Usage(usage) => { RunEvent::Usage(usage) => {
if !self.context.compacting { if !self.context.compacting {
if let Some(output_tokens) = usage.output_tokens { if let Some(output_tokens) = usage.output_tokens {
+20 -2
View File
@@ -20,6 +20,9 @@ use crate::{
type BlobSetSender = oneshot::Sender<Result<()>>; type BlobSetSender = oneshot::Sender<Result<()>>;
const SET_TIMEOUT: Duration = Duration::from_secs(30 * 60);
const GET_TIMEOUT: Duration = Duration::from_secs(10 * 60);
#[derive(Clone)] #[derive(Clone)]
pub struct BlobSynchronizer { pub struct BlobSynchronizer {
inner: Arc<Inner>, inner: Arc<Inner>,
@@ -130,7 +133,7 @@ impl BlobSynchronizer {
let result = tokio::select! { let result = tokio::select! {
result = receiver => result.map_err(|_| Error::Protocol("KV SET response channel closed".into()))?, result = receiver => result.map_err(|_| Error::Protocol("KV SET response channel closed".into()))?,
_ = cancellation.cancelled() => Err(Error::Cancelled), _ = cancellation.cancelled() => Err(Error::Cancelled),
_ = tokio::time::sleep(Duration::from_secs(60)) => Err(Error::Protocol(format!("KV SET timed out: {}", blob_id.to_base64()))), _ = tokio::time::sleep(SET_TIMEOUT) => Err(Error::Protocol(format!("KV SET timed out: {}", blob_id.to_base64()))),
}; };
if result.is_err() { if result.is_err() {
self.inner.set_requests.lock().await.remove(&id); self.inner.set_requests.lock().await.remove(&id);
@@ -183,7 +186,7 @@ impl BlobSynchronizer {
let result = tokio::select! { let result = tokio::select! {
result = receiver => result.map_err(|_| Error::Protocol("KV GET response channel closed".into()))?, result = receiver => result.map_err(|_| Error::Protocol("KV GET response channel closed".into()))?,
_ = cancellation.cancelled() => Err(Error::Cancelled), _ = cancellation.cancelled() => Err(Error::Cancelled),
_ = tokio::time::sleep(Duration::from_secs(60)) => Err(Error::Protocol(format!("KV GET timed out: {}", blob_id.to_base64()))), _ = tokio::time::sleep(GET_TIMEOUT) => Err(Error::Protocol(format!("KV GET timed out: {}", blob_id.to_base64()))),
}; };
if result.is_err() { if result.is_err() {
self.inner.get_requests.lock().await.remove(&id); self.inner.get_requests.lock().await.remove(&id);
@@ -318,3 +321,18 @@ impl BlobSynchronizer {
Ok(()) Ok(())
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn set_timeout_allows_slow_cursor_acknowledgements() {
assert_eq!(SET_TIMEOUT, Duration::from_secs(30 * 60));
}
#[test]
fn get_timeout_allows_slow_cursor_responses() {
assert_eq!(GET_TIMEOUT, Duration::from_secs(10 * 60));
}
}
+12 -4
View File
@@ -40,15 +40,23 @@ pub(super) fn estimated_tokens(
.unwrap_or_else(|| estimate_context_tokens(&prepared.prompt, projected_messages)) .unwrap_or_else(|| estimate_context_tokens(&prepared.prompt, projected_messages))
} }
pub(super) fn compaction_estimate(
prepared: &PreparedRun,
projected_messages: &[ProjectedMessage],
anchor: Option<ContextUsageAnchor>,
) -> Option<u64> {
let budget = input_budget(prepared)?;
let estimated = estimated_tokens(prepared, projected_messages, anchor);
(estimated > budget).then_some(estimated)
}
#[cfg(test)]
pub(super) fn should_compact( pub(super) fn should_compact(
prepared: &PreparedRun, prepared: &PreparedRun,
projected_messages: &[ProjectedMessage], projected_messages: &[ProjectedMessage],
anchor: Option<ContextUsageAnchor>, anchor: Option<ContextUsageAnchor>,
) -> bool { ) -> bool {
let Some(budget) = input_budget(prepared) else { compaction_estimate(prepared, projected_messages, anchor).is_some()
return false;
};
estimated_tokens(prepared, projected_messages, anchor) > budget
} }
pub(super) fn validate_compacted( pub(super) fn validate_compacted(
+30 -17
View File
@@ -183,9 +183,21 @@ impl RunEngine {
Ok(history) => history, Ok(history) => history,
Err(error) => return (RunOutcome::Failed(error.into()), usage), Err(error) => return (RunOutcome::Failed(error.into()), usage),
}; };
if prepared.action != RunAction::Compact let compaction_estimate = (prepared.action != RunAction::Compact)
&& super::compaction::should_compact(prepared, &history, context_usage_anchor) .then(|| {
{ super::compaction::compaction_estimate(prepared, &history, context_usage_anchor)
})
.flatten();
if let Some(estimated_tokens) = compaction_estimate {
if emit(
client,
RunEvent::UsageSnapshot(context_usage_snapshot(estimated_tokens)),
)
.await
.is_err()
{
return (client_failure(), usage);
}
match self match self
.auto_compact(prepared, checkpoint, &messages, client, cancellation) .auto_compact(prepared, checkpoint, &messages, client, cancellation)
.await .await
@@ -703,20 +715,6 @@ impl RunEngine {
emit(client, RunEvent::AutoCompactionStarted) emit(client, RunEvent::AutoCompactionStarted)
.await .await
.map_err(|_| client_failure())?; .map_err(|_| client_failure())?;
emit(
client,
RunEvent::Usage(Usage {
input_tokens: Some(0),
context_input_tokens: Some(0),
output_tokens: Some(0),
total_tokens: Some(0),
cache_read_tokens: Some(0),
cache_write_tokens: Some(0),
reasoning_tokens: Some(0),
}),
)
.await
.map_err(|_| client_failure())?;
let provider_call_index = self let provider_call_index = self
.store .store
.begin_provider_call(&prepared.run_id) .begin_provider_call(&prepared.run_id)
@@ -862,6 +860,9 @@ impl RunEngine {
emit(client, RunEvent::AutoCompactionCompleted) emit(client, RunEvent::AutoCompactionCompleted)
.await .await
.map_err(|_| client_failure())?; .map_err(|_| client_failure())?;
emit(client, RunEvent::UsageSnapshot(context_usage_snapshot(0)))
.await
.map_err(|_| client_failure())?;
checkpoint = super::messages::append_batches( checkpoint = super::messages::append_batches(
&self.store, &self.store,
prepared, prepared,
@@ -922,6 +923,18 @@ async fn hydrate_tool_images(
Ok(()) Ok(())
} }
fn context_usage_snapshot(tokens: u64) -> Usage {
Usage {
input_tokens: Some(tokens),
context_input_tokens: Some(tokens),
output_tokens: Some(0),
total_tokens: Some(tokens),
cache_read_tokens: Some(0),
cache_write_tokens: Some(0),
reasoning_tokens: Some(0),
}
}
fn update_context_usage_anchor( fn update_context_usage_anchor(
anchor: &mut Option<ContextUsageAnchor>, anchor: &mut Option<ContextUsageAnchor>,
usage: Usage, usage: Usage,
+1
View File
@@ -129,6 +129,7 @@ pub enum RunEvent {
ToolCallEnd { ToolCallEnd {
index: usize, index: usize,
}, },
UsageSnapshot(Usage),
Usage(Usage), Usage(Usage),
ExecuteToolRound { ExecuteToolRound {
round_id: ToolRoundId, round_id: ToolRoundId,
+10 -4
View File
@@ -268,9 +268,14 @@ async fn automatic_compaction_preflights_provider_input_and_records_rebuilt_toke
assert_eq!(second.summary_started, 1); assert_eq!(second.summary_started, 1);
assert_eq!(second.summary_completed, 1); assert_eq!(second.summary_completed, 1);
assert_eq!( assert_eq!(
&second.interaction_events[..2], &second.interaction_events[..4],
&["summary_started", "token_delta:0"], &[
"automatic compaction must immediately reset Cursor usage" "token_delta:0",
"summary_started",
"summary_completed",
"token_delta:0",
],
"automatic compaction must publish estimated usage before summarizing and zero usage after"
); );
let compacted_tokens = second let compacted_tokens = second
.checkpoints .checkpoints
@@ -530,7 +535,8 @@ async fn run(
output.summary.push_str(&delta.summary) output.summary.push_str(&delta.summary)
} }
Some(pb::interaction_update::Message::SummaryCompleted(_)) => { Some(pb::interaction_update::Message::SummaryCompleted(_)) => {
output.summary_completed += 1 output.summary_completed += 1;
output.interaction_events.push("summary_completed".into());
} }
Some(pb::interaction_update::Message::TurnEnded(_)) => output.turn_ended += 1, Some(pb::interaction_update::Message::TurnEnded(_)) => output.turn_ended += 1,
Some(pb::interaction_update::Message::TokenDelta(delta)) => { Some(pb::interaction_update::Message::TokenDelta(delta)) => {