fix: recover partial session log writes

This commit is contained in:
2026-08-05 18:15:49 +09:00
parent 36df79e561
commit fd391ef705
10 changed files with 488 additions and 104 deletions
+45 -27
View File
@@ -50,6 +50,9 @@ pub enum EngineError {
/// Config warnings (unsupported options)
#[error("Config warnings: {}", .0.iter().map(|w| w.to_string()).collect::<Vec<_>>().join(", "))]
ConfigWarnings(Vec<ConfigWarning>),
/// A durable-history observer rejected an item before it entered history.
#[error("History append failed: {0}")]
HistoryAppend(String),
}
/// Tool registration error
@@ -222,10 +225,10 @@ pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
/// truncation have been applied — i.e. on the same data that
/// enters history.
tool_result_cbs: Vec<Box<dyn Fn(&ToolResult) + Send + Sync>>,
/// History-append callbacks. Invoked for non-streamed items when they
/// are appended to persistent engine history, so upper layers can
/// broadcast those items using history itself as the source of truth.
history_append_cbs: Vec<Box<dyn Fn(&Item) + Send + Sync>>,
/// History-append callbacks. Invoked before non-streamed items enter
/// engine history. An error rejects the item and aborts the turn, allowing
/// upper layers to make durable storage the commit gate.
history_append_cbs: Vec<Box<dyn Fn(&Item) -> Result<(), String> + Send + Sync>>,
/// Request configuration (max_tokens, temperature, etc.)
request_config: RequestConfig,
/// Whether the previous run was interrupted
@@ -498,23 +501,31 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
}
}
/// Register a callback invoked for items appended directly to engine
/// history outside streaming timeline callbacks.
pub fn on_history_append(&mut self, callback: impl Fn(&Item) + Send + Sync + 'static) {
/// Register a fallible callback invoked before an item enters engine
/// history. Returning an error rejects that item and aborts the turn.
pub fn on_history_append(
&mut self,
callback: impl Fn(&Item) -> Result<(), String> + Send + Sync + 'static,
) {
self.history_append_cbs.push(Box::new(callback));
}
fn emit_history_append(&self, item: &Item) {
fn emit_history_append(&self, item: &Item) -> Result<(), EngineError> {
for cb in &self.history_append_cbs {
cb(item);
cb(item).map_err(EngineError::HistoryAppend)?;
}
Ok(())
}
fn append_history_items(&mut self, items: impl IntoIterator<Item = Item>) {
fn append_history_items(
&mut self,
items: impl IntoIterator<Item = Item>,
) -> Result<(), EngineError> {
for item in items {
self.emit_history_append(&item);
self.emit_history_append(&item)?;
self.history.push(item);
}
Ok(())
}
fn request_trace_payload(&self, request: &Request) -> Value {
@@ -1125,9 +1136,13 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
// These are committed *before* the per-request clone so they
// participate in the LLM request below and get persisted by
// the caller that owns durable history.
let pending = self.interceptor.pending_history_appends().await;
let pending = self
.interceptor
.pending_history_appends()
.await
.map_err(EngineError::HistoryAppend)?;
if !pending.is_empty() {
self.append_history_items(pending);
self.append_history_items(pending)?;
}
// Clone the history into a per-request context. Everything
@@ -1202,7 +1217,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
return Err(EngineError::Aborted(reason));
}
PreRequestAction::YieldWith(items) => {
self.append_history_items(items.clone());
self.append_history_items(items.clone())?;
request_context.extend(items);
info!("Yielded by interceptor after pre-request history append");
for cb in &self.turn_end_cbs {
@@ -1220,7 +1235,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
return Ok(EngineResult::Yielded);
}
PreRequestAction::ContinueWith(items) => {
self.append_history_items(items.clone());
self.append_history_items(items.clone())?;
request_context.extend(items);
}
PreRequestAction::Continue => {}
@@ -1280,7 +1295,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
let assistant_items =
self.build_assistant_items(&reasoning_items, &text_blocks, &[]);
if !assistant_items.is_empty() {
self.append_history_items(assistant_items);
self.append_history_items(assistant_items)?;
}
self.emit_llm_continuation(
current_llm_call,
@@ -1307,7 +1322,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
let tool_calls = self.tool_call_collector.take_collected();
let assistant_items =
self.build_assistant_items(&reasoning_items, &text_blocks, &tool_calls);
self.append_history_items(assistant_items);
self.append_history_items(assistant_items)?;
if tool_calls.is_empty() {
match self.interceptor.on_turn_end(&self.history).await {
@@ -1316,7 +1331,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
return Ok(EngineResult::Finished);
}
TurnEndAction::ContinueWithMessages(additional) => {
self.append_history_items(additional);
self.append_history_items(additional)?;
continue;
}
TurnEndAction::Pause => {
@@ -1610,7 +1625,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
result.is_error,
)
});
self.append_history_items(items);
self.append_history_items(items)?;
Ok(None)
}
Err(err) => {
@@ -1815,12 +1830,15 @@ impl<C: LlmClient> Engine<C, Mutable> {
self.history = items;
}
/// Append items to history and notify history-append observers for each
/// item before it lands. This is the only public Mutable-state API for
/// growing engine history; callers that need session-log persistence must
/// install [`on_history_append`](Self::on_history_append) before calling it.
pub fn append_history(&mut self, items: impl IntoIterator<Item = Item>) {
self.append_history_items(items);
/// Append items to history after every history-append observer accepts the
/// item. This is the only public Mutable-state API for growing engine
/// history; callers that need session-log persistence must install
/// [`on_history_append`](Self::on_history_append) before calling it.
pub fn append_history(
&mut self,
items: impl IntoIterator<Item = Item>,
) -> Result<(), EngineError> {
self.append_history_items(items)
}
/// Truncate history without emitting append callbacks.
@@ -1969,9 +1987,9 @@ impl<C: LlmClient> Engine<C, Locked> {
PromptAction::Continue => Vec::new(),
PromptAction::ContinueWith(items) => items,
};
self.append_history_items(std::iter::once(user_item));
self.append_history_items(std::iter::once(user_item))?;
if !extras.is_empty() {
self.append_history_items(extras);
self.append_history_items(extras)?;
}
let result = self.run_turn_loop().await;
self.finalize_interruption(result).await
+2 -2
View File
@@ -158,8 +158,8 @@ pub trait Interceptor: Send + Sync {
/// reproducible per-request transformations (pruning, content
/// trimming, cache anchors) that depend only on the existing
/// history.
async fn pending_history_appends(&self) -> Vec<Item> {
Vec::new()
async fn pending_history_appends(&self) -> Result<Vec<Item>, String> {
Ok(Vec::new())
}
/// Called before each LLM request. The context starts as a clone
+76 -15
View File
@@ -44,12 +44,18 @@ fn test_mutable_history_manipulation() {
assert!(engine.history().is_empty());
// Add to history
engine.append_history(vec![Item::user_message("Hello")]);
engine.append_history(vec![Item::assistant_message("Hi there!")]);
engine
.append_history(vec![Item::user_message("Hello")])
.unwrap();
engine
.append_history(vec![Item::assistant_message("Hi there!")])
.unwrap();
assert_eq!(engine.history().len(), 2);
// Append to history via the callback-aware API.
engine.append_history(vec![Item::user_message("How are you?")]);
engine
.append_history(vec![Item::user_message("How are you?")])
.unwrap();
assert_eq!(engine.history().len(), 3);
// Clear history
@@ -86,15 +92,20 @@ fn test_mutable_append_history() {
if let Some(text) = item.as_text() {
observed_for_callback.lock().unwrap().push(text.to_string());
}
Ok(())
});
engine.append_history(vec![Item::user_message("First")]);
engine
.append_history(vec![Item::user_message("First")])
.unwrap();
engine.append_history(vec![
engine
.append_history(vec![
Item::assistant_message("Response 1"),
Item::user_message("Second"),
Item::assistant_message("Response 2"),
]);
])
.unwrap();
assert_eq!(engine.history().len(), 4);
assert_eq!(
@@ -157,6 +168,40 @@ fn test_mutable_can_register_tool() {
engine.register_tool(tool.definition());
}
/// A durable-history failure on a tool call must stop the turn before the
/// tool can produce an external side effect.
#[tokio::test]
async fn history_append_failure_stops_before_tool_execution() {
let client = MockLlmClient::new(vec![
Event::tool_use_start(0, "call_1", "count_tool"),
Event::tool_input_delta(0, r#"{}"#),
Event::tool_use_stop(0),
Event::Status(StatusEvent {
status: ResponseStatus::Completed,
}),
]);
let tool = CountingTool::new("count_tool");
let mut engine = Engine::new(client);
engine.register_tool(tool.definition());
engine.on_history_append(|item| {
if item.is_tool_call() {
Err("simulated ENOSPC".to_string())
} else {
Ok(())
}
});
let mut engine = engine.lock();
let error = engine.run("use the tool").await.unwrap_err();
assert!(
matches!(error, EngineError::HistoryAppend(ref message) if message == "simulated ENOSPC")
);
assert_eq!(tool.call_count(), 0);
assert_eq!(engine.history().len(), 1);
assert_eq!(engine.history()[0].as_text(), Some("use the tool"));
}
// =============================================================================
// State Transition Tests
// =============================================================================
@@ -168,8 +213,12 @@ fn test_lock_transition() {
let mut engine = Engine::new(client);
engine.set_system_prompt("System");
engine.append_history(vec![Item::user_message("Hello")]);
engine.append_history(vec![Item::assistant_message("Hi")]);
engine
.append_history(vec![Item::user_message("Hello")])
.unwrap();
engine
.append_history(vec![Item::assistant_message("Hi")])
.unwrap();
// Lock
let locked_engine = engine.lock();
@@ -186,14 +235,18 @@ fn test_unlock_transition() {
let client = MockLlmClient::new(vec![]);
let mut engine = Engine::new(client);
engine.append_history(vec![Item::user_message("Hello")]);
engine
.append_history(vec![Item::user_message("Hello")])
.unwrap();
let locked_engine = engine.lock();
// Unlock
let mut engine = locked_engine.unlock();
// History operations are available again in Mutable state
engine.append_history(vec![Item::assistant_message("Hi")]);
engine
.append_history(vec![Item::assistant_message("Hi")])
.unwrap();
engine.clear_history();
assert!(engine.history().is_empty());
}
@@ -316,8 +369,12 @@ async fn test_locked_prefix_len_tracking() {
let mut engine = Engine::new(client);
// Add items beforehand
engine.append_history(vec![Item::user_message("Pre-existing message 1")]);
engine.append_history(vec![Item::assistant_message("Pre-existing response 1")]);
engine
.append_history(vec![Item::user_message("Pre-existing message 1")])
.unwrap();
engine
.append_history(vec![Item::assistant_message("Pre-existing response 1")])
.unwrap();
assert_eq!(engine.history().len(), 2);
@@ -387,10 +444,12 @@ async fn test_unlock_edit_relock() {
]]);
let mut engine = Engine::new(client);
engine.append_history(vec![
engine
.append_history(vec![
Item::user_message("Hello"),
Item::assistant_message("Hi"),
]);
])
.unwrap();
// Lock -> Unlock
let locked = engine.lock();
@@ -400,7 +459,9 @@ async fn test_unlock_edit_relock() {
// Edit history
unlocked.clear_history();
unlocked.append_history(vec![Item::user_message("Fresh start")]);
unlocked
.append_history(vec![Item::user_message("Fresh start")])
.unwrap();
// Re-lock
let relocked = unlocked.lock();
+107 -12
View File
@@ -20,17 +20,23 @@ use crate::segment_log::LogEntry;
use crate::store::{Store, StoreError};
use crate::{SegmentId, SessionId};
use std::fs;
use std::io::Write;
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::SystemTime;
/// Filesystem-backed JSONL store.
///
/// Each segment is stored as a single `.jsonl` file with one [`LogEntry`]
/// per line. Writes use append mode for crash safety.
/// per line. A trailing line is committed only once its newline has been
/// written; readers ignore an unterminated tail and the next append removes it.
#[derive(Clone)]
pub struct FsStore {
root: PathBuf,
/// Serialises append repair + write + rollback across clones. A failed
/// `write_all` may have extended the file, so rollback is safe only while
/// no sibling writer can append behind it.
append_lock: Arc<Mutex<()>>,
}
impl FsStore {
@@ -39,7 +45,10 @@ impl FsStore {
pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> {
let root = root.into();
fs::create_dir_all(&root)?;
Ok(Self { root })
Ok(Self {
root,
append_lock: Arc::new(Mutex::new(())),
})
}
/// Return the filesystem root used by this store.
@@ -101,22 +110,62 @@ impl FsStore {
}
fn append_line(&self, path: &Path, line: &str) -> Result<(), StoreError> {
let _guard = self
.append_lock
.lock()
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
let mut file = fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.append(true)
.open(path)?;
file.write_all(line.as_bytes())?;
file.write_all(b"\n")?;
// Append-mode write is the durability boundary; an explicit
// `sync_all` here would multiply latency by ~10× for no gain
// since the kernel already orders concurrent `O_APPEND` writes.
let committed_len = Self::truncate_uncommitted_tail(&mut file)?;
let mut record = Vec::with_capacity(line.len() + 1);
record.extend_from_slice(line.as_bytes());
record.push(b'\n');
if let Err(write_error) = file.write_all(&record) {
return match file.set_len(committed_len) {
Ok(()) => Err(write_error.into()),
Err(rollback_error) => Err(std::io::Error::new(
rollback_error.kind(),
format!(
"session append failed ({write_error}) and partial-write rollback failed: {rollback_error}"
),
)
.into()),
};
}
Ok(())
}
fn parse_jsonl<T: serde::de::DeserializeOwned>(content: &str) -> Result<Vec<T>, StoreError> {
/// Return only newline-terminated records. A process interruption or
/// ENOSPC can leave the final UTF-8 code point / JSON object incomplete;
/// without a newline that record never crossed the commit boundary.
fn complete_jsonl_prefix(content: &[u8]) -> &[u8] {
if content.last() == Some(&b'\n') {
return content;
}
match content.iter().rposition(|byte| *byte == b'\n') {
Some(index) => &content[..=index],
None => &[],
}
}
fn parse_jsonl<T: serde::de::DeserializeOwned>(content: &[u8]) -> Result<Vec<T>, StoreError> {
let complete = Self::complete_jsonl_prefix(content);
let content = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt {
line: complete[..error.valid_up_to()]
.iter()
.filter(|byte| **byte == b'\n')
.count()
+ 1,
message: error.to_string(),
})?;
let mut entries = Vec::new();
for (i, line) in content.lines().enumerate() {
if line.trim().is_empty() {
@@ -130,6 +179,43 @@ impl FsStore {
}
Ok(entries)
}
/// Remove a prior unterminated record and return the committed file size.
/// Scans backwards in bounded chunks so repairing a large session does not
/// require loading it into memory.
fn truncate_uncommitted_tail(file: &mut fs::File) -> std::io::Result<u64> {
const SCAN_BYTES: usize = 8 * 1024;
let len = file.metadata()?.len();
if len == 0 {
return Ok(0);
}
file.seek(SeekFrom::End(-1))?;
let mut last = [0_u8; 1];
file.read_exact(&mut last)?;
if last[0] == b'\n' {
return Ok(len);
}
let mut end = len;
let mut buffer = [0_u8; SCAN_BYTES];
while end > 0 {
let start = end.saturating_sub(SCAN_BYTES as u64);
let chunk_len = (end - start) as usize;
file.seek(SeekFrom::Start(start))?;
file.read_exact(&mut buffer[..chunk_len])?;
if let Some(index) = buffer[..chunk_len].iter().rposition(|byte| *byte == b'\n') {
let committed_len = start + index as u64 + 1;
file.set_len(committed_len)?;
return Ok(committed_len);
}
end = start;
}
file.set_len(0)?;
Ok(0)
}
}
impl Store for FsStore {
@@ -152,7 +238,7 @@ impl Store for FsStore {
if !path.exists() {
return Err(StoreError::NotFound(segment_id));
}
let content = fs::read_to_string(&path)?;
let content = fs::read(&path)?;
Self::parse_jsonl(&content)
}
@@ -251,8 +337,17 @@ impl Store for FsStore {
if !path.exists() {
return Err(StoreError::NotFound(segment_id));
}
let content = fs::read_to_string(&path)?;
Ok(content.lines().filter(|l| !l.trim().is_empty()).count())
let content = fs::read(&path)?;
let complete = Self::complete_jsonl_prefix(&content);
let complete = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt {
line: complete[..error.valid_up_to()]
.iter()
.filter(|byte| **byte == b'\n')
.count()
+ 1,
message: error.to_string(),
})?;
Ok(complete.lines().filter(|l| !l.trim().is_empty()).count())
}
fn append_trace(
+38 -5
View File
@@ -69,7 +69,10 @@ pub enum LogEntry {
/// Field name is `trigger` (not `kind`) because the LogEntry
/// serde tag already occupies `"kind"`.
///
/// Marker only — replay does not mutate `RestoredState`.
/// Replay marks the run interrupted until a terminal `RunCompleted`,
/// `RunErrored`, or `PausedTurnAbandoned` entry proves how it ended. This
/// makes a process/disk failure between Invoke and its terminal record
/// restore conservatively instead of re-running a dangling tool call.
Invoke { ts: u64, trigger: InvokeKind },
/// User input accepted at submit time. Carries the original typed
@@ -236,9 +239,9 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState {
state.history = history.iter().cloned().map(Item::from).collect();
}
LogEntry::Invoke { .. } => {
// Marker only; no state mutation. The trailing
// UserInput / SystemItem / TurnEnd entries carry all
// replay-relevant data.
// A terminal run record below clears or refines this. If the
// log ends first, restore must treat the turn as interrupted.
state.last_run_interrupted = true;
}
LogEntry::UserInput { segments, .. } => {
let text = Segment::flatten_to_text(segments);
@@ -368,6 +371,35 @@ mod tests {
assert!(!state.last_run_interrupted);
}
#[test]
fn replay_incomplete_invoke_is_interrupted() {
let state = collect_state(&[
LogEntry::SegmentStart {
ts: 1000,
session_id: uuid::Uuid::nil(),
system_prompt: None,
config: RequestConfig::default(),
history: vec![],
forked_from: None,
compacted_from: None,
},
LogEntry::Invoke {
ts: 2000,
trigger: InvokeKind::UserSend,
},
LogEntry::UserInput {
ts: 2001,
segments: vec![Segment::text("run a tool")],
},
LogEntry::AssistantItem {
ts: 3000,
item: Item::tool_call("call_1", "side_effect", "{}").into(),
},
]);
assert!(state.last_run_interrupted);
}
#[test]
fn replay_with_tool_calls() {
let state = collect_state(&[
@@ -546,7 +578,7 @@ mod tests {
}
#[test]
fn replay_invoke_marker_does_not_mutate_state() {
fn replay_invoke_marker_only_mutates_interrupted_state() {
let state = collect_state(&[
LogEntry::SegmentStart {
ts: 0,
@@ -576,6 +608,7 @@ mod tests {
]);
assert_eq!(state.history.len(), 1);
assert_eq!(state.turn_count, 1);
assert!(state.last_run_interrupted);
}
#[test]
+2 -2
View File
@@ -40,8 +40,8 @@ pub enum StoreError {
pub trait Store: Send + Sync {
/// Append a single log entry to the segment log.
///
/// One line per call. The kernel orders concurrent `O_APPEND` writes
/// for lines < `PIPE_BUF`, so user-space serialization is unnecessary.
/// One committed line per successful call. Implementations must not expose
/// a failed call's partial record as committed data on later reads.
fn append(
&self,
session_id: SessionId,
@@ -3,6 +3,7 @@ use llm_engine::llm_client::types::{Item, RequestConfig};
use session_store::{
FsStore, LogEntry, Store, TraceEntry, collect_state, new_segment_id, new_session_id,
};
use std::io::Write;
fn nil_session_start(ts: u64, session_id: uuid::Uuid) -> LogEntry {
LogEntry::SegmentStart {
@@ -224,6 +225,71 @@ fn read_entry_count_matches_append_tally() {
assert_eq!(store.read_entry_count(sid, segid).unwrap(), entries.len());
}
#[test]
fn unterminated_utf8_tail_is_ignored_and_replaced_on_append() {
let dir = tempfile::tempdir().unwrap();
let store = FsStore::new(dir.path()).unwrap();
let sid = new_session_id();
let segid = new_segment_id();
let path = dir
.path()
.join(sid.to_string())
.join(format!("{segid}.jsonl"));
store
.append(sid, segid, &nil_session_start(1, sid))
.unwrap();
std::fs::OpenOptions::new()
.append(true)
.open(&path)
.unwrap()
// First byte of a three-byte UTF-8 code point, matching an ENOSPC
// partial write observed in a real session log.
.write_all(&[0xe3])
.unwrap();
assert_eq!(store.read_all(sid, segid).unwrap().len(), 1);
assert_eq!(store.read_entry_count(sid, segid).unwrap(), 1);
let next = LogEntry::UserInput {
ts: 2,
segments: vec![protocol::Segment::text("recovered")],
};
store.append(sid, segid, &next).unwrap();
let bytes = std::fs::read(&path).unwrap();
assert!(std::str::from_utf8(&bytes).is_ok());
assert_eq!(store.read_all(sid, segid).unwrap().len(), 2);
assert_eq!(store.read_entry_count(sid, segid).unwrap(), 2);
}
#[test]
fn newline_terminated_invalid_utf8_is_reported_as_corruption() {
let dir = tempfile::tempdir().unwrap();
let store = FsStore::new(dir.path()).unwrap();
let sid = new_session_id();
let segid = new_segment_id();
let path = dir
.path()
.join(sid.to_string())
.join(format!("{segid}.jsonl"));
store
.append(sid, segid, &nil_session_start(1, sid))
.unwrap();
std::fs::OpenOptions::new()
.append(true)
.open(path)
.unwrap()
.write_all(&[0xe3, b'\n'])
.unwrap();
assert!(matches!(
store.read_all(sid, segid),
Err(session_store::StoreError::Corrupt { line: 2, .. })
));
}
#[test]
fn lookup_session_of_finds_owning_session() {
let dir = tempfile::tempdir().unwrap();
+27 -15
View File
@@ -112,13 +112,14 @@ impl WorkerInterceptor {
/// `Item::system_message`s reach the worker via
/// `ContinueWith` / `pending_history_appends`, so on-disk order
/// matches worker-history order.
fn commit_system_items(&self, items: &[SystemItem]) {
fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> {
let Some(writer) = self.log_writer.as_ref() else {
return;
return Ok(());
};
for item in items {
writer.commit_system_item(item.clone());
writer.commit_system_item(item.clone())?;
}
Ok(())
}
fn current_turn_index(&self) -> usize {
@@ -194,15 +195,17 @@ impl Interceptor for WorkerInterceptor {
// `Item::system_message`s, so on-disk order matches
// worker-history order.
let items: Vec<Item> = extras.iter().map(SystemItem::to_history_item).collect();
self.commit_system_items(&extras);
PromptAction::ContinueWith(items)
match self.commit_system_items(&extras) {
Ok(()) => PromptAction::ContinueWith(items),
Err(error) => PromptAction::Cancel(format!("session persistence failed: {error}")),
}
}
}
async fn pending_history_appends(&self) -> Vec<Item> {
async fn pending_history_appends(&self) -> Result<Vec<Item>, String> {
let drained = self.pending_notifies.drain();
if drained.is_empty() {
return Vec::new();
return Ok(Vec::new());
}
let mut system_items: Vec<SystemItem> = Vec::with_capacity(drained.len());
@@ -231,8 +234,9 @@ impl Interceptor for WorkerInterceptor {
}
}
}
self.commit_system_items(&system_items);
items
self.commit_system_items(&system_items)
.map_err(|error| format!("session persistence failed: {error}"))?;
Ok(items)
}
async fn pre_llm_request(&self, context: &mut Vec<Item>) -> PreRequestAction {
@@ -278,7 +282,9 @@ impl Interceptor for WorkerInterceptor {
let current_tokens = self.estimated_tokens(effective_context.as_ref());
if self.request_threshold_exceeded(current_tokens, effective_context.as_ref()) {
self.commit_system_items(&system_items);
if let Err(error) = self.commit_system_items(&system_items) {
return PreRequestAction::Cancel(format!("session persistence failed: {error}"));
}
return if appended_items.is_empty() {
PreRequestAction::Yield
} else {
@@ -292,8 +298,10 @@ impl Interceptor for WorkerInterceptor {
if system_items.is_empty() {
return PreRequestAction::Continue;
}
self.commit_system_items(&system_items);
PreRequestAction::ContinueWith(appended_items)
match self.commit_system_items(&system_items) {
Ok(()) => PreRequestAction::ContinueWith(appended_items),
Err(error) => PreRequestAction::Cancel(format!("session persistence failed: {error}")),
}
}
async fn pre_tool_call(&self, info: &mut ToolCallInfo) -> PreToolAction {
@@ -451,13 +459,17 @@ mod tests {
}
impl SystemItemCommitter for RecordingSystemItemCommitter {
fn commit_log_entry(&self, entry: session_store::LogEntry) {
fn commit_log_entry(
&self,
entry: session_store::LogEntry,
) -> Result<(), session_store::StoreError> {
if let session_store::LogEntry::SystemItem { item, .. } = entry {
self.committed
.lock()
.expect("committed system-item list poisoned")
.push(item);
}
Ok(())
}
}
@@ -1034,7 +1046,7 @@ mod tests {
None,
);
let items = interceptor.pending_history_appends().await;
let items = interceptor.pending_history_appends().await.unwrap();
assert_eq!(items.len(), 2);
let first = items[0].as_text().unwrap_or_default();
let second = items[1].as_text().unwrap_or_default();
@@ -1048,7 +1060,7 @@ mod tests {
);
// Empty buffer → empty Vec (no synthesised items).
let again = interceptor.pending_history_appends().await;
let again = interceptor.pending_history_appends().await.unwrap();
assert!(again.is_empty());
}
+74 -21
View File
@@ -568,9 +568,8 @@ where
St: Store + Clone,
{
/// Append `entry` to the log: disk write → counter bump → in-memory
/// mirror push → broadcast. The kernel orders concurrent `O_APPEND`
/// writes for `< PIPE_BUF` lines, so no user-space serialization is
/// needed across appenders.
/// mirror push → broadcast. The Store owns physical write ordering and
/// partial-write recovery; publication happens only after it returns Ok.
pub fn append_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
let loc = self.state.location();
self.store.append(loc.session_id, loc.segment_id, &entry)?;
@@ -602,13 +601,13 @@ where
/// interceptor commit `SystemItem`s without being generic over the
/// concrete `Store` type.
pub trait SystemItemCommitter: Send + Sync {
fn commit_log_entry(&self, entry: LogEntry);
fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError>;
fn commit_system_item(&self, item: SystemItem) {
fn commit_system_item(&self, item: SystemItem) -> Result<(), StoreError> {
self.commit_log_entry(LogEntry::SystemItem {
ts: segment_log::now_millis(),
item,
});
})
}
}
@@ -616,10 +615,8 @@ impl<St> SystemItemCommitter for LogWriterHandle<St>
where
St: Store + Clone + Send + Sync + 'static,
{
fn commit_log_entry(&self, entry: LogEntry) {
if let Err(err) = self.append_entry(entry) {
warn!(error = %err, "session log entry commit failed; dropping");
}
fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
self.append_entry(entry)
}
}
@@ -914,7 +911,7 @@ impl<C: LlmClient + Clone + 'static, St: Store + Clone + 'static> Worker<C, St>
let writer = self.log_writer_handle();
self.engine_mut().on_history_append(move |item| {
if item.is_user_message() {
return;
return Ok(());
}
if matches!(
item,
@@ -923,12 +920,12 @@ impl<C: LlmClient + Clone + 'static, St: Store + Clone + 'static> Worker<C, St>
..
}
) {
return;
return Ok(());
}
let entry = session_store::classify_history_item(item, segment_log::now_millis());
if let Err(err) = writer.append_entry(entry) {
warn!(error = %err, "history append commit failed; dropping");
}
writer
.append_entry(entry)
.map_err(|error| error.to_string())
});
if self.manifest.session.record_event_trace {
let writer = self.log_writer_handle();
@@ -1206,7 +1203,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
},
})?;
self.engine_mut()
.append_history(std::iter::once(llm_engine::Item::system_message(body)));
.append_history(std::iter::once(llm_engine::Item::system_message(body)))?;
Ok(activation)
}
@@ -1251,9 +1248,8 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
}
/// Append `entry` to the session log AND publish it through the
/// broadcast sink. No user-space serialization is needed across
/// concurrent appenders — the kernel orders `O_APPEND` writes for
/// lines smaller than `PIPE_BUF`.
/// broadcast sink. The Store is the commit boundary: a failed write is
/// never counted or published.
pub(crate) fn commit_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
let loc = self.segment_state.location();
self.store.append(loc.session_id, loc.segment_id, &entry)?;
@@ -2083,7 +2079,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
&tool_result_summary,
);
if !closures.is_empty() {
self.engine_mut().append_history(closures);
self.engine_mut().append_history(closures)?;
}
self.commit_entry(LogEntry::SystemItem {
ts: segment_log::now_millis(),
@@ -2094,7 +2090,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
self.engine_mut()
.append_history(std::iter::once(llm_engine::Item::system_message(
system_note,
)));
)))?;
Ok(())
}
@@ -2164,6 +2160,12 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
),
"run_for_notification expects a non-UserSend InvokeKind; got {kind:?}"
);
// This is a fresh Invoke, not an explicit resume of the interrupted
// turn. Close any dangling tool calls before an auto-run notification
// can enter `Engine::resume` and execute them again after a crash.
if self.engine.as_ref().unwrap().last_run_interrupted() {
self.apply_interrupt_prep()?;
}
self.prepare_for_run().await?;
// IDLE → active marker for the buffered notification / worker-event
@@ -5914,6 +5916,57 @@ mod build_summary_prompt_tests {
assert_eq!(interrupt_system_count, 1);
}
#[tokio::test]
async fn notification_run_closes_interrupted_tool_call_before_engine_resume() {
let dir = tempfile::tempdir().unwrap();
let manifest = minimal_manifest();
let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap();
let cwd = dir.path().join("workspace");
std::fs::create_dir_all(&cwd).unwrap();
let scope = Scope::writable(&cwd).unwrap();
let authority = WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone());
let mut worker = Worker::new(
manifest,
Engine::new(NoopClient),
store,
WorkerWorkspaceContext::local_filesystem(None),
authority,
scope,
)
.await
.unwrap();
worker.ensure_segment_head().unwrap();
worker.wire_history_persistence();
let dangling_call = Item::tool_call("call-1", "SideEffect", "{}");
worker
.commit_entry(LogEntry::AssistantItem {
ts: segment_log::now_millis(),
item: dangling_call.clone().into(),
})
.unwrap();
worker.engine_mut().set_history(vec![dangling_call]);
worker.engine_mut().set_last_run_interrupted(true);
worker
.run_for_notification(protocol::InvokeKind::Notify)
.await
.unwrap();
let history = worker.engine().history();
assert!(matches!(
history.get(1),
Some(Item::ToolResult { call_id, .. }) if call_id == "call-1"
));
assert!(matches!(
history.get(2),
Some(Item::Message {
role: Role::System,
..
})
));
}
#[derive(Clone, Copy)]
struct ResidentInjectionGates {
summary: bool,
@@ -0,0 +1,46 @@
# ENOSPC で Session JSONL の末尾が UTF-8 途中切れになる
## 観測
`Companion1` (`worker-runtime-30`) の直近 Segment
`019fce34-eb04-7420-9cc5-0e7e12f5bd67` は、Workdir の Edit tool が
`no storage space` で失敗した後、会話ログ `.jsonl` の末尾が `0xe3` 1 byte
だけで終わっていた。これは 3 byte UTF-8 文字の先頭 byte であり、実際の末尾は
「Workdir のストレー」の次の文字の途中で切れていた。
同 Segment の `.trace.jsonl` は valid UTF-8 だった。再開時の
`stream did not contain valid UTF-8` は provider stream ではなく、Session log を
`fs::read_to_string` した際のエラーだった。
## 原因
`FsStore::append_line` は JSON 本文と改行を別々に `write_all` し、ENOSPC で
部分書き込みになっても元の file length へ戻していなかった。reader も file 全体を
UTF-8 String として読むため、newline に到達していない未コミット末尾だけで Segment
全体を復元不能にしていた。
さらに Engine の history append callback と SystemItem committer は、Store error を
warning にして drop していた。このため disk 上の history を更新できなくても memory
上の history と tool loop が先へ進み得た。
## 改善
- newline を JSONL record の commit marker とする。
- reader は newline 未到達の末尾を未コミット record として無視する。
- 次回 append 前に未コミット末尾を最後の newline まで truncate する。
- append の partial write は append 開始時の file length へ rollback する。
- repair/write/rollback は `FsStore` clone 間で直列化する。
- Engine history append を fallible にし、Store write 成功前には item を memory history
に入れない。
- tool call の永続化に失敗した turn は tool 実行前に停止する。
- SystemItem の commit failure も transient context injection にせず turn error にする。
- `Invoke` 後に terminal run record が無い Segment は restore 時に interrupted とする。
これにより crash/ENOSPC 後の dangling tool call は新しい user turn の前に閉じられ、
side effect を無条件に再実行しない。
## 境界
この修正は process interruption と ENOSPC による trailing partial record を対象にする。
newline 済み record 内部の破損は silent recovery せず `StoreError::Corrupt` のまま扱う。
また append ごとの `fsync` は追加していないため、突然の電源断に対する block-level
durability まで保証するものではない。