fix: recover partial session log writes
This commit is contained in:
@@ -50,6 +50,9 @@ pub enum EngineError {
|
|||||||
/// Config warnings (unsupported options)
|
/// Config warnings (unsupported options)
|
||||||
#[error("Config warnings: {}", .0.iter().map(|w| w.to_string()).collect::<Vec<_>>().join(", "))]
|
#[error("Config warnings: {}", .0.iter().map(|w| w.to_string()).collect::<Vec<_>>().join(", "))]
|
||||||
ConfigWarnings(Vec<ConfigWarning>),
|
ConfigWarnings(Vec<ConfigWarning>),
|
||||||
|
/// A durable-history observer rejected an item before it entered history.
|
||||||
|
#[error("History append failed: {0}")]
|
||||||
|
HistoryAppend(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Tool registration error
|
/// Tool registration error
|
||||||
@@ -222,10 +225,10 @@ pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
|
|||||||
/// truncation have been applied — i.e. on the same data that
|
/// truncation have been applied — i.e. on the same data that
|
||||||
/// enters history.
|
/// enters history.
|
||||||
tool_result_cbs: Vec<Box<dyn Fn(&ToolResult) + Send + Sync>>,
|
tool_result_cbs: Vec<Box<dyn Fn(&ToolResult) + Send + Sync>>,
|
||||||
/// History-append callbacks. Invoked for non-streamed items when they
|
/// History-append callbacks. Invoked before non-streamed items enter
|
||||||
/// are appended to persistent engine history, so upper layers can
|
/// engine history. An error rejects the item and aborts the turn, allowing
|
||||||
/// broadcast those items using history itself as the source of truth.
|
/// upper layers to make durable storage the commit gate.
|
||||||
history_append_cbs: Vec<Box<dyn Fn(&Item) + Send + Sync>>,
|
history_append_cbs: Vec<Box<dyn Fn(&Item) -> Result<(), String> + Send + Sync>>,
|
||||||
/// Request configuration (max_tokens, temperature, etc.)
|
/// Request configuration (max_tokens, temperature, etc.)
|
||||||
request_config: RequestConfig,
|
request_config: RequestConfig,
|
||||||
/// Whether the previous run was interrupted
|
/// Whether the previous run was interrupted
|
||||||
@@ -498,23 +501,31 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register a callback invoked for items appended directly to engine
|
/// Register a fallible callback invoked before an item enters engine
|
||||||
/// history outside streaming timeline callbacks.
|
/// history. Returning an error rejects that item and aborts the turn.
|
||||||
pub fn on_history_append(&mut self, callback: impl Fn(&Item) + Send + Sync + 'static) {
|
pub fn on_history_append(
|
||||||
|
&mut self,
|
||||||
|
callback: impl Fn(&Item) -> Result<(), String> + Send + Sync + 'static,
|
||||||
|
) {
|
||||||
self.history_append_cbs.push(Box::new(callback));
|
self.history_append_cbs.push(Box::new(callback));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn emit_history_append(&self, item: &Item) {
|
fn emit_history_append(&self, item: &Item) -> Result<(), EngineError> {
|
||||||
for cb in &self.history_append_cbs {
|
for cb in &self.history_append_cbs {
|
||||||
cb(item);
|
cb(item).map_err(EngineError::HistoryAppend)?;
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn append_history_items(&mut self, items: impl IntoIterator<Item = Item>) {
|
fn append_history_items(
|
||||||
|
&mut self,
|
||||||
|
items: impl IntoIterator<Item = Item>,
|
||||||
|
) -> Result<(), EngineError> {
|
||||||
for item in items {
|
for item in items {
|
||||||
self.emit_history_append(&item);
|
self.emit_history_append(&item)?;
|
||||||
self.history.push(item);
|
self.history.push(item);
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn request_trace_payload(&self, request: &Request) -> Value {
|
fn request_trace_payload(&self, request: &Request) -> Value {
|
||||||
@@ -1125,9 +1136,13 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
// These are committed *before* the per-request clone so they
|
// These are committed *before* the per-request clone so they
|
||||||
// participate in the LLM request below and get persisted by
|
// participate in the LLM request below and get persisted by
|
||||||
// the caller that owns durable history.
|
// the caller that owns durable history.
|
||||||
let pending = self.interceptor.pending_history_appends().await;
|
let pending = self
|
||||||
|
.interceptor
|
||||||
|
.pending_history_appends()
|
||||||
|
.await
|
||||||
|
.map_err(EngineError::HistoryAppend)?;
|
||||||
if !pending.is_empty() {
|
if !pending.is_empty() {
|
||||||
self.append_history_items(pending);
|
self.append_history_items(pending)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clone the history into a per-request context. Everything
|
// Clone the history into a per-request context. Everything
|
||||||
@@ -1202,7 +1217,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
return Err(EngineError::Aborted(reason));
|
return Err(EngineError::Aborted(reason));
|
||||||
}
|
}
|
||||||
PreRequestAction::YieldWith(items) => {
|
PreRequestAction::YieldWith(items) => {
|
||||||
self.append_history_items(items.clone());
|
self.append_history_items(items.clone())?;
|
||||||
request_context.extend(items);
|
request_context.extend(items);
|
||||||
info!("Yielded by interceptor after pre-request history append");
|
info!("Yielded by interceptor after pre-request history append");
|
||||||
for cb in &self.turn_end_cbs {
|
for cb in &self.turn_end_cbs {
|
||||||
@@ -1220,7 +1235,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
return Ok(EngineResult::Yielded);
|
return Ok(EngineResult::Yielded);
|
||||||
}
|
}
|
||||||
PreRequestAction::ContinueWith(items) => {
|
PreRequestAction::ContinueWith(items) => {
|
||||||
self.append_history_items(items.clone());
|
self.append_history_items(items.clone())?;
|
||||||
request_context.extend(items);
|
request_context.extend(items);
|
||||||
}
|
}
|
||||||
PreRequestAction::Continue => {}
|
PreRequestAction::Continue => {}
|
||||||
@@ -1280,7 +1295,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
let assistant_items =
|
let assistant_items =
|
||||||
self.build_assistant_items(&reasoning_items, &text_blocks, &[]);
|
self.build_assistant_items(&reasoning_items, &text_blocks, &[]);
|
||||||
if !assistant_items.is_empty() {
|
if !assistant_items.is_empty() {
|
||||||
self.append_history_items(assistant_items);
|
self.append_history_items(assistant_items)?;
|
||||||
}
|
}
|
||||||
self.emit_llm_continuation(
|
self.emit_llm_continuation(
|
||||||
current_llm_call,
|
current_llm_call,
|
||||||
@@ -1307,7 +1322,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
let tool_calls = self.tool_call_collector.take_collected();
|
let tool_calls = self.tool_call_collector.take_collected();
|
||||||
let assistant_items =
|
let assistant_items =
|
||||||
self.build_assistant_items(&reasoning_items, &text_blocks, &tool_calls);
|
self.build_assistant_items(&reasoning_items, &text_blocks, &tool_calls);
|
||||||
self.append_history_items(assistant_items);
|
self.append_history_items(assistant_items)?;
|
||||||
|
|
||||||
if tool_calls.is_empty() {
|
if tool_calls.is_empty() {
|
||||||
match self.interceptor.on_turn_end(&self.history).await {
|
match self.interceptor.on_turn_end(&self.history).await {
|
||||||
@@ -1316,7 +1331,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
return Ok(EngineResult::Finished);
|
return Ok(EngineResult::Finished);
|
||||||
}
|
}
|
||||||
TurnEndAction::ContinueWithMessages(additional) => {
|
TurnEndAction::ContinueWithMessages(additional) => {
|
||||||
self.append_history_items(additional);
|
self.append_history_items(additional)?;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
TurnEndAction::Pause => {
|
TurnEndAction::Pause => {
|
||||||
@@ -1610,7 +1625,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
result.is_error,
|
result.is_error,
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
self.append_history_items(items);
|
self.append_history_items(items)?;
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
@@ -1815,12 +1830,15 @@ impl<C: LlmClient> Engine<C, Mutable> {
|
|||||||
self.history = items;
|
self.history = items;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Append items to history and notify history-append observers for each
|
/// Append items to history after every history-append observer accepts the
|
||||||
/// item before it lands. This is the only public Mutable-state API for
|
/// item. This is the only public Mutable-state API for growing engine
|
||||||
/// growing engine history; callers that need session-log persistence must
|
/// history; callers that need session-log persistence must install
|
||||||
/// install [`on_history_append`](Self::on_history_append) before calling it.
|
/// [`on_history_append`](Self::on_history_append) before calling it.
|
||||||
pub fn append_history(&mut self, items: impl IntoIterator<Item = Item>) {
|
pub fn append_history(
|
||||||
self.append_history_items(items);
|
&mut self,
|
||||||
|
items: impl IntoIterator<Item = Item>,
|
||||||
|
) -> Result<(), EngineError> {
|
||||||
|
self.append_history_items(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Truncate history without emitting append callbacks.
|
/// Truncate history without emitting append callbacks.
|
||||||
@@ -1969,9 +1987,9 @@ impl<C: LlmClient> Engine<C, Locked> {
|
|||||||
PromptAction::Continue => Vec::new(),
|
PromptAction::Continue => Vec::new(),
|
||||||
PromptAction::ContinueWith(items) => items,
|
PromptAction::ContinueWith(items) => items,
|
||||||
};
|
};
|
||||||
self.append_history_items(std::iter::once(user_item));
|
self.append_history_items(std::iter::once(user_item))?;
|
||||||
if !extras.is_empty() {
|
if !extras.is_empty() {
|
||||||
self.append_history_items(extras);
|
self.append_history_items(extras)?;
|
||||||
}
|
}
|
||||||
let result = self.run_turn_loop().await;
|
let result = self.run_turn_loop().await;
|
||||||
self.finalize_interruption(result).await
|
self.finalize_interruption(result).await
|
||||||
|
|||||||
@@ -158,8 +158,8 @@ pub trait Interceptor: Send + Sync {
|
|||||||
/// reproducible per-request transformations (pruning, content
|
/// reproducible per-request transformations (pruning, content
|
||||||
/// trimming, cache anchors) that depend only on the existing
|
/// trimming, cache anchors) that depend only on the existing
|
||||||
/// history.
|
/// history.
|
||||||
async fn pending_history_appends(&self) -> Vec<Item> {
|
async fn pending_history_appends(&self) -> Result<Vec<Item>, String> {
|
||||||
Vec::new()
|
Ok(Vec::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Called before each LLM request. The context starts as a clone
|
/// Called before each LLM request. The context starts as a clone
|
||||||
|
|||||||
@@ -44,12 +44,18 @@ fn test_mutable_history_manipulation() {
|
|||||||
assert!(engine.history().is_empty());
|
assert!(engine.history().is_empty());
|
||||||
|
|
||||||
// Add to history
|
// Add to history
|
||||||
engine.append_history(vec![Item::user_message("Hello")]);
|
engine
|
||||||
engine.append_history(vec![Item::assistant_message("Hi there!")]);
|
.append_history(vec![Item::user_message("Hello")])
|
||||||
|
.unwrap();
|
||||||
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Hi there!")])
|
||||||
|
.unwrap();
|
||||||
assert_eq!(engine.history().len(), 2);
|
assert_eq!(engine.history().len(), 2);
|
||||||
|
|
||||||
// Append to history via the callback-aware API.
|
// Append to history via the callback-aware API.
|
||||||
engine.append_history(vec![Item::user_message("How are you?")]);
|
engine
|
||||||
|
.append_history(vec![Item::user_message("How are you?")])
|
||||||
|
.unwrap();
|
||||||
assert_eq!(engine.history().len(), 3);
|
assert_eq!(engine.history().len(), 3);
|
||||||
|
|
||||||
// Clear history
|
// Clear history
|
||||||
@@ -86,15 +92,20 @@ fn test_mutable_append_history() {
|
|||||||
if let Some(text) = item.as_text() {
|
if let Some(text) = item.as_text() {
|
||||||
observed_for_callback.lock().unwrap().push(text.to_string());
|
observed_for_callback.lock().unwrap().push(text.to_string());
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
});
|
});
|
||||||
|
|
||||||
engine.append_history(vec![Item::user_message("First")]);
|
engine
|
||||||
|
.append_history(vec![Item::user_message("First")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
engine.append_history(vec![
|
engine
|
||||||
Item::assistant_message("Response 1"),
|
.append_history(vec![
|
||||||
Item::user_message("Second"),
|
Item::assistant_message("Response 1"),
|
||||||
Item::assistant_message("Response 2"),
|
Item::user_message("Second"),
|
||||||
]);
|
Item::assistant_message("Response 2"),
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(engine.history().len(), 4);
|
assert_eq!(engine.history().len(), 4);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -157,6 +168,40 @@ fn test_mutable_can_register_tool() {
|
|||||||
engine.register_tool(tool.definition());
|
engine.register_tool(tool.definition());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A durable-history failure on a tool call must stop the turn before the
|
||||||
|
/// tool can produce an external side effect.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn history_append_failure_stops_before_tool_execution() {
|
||||||
|
let client = MockLlmClient::new(vec![
|
||||||
|
Event::tool_use_start(0, "call_1", "count_tool"),
|
||||||
|
Event::tool_input_delta(0, r#"{}"#),
|
||||||
|
Event::tool_use_stop(0),
|
||||||
|
Event::Status(StatusEvent {
|
||||||
|
status: ResponseStatus::Completed,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
let tool = CountingTool::new("count_tool");
|
||||||
|
let mut engine = Engine::new(client);
|
||||||
|
engine.register_tool(tool.definition());
|
||||||
|
engine.on_history_append(|item| {
|
||||||
|
if item.is_tool_call() {
|
||||||
|
Err("simulated ENOSPC".to_string())
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut engine = engine.lock();
|
||||||
|
let error = engine.run("use the tool").await.unwrap_err();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(error, EngineError::HistoryAppend(ref message) if message == "simulated ENOSPC")
|
||||||
|
);
|
||||||
|
assert_eq!(tool.call_count(), 0);
|
||||||
|
assert_eq!(engine.history().len(), 1);
|
||||||
|
assert_eq!(engine.history()[0].as_text(), Some("use the tool"));
|
||||||
|
}
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// State Transition Tests
|
// State Transition Tests
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
@@ -168,8 +213,12 @@ fn test_lock_transition() {
|
|||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
|
|
||||||
engine.set_system_prompt("System");
|
engine.set_system_prompt("System");
|
||||||
engine.append_history(vec![Item::user_message("Hello")]);
|
engine
|
||||||
engine.append_history(vec![Item::assistant_message("Hi")]);
|
.append_history(vec![Item::user_message("Hello")])
|
||||||
|
.unwrap();
|
||||||
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Hi")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Lock
|
// Lock
|
||||||
let locked_engine = engine.lock();
|
let locked_engine = engine.lock();
|
||||||
@@ -186,14 +235,18 @@ fn test_unlock_transition() {
|
|||||||
let client = MockLlmClient::new(vec![]);
|
let client = MockLlmClient::new(vec![]);
|
||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
|
|
||||||
engine.append_history(vec![Item::user_message("Hello")]);
|
engine
|
||||||
|
.append_history(vec![Item::user_message("Hello")])
|
||||||
|
.unwrap();
|
||||||
let locked_engine = engine.lock();
|
let locked_engine = engine.lock();
|
||||||
|
|
||||||
// Unlock
|
// Unlock
|
||||||
let mut engine = locked_engine.unlock();
|
let mut engine = locked_engine.unlock();
|
||||||
|
|
||||||
// History operations are available again in Mutable state
|
// History operations are available again in Mutable state
|
||||||
engine.append_history(vec![Item::assistant_message("Hi")]);
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Hi")])
|
||||||
|
.unwrap();
|
||||||
engine.clear_history();
|
engine.clear_history();
|
||||||
assert!(engine.history().is_empty());
|
assert!(engine.history().is_empty());
|
||||||
}
|
}
|
||||||
@@ -316,8 +369,12 @@ async fn test_locked_prefix_len_tracking() {
|
|||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
|
|
||||||
// Add items beforehand
|
// Add items beforehand
|
||||||
engine.append_history(vec![Item::user_message("Pre-existing message 1")]);
|
engine
|
||||||
engine.append_history(vec![Item::assistant_message("Pre-existing response 1")]);
|
.append_history(vec![Item::user_message("Pre-existing message 1")])
|
||||||
|
.unwrap();
|
||||||
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Pre-existing response 1")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(engine.history().len(), 2);
|
assert_eq!(engine.history().len(), 2);
|
||||||
|
|
||||||
@@ -387,10 +444,12 @@ async fn test_unlock_edit_relock() {
|
|||||||
]]);
|
]]);
|
||||||
|
|
||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
engine.append_history(vec![
|
engine
|
||||||
Item::user_message("Hello"),
|
.append_history(vec![
|
||||||
Item::assistant_message("Hi"),
|
Item::user_message("Hello"),
|
||||||
]);
|
Item::assistant_message("Hi"),
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Lock -> Unlock
|
// Lock -> Unlock
|
||||||
let locked = engine.lock();
|
let locked = engine.lock();
|
||||||
@@ -400,7 +459,9 @@ async fn test_unlock_edit_relock() {
|
|||||||
|
|
||||||
// Edit history
|
// Edit history
|
||||||
unlocked.clear_history();
|
unlocked.clear_history();
|
||||||
unlocked.append_history(vec![Item::user_message("Fresh start")]);
|
unlocked
|
||||||
|
.append_history(vec![Item::user_message("Fresh start")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Re-lock
|
// Re-lock
|
||||||
let relocked = unlocked.lock();
|
let relocked = unlocked.lock();
|
||||||
|
|||||||
@@ -20,17 +20,23 @@ use crate::segment_log::LogEntry;
|
|||||||
use crate::store::{Store, StoreError};
|
use crate::store::{Store, StoreError};
|
||||||
use crate::{SegmentId, SessionId};
|
use crate::{SegmentId, SessionId};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::Write;
|
use std::io::{Read, Seek, SeekFrom, Write};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
|
|
||||||
/// Filesystem-backed JSONL store.
|
/// Filesystem-backed JSONL store.
|
||||||
///
|
///
|
||||||
/// Each segment is stored as a single `.jsonl` file with one [`LogEntry`]
|
/// Each segment is stored as a single `.jsonl` file with one [`LogEntry`]
|
||||||
/// per line. Writes use append mode for crash safety.
|
/// per line. A trailing line is committed only once its newline has been
|
||||||
|
/// written; readers ignore an unterminated tail and the next append removes it.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct FsStore {
|
pub struct FsStore {
|
||||||
root: PathBuf,
|
root: PathBuf,
|
||||||
|
/// Serialises append repair + write + rollback across clones. A failed
|
||||||
|
/// `write_all` may have extended the file, so rollback is safe only while
|
||||||
|
/// no sibling writer can append behind it.
|
||||||
|
append_lock: Arc<Mutex<()>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FsStore {
|
impl FsStore {
|
||||||
@@ -39,7 +45,10 @@ impl FsStore {
|
|||||||
pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> {
|
pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> {
|
||||||
let root = root.into();
|
let root = root.into();
|
||||||
fs::create_dir_all(&root)?;
|
fs::create_dir_all(&root)?;
|
||||||
Ok(Self { root })
|
Ok(Self {
|
||||||
|
root,
|
||||||
|
append_lock: Arc::new(Mutex::new(())),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the filesystem root used by this store.
|
/// Return the filesystem root used by this store.
|
||||||
@@ -101,22 +110,62 @@ impl FsStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn append_line(&self, path: &Path, line: &str) -> Result<(), StoreError> {
|
fn append_line(&self, path: &Path, line: &str) -> Result<(), StoreError> {
|
||||||
|
let _guard = self
|
||||||
|
.append_lock
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
fs::create_dir_all(parent)?;
|
fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
let mut file = fs::OpenOptions::new()
|
let mut file = fs::OpenOptions::new()
|
||||||
.create(true)
|
.create(true)
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
.append(true)
|
.append(true)
|
||||||
.open(path)?;
|
.open(path)?;
|
||||||
file.write_all(line.as_bytes())?;
|
let committed_len = Self::truncate_uncommitted_tail(&mut file)?;
|
||||||
file.write_all(b"\n")?;
|
let mut record = Vec::with_capacity(line.len() + 1);
|
||||||
// Append-mode write is the durability boundary; an explicit
|
record.extend_from_slice(line.as_bytes());
|
||||||
// `sync_all` here would multiply latency by ~10× for no gain
|
record.push(b'\n');
|
||||||
// since the kernel already orders concurrent `O_APPEND` writes.
|
|
||||||
|
if let Err(write_error) = file.write_all(&record) {
|
||||||
|
return match file.set_len(committed_len) {
|
||||||
|
Ok(()) => Err(write_error.into()),
|
||||||
|
Err(rollback_error) => Err(std::io::Error::new(
|
||||||
|
rollback_error.kind(),
|
||||||
|
format!(
|
||||||
|
"session append failed ({write_error}) and partial-write rollback failed: {rollback_error}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.into()),
|
||||||
|
};
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_jsonl<T: serde::de::DeserializeOwned>(content: &str) -> Result<Vec<T>, StoreError> {
|
/// Return only newline-terminated records. A process interruption or
|
||||||
|
/// ENOSPC can leave the final UTF-8 code point / JSON object incomplete;
|
||||||
|
/// without a newline that record never crossed the commit boundary.
|
||||||
|
fn complete_jsonl_prefix(content: &[u8]) -> &[u8] {
|
||||||
|
if content.last() == Some(&b'\n') {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
match content.iter().rposition(|byte| *byte == b'\n') {
|
||||||
|
Some(index) => &content[..=index],
|
||||||
|
None => &[],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_jsonl<T: serde::de::DeserializeOwned>(content: &[u8]) -> Result<Vec<T>, StoreError> {
|
||||||
|
let complete = Self::complete_jsonl_prefix(content);
|
||||||
|
let content = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt {
|
||||||
|
line: complete[..error.valid_up_to()]
|
||||||
|
.iter()
|
||||||
|
.filter(|byte| **byte == b'\n')
|
||||||
|
.count()
|
||||||
|
+ 1,
|
||||||
|
message: error.to_string(),
|
||||||
|
})?;
|
||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
for (i, line) in content.lines().enumerate() {
|
for (i, line) in content.lines().enumerate() {
|
||||||
if line.trim().is_empty() {
|
if line.trim().is_empty() {
|
||||||
@@ -130,6 +179,43 @@ impl FsStore {
|
|||||||
}
|
}
|
||||||
Ok(entries)
|
Ok(entries)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Remove a prior unterminated record and return the committed file size.
|
||||||
|
/// Scans backwards in bounded chunks so repairing a large session does not
|
||||||
|
/// require loading it into memory.
|
||||||
|
fn truncate_uncommitted_tail(file: &mut fs::File) -> std::io::Result<u64> {
|
||||||
|
const SCAN_BYTES: usize = 8 * 1024;
|
||||||
|
|
||||||
|
let len = file.metadata()?.len();
|
||||||
|
if len == 0 {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
file.seek(SeekFrom::End(-1))?;
|
||||||
|
let mut last = [0_u8; 1];
|
||||||
|
file.read_exact(&mut last)?;
|
||||||
|
if last[0] == b'\n' {
|
||||||
|
return Ok(len);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut end = len;
|
||||||
|
let mut buffer = [0_u8; SCAN_BYTES];
|
||||||
|
while end > 0 {
|
||||||
|
let start = end.saturating_sub(SCAN_BYTES as u64);
|
||||||
|
let chunk_len = (end - start) as usize;
|
||||||
|
file.seek(SeekFrom::Start(start))?;
|
||||||
|
file.read_exact(&mut buffer[..chunk_len])?;
|
||||||
|
if let Some(index) = buffer[..chunk_len].iter().rposition(|byte| *byte == b'\n') {
|
||||||
|
let committed_len = start + index as u64 + 1;
|
||||||
|
file.set_len(committed_len)?;
|
||||||
|
return Ok(committed_len);
|
||||||
|
}
|
||||||
|
end = start;
|
||||||
|
}
|
||||||
|
|
||||||
|
file.set_len(0)?;
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Store for FsStore {
|
impl Store for FsStore {
|
||||||
@@ -152,7 +238,7 @@ impl Store for FsStore {
|
|||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
return Err(StoreError::NotFound(segment_id));
|
return Err(StoreError::NotFound(segment_id));
|
||||||
}
|
}
|
||||||
let content = fs::read_to_string(&path)?;
|
let content = fs::read(&path)?;
|
||||||
Self::parse_jsonl(&content)
|
Self::parse_jsonl(&content)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,8 +337,17 @@ impl Store for FsStore {
|
|||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
return Err(StoreError::NotFound(segment_id));
|
return Err(StoreError::NotFound(segment_id));
|
||||||
}
|
}
|
||||||
let content = fs::read_to_string(&path)?;
|
let content = fs::read(&path)?;
|
||||||
Ok(content.lines().filter(|l| !l.trim().is_empty()).count())
|
let complete = Self::complete_jsonl_prefix(&content);
|
||||||
|
let complete = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt {
|
||||||
|
line: complete[..error.valid_up_to()]
|
||||||
|
.iter()
|
||||||
|
.filter(|byte| **byte == b'\n')
|
||||||
|
.count()
|
||||||
|
+ 1,
|
||||||
|
message: error.to_string(),
|
||||||
|
})?;
|
||||||
|
Ok(complete.lines().filter(|l| !l.trim().is_empty()).count())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn append_trace(
|
fn append_trace(
|
||||||
|
|||||||
@@ -69,7 +69,10 @@ pub enum LogEntry {
|
|||||||
/// Field name is `trigger` (not `kind`) because the LogEntry
|
/// Field name is `trigger` (not `kind`) because the LogEntry
|
||||||
/// serde tag already occupies `"kind"`.
|
/// serde tag already occupies `"kind"`.
|
||||||
///
|
///
|
||||||
/// Marker only — replay does not mutate `RestoredState`.
|
/// Replay marks the run interrupted until a terminal `RunCompleted`,
|
||||||
|
/// `RunErrored`, or `PausedTurnAbandoned` entry proves how it ended. This
|
||||||
|
/// makes a process/disk failure between Invoke and its terminal record
|
||||||
|
/// restore conservatively instead of re-running a dangling tool call.
|
||||||
Invoke { ts: u64, trigger: InvokeKind },
|
Invoke { ts: u64, trigger: InvokeKind },
|
||||||
|
|
||||||
/// User input accepted at submit time. Carries the original typed
|
/// User input accepted at submit time. Carries the original typed
|
||||||
@@ -236,9 +239,9 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState {
|
|||||||
state.history = history.iter().cloned().map(Item::from).collect();
|
state.history = history.iter().cloned().map(Item::from).collect();
|
||||||
}
|
}
|
||||||
LogEntry::Invoke { .. } => {
|
LogEntry::Invoke { .. } => {
|
||||||
// Marker only; no state mutation. The trailing
|
// A terminal run record below clears or refines this. If the
|
||||||
// UserInput / SystemItem / TurnEnd entries carry all
|
// log ends first, restore must treat the turn as interrupted.
|
||||||
// replay-relevant data.
|
state.last_run_interrupted = true;
|
||||||
}
|
}
|
||||||
LogEntry::UserInput { segments, .. } => {
|
LogEntry::UserInput { segments, .. } => {
|
||||||
let text = Segment::flatten_to_text(segments);
|
let text = Segment::flatten_to_text(segments);
|
||||||
@@ -368,6 +371,35 @@ mod tests {
|
|||||||
assert!(!state.last_run_interrupted);
|
assert!(!state.last_run_interrupted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_incomplete_invoke_is_interrupted() {
|
||||||
|
let state = collect_state(&[
|
||||||
|
LogEntry::SegmentStart {
|
||||||
|
ts: 1000,
|
||||||
|
session_id: uuid::Uuid::nil(),
|
||||||
|
system_prompt: None,
|
||||||
|
config: RequestConfig::default(),
|
||||||
|
history: vec![],
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
},
|
||||||
|
LogEntry::Invoke {
|
||||||
|
ts: 2000,
|
||||||
|
trigger: InvokeKind::UserSend,
|
||||||
|
},
|
||||||
|
LogEntry::UserInput {
|
||||||
|
ts: 2001,
|
||||||
|
segments: vec![Segment::text("run a tool")],
|
||||||
|
},
|
||||||
|
LogEntry::AssistantItem {
|
||||||
|
ts: 3000,
|
||||||
|
item: Item::tool_call("call_1", "side_effect", "{}").into(),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert!(state.last_run_interrupted);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replay_with_tool_calls() {
|
fn replay_with_tool_calls() {
|
||||||
let state = collect_state(&[
|
let state = collect_state(&[
|
||||||
@@ -546,7 +578,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replay_invoke_marker_does_not_mutate_state() {
|
fn replay_invoke_marker_only_mutates_interrupted_state() {
|
||||||
let state = collect_state(&[
|
let state = collect_state(&[
|
||||||
LogEntry::SegmentStart {
|
LogEntry::SegmentStart {
|
||||||
ts: 0,
|
ts: 0,
|
||||||
@@ -576,6 +608,7 @@ mod tests {
|
|||||||
]);
|
]);
|
||||||
assert_eq!(state.history.len(), 1);
|
assert_eq!(state.history.len(), 1);
|
||||||
assert_eq!(state.turn_count, 1);
|
assert_eq!(state.turn_count, 1);
|
||||||
|
assert!(state.last_run_interrupted);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -40,8 +40,8 @@ pub enum StoreError {
|
|||||||
pub trait Store: Send + Sync {
|
pub trait Store: Send + Sync {
|
||||||
/// Append a single log entry to the segment log.
|
/// Append a single log entry to the segment log.
|
||||||
///
|
///
|
||||||
/// One line per call. The kernel orders concurrent `O_APPEND` writes
|
/// One committed line per successful call. Implementations must not expose
|
||||||
/// for lines < `PIPE_BUF`, so user-space serialization is unnecessary.
|
/// a failed call's partial record as committed data on later reads.
|
||||||
fn append(
|
fn append(
|
||||||
&self,
|
&self,
|
||||||
session_id: SessionId,
|
session_id: SessionId,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ use llm_engine::llm_client::types::{Item, RequestConfig};
|
|||||||
use session_store::{
|
use session_store::{
|
||||||
FsStore, LogEntry, Store, TraceEntry, collect_state, new_segment_id, new_session_id,
|
FsStore, LogEntry, Store, TraceEntry, collect_state, new_segment_id, new_session_id,
|
||||||
};
|
};
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
fn nil_session_start(ts: u64, session_id: uuid::Uuid) -> LogEntry {
|
fn nil_session_start(ts: u64, session_id: uuid::Uuid) -> LogEntry {
|
||||||
LogEntry::SegmentStart {
|
LogEntry::SegmentStart {
|
||||||
@@ -224,6 +225,71 @@ fn read_entry_count_matches_append_tally() {
|
|||||||
assert_eq!(store.read_entry_count(sid, segid).unwrap(), entries.len());
|
assert_eq!(store.read_entry_count(sid, segid).unwrap(), entries.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unterminated_utf8_tail_is_ignored_and_replaced_on_append() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = FsStore::new(dir.path()).unwrap();
|
||||||
|
let sid = new_session_id();
|
||||||
|
let segid = new_segment_id();
|
||||||
|
let path = dir
|
||||||
|
.path()
|
||||||
|
.join(sid.to_string())
|
||||||
|
.join(format!("{segid}.jsonl"));
|
||||||
|
|
||||||
|
store
|
||||||
|
.append(sid, segid, &nil_session_start(1, sid))
|
||||||
|
.unwrap();
|
||||||
|
std::fs::OpenOptions::new()
|
||||||
|
.append(true)
|
||||||
|
.open(&path)
|
||||||
|
.unwrap()
|
||||||
|
// First byte of a three-byte UTF-8 code point, matching an ENOSPC
|
||||||
|
// partial write observed in a real session log.
|
||||||
|
.write_all(&[0xe3])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(store.read_all(sid, segid).unwrap().len(), 1);
|
||||||
|
assert_eq!(store.read_entry_count(sid, segid).unwrap(), 1);
|
||||||
|
|
||||||
|
let next = LogEntry::UserInput {
|
||||||
|
ts: 2,
|
||||||
|
segments: vec![protocol::Segment::text("recovered")],
|
||||||
|
};
|
||||||
|
store.append(sid, segid, &next).unwrap();
|
||||||
|
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
assert!(std::str::from_utf8(&bytes).is_ok());
|
||||||
|
assert_eq!(store.read_all(sid, segid).unwrap().len(), 2);
|
||||||
|
assert_eq!(store.read_entry_count(sid, segid).unwrap(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn newline_terminated_invalid_utf8_is_reported_as_corruption() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = FsStore::new(dir.path()).unwrap();
|
||||||
|
let sid = new_session_id();
|
||||||
|
let segid = new_segment_id();
|
||||||
|
let path = dir
|
||||||
|
.path()
|
||||||
|
.join(sid.to_string())
|
||||||
|
.join(format!("{segid}.jsonl"));
|
||||||
|
|
||||||
|
store
|
||||||
|
.append(sid, segid, &nil_session_start(1, sid))
|
||||||
|
.unwrap();
|
||||||
|
std::fs::OpenOptions::new()
|
||||||
|
.append(true)
|
||||||
|
.open(path)
|
||||||
|
.unwrap()
|
||||||
|
.write_all(&[0xe3, b'\n'])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
store.read_all(sid, segid),
|
||||||
|
Err(session_store::StoreError::Corrupt { line: 2, .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn lookup_session_of_finds_owning_session() {
|
fn lookup_session_of_finds_owning_session() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -112,13 +112,14 @@ impl WorkerInterceptor {
|
|||||||
/// `Item::system_message`s reach the worker via
|
/// `Item::system_message`s reach the worker via
|
||||||
/// `ContinueWith` / `pending_history_appends`, so on-disk order
|
/// `ContinueWith` / `pending_history_appends`, so on-disk order
|
||||||
/// matches worker-history order.
|
/// matches worker-history order.
|
||||||
fn commit_system_items(&self, items: &[SystemItem]) {
|
fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> {
|
||||||
let Some(writer) = self.log_writer.as_ref() else {
|
let Some(writer) = self.log_writer.as_ref() else {
|
||||||
return;
|
return Ok(());
|
||||||
};
|
};
|
||||||
for item in items {
|
for item in items {
|
||||||
writer.commit_system_item(item.clone());
|
writer.commit_system_item(item.clone())?;
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn current_turn_index(&self) -> usize {
|
fn current_turn_index(&self) -> usize {
|
||||||
@@ -194,15 +195,17 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
// `Item::system_message`s, so on-disk order matches
|
// `Item::system_message`s, so on-disk order matches
|
||||||
// worker-history order.
|
// worker-history order.
|
||||||
let items: Vec<Item> = extras.iter().map(SystemItem::to_history_item).collect();
|
let items: Vec<Item> = extras.iter().map(SystemItem::to_history_item).collect();
|
||||||
self.commit_system_items(&extras);
|
match self.commit_system_items(&extras) {
|
||||||
PromptAction::ContinueWith(items)
|
Ok(()) => PromptAction::ContinueWith(items),
|
||||||
|
Err(error) => PromptAction::Cancel(format!("session persistence failed: {error}")),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pending_history_appends(&self) -> Vec<Item> {
|
async fn pending_history_appends(&self) -> Result<Vec<Item>, String> {
|
||||||
let drained = self.pending_notifies.drain();
|
let drained = self.pending_notifies.drain();
|
||||||
if drained.is_empty() {
|
if drained.is_empty() {
|
||||||
return Vec::new();
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut system_items: Vec<SystemItem> = Vec::with_capacity(drained.len());
|
let mut system_items: Vec<SystemItem> = Vec::with_capacity(drained.len());
|
||||||
@@ -231,8 +234,9 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.commit_system_items(&system_items);
|
self.commit_system_items(&system_items)
|
||||||
items
|
.map_err(|error| format!("session persistence failed: {error}"))?;
|
||||||
|
Ok(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pre_llm_request(&self, context: &mut Vec<Item>) -> PreRequestAction {
|
async fn pre_llm_request(&self, context: &mut Vec<Item>) -> PreRequestAction {
|
||||||
@@ -278,7 +282,9 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
let current_tokens = self.estimated_tokens(effective_context.as_ref());
|
let current_tokens = self.estimated_tokens(effective_context.as_ref());
|
||||||
|
|
||||||
if self.request_threshold_exceeded(current_tokens, effective_context.as_ref()) {
|
if self.request_threshold_exceeded(current_tokens, effective_context.as_ref()) {
|
||||||
self.commit_system_items(&system_items);
|
if let Err(error) = self.commit_system_items(&system_items) {
|
||||||
|
return PreRequestAction::Cancel(format!("session persistence failed: {error}"));
|
||||||
|
}
|
||||||
return if appended_items.is_empty() {
|
return if appended_items.is_empty() {
|
||||||
PreRequestAction::Yield
|
PreRequestAction::Yield
|
||||||
} else {
|
} else {
|
||||||
@@ -292,8 +298,10 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
if system_items.is_empty() {
|
if system_items.is_empty() {
|
||||||
return PreRequestAction::Continue;
|
return PreRequestAction::Continue;
|
||||||
}
|
}
|
||||||
self.commit_system_items(&system_items);
|
match self.commit_system_items(&system_items) {
|
||||||
PreRequestAction::ContinueWith(appended_items)
|
Ok(()) => PreRequestAction::ContinueWith(appended_items),
|
||||||
|
Err(error) => PreRequestAction::Cancel(format!("session persistence failed: {error}")),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pre_tool_call(&self, info: &mut ToolCallInfo) -> PreToolAction {
|
async fn pre_tool_call(&self, info: &mut ToolCallInfo) -> PreToolAction {
|
||||||
@@ -451,13 +459,17 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SystemItemCommitter for RecordingSystemItemCommitter {
|
impl SystemItemCommitter for RecordingSystemItemCommitter {
|
||||||
fn commit_log_entry(&self, entry: session_store::LogEntry) {
|
fn commit_log_entry(
|
||||||
|
&self,
|
||||||
|
entry: session_store::LogEntry,
|
||||||
|
) -> Result<(), session_store::StoreError> {
|
||||||
if let session_store::LogEntry::SystemItem { item, .. } = entry {
|
if let session_store::LogEntry::SystemItem { item, .. } = entry {
|
||||||
self.committed
|
self.committed
|
||||||
.lock()
|
.lock()
|
||||||
.expect("committed system-item list poisoned")
|
.expect("committed system-item list poisoned")
|
||||||
.push(item);
|
.push(item);
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1034,7 +1046,7 @@ mod tests {
|
|||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
|
|
||||||
let items = interceptor.pending_history_appends().await;
|
let items = interceptor.pending_history_appends().await.unwrap();
|
||||||
assert_eq!(items.len(), 2);
|
assert_eq!(items.len(), 2);
|
||||||
let first = items[0].as_text().unwrap_or_default();
|
let first = items[0].as_text().unwrap_or_default();
|
||||||
let second = items[1].as_text().unwrap_or_default();
|
let second = items[1].as_text().unwrap_or_default();
|
||||||
@@ -1048,7 +1060,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Empty buffer → empty Vec (no synthesised items).
|
// Empty buffer → empty Vec (no synthesised items).
|
||||||
let again = interceptor.pending_history_appends().await;
|
let again = interceptor.pending_history_appends().await.unwrap();
|
||||||
assert!(again.is_empty());
|
assert!(again.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+74
-21
@@ -568,9 +568,8 @@ where
|
|||||||
St: Store + Clone,
|
St: Store + Clone,
|
||||||
{
|
{
|
||||||
/// Append `entry` to the log: disk write → counter bump → in-memory
|
/// Append `entry` to the log: disk write → counter bump → in-memory
|
||||||
/// mirror push → broadcast. The kernel orders concurrent `O_APPEND`
|
/// mirror push → broadcast. The Store owns physical write ordering and
|
||||||
/// writes for `< PIPE_BUF` lines, so no user-space serialization is
|
/// partial-write recovery; publication happens only after it returns Ok.
|
||||||
/// needed across appenders.
|
|
||||||
pub fn append_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
pub fn append_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
||||||
let loc = self.state.location();
|
let loc = self.state.location();
|
||||||
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
||||||
@@ -602,13 +601,13 @@ where
|
|||||||
/// interceptor commit `SystemItem`s without being generic over the
|
/// interceptor commit `SystemItem`s without being generic over the
|
||||||
/// concrete `Store` type.
|
/// concrete `Store` type.
|
||||||
pub trait SystemItemCommitter: Send + Sync {
|
pub trait SystemItemCommitter: Send + Sync {
|
||||||
fn commit_log_entry(&self, entry: LogEntry);
|
fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError>;
|
||||||
|
|
||||||
fn commit_system_item(&self, item: SystemItem) {
|
fn commit_system_item(&self, item: SystemItem) -> Result<(), StoreError> {
|
||||||
self.commit_log_entry(LogEntry::SystemItem {
|
self.commit_log_entry(LogEntry::SystemItem {
|
||||||
ts: segment_log::now_millis(),
|
ts: segment_log::now_millis(),
|
||||||
item,
|
item,
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -616,10 +615,8 @@ impl<St> SystemItemCommitter for LogWriterHandle<St>
|
|||||||
where
|
where
|
||||||
St: Store + Clone + Send + Sync + 'static,
|
St: Store + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
fn commit_log_entry(&self, entry: LogEntry) {
|
fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
||||||
if let Err(err) = self.append_entry(entry) {
|
self.append_entry(entry)
|
||||||
warn!(error = %err, "session log entry commit failed; dropping");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -914,7 +911,7 @@ impl<C: LlmClient + Clone + 'static, St: Store + Clone + 'static> Worker<C, St>
|
|||||||
let writer = self.log_writer_handle();
|
let writer = self.log_writer_handle();
|
||||||
self.engine_mut().on_history_append(move |item| {
|
self.engine_mut().on_history_append(move |item| {
|
||||||
if item.is_user_message() {
|
if item.is_user_message() {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
if matches!(
|
if matches!(
|
||||||
item,
|
item,
|
||||||
@@ -923,12 +920,12 @@ impl<C: LlmClient + Clone + 'static, St: Store + Clone + 'static> Worker<C, St>
|
|||||||
..
|
..
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
let entry = session_store::classify_history_item(item, segment_log::now_millis());
|
let entry = session_store::classify_history_item(item, segment_log::now_millis());
|
||||||
if let Err(err) = writer.append_entry(entry) {
|
writer
|
||||||
warn!(error = %err, "history append commit failed; dropping");
|
.append_entry(entry)
|
||||||
}
|
.map_err(|error| error.to_string())
|
||||||
});
|
});
|
||||||
if self.manifest.session.record_event_trace {
|
if self.manifest.session.record_event_trace {
|
||||||
let writer = self.log_writer_handle();
|
let writer = self.log_writer_handle();
|
||||||
@@ -1206,7 +1203,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
},
|
},
|
||||||
})?;
|
})?;
|
||||||
self.engine_mut()
|
self.engine_mut()
|
||||||
.append_history(std::iter::once(llm_engine::Item::system_message(body)));
|
.append_history(std::iter::once(llm_engine::Item::system_message(body)))?;
|
||||||
Ok(activation)
|
Ok(activation)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1251,9 +1248,8 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Append `entry` to the session log AND publish it through the
|
/// Append `entry` to the session log AND publish it through the
|
||||||
/// broadcast sink. No user-space serialization is needed across
|
/// broadcast sink. The Store is the commit boundary: a failed write is
|
||||||
/// concurrent appenders — the kernel orders `O_APPEND` writes for
|
/// never counted or published.
|
||||||
/// lines smaller than `PIPE_BUF`.
|
|
||||||
pub(crate) fn commit_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
pub(crate) fn commit_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
||||||
let loc = self.segment_state.location();
|
let loc = self.segment_state.location();
|
||||||
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
||||||
@@ -2083,7 +2079,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
&tool_result_summary,
|
&tool_result_summary,
|
||||||
);
|
);
|
||||||
if !closures.is_empty() {
|
if !closures.is_empty() {
|
||||||
self.engine_mut().append_history(closures);
|
self.engine_mut().append_history(closures)?;
|
||||||
}
|
}
|
||||||
self.commit_entry(LogEntry::SystemItem {
|
self.commit_entry(LogEntry::SystemItem {
|
||||||
ts: segment_log::now_millis(),
|
ts: segment_log::now_millis(),
|
||||||
@@ -2094,7 +2090,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
self.engine_mut()
|
self.engine_mut()
|
||||||
.append_history(std::iter::once(llm_engine::Item::system_message(
|
.append_history(std::iter::once(llm_engine::Item::system_message(
|
||||||
system_note,
|
system_note,
|
||||||
)));
|
)))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2164,6 +2160,12 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
),
|
),
|
||||||
"run_for_notification expects a non-UserSend InvokeKind; got {kind:?}"
|
"run_for_notification expects a non-UserSend InvokeKind; got {kind:?}"
|
||||||
);
|
);
|
||||||
|
// This is a fresh Invoke, not an explicit resume of the interrupted
|
||||||
|
// turn. Close any dangling tool calls before an auto-run notification
|
||||||
|
// can enter `Engine::resume` and execute them again after a crash.
|
||||||
|
if self.engine.as_ref().unwrap().last_run_interrupted() {
|
||||||
|
self.apply_interrupt_prep()?;
|
||||||
|
}
|
||||||
self.prepare_for_run().await?;
|
self.prepare_for_run().await?;
|
||||||
|
|
||||||
// IDLE → active marker for the buffered notification / worker-event
|
// IDLE → active marker for the buffered notification / worker-event
|
||||||
@@ -5914,6 +5916,57 @@ mod build_summary_prompt_tests {
|
|||||||
assert_eq!(interrupt_system_count, 1);
|
assert_eq!(interrupt_system_count, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn notification_run_closes_interrupted_tool_call_before_engine_resume() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let manifest = minimal_manifest();
|
||||||
|
let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap();
|
||||||
|
let cwd = dir.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&cwd).unwrap();
|
||||||
|
let scope = Scope::writable(&cwd).unwrap();
|
||||||
|
let authority = WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone());
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
Engine::new(NoopClient),
|
||||||
|
store,
|
||||||
|
WorkerWorkspaceContext::local_filesystem(None),
|
||||||
|
authority,
|
||||||
|
scope,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
worker.ensure_segment_head().unwrap();
|
||||||
|
worker.wire_history_persistence();
|
||||||
|
let dangling_call = Item::tool_call("call-1", "SideEffect", "{}");
|
||||||
|
worker
|
||||||
|
.commit_entry(LogEntry::AssistantItem {
|
||||||
|
ts: segment_log::now_millis(),
|
||||||
|
item: dangling_call.clone().into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
worker.engine_mut().set_history(vec![dangling_call]);
|
||||||
|
worker.engine_mut().set_last_run_interrupted(true);
|
||||||
|
|
||||||
|
worker
|
||||||
|
.run_for_notification(protocol::InvokeKind::Notify)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let history = worker.engine().history();
|
||||||
|
assert!(matches!(
|
||||||
|
history.get(1),
|
||||||
|
Some(Item::ToolResult { call_id, .. }) if call_id == "call-1"
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
history.get(2),
|
||||||
|
Some(Item::Message {
|
||||||
|
role: Role::System,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct ResidentInjectionGates {
|
struct ResidentInjectionGates {
|
||||||
summary: bool,
|
summary: bool,
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# ENOSPC で Session JSONL の末尾が UTF-8 途中切れになる
|
||||||
|
|
||||||
|
## 観測
|
||||||
|
|
||||||
|
`Companion1` (`worker-runtime-30`) の直近 Segment
|
||||||
|
`019fce34-eb04-7420-9cc5-0e7e12f5bd67` は、Workdir の Edit tool が
|
||||||
|
`no storage space` で失敗した後、会話ログ `.jsonl` の末尾が `0xe3` 1 byte
|
||||||
|
だけで終わっていた。これは 3 byte UTF-8 文字の先頭 byte であり、実際の末尾は
|
||||||
|
「Workdir のストレー」の次の文字の途中で切れていた。
|
||||||
|
|
||||||
|
同 Segment の `.trace.jsonl` は valid UTF-8 だった。再開時の
|
||||||
|
`stream did not contain valid UTF-8` は provider stream ではなく、Session log を
|
||||||
|
`fs::read_to_string` した際のエラーだった。
|
||||||
|
|
||||||
|
## 原因
|
||||||
|
|
||||||
|
`FsStore::append_line` は JSON 本文と改行を別々に `write_all` し、ENOSPC で
|
||||||
|
部分書き込みになっても元の file length へ戻していなかった。reader も file 全体を
|
||||||
|
UTF-8 String として読むため、newline に到達していない未コミット末尾だけで Segment
|
||||||
|
全体を復元不能にしていた。
|
||||||
|
|
||||||
|
さらに Engine の history append callback と SystemItem committer は、Store error を
|
||||||
|
warning にして drop していた。このため disk 上の history を更新できなくても memory
|
||||||
|
上の history と tool loop が先へ進み得た。
|
||||||
|
|
||||||
|
## 改善
|
||||||
|
|
||||||
|
- newline を JSONL record の commit marker とする。
|
||||||
|
- reader は newline 未到達の末尾を未コミット record として無視する。
|
||||||
|
- 次回 append 前に未コミット末尾を最後の newline まで truncate する。
|
||||||
|
- append の partial write は append 開始時の file length へ rollback する。
|
||||||
|
- repair/write/rollback は `FsStore` clone 間で直列化する。
|
||||||
|
- Engine history append を fallible にし、Store write 成功前には item を memory history
|
||||||
|
に入れない。
|
||||||
|
- tool call の永続化に失敗した turn は tool 実行前に停止する。
|
||||||
|
- SystemItem の commit failure も transient context injection にせず turn error にする。
|
||||||
|
- `Invoke` 後に terminal run record が無い Segment は restore 時に interrupted とする。
|
||||||
|
これにより crash/ENOSPC 後の dangling tool call は新しい user turn の前に閉じられ、
|
||||||
|
side effect を無条件に再実行しない。
|
||||||
|
|
||||||
|
## 境界
|
||||||
|
|
||||||
|
この修正は process interruption と ENOSPC による trailing partial record を対象にする。
|
||||||
|
newline 済み record 内部の破損は silent recovery せず `StoreError::Corrupt` のまま扱う。
|
||||||
|
また append ごとの `fsync` は追加していないため、突然の電源断に対する block-level
|
||||||
|
durability まで保証するものではない。
|
||||||
Reference in New Issue
Block a user