feat: add provenance-aware worker history

This commit is contained in:
2026-08-27 14:54:24 +09:00
parent 116d610ad0
commit e365189276
46 changed files with 2560 additions and 658 deletions
+180
View File
@@ -0,0 +1,180 @@
//! Serializable history entries with restore-authoritative logical identity and origin.
use serde::{Deserialize, Serialize};
use crate::{LoggedItem, SessionId};
/// Stable logical identity of one model-visible history entry.
///
/// This value is generated at the trusted Worker session boundary and copied
/// unchanged across fork, rewind, compaction retention, restore, and reboot.
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct LoggedSessionHistoryEntryId(pub String);
impl LoggedSessionHistoryEntryId {
pub fn new() -> Self {
Self(uuid::Uuid::now_v7().to_string())
}
}
impl Default for LoggedSessionHistoryEntryId {
fn default() -> Self {
Self::new()
}
}
/// Bounded subject snapshot. It is evidence, not a live authorization handle.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LoggedWorkerSubject {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub runtime_id: Option<String>,
pub worker_id: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum LoggedSessionHistoryOrigin {
HumanInput {
account_id: String,
},
WorkerInput {
actor: LoggedWorkerSubject,
},
FlowInstruction {
selector: String,
definition_id: String,
definition_revision: u64,
instance_id: String,
state_id: String,
},
BackendInstruction {
#[serde(default, skip_serializing_if = "Option::is_none")]
operation_id: Option<String>,
},
ModelOutput {
worker: LoggedWorkerSubject,
},
ToolOutput {
worker: LoggedWorkerSubject,
},
DerivedSummary,
LegacyUnknown,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LoggedHistoryDerivation {
pub sources: Vec<LoggedSessionHistoryEntryId>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LoggedSessionHistoryMetadata {
pub entry_id: LoggedSessionHistoryEntryId,
pub origin: LoggedSessionHistoryOrigin,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub derivation: Option<LoggedHistoryDerivation>,
}
impl LoggedSessionHistoryMetadata {
pub fn legacy_unknown() -> Self {
Self {
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
}
}
}
/// Persisted item and metadata are one value so transforms cannot reorder or
/// truncate one without the other.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct LoggedHistoryEntry {
pub item: LoggedItem,
pub metadata: LoggedSessionHistoryMetadata,
}
/// Typed system-item history record. The typed system event remains available
/// to client replay while its model-visible projection carries the same stable
/// metadata used by live history.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LoggedSystemHistoryEntry {
pub item: crate::SystemItem,
pub metadata: LoggedSessionHistoryMetadata,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::LoggedRole;
use agen::llm_client::RequestConfig;
#[test]
fn logged_history_entry_round_trip_preserves_id_origin_and_derivation() {
let source_id = LoggedSessionHistoryEntryId::new();
let entry = LoggedHistoryEntry {
item: LoggedItem::Message {
role: LoggedRole::User,
content: vec![crate::LoggedContentPart::Text {
text: "preference".into(),
}],
},
metadata: LoggedSessionHistoryMetadata {
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".into(),
},
derivation: Some(LoggedHistoryDerivation {
sources: vec![source_id.clone()],
}),
},
};
let encoded = serde_json::to_vec(&entry).unwrap();
let decoded: LoggedHistoryEntry = serde_json::from_slice(&encoded).unwrap();
assert_eq!(decoded, entry);
assert_eq!(
decoded.metadata.derivation.unwrap().sources,
vec![source_id]
);
}
#[test]
fn annotated_segment_start_is_restore_visible_without_projecting_metadata() {
let session_id = uuid::Uuid::now_v7();
let history_entry = legacy_logged_history(LoggedItem::Message {
role: LoggedRole::Assistant,
content: vec![crate::LoggedContentPart::Text {
text: "answer".into(),
}],
});
let state = crate::collect_state(&[crate::LogEntry::AnnotatedSegmentStart {
ts: 1,
session_id,
system_prompt: None,
config: RequestConfig::default(),
history: vec![history_entry],
forked_from: None,
compacted_from: None,
}]);
assert_eq!(state.history[0].as_text(), Some("answer"));
}
}
/// Legacy Session Logs did not persist annotations. Decode helpers explicitly
/// create `LegacyUnknown`; they never infer Human/System authority from role or
/// plaintext.
pub fn legacy_logged_history(item: LoggedItem) -> LoggedHistoryEntry {
LoggedHistoryEntry {
item,
metadata: LoggedSessionHistoryMetadata::legacy_unknown(),
}
}
pub fn legacy_segment_history(
session_id: SessionId,
items: impl IntoIterator<Item = LoggedItem>,
) -> Vec<LoggedHistoryEntry> {
let _ = session_id;
items.into_iter().map(legacy_logged_history).collect()
}
+6
View File
@@ -32,6 +32,7 @@
pub mod event_trace;
pub mod fs_store;
pub mod history;
pub mod logged_item;
pub mod segment;
pub mod segment_log;
@@ -44,6 +45,11 @@ pub use agen::UsageRecord;
pub use agen::llm_client::types::{ContentPart, Item, Role};
pub use event_trace::{TraceEntry, TracePayload};
pub use fs_store::FsStore;
pub use history::{
LoggedHistoryDerivation, LoggedHistoryEntry, LoggedSessionHistoryEntryId,
LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, LoggedSystemHistoryEntry,
LoggedWorkerSubject, legacy_logged_history, legacy_segment_history,
};
pub use logged_item::{LoggedContentPart, LoggedItem, LoggedRole, from_logged, to_logged};
pub use segment::{
SegmentStartState, append_entry, append_system_item, classify_history_item,
+77
View File
@@ -14,6 +14,7 @@ use agen::{EngineResult, UsageRecord};
use protocol::{InvokeKind, Segment};
use serde::{Deserialize, Serialize};
use crate::history::{LoggedHistoryEntry, LoggedSystemHistoryEntry};
use crate::logged_item::LoggedItem;
use crate::system_item::SystemItem;
@@ -70,6 +71,20 @@ pub enum LogEntry {
compacted_from: Option<SegmentOrigin>,
},
/// Schema-v2 segment seed. Retained entries keep their stable logical
/// identity and origin across fork/compaction/restore.
AnnotatedSegmentStart {
ts: u64,
session_id: crate::SessionId,
system_prompt: Option<String>,
config: RequestConfig,
history: Vec<LoggedHistoryEntry>,
#[serde(default, skip_serializing_if = "Option::is_none")]
forked_from: Option<SegmentOrigin>,
#[serde(default, skip_serializing_if = "Option::is_none")]
compacted_from: Option<SegmentOrigin>,
},
/// IDLE → active marker. Records the start of a new self-driving
/// cycle (Invoke range). The range extends implicitly until the
/// next `Invoke` entry; this entry carries the trigger only — the
@@ -105,14 +120,37 @@ pub enum LogEntry {
extensions: Vec<SessionExtension>,
},
/// Schema-v2 user submission with its exact model-visible entries. Typed
/// Flow instructions and caller-attributed input remain separate entries.
AnnotatedUserInput {
ts: u64,
segments: Vec<Segment>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
extensions: Vec<SessionExtension>,
history: Vec<LoggedHistoryEntry>,
},
/// Schema-v2 model output and metadata committed as one journal record.
AnnotatedAssistantItem { ts: u64, entry: LoggedHistoryEntry },
/// One assistant-side item appended to history — assistant message,
/// reasoning, or tool call. Singular: one entry per history item so
/// the wire-side `Event::*` lane and on-disk LogEntry stay 1:1.
AssistantItem { ts: u64, item: LoggedItem },
/// Schema-v2 tool output and metadata committed as one journal record.
AnnotatedToolResult { ts: u64, entry: LoggedHistoryEntry },
/// One tool-execution result appended to history.
ToolResult { ts: u64, item: LoggedItem },
/// Schema-v2 typed system event and model-visible metadata committed
/// together.
AnnotatedSystemItem {
ts: u64,
entry: LoggedSystemHistoryEntry,
},
/// One typed agent-injected system item: notification, child-Worker
/// lifecycle event, `@<path>` / `/<slug>` resolution payload. Each
/// `SystemItem` carries kind metadata that the LLM
@@ -278,6 +316,22 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState {
state.config = config.clone();
state.history = history.iter().cloned().map(Item::from).collect();
}
LogEntry::AnnotatedSegmentStart {
session_id,
system_prompt,
config,
history,
..
} => {
state.session_id = Some(*session_id);
state.system_prompt = system_prompt.clone();
state.config = config.clone();
state.history = history
.iter()
.cloned()
.map(|entry| Item::from(entry.item))
.collect();
}
LogEntry::Invoke { .. } => {
// A terminal run record below clears or refines this. If the
// log ends first, restore must treat the turn as interrupted.
@@ -298,6 +352,29 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState {
.map(|extension| (extension.domain.clone(), extension.payload.clone())),
);
}
LogEntry::AnnotatedUserInput {
segments,
extensions,
history,
..
} => {
state
.history
.extend(history.iter().cloned().map(|entry| Item::from(entry.item)));
state.user_segments.push(segments.clone());
state.extensions.extend(
extensions
.iter()
.map(|extension| (extension.domain.clone(), extension.payload.clone())),
);
}
LogEntry::AnnotatedAssistantItem { entry, .. }
| LogEntry::AnnotatedToolResult { entry, .. } => {
state.history.push(Item::from(entry.item.clone()));
}
LogEntry::AnnotatedSystemItem { entry, .. } => {
state.history.push(entry.item.to_history_item());
}
LogEntry::AssistantItem { item, .. } => {
state.history.push(Item::from(item.clone()));
}
@@ -20,7 +20,8 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::SystemTime;
const SESSION_SCHEMA_VERSION: u32 = 1;
const SESSION_SCHEMA_VERSION: u32 = 2;
const LEGACY_SESSION_SCHEMA_VERSION: u32 = 1;
const SESSION_FILE: &str = "session.json";
const SEGMENTS_DIR: &str = "segments";
@@ -44,15 +45,22 @@ impl WorkerSessionStore {
fs::create_dir_all(root.join(SEGMENTS_DIR))?;
let session_id = match fs::read(root.join(SESSION_FILE)) {
Ok(bytes) => {
let manifest: SessionManifest = serde_json::from_slice(&bytes)?;
if manifest.schema_version != SESSION_SCHEMA_VERSION {
return Err(StoreError::Corrupt {
line: 0,
message: format!(
"unsupported Worker Session schema version {}, expected {}",
manifest.schema_version, SESSION_SCHEMA_VERSION
),
});
let mut manifest: SessionManifest = serde_json::from_slice(&bytes)?;
match manifest.schema_version {
SESSION_SCHEMA_VERSION => {}
LEGACY_SESSION_SCHEMA_VERSION => {
validate_legacy_segment_logs(&root)?;
manifest.schema_version = SESSION_SCHEMA_VERSION;
atomic_write_json(&root.join(SESSION_FILE), &manifest)?;
}
version => {
return Err(StoreError::Corrupt {
line: 0,
message: format!(
"unsupported Worker Session schema version {version}, expected {SESSION_SCHEMA_VERSION}"
),
});
}
}
Some(manifest.session_id)
}
@@ -278,6 +286,37 @@ impl Store for WorkerSessionStore {
}
}
fn validate_legacy_segment_logs(root: &Path) -> Result<(), StoreError> {
let segments = root.join(SEGMENTS_DIR);
if !segments.exists() {
return Ok(());
}
for entry in fs::read_dir(&segments)? {
let entry = entry?;
let path = entry.path();
let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
if !name.ends_with(".jsonl") || name.ends_with(".trace.jsonl") {
continue;
}
let contents = fs::read_to_string(&path)?;
for (line_index, line) in contents.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
serde_json::from_str::<LogEntry>(line).map_err(|error| StoreError::Corrupt {
line: line_index + 1,
message: format!(
"cannot migrate legacy Worker Session log {}: {error}",
path.display()
),
})?;
}
}
Ok(())
}
fn atomic_write_json<T: Serialize>(path: &Path, value: &T) -> Result<(), StoreError> {
let mut bytes = serde_json::to_vec_pretty(value)?;
bytes.push(b'\n');
@@ -405,6 +444,54 @@ mod tests {
assert_eq!(store.list_sessions().unwrap(), vec![session_id]);
}
#[test]
fn schema_v1_logs_are_validated_and_promoted_to_v2() {
let root = tempfile::tempdir().unwrap();
let session_id = new_session_id();
let segment_id = new_segment_id();
WorkerSessionStore::new(root.path())
.unwrap()
.create_segment(session_id, segment_id, &[])
.unwrap();
let manifest_path = root.path().join(SESSION_FILE);
let mut manifest: SessionManifest =
serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap();
manifest.schema_version = LEGACY_SESSION_SCHEMA_VERSION;
atomic_write_json(&manifest_path, &manifest).unwrap();
let reopened = WorkerSessionStore::new(root.path()).unwrap();
assert_eq!(reopened.session_id().unwrap(), Some(session_id));
let migrated: SessionManifest =
serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap();
assert_eq!(migrated.schema_version, SESSION_SCHEMA_VERSION);
}
#[test]
fn schema_v1_migration_rejects_corrupt_log_before_manifest_update() {
let root = tempfile::tempdir().unwrap();
let session_id = new_session_id();
let manifest = SessionManifest {
schema_version: LEGACY_SESSION_SCHEMA_VERSION,
session_id,
};
atomic_write_json(&root.path().join(SESSION_FILE), &manifest).unwrap();
fs::create_dir_all(root.path().join(SEGMENTS_DIR)).unwrap();
fs::write(
root.path().join(SEGMENTS_DIR).join("broken.jsonl"),
"{not-json}\n",
)
.unwrap();
let error = match WorkerSessionStore::new(root.path()) {
Ok(_) => panic!("corrupt legacy Session log must reject migration"),
Err(error) => error,
};
assert!(matches!(error, StoreError::Corrupt { .. }));
let persisted: SessionManifest =
serde_json::from_slice(&fs::read(root.path().join(SESSION_FILE)).unwrap()).unwrap();
assert_eq!(persisted.schema_version, LEGACY_SESSION_SCHEMA_VERSION);
}
#[test]
fn reopen_preserves_session_and_segment_ids() {
let root = tempfile::tempdir().unwrap();
+62 -28
View File
@@ -1,12 +1,13 @@
mod common;
use std::ops::{Deref, DerefMut};
use std::sync::Arc;
use agen::Engine;
use agen::interceptor::{Interceptor, TurnEndAction};
use agen::llm_client::event::{Event, ResponseStatus, StatusEvent};
use agen::llm_client::types::{Item, RequestConfig};
use agen::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use agen::{Engine, History};
use async_trait::async_trait;
use common::MockLlmClient;
use session_store::{FsStore, LogEntry, SegmentStartState, Store, collect_state};
@@ -94,15 +95,47 @@ fn make_store() -> (tempfile::TempDir, FsStore) {
(dir, store)
}
struct TestWorker {
engine: Engine<MockLlmClient>,
history: History,
}
impl TestWorker {
fn new(engine: Engine<MockLlmClient>) -> Self {
Self {
engine,
history: History::new(),
}
}
fn history(&self) -> Vec<Item> {
self.history.items_cloned()
}
}
impl Deref for TestWorker {
type Target = Engine<MockLlmClient>;
fn deref(&self) -> &Self::Target {
&self.engine
}
}
impl DerefMut for TestWorker {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.engine
}
}
/// Run a worker turn and persist via session-store functions.
/// Takes ownership of the worker (needed for lock/unlock) and returns it.
async fn run_and_persist(
worker: Engine<MockLlmClient>,
mut worker: TestWorker,
store: &FsStore,
session_id: session_store::SessionId,
segment_id: session_store::SegmentId,
input: &str,
) -> (Engine<MockLlmClient>, agen::EngineRunExit) {
) -> (TestWorker, agen::EngineResult) {
// Mirror Worker's run-entry contract: log the user input as segments
// before the worker pushes its flattened user_message; save_delta
// skips the resulting user_message item to avoid double-write.
@@ -114,13 +147,14 @@ async fn run_and_persist(
)
.unwrap();
let history_before = worker.history().len();
let history_before = worker.history.len();
let mut locked = worker.lock();
let result = locked.run(input).await;
let worker = locked.unlock();
let mut locked = worker.engine.lock(&worker.history);
let result = locked.run(&mut worker.history, input).await;
worker.engine = locked.unlock();
let new_items = &worker.history()[history_before..];
let projected = worker.history();
let new_items = &projected[history_before..];
session_store::save_delta(store, session_id, segment_id, new_items).unwrap();
session_store::save_turn_end(store, session_id, segment_id, worker.turn_count()).unwrap();
@@ -178,14 +212,14 @@ async fn run_and_persist(
async fn session_run_logs_entries() {
let (_dir, store) = make_store();
let client = MockLlmClient::new(simple_text_events());
let worker = Engine::new(client);
let worker = TestWorker::new(Engine::new(client));
let (sid, segid) = session_store::create_segment(
&store,
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -222,7 +256,7 @@ async fn session_run_logs_entries() {
async fn session_restore_round_trip() {
let (_dir, store) = make_store();
let client = MockLlmClient::new(simple_text_events());
let mut worker = Engine::new(client);
let mut worker = TestWorker::new(Engine::new(client));
worker.set_system_prompt("You are helpful.");
let (sid, segid) = session_store::create_segment(
@@ -230,7 +264,7 @@ async fn session_restore_round_trip() {
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -261,7 +295,7 @@ async fn session_restore_round_trip() {
async fn session_run_with_tool_call() {
let (_dir, store) = make_store();
let client = MockLlmClient::with_responses(tool_call_events());
let mut worker = Engine::new(client);
let mut worker = TestWorker::new(Engine::new(client));
worker.register_tool(weather_tool_definition());
let (sid, segid) = session_store::create_segment(
@@ -269,7 +303,7 @@ async fn session_run_with_tool_call() {
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -295,7 +329,7 @@ async fn session_resume_after_pause() {
// First run: tool call with pause policy → Paused
let client = MockLlmClient::with_responses(tool_call_events());
let mut worker = Engine::new(client);
let mut worker = TestWorker::new(Engine::new(client));
worker.register_tool(weather_tool_definition());
worker.set_interceptor(PausePolicy);
@@ -304,7 +338,7 @@ async fn session_resume_after_pause() {
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -335,7 +369,7 @@ async fn session_resume_after_pause() {
async fn session_fork_creates_new_session() {
let (_dir, store) = make_store();
let client = MockLlmClient::new(simple_text_events());
let mut worker = Engine::new(client);
let mut worker = TestWorker::new(Engine::new(client));
worker.set_system_prompt("System prompt");
let (sid, segid) = session_store::create_segment(
@@ -343,7 +377,7 @@ async fn session_fork_creates_new_session() {
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -356,7 +390,7 @@ async fn session_fork_creates_new_session() {
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -377,14 +411,14 @@ async fn session_fork_creates_new_session() {
async fn session_fork_at_truncates_within_session() {
let (_dir, store) = make_store();
let client = MockLlmClient::new(simple_text_events());
let worker = Engine::new(client);
let worker = TestWorker::new(Engine::new(client));
let (sid, segid) = session_store::create_segment(
&store,
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -422,14 +456,14 @@ async fn session_fork_at_truncates_within_session() {
async fn session_config_changed_logged() {
let (_dir, store) = make_store();
let client = MockLlmClient::new(vec![]);
let mut worker = Engine::new(client);
let mut worker = TestWorker::new(Engine::new(client));
let (sid, segid) = session_store::create_segment(
&store,
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();
@@ -455,14 +489,14 @@ async fn session_auto_forks_on_conflict() {
// Create a segment
let client_a = MockLlmClient::new(simple_text_events());
let worker_a = Engine::new(client_a);
let worker_a = TestWorker::new(Engine::new(client_a));
let (sid, original_segid) = session_store::create_segment(
&store,
SegmentStartState {
system_prompt: worker_a.get_system_prompt(),
config: worker_a.request_config(),
history: worker_a.history(),
history: &worker_a.history(),
},
)
.unwrap();
@@ -488,7 +522,7 @@ async fn session_auto_forks_on_conflict() {
SegmentStartState {
system_prompt: worker_a.get_system_prompt(),
config: worker_a.request_config(),
history: worker_a.history(),
history: &worker_a.history(),
},
)
.unwrap();
@@ -540,14 +574,14 @@ async fn session_auto_forks_on_conflict() {
async fn nested_past_fork_leaves_ancestors_immutable() {
let (_dir, store) = make_store();
let client = MockLlmClient::new(simple_text_events());
let worker = Engine::new(client);
let worker = TestWorker::new(Engine::new(client));
let (sid, root_segid) = session_store::create_segment(
&store,
SegmentStartState {
system_prompt: worker.get_system_prompt(),
config: worker.request_config(),
history: worker.history(),
history: &worker.history(),
},
)
.unwrap();