Merge remote-tracking branch 'origin/develop' into ticket/t-768-subject-grants

This commit is contained in:
2026-10-20 06:38:12 +09:00
72 changed files with 6060 additions and 959 deletions
+19 -7
View File
@@ -408,7 +408,7 @@ pub enum Method {
/// Transport is fire-and-forget; receivers must tolerate out-of-order /// Transport is fire-and-forget; receivers must tolerate out-of-order
/// delivery (e.g. `TurnEnded` arriving after `ShutDown` for the same /// delivery (e.g. `TurnEnded` arriving after `ShutDown` for the same
/// child Worker). /// child Worker).
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
#[serde(tag = "kind", rename_all = "snake_case")] #[serde(tag = "kind", rename_all = "snake_case")]
@@ -982,6 +982,17 @@ pub enum SessionEntryProvenance {
LegacyUnknown, LegacyUnknown,
} }
/// Public use decisions contain no Feature identity or private payload. Absence
/// denotes the older projection contract, not permission to expose System input.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct SessionEntryUsePolicy {
pub observation_readable: bool,
pub memory_evidence: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
@@ -994,6 +1005,8 @@ pub struct SessionSnapshotEntry {
pub provenance: SessionEntryProvenance, pub provenance: SessionEntryProvenance,
#[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub derived_from: Vec<String>, pub derived_from: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub use_policy: Option<SessionEntryUsePolicy>,
#[serde(flatten)] #[serde(flatten)]
pub data: SessionSnapshotEntryData, pub data: SessionSnapshotEntryData,
} }
@@ -1180,12 +1193,10 @@ pub enum Event {
}, },
/// One agent-injected system item committed to history. /// One agent-injected system item committed to history.
/// ///
/// Carries the JSON form of `session_store::SystemItem`. Covers /// Carries the allowlisted JSON form of `session_store::PublicSystemItem`,
/// `Method::Notify` echoes, child-Worker lifecycle events from /// not the storage record. Its descriptive `kind` is independent from the
/// `Method::WorkerEvent`, `@<path>` resolution payloads, and any future /// typed `presentation` used for notification, reminder, or context rendering.
/// Clients dispatch on the `kind` tag for typed rendering instead /// Private Feature payloads and prompt provenance are never event data.
/// of parsing free-text prefixes like `[Notification] …` or
/// `[File: …]`.
/// ///
/// One event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and /// One event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and
/// wire-side are 1:1. /// wire-side are 1:1.
@@ -2621,6 +2632,7 @@ mod tests {
session: SessionSnapshot { session: SessionSnapshot {
pending_submissions: PendingSubmissionsSnapshot::default(), pending_submissions: PendingSubmissionsSnapshot::default(),
entries: vec![SessionSnapshotEntry { entries: vec![SessionSnapshotEntry {
use_policy: None,
entry_id: "entry-1".into(), entry_id: "entry-1".into(),
timestamp: 1, timestamp: 1,
provenance: SessionEntryProvenance::HumanInput, provenance: SessionEntryProvenance::HumanInput,
+7 -6
View File
@@ -17,12 +17,12 @@ use crate::{
PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission, PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission,
ReasoningConfig, RewindSummary, RunFailureKind, RunResult, RunResumeSource, RunYieldReason, ReasoningConfig, RewindSummary, RunFailureKind, RunResult, RunResumeSource, RunYieldReason,
ScopeRule, Segment, SessionContentPart, SessionConversationTurn, SessionEntryProvenance, ScopeRule, Segment, SessionContentPart, SessionConversationTurn, SessionEntryProvenance,
SessionHistoryLineageBoundary, SessionHistoryPage, SessionMessageRole, SessionSnapshot, SessionEntryUsePolicy, SessionHistoryLineageBoundary, SessionHistoryPage, SessionMessageRole,
SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment,
SymlinkPolicy, ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef, SubmissionDisposition, SymlinkPolicy, ToolResultDisposition, TurnResult,
WorkerBusyState, WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope, UploadedFileAvailability, UploadedFileRef, WorkerBusyState, WorkerCommandAcknowledgement,
WorkerCommandKind, WorkerEvent, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind, WorkerEvent,
WorkerStateSnapshot, WorkerStatus, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus,
subscription::{ subscription::{
EventSubscriptionSelector, MAX_CORRELATION_ID_BYTES, MAX_REJECTION_MESSAGE_BYTES, EventSubscriptionSelector, MAX_CORRELATION_ID_BYTES, MAX_REJECTION_MESSAGE_BYTES,
MAX_RESOURCE_ID_BYTES, MAX_SUBSCRIPTION_COLLECTION_ITEMS, MAX_RESOURCE_ID_BYTES, MAX_SUBSCRIPTION_COLLECTION_ITEMS,
@@ -117,6 +117,7 @@ pub fn generated_protocol_types() -> String {
push_decl::<InFlightCompaction>(&cfg, &mut output); push_decl::<InFlightCompaction>(&cfg, &mut output);
push_decl::<InFlightSnapshot>(&cfg, &mut output); push_decl::<InFlightSnapshot>(&cfg, &mut output);
push_decl::<SessionEntryProvenance>(&cfg, &mut output); push_decl::<SessionEntryProvenance>(&cfg, &mut output);
push_decl::<SessionEntryUsePolicy>(&cfg, &mut output);
push_decl::<SessionMessageRole>(&cfg, &mut output); push_decl::<SessionMessageRole>(&cfg, &mut output);
push_decl::<SessionContentPart>(&cfg, &mut output); push_decl::<SessionContentPart>(&cfg, &mut output);
push_decl::<SessionToolAttachment>(&cfg, &mut output); push_decl::<SessionToolAttachment>(&cfg, &mut output);
+4
View File
@@ -361,6 +361,7 @@ impl Store for FsStore {
segment_id: SegmentId, segment_id: SegmentId,
entry: &LogEntry, entry: &LogEntry,
) -> Result<(), StoreError> { ) -> Result<(), StoreError> {
entry.validate_input_metadata()?;
let _retention = self.retention().lock_live(session_id)?; let _retention = self.retention().lock_live(session_id)?;
let line = serde_json::to_string(entry)?; let line = serde_json::to_string(entry)?;
self.append_line(&self.log_path(session_id, segment_id), &line) self.append_line(&self.log_path(session_id, segment_id), &line)
@@ -448,6 +449,9 @@ impl Store for FsStore {
segment_id: SegmentId, segment_id: SegmentId,
entries: &[LogEntry], entries: &[LogEntry],
) -> Result<(), StoreError> { ) -> Result<(), StoreError> {
for entry in entries {
entry.validate_input_metadata()?;
}
let _retention = self.retention().lock_live(session_id)?; let _retention = self.retention().lock_live(session_id)?;
let path = self.log_path(session_id, segment_id); let path = self.log_path(session_id, segment_id);
if let Some(parent) = path.parent() { if let Some(parent) = path.parent() {
+202
View File
@@ -67,6 +67,10 @@ pub enum LoggedSessionHistoryOrigin {
ToolOutput { ToolOutput {
worker: LoggedWorkerSubject, worker: LoggedWorkerSubject,
}, },
/// Descriptive producer identity; use policies live in separate metadata.
Provided {
producer: String,
},
/// Host-produced opening context summary; not auto-read or TaskStore material. /// Host-produced opening context summary; not auto-read or TaskStore material.
CompactionSummary, CompactionSummary,
/// Frozen Host Journal reference, never independent observation evidence. /// Frozen Host Journal reference, never independent observation evidence.
@@ -81,19 +85,32 @@ pub struct LoggedHistoryDerivation {
} }
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LoggedSessionHistoryMetadata { pub struct LoggedSessionHistoryMetadata {
pub entry_id: LoggedSessionHistoryEntryId, pub entry_id: LoggedSessionHistoryEntryId,
pub origin: LoggedSessionHistoryOrigin, pub origin: LoggedSessionHistoryOrigin,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub derivation: Option<LoggedHistoryDerivation>, pub derivation: Option<LoggedHistoryDerivation>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_metadata: Option<crate::HistoryInputMetadata>,
} }
impl LoggedSessionHistoryMetadata { impl LoggedSessionHistoryMetadata {
pub(crate) fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
if matches!(self.origin, LoggedSessionHistoryOrigin::Provided { .. })
&& self.input_metadata.is_none()
{
return Err(crate::InputMetadataError::MissingMetadata);
}
Ok(())
}
pub fn legacy_unknown() -> Self { pub fn legacy_unknown() -> Self {
Self { Self {
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
input_metadata: None,
} }
} }
} }
@@ -101,20 +118,146 @@ impl LoggedSessionHistoryMetadata {
/// Persisted item and metadata are one value so transforms cannot reorder or /// Persisted item and metadata are one value so transforms cannot reorder or
/// truncate one without the other. /// truncate one without the other.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(try_from = "HistoryEntryWire")]
pub struct LoggedHistoryEntry { pub struct LoggedHistoryEntry {
pub item: LoggedItem, pub item: LoggedItem,
pub metadata: LoggedSessionHistoryMetadata, pub metadata: LoggedSessionHistoryMetadata,
} }
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct HistoryEntryWire {
item: LoggedItem,
metadata: LoggedSessionHistoryMetadata,
}
impl TryFrom<HistoryEntryWire> for LoggedHistoryEntry {
type Error = crate::InputMetadataError;
fn try_from(wire: HistoryEntryWire) -> Result<Self, Self::Error> {
let entry = Self {
item: wire.item,
metadata: wire.metadata,
};
entry.validate_input_metadata()?;
Ok(entry)
}
}
impl LoggedHistoryEntry {
pub fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
self.metadata.validate_input_metadata()?;
if let Some(input) = &self.metadata.input_metadata {
match &self.item {
LoggedItem::Message {
role: crate::LoggedRole::System,
content,
} => {
let [crate::LoggedContentPart::Text { text }] = content.as_slice() else {
return Err(crate::InputMetadataError::InconsistentRecord);
};
if text.len() > crate::input_metadata::MAX_SYSTEM_INPUT_BYTES {
return Err(crate::InputMetadataError::ResourceLimit);
}
}
// Other logical roles may carry use policy, but cannot masquerade
// as the System-only public/payload extension contract.
_ if input.feature_payload().is_some() || input.public_item().is_some() => {
return Err(crate::InputMetadataError::InconsistentRecord);
}
_ => {}
}
}
Ok(())
}
}
/// Typed system-item history record. The typed system event remains available /// Typed system-item history record. The typed system event remains available
/// to client replay while its model-visible projection carries the same stable /// to client replay while its model-visible projection carries the same stable
/// metadata used by live history. /// metadata used by live history.
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(try_from = "SystemEntryWire")]
pub struct LoggedSystemHistoryEntry { pub struct LoggedSystemHistoryEntry {
pub item: crate::SystemItem, pub item: crate::SystemItem,
pub metadata: LoggedSessionHistoryMetadata, pub metadata: LoggedSessionHistoryMetadata,
} }
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct SystemEntryWire {
item: crate::SystemItem,
metadata: LoggedSessionHistoryMetadata,
}
impl TryFrom<SystemEntryWire> for LoggedSystemHistoryEntry {
type Error = crate::InputMetadataError;
fn try_from(wire: SystemEntryWire) -> Result<Self, Self::Error> {
let entry = Self {
item: wire.item,
metadata: wire.metadata,
};
entry.validate_input_metadata()?;
Ok(entry)
}
}
impl LoggedSystemHistoryEntry {
pub fn from_input(
origin: LoggedSessionHistoryOrigin,
derivation: Option<LoggedHistoryDerivation>,
input: crate::SystemInputEnvelope,
) -> Self {
let (body, input_metadata) = input.into_parts();
Self {
item: crate::SystemItem::Input { body },
metadata: LoggedSessionHistoryMetadata {
entry_id: LoggedSessionHistoryEntryId::new(),
origin,
derivation,
input_metadata: Some(input_metadata),
},
}
}
pub fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
self.metadata.validate_input_metadata()?;
match (&self.item, &self.metadata.input_metadata) {
(crate::SystemItem::Input { body }, Some(_)) => {
if body.len() > crate::input_metadata::MAX_SYSTEM_INPUT_BYTES {
return Err(crate::InputMetadataError::ResourceLimit);
}
}
(crate::SystemItem::Input { .. }, None) => {
return Err(crate::InputMetadataError::MissingMetadata);
}
(_, Some(_)) => return Err(crate::InputMetadataError::InconsistentRecord),
(_, None) => {}
}
Ok(())
}
pub fn to_history_entry(&self) -> LoggedHistoryEntry {
LoggedHistoryEntry {
item: self.item.to_history_item().into(),
metadata: self.metadata.clone(),
}
}
pub fn public_item(&self) -> crate::PublicSystemItem {
if matches!(self.item, crate::SystemItem::Input { .. }) {
return self
.metadata
.input_metadata
.as_ref()
.and_then(crate::HistoryInputMetadata::public_item)
.cloned()
.unwrap_or_else(|| self.item.public_item());
}
self.item.public_item()
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -132,6 +275,7 @@ mod tests {
}], }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::HumanInput { origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".into(), account_id: "account-1".into(),
@@ -150,6 +294,64 @@ mod tests {
); );
} }
#[test]
fn external_system_records_reject_missing_policy_role_mismatch_and_overlimit_body() {
let valid = serde_json::json!({
"item": { "kind": "input", "body": "context" },
"metadata": {
"entry_id": "stable-entry", "origin": { "kind": "provided", "producer": "owner" },
"input_metadata": { "usage": {
"recording_material": "exclude", "memory_evidence": "exclude",
"observation": "hidden", "compaction": "reference_input"
} }
}
});
let restored: LoggedSystemHistoryEntry = serde_json::from_value(valid.clone()).unwrap();
let flat = serde_json::to_value(restored.to_history_entry()).unwrap();
assert!(serde_json::from_value::<LoggedHistoryEntry>(flat.clone()).is_ok());
for generic in [true, false] {
let wire = if generic { &valid } else { &flat };
let decode = |wire| {
if generic {
serde_json::from_value::<LoggedSystemHistoryEntry>(wire).is_err()
} else {
serde_json::from_value::<LoggedHistoryEntry>(wire).is_err()
}
};
let mut missing = wire.clone();
missing["metadata"]
.as_object_mut()
.unwrap()
.remove("input_metadata");
assert!(decode(missing));
let mut absent_policy = wire.clone();
absent_policy["metadata"]["input_metadata"]
.as_object_mut()
.unwrap()
.remove("usage");
assert!(decode(absent_policy));
let mut invalid_role = wire.clone();
invalid_role["item"] =
serde_json::to_value(LoggedItem::from(agen::Item::user_message("not system")))
.unwrap();
invalid_role["metadata"]["input_metadata"]["public"] = serde_json::json!({
"kind": "context", "body": "public system projection", "presentation": {"kind": "context"}
});
assert!(decode(invalid_role));
let mut oversized = wire.clone();
if generic {
oversized["item"]["body"] = "x"
.repeat(crate::input_metadata::MAX_SYSTEM_INPUT_BYTES + 1)
.into();
} else {
oversized["item"]["content"][0]["text"] = "x"
.repeat(crate::input_metadata::MAX_SYSTEM_INPUT_BYTES + 1)
.into();
}
assert!(decode(oversized));
}
}
#[test] #[test]
fn annotated_segment_start_is_restore_visible_without_projecting_metadata() { fn annotated_segment_start_is_restore_visible_without_projecting_metadata() {
let session_id = uuid::Uuid::now_v7(); let session_id = uuid::Uuid::now_v7();
+563
View File
@@ -0,0 +1,563 @@
//! Validated, Feature-neutral history-input policy and System extensions.
//!
//! The enclosing history annotation owns identity, origin and derivation. None of
//! those descriptive facts selects a use policy here. Model text, independent use
//! policies, private Feature data and an explicit public projection travel as one
//! value. Restoring this value does not load a Feature or invoke domain behavior.
//!
//! Payload validation bounds the decoded value; enclosing log readers must also
//! enforce their record allocation limit before decoding untrusted persisted input.
use agen::Item;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::PublicSystemItem;
pub const MAX_FEATURE_PAYLOAD_BYTES: usize = 128 * 1024;
pub const MAX_SYSTEM_INPUT_BYTES: usize = 4 * 1024 * 1024;
pub const MAX_SYSTEM_PUBLIC_BYTES: usize = 256 * 1024;
const MAX_SCHEMA_BYTES: usize = 128;
const MAX_JSON_DEPTH: usize = 16;
const MAX_JSON_NODES: usize = 8192;
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum InputMetadataError {
#[error("history input payload schema is invalid")]
InvalidSchema,
#[error("history input payload version must be nonzero")]
InvalidVersion,
#[error("system input resource limit exceeded")]
ResourceLimit,
#[error("history input payload could not be serialized")]
Serialization,
#[error("history input payload schema or version is unsupported by this consumer")]
UnsupportedPayload,
#[error("history input payload does not match the owner's typed contract")]
InvalidTypedPayload,
#[error("provided input is missing explicit metadata")]
MissingMetadata,
#[error("input metadata does not match its logical record")]
InconsistentRecord,
}
/// One private payload owned by a Feature's versioned typed decoder.
/// Unknown valid schemas/versions remain data, never execution instructions.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(try_from = "PayloadWire")]
pub struct FeatureInputPayload {
schema: String,
version: u32,
value: Value,
}
impl std::fmt::Debug for FeatureInputPayload {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FeatureInputPayload")
.field("schema", &self.schema)
.field("version", &self.version)
.finish_non_exhaustive()
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct PayloadWire {
schema: String,
version: u32,
value: Value,
}
impl TryFrom<PayloadWire> for FeatureInputPayload {
type Error = InputMetadataError;
fn try_from(wire: PayloadWire) -> Result<Self, Self::Error> {
if wire.schema.is_empty()
|| wire.schema.len() > MAX_SCHEMA_BYTES
|| !wire.schema.as_bytes()[0].is_ascii_lowercase()
|| !wire.schema.bytes().all(|byte| {
byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-_.:/".contains(&byte)
})
{
return Err(InputMetadataError::InvalidSchema);
}
if wire.version == 0 {
return Err(InputMetadataError::InvalidVersion);
}
validate_json(&wire.value, MAX_FEATURE_PAYLOAD_BYTES)?;
Ok(Self {
schema: wire.schema,
version: wire.version,
value: wire.value,
})
}
}
impl FeatureInputPayload {
pub fn encode<T: Serialize>(
schema: impl Into<String>,
version: u32,
value: &T,
) -> Result<Self, InputMetadataError> {
PayloadWire {
schema: schema.into(),
version,
value: serde_json::to_value(value).map_err(|_| InputMetadataError::Serialization)?,
}
.try_into()
}
pub fn schema(&self) -> &str {
&self.schema
}
pub fn version(&self) -> u32 {
self.version
}
/// The owner explicitly selects its contract before decoding. A valid unknown
/// payload can be retained and round-tripped without calling this operation.
pub fn decode<T: DeserializeOwned>(
&self,
schema: &str,
version: u32,
) -> Result<T, InputMetadataError> {
if self.schema != schema || self.version != version {
return Err(InputMetadataError::UnsupportedPayload);
}
serde_json::from_value(self.value.clone())
.map_err(|_| InputMetadataError::InvalidTypedPayload)
}
}
/// Eligibility is supplied independently for recording and Memory evidence.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InputEvidenceUse {
Include,
Exclude,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InputObservationUse {
Readable,
Hidden,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InputCompactionUse {
/// Ordinary history eligible for the selected retained tail.
Retain,
/// A full opening summary consumed separately and superseded by compaction.
OpeningSummary,
/// Full historical context for the compactor, not an accumulating retained tail.
ReferenceInput,
}
/// No permissive defaults: malformed/unknown policies cannot turn into eligibility.
/// Public presentation is deliberately not one of these use decisions.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HistoryInputUsePolicy {
pub recording_material: InputEvidenceUse,
pub memory_evidence: InputEvidenceUse,
pub observation: InputObservationUse,
pub compaction: InputCompactionUse,
}
/// Use policy is independent of logical role. Non-System derived Tool context
/// carries policy only; private Feature payload and public presentation belong
/// to the System envelope and are rejected on other roles at the history boundary.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(try_from = "MetadataWire")]
pub struct HistoryInputMetadata {
usage: HistoryInputUsePolicy,
feature_payload: Option<FeatureInputPayload>,
public: Option<PublicSystemItem>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct MetadataWire {
usage: HistoryInputUsePolicy,
feature_payload: Option<FeatureInputPayload>,
public: Option<PublicSystemItem>,
}
impl TryFrom<MetadataWire> for HistoryInputMetadata {
type Error = InputMetadataError;
fn try_from(wire: MetadataWire) -> Result<Self, Self::Error> {
if let Some(public) = &wire.public {
// Kind is a descriptive label, not a Feature dispatch or authority key.
if public.kind.is_empty()
|| public.kind.len() > MAX_SCHEMA_BYTES
|| public.kind.chars().any(char::is_control)
{
return Err(InputMetadataError::InvalidSchema);
}
let value =
serde_json::to_value(public).map_err(|_| InputMetadataError::Serialization)?;
validate_json(&value, MAX_SYSTEM_PUBLIC_BYTES)?;
}
Ok(Self {
usage: wire.usage,
feature_payload: wire.feature_payload,
public: wire.public,
})
}
}
impl HistoryInputMetadata {
pub fn new(
usage: HistoryInputUsePolicy,
feature_payload: Option<FeatureInputPayload>,
public: Option<PublicSystemItem>,
) -> Result<Self, InputMetadataError> {
MetadataWire {
usage,
feature_payload,
public,
}
.try_into()
}
pub fn usage(&self) -> HistoryInputUsePolicy {
self.usage
}
pub fn feature_payload(&self) -> Option<&FeatureInputPayload> {
self.feature_payload.as_ref()
}
pub fn public_item(&self) -> Option<&PublicSystemItem> {
self.public.as_ref()
}
pub fn public_use_policy(&self) -> protocol::SessionEntryUsePolicy {
protocol::SessionEntryUsePolicy {
observation_readable: self.usage.observation == InputObservationUse::Readable,
memory_evidence: self.usage.memory_evidence == InputEvidenceUse::Include,
}
}
pub fn public_system_use_policy(&self) -> protocol::SessionEntryUsePolicy {
if self.public.is_some() {
self.public_use_policy()
} else {
protocol::SessionEntryUsePolicy {
observation_readable: false,
memory_evidence: false,
}
}
}
pub fn public_projection(&self) -> PublicSystemItem {
self.public.clone().unwrap_or_else(|| PublicSystemItem {
kind: "input".into(),
body: String::new(),
presentation: crate::PublicSystemPresentation::Ignored,
})
}
}
/// System role is invariant; provenance and use policies are attached data.
/// An absent public projection never implicitly publishes the model body.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(try_from = "InputWire")]
pub struct SystemInputEnvelope {
body: String,
metadata: HistoryInputMetadata,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct InputWire {
body: String,
metadata: HistoryInputMetadata,
}
impl TryFrom<InputWire> for SystemInputEnvelope {
type Error = InputMetadataError;
fn try_from(wire: InputWire) -> Result<Self, Self::Error> {
if wire.body.len() > MAX_SYSTEM_INPUT_BYTES {
return Err(InputMetadataError::ResourceLimit);
}
Ok(Self {
body: wire.body,
metadata: wire.metadata,
})
}
}
impl SystemInputEnvelope {
pub fn new(
body: impl Into<String>,
metadata: HistoryInputMetadata,
) -> Result<Self, InputMetadataError> {
InputWire {
body: body.into(),
metadata,
}
.try_into()
}
pub fn body(&self) -> &str {
&self.body
}
pub fn metadata(&self) -> &HistoryInputMetadata {
&self.metadata
}
pub fn to_history_item(&self) -> Item {
Item::system_message(self.body.clone())
}
pub fn into_parts(self) -> (String, HistoryInputMetadata) {
(self.body, self.metadata)
}
}
fn validate_json(value: &Value, max_bytes: usize) -> Result<(), InputMetadataError> {
fn walk(value: &Value, depth: usize, remaining: &mut usize) -> Result<(), InputMetadataError> {
if depth > MAX_JSON_DEPTH || *remaining == 0 {
return Err(InputMetadataError::ResourceLimit);
}
*remaining -= 1;
match value {
Value::Array(values) => {
for value in values {
walk(value, depth + 1, remaining)?;
}
}
Value::Object(values) => {
for value in values.values() {
walk(value, depth + 1, remaining)?;
}
}
_ => {}
}
Ok(())
}
let mut remaining = MAX_JSON_NODES;
walk(value, 0, &mut remaining)?;
let bytes = serde_json::to_vec(value).map_err(|_| InputMetadataError::Serialization)?;
if bytes.len() > max_bytes {
return Err(InputMetadataError::ResourceLimit);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{LoggedItem, LoggedRole, PublicSystemPresentation};
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct OwnerPayload {
source_digest: String,
omitted: bool,
}
fn reference_policy() -> HistoryInputUsePolicy {
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::ReferenceInput,
}
}
#[test]
fn unknown_valid_payload_round_trips_without_domain_decode_or_public_exposure() {
let payload = FeatureInputPayload::encode(
"future-owner:context",
99,
&OwnerPayload {
source_digest: "private-digest".into(),
omitted: true,
},
)
.unwrap();
let envelope = SystemInputEnvelope::new(
"private model context",
HistoryInputMetadata::new(reference_policy(), Some(payload), None).unwrap(),
)
.unwrap();
let restored: SystemInputEnvelope =
serde_json::from_slice(&serde_json::to_vec(&envelope).unwrap()).unwrap();
assert_eq!(restored.body(), "private model context");
assert!(restored.metadata().public_item().is_none());
let saved_payload = restored.metadata().feature_payload().unwrap();
assert_eq!(saved_payload.schema(), "future-owner:context");
assert_eq!(saved_payload.version(), 99);
assert_eq!(
saved_payload.decode::<OwnerPayload>("known-owner:context", 1),
Err(InputMetadataError::UnsupportedPayload)
);
assert_eq!(
serde_json::to_value(&restored).unwrap(),
serde_json::to_value(&envelope).unwrap()
);
assert!(!format!("{saved_payload:?}").contains("private-digest"));
}
#[test]
fn owner_decodes_typed_payload_only_for_its_explicit_contract() {
let input = OwnerPayload {
source_digest: "rev".into(),
omitted: false,
};
let payload = FeatureInputPayload::encode("owner:context", 1, &input).unwrap();
assert_eq!(
payload.decode::<OwnerPayload>("owner:context", 1).unwrap(),
input
);
let invalid = FeatureInputPayload::encode(
"owner:context",
1,
&serde_json::json!({ "omitted": "not a bool" }),
)
.unwrap();
assert_eq!(
invalid.decode::<OwnerPayload>("owner:context", 1),
Err(InputMetadataError::InvalidTypedPayload)
);
}
#[test]
fn malformed_or_oversized_payload_is_rejected_at_construction_and_restore() {
for (schema, version, value) in [
("", 1, Value::Null),
("invalid schema", 1, Value::Null),
("owner:context", 0, Value::Null),
(
"owner:context",
1,
Value::String("x".repeat(MAX_FEATURE_PAYLOAD_BYTES)),
),
(
"owner:context",
1,
Value::Array(vec![Value::Null; MAX_JSON_NODES]),
),
(
"owner:context",
1,
(0..=MAX_JSON_DEPTH).fold(Value::Null, |inner, _| Value::Array(vec![inner])),
),
] {
assert!(FeatureInputPayload::encode(schema, version, &value).is_err());
assert!(
serde_json::from_value::<FeatureInputPayload>(
serde_json::json!({ "schema": schema, "version": version, "value": value })
)
.is_err()
);
}
}
#[test]
fn policy_is_required_and_unknown_or_malformed_fields_fail_closed() {
let valid = serde_json::to_value(reference_policy()).unwrap();
for field in [
"recording_material",
"memory_evidence",
"observation",
"compaction",
] {
let mut absent = valid.clone();
absent.as_object_mut().unwrap().remove(field);
assert!(serde_json::from_value::<HistoryInputUsePolicy>(absent).is_err());
let mut unknown = valid.clone();
unknown[field] = Value::String("future-allow".into());
assert!(serde_json::from_value::<HistoryInputUsePolicy>(unknown).is_err());
}
let mut extra = valid;
extra["feature_name"] = Value::String("infer-policy-from-me".into());
assert!(serde_json::from_value::<HistoryInputUsePolicy>(extra).is_err());
}
#[test]
fn feature_name_and_model_body_do_not_select_policies_or_public_projection() {
for (schema, body) in [
("owner:journal", "past context"),
("unrelated:skill", "do not extract this; legitimate wording"),
] {
let public = PublicSystemItem {
kind: "context-label".into(),
body: "explicit public text".into(),
presentation: PublicSystemPresentation::Context,
};
let metadata = HistoryInputMetadata::new(
reference_policy(),
Some(
FeatureInputPayload::encode(
schema,
1,
&OwnerPayload {
source_digest: "private".into(),
omitted: false,
},
)
.unwrap(),
),
Some(public),
)
.unwrap();
let input = SystemInputEnvelope::new(body, metadata).unwrap();
assert_eq!(input.metadata().usage(), reference_policy());
let history: LoggedItem = input.to_history_item().into();
assert!(matches!(
history,
LoggedItem::Message {
role: LoggedRole::System,
..
}
));
assert_eq!(input.to_history_item().as_text(), Some(body));
let public = serde_json::to_string(input.metadata().public_item().unwrap()).unwrap();
assert!(!public.contains("private"));
assert!(!public.contains(schema));
assert!(!public.contains(body));
}
}
#[test]
fn use_policies_remain_independent_of_each_other_and_of_public_display() {
let usage = HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Include,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Readable,
compaction: InputCompactionUse::Retain,
};
let metadata = HistoryInputMetadata::new(usage, None, None).unwrap();
let restored: HistoryInputMetadata =
serde_json::from_value(serde_json::to_value(&metadata).unwrap()).unwrap();
assert_eq!(restored.usage(), usage);
assert!(restored.public_item().is_none());
assert!(restored.feature_payload().is_none());
}
#[test]
fn model_body_and_public_projection_have_independent_resource_limits() {
let metadata = HistoryInputMetadata::new(reference_policy(), None, None).unwrap();
assert!(
SystemInputEnvelope::new("x".repeat(MAX_SYSTEM_INPUT_BYTES + 1), metadata).is_err()
);
let public = PublicSystemItem {
kind: "context".into(),
body: "x".repeat(MAX_SYSTEM_PUBLIC_BYTES),
presentation: PublicSystemPresentation::Context,
};
assert!(HistoryInputMetadata::new(reference_policy(), None, Some(public)).is_err());
let invalid = serde_json::json!({ "body": "small", "metadata": { "public": null, "feature_payload": null } });
assert!(serde_json::from_value::<SystemInputEnvelope>(invalid).is_err());
}
}
+354 -21
View File
@@ -1,12 +1,12 @@
//! Versioned decoder for Session schemas that predate canonical annotated history. //! Versioned decoder for Session schemas v1-5, including frozen Feature wire.
//! //!
//! These types are intentionally private to `session-store`. Current writers, //! These types are intentionally private to `session-store`. Current writers,
//! replay, and public projections use [`crate::LogEntry`] exclusively; only the //! replay, and public projections use [`crate::LogEntry`] exclusively; only the
//! Worker Session schema migration is allowed to deserialize these shapes. //! explicit Session schema migration is allowed to deserialize these shapes.
use agen::llm_client::types::RequestConfig; use agen::llm_client::types::RequestConfig;
use protocol::Segment; use protocol::Segment;
use serde::Deserialize; use serde::{Deserialize, Serialize};
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
@@ -17,6 +17,212 @@ use crate::{
SegmentOrigin, SessionExtension, SessionId, SystemItem, SegmentOrigin, SessionExtension, SessionId, SystemItem,
}; };
// Frozen v1-4 Feature wire. These types are deliberately not exported and are
// never used by LogEntry/SystemItem's current deserializers. Payload structs
// retain the owner's v1 shape rather than erasing typed fields into arbitrary JSON.
#[derive(Debug, Serialize, Deserialize)]
struct JournalStartupPayload {
snapshot: serde_json::Value,
snapshot_sha256: String,
prompt_provenance: crate::PromptRenderProvenance,
}
#[derive(Debug, Serialize, Deserialize)]
struct ResidentSummaryPayload {
#[serde(default)]
prompt_provenance: Option<crate::PromptRenderProvenance>,
}
#[derive(Debug, Serialize, Deserialize)]
struct SubjectBehaviorPayload {
#[serde(default)]
behavior_md: String,
#[serde(default)]
prompt_provenance: Option<crate::PromptRenderProvenance>,
}
#[derive(Debug, Serialize, Deserialize)]
struct SkillActivationPayload {
name: String,
}
#[derive(Debug, Serialize, Deserialize)]
struct InvocationResultPayload {
result: protocol::FeatureInvocationResult,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum LegacyFeatureSystemItem {
JournalStartup {
body: String,
#[serde(flatten)]
payload: JournalStartupPayload,
},
ResidentSummaryRefresh {
body: String,
#[serde(flatten)]
payload: ResidentSummaryPayload,
},
SubjectBehaviorRefresh {
body: String,
#[serde(flatten)]
payload: SubjectBehaviorPayload,
},
SkillActivation {
body: String,
#[serde(flatten)]
payload: SkillActivationPayload,
},
FeatureInvocationResult {
body: String,
#[serde(flatten)]
payload: InvocationResultPayload,
},
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum LegacySystemItem {
Feature(LegacyFeatureSystemItem),
// Includes already-migrated Input records for interrupted migration retry.
Core(SystemItem),
}
impl LegacySystemItem {
fn into_entry(
self,
mut metadata: LoggedSessionHistoryMetadata,
) -> Result<LoggedSystemHistoryEntry, crate::InputMetadataError> {
use crate::InputCompactionUse::{ReferenceInput, Retain};
let item = match self {
Self::Core(item) => item,
Self::Feature(feature) => {
metadata.validate_input_metadata()?;
if metadata.input_metadata.is_some() {
return Err(crate::InputMetadataError::InconsistentRecord);
}
let (body, input_metadata) = match feature {
LegacyFeatureSystemItem::JournalStartup { body, payload } => {
migrated_feature_input(
body,
"journal_startup",
"journal:startup",
payload,
ReferenceInput,
)?
}
LegacyFeatureSystemItem::ResidentSummaryRefresh { body, payload } => {
migrated_feature_input(
body,
"resident_summary_refresh",
"memory:resident-summary",
payload,
Retain,
)?
}
LegacyFeatureSystemItem::SubjectBehaviorRefresh { body, payload } => {
migrated_feature_input(
body,
"subject_behavior_refresh",
"subject:behavior",
payload,
Retain,
)?
}
LegacyFeatureSystemItem::SkillActivation { body, payload } => {
migrated_feature_input(
body,
"skill_activation",
"skill:activation",
payload,
Retain,
)?
}
LegacyFeatureSystemItem::FeatureInvocationResult { body, payload } => {
migrated_feature_input(
body,
"feature_invocation_result",
"feature:invocation-result",
payload,
Retain,
)?
}
};
// Identity/origin/derivation are retained, never re-inferred.
metadata.input_metadata = Some(input_metadata);
SystemItem::Input { body }
}
};
let entry = LoggedSystemHistoryEntry { item, metadata };
entry.validate_input_metadata()?;
Ok(entry)
}
}
fn migrated_feature_input<T: Serialize>(
body: String,
label: &str,
schema: &str,
payload: T,
compaction: crate::InputCompactionUse,
) -> Result<(String, crate::HistoryInputMetadata), crate::InputMetadataError> {
let metadata = crate::HistoryInputMetadata::new(
migration_policy(compaction),
Some(crate::FeatureInputPayload::encode(schema, 1, &payload)?),
Some(crate::PublicSystemItem {
kind: label.into(),
body: body.clone(),
presentation: crate::PublicSystemPresentation::Context,
}),
)?;
Ok((body, metadata))
}
/// Annotated old wire used by v3/v4, and by already-converted files when an
/// older marker survived an interrupted migration. No ordinary decoder uses it.
#[derive(Debug, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum LegacyAnnotatedLogEntry {
AnnotatedSystemItem {
ts: u64,
entry: LegacySystemHistoryEntry,
#[serde(default)]
extensions: Vec<SessionExtension>,
},
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacySystemHistoryEntry {
item: LegacySystemItem,
metadata: LoggedSessionHistoryMetadata,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum LegacyAnnotatedOrCurrent {
Annotated(LegacyAnnotatedLogEntry),
Current(LogEntry),
}
impl LegacyAnnotatedOrCurrent {
fn into_current(self) -> Result<LogEntry, crate::InputMetadataError> {
match self {
Self::Annotated(LegacyAnnotatedLogEntry::AnnotatedSystemItem {
ts,
entry,
extensions,
}) => Ok(LogEntry::AnnotatedSystemItem {
ts,
entry: entry.item.into_entry(entry.metadata)?,
extensions,
}),
Self::Current(entry) => Ok(entry),
}
}
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")] #[serde(tag = "kind", rename_all = "snake_case")]
enum LegacyHistoryLogEntry { enum LegacyHistoryLogEntry {
@@ -47,7 +253,9 @@ enum LegacyHistoryLogEntry {
}, },
SystemItem { SystemItem {
ts: u64, ts: u64,
item: SystemItem, item: LegacySystemItem,
#[serde(default)]
extensions: Vec<SessionExtension>,
}, },
} }
@@ -57,7 +265,7 @@ enum LegacyHistoryLogEntry {
#[serde(untagged)] #[serde(untagged)]
enum LegacySessionLogEntryV1 { enum LegacySessionLogEntryV1 {
History(LegacyHistoryLogEntry), History(LegacyHistoryLogEntry),
Current(LogEntry), Current(LegacyAnnotatedOrCurrent),
} }
/// Schema v2 retained the v1 history shapes while adding non-history records. /// Schema v2 retained the v1 history shapes while adding non-history records.
@@ -67,7 +275,7 @@ enum LegacySessionLogEntryV1 {
#[serde(untagged)] #[serde(untagged)]
enum LegacySessionLogEntryV2 { enum LegacySessionLogEntryV2 {
History(LegacyHistoryLogEntry), History(LegacyHistoryLogEntry),
Current(LogEntry), Current(LegacyAnnotatedOrCurrent),
} }
pub(crate) fn decode_entry( pub(crate) fn decode_entry(
@@ -77,24 +285,75 @@ pub(crate) fn decode_entry(
segment_id: SegmentId, segment_id: SegmentId,
line_index: usize, line_index: usize,
) -> Result<LogEntry, serde_json::Error> { ) -> Result<LogEntry, serde_json::Error> {
let mut wire: serde_json::Value = serde_json::from_str(line)?;
migrate_legacy_metadata_field(&mut wire)?;
let entry = match schema_version { let entry = match schema_version {
1 => match serde_json::from_str::<LegacySessionLogEntryV1>(line)? { 1 => match serde_json::from_value::<LegacySessionLogEntryV1>(wire)? {
LegacySessionLogEntryV1::History(entry) => Entry::History(entry), LegacySessionLogEntryV1::History(entry) => Entry::History(entry),
LegacySessionLogEntryV1::Current(entry) => Entry::Current(entry), LegacySessionLogEntryV1::Current(entry) => Entry::Current(entry),
}, },
2 => match serde_json::from_str::<LegacySessionLogEntryV2>(line)? { 2 => match serde_json::from_value::<LegacySessionLogEntryV2>(wire)? {
LegacySessionLogEntryV2::History(entry) => Entry::History(entry), LegacySessionLogEntryV2::History(entry) => Entry::History(entry),
LegacySessionLogEntryV2::Current(entry) => Entry::Current(entry), LegacySessionLogEntryV2::Current(entry) => Entry::Current(entry),
}, },
3 => Entry::Current(serde_json::from_str(line)?), 3 | 4 => Entry::Current(serde_json::from_value(wire)?),
5 => return serde_json::from_value(wire),
_ => unreachable!("legacy decoder called for unsupported schema {schema_version}"), _ => unreachable!("legacy decoder called for unsupported schema {schema_version}"),
}; };
Ok(match entry { match entry {
Entry::History(entry) => { Entry::History(entry) => {
canonicalize_history_entry(session_id, segment_id, line_index, entry) canonicalize_history_entry(session_id, segment_id, line_index, entry)
} }
Entry::Current(entry) => entry, Entry::Current(entry) => entry.into_current(),
}) }
.map_err(serde::de::Error::custom)
}
// The v5 System-only metadata field was widened to role-neutral input metadata
// in v6. Touch only known history slots, never arbitrary Feature payload objects.
// Both keys are an ambiguous source, not permission to discard either value.
fn migrate_legacy_metadata_field(record: &mut serde_json::Value) -> Result<(), serde_json::Error> {
fn entry(value: &mut serde_json::Value) -> Result<(), serde_json::Error> {
if let Some(metadata) = value
.get_mut("metadata")
.and_then(serde_json::Value::as_object_mut)
{
if let Some(old) = metadata.remove("system_input") {
if metadata.contains_key("input_metadata") {
return Err(serde::de::Error::custom(
"ambiguous legacy input metadata fields",
));
}
metadata.insert("input_metadata".into(), old);
}
}
Ok(())
}
let kind = record
.get("kind")
.and_then(serde_json::Value::as_str)
.unwrap_or_default()
.to_owned();
if matches!(
kind.as_str(),
"annotated_system_item"
| "annotated_assistant_item"
| "annotated_tool_result"
| "tool_result_corrected"
) {
if let Some(value) = record.get_mut("entry") {
entry(value)?;
}
}
if let Some(history) = record
.get_mut("history")
.and_then(serde_json::Value::as_array_mut)
{
for value in history {
entry(value)?;
}
}
Ok(())
} }
/// Schema 3 used DerivedSummary for Host compaction seeds. Only the trusted /// Schema 3 used DerivedSummary for Host compaction seeds. Only the trusted
@@ -152,9 +411,81 @@ pub(crate) fn migrate_compaction_origins(
} }
} }
/// Explicit legacy-to-current boundary. No Feature inference belongs in ordinary
/// readers/restoration. Invalid or oversized payloads fail migration; the generic
/// payload/body/public limits are deliberately not relaxed for legacy data.
pub(crate) fn migrate_system_inputs(entry: &mut LogEntry) -> Result<(), crate::InputMetadataError> {
entry.validate_input_metadata()?;
match entry {
// Feature wire has already been converted by the explicit old-schema
// decoder. Ordinary LogEntry can contain only generic Input/core events.
LogEntry::AnnotatedSystemItem { entry, .. } => entry.validate_input_metadata()?,
LogEntry::AnnotatedSegmentStart { history, .. }
| LogEntry::AnnotatedUserInput { history, .. } => {
for entry in history {
migrate_flattened_system_input(entry)?;
}
}
LogEntry::AnnotatedAssistantItem { entry, .. }
| LogEntry::AnnotatedToolResult { entry, .. }
| LogEntry::ToolResultCorrected { entry, .. } => {
migrate_flattened_system_input(entry)?;
}
_ => {}
}
entry.validate_input_metadata()
}
fn migration_policy(compaction: crate::InputCompactionUse) -> crate::HistoryInputUsePolicy {
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
compaction,
}
}
fn migrate_flattened_system_input(
entry: &mut LoggedHistoryEntry,
) -> Result<(), crate::InputMetadataError> {
// Check before supplying legacy policy: Provided without metadata is a
// malformed NEW record, not permission to synthesize a producer policy.
entry.validate_input_metadata()?;
if entry.metadata.input_metadata.is_none()
&& (matches!(
entry.item,
LoggedItem::Message {
role: crate::LoggedRole::System,
..
}
) || matches!(
entry.metadata.origin,
LoggedSessionHistoryOrigin::DerivedSummary
| LoggedSessionHistoryOrigin::CompactionSummary
))
{
let compaction = match entry.metadata.origin {
LoggedSessionHistoryOrigin::JournalContext => crate::InputCompactionUse::ReferenceInput,
LoggedSessionHistoryOrigin::CompactionSummary => {
crate::InputCompactionUse::OpeningSummary
}
_ => crate::InputCompactionUse::Retain,
};
// Legacy derived context was excluded from independent evidence. Preserve
// that established decision explicitly, including synthetic Tool roles;
// never guess from Tool names or body text. Payload/public stay absent.
entry.metadata.input_metadata = Some(crate::HistoryInputMetadata::new(
migration_policy(compaction),
None,
None,
)?);
}
entry.validate_input_metadata()
}
enum Entry { enum Entry {
History(LegacyHistoryLogEntry), History(LegacyHistoryLogEntry),
Current(LogEntry), Current(LegacyAnnotatedOrCurrent),
} }
fn legacy_metadata( fn legacy_metadata(
@@ -167,6 +498,7 @@ fn legacy_metadata(
identity.extend_from_slice(&(line_index as u64).to_be_bytes()); identity.extend_from_slice(&(line_index as u64).to_be_bytes());
identity.extend_from_slice(&(item_index as u64).to_be_bytes()); identity.extend_from_slice(&(item_index as u64).to_be_bytes());
LoggedSessionHistoryMetadata { LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(format!( entry_id: LoggedSessionHistoryEntryId(format!(
"l-{}", "l-{}",
base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, identity) base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, identity)
@@ -181,8 +513,8 @@ fn canonicalize_history_entry(
segment_id: SegmentId, segment_id: SegmentId,
line_index: usize, line_index: usize,
entry: LegacyHistoryLogEntry, entry: LegacyHistoryLogEntry,
) -> LogEntry { ) -> Result<LogEntry, crate::InputMetadataError> {
match entry { Ok(match entry {
LegacyHistoryLogEntry::SegmentStart { LegacyHistoryLogEntry::SegmentStart {
ts, ts,
session_id, session_id,
@@ -236,13 +568,14 @@ fn canonicalize_history_entry(
metadata: legacy_metadata(segment_id, line_index, 0), metadata: legacy_metadata(segment_id, line_index, 0),
}, },
}, },
LegacyHistoryLogEntry::SystemItem { ts, item } => LogEntry::AnnotatedSystemItem { LegacyHistoryLogEntry::SystemItem {
ts, ts,
entry: LoggedSystemHistoryEntry {
item, item,
metadata: legacy_metadata(segment_id, line_index, 0), extensions,
} => LogEntry::AnnotatedSystemItem {
ts,
entry: item.into_entry(legacy_metadata(segment_id, line_index, 0))?,
extensions,
}, },
extensions: Vec::new(), })
},
}
} }
@@ -1,5 +1,7 @@
use super::*; use super::*;
mod v5;
fn seed( fn seed(
origin: LoggedSessionHistoryOrigin, origin: LoggedSessionHistoryOrigin,
compacted: bool, compacted: bool,
@@ -22,6 +24,7 @@ fn seed(
content: vec![crate::LoggedContentPart::Text { text: text.into() }], content: vec![crate::LoggedContentPart::Text { text: text.into() }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("summary".into()), entry_id: LoggedSessionHistoryEntryId("summary".into()),
origin, origin,
derivation: Some(crate::LoggedHistoryDerivation { derivation: Some(crate::LoggedHistoryDerivation {
@@ -0,0 +1,3 @@
{"kind":"system_item","ts":1,"item":{"kind":"skill_activation","name":"legacy-skill","body":"legacy skill body"}}
{"kind":"user_input","ts":2,"segments":[{"kind":"text","content":"legacy user body"}],"extensions":[{"domain":"old","payload":{"kept":true}}]}
{"kind":"assistant_item","ts":3,"item":{"kind":"message","role":"assistant","content":[{"kind":"text","text":"legacy reply"}]}}
@@ -0,0 +1,8 @@
{"kind":"annotated_system_item","ts":101,"entry":{"item":{"kind":"journal_startup","body":"journal body\nunchanged","snapshot":{"entries":[{"text":"private snapshot"}]},"snapshot_sha256":"snapshot-digest","prompt_provenance":{"workspace_id":"workspace","source_digest":"source","projection_digest":"projection","logical_name":"journal_startup"}},"metadata":{"entry_id":"journal-id","origin":{"kind":"legacy_unknown"},"derivation":{"sources":["source-id"]}}},"extensions":[{"domain":"migration-fixture","payload":{"kept":true}}]}
{"kind":"annotated_system_item","ts":102,"entry":{"item":{"kind":"resident_summary_refresh","body":"resident body"},"metadata":{"entry_id":"resident-id","origin":{"kind":"derived_summary"}}}}
{"kind":"annotated_system_item","ts":103,"entry":{"item":{"kind":"subject_behavior_refresh","body":"rendered subject body","behavior_md":"exact behavior","prompt_provenance":{"source_digest":"subject-source","projection_digest":"subject-projection","logical_name":"subject_behavior"}},"metadata":{"entry_id":"subject-id","origin":{"kind":"backend_instruction","operation_id":"op"}}}}
{"kind":"annotated_system_item","ts":104,"entry":{"item":{"kind":"skill_activation","body":"skill body","name":"example-skill"},"metadata":{"entry_id":"skill-id","origin":{"kind":"legacy_unknown"}}}}
{"kind":"annotated_system_item","ts":105,"entry":{"item":{"kind":"feature_invocation_result","body":"result body","result":{"invocation_id":"invoke-id","identity":"builtin:test/prepare","status":"outcome_unknown","message":"may have committed","context":null}},"metadata":{"entry_id":"result-id","origin":{"kind":"legacy_unknown"}}}}
{"kind":"annotated_user_input","ts":106,"segments":[],"history":[{"item":{"kind":"message","role":"system","content":[{"kind":"text","text":"[Journal] prefix is not Feature evidence"}]},"metadata":{"entry_id":"flat-journal","origin":{"kind":"journal_context"},"derivation":{"sources":["journal-id"]}}},{"item":{"kind":"message","role":"system","content":[{"kind":"text","text":"summary body"}]},"metadata":{"entry_id":"flat-summary","origin":{"kind":"compaction_summary"}}},{"item":{"kind":"message","role":"system","content":[{"kind":"text","text":"[Skill: example] never infer Feature"}]},"metadata":{"entry_id":"flat-other","origin":{"kind":"legacy_unknown"}}}]}
{"kind":"annotated_system_item","ts":107,"entry":{"item":{"kind":"notification","message":"raw message","body":"wrapped message"},"metadata":{"entry_id":"core-id","origin":{"kind":"legacy_unknown"}}}}
{"kind":"annotated_system_item","ts":108,"entry":{"item":{"kind":"input","body":"already generic body"},"metadata":{"entry_id":"generic-id","origin":{"kind":"provided","producer":"future-owner"},"derivation":{"sources":["source-id"]},"system_input":{"usage":{"recording_material":"include","memory_evidence":"exclude","observation":"readable","compaction":"retain"},"feature_payload":{"schema":"future:unknown","version":99,"value":{"opaque":"retained"}},"public":null}}}}
@@ -0,0 +1,5 @@
{"kind":"annotated_system_item","ts":201,"entry":{"item":{"kind":"input","body":"frozen v5 reference"},"metadata":{"entry_id":"reference-v5","origin":{"kind":"provided","producer":"owner-v5"},"system_input":{"usage":{"recording_material":"exclude","memory_evidence":"exclude","observation":"hidden","compaction":"reference_input"},"feature_payload":{"schema":"future:owner","version":8,"value":{"system_input":"opaque owner field must not be renamed"}},"public":null}}}}
{"kind":"annotated_assistant_item","ts":202,"entry":{"item":{"kind":"tool_call","call_id":"original-call","name":"ArbitraryTool","arguments":"{}"},"metadata":{"entry_id":"original-call-entry","origin":{"kind":"model_output","worker":{"worker_id":"body"}}}}}
{"kind":"annotated_tool_result","ts":203,"entry":{"item":{"kind":"tool_result","call_id":"original-call","summary":"original result"},"metadata":{"entry_id":"original-result-entry","origin":{"kind":"tool_output","worker":{"worker_id":"body"}}}}}
{"kind":"annotated_assistant_item","ts":204,"entry":{"item":{"kind":"tool_call","call_id":"derived-call","name":"ArbitraryTool","arguments":"{}"},"metadata":{"entry_id":"derived-call-entry","origin":{"kind":"derived_summary"},"derivation":{"sources":["original-call-entry","original-result-entry"]}}}}
{"kind":"annotated_tool_result","ts":205,"entry":{"item":{"kind":"tool_result","call_id":"derived-call","summary":"derived retrieval result"},"metadata":{"entry_id":"derived-result-entry","origin":{"kind":"derived_summary"},"derivation":{"sources":["original-call-entry","original-result-entry"]}}}}
@@ -0,0 +1,703 @@
//! Fixed old wire bytes, not fixtures generated by the current serializer.
use super::*;
use crate::{FsStore, InputCompactionUse, Store, WorkerSessionStore};
use serde_json::{Value, json};
use std::{fs, path::Path};
const V4: &str = include_str!("fixtures/v4-system-inputs.jsonl");
const V1: &str = include_str!("fixtures/v1-system-inputs.jsonl");
fn decode_fixture(version: u32, source: &str, segment: SegmentId) -> Vec<LogEntry> {
source
.lines()
.enumerate()
.map(|(index, line)| decode_entry(version, line, SessionId::nil(), segment, index).unwrap())
.collect()
}
fn encode(entries: &[LogEntry]) -> Vec<u8> {
let mut bytes = Vec::new();
for entry in entries {
serde_json::to_writer(&mut bytes, entry).unwrap();
bytes.push(b'\n');
}
bytes
}
fn migrate(entries: &mut [LogEntry]) {
for entry in entries {
migrate_system_inputs(entry).unwrap();
entry.validate_input_metadata().unwrap();
}
}
fn write_worker(root: &Path, version: u32, source: &[u8]) -> (SessionId, SegmentId) {
let session = SessionId::now_v7();
let segment = SegmentId::from_u128(1);
fs::create_dir_all(root.join("segments")).unwrap();
fs::write(
root.join("session.json"),
serde_json::to_vec(&json!({
"schema_version": version, "session_id": session,
}))
.unwrap(),
)
.unwrap();
fs::write(
root.join("segments").join(format!("{segment}.jsonl")),
source,
)
.unwrap();
(session, segment)
}
fn assert_marker(root: &Path, version: u32) {
let manifest: Value =
serde_json::from_slice(&fs::read(root.join("session.json")).unwrap()).unwrap();
assert_eq!(manifest["schema_version"], version);
}
#[test]
fn v4_fixed_wire_preserves_identity_body_payload_public_and_independent_policy() {
let mut entries = decode_fixture(4, V4, SegmentId::nil());
let before = entries.clone();
migrate(&mut entries);
let schemas = [
"journal:startup",
"memory:resident-summary",
"subject:behavior",
"skill:activation",
"feature:invocation-result",
];
let old_wire: Vec<Value> = V4
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
for index in 0..5 {
let LogEntry::AnnotatedSystemItem {
ts,
entry,
extensions,
} = &entries[index]
else {
panic!()
};
let old = &old_wire[index];
let old_item = &old["entry"]["item"];
let old_metadata: LoggedSessionHistoryMetadata =
serde_json::from_value(old["entry"]["metadata"].clone()).unwrap();
let old_ext: Vec<crate::SessionExtension> = old
.get("extensions")
.map(|value| serde_json::from_value(value.clone()).unwrap())
.unwrap_or_default();
assert_eq!(*ts, old["ts"].as_u64().unwrap());
assert_eq!(extensions, &old_ext);
assert!(matches!(entry.item, SystemItem::Input { .. }));
assert_eq!(
entry.item.history_text(),
old_item["body"].as_str().unwrap()
);
assert_eq!(entry.metadata.entry_id, old_metadata.entry_id);
assert_eq!(entry.metadata.origin, old_metadata.origin);
assert_eq!(entry.metadata.derivation, old_metadata.derivation);
let metadata = entry.metadata.input_metadata.as_ref().unwrap();
let public = metadata.public_item().unwrap();
assert_eq!(public.kind, old_item["kind"].as_str().unwrap());
assert_eq!(public.body, old_item["body"].as_str().unwrap());
assert_eq!(
public.presentation,
crate::PublicSystemPresentation::Context
);
assert_eq!(
metadata.usage(),
migration_policy(if index == 0 {
InputCompactionUse::ReferenceInput
} else {
InputCompactionUse::Retain
})
);
let payload = metadata.feature_payload().unwrap();
assert_eq!(payload.schema(), schemas[index]);
assert_eq!(payload.version(), 1);
let value: Value = payload.decode(schemas[index], 1).unwrap();
// Independently require each frozen owner's typed v1 payload contract.
match index {
0 => {
let _: JournalStartupPayload = payload.decode(schemas[index], 1).unwrap();
}
1 => {
let _: ResidentSummaryPayload = payload.decode(schemas[index], 1).unwrap();
}
2 => {
let _: SubjectBehaviorPayload = payload.decode(schemas[index], 1).unwrap();
}
3 => {
let _: SkillActivationPayload = payload.decode(schemas[index], 1).unwrap();
}
4 => {
let _: InvocationResultPayload = payload.decode(schemas[index], 1).unwrap();
}
_ => unreachable!(),
}
let mut expected = old_item.clone();
expected.as_object_mut().unwrap().remove("kind");
expected.as_object_mut().unwrap().remove("body");
// Optional provenance is an explicit Option in the v1 payload contract.
if index == 1 {
expected["prompt_provenance"] = Value::Null;
}
if index == 4 {
let result: protocol::FeatureInvocationResult =
serde_json::from_value(old_item["result"].clone()).unwrap();
let typed: InvocationResultPayload = payload.decode(schemas[index], 1).unwrap();
assert_eq!(typed.result, result);
// Preserve the protocol's typed serialization contract, including
// its canonical representation of optional result context.
expected["result"] = serde_json::to_value(result).unwrap();
}
assert_eq!(value, expected);
assert!(value.get("body").is_none());
}
let LogEntry::AnnotatedUserInput { history, .. } = &entries[5] else {
panic!()
};
let LogEntry::AnnotatedUserInput { history: old, .. } = &before[5] else {
panic!()
};
for (index, compaction) in [
InputCompactionUse::ReferenceInput,
InputCompactionUse::OpeningSummary,
InputCompactionUse::Retain,
]
.into_iter()
.enumerate()
{
assert_eq!(history[index].item, old[index].item);
assert_eq!(
history[index].metadata.entry_id,
old[index].metadata.entry_id
);
assert_eq!(history[index].metadata.origin, old[index].metadata.origin);
assert_eq!(
history[index].metadata.derivation,
old[index].metadata.derivation
);
let metadata = history[index].metadata.input_metadata.as_ref().unwrap();
assert_eq!(metadata.usage(), migration_policy(compaction));
assert!(metadata.public_item().is_none());
assert!(metadata.feature_payload().is_none());
}
// Core notification is unchanged; existing generic Input and unknown valid
// owner payload are retained without decoding/reclassifying a Feature.
assert_eq!(encode(&entries[6..]), encode(&before[6..]));
let once = encode(&entries);
migrate(&mut entries);
assert_eq!(encode(&entries), once);
}
#[test]
fn v1_through_v4_worker_migration_reopen_and_mixed_file_retry() {
for version in 1..=4 {
let source = if version <= 2 { V1 } else { V4 };
let root = tempfile::tempdir().unwrap();
let (session, segment) = write_worker(root.path(), version, source.as_bytes());
let path = root
.path()
.join("segments")
.join(format!("{segment}.jsonl"));
// Simulate an interrupted publication: v5 records precede legacy records
// in one file, and a fully new file coexists with an untouched old file.
let mut converted = decode_fixture(version, source, segment);
migrate(&mut converted);
let mut mixed = encode(&converted[..1]);
mixed.extend_from_slice(source.split_once('\n').unwrap().1.as_bytes());
fs::write(&path, mixed).unwrap();
let new_path = root
.path()
.join("segments")
.join(format!("{}.jsonl", SegmentId::from_u128(2)));
let new_bytes = encode(&converted);
fs::write(&new_path, &new_bytes).unwrap();
let old_path = root
.path()
.join("segments")
.join(format!("{}.jsonl", SegmentId::from_u128(3)));
fs::write(&old_path, source).unwrap();
let store = WorkerSessionStore::new(root.path()).unwrap();
assert_marker(root.path(), 6);
let restored = store.read_all(session, segment).unwrap();
assert_eq!(encode(&restored), new_bytes);
assert_eq!(fs::read(&new_path).unwrap(), new_bytes);
let once = fs::read(&path).unwrap();
let old_once = fs::read(&old_path).unwrap();
drop(store);
let reopened = WorkerSessionStore::new(root.path()).unwrap();
assert_eq!(encode(&reopened.read_all(session, segment).unwrap()), once);
assert_eq!(fs::read(&path).unwrap(), once);
assert_eq!(fs::read(&old_path).unwrap(), old_once);
}
}
#[test]
fn all_segments_preflight_keeps_old_bytes_on_decode_or_conversion_failure() {
let mut oversized: Value = serde_json::from_str(V4.lines().next().unwrap()).unwrap();
oversized["entry"]["item"]["snapshot"] =
json!({"large": "x".repeat(crate::input_metadata::MAX_FEATURE_PAYLOAD_BYTES + 1)});
let mut unknown: Value = serde_json::from_str(V4.lines().next().unwrap()).unwrap();
unknown["entry"]["item"]["kind"] = json!("unsupported_future_feature");
let mut malformed = unknown.clone();
malformed["entry"]["item"] = json!({"kind":"journal_startup", "body":"missing snapshot"});
let mut missing_metadata: Value = serde_json::from_str(V4.lines().last().unwrap()).unwrap();
missing_metadata["entry"]["metadata"]
.as_object_mut()
.unwrap()
.remove("system_input");
let mut missing_input = missing_metadata.clone();
missing_input["entry"]["metadata"]["origin"] = json!({"kind":"legacy_unknown"});
let mut invalid_policy: Value = serde_json::from_str(V4.lines().last().unwrap()).unwrap();
invalid_policy["entry"]["metadata"]["system_input"]["usage"]["compaction"] = json!("unknown");
let mut invalid_payload = invalid_policy.clone();
invalid_payload["entry"]["metadata"]["system_input"]["usage"]["compaction"] = json!("retain");
invalid_payload["entry"]["metadata"]["system_input"]["feature_payload"]["version"] = json!(0);
let mut flat_provided: Value = serde_json::from_str(V4.lines().nth(5).unwrap()).unwrap();
flat_provided["history"][0]["metadata"]["origin"] =
json!({"kind":"provided", "producer":"owner"});
for bad in [
"{broken-json}\n".to_owned(),
format!("{}\n", oversized),
format!("{}\n", unknown),
format!("{}\n", malformed),
format!("{}\n", missing_metadata),
format!("{}\n", invalid_policy),
format!("{}\n", invalid_payload),
format!("{}\n", flat_provided),
format!("{}\n", missing_input),
] {
let root = tempfile::tempdir().unwrap();
let (_, segment) = write_worker(root.path(), 4, V4.as_bytes());
let first = root
.path()
.join("segments")
.join(format!("{segment}.jsonl"));
let later = root
.path()
.join("segments")
.join(format!("{}.jsonl", SegmentId::from_u128(2)));
fs::write(&later, &bad).unwrap();
let marker = fs::read(root.path().join("session.json")).unwrap();
assert!(WorkerSessionStore::new(root.path()).is_err());
assert_eq!(fs::read(&first).unwrap(), V4.as_bytes());
assert_eq!(fs::read(&later).unwrap(), bad.as_bytes());
assert_eq!(fs::read(root.path().join("session.json")).unwrap(), marker);
}
}
#[test]
fn unsupported_schema_preserves_all_old_bytes() {
for version in [0, 7, 99] {
let root = tempfile::tempdir().unwrap();
let (_, segment) = write_worker(root.path(), version, V4.as_bytes());
let path = root
.path()
.join("segments")
.join(format!("{segment}.jsonl"));
let marker = fs::read(root.path().join("session.json")).unwrap();
assert!(WorkerSessionStore::new(root.path()).is_err());
assert_eq!(fs::read(&path).unwrap(), V4.as_bytes());
assert_eq!(fs::read(root.path().join("session.json")).unwrap(), marker);
}
}
#[test]
fn standalone_store_uses_same_v1_through_v4_migration_and_reopens() {
// None represents the existing unmarked annotated standalone layout (v3).
for version in [None, Some(1), Some(2), Some(3), Some(4)] {
let root = tempfile::tempdir().unwrap();
let session = SessionId::now_v7();
let segment = SegmentId::from_u128(1);
let dir = root.path().join(session.to_string());
fs::create_dir_all(&dir).unwrap();
let source = if version.is_some_and(|v| v <= 2) {
V1
} else {
V4
};
let path = dir.join(format!("{segment}.jsonl"));
fs::write(&path, source).unwrap();
if let Some(version) = version {
fs::write(dir.join("session-schema.json"), version.to_string()).unwrap();
}
let store = FsStore::new(root.path()).unwrap();
let entries = store.read_all(session, segment).unwrap();
for entry in entries {
entry.validate_input_metadata().unwrap();
}
assert_eq!(
fs::read_to_string(dir.join("session-schema.json"))
.unwrap()
.trim(),
"6"
);
let once = fs::read(&path).unwrap();
drop(store);
FsStore::new(root.path()).unwrap();
assert_eq!(fs::read(&path).unwrap(), once);
}
}
#[test]
fn compaction_origin_upgrade_is_retained_only_for_pre_v4_sources() {
for version in [3, 4] {
let root = tempfile::tempdir().unwrap();
let creation = seed(
LoggedSessionHistoryOrigin::DerivedSummary,
true,
crate::LoggedRole::System,
"[Compacted context summary]\n\nold summary",
);
let inherited = seed(
LoggedSessionHistoryOrigin::DerivedSummary,
false,
crate::LoggedRole::System,
"[Compacted context summary]\n\nold summary",
);
let (session, segment) =
write_worker(root.path(), version, &encode(&[creation, inherited]));
let store = WorkerSessionStore::new(root.path()).unwrap();
for record in store.read_all(session, segment).unwrap() {
let LogEntry::AnnotatedSegmentStart { history, .. } = record else {
panic!()
};
let metadata = &history[0].metadata;
assert_eq!(metadata.entry_id.0, "summary");
assert_eq!(metadata.derivation.as_ref().unwrap().sources[0].0, "source");
assert_eq!(
metadata.origin,
if version == 3 {
LoggedSessionHistoryOrigin::CompactionSummary
} else {
LoggedSessionHistoryOrigin::DerivedSummary
}
);
assert_eq!(
metadata.input_metadata.as_ref().unwrap().usage().compaction,
if version == 3 {
InputCompactionUse::OpeningSummary
} else {
InputCompactionUse::Retain
}
);
}
}
}
#[test]
fn standalone_preflight_and_unsupported_marker_leave_sources_unchanged() {
for version in [4, 99] {
let root = tempfile::tempdir().unwrap();
let session = SessionId::now_v7();
let dir = root.path().join(session.to_string());
fs::create_dir_all(&dir).unwrap();
let first = dir.join(format!("{}.jsonl", SegmentId::from_u128(1)));
let later = dir.join(format!("{}.jsonl", SegmentId::from_u128(2)));
fs::write(&first, V4).unwrap();
fs::write(&later, "{broken-json}\n").unwrap();
let marker = version.to_string();
fs::write(dir.join("session-schema.json"), &marker).unwrap();
assert!(FsStore::new(root.path()).is_err());
assert_eq!(fs::read(&first).unwrap(), V4.as_bytes());
assert_eq!(fs::read_to_string(&later).unwrap(), "{broken-json}\n");
assert_eq!(
fs::read_to_string(dir.join("session-schema.json")).unwrap(),
marker
);
}
}
#[test]
fn current_v5_rejects_all_legacy_feature_kinds_without_rewriting_sources() {
for line in V4.lines().take(5) {
// Current serde, writable open and read-only restore share the same
// current wire contract. None may dispatch to the old Feature decoder.
assert!(serde_json::from_str::<LogEntry>(line).is_err());
let bytes = format!("{line}\n");
let root = tempfile::tempdir().unwrap();
let (session, segment) = write_worker(root.path(), 6, bytes.as_bytes());
let path = root
.path()
.join("segments")
.join(format!("{segment}.jsonl"));
let marker = fs::read(root.path().join("session.json")).unwrap();
let readonly = WorkerSessionStore::open_read_only(root.path()).unwrap();
assert!(readonly.read_all_read_only(session, segment).is_err());
assert!(WorkerSessionStore::new(root.path()).is_err());
assert_eq!(fs::read(&path).unwrap(), bytes.as_bytes());
assert_eq!(fs::read(root.path().join("session.json")).unwrap(), marker);
let standalone = tempfile::tempdir().unwrap();
let dir = standalone.path().join(session.to_string());
fs::create_dir_all(&dir).unwrap();
let path = dir.join(format!("{segment}.jsonl"));
fs::write(&path, &bytes).unwrap();
fs::write(dir.join("session-schema.json"), "5").unwrap();
assert!(FsStore::new(standalone.path()).is_err());
assert_eq!(fs::read(&path).unwrap(), bytes.as_bytes());
assert_eq!(
fs::read_to_string(dir.join("session-schema.json")).unwrap(),
"5"
);
}
}
#[test]
fn v1_v2_feature_wire_uses_the_same_typed_payloads_and_stable_legacy_identity() {
let segment = SegmentId::from_u128(42);
for version in [1, 2] {
for (index, line) in V4.lines().take(5).enumerate() {
let annotated: Value = serde_json::from_str(line).unwrap();
let wire = json!({
"kind": "system_item", "ts": annotated["ts"],
"item": annotated["entry"]["item"],
"extensions": [{"domain":"old-owner", "payload":{"kept":true}}],
})
.to_string();
let entry = decode_entry(version, &wire, SessionId::nil(), segment, index).unwrap();
let retry = decode_entry(version, &wire, SessionId::nil(), segment, index).unwrap();
assert_eq!(encode(&[entry.clone()]), encode(&[retry]));
let LogEntry::AnnotatedSystemItem {
ts,
entry,
extensions,
} = entry
else {
panic!()
};
assert_eq!(ts, annotated["ts"].as_u64().unwrap());
assert!(matches!(entry.item, SystemItem::Input { .. }));
assert_eq!(
entry.item.history_text(),
annotated["entry"]["item"]["body"].as_str().unwrap()
);
assert_eq!(
entry.metadata.entry_id,
legacy_metadata(segment, index, 0).entry_id
);
assert_eq!(
entry.metadata.origin,
LoggedSessionHistoryOrigin::LegacyUnknown
);
assert!(entry.metadata.derivation.is_none());
assert_eq!(extensions[0].domain, "old-owner");
assert_eq!(extensions[0].payload, json!({"kept":true}));
let LogEntry::AnnotatedSystemItem { entry: v4, .. } =
decode_entry(4, line, SessionId::nil(), segment, index).unwrap()
else {
panic!()
};
assert_eq!(entry.metadata.input_metadata, v4.metadata.input_metadata);
}
}
}
#[test]
fn migration_rejects_malformed_typed_feature_fields_instead_of_retaining_unchecked_json() {
let mut items: Vec<Value> = V4
.lines()
.take(5)
.map(|line| serde_json::from_str::<Value>(line).unwrap()["entry"]["item"].clone())
.collect();
items[0]["snapshot_sha256"] = json!(17);
items[1]["prompt_provenance"] = json!({"logical_name":"missing digests"});
items[2]["behavior_md"] = json!(["not a string"]);
items[3]["name"] = json!({"not":"a name"});
items[4]["result"]["status"] = json!("unsupported_status");
for version in 1..=4 {
for item in &items {
let wire = if version <= 2 {
json!({"kind":"system_item", "ts":1, "item":item})
} else {
json!({"kind":"annotated_system_item", "ts":1, "entry":{
"item":item, "metadata":{
"entry_id":"malformed-id", "origin":{"kind":"legacy_unknown"},
},
}})
}
.to_string();
assert!(decode_entry(version, &wire, SessionId::nil(), SegmentId::nil(), 0).is_err());
}
}
}
#[test]
fn frozen_behavior_counter_is_accepted_only_at_the_old_schema_boundary() {
for version in 1..=4 {
let item = json!({
"kind":"subject_behavior_refresh", "body":"exact archived body",
"behavior_revision":7,
});
let wire = if version <= 2 {
json!({"kind":"system_item", "ts":1, "item":item})
} else {
json!({"kind":"annotated_system_item", "ts":1, "entry":{
"item":item, "metadata":{
"entry_id":"behavior-id", "origin":{"kind":"legacy_unknown"},
},
}})
}
.to_string();
assert!(serde_json::from_str::<LogEntry>(&wire).is_err());
let LogEntry::AnnotatedSystemItem { entry, .. } =
decode_entry(version, &wire, SessionId::nil(), SegmentId::nil(), 0).unwrap()
else {
panic!()
};
assert_eq!(entry.item.history_text(), "exact archived body");
let payload = entry
.metadata
.input_metadata
.as_ref()
.unwrap()
.feature_payload()
.unwrap();
let typed: SubjectBehaviorPayload = payload.decode("subject:behavior", 1).unwrap();
assert_eq!(typed.behavior_md, "");
assert!(typed.prompt_provenance.is_none());
assert_eq!(
serde_json::to_value(typed).unwrap(),
json!({
"behavior_md":"", "prompt_provenance":null,
})
);
}
}
#[test]
fn current_schema_rejects_generic_input_without_metadata_instead_of_migrating_it() {
let mut missing: Value = serde_json::from_str(V4.lines().last().unwrap()).unwrap();
missing["entry"]["metadata"]
.as_object_mut()
.unwrap()
.remove("system_input");
let bytes = format!("{}\n", missing);
let root = tempfile::tempdir().unwrap();
let (_, segment) = write_worker(root.path(), 6, bytes.as_bytes());
assert!(WorkerSessionStore::new(root.path()).is_err());
assert_eq!(
fs::read(
root.path()
.join("segments")
.join(format!("{segment}.jsonl"))
)
.unwrap(),
bytes.as_bytes()
);
assert_marker(root.path(), 6);
}
const V5: &str = include_str!("fixtures/v5-role-metadata.jsonl");
#[test]
fn v5_metadata_and_derived_tool_context_upgrade_without_changing_roles_or_original_evidence() {
for standalone in [false, true] {
let root = tempfile::tempdir().unwrap();
let (session, segment) = if standalone {
let session = SessionId::now_v7();
let segment = SegmentId::now_v7();
let dir = root.path().join(session.to_string());
fs::create_dir_all(&dir).unwrap();
fs::write(dir.join("session-schema.json"), b"5").unwrap();
fs::write(dir.join(format!("{segment}.jsonl")), V5).unwrap();
(session, segment)
} else {
write_worker(root.path(), 5, V5.as_bytes())
};
let entries = if standalone {
FsStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
} else {
WorkerSessionStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
};
let originals: Vec<Value> = V5
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(entries.len(), originals.len());
for (index, (entry, old)) in entries.iter().zip(originals).enumerate() {
let wire = serde_json::to_value(entry).unwrap();
assert_eq!(wire["ts"], old["ts"]);
assert_eq!(wire["entry"]["item"], old["entry"]["item"]);
for field in ["entry_id", "origin", "derivation"] {
assert_eq!(
wire["entry"]["metadata"][field],
old["entry"]["metadata"][field]
);
}
let metadata = &wire["entry"]["metadata"];
if index == 0 {
assert_eq!(
metadata["input_metadata"],
old["entry"]["metadata"]["system_input"]
);
assert_eq!(
metadata["input_metadata"]["feature_payload"]["value"]["system_input"],
"opaque owner field must not be renamed"
);
} else if index < 3 {
assert!(
metadata.get("input_metadata").is_none(),
"ordinary model Tool originals must not acquire exclusion policy"
);
} else {
assert_eq!(
metadata["input_metadata"]["usage"],
json!({
"recording_material":"exclude", "memory_evidence":"exclude", "observation":"hidden", "compaction":"retain"
})
);
}
}
let before = encode(&entries);
let reopened = if standalone {
FsStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
} else {
WorkerSessionStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
};
assert_eq!(encode(&reopened), before);
}
}
#[test]
fn legacy_metadata_alias_conflict_fails_preflight_without_dropping_either_value() {
let mut bad: Value = serde_json::from_str(V5.lines().next().unwrap()).unwrap();
bad["entry"]["metadata"]["input_metadata"] = bad["entry"]["metadata"]["system_input"].clone();
for version in [4, 5] {
let root = tempfile::tempdir().unwrap();
let (_, segment) = write_worker(root.path(), version, V5.as_bytes());
let path = root
.path()
.join("segments")
.join(format!("{segment}.jsonl"));
let later = root
.path()
.join("segments")
.join(format!("{}.jsonl", SegmentId::from_u128(2)));
let bad_bytes = format!("{bad}\n");
fs::write(&later, &bad_bytes).unwrap();
assert!(WorkerSessionStore::new(root.path()).is_err());
assert_eq!(fs::read(path).unwrap(), V5.as_bytes());
assert_eq!(fs::read(later).unwrap(), bad_bytes.as_bytes());
assert_marker(root.path(), version);
}
}
+7 -1
View File
@@ -41,6 +41,7 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res
pub mod event_trace; pub mod event_trace;
pub mod fs_store; pub mod fs_store;
pub mod history; pub mod history;
pub mod input_metadata;
mod legacy_session_log; mod legacy_session_log;
pub mod logged_item; pub mod logged_item;
mod paste_artifact; mod paste_artifact;
@@ -65,6 +66,10 @@ pub use history::{
LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, LoggedSystemHistoryEntry, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, LoggedSystemHistoryEntry,
LoggedWorkerSubject, LoggedWorkerSubject,
}; };
pub use input_metadata::{
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
InputEvidenceUse, InputMetadataError, InputObservationUse, SystemInputEnvelope,
};
pub use logged_item::{LoggedContentPart, LoggedItem, LoggedRole, from_logged, to_logged}; pub use logged_item::{LoggedContentPart, LoggedItem, LoggedRole, from_logged, to_logged};
pub use paste_artifact::PasteArtifactLimits; pub use paste_artifact::PasteArtifactLimits;
pub use public_index::{ pub use public_index::{
@@ -98,7 +103,8 @@ pub use segment::{
pub use segment_log::{LogEntry, RestoredState, SegmentOrigin, SessionExtension, collect_state}; pub use segment_log::{LogEntry, RestoredState, SegmentOrigin, SessionExtension, collect_state};
pub use store::{Store, StoreError}; pub use store::{Store, StoreError};
pub use system_item::{ pub use system_item::{
PromptRenderProvenance, SystemItem, SystemReminder, SystemReminderSource, render_worker_event, PromptRenderProvenance, PublicSystemItem, PublicSystemPresentation, SystemItem, SystemReminder,
SystemReminderSource, render_worker_event,
}; };
pub use uploaded_file::{ pub use uploaded_file::{
DEFAULT_MAX_FILES_PER_SUBMISSION, DEFAULT_MAX_SESSION_ARTIFACT_BYTES, DEFAULT_MAX_FILES_PER_SUBMISSION, DEFAULT_MAX_SESSION_ARTIFACT_BYTES,
+2
View File
@@ -1296,6 +1296,7 @@ mod tests {
LoggedHistoryEntry { LoggedHistoryEntry {
item, item,
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.to_owned()), entry_id: LoggedSessionHistoryEntryId(id.to_owned()),
origin, origin,
derivation: None, derivation: None,
@@ -1537,6 +1538,7 @@ mod tests {
prompt_provenance: None, prompt_provenance: None,
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("system-item".into()), entry_id: LoggedSessionHistoryEntryId("system-item".into()),
origin: LoggedSessionHistoryOrigin::BackendInstruction { origin: LoggedSessionHistoryOrigin::BackendInstruction {
operation_id: None, operation_id: None,
+110 -28
View File
@@ -148,15 +148,24 @@ struct SegmentHeader {
file_len: u64, file_len: u64,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublicMaterialUse {
Recording,
MemoryEvidence,
}
/// A fixed material catalog. Opening a new catalog observes newly appended /// A fixed material catalog. Opening a new catalog observes newly appended
/// records; reading a body through this catalog verifies its saved digest and /// records; reading a body through this catalog verifies its saved digest and
/// byte length again. Missing/corrected/deleted sources are explicit failures. /// byte length again. Missing/corrected/deleted sources are explicit failures.
/// Evidence eligibility never publishes private model text or Feature payload:
/// System inputs also require an explicit public projection for every public use.
pub struct SessionPublicMaterialReader { pub struct SessionPublicMaterialReader {
session_id: SessionId, session_id: SessionId,
active_segment_id: SegmentId, active_segment_id: SegmentId,
layout: Layout, layout: Layout,
limits: PublicMaterialLimits, limits: PublicMaterialLimits,
materials: Vec<Material>, materials: Vec<Material>,
purpose: PublicMaterialUse,
retention: Option<crate::retention::SessionRetentionStore>, retention: Option<crate::retention::SessionRetentionStore>,
} }
@@ -180,7 +189,13 @@ impl SessionPublicMaterialReader {
active_segment_id: SegmentId, active_segment_id: SegmentId,
limits: PublicMaterialLimits, limits: PublicMaterialLimits,
) -> Result<Self, PublicMaterialError> { ) -> Result<Self, PublicMaterialError> {
Self::open_worker_guarded(session_root, active_segment_id, limits, None) Self::open_worker_guarded(
session_root,
active_segment_id,
limits,
None,
PublicMaterialUse::Recording,
)
} }
/// Uses the Host's aggregate-external control root for the whole scan and every body read. /// Uses the Host's aggregate-external control root for the whole scan and every body read.
@@ -191,7 +206,30 @@ impl SessionPublicMaterialReader {
limits: PublicMaterialLimits, limits: PublicMaterialLimits,
retention: crate::retention::SessionRetentionStore, retention: crate::retention::SessionRetentionStore,
) -> Result<Self, PublicMaterialError> { ) -> Result<Self, PublicMaterialError> {
Self::open_worker_guarded(session_root, active_segment_id, limits, Some(retention)) Self::open_worker_for_use(
session_root,
active_segment_id,
limits,
retention,
PublicMaterialUse::Recording,
)
}
/// Selects the evidence purpose independently of public display and observation.
pub fn open_worker_for_use(
session_root: &Path,
active_segment_id: SegmentId,
limits: PublicMaterialLimits,
retention: crate::retention::SessionRetentionStore,
purpose: PublicMaterialUse,
) -> Result<Self, PublicMaterialError> {
Self::open_worker_guarded(
session_root,
active_segment_id,
limits,
Some(retention),
purpose,
)
} }
fn open_worker_guarded( fn open_worker_guarded(
@@ -199,6 +237,7 @@ impl SessionPublicMaterialReader {
active_segment_id: SegmentId, active_segment_id: SegmentId,
limits: PublicMaterialLimits, limits: PublicMaterialLimits,
retention: Option<crate::retention::SessionRetentionStore>, retention: Option<crate::retention::SessionRetentionStore>,
purpose: PublicMaterialUse,
) -> Result<Self, PublicMaterialError> { ) -> Result<Self, PublicMaterialError> {
let store = WorkerSessionStore::open_read_only(session_root) let store = WorkerSessionStore::open_read_only(session_root)
.map_err(|_| PublicMaterialError::Unavailable)?; .map_err(|_| PublicMaterialError::Unavailable)?;
@@ -212,6 +251,7 @@ impl SessionPublicMaterialReader {
Layout::Worker(session_root.into()), Layout::Worker(session_root.into()),
limits, limits,
retention, retention,
purpose,
) )
} }
@@ -237,6 +277,25 @@ impl SessionPublicMaterialReader {
active_segment_id: SegmentId, active_segment_id: SegmentId,
limits: PublicMaterialLimits, limits: PublicMaterialLimits,
retention: crate::retention::SessionRetentionStore, retention: crate::retention::SessionRetentionStore,
) -> Result<Self, PublicMaterialError> {
Self::open_fs_for_use(
sessions_root,
session_id,
active_segment_id,
limits,
retention,
PublicMaterialUse::Recording,
)
}
/// Uses retained originals with the independently selected evidence policy.
pub fn open_fs_for_use(
sessions_root: &Path,
session_id: SessionId,
active_segment_id: SegmentId,
limits: PublicMaterialLimits,
retention: crate::retention::SessionRetentionStore,
purpose: PublicMaterialUse,
) -> Result<Self, PublicMaterialError> { ) -> Result<Self, PublicMaterialError> {
Self::open( Self::open(
session_id, session_id,
@@ -244,6 +303,7 @@ impl SessionPublicMaterialReader {
Layout::Fs(sessions_root.join(session_id.to_string())), Layout::Fs(sessions_root.join(session_id.to_string())),
limits, limits,
Some(retention), Some(retention),
purpose,
) )
} }
@@ -263,6 +323,7 @@ impl SessionPublicMaterialReader {
layout: Layout, layout: Layout,
limits: PublicMaterialLimits, limits: PublicMaterialLimits,
retention: Option<crate::retention::SessionRetentionStore>, retention: Option<crate::retention::SessionRetentionStore>,
purpose: PublicMaterialUse,
) -> Result<Self, PublicMaterialError> { ) -> Result<Self, PublicMaterialError> {
let _guard = retention let _guard = retention
.as_ref() .as_ref()
@@ -310,11 +371,10 @@ impl SessionPublicMaterialReader {
let source_ids: Option<Vec<String>> = if compact { let source_ids: Option<Vec<String>> = if compact {
let summaries: Vec<_> = history let summaries: Vec<_> = history
.iter() .iter()
.filter(|e| { .filter(|entry| {
matches!( entry.metadata.input_metadata.as_ref().is_some_and(|input| {
e.metadata.origin, input.usage().compaction == crate::InputCompactionUse::OpeningSummary
LoggedSessionHistoryOrigin::CompactionSummary })
)
}) })
.collect(); .collect();
if summaries.len() != 1 { if summaries.len() != 1 {
@@ -435,23 +495,19 @@ impl SessionPublicMaterialReader {
} => turn = *total_turn_count, } => turn = *total_turn_count,
_ => {} _ => {}
} }
let system_history;
let entries: &[LoggedHistoryEntry] = match &record { let entries: &[LoggedHistoryEntry] = match &record {
LogEntry::AnnotatedSegmentStart { history, .. } LogEntry::AnnotatedSegmentStart { history, .. }
| LogEntry::AnnotatedUserInput { history, .. } => history, | LogEntry::AnnotatedUserInput { history, .. } => history,
LogEntry::AnnotatedAssistantItem { entry, .. } LogEntry::AnnotatedAssistantItem { entry, .. }
| LogEntry::AnnotatedToolResult { entry, .. } | LogEntry::AnnotatedToolResult { entry, .. }
| LogEntry::ToolResultCorrected { entry, .. } => std::slice::from_ref(entry), | LogEntry::ToolResultCorrected { entry, .. } => std::slice::from_ref(entry),
LogEntry::AnnotatedSystemItem { entry, .. } => {
system_history = entry.to_history_entry();
std::slice::from_ref(&system_history)
}
_ => &[], _ => &[],
}; };
if let LogEntry::AnnotatedSystemItem { entry, .. } = &record {
charge(&mut examined, limits.max_entries)?;
if entry.metadata.entry_id.0.is_empty()
|| !history_seen.insert(entry.metadata.entry_id.0.clone())
{
return Err(PublicMaterialError::Inconsistent);
}
history_ids.push(entry.metadata.entry_id.0.clone());
}
for (history_index, entry) in entries.iter().enumerate() { for (history_index, entry) in entries.iter().enumerate() {
charge(&mut examined, limits.max_entries)?; charge(&mut examined, limits.max_entries)?;
let id = &entry.metadata.entry_id.0; let id = &entry.metadata.entry_id.0;
@@ -468,7 +524,7 @@ impl SessionPublicMaterialReader {
if in_run && ai_materialized(&entry.item) { if in_run && ai_materialized(&entry.item) {
safe = true; safe = true;
} }
let Some((sha256, total_bytes)) = canonical_metadata(entry)? else { let Some((sha256, total_bytes)) = canonical_metadata(entry, purpose)? else {
continue; continue;
}; };
let location = Location { let location = Location {
@@ -621,6 +677,7 @@ impl SessionPublicMaterialReader {
layout, layout,
limits, limits,
materials, materials,
purpose,
retention: retention.clone(), retention: retention.clone(),
}) })
} }
@@ -660,6 +717,7 @@ impl SessionPublicMaterialReader {
let Some((_, record)) = reader.next()? else { let Some((_, record)) = reader.next()? else {
return Err(PublicMaterialError::Missing); return Err(PublicMaterialError::Missing);
}; };
let system_history;
let (body, origin) = if let Some(boundary) = run_boundary(&record) { let (body, origin) = if let Some(boundary) = run_boundary(&record) {
if boundary_reference( if boundary_reference(
&record, &record,
@@ -683,12 +741,16 @@ impl SessionPublicMaterialReader {
LogEntry::AnnotatedAssistantItem { entry, .. } LogEntry::AnnotatedAssistantItem { entry, .. }
| LogEntry::AnnotatedToolResult { entry, .. } | LogEntry::AnnotatedToolResult { entry, .. }
| LogEntry::ToolResultCorrected { entry, .. } => Some(entry), | LogEntry::ToolResultCorrected { entry, .. } => Some(entry),
LogEntry::AnnotatedSystemItem { entry, .. } => {
system_history = entry.to_history_entry();
Some(&system_history)
}
_ => None, _ => None,
} }
.filter(|e| format!("E{}", e.metadata.entry_id.0) == descriptor.identity.entry_ref) .filter(|e| format!("E{}", e.metadata.entry_id.0) == descriptor.identity.entry_ref)
.ok_or(PublicMaterialError::Changed)?; .ok_or(PublicMaterialError::Changed)?;
( (
canonical_body(entry).ok_or(PublicMaterialError::Changed)?, canonical_body(entry, self.purpose).ok_or(PublicMaterialError::Changed)?,
PublicMaterialOrigin::History { PublicMaterialOrigin::History {
origin: entry.metadata.origin.clone(), origin: entry.metadata.origin.clone(),
}, },
@@ -765,7 +827,7 @@ enum CanonicalBody<'a> {
Message { Message {
origin: &'a LoggedSessionHistoryOrigin, origin: &'a LoggedSessionHistoryOrigin,
role: LoggedRole, role: LoggedRole,
content: &'a [crate::LoggedContentPart], content: std::borrow::Cow<'a, [crate::LoggedContentPart]>,
}, },
ToolCall { ToolCall {
origin: &'a LoggedSessionHistoryOrigin, origin: &'a LoggedSessionHistoryOrigin,
@@ -828,15 +890,34 @@ fn boundary_reference(
.unwrap_or_else(|| format!("El-run-{session}-{segment}-{offset}")) .unwrap_or_else(|| format!("El-run-{session}-{segment}-{offset}"))
} }
fn canonical_body(entry: &LoggedHistoryEntry) -> Option<CanonicalBody<'_>> { fn canonical_body(
if matches!( entry: &LoggedHistoryEntry,
entry.metadata.origin, purpose: PublicMaterialUse,
LoggedSessionHistoryOrigin::DerivedSummary ) -> Option<CanonicalBody<'_>> {
| LoggedSessionHistoryOrigin::CompactionSummary if let Some(input) = &entry.metadata.input_metadata {
| LoggedSessionHistoryOrigin::JournalContext let usage = input.usage();
) { let allowed = match purpose {
PublicMaterialUse::Recording => usage.recording_material,
PublicMaterialUse::MemoryEvidence => usage.memory_evidence,
};
if allowed == crate::InputEvidenceUse::Exclude {
return None; return None;
} }
if let LoggedItem::Message {
role: LoggedRole::System,
..
} = &entry.item
{
let public = input.public_item()?;
return Some(CanonicalBody::Message {
origin: &entry.metadata.origin,
role: LoggedRole::System,
content: std::borrow::Cow::Owned(vec![crate::LoggedContentPart::Text {
text: public.body.clone(),
}]),
});
}
}
match &entry.item { match &entry.item {
LoggedItem::Message { LoggedItem::Message {
role: LoggedRole::System, role: LoggedRole::System,
@@ -847,7 +928,7 @@ fn canonical_body(entry: &LoggedHistoryEntry) -> Option<CanonicalBody<'_>> {
LoggedItem::Message { role, content } => Some(CanonicalBody::Message { LoggedItem::Message { role, content } => Some(CanonicalBody::Message {
origin: &entry.metadata.origin, origin: &entry.metadata.origin,
role: *role, role: *role,
content, content: std::borrow::Cow::Borrowed(content),
}), }),
LoggedItem::ToolCall { LoggedItem::ToolCall {
call_id, call_id,
@@ -880,8 +961,9 @@ fn canonical_body(entry: &LoggedHistoryEntry) -> Option<CanonicalBody<'_>> {
fn canonical_metadata( fn canonical_metadata(
entry: &LoggedHistoryEntry, entry: &LoggedHistoryEntry,
purpose: PublicMaterialUse,
) -> Result<Option<(String, u64)>, PublicMaterialError> { ) -> Result<Option<(String, u64)>, PublicMaterialError> {
let Some(body) = canonical_body(entry) else { let Some(body) = canonical_body(entry, purpose) else {
return Ok(None); return Ok(None);
}; };
let mut sink = BodySink::new(0, 0); let mut sink = BodySink::new(0, 0);
+263 -19
View File
@@ -9,6 +9,7 @@ fn entry(id: &str, item: LoggedItem) -> LoggedHistoryEntry {
LoggedHistoryEntry { LoggedHistoryEntry {
item, item,
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.into()), entry_id: LoggedSessionHistoryEntryId(id.into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput, origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None, derivation: None,
@@ -24,6 +25,27 @@ fn message(id: &str, role: LoggedRole, text: &str) -> LoggedHistoryEntry {
}, },
) )
} }
fn system_context(
id: &str,
text: &str,
compaction: crate::InputCompactionUse,
) -> LoggedHistoryEntry {
let mut entry = message(id, LoggedRole::System, text);
entry.metadata.input_metadata = Some(
crate::HistoryInputMetadata::new(
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
compaction,
},
None,
None,
)
.unwrap(),
);
entry
}
fn seed(session: SessionId, history: Vec<LoggedHistoryEntry>) -> LogEntry { fn seed(session: SessionId, history: Vec<LoggedHistoryEntry>) -> LogEntry {
LogEntry::AnnotatedSegmentStart { LogEntry::AnnotatedSegmentStart {
ts: 1, ts: 1,
@@ -347,7 +369,11 @@ fn yield_resume_cancel_keep_original_boundary_identity_across_compaction() {
.descriptors() .descriptors()
.cloned() .cloned()
.collect(); .collect();
let mut summary = message("summary", LoggedRole::User, "must not replace originals"); let mut summary = system_context(
"summary",
"must not replace originals",
crate::InputCompactionUse::OpeningSummary,
);
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
summary.metadata.derivation = Some(crate::LoggedHistoryDerivation { summary.metadata.derivation = Some(crate::LoggedHistoryDerivation {
sources: vec![ sources: vec![
@@ -543,14 +569,18 @@ fn reasoning_proves_rollback_safety_but_private_and_derived_items_are_not_materi
let store = FsStore::new(temp.path()).unwrap(); let store = FsStore::new(temp.path()).unwrap();
let session = crate::new_session_id(); let session = crate::new_session_id();
let segment = crate::new_segment_id(); let segment = crate::new_segment_id();
let mut derived = message("derived", LoggedRole::User, "summary"); let mut derived = system_context("derived", "summary", crate::InputCompactionUse::Retain);
derived.metadata.origin = LoggedSessionHistoryOrigin::DerivedSummary; derived.metadata.origin = LoggedSessionHistoryOrigin::DerivedSummary;
let mut journal = message("journal", LoggedRole::User, "frozen Journal reference"); let mut journal = system_context(
"journal",
"frozen Journal reference",
crate::InputCompactionUse::ReferenceInput,
);
journal.metadata.origin = LoggedSessionHistoryOrigin::JournalContext; journal.metadata.origin = LoggedSessionHistoryOrigin::JournalContext;
let mut summary = message( let mut summary = system_context(
"compaction-summary", "compaction-summary",
LoggedRole::User,
"complementary summary", "complementary summary",
crate::InputCompactionUse::OpeningSummary,
); );
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
let reasoning = entry( let reasoning = entry(
@@ -579,18 +609,23 @@ fn reasoning_proves_rollback_safety_but_private_and_derived_items_are_not_materi
LogEntry::AnnotatedSystemItem { LogEntry::AnnotatedSystemItem {
ts: 1, ts: 1,
entry: crate::LoggedSystemHistoryEntry { entry: crate::LoggedSystemHistoryEntry {
item: crate::SystemItem::JournalStartup { item: crate::SystemItem::Input {
body: "previous Journal must not become new recording material".into(), body: "previous Journal must not become new recording material".into(),
snapshot: serde_json::json!({"journals": []}),
snapshot_sha256: "retained-digest".into(),
prompt_provenance: crate::PromptRenderProvenance {
workspace_id: None,
source_digest: "source".into(),
projection_digest: "projection".into(),
logical_name: "internal.subjektiv_journal_startup".into(),
},
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: Some(
crate::HistoryInputMetadata::new(
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
compaction: crate::InputCompactionUse::ReferenceInput,
},
None,
None,
)
.unwrap(),
),
entry_id: LoggedSessionHistoryEntryId("startup-journal".into()), entry_id: LoggedSessionHistoryEntryId("startup-journal".into()),
origin: LoggedSessionHistoryOrigin::DerivedSummary, origin: LoggedSessionHistoryOrigin::DerivedSummary,
derivation: None, derivation: None,
@@ -777,6 +812,19 @@ fn schema_v3_compaction_migrates_persistently_and_strict_reader_keeps_original_i
panic!() panic!()
}; };
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
summary.metadata.input_metadata = Some(
crate::HistoryInputMetadata::new(
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
compaction: crate::InputCompactionUse::OpeningSummary,
},
None,
None,
)
.unwrap(),
);
assert_eq!(history, &[summary]); assert_eq!(history, &[summary]);
// Run-transition ids synthesized from byte offsets must be frozen before // Run-transition ids synthesized from byte offsets must be frozen before
// the changed summary record shifts every subsequent record offset. // the changed summary record shifts every subsequent record offset.
@@ -825,8 +873,14 @@ fn midturn_compaction_uses_exact_source_ids_and_preserves_discarded_originals()
], ],
) )
.unwrap(); .unwrap();
let mut summary = message("summary", LoggedRole::User, "never use as original"); let mut summary = system_context(
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; "summary",
"never use as original",
crate::InputCompactionUse::OpeningSummary,
);
summary.metadata.origin = LoggedSessionHistoryOrigin::Provided {
producer: "renamed-summary-owner".into(),
};
summary.metadata.derivation = Some(crate::LoggedHistoryDerivation { summary.metadata.derivation = Some(crate::LoggedHistoryDerivation {
sources: vec![ sources: vec![
LoggedSessionHistoryEntryId("u".into()), LoggedSessionHistoryEntryId("u".into()),
@@ -1030,7 +1084,11 @@ fn zero_turn_fork_from_seed_and_consecutive_compactions_preserve_original_identi
for index in 0..3 { for index in 0..3 {
let compact = crate::new_segment_id(); let compact = crate::new_segment_id();
let summary_id = format!("summary-{index}"); let summary_id = format!("summary-{index}");
let mut summary = message(&summary_id, LoggedRole::User, "not original"); let mut summary = system_context(
&summary_id,
"not original",
crate::InputCompactionUse::OpeningSummary,
);
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
summary.metadata.derivation = Some(crate::LoggedHistoryDerivation { summary.metadata.derivation = Some(crate::LoggedHistoryDerivation {
sources: source_ids sources: source_ids
@@ -1178,7 +1236,11 @@ fn correction_preserves_original_identity_across_compaction_without_summary_fall
], ],
) )
.unwrap(); .unwrap();
let mut summary = message("summary", LoggedRole::User, "not original"); let mut summary = system_context(
"summary",
"not original",
crate::InputCompactionUse::OpeningSummary,
);
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
summary.metadata.derivation = Some(crate::LoggedHistoryDerivation { summary.metadata.derivation = Some(crate::LoggedHistoryDerivation {
sources: vec![LoggedSessionHistoryEntryId("result".into())], sources: vec![LoggedSessionHistoryEntryId("result".into())],
@@ -1226,7 +1288,11 @@ fn compaction_without_exact_derivation_or_with_invalid_turn_is_rejected() {
), ),
] { ] {
let segment = crate::new_segment_id(); let segment = crate::new_segment_id();
let mut summary = message("summary", LoggedRole::User, "not original"); let mut summary = system_context(
"summary",
"not original",
crate::InputCompactionUse::OpeningSummary,
);
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
summary.metadata.derivation = derivation; summary.metadata.derivation = derivation;
let mut start = seed(session, vec![summary]); let mut start = seed(session, vec![summary]);
@@ -1243,3 +1309,181 @@ fn compaction_without_exact_derivation_or_with_invalid_turn_is_rejected() {
)); ));
} }
} }
#[test]
fn generic_system_material_uses_explicit_evidence_purpose_not_producer_or_body() {
use crate::{
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
InputEvidenceUse, InputObservationUse, LoggedSystemHistoryEntry, SystemInputEnvelope,
};
for worker_layout in [false, true] {
let temp = tempfile::tempdir().unwrap();
let session = crate::new_session_id();
let segment = crate::new_segment_id();
let fork = crate::new_segment_id();
let make =
|producer: &str, recording_material, memory_evidence, body: &str, public: bool| {
LoggedSystemHistoryEntry::from_input(
LoggedSessionHistoryOrigin::Provided {
producer: producer.into(),
},
None,
SystemInputEnvelope::new(
format!("private model text: {body}"),
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material,
memory_evidence,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::Retain,
},
Some(
FeatureInputPayload::encode(
"unknown:future",
99,
&serde_json::json!({"private": "not-public"}),
)
.unwrap(),
),
public.then(|| crate::PublicSystemItem {
kind: "context".into(),
body: body.into(),
presentation: crate::PublicSystemPresentation::Context,
}),
)
.unwrap(),
)
.unwrap(),
)
};
let inputs = [
make(
"same-owner",
InputEvidenceUse::Include,
InputEvidenceUse::Exclude,
"do not extract this; original material",
true,
),
make(
"same-owner",
InputEvidenceUse::Exclude,
InputEvidenceUse::Include,
"memory-only original",
true,
),
make(
"renamed-owner",
InputEvidenceUse::Exclude,
InputEvidenceUse::Exclude,
"reference without warning text",
true,
),
make(
"private-only",
InputEvidenceUse::Include,
InputEvidenceUse::Include,
"must not publish without projection",
false,
),
];
let root = if worker_layout {
temp.path().join("worker")
} else {
temp.path().join(session.to_string())
};
let store: Box<dyn Store> = if worker_layout {
Box::new(WorkerSessionStore::new(&root).unwrap())
} else {
Box::new(FsStore::new(temp.path()).unwrap())
};
store
.create_segment(session, segment, &[seed(session, vec![])])
.unwrap();
for entry in &inputs {
store
.append(
session,
segment,
&LogEntry::AnnotatedSystemItem {
ts: 2,
entry: entry.clone(),
extensions: vec![],
},
)
.unwrap();
}
store
.append(
session,
segment,
&LogEntry::TurnEnd {
ts: 3,
turn_count: 1,
},
)
.unwrap();
let mut fork_seed = seed(
session,
inputs
.iter()
.map(LoggedSystemHistoryEntry::to_history_entry)
.collect(),
);
if let LogEntry::AnnotatedSegmentStart { forked_from, .. } = &mut fork_seed {
*forked_from = Some(crate::SegmentOrigin {
segment_id: segment,
at_turn_index: 1,
});
}
store.create_segment(session, fork, &[fork_seed]).unwrap();
let open = |active, purpose| {
if worker_layout {
SessionPublicMaterialReader::open_worker_for_use(
&root,
active,
Default::default(),
crate::retention::fs_retention(temp.path()),
purpose,
)
.unwrap()
} else {
SessionPublicMaterialReader::open_fs_for_use(
temp.path(),
session,
active,
Default::default(),
crate::retention::fs_retention(temp.path()),
purpose,
)
.unwrap()
}
};
for (purpose, selected, expected_text) in [
(
PublicMaterialUse::Recording,
0,
"do not extract this; original material",
),
(PublicMaterialUse::MemoryEvidence, 1, "memory-only original"),
] {
let original = open(segment, purpose);
let inherited = open(fork, purpose);
assert_eq!(
ids(&original),
vec![format!("E{}", inputs[selected].metadata.entry_id.0)]
);
assert_eq!(ids(&inherited), ids(&original));
let descriptor = inherited.descriptors().next().unwrap();
let body = inherited
.read_content(descriptor, 0, 16 * 1024)
.unwrap()
.content;
let value: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(value["role"], "system");
assert_eq!(value["content"][0]["text"], expected_text);
assert!(!body.contains("not-public"));
assert!(!body.contains("private model text"));
assert!(!body.contains("unknown:future"));
}
}
}
+73 -21
View File
@@ -27,7 +27,7 @@ use thiserror::Error;
use crate::{ use crate::{
LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole, LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole,
LoggedSessionHistoryOrigin, SegmentId, SegmentOrigin, SessionId, StoreError, SystemItem, LoggedSessionHistoryOrigin, SegmentId, SegmentOrigin, SessionId, StoreError,
WorkerAggregateStore, WorkerSessionStore, WorkerStoreError, WorkerAggregateStore, WorkerSessionStore, WorkerStoreError,
}; };
@@ -1165,7 +1165,7 @@ fn capture_tool_correction(
let LogEntry::ToolResultCorrected { entry, .. } = record else { let LogEntry::ToolResultCorrected { entry, .. } = record else {
return false; return false;
}; };
if let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) { if let Some(data) = project_history_item(entry) {
// We read backwards: the first correction seen is the latest one. // We read backwards: the first correction seen is the latest one.
corrections corrections
.entry(entry.metadata.entry_id.0.clone()) .entry(entry.metadata.entry_id.0.clone())
@@ -1600,12 +1600,12 @@ pub(crate) fn project_session_snapshot_for_segment(
} => extend_history(&mut entries, history, Some(segments), *ts), } => extend_history(&mut entries, history, Some(segments), *ts),
LogEntry::AnnotatedAssistantItem { ts, entry } LogEntry::AnnotatedAssistantItem { ts, entry }
| LogEntry::AnnotatedToolResult { ts, entry } => { | LogEntry::AnnotatedToolResult { ts, entry } => {
if let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) { if let Some(data) = project_history_item(entry) {
entries.push(history_entry(entry, *ts, data)); entries.push(history_entry(entry, *ts, data));
} }
} }
LogEntry::ToolResultCorrected { ts, entry } => { LogEntry::ToolResultCorrected { ts, entry } => {
if let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) { if let Some(data) = project_history_item(entry) {
if let Some(existing) = entries if let Some(existing) = entries
.iter_mut() .iter_mut()
.find(|existing| existing.entry_id == entry.metadata.entry_id.0) .find(|existing| existing.entry_id == entry.metadata.entry_id.0)
@@ -1618,11 +1618,16 @@ pub(crate) fn project_session_snapshot_for_segment(
} }
} }
LogEntry::AnnotatedSystemItem { ts, entry, .. } => entries.push(system_entry( LogEntry::AnnotatedSystemItem { ts, entry, .. } => entries.push(system_entry(
&entry.item, &entry.public_item(),
entry.metadata.entry_id.0.clone(), entry.metadata.entry_id.0.clone(),
*ts, *ts,
provenance(&entry.metadata.origin), provenance(&entry.metadata.origin),
derivation_ids(entry), derivation_ids(entry),
entry
.metadata
.input_metadata
.as_ref()
.map(crate::HistoryInputMetadata::public_system_use_policy),
)), )),
LogEntry::RunYielded { LogEntry::RunYielded {
ts, ts,
@@ -1775,7 +1780,7 @@ fn extend_history(
segments: input_segments.cloned().unwrap_or_default(), segments: input_segments.cloned().unwrap_or_default(),
} }
} else { } else {
let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) else { let Some(data) = project_history_item(entry) else {
continue; continue;
}; };
data data
@@ -1790,6 +1795,19 @@ fn history_entry(
data: SessionSnapshotEntryData, data: SessionSnapshotEntryData,
) -> SessionSnapshotEntry { ) -> SessionSnapshotEntry {
SessionSnapshotEntry { SessionSnapshotEntry {
use_policy: entry.metadata.input_metadata.as_ref().map(|input| {
if matches!(
entry.item,
LoggedItem::Message {
role: LoggedRole::System,
..
}
) {
input.public_system_use_policy()
} else {
input.public_use_policy()
}
}),
entry_id: entry.metadata.entry_id.0.clone(), entry_id: entry.metadata.entry_id.0.clone(),
timestamp, timestamp,
provenance: provenance(&entry.metadata.origin), provenance: provenance(&entry.metadata.origin),
@@ -1833,6 +1851,7 @@ fn run_transition_entry(
data: SessionSnapshotEntryData, data: SessionSnapshotEntryData,
) -> SessionSnapshotEntry { ) -> SessionSnapshotEntry {
SessionSnapshotEntry { SessionSnapshotEntry {
use_policy: None,
entry_id: persisted_entry_id entry_id: persisted_entry_id
.map(|entry_id| entry_id.0.clone()) .map(|entry_id| entry_id.0.clone())
.unwrap_or_else(|| legacy_entry_id(session_key, segment_id, log_index, 0)), .unwrap_or_else(|| legacy_entry_id(session_key, segment_id, log_index, 0)),
@@ -1852,6 +1871,7 @@ fn legacy_entry(
data: SessionSnapshotEntryData, data: SessionSnapshotEntryData,
) -> SessionSnapshotEntry { ) -> SessionSnapshotEntry {
SessionSnapshotEntry { SessionSnapshotEntry {
use_policy: None,
entry_id: legacy_entry_id(session_key, segment_id, log_index, item_index), entry_id: legacy_entry_id(session_key, segment_id, log_index, item_index),
timestamp, timestamp,
provenance: SessionEntryProvenance::LegacyUnknown, provenance: SessionEntryProvenance::LegacyUnknown,
@@ -1884,9 +1904,8 @@ fn provenance(origin: &LoggedSessionHistoryOrigin) -> SessionEntryProvenance {
LoggedSessionHistoryOrigin::FlowInstruction { .. } => { LoggedSessionHistoryOrigin::FlowInstruction { .. } => {
SessionEntryProvenance::FlowInstruction SessionEntryProvenance::FlowInstruction
} }
LoggedSessionHistoryOrigin::BackendInstruction { .. } => { LoggedSessionHistoryOrigin::BackendInstruction { .. }
SessionEntryProvenance::BackendInstruction | LoggedSessionHistoryOrigin::Provided { .. } => SessionEntryProvenance::BackendInstruction,
}
LoggedSessionHistoryOrigin::ModelOutput { .. } => SessionEntryProvenance::ModelOutput, LoggedSessionHistoryOrigin::ModelOutput { .. } => SessionEntryProvenance::ModelOutput,
LoggedSessionHistoryOrigin::ToolOutput { .. } => SessionEntryProvenance::ToolOutput, LoggedSessionHistoryOrigin::ToolOutput { .. } => SessionEntryProvenance::ToolOutput,
LoggedSessionHistoryOrigin::CompactionSummary LoggedSessionHistoryOrigin::CompactionSummary
@@ -1914,6 +1933,35 @@ pub fn session_tool_attachment_id(
URL_SAFE_NO_PAD.encode(hash.finalize()) URL_SAFE_NO_PAD.encode(hash.finalize())
} }
fn project_history_item(entry: &LoggedHistoryEntry) -> Option<SessionSnapshotEntryData> {
if let Some(input) = &entry.metadata.input_metadata {
if matches!(
entry.item,
LoggedItem::Message {
role: LoggedRole::System,
..
}
) {
let public = input.public_projection();
return Some(SessionSnapshotEntryData::SystemItem {
item_kind: public.kind.clone(),
content: public.body.clone(),
data: Some(
serde_json::to_value(public).expect("public system input is serializable"),
),
});
}
let usage = input.usage();
if usage.recording_material == crate::InputEvidenceUse::Exclude
&& usage.memory_evidence == crate::InputEvidenceUse::Exclude
&& usage.observation == crate::InputObservationUse::Hidden
{
return None;
}
}
project_item(&entry.metadata.entry_id.0, &entry.item)
}
fn project_item(entry_id: &str, item: &LoggedItem) -> Option<SessionSnapshotEntryData> { fn project_item(entry_id: &str, item: &LoggedItem) -> Option<SessionSnapshotEntryData> {
match item { match item {
LoggedItem::Message { role, content } => { LoggedItem::Message { role, content } => {
@@ -1986,30 +2034,26 @@ fn project_item(entry_id: &str, item: &LoggedItem) -> Option<SessionSnapshotEntr
} }
fn system_entry( fn system_entry(
item: &SystemItem, item: &crate::PublicSystemItem,
entry_id: String, entry_id: String,
timestamp: u64, timestamp: u64,
provenance: SessionEntryProvenance, provenance: SessionEntryProvenance,
derived_from: Vec<String>, derived_from: Vec<String>,
use_policy: Option<protocol::SessionEntryUsePolicy>,
) -> SessionSnapshotEntry { ) -> SessionSnapshotEntry {
let mut data = serde_json::to_value(item).ok(); let public = item.clone();
if let Some(serde_json::Value::Object(object)) = data.as_mut() { let data = Some(serde_json::to_value(&public).expect("public system input is serializable"));
object.remove("prompt_provenance"); let item_kind = public.kind;
} let content = public.body;
let item_kind = data
.as_ref()
.and_then(|value| value.get("kind"))
.and_then(serde_json::Value::as_str)
.unwrap_or("system_item")
.to_owned();
SessionSnapshotEntry { SessionSnapshotEntry {
use_policy,
entry_id, entry_id,
timestamp, timestamp,
provenance, provenance,
derived_from, derived_from,
data: SessionSnapshotEntryData::SystemItem { data: SessionSnapshotEntryData::SystemItem {
item_kind, item_kind,
content: item.history_text(), content,
data, data,
}, },
} }
@@ -2291,6 +2335,7 @@ mod tests {
content: vec![LoggedContentPart::Text { text: text.into() }], content: vec![LoggedContentPart::Text { text: text.into() }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(entry_id.into()), entry_id: LoggedSessionHistoryEntryId(entry_id.into()),
origin, origin,
derivation: None, derivation: None,
@@ -3589,6 +3634,7 @@ mod tests {
.map(|item| LoggedHistoryEntry { .map(|item| LoggedHistoryEntry {
item, item,
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -3643,6 +3689,7 @@ mod tests {
}], }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: user_entry_id, entry_id: user_entry_id,
origin: LoggedSessionHistoryOrigin::HumanInput { origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".into(), account_id: "account-1".into(),
@@ -3712,6 +3759,7 @@ mod tests {
}], }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::FlowInstruction { origin: LoggedSessionHistoryOrigin::FlowInstruction {
selector: "builtin:coder-review".into(), selector: "builtin:coder-review".into(),
@@ -3731,6 +3779,7 @@ mod tests {
}], }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: user_entry_id.clone(), entry_id: user_entry_id.clone(),
origin: origin.clone(), origin: origin.clone(),
derivation: Some(LoggedHistoryDerivation { derivation: Some(LoggedHistoryDerivation {
@@ -3761,6 +3810,7 @@ mod tests {
fn annotated_projection_preserves_identity_and_provenance() { fn annotated_projection_preserves_identity_and_provenance() {
let session_id = crate::new_session_id(); let session_id = crate::new_session_id();
let metadata = LoggedSessionHistoryMetadata { let metadata = LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::ModelOutput { origin: LoggedSessionHistoryOrigin::ModelOutput {
worker: LoggedWorkerSubject { worker: LoggedWorkerSubject {
@@ -4323,6 +4373,7 @@ mod tests {
is_error: false, is_error: false,
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("ancestor-image".into()), entry_id: LoggedSessionHistoryEntryId("ancestor-image".into()),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -4417,6 +4468,7 @@ mod tests {
is_error: false, is_error: false,
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("tool-images".into()), entry_id: LoggedSessionHistoryEntryId("tool-images".into()),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -103,6 +103,7 @@ fn finish(
stats.insert( stats.insert(
anchor.clone(), anchor.clone(),
SessionSnapshotEntry { SessionSnapshotEntry {
use_policy: None,
entry_id: format!("run-stats:{anchor}"), entry_id: format!("run-stats:{anchor}"),
timestamp: ended_at, timestamp: ended_at,
provenance: SessionEntryProvenance::LegacyUnknown, provenance: SessionEntryProvenance::LegacyUnknown,
+18
View File
@@ -254,6 +254,23 @@ pub struct SegmentOrigin {
} }
impl LogEntry { impl LogEntry {
/// Validate generic System records before a Store publishes any bytes.
pub fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
match self {
Self::AnnotatedSegmentStart { history, .. }
| Self::AnnotatedUserInput { history, .. } => {
for entry in history {
entry.validate_input_metadata()?;
}
}
Self::AnnotatedAssistantItem { entry, .. }
| Self::AnnotatedToolResult { entry, .. }
| Self::ToolResultCorrected { entry, .. } => entry.validate_input_metadata()?,
Self::AnnotatedSystemItem { entry, .. } => entry.validate_input_metadata()?,
_ => {}
}
Ok(())
}
/// Allocate the durable public identity for a newly committed Run /// Allocate the durable public identity for a newly committed Run
/// transition. Calling this repeatedly is idempotent so the exact value /// transition. Calling this repeatedly is idempotent so the exact value
/// written to storage is also the value published to live subscribers. /// written to storage is also the value published to live subscribers.
@@ -537,6 +554,7 @@ mod tests {
LoggedHistoryEntry { LoggedHistoryEntry {
item: LoggedItem::from(item), item: LoggedItem::from(item),
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
+3
View File
@@ -27,6 +27,9 @@ pub enum StoreError {
#[error("serialization error: {0}")] #[error("serialization error: {0}")]
Serde(#[from] serde_json::Error), Serde(#[from] serde_json::Error),
#[error("invalid history input metadata: {0}")]
InvalidInputMetadata(#[from] crate::InputMetadataError),
#[error("segment not found: {0}")] #[error("segment not found: {0}")]
NotFound(SegmentId), NotFound(SegmentId),
+90 -101
View File
@@ -86,20 +86,16 @@ pub struct PromptRenderProvenance {
/// `notify_wrapper` template) re-shape the live rendering on a later /// `notify_wrapper` template) re-shape the live rendering on a later
/// resume. /// resume.
/// ///
/// New variants get added here as fresh injection kinds come online /// Feature-owned inputs use `Input` plus explicit annotated metadata; adding a
/// (e.g. `TaskReminder`). The `kind` JSON tag is the snake_case form of /// Feature must not extend this enum or ordinary replay. Only core events have
/// the variant name. /// dedicated variants. Pre-v5 Feature kinds exist exclusively in the migration
/// decoder, not in this current wire contract.
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")] #[serde(tag = "kind", rename_all = "snake_case")]
pub enum SystemItem { pub enum SystemItem {
/// One-time derived Journal reference context. The bounded Host snapshot is /// Generic System body; private metadata and public presentation are supplied
/// retained as JSON independently of its rendered body and prompt projection. /// independently by the producer in the same annotated history record.
JournalStartup { Input { body: String },
body: String,
snapshot: serde_json::Value,
snapshot_sha256: String,
prompt_provenance: PromptRenderProvenance,
},
/// Free-form notification sent in by an external caller via /// Free-form notification sent in by an external caller via
/// `Method::Notify`. `message` is the raw caller-supplied text; /// `Method::Notify`. `message` is the raw caller-supplied text;
/// `body` is the wrapped LLM-context form (Worker renders it via /// `body` is the wrapped LLM-context form (Worker renders it via
@@ -122,14 +118,6 @@ pub enum SystemItem {
prompt_provenance: Option<PromptRenderProvenance>, prompt_provenance: Option<PromptRenderProvenance>,
}, },
/// Result of an explicitly selected chat Feature invocation. The typed
/// result is retained for clients; `body` is the exact bounded context made
/// visible to the Worker model.
FeatureInvocationResult {
result: protocol::FeatureInvocationResult,
body: String,
},
/// `@<path>` file reference resolution. `body` is the rendered /// `@<path>` file reference resolution. `body` is the rendered
/// LLM-context text (`[File: <path>]\n…` for regular files, /// LLM-context text (`[File: <path>]\n…` for regular files,
/// `[Dir: <path>]\n…` for directory listings, possibly with a /// `[Dir: <path>]\n…` for directory listings, possibly with a
@@ -137,10 +125,6 @@ pub enum SystemItem {
/// byte-identical to what was sent. /// byte-identical to what was sent.
FileAttachment { path: String, body: String }, FileAttachment { path: String, body: String },
/// Explicit Agent Skill activation. `body` is the exact LLM-facing
/// Skill text committed before any subsequent LLM request can use it.
SkillActivation { name: String, body: String },
/// Historical persisted Knowledge reference resolution. Knowledge is no /// Historical persisted Knowledge reference resolution. Knowledge is no
/// longer active, so restored sessions ignore this item instead of replaying /// longer active, so restored sessions ignore this item instead of replaying
/// archived record text into model context. /// archived record text into model context.
@@ -164,29 +148,6 @@ pub enum SystemItem {
prompt_provenance: Option<PromptRenderProvenance>, prompt_provenance: Option<PromptRenderProvenance>,
}, },
/// Restore-boundary refresh of the current resident Memory projection.
/// The exact rendered body is retained so replay is byte-identical; a
/// refresh with no ready summary explicitly supersedes older resident
/// context without deleting or rewriting history.
ResidentSummaryRefresh {
body: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
prompt_provenance: Option<PromptRenderProvenance>,
},
/// A Host-fetched change to user-managed Subject behavior. Each changed
/// body is appended rather than rewriting prior prompt/session content.
SubjectBehaviorRefresh {
body: String,
/// Exact user-managed behavior acknowledged after durable commit. Old
/// logs omit this value; an empty value is not an acknowledgement of a
/// nonempty behavior and does not replace their stored rendered body.
#[serde(default)]
behavior_md: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
prompt_provenance: Option<PromptRenderProvenance>,
},
/// Synthetic note inserted after an interrupted turn before the next /// Synthetic note inserted after an interrupted turn before the next
/// user input. `body` is the exact LLM-context text explaining that the /// user input. `body` is the exact LLM-context text explaining that the
/// previous turn was cut short. /// previous turn was cut short.
@@ -197,24 +158,67 @@ pub enum SystemItem {
}, },
} }
/// Allowlisted presentation data shared by live events and retained snapshots.
/// Private Feature payloads and render provenance never cross this boundary.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct PublicSystemItem {
pub kind: String,
pub body: String,
pub presentation: PublicSystemPresentation,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum PublicSystemPresentation {
Notification { message: String },
WorkerEvent { event: WorkerEvent },
Context,
TaskReminder,
Ignored,
}
impl SystemItem { impl SystemItem {
pub fn public_item(&self) -> PublicSystemItem {
let presentation = match self {
Self::Notification { message, .. } => PublicSystemPresentation::Notification {
message: message.clone(),
},
Self::WorkerEvent { event, .. } => PublicSystemPresentation::WorkerEvent {
event: event.clone(),
},
Self::TaskReminder { .. } => PublicSystemPresentation::TaskReminder,
Self::Input { .. }
| Self::LegacyKnowledgeIgnored { .. }
| Self::LegacyIgnored { .. } => PublicSystemPresentation::Ignored,
Self::FileAttachment { .. } | Self::Interrupt { .. } => {
PublicSystemPresentation::Context
}
};
PublicSystemItem {
kind: self.kind_label().to_owned(),
body: if matches!(presentation, PublicSystemPresentation::Ignored) {
String::new()
} else {
self.history_text()
},
presentation,
}
}
/// Free-text body the LLM sees inside its `role:system` message /// Free-text body the LLM sees inside its `role:system` message
/// for this item. Returns the variant's stored `body` verbatim. /// for this item. Returns the variant's stored `body` verbatim.
pub fn history_text(&self) -> String { pub fn history_text(&self) -> String {
match self { match self {
SystemItem::JournalStartup { body, .. } => body.clone(), SystemItem::Input { body } => body.clone(),
SystemItem::Notification { body, .. } => body.clone(), SystemItem::Notification { body, .. } => body.clone(),
SystemItem::WorkerEvent { body, .. } => body.clone(), SystemItem::WorkerEvent { body, .. } => body.clone(),
SystemItem::FeatureInvocationResult { body, .. } => body.clone(),
SystemItem::FileAttachment { body, .. } => body.clone(), SystemItem::FileAttachment { body, .. } => body.clone(),
SystemItem::SkillActivation { body, .. } => body.clone(),
SystemItem::LegacyKnowledgeIgnored { .. } => String::new(), SystemItem::LegacyKnowledgeIgnored { .. } => String::new(),
SystemItem::LegacyIgnored { slug } => { SystemItem::LegacyIgnored { slug } => {
format!("Ignored legacy procedure item: /{slug}") format!("Ignored legacy procedure item: /{slug}")
} }
SystemItem::TaskReminder { body, .. } => body.clone(), SystemItem::TaskReminder { body, .. } => body.clone(),
SystemItem::ResidentSummaryRefresh { body, .. } => body.clone(),
SystemItem::SubjectBehaviorRefresh { body, .. } => body.clone(),
SystemItem::Interrupt { body, .. } => body.clone(), SystemItem::Interrupt { body, .. } => body.clone(),
} }
} }
@@ -229,17 +233,13 @@ impl SystemItem {
/// wire — keep flexible. /// wire — keep flexible.
pub fn kind_label(&self) -> &'static str { pub fn kind_label(&self) -> &'static str {
match self { match self {
SystemItem::JournalStartup { .. } => "journal_startup", SystemItem::Input { .. } => "input",
SystemItem::Notification { .. } => "notification", SystemItem::Notification { .. } => "notification",
SystemItem::WorkerEvent { .. } => "worker_event", SystemItem::WorkerEvent { .. } => "worker_event",
SystemItem::FeatureInvocationResult { .. } => "feature_invocation_result",
SystemItem::FileAttachment { .. } => "file_attachment", SystemItem::FileAttachment { .. } => "file_attachment",
SystemItem::SkillActivation { .. } => "skill_activation",
SystemItem::LegacyKnowledgeIgnored { .. } => "legacy_knowledge_ignored", SystemItem::LegacyKnowledgeIgnored { .. } => "legacy_knowledge_ignored",
SystemItem::LegacyIgnored { .. } => "legacy_ignored", SystemItem::LegacyIgnored { .. } => "legacy_ignored",
SystemItem::TaskReminder { .. } => "task_reminder", SystemItem::TaskReminder { .. } => "task_reminder",
SystemItem::ResidentSummaryRefresh { .. } => "resident_summary_refresh",
SystemItem::SubjectBehaviorRefresh { .. } => "subject_behavior_refresh",
SystemItem::Interrupt { .. } => "interrupt", SystemItem::Interrupt { .. } => "interrupt",
} }
} }
@@ -297,33 +297,45 @@ mod tests {
} }
#[test] #[test]
fn subject_behavior_refresh_round_trips_with_exact_body() { fn current_system_item_rejects_frozen_feature_kinds() {
let item = SystemItem::SubjectBehaviorRefresh { // Even well-formed old payloads cannot enter normal v5 decode/replay.
body: "current Subject context".to_string(), for line in include_str!("legacy_session_log/tests/fixtures/v4-system-inputs.jsonl")
behavior_md: "exact behavior".to_string(), .lines()
prompt_provenance: None, .take(5)
}; {
let raw = serde_json::to_string(&item).unwrap(); let record: serde_json::Value = serde_json::from_str(line).unwrap();
let parsed: SystemItem = serde_json::from_str(&raw).unwrap(); assert!(serde_json::from_value::<SystemItem>(record["entry"]["item"].clone()).is_err());
assert_eq!(parsed.history_text(), "current Subject context");
assert!( assert!(
matches!(parsed, SystemItem::SubjectBehaviorRefresh { behavior_md, .. } if behavior_md == "exact behavior") serde_json::from_value::<crate::LoggedSystemHistoryEntry>(record["entry"].clone())
.is_err()
); );
assert!(serde_json::from_str::<crate::LogEntry>(line).is_err());
}
} }
#[test] #[test]
fn frozen_subject_behavior_log_reads_old_counter_without_republishing_it() { fn public_system_input_decoder_rejects_private_fields_and_unknown_presentations() {
let old = r#"{"kind":"subject_behavior_refresh","body":"exact archived body","behavior_revision":7}"#; let valid = serde_json::json!({
let item: SystemItem = serde_json::from_str(old).unwrap(); "kind": "new-feature-label",
assert_eq!(item.history_text(), "exact archived body"); "body": "explicit public content",
assert_eq!( "presentation": { "kind": "context" },
serde_json::to_value(item).unwrap(), });
serde_json::json!({ let public: PublicSystemItem = serde_json::from_value(valid.clone()).unwrap();
"kind": "subject_behavior_refresh", assert_eq!(public.body, "explicit public content");
"body": "exact archived body", for private_field in [
"behavior_md": "" "payload",
}) "prompt_provenance",
); "policy",
"snapshot",
"behavior_md",
] {
let mut malformed = valid.clone();
malformed[private_field] = serde_json::json!({ "private": true });
assert!(serde_json::from_value::<PublicSystemItem>(malformed).is_err());
}
let mut unknown = valid;
unknown["presentation"] = serde_json::json!({ "kind": "execute_feature_payload" });
assert!(serde_json::from_value::<PublicSystemItem>(unknown).is_err());
} }
#[test] #[test]
@@ -419,29 +431,6 @@ mod tests {
} }
} }
#[test]
fn invocation_result_retains_typed_outcome_and_exact_context() {
let result = protocol::FeatureInvocationResult {
invocation_id: "stable-invoke".into(),
identity: protocol::FeatureInvocationIdentity("builtin:test/prepare".into()),
status: protocol::FeatureInvocationStatus::OutcomeUnknown,
message: "operation may have committed".into(),
context: None,
};
let body = "Outcome unknown; do not retry automatically";
let item = SystemItem::FeatureInvocationResult {
result: result.clone(),
body: body.into(),
};
let restored: SystemItem =
serde_json::from_str(&serde_json::to_string(&item).unwrap()).unwrap();
assert_eq!(restored.history_text(), body);
assert_eq!(restored.kind_label(), "feature_invocation_result");
assert!(
matches!(restored, SystemItem::FeatureInvocationResult { result: saved, .. } if saved == result)
);
}
#[test] #[test]
fn round_trip_worker_event() { fn round_trip_worker_event() {
let item = SystemItem::WorkerEvent { let item = SystemItem::WorkerEvent {
+17 -39
View File
@@ -210,18 +210,6 @@ impl WorkerMetadata {
} }
} }
fn replace_active_session(metadata: &mut WorkerMetadata, active: Option<WorkerActiveSegmentRef>) {
if let Some(session_id) = active.as_ref().map(|active| active.session_id)
&& metadata
.subjektiv_session_attribution
.as_ref()
.is_some_and(|state| state.session_id() != session_id)
{
metadata.subjektiv_session_attribution = None;
}
metadata.active = active;
}
/// Sync persistence backend for Worker metadata. /// Sync persistence backend for Worker metadata.
pub trait WorkerMetadataStore: Send + Sync { pub trait WorkerMetadataStore: Send + Sync {
/// Create or replace metadata for its `worker_name` key. /// Create or replace metadata for its `worker_name` key.
@@ -297,7 +285,7 @@ pub trait WorkerMetadataStore: Send + Sync {
workspace_root: Option<PathBuf>, workspace_root: Option<PathBuf>,
) -> Result<WorkerMetadata, WorkerStoreError> { ) -> Result<WorkerMetadata, WorkerStoreError> {
self.update_by_name(worker_name, |metadata| { self.update_by_name(worker_name, |metadata| {
replace_active_session(metadata, active); metadata.active = active;
metadata.resolved_manifest_snapshot = resolved_manifest_snapshot; metadata.resolved_manifest_snapshot = resolved_manifest_snapshot;
if let Some(workspace_id) = workspace_id { if let Some(workspace_id) = workspace_id {
metadata.workspace_id = Some(workspace_id); metadata.workspace_id = Some(workspace_id);
@@ -317,7 +305,7 @@ pub trait WorkerMetadataStore: Send + Sync {
workspace_id: Option<String>, workspace_id: Option<String>,
) -> Result<WorkerMetadata, WorkerStoreError> { ) -> Result<WorkerMetadata, WorkerStoreError> {
self.update_by_name(worker_name, |metadata| { self.update_by_name(worker_name, |metadata| {
replace_active_session(metadata, active); metadata.active = active;
metadata.resolved_manifest_snapshot = resolved_manifest_snapshot; metadata.resolved_manifest_snapshot = resolved_manifest_snapshot;
if let Some(workspace_id) = workspace_id { if let Some(workspace_id) = workspace_id {
metadata.workspace_id = Some(workspace_id); metadata.workspace_id = Some(workspace_id);
@@ -334,7 +322,7 @@ pub trait WorkerMetadataStore: Send + Sync {
workspace_root: Option<PathBuf>, workspace_root: Option<PathBuf>,
) -> Result<WorkerMetadata, WorkerStoreError> { ) -> Result<WorkerMetadata, WorkerStoreError> {
self.update_by_name(worker_name, |metadata| { self.update_by_name(worker_name, |metadata| {
replace_active_session(metadata, active); metadata.active = active;
metadata.resolved_manifest_snapshot = resolved_manifest_snapshot; metadata.resolved_manifest_snapshot = resolved_manifest_snapshot;
if let Some(workspace_root) = workspace_root { if let Some(workspace_root) = workspace_root {
metadata.workspace_root = Some(workspace_root); metadata.workspace_root = Some(workspace_root);
@@ -342,18 +330,6 @@ pub trait WorkerMetadataStore: Send + Sync {
}) })
} }
/// Set the Host-owned Subjektiv Session attribution lifecycle state while
/// preserving the active Session pointer and every unrelated field.
fn set_subjektiv_session_attribution(
&self,
worker_name: &str,
state: SubjektivSessionAttributionState,
) -> Result<WorkerMetadata, WorkerStoreError> {
self.update_by_name(worker_name, |metadata| {
metadata.subjektiv_session_attribution = Some(state);
})
}
/// Set spawned-child registry state while preserving active pointer and manifest snapshot. /// Set spawned-child registry state while preserving active pointer and manifest snapshot.
fn set_spawned_children( fn set_spawned_children(
&self, &self,
@@ -1087,13 +1063,13 @@ mod tests {
)) ))
.unwrap(); .unwrap();
store store
.set_subjektiv_session_attribution( .update_by_name("subject-worker", |metadata| {
"subject-worker", metadata.subjektiv_session_attribution =
SubjektivSessionAttributionState::OutcomeUnknown { Some(SubjektivSessionAttributionState::OutcomeUnknown {
session_id, session_id,
diagnostic: "request interrupted".to_string(), diagnostic: "request interrupted".to_string(),
}, });
) })
.unwrap(); .unwrap();
store store
.set_active( .set_active(
@@ -1117,7 +1093,7 @@ mod tests {
} }
#[test] #[test]
fn new_active_session_clears_prior_session_attribution_state() { fn generic_active_update_does_not_interpret_host_attribution() {
let tmp = tempfile::TempDir::new().unwrap(); let tmp = tempfile::TempDir::new().unwrap();
let store = FsWorkerStore::new(tmp.path()).unwrap(); let store = FsWorkerStore::new(tmp.path()).unwrap();
let old_session_id = crate::new_session_id(); let old_session_id = crate::new_session_id();
@@ -1129,13 +1105,13 @@ mod tests {
)) ))
.unwrap(); .unwrap();
store store
.set_subjektiv_session_attribution( .update_by_name("subject-worker", |metadata| {
"subject-worker", metadata.subjektiv_session_attribution =
SubjektivSessionAttributionState::Confirmed { Some(SubjektivSessionAttributionState::Confirmed {
session_id: old_session_id, session_id: old_session_id,
subject_id: "subject-1".to_string(), subject_id: "subject-1".to_string(),
}, });
) })
.unwrap(); .unwrap();
store store
@@ -1151,7 +1127,9 @@ mod tests {
restored.active, restored.active,
Some(WorkerActiveSegmentRef::pending_segment(new_session_id)) Some(WorkerActiveSegmentRef::pending_segment(new_session_id))
); );
assert!(restored.subjektiv_session_attribution.is_none()); assert!(
matches!(restored.subjektiv_session_attribution, Some(SubjektivSessionAttributionState::Confirmed { session_id, .. }) if session_id == old_session_id)
);
} }
#[test] #[test]
+198 -14
View File
@@ -36,7 +36,9 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::time::SystemTime; use std::time::SystemTime;
const SESSION_SCHEMA_VERSION: u32 = 4; const SESSION_SCHEMA_VERSION: u32 = 6;
const SYSTEM_INPUT_SESSION_SCHEMA_VERSION: u32 = 5;
const COMPACTION_SESSION_SCHEMA_VERSION: u32 = 4;
const ANNOTATED_SESSION_SCHEMA_VERSION: u32 = 3; const ANNOTATED_SESSION_SCHEMA_VERSION: u32 = 3;
const PREVIOUS_SESSION_SCHEMA_VERSION: u32 = 2; const PREVIOUS_SESSION_SCHEMA_VERSION: u32 = 2;
const LEGACY_SESSION_SCHEMA_VERSION: u32 = 1; const LEGACY_SESSION_SCHEMA_VERSION: u32 = 1;
@@ -204,7 +206,15 @@ impl WorkerSessionStore {
/// Open the Session store rooted at `<worker-aggregate>/session`. /// Open the Session store rooted at `<worker-aggregate>/session`.
pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> { pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> {
let root = root.into(); Self::new_with_migration_writer(root.into(), atomic_write_bytes)
}
// Keep fault injection at the storage publication boundary, without global
// flags or a public alternate restore API. Production always uses atomic writes.
fn new_with_migration_writer(
root: PathBuf,
write_segment: impl FnMut(&Path, &[u8]) -> Result<(), StoreError>,
) -> Result<Self, StoreError> {
fs::create_dir_all(root.join(SEGMENTS_DIR))?; fs::create_dir_all(root.join(SEGMENTS_DIR))?;
let session_id = match fs::read(root.join(SESSION_FILE)) { let session_id = match fs::read(root.join(SESSION_FILE)) {
Ok(bytes) => { Ok(bytes) => {
@@ -215,11 +225,14 @@ impl WorkerSessionStore {
} }
LEGACY_SESSION_SCHEMA_VERSION LEGACY_SESSION_SCHEMA_VERSION
| PREVIOUS_SESSION_SCHEMA_VERSION | PREVIOUS_SESSION_SCHEMA_VERSION
| ANNOTATED_SESSION_SCHEMA_VERSION => { | ANNOTATED_SESSION_SCHEMA_VERSION
migrate_segment_logs_to_v4( | COMPACTION_SESSION_SCHEMA_VERSION
| SYSTEM_INPUT_SESSION_SCHEMA_VERSION => {
migrate_segment_logs_to_current(
segment_log_paths(&root)?, segment_log_paths(&root)?,
manifest.session_id, manifest.session_id,
manifest.schema_version, manifest.schema_version,
write_segment,
)?; )?;
manifest.schema_version = SESSION_SCHEMA_VERSION; manifest.schema_version = SESSION_SCHEMA_VERSION;
atomic_write_json(&root.join(SESSION_FILE), &manifest)?; atomic_write_json(&root.join(SESSION_FILE), &manifest)?;
@@ -742,6 +755,7 @@ impl Store for WorkerSessionStore {
segment_id: SegmentId, segment_id: SegmentId,
entry: &LogEntry, entry: &LogEntry,
) -> Result<(), StoreError> { ) -> Result<(), StoreError> {
entry.validate_input_metadata()?;
let _retention = self.lock_publication(session_id)?; let _retention = self.lock_publication(session_id)?;
self.ensure_session(session_id, true)?; self.ensure_session(session_id, true)?;
self.append_log_entry(&self.log_path(segment_id), entry) self.append_log_entry(&self.log_path(segment_id), entry)
@@ -800,6 +814,9 @@ impl Store for WorkerSessionStore {
segment_id: SegmentId, segment_id: SegmentId,
entries: &[LogEntry], entries: &[LogEntry],
) -> Result<(), StoreError> { ) -> Result<(), StoreError> {
for entry in entries {
entry.validate_input_metadata()?;
}
let _retention = self.lock_publication(session_id)?; let _retention = self.lock_publication(session_id)?;
self.ensure_session(session_id, true)?; self.ensure_session(session_id, true)?;
let mut content = Vec::new(); let mut content = Vec::new();
@@ -1158,10 +1175,11 @@ fn segment_log_paths_in_directory(
Ok(paths) Ok(paths)
} }
fn migrate_segment_logs_to_v4( fn migrate_segment_logs_to_current(
paths: Vec<(SegmentId, PathBuf)>, paths: Vec<(SegmentId, PathBuf)>,
session_id: SessionId, session_id: SessionId,
source_schema_version: u32, source_schema_version: u32,
mut write_segment: impl FnMut(&Path, &[u8]) -> Result<(), StoreError>,
) -> Result<(), StoreError> { ) -> Result<(), StoreError> {
struct MigrationPlan { struct MigrationPlan {
path: PathBuf, path: PathBuf,
@@ -1192,12 +1210,19 @@ fn migrate_segment_logs_to_v4(
} }
// Resolve trusted creation boundaries first, then update inherited copies by // Resolve trusted creation boundaries first, then update inherited copies by
// logical identity too (fork/rewind seeds retain the same entry id). // logical identity too (fork/rewind seeds retain the same entry id).
let summaries = crate::legacy_session_log::compaction_summary_ids( let summaries = if source_schema_version < COMPACTION_SESSION_SCHEMA_VERSION {
crate::legacy_session_log::compaction_summary_ids(
plans.iter().flat_map(|plan| plan.entries.iter()), plans.iter().flat_map(|plan| plan.entries.iter()),
); )
} else {
// v4 already made the explicit origin migration. Preserve its origins.
Default::default()
};
for plan in &mut plans { for plan in &mut plans {
crate::legacy_session_log::migrate_compaction_origins(&mut plan.entries, &summaries); crate::legacy_session_log::migrate_compaction_origins(&mut plan.entries, &summaries);
for entry in std::mem::take(&mut plan.entries) { for mut entry in std::mem::take(&mut plan.entries) {
crate::legacy_session_log::migrate_system_inputs(&mut entry)?;
entry.validate_input_metadata()?;
serde_json::to_writer(&mut plan.output, &entry)?; serde_json::to_writer(&mut plan.output, &entry)?;
plan.output.push(b'\n'); plan.output.push(b'\n');
} }
@@ -1219,7 +1244,7 @@ fn migrate_segment_logs_to_v4(
} }
for plan in plans { for plan in plans {
atomic_write_bytes(&plan.path, &plan.output)?; write_segment(&plan.path, &plan.output)?;
} }
Ok(()) Ok(())
} }
@@ -1236,9 +1261,20 @@ pub(crate) fn migrate_fs_session(root: &Path, session_id: SessionId) -> Result<(
Err(error) => return Err(error.into()), Err(error) => return Err(error.into()),
}; };
match version { match version {
SESSION_SCHEMA_VERSION => Ok(()), SESSION_SCHEMA_VERSION => {
ANNOTATED_SESSION_SCHEMA_VERSION => { validate_canonical_segment_paths(segment_log_paths_in_directory(root)?)
migrate_segment_logs_to_v4(segment_log_paths_in_directory(root)?, session_id, version)?; }
LEGACY_SESSION_SCHEMA_VERSION
| PREVIOUS_SESSION_SCHEMA_VERSION
| ANNOTATED_SESSION_SCHEMA_VERSION
| COMPACTION_SESSION_SCHEMA_VERSION
| SYSTEM_INPUT_SESSION_SCHEMA_VERSION => {
migrate_segment_logs_to_current(
segment_log_paths_in_directory(root)?,
session_id,
version,
atomic_write_bytes,
)?;
atomic_write_json(&marker, &SESSION_SCHEMA_VERSION) atomic_write_json(&marker, &SESSION_SCHEMA_VERSION)
} }
_ => Err(StoreError::Corrupt { _ => Err(StoreError::Corrupt {
@@ -1249,8 +1285,15 @@ pub(crate) fn migrate_fs_session(root: &Path, session_id: SessionId) -> Result<(
} }
fn validate_canonical_segment_logs(root: &Path) -> Result<(), StoreError> { fn validate_canonical_segment_logs(root: &Path) -> Result<(), StoreError> {
for (_, path) in segment_log_paths(root)? { validate_canonical_segment_paths(segment_log_paths(root)?)
let _: Vec<LogEntry> = parse_jsonl(&fs::read(&path)?)?; }
fn validate_canonical_segment_paths(paths: Vec<(SegmentId, PathBuf)>) -> Result<(), StoreError> {
for (_, path) in paths {
let entries: Vec<LogEntry> = parse_jsonl(&fs::read(&path)?)?;
for entry in entries {
entry.validate_input_metadata()?;
}
} }
Ok(()) Ok(())
} }
@@ -1430,6 +1473,7 @@ mod tests {
LoggedHistoryEntry { LoggedHistoryEntry {
item: LoggedItem::from(item), item: LoggedItem::from(item),
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -2017,6 +2061,146 @@ mod tests {
); );
} }
#[test]
fn v5_migration_retries_after_second_file_publication_fails() {
fn identity_and_body(entries: &[LogEntry]) -> Vec<serde_json::Value> {
entries
.iter()
.flat_map(|record| match record {
LogEntry::AnnotatedSystemItem { entry, .. } => vec![entry.to_history_entry()],
LogEntry::AnnotatedUserInput { history, .. } => history.clone(),
_ => panic!("unexpected record in fixed migration fixture"),
})
.map(|entry| {
serde_json::json!({
"entry_id": entry.metadata.entry_id,
"origin": entry.metadata.origin,
"derivation": entry.metadata.derivation,
"item": entry.item,
})
})
.collect()
}
const SOURCE: &str =
include_str!("legacy_session_log/tests/fixtures/v4-system-inputs.jsonl");
let root = tempfile::tempdir().unwrap();
let session_id = new_session_id();
let first_id = SegmentId::from_u128(1);
let second_id = SegmentId::from_u128(2);
fs::create_dir_all(root.path().join(SEGMENTS_DIR)).unwrap();
let marker_path = root.path().join(SESSION_FILE);
atomic_write_json(
&marker_path,
&SessionManifest {
schema_version: COMPACTION_SESSION_SCHEMA_VERSION,
session_id,
},
)
.unwrap();
let old_marker = fs::read(&marker_path).unwrap();
let first_path = root
.path()
.join(SEGMENTS_DIR)
.join(format!("{first_id}.jsonl"));
let second_path = root
.path()
.join(SEGMENTS_DIR)
.join(format!("{second_id}.jsonl"));
let second_source = SOURCE.replace("-id", "-second-id");
fs::write(&first_path, SOURCE).unwrap();
fs::write(&second_path, &second_source).unwrap();
let old_first = fs::read(&first_path).unwrap();
let old_second = fs::read(&second_path).unwrap();
let first_identity =
identity_and_body(&parse_legacy_jsonl(4, session_id, first_id, &old_first).unwrap());
let second_identity =
identity_and_body(&parse_legacy_jsonl(4, session_id, second_id, &old_second).unwrap());
let mut writes = 0;
let mut first_published = None;
let failure = WorkerSessionStore::new_with_migration_writer(
root.path().to_path_buf(),
|path, bytes| {
writes += 1;
match writes {
1 => {
assert_eq!(path, first_path.as_path());
// Both files are still old when the first publication starts.
assert_eq!(fs::read(&first_path).unwrap(), old_first);
assert_eq!(fs::read(&second_path).unwrap(), old_second);
atomic_write_bytes(path, bytes)?;
first_published = Some(bytes.to_vec());
Ok(())
}
2 => {
assert_eq!(path, second_path.as_path());
assert_eq!(
fs::read(&first_path).unwrap(),
*first_published.as_ref().unwrap()
);
assert_eq!(fs::read(&second_path).unwrap(), old_second);
// Fail only this atomic-write boundary, after complete
// preflight and a real successful first-file publication.
Err(std::io::Error::other("injected second migration write failure").into())
}
_ => panic!("publication continued after its write failure"),
}
},
);
let error = match failure {
Ok(_) => panic!("the injected publication failure must reject store open"),
Err(error) => error,
};
assert!(
error
.to_string()
.contains("injected second migration write failure")
);
assert_eq!(writes, 2);
let first_published = first_published.unwrap();
assert_ne!(first_published, old_first);
assert_eq!(fs::read(&first_path).unwrap(), first_published);
assert_eq!(fs::read(&second_path).unwrap(), old_second);
assert_eq!(fs::read(&marker_path).unwrap(), old_marker);
let updated_first: Vec<LogEntry> = parse_jsonl(&first_published).unwrap();
for entry in &updated_first {
entry.validate_input_metadata().unwrap();
}
assert!(matches!(
&updated_first[0],
LogEntry::AnnotatedSystemItem {
entry: crate::LoggedSystemHistoryEntry {
item: crate::SystemItem::Input { .. },
..
},
..
}
));
assert_eq!(identity_and_body(&updated_first), first_identity);
// Retry via the public production constructor, without any injection or
// hand-built mixed fixture. The old marker drives the same migration.
let store = WorkerSessionStore::new(root.path()).unwrap();
let first = store.read_all(session_id, first_id).unwrap();
let second = store.read_all(session_id, second_id).unwrap();
assert_eq!(identity_and_body(&first), first_identity);
assert_eq!(identity_and_body(&second), second_identity);
for entry in first.iter().chain(second.iter()) {
entry.validate_input_metadata().unwrap();
}
assert_eq!(fs::read(&first_path).unwrap(), first_published);
let second_published = fs::read(&second_path).unwrap();
assert_ne!(second_published, old_second);
let manifest: SessionManifest =
serde_json::from_slice(&fs::read(&marker_path).unwrap()).unwrap();
assert_eq!(manifest.schema_version, SESSION_SCHEMA_VERSION);
drop(store);
WorkerSessionStore::new(root.path()).unwrap();
assert_eq!(fs::read(&first_path).unwrap(), first_published);
assert_eq!(fs::read(&second_path).unwrap(), second_published);
}
#[test] #[test]
fn current_jsonl_requires_annotations_across_append_rewrite_and_reopen() { fn current_jsonl_requires_annotations_across_append_rewrite_and_reopen() {
let root = tempfile::tempdir().unwrap(); let root = tempfile::tempdir().unwrap();
+175
View File
@@ -11,6 +11,7 @@ fn annotated(item: Item) -> LoggedHistoryEntry {
LoggedHistoryEntry { LoggedHistoryEntry {
item: LoggedItem::from(item), item: LoggedItem::from(item),
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -421,3 +422,177 @@ fn lookup_session_of_finds_owning_session() {
assert_eq!(store.lookup_session_of(segid).unwrap(), Some(sid)); assert_eq!(store.lookup_session_of(segid).unwrap(), Some(sid));
} }
// Storage/projection component test: both production layouts, temporary originals,
// no Feature, model, runtime or Host authority is substituted into this boundary.
#[test]
fn generic_system_append_and_fork_preserve_metadata_without_public_payload_leak() {
use session_store::{
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
InputEvidenceUse, InputObservationUse, LoggedHistoryDerivation, LoggedSystemHistoryEntry,
PublicSystemItem, PublicSystemPresentation, SystemInputEnvelope, WorkerSessionStore,
};
for worker_layout in [false, true] {
let dir = tempfile::tempdir().unwrap();
let sid = new_session_id();
let segment = new_segment_id();
let fork = new_segment_id();
let store: Box<dyn Store> = if worker_layout {
Box::new(WorkerSessionStore::new(dir.path()).unwrap())
} else {
Box::new(FsStore::new(dir.path()).unwrap())
};
store
.create_segment(sid, segment, &[nil_session_start(1, sid)])
.unwrap();
let entry = LoggedSystemHistoryEntry::from_input(
LoggedSessionHistoryOrigin::Provided {
producer: "new-feature".into(),
},
Some(LoggedHistoryDerivation {
sources: vec![LoggedSessionHistoryEntryId("source-entry".into())],
}),
SystemInputEnvelope::new(
"private model body",
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::ReferenceInput,
},
Some(
FeatureInputPayload::encode(
"new-feature:context",
101,
&serde_json::json!({"private": "secret-detail"}),
)
.unwrap(),
),
Some(PublicSystemItem {
kind: "context".into(),
body: "explicit public body".into(),
presentation: PublicSystemPresentation::Context,
}),
)
.unwrap(),
)
.unwrap(),
);
store
.append(
sid,
segment,
&LogEntry::AnnotatedSystemItem {
ts: 2,
entry: entry.clone(),
extensions: vec![],
},
)
.unwrap();
let saved = store.read_all(sid, segment).unwrap();
let LogEntry::AnnotatedSystemItem {
entry: restored, ..
} = &saved[1]
else {
panic!("missing system input")
};
assert_eq!(restored.to_history_entry(), entry.to_history_entry());
assert_eq!(
collect_state(&saved).history[0].as_text(),
Some("private model body")
);
let mut fork_seed = nil_session_start(3, sid);
if let LogEntry::AnnotatedSegmentStart {
history,
forked_from,
..
} = &mut fork_seed
{
history.push(restored.to_history_entry());
*forked_from = Some(session_store::SegmentOrigin {
segment_id: segment,
at_turn_index: 0,
});
}
store.create_segment(sid, fork, &[fork_seed]).unwrap();
let fork_saved = store.read_all(sid, fork).unwrap();
let LogEntry::AnnotatedSegmentStart { history, .. } = &fork_saved[0] else {
unreachable!()
};
assert_eq!(history, &[entry.to_history_entry()]);
let original = session_store::public_snapshot::project_session_snapshot(sid, &saved);
let inherited = session_store::public_snapshot::project_session_snapshot(sid, &fork_saved);
let public = serde_json::to_string(&original.entries).unwrap();
assert!(public.contains("explicit public body"));
for private in ["private model body", "secret-detail", "new-feature:context"] {
assert!(!public.contains(private), "private field leaked: {private}");
}
assert_eq!(original.entries.len(), 1);
assert_eq!(inherited.entries.len(), 1);
assert_eq!(original.entries[0].entry_id, inherited.entries[0].entry_id);
assert_eq!(
original.entries[0].derived_from,
inherited.entries[0].derived_from
);
assert_eq!(
original.entries[0].provenance,
inherited.entries[0].provenance
);
assert_eq!(
serde_json::to_value(&original.entries[0].data).unwrap(),
serde_json::to_value(&inherited.entries[0].data).unwrap()
);
}
}
#[test]
fn invalid_generic_system_input_rejects_append_create_and_external_json_without_changing_originals()
{
use session_store::{StoreError, SystemItem, WorkerSessionStore};
for worker_layout in [false, true] {
let dir = tempfile::tempdir().unwrap();
let sid = new_session_id();
let segment = new_segment_id();
let store: Box<dyn Store> = if worker_layout {
Box::new(WorkerSessionStore::new(dir.path()).unwrap())
} else {
Box::new(FsStore::new(dir.path()).unwrap())
};
let seed = nil_session_start(1, sid);
store
.create_segment(sid, segment, std::slice::from_ref(&seed))
.unwrap();
let path = if worker_layout {
dir.path().join("segments").join(format!("{segment}.jsonl"))
} else {
dir.path()
.join(sid.to_string())
.join(format!("{segment}.jsonl"))
};
let before = std::fs::read(&path).unwrap();
let invalid = LogEntry::AnnotatedSystemItem {
ts: 2,
extensions: vec![],
entry: session_store::LoggedSystemHistoryEntry {
item: SystemItem::Input {
body: "no policy".into(),
},
metadata: LoggedSessionHistoryMetadata::legacy_unknown(),
},
};
assert!(matches!(
store.append(sid, segment, &invalid),
Err(StoreError::InvalidInputMetadata(_))
));
assert!(matches!(
store.create_segment(sid, segment, &[seed, invalid.clone()]),
Err(StoreError::InvalidInputMetadata(_))
));
assert!(
serde_json::from_value::<LogEntry>(serde_json::to_value(invalid).unwrap()).is_err()
);
assert_eq!(std::fs::read(path).unwrap(), before);
assert_eq!(store.read_all(sid, segment).unwrap().len(), 1);
}
}
@@ -24,6 +24,7 @@ fn annotated(items: &[Item]) -> Vec<session_store::LoggedHistoryEntry> {
.map(|item| session_store::LoggedHistoryEntry { .map(|item| session_store::LoggedHistoryEntry {
item: session_store::LoggedItem::from(item), item: session_store::LoggedItem::from(item),
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId::new(), entry_id: session_store::LoggedSessionHistoryEntryId::new(),
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown, origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -212,6 +212,7 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry {
content: vec![session_store::LoggedContentPart::Text { text: text.into() }], content: vec![session_store::LoggedContentPart::Text { text: text.into() }],
}, },
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId(id.into()), entry_id: session_store::LoggedSessionHistoryEntryId(id.into()),
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None, derivation: None,
@@ -215,6 +215,7 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
} }
protocol::Event::TextDelta { text } if text.contains(PARTIAL) => break, protocol::Event::TextDelta { text } if text.contains(PARTIAL) => break,
protocol::Event::RunEnd { .. } => panic!("dialog ended before cancellation"), protocol::Event::RunEnd { .. } => panic!("dialog ended before cancellation"),
protocol::Event::SessionEntryCommitted { entry } if matches!(entry.data, protocol::SessionSnapshotEntryData::RunError { .. }) => panic!("dialog failed before cancellation: {entry:?}"),
protocol::Event::Error { code, message } => panic!("{code:?}: {message}"), protocol::Event::Error { code, message } => panic!("{code:?}: {message}"),
_ => {} _ => {}
} }
@@ -294,6 +295,32 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
.iter() .iter()
.flat_map(|segment| source_store.read_all(session, *segment).unwrap()) .flat_map(|segment| source_store.read_all(session, *segment).unwrap())
.collect(); .collect();
for history in raw.iter().filter_map(|record| match record {
session_store::LogEntry::AnnotatedSegmentStart { history, compacted_from: Some(_), .. } => Some(history),
_ => None,
}) {
let mut calls = std::collections::HashSet::new();
let mut results = 0;
for entry in history {
let is_tool = matches!(entry.item, session_store::LoggedItem::ToolCall { .. } | session_store::LoggedItem::ToolResult { .. });
let Some(metadata) = entry.metadata.input_metadata.as_ref().filter(|_| is_tool) else { continue; };
assert_eq!(metadata.usage().recording_material, session_store::InputEvidenceUse::Exclude);
assert_eq!(metadata.usage().memory_evidence, session_store::InputEvidenceUse::Exclude);
assert_eq!(metadata.usage().observation, session_store::InputObservationUse::Hidden);
assert!(metadata.public_item().is_none());
assert!(entry.metadata.derivation.is_some());
match &entry.item {
session_store::LoggedItem::ToolCall { call_id, .. } => { calls.insert(call_id.clone()); }
session_store::LoggedItem::ToolResult { call_id, .. } => {
assert!(calls.contains(call_id), "derived Tool result must follow its real logical call");
results += 1;
}
_ => unreachable!(),
}
}
assert!(!calls.is_empty());
assert!(results >= calls.len());
}
assert_eq!( assert_eq!(
raw.iter() raw.iter()
.filter(|entry| matches!( .filter(|entry| matches!(
@@ -11,7 +11,7 @@ use agen::llm_client::{
types::Item, types::Item,
}; };
use futures::Stream; use futures::Stream;
use session_store::{CombinedStore, FsWorkerStore, LogEntry, LoggedSessionHistoryOrigin}; use session_store::{CombinedStore, FsWorkerStore, LogEntry};
use std::pin::Pin; use std::pin::Pin;
const BEFORE: &str = "FROZEN_POSITION: B partially implemented; tests not run; result unconfirmed"; const BEFORE: &str = "FROZEN_POSITION: B partially implemented; tests not run; result unconfirmed";
@@ -139,14 +139,14 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab
let worker = prepared.worker(); let worker = prepared.worker();
assert_ne!(worker.segment_id(), source_segment); assert_ne!(worker.segment_id(), source_segment);
let seed = worker.store().read_all(worker.session_id(), worker.segment_id()).unwrap().into_iter().find_map(|entry| match entry { LogEntry::AnnotatedSegmentStart { history, .. } => Some(history), _ => None }).expect("durable compacted seed"); let seed = worker.store().read_all(worker.session_id(), worker.segment_id()).unwrap().into_iter().find_map(|entry| match entry { LogEntry::AnnotatedSegmentStart { history, .. } => Some(history), _ => None }).expect("durable compacted seed");
let journal_items: Vec<_> = seed.iter().filter(|entry| entry.metadata.origin == LoggedSessionHistoryOrigin::JournalContext).collect(); let journal_items: Vec<_> = seed.iter().filter(|entry| entry.metadata.input_metadata.as_ref().is_some_and(|input| input.usage().compaction == session_store::InputCompactionUse::ReferenceInput)).collect();
assert_eq!(journal_items.len(), 1); assert_eq!(journal_items.len(), 1);
let journal_item = Item::from(journal_items[0].item.clone()); let journal_item = Item::from(journal_items[0].item.clone());
let body = journal_item.as_text().unwrap(); let body = journal_item.as_text().unwrap();
assert!(body.contains(&expected_json), "durable seed contains the full selected JSON and processing checkpoint"); assert!(body.contains(&expected_json), "durable seed contains the full selected JSON and processing checkpoint");
assert!(body.contains(BEFORE)); assert!(body.contains(BEFORE));
assert!(!body.contains(AFTER)); assert!(!body.contains(AFTER));
assert!(seed.iter().any(|entry| entry.metadata.origin == LoggedSessionHistoryOrigin::CompactionSummary && Item::from(entry.item.clone()).as_text().is_some_and(|text| text.contains(COMPLEMENT)))); assert!(seed.iter().any(|entry| entry.metadata.input_metadata.as_ref().is_some_and(|input| input.usage().compaction == session_store::InputCompactionUse::OpeningSummary) && Item::from(entry.item.clone()).as_text().is_some_and(|text| text.contains(COMPLEMENT))));
let requests = model.requests.lock().unwrap(); let requests = model.requests.lock().unwrap();
let input = requests[0].items.iter().find_map(|item| match item { let input = requests[0].items.iter().find_map(|item| match item {
Item::Message { role: agen::llm_client::types::Role::User, .. } => item.as_text().filter(|text| text.contains("## Conversation overview/index")), Item::Message { role: agen::llm_client::types::Role::User, .. } => item.as_text().filter(|text| text.contains("## Conversation overview/index")),
@@ -79,7 +79,7 @@ async fn recorded_idle_body_handoff_preserves_open_after_source_deletion_and_see
let sessions = session_store::FsStore::new_with_retention(workers.sessions_dir(second.worker_id()), crate::store::session_retention(&state)).unwrap(); let sessions = session_store::FsStore::new_with_retention(workers.sessions_dir(second.worker_id()), crate::store::session_retention(&state)).unwrap();
let pointer = session_store::FsWorkerStore::new(workers.worker_metadata_dir(second.worker_id())).unwrap().read_by_name(&second.record().storage_key).unwrap().unwrap().active.unwrap(); let pointer = session_store::FsWorkerStore::new(workers.worker_metadata_dir(second.worker_id())).unwrap().read_by_name(&second.record().storage_key).unwrap().unwrap().active.unwrap();
let entries = sessions.read_all(session, pointer.segment_id.unwrap()).unwrap(); let entries = sessions.read_all(session, pointer.segment_id.unwrap()).unwrap();
let journal_items = entries.iter().filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, session_store::SystemItem::JournalStartup { .. }))).count(); let journal_items = entries.iter().filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSystemItem { entry, .. } if entry.metadata.input_metadata.as_ref().and_then(|input| input.feature_payload()).is_some_and(|payload| payload.schema() == "journal:startup"))).count();
assert_eq!(journal_items, 1); assert_eq!(journal_items, 1);
assert_eq!(catalog.store.subject(&subject.id).unwrap().unwrap().behavior_md, "BEHAVIOR_ORIGINAL_UNCHANGED"); assert_eq!(catalog.store.subject(&subject.id).unwrap().unwrap().behavior_md, "BEHAVIOR_ORIGINAL_UNCHANGED");
second.shutdown().await.unwrap(); second.shutdown().await.unwrap();
@@ -1621,6 +1621,7 @@ allow = []
let history = |id: &str, item: LoggedItem| LoggedHistoryEntry { let history = |id: &str, item: LoggedItem| LoggedHistoryEntry {
item, item,
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.into()), entry_id: LoggedSessionHistoryEntryId(id.into()),
origin: LoggedSessionHistoryOrigin::BackendInstruction { operation_id: None }, origin: LoggedSessionHistoryOrigin::BackendInstruction { operation_id: None },
derivation: None, derivation: None,
+36 -20
View File
@@ -3013,7 +3013,8 @@ impl App {
/// `[File: …]` parsing path: each kind maps directly to a typed /// `[File: …]` parsing path: each kind maps directly to a typed
/// block (`Block::Notify`, `Block::WorkerEvent`, …). /// block (`Block::Notify`, `Block::WorkerEvent`, …).
fn apply_system_item(&mut self, value: &serde_json::Value) { fn apply_system_item(&mut self, value: &serde_json::Value) {
let Ok(item) = serde_json::from_value::<session_store::SystemItem>(value.clone()) else { let Ok(item) = serde_json::from_value::<session_store::PublicSystemItem>(value.clone())
else {
// Unknown / forward-compat shape: fall back to rendering the // Unknown / forward-compat shape: fall back to rendering the
// raw text payload (if any) as a generic system message. // raw text payload (if any) as a generic system message.
if let Some(text) = value.get("body").and_then(|b| b.as_str()) { if let Some(text) = value.get("body").and_then(|b| b.as_str()) {
@@ -3024,29 +3025,23 @@ impl App {
} }
return; return;
}; };
match item { let body = item.body;
session_store::SystemItem::Notification { message, .. } => { match item.presentation {
session_store::PublicSystemPresentation::Notification { message } => {
self.blocks.push(Block::Notify { message }); self.blocks.push(Block::Notify { message });
} }
session_store::SystemItem::WorkerEvent { event, .. } => { session_store::PublicSystemPresentation::WorkerEvent { event } => {
self.blocks.push(Block::WorkerEvent { event }); self.blocks.push(Block::WorkerEvent { event });
} }
session_store::SystemItem::FeatureInvocationResult { body, .. } session_store::PublicSystemPresentation::Context => {
| session_store::SystemItem::FileAttachment { body, .. }
| session_store::SystemItem::SkillActivation { body, .. }
| session_store::SystemItem::JournalStartup { body, .. }
| session_store::SystemItem::ResidentSummaryRefresh { body, .. }
| session_store::SystemItem::SubjectBehaviorRefresh { body, .. }
| session_store::SystemItem::Interrupt { body, .. } => {
self.task_store.apply_system_message_text(&body); self.task_store.apply_system_message_text(&body);
self.blocks.push(Block::SystemMessage { text: body }); self.blocks.push(Block::SystemMessage { text: body });
} }
session_store::SystemItem::TaskReminder { body, .. } => { session_store::PublicSystemPresentation::TaskReminder => {
self.task_store.apply_system_message_text(&body); self.task_store.apply_system_message_text(&body);
self.blocks.push(Block::TaskReminder { text: body }); self.blocks.push(Block::TaskReminder { text: body });
} }
session_store::SystemItem::LegacyIgnored { .. } => {} session_store::PublicSystemPresentation::Ignored => {}
session_store::SystemItem::LegacyKnowledgeIgnored { .. } => {}
} }
} }
@@ -3572,6 +3567,7 @@ mod completion_flow_tests {
session_store::LoggedHistoryEntry { session_store::LoggedHistoryEntry {
item: session_store::LoggedItem::from(item), item: session_store::LoggedItem::from(item),
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId::new(), entry_id: session_store::LoggedSessionHistoryEntryId::new(),
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown, origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -4329,6 +4325,7 @@ mod completion_flow_tests {
})); }));
app.handle_worker_event(Event::SessionEntryCommitted { app.handle_worker_event(Event::SessionEntryCommitted {
entry: protocol::SessionSnapshotEntry { entry: protocol::SessionSnapshotEntry {
use_policy: None,
entry_id: "run-failure".into(), entry_id: "run-failure".into(),
timestamp: 2, timestamp: 2,
provenance: protocol::SessionEntryProvenance::LegacyUnknown, provenance: protocol::SessionEntryProvenance::LegacyUnknown,
@@ -4450,12 +4447,12 @@ mod completion_flow_tests {
} }
#[test] #[test]
fn live_legacy_workflow_system_item_is_ignored() { fn retired_system_input_is_ignored() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
let item = serde_json::json!({ let item = serde_json::json!({
"kind": "workflow", "kind": "retired_input",
"slug": "build", "body": "",
"body": "[Workflow /build]\nRun the build", "presentation": { "kind": "ignored" },
}); });
app.handle_worker_event(Event::SystemItem { app.handle_worker_event(Event::SystemItem {
entry_id: None, entry_id: None,
@@ -4861,12 +4858,30 @@ mod completion_flow_tests {
); );
} }
#[test]
fn feature_label_does_not_select_system_input_presentation() {
for label in ["journal_startup", "new-unregistered-feature"] {
let mut app = App::new("test".into());
app.handle_worker_event(Event::SystemItem {
entry_id: None,
item: serde_json::json!({
"kind": label,
"body": "explicit public content",
"presentation": { "kind": "context" },
}),
});
assert!(
matches!(app.blocks.as_slice(), [Block::SystemMessage { text }] if text == "explicit public content")
);
}
}
#[test] #[test]
fn live_system_item_notification_appends_notify_block() { fn live_system_item_notification_appends_notify_block() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
let item = serde_json::json!({ let item = serde_json::json!({
"kind": "notification", "kind": "notification",
"message": "hi", "presentation": { "kind": "notification", "message": "hi" },
"body": "[Notification] hi", "body": "[Notification] hi",
}); });
app.handle_worker_event(Event::SystemItem { app.handle_worker_event(Event::SystemItem {
@@ -4884,7 +4899,7 @@ mod completion_flow_tests {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
let item = serde_json::json!({ let item = serde_json::json!({
"kind": "worker_event", "kind": "worker_event",
"event": { "kind": "turn_ended", "worker_name": "child" }, "presentation": { "kind": "worker_event", "event": { "kind": "turn_ended", "worker_name": "child" } },
"body": "[Notification] worker `child` finished a turn", "body": "[Notification] worker `child` finished a turn",
}); });
app.handle_worker_event(Event::SystemItem { app.handle_worker_event(Event::SystemItem {
@@ -5285,6 +5300,7 @@ mod completion_flow_tests {
item: serde_json::json!({ item: serde_json::json!({
"kind": "task_reminder", "kind": "task_reminder",
"body": snapshot, "body": snapshot,
"presentation": { "kind": "task_reminder" },
}), }),
}); });
+44 -11
View File
@@ -657,18 +657,32 @@ fn plan_runtime_store_migration(
)); ));
} }
if current_schema_version == SCHEMA_VERSION { if current_schema_version == SCHEMA_VERSION {
// Runtime and Session schemas evolve independently. Probe only through
// Session Store's non-mutating current-schema admission boundary; any
// conversion still runs on the existing owner-locked staging tree.
let migration_required = crate::retention::sessions_require_migration(root)?;
let excluded_ephemeral_paths = if migration_required {
runtime_tree_exclusions(root)?
} else {
Vec::new()
};
let archived_orphan_worker_count = if migration_required {
orphaned_worker_directories(root)?.len()
} else {
0
};
let plan = FsRuntimeStoreMigrationPlan { let plan = FsRuntimeStoreMigrationPlan {
current_schema_version, current_schema_version,
target_schema_version: SCHEMA_VERSION, target_schema_version: SCHEMA_VERSION,
migration_required: false, migration_required,
worker_count: 0, worker_count: 0,
archived_orphan_worker_count: 0, archived_orphan_worker_count,
migrated_worker_aggregate_count: 0, migrated_worker_aggregate_count: 0,
migrated_diagnostic_worker_ref_count: 0, migrated_diagnostic_worker_ref_count: 0,
cleared_diagnostic_worker_ref_count: 0, cleared_diagnostic_worker_ref_count: 0,
mapping_digest: legacy_worker_identity_mapping_digest(&[]), mapping_digest: legacy_worker_identity_mapping_digest(&[]),
mappings: Vec::new(), mappings: Vec::new(),
excluded_ephemeral_paths: Vec::new(), excluded_ephemeral_paths,
}; };
return Ok((plan, Vec::new())); return Ok((plan, Vec::new()));
} }
@@ -1808,7 +1822,9 @@ fn migrate_runtime_store(
Ok(plan) Ok(plan)
} }
fn archive_orphaned_worker_directories(root: &Path) -> Result<usize, RuntimeError> { fn orphaned_worker_directories(
root: &Path,
) -> Result<Vec<(std::ffi::OsString, PathBuf)>, RuntimeError> {
let workers_dir = root.join(WORKERS_DIR); let workers_dir = root.join(WORKERS_DIR);
let mut orphaned = fs::read_dir(&workers_dir) let mut orphaned = fs::read_dir(&workers_dir)
.map_err(|source| runtime_io_error("read Worker records", &workers_dir, source))? .map_err(|source| runtime_io_error("read Worker records", &workers_dir, source))?
@@ -1821,6 +1837,11 @@ fn archive_orphaned_worker_directories(root: &Path) -> Result<usize, RuntimeErro
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
orphaned.sort_by(|(left, _), (right, _)| left.cmp(right)); orphaned.sort_by(|(left, _), (right, _)| left.cmp(right));
Ok(orphaned)
}
fn archive_orphaned_worker_directories(root: &Path) -> Result<usize, RuntimeError> {
let orphaned = orphaned_worker_directories(root)?;
if orphaned.is_empty() { if orphaned.is_empty() {
return Ok(0); return Ok(0);
} }
@@ -1855,6 +1876,21 @@ fn migrate_runtime_store_in_place(
if !plan.migration_required { if !plan.migration_required {
return Ok(plan); return Ok(plan);
} }
if plan.current_schema_version == SCHEMA_VERSION {
// Session-only migration: do not reserialize Runtime/Worker identity,
// Restore guards, recovery fences, or retained execution receipts.
// As in a Runtime schema migration, retain orphan trees outside the
// Worker catalog before validating the completed staging tree.
let archived = archive_orphaned_worker_directories(root)?;
if archived != plan.archived_orphan_worker_count {
return Err(runtime_store_corrupt(
root,
"orphan count changed during Session migration".into(),
));
}
crate::retention::migrate_staged_sessions(root)?;
return Ok(plan);
}
let runtime_path = root.join(RUNTIME_FILE); let runtime_path = root.join(RUNTIME_FILE);
let bytes = let bytes =
fs::read(&runtime_path).map_err(|error| runtime_io_error("read", &runtime_path, error))?; fs::read(&runtime_path).map_err(|error| runtime_io_error("read", &runtime_path, error))?;
@@ -1948,11 +1984,9 @@ fn migrate_runtime_store_in_place(
), ),
)); ));
} }
// Schema 11 already migrated Sessions; the recovery-fence bump must not // A Runtime schema bump says nothing about the retained Session schema.
// rewrite retained history or archive attestations. // Session Store owns versioned conversion, including v4 -> generic v5.
if plan.current_schema_version != SOURCE_SCHEMA_VERSION {
crate::retention::migrate_staged_sessions(root)?; crate::retention::migrate_staged_sessions(root)?;
}
let (document, _) = migrate_runtime_document( let (document, _) = migrate_runtime_document(
document, document,
plan.current_schema_version, plan.current_schema_version,
@@ -3011,9 +3045,8 @@ mod tests {
} else { } else {
drop(result.unwrap()); drop(result.unwrap());
assert_eq!(stored_schema_version(&root), u64::from(SCHEMA_VERSION)); assert_eq!(stored_schema_version(&root), u64::from(SCHEMA_VERSION));
let manifest: serde_json::Value = session_store::WorkerSessionStore::open_read_only(&session)
read_json(&session.join("session.json"), "test").unwrap(); .expect("startup migration must admit the current Session schema");
assert_eq!(manifest["schema_version"], 4);
let once = fs::read(session.join("session.json")).unwrap(); let once = fs::read(session.join("session.json")).unwrap();
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap()); drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap());
assert_eq!(fs::read(session.join("session.json")).unwrap(), once); assert_eq!(fs::read(session.join("session.json")).unwrap(), once);
@@ -35,6 +35,7 @@ fn message(id: &str, role: LoggedRole, text: &str) -> LoggedHistoryEntry {
content: vec![LoggedContentPart::Text { text: text.into() }], content: vec![LoggedContentPart::Text { text: text.into() }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.into()), entry_id: LoggedSessionHistoryEntryId(id.into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput, origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None, derivation: None,
+47 -9
View File
@@ -1204,9 +1204,13 @@ fn archive_storage(
} }
} }
/// Called only on the private Runtime migration staging tree under its owner // Share discovery and erasure fencing between the read-only Runtime preflight
/// lock. Failure discards the whole tree, including manifests and attestations. // and its private staging pass. Session Store remains the authority for schema
pub(crate) fn migrate_staged_sessions(root: &Path) -> Result<(), RuntimeError> { // admission and conversion; Runtime must not duplicate its version table.
fn for_each_live_session(
root: &Path,
mut visit: impl FnMut(&Path, bool) -> Result<(), RuntimeError>,
) -> Result<(), RuntimeError> {
for directory in ["workers", "orphaned-workers", "archives/workers"] { for directory in ["workers", "orphaned-workers", "archives/workers"] {
let directory = root.join(directory); let directory = root.join(directory);
let entries = match fs::read_dir(&directory) { let entries = match fs::read_dir(&directory) {
@@ -1253,15 +1257,44 @@ pub(crate) fn migrate_staged_sessions(root: &Path) -> Result<(), RuntimeError> {
Err(session_store::StoreError::SessionErased(_)) => continue, Err(session_store::StoreError::SessionErased(_)) => continue,
Err(error) => return Err(retention_error(error)), Err(error) => return Err(retention_error(error)),
} }
let archive = if directory == root.join("archives/workers") { visit(&aggregate, directory == root.join("archives/workers"))?;
}
}
Ok(())
}
/// Read-only admission probe: a rejected Session needs the existing staged
/// migration attempt (which will also reject corrupt/unsupported inputs).
/// Never create a writable Session store or rewrite archive attestations here.
pub(crate) fn sessions_require_migration(root: &Path) -> Result<bool, RuntimeError> {
let mut required = false;
for_each_live_session(root, |aggregate, _| {
required |=
session_store::WorkerSessionStore::open_read_only(aggregate.join("session")).is_err();
Ok(())
})?;
Ok(required)
}
/// Called only on the private Runtime migration staging tree under its owner
/// lock. Failure discards the whole tree, including manifests and attestations.
pub(crate) fn migrate_staged_sessions(root: &Path) -> Result<(), RuntimeError> {
for_each_live_session(root, |aggregate, is_archive| {
let session = aggregate.join("session");
let archive = if is_archive {
let manifest: WorkerSessionArchiveManifest = read_json( let manifest: WorkerSessionArchiveManifest = read_json(
&aggregate.join("manifest.json"), &aggregate.join("manifest.json"),
"read archive before Session migration", "read archive before Session migration",
)?; )?;
Some(validate_existing_archive(&aggregate, &manifest, false)?) Some(validate_existing_archive(aggregate, &manifest, false)?)
} else { } else {
None None
}; };
// Current Sessions need no writable admission, and their storage
// attestations must remain byte-for-byte unchanged.
if session_store::WorkerSessionStore::open_read_only(&session).is_ok() {
return Ok(());
}
session_store::WorkerSessionStore::new(&session).map_err(|error| { session_store::WorkerSessionStore::new(&session).map_err(|error| {
RuntimeError::StoreCorrupt { RuntimeError::StoreCorrupt {
operation: "migrate Session schema", operation: "migrate Session schema",
@@ -1286,9 +1319,8 @@ pub(crate) fn migrate_staged_sessions(root: &Path) -> Result<(), RuntimeError> {
"commit migrated archive storage", "commit migrated archive storage",
)?; )?;
} }
}
}
Ok(()) Ok(())
})
} }
fn validate_existing_archive( fn validate_existing_archive(
@@ -1753,12 +1785,16 @@ fn temporary_path(path: &Path) -> PathBuf {
path.with_file_name(format!(".{name}.tmp-{}-{sequence}", std::process::id())) path.with_file_name(format!(".{name}.tmp-{}-{sequence}", std::process::id()))
} }
#[cfg(all(test, feature = "fs-store"))]
#[path = "retention/runtime_v5_tests.rs"]
mod runtime_v5_tests;
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use std::sync::{Arc, Barrier}; use std::sync::{Arc, Barrier};
fn write_json(path: &Path, value: &impl Serialize) { pub(super) fn write_json(path: &Path, value: &impl Serialize) {
fs::create_dir_all(path.parent().unwrap()).unwrap(); fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, serde_json::to_vec_pretty(value).unwrap()).unwrap(); fs::write(path, serde_json::to_vec_pretty(value).unwrap()).unwrap();
} }
@@ -1786,7 +1822,7 @@ mod tests {
fs::write(worker.join("runs/attempt-a/worker.sock"), b"not retained").unwrap(); fs::write(worker.join("runs/attempt-a/worker.sock"), b"not retained").unwrap();
} }
fn request( pub(super) fn request(
worker_id: WorkerId, worker_id: WorkerId,
disposition: SessionDisposition, disposition: SessionDisposition,
) -> WorkerRetentionExecutionRequest { ) -> WorkerRetentionExecutionRequest {
@@ -1831,6 +1867,7 @@ mod tests {
}], }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("original".into()), entry_id: LoggedSessionHistoryEntryId("original".into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput, origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None, derivation: None,
@@ -1844,6 +1881,7 @@ mod tests {
}], }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("summary".into()), entry_id: LoggedSessionHistoryEntryId("summary".into()),
origin: LoggedSessionHistoryOrigin::DerivedSummary, origin: LoggedSessionHistoryOrigin::DerivedSummary,
derivation: Some(session_store::LoggedHistoryDerivation { derivation: Some(session_store::LoggedHistoryDerivation {
@@ -0,0 +1,2 @@
{"kind":"annotated_system_item","ts":101,"entry":{"item":{"kind":"journal_startup","body":"journal body\nunchanged","snapshot":{"entries":[{"text":"private snapshot"}]},"snapshot_sha256":"snapshot-digest","prompt_provenance":{"workspace_id":"workspace","source_digest":"source","projection_digest":"projection","logical_name":"journal_startup"}},"metadata":{"entry_id":"journal-id","origin":{"kind":"legacy_unknown"},"derivation":{"sources":["source-id"]}}},"extensions":[{"domain":"migration-fixture","payload":{"kept":true}}]}
{"kind":"annotated_system_item","ts":103,"entry":{"item":{"kind":"subject_behavior_refresh","body":"rendered subject body","behavior_md":"exact behavior","prompt_provenance":{"source_digest":"subject-source","projection_digest":"subject-projection","logical_name":"subject_behavior"}},"metadata":{"entry_id":"subject-id","origin":{"kind":"backend_instruction","operation_id":"op"}}}}
@@ -0,0 +1,634 @@
//! Runtime integration, not a second implementation of Session migration.
//! The Feature records are frozen v4 wire copied from the Session Store fixture;
//! only the empty segment envelope is built with current APIs.
use super::tests::{request, write_json};
use super::*;
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
use serde_json::{Value, json};
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
use std::collections::BTreeMap;
const V4_SYSTEM_INPUTS: &str = include_str!("fixtures/system-input-v4.jsonl");
fn write_v4_session(aggregate: &Path) -> (SessionId, SegmentId) {
let root = aggregate.join("session");
let session = session_store::new_session_id();
let segment = session_store::new_segment_id();
let store = WorkerSessionStore::new(&root).unwrap();
store
.create_segment(
session,
segment,
&[LogEntry::AnnotatedSegmentStart {
ts: 1,
session_id: session,
system_prompt: None,
config: Default::default(),
history: vec![],
forked_from: None,
compacted_from: None,
}],
)
.unwrap();
let path = root.join("segments").join(format!("{segment}.jsonl"));
let mut bytes = fs::read(&path).unwrap();
bytes.extend_from_slice(V4_SYSTEM_INPUTS.as_bytes());
fs::write(path, bytes).unwrap();
write_json(
&root.join("session.json"),
&json!({
"schema_version": 4, "session_id": session,
}),
);
(session, segment)
}
fn stopped_worker(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId) {
let worker = WorkerId::now_v7();
let aggregate = root.join("workers").join(worker.to_string());
// Serialize request defaults just as a production aggregate writer does.
let request: crate::catalog::CreateWorkerRequest = serde_json::from_value(json!({
"worker_id": worker,
"workspace_api": {"workspace_id": "workspace-a", "base_url": "https://workspace.invalid"},
"create_fingerprint": "runtime-v5-fixture",
"profile": {"kind": "builtin", "value": "builtin:coder"},
"profile_source": {"kind": "workspace_config", "archive": {
"id": "archive-1", "digest": "sha256:archive", "size_bytes": 0,
"source_graph": {"source_count": 0, "total_source_bytes": 0,
"entrypoints": {}, "import_count": 0}
}}
}))
.unwrap();
write_json(
&aggregate.join("worker.json"),
&json!({
"schema_version": schema,
"worker_ref": {"worker_id": worker}, "worker_id": worker,
"profile": request.profile,
"profile_source": request.profile_source.reference(),
"display_name": null, "config_bundle": null,
"created_at_ms": 1234, "status": "stopped", "workspace_id": "workspace-a",
"last_finished_submission_request_id": "retained-submission",
"restore_guard": {"observation_token": "retained-token",
"active_request_id": null, "owners": {}},
"execution_state": {"state": "available", "execution": {
"request": request, "binding": {}, "restore_intent": "explicit"
}}
}),
);
let (session, segment) = write_v4_session(&aggregate);
let mut metadata = session_store::WorkerMetadata::new(
format!("worker-runtime-{worker}"),
Some(session_store::WorkerActiveSegmentRef::active_segment(
session, segment,
)),
);
metadata.workspace_id = Some("workspace-a".into());
write_json(&aggregate.join("metadata.json"), &metadata);
(worker, session, segment)
}
fn runtime_fixture(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId) {
fs::create_dir_all(root.join("workers")).unwrap();
let fenced_worker = WorkerId::now_v7();
write_json(
&root.join("runtime.json"),
&json!({
"schema_version": schema, "display_name": "retained Runtime",
"backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
"config_bundles": {}, "workspace_owners": {"workspace-a": "owner-a"},
"diagnostics": [], "subject_releases": [],
"subject_create_recovery_fences": {fenced_worker.to_string(): {
"request": {"workspace_id": "workspace-a", "worker_id": fenced_worker,
"create_fingerprint": "retained-create", "operation_id": "retained-recovery"},
"response": null
}}
}),
);
stopped_worker(root, schema)
}
// Include every file, not just schema markers: failures must preserve original
// history, metadata, archive receipts, checksums and retention evidence together.
fn tree_bytes(root: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
fn walk(root: &Path, at: &Path, result: &mut BTreeMap<PathBuf, Vec<u8>>) {
for entry in fs::read_dir(at).unwrap() {
let entry = entry.unwrap();
if entry.file_type().unwrap().is_dir() {
walk(root, &entry.path(), result);
} else {
result.insert(
entry.path().strip_prefix(root).unwrap().to_path_buf(),
fs::read(entry.path()).unwrap(),
);
}
}
}
let mut result = BTreeMap::new();
walk(root, root, &mut result);
result
}
fn assert_generic(root: &Path, session: SessionId, segment: SegmentId) {
use session_store::{InputCompactionUse, InputEvidenceUse, InputObservationUse};
let store = WorkerSessionStore::open_read_only(root).unwrap();
let entries = store.read_all_read_only(session, segment).unwrap();
assert_eq!(entries.len(), 3);
let originals: Vec<Value> = V4_SYSTEM_INPUTS
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
for (log, old) in entries[1..].iter().zip(originals) {
let LogEntry::AnnotatedSystemItem { entry, .. } = log else {
panic!("lost System record: {log:?}");
};
let current = serde_json::to_value(log).unwrap();
assert_eq!(current["ts"], old["ts"]);
for field in ["entry_id", "origin", "derivation"] {
assert_eq!(
current["entry"]["metadata"][field],
old["entry"]["metadata"][field]
);
}
assert_eq!(current["entry"]["item"]["kind"], "input");
assert_eq!(
entry.item.history_text(),
old["entry"]["item"]["body"].as_str().unwrap()
);
let input = entry.metadata.input_metadata.as_ref().unwrap();
let usage = input.usage();
assert_eq!(usage.recording_material, InputEvidenceUse::Exclude);
assert_eq!(usage.memory_evidence, InputEvidenceUse::Exclude);
assert_eq!(usage.observation, InputObservationUse::Hidden);
let kind = old["entry"]["item"]["kind"].as_str().unwrap();
let schema = if kind == "journal_startup" {
assert_eq!(usage.compaction, InputCompactionUse::ReferenceInput);
assert_eq!(current["extensions"], old["extensions"]);
"journal:startup"
} else {
assert_eq!(usage.compaction, InputCompactionUse::Retain);
"subject:behavior"
};
let payload = input.feature_payload().unwrap();
assert_eq!(payload.schema(), schema);
assert_eq!(payload.version(), 1);
let mut expected_payload = old["entry"]["item"].clone();
expected_payload.as_object_mut().unwrap().remove("kind");
expected_payload.as_object_mut().unwrap().remove("body");
assert_eq!(
payload.decode::<Value>(schema, 1).unwrap(),
expected_payload
);
let public = serde_json::to_value(input.public_item().unwrap()).unwrap();
assert_eq!(public["kind"], kind);
assert_eq!(public["body"], old["entry"]["item"]["body"]);
assert_eq!(public["presentation"], json!({"kind": "context"}));
}
}
fn completed_v4_archive(
root: &Path,
schema: u32,
) -> (
SessionId,
SegmentId,
WorkerRetentionExecutionRequest,
WorkerRetentionExecutionResult,
) {
let (worker, session, segment) = stopped_worker(root, schema);
let provider = FsWorkerRetentionProvider::new(root);
let request = request(worker, SessionDisposition::Archive);
// Frozen old-runtime receipt: ingestion and removal completed before the
// current strict decoder existed. Do not ask the v5 reader to mint it.
let manifest =
commit_session_archive(&provider, &request, &provider.worker_dir(worker)).unwrap();
let completed = WorkerRetentionExecutionResult {
operation_id: request.operation_id.clone(),
input_fingerprint: request.input_fingerprint.clone(),
worker_id: worker,
session_disposition: SessionDisposition::Archive,
diagnostics_disposition: DiagnosticsDisposition::Purge,
archive: Some(manifest),
source_removed: true,
diagnostics_retained: false,
};
write_json(
&provider.operation_path(&request.operation_id).unwrap(),
&RetentionOperationReceipt {
schema_version: OPERATION_SCHEMA_VERSION,
request: request.clone(),
result: completed.clone(),
},
);
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
(session, segment, request, completed)
}
#[test]
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
for runtime_schema in [11, 12] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
let orphan = root
.join("orphaned-workers")
.join(WorkerId::now_v7().to_string());
let (orphan_session, orphan_segment) = write_v4_session(&orphan);
// Include an unregistered tree still under workers, not just an
// already-archived orphan. Both Runtime source schemas must fence it
// out of the live catalog without discarding its Session.
let loose_id = WorkerId::now_v7();
let loose = root.join("workers").join(loose_id.to_string());
let (loose_session, loose_segment) = write_v4_session(&loose);
let retained_loose = root.join("orphaned-workers").join(loose_id.to_string());
let (archived_session, archived_segment, request, completed) =
completed_v4_archive(&root, runtime_schema);
let provider = FsWorkerRetentionProvider::new(&root);
let archive = provider.archive_dir("archive-a").unwrap();
// Also cover an archive which already has a local storage attestation
// from an earlier schema migration. Keep ingestion bytes immutable;
// harmless marker whitespace makes current storage differ from them.
if runtime_schema == 12 {
let marker = archive.join("session/session.json");
let mut bytes = fs::read(&marker).unwrap();
bytes.push(b'\n');
fs::write(marker, bytes).unwrap();
let ingestion = completed.archive.clone().unwrap();
let (checksum_sha256, bytes, file_count) = checksum_files(
&collect_files(&archive.join("session"), "test").unwrap(),
"test",
)
.unwrap();
write_json(
&archive.join(ARCHIVE_STORAGE_FILE),
&MigratedArchiveStorage {
schema_version: 1,
ingestion,
checksum_sha256,
bytes,
file_count,
},
);
}
let storage_before = fs::read(archive.join(ARCHIVE_STORAGE_FILE)).ok();
let manifest_before = fs::read(archive.join("manifest.json")).unwrap();
let receipt_path = provider.operation_path(&request.operation_id).unwrap();
let receipt_before = fs::read(&receipt_path).unwrap();
let stopped = root.join("workers").join(worker.to_string());
assert!(WorkerSessionStore::open_read_only(stopped.join("session")).is_err());
// Current decode itself must not be made permissive to fix routing.
for line in V4_SYSTEM_INPUTS.lines() {
assert!(serde_json::from_str::<LogEntry>(line).is_err());
}
let before = tree_bytes(&root);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
let plan = FsRuntimeStore::migration_plan(&options).unwrap();
assert!(plan.migration_required);
assert_eq!(plan.archived_orphan_worker_count, 1);
assert_eq!(plan.current_schema_version, runtime_schema);
assert_eq!(tree_bytes(&root), before, "preflight mutated source");
// Exercise normal startup, not just the private migration helper.
let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap();
assert!(
matches!(
FsRuntimeStore::migrate(&options),
Err(RuntimeError::RuntimeStoreAlreadyOpen { .. })
),
"owner lock not fenced"
);
drop(opened);
assert_generic(&stopped.join("session"), session, segment);
assert_generic(&orphan.join("session"), orphan_session, orphan_segment);
assert!(!loose.exists());
assert_generic(
&retained_loose.join("session"),
loose_session,
loose_segment,
);
assert_generic(&archive.join("session"), archived_session, archived_segment);
assert_eq!(
fs::read(archive.join("manifest.json")).unwrap(),
manifest_before
);
assert_eq!(fs::read(&receipt_path).unwrap(), receipt_before);
let (manifest, archive_session) = provider.session_archive("archive-a").unwrap();
assert_eq!(completed.archive.as_ref(), Some(&manifest));
assert_eq!(provider.execute(&request).unwrap(), completed);
if let Some(storage_before) = storage_before {
assert_ne!(
fs::read(archive.join(ARCHIVE_STORAGE_FILE)).unwrap(),
storage_before
);
}
let storage: MigratedArchiveStorage =
read_json(&archive.join(ARCHIVE_STORAGE_FILE), "test").unwrap();
assert_eq!(storage.ingestion, manifest);
let (checksum, bytes, count) =
checksum_files(&collect_files(&archive_session, "test").unwrap(), "test").unwrap();
assert_eq!(
(storage.checksum_sha256, storage.bytes, storage.file_count),
(checksum, bytes, count)
);
if runtime_schema == 12 {
for name in [
PathBuf::from("runtime.json"),
PathBuf::from(format!("workers/{worker}/worker.json")),
PathBuf::from(format!("workers/{worker}/metadata.json")),
] {
assert_eq!(fs::read(root.join(&name)).unwrap(), before[&name]);
}
}
let once = tree_bytes(&root);
assert!(
!FsRuntimeStore::migrate(&options)
.unwrap()
.migration_required
);
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
// session-storage, not the immutable receipt's old checksum, attests
// current bytes. Tampering with those bytes must remain detectable.
let path = archive_session
.join("segments")
.join(format!("{archived_segment}.jsonl"));
fs::write(path, b"tampered\n").unwrap();
assert!(provider.session_archive("archive-a").is_err());
}
}
#[test]
fn runtime_v5_archive_failure_discards_staging_without_losing_originals_and_retry_converges() {
for runtime_schema in [11, 12] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
let (archived_session, archived_segment, request, completed) =
completed_v4_archive(&root, runtime_schema);
let provider = FsWorkerRetentionProvider::new(&root);
let archive = provider.archive_dir("archive-a").unwrap();
let path = archive
.join("session/segments")
.join(format!("{archived_segment}.jsonl"));
let original = fs::read(&path).unwrap();
fs::write(&path, b"corrupt archive\n").unwrap();
let before = tree_bytes(&root);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
// workers are visited before archives: earlier successful staged
// conversion must not leak into the original tree on this later error.
assert!(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").is_err());
assert_eq!(tree_bytes(&root), before);
assert!(!archive.join(ARCHIVE_STORAGE_FILE).exists());
fs::write(&path, original).unwrap();
FsRuntimeStore::migrate(&options).unwrap();
assert_generic(
&root
.join("workers")
.join(worker.to_string())
.join("session"),
session,
segment,
);
assert_generic(&archive.join("session"), archived_session, archived_segment);
assert_eq!(provider.execute(&request).unwrap(), completed);
}
}
#[test]
fn runtime_v5_typed_legacy_payload_failure_keeps_valid_archive_evidence_on_retries() {
for (from, to) in [
(
"\"snapshot_sha256\":\"snapshot-digest\"",
"\"snapshot_sha256\":42",
),
("\"behavior_md\":\"exact behavior\"", "\"behavior_md\":[]"),
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
runtime_fixture(&root, 12);
let (worker, _, segment) = stopped_worker(&root, 12);
let provider = FsWorkerRetentionProvider::new(&root);
let path = provider
.worker_dir(worker)
.join("session/segments")
.join(format!("{segment}.jsonl"));
let original = fs::read_to_string(&path).unwrap();
assert!(original.contains(from));
fs::write(&path, original.replace(from, to)).unwrap();
let request = request(worker, SessionDisposition::Archive);
let manifest =
commit_session_archive(&provider, &request, &provider.worker_dir(worker)).unwrap();
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
let archive = provider.archive_dir("archive-a").unwrap();
// This archive checksum is valid; failure must come from the legacy
// typed decoder, not from deliberately broken ingestion evidence.
validate_existing_archive(&archive, &manifest, false).unwrap();
let before = tree_bytes(&root);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
for _ in 0..2 {
let error = FsRuntimeStore::migrate(&options).unwrap_err();
let RuntimeError::StoreCorrupt {
operation, path, ..
} = error
else {
panic!("unexpected failure: {error}");
};
assert_eq!(operation, "migrate Session schema");
assert!(path.starts_with(temp.path().join(".runtime.schema-v12-staging")));
assert_eq!(tree_bytes(&root), before);
validate_existing_archive(&archive, &manifest, false).unwrap();
}
}
}
#[test]
fn runtime_v5_probe_and_staging_skip_erased_originals() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
let erased = root
.join("orphaned-workers")
.join(WorkerId::now_v7().to_string());
let (session, segment) = write_v4_session(&erased);
runtime_session_retention(&root)
.delete(
session,
session_store::retention::SessionDeletionIntent::ErasePermanently,
|| Ok(()),
)
.unwrap();
fs::write(
erased
.join("session/segments")
.join(format!("{segment}.jsonl")),
b"fenced malformed JSON\n",
)
.unwrap();
let before = tree_bytes(&erased);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
FsRuntimeStore::migrate(&options).unwrap();
assert_generic(
&root
.join("workers")
.join(worker.to_string())
.join("session"),
live_session,
live_segment,
);
assert_eq!(tree_bytes(&erased), before);
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
assert!(matches!(
runtime_session_retention(&root).ensure_live(session),
Err(session_store::StoreError::SessionErased(_))
));
}
#[test]
fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
for corruption in ["broken-json", "future-schema"] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let session_root = root
.join("workers")
.join(worker.to_string())
.join("session");
if corruption == "broken-json" {
fs::write(
session_root
.join("segments")
.join(format!("{segment}.jsonl")),
b"{not valid JSON\n",
)
.unwrap();
} else {
write_json(
&session_root.join("session.json"),
&json!({
"schema_version": 999, "session_id": session,
}),
);
}
let before = tree_bytes(&root);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
assert_eq!(tree_bytes(&root), before);
assert!(FsRuntimeStore::migrate(&options).is_err());
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
}
}
#[cfg(unix)]
#[test]
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, _, _) = runtime_fixture(&root, 12);
let before = tree_bytes(&root);
let runs = root.join("workers").join(worker.to_string()).join("runs");
fs::create_dir_all(runs.join("1")).unwrap();
let listener = std::os::unix::net::UnixListener::bind(runs.join("1/worker.sock")).unwrap();
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
let error = FsRuntimeStore::migrate(&options).unwrap_err();
assert!(error.to_string().contains("active Worker socket"));
drop(listener);
fs::remove_dir_all(runs).unwrap();
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
}
#[test]
fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializing_owner_evidence()
{
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let aggregate = root.join("workers").join(worker.to_string());
let path = aggregate
.join("session/segments")
.join(format!("{segment}.jsonl"));
let old = fs::read_to_string(&path).unwrap();
let header = old.lines().next().unwrap();
let source = include_str!(
"../../../session-store/src/legacy_session_log/tests/fixtures/v5-role-metadata.jsonl"
);
fs::write(&path, format!("{header}\n{source}")).unwrap();
write_json(
&aggregate.join("session/session.json"),
&json!({"schema_version":5, "session_id": session}),
);
let owner_before = fs::read(root.join("runtime.json")).unwrap();
let worker_before = fs::read(aggregate.join("worker.json")).unwrap();
let metadata_before = fs::read(aggregate.join("metadata.json")).unwrap();
assert!(WorkerSessionStore::open_read_only(aggregate.join("session")).is_err());
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
FsRuntimeStore::migrate(&options).unwrap();
let store = WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap();
let entries = store.read_all_read_only(session, segment).unwrap();
assert_eq!(entries.len(), 6);
for (current, old) in entries[1..].iter().zip(source.lines()) {
let current = serde_json::to_value(current).unwrap();
let old: Value = serde_json::from_str(old).unwrap();
assert_eq!(current["ts"], old["ts"]);
assert_eq!(current["entry"]["item"], old["entry"]["item"]);
for field in ["entry_id", "origin", "derivation"] {
assert_eq!(
current["entry"]["metadata"][field],
old["entry"]["metadata"][field]
);
}
if current["entry"]["metadata"]["origin"]["kind"] == "derived_summary" {
assert_eq!(
current["entry"]["metadata"]["input_metadata"]["usage"]["recording_material"],
"exclude"
);
assert_eq!(
current["entry"]["metadata"]["input_metadata"]["usage"]["memory_evidence"],
"exclude"
);
}
}
assert_eq!(fs::read(root.join("runtime.json")).unwrap(), owner_before);
assert_eq!(
fs::read(aggregate.join("worker.json")).unwrap(),
worker_before
);
assert_eq!(
fs::read(aggregate.join("metadata.json")).unwrap(),
metadata_before
);
let once = tree_bytes(&aggregate.join("session"));
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
FsRuntimeStore::migrate(&options).unwrap();
assert_eq!(tree_bytes(&aggregate.join("session")), once);
}
+1
View File
@@ -12422,6 +12422,7 @@ mod tests {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![protocol::SessionSnapshotEntry { entries: vec![protocol::SessionSnapshotEntry {
use_policy: None,
entry_id: "restored-log-entry".to_owned(), entry_id: "restored-log-entry".to_owned(),
timestamp: 1, timestamp: 1,
provenance: protocol::SessionEntryProvenance::LegacyUnknown, provenance: protocol::SessionEntryProvenance::LegacyUnknown,
@@ -398,6 +398,7 @@ mod tests {
content: vec![LoggedContentPart::Text { text: text.into() }], content: vec![LoggedContentPart::Text { text: text.into() }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.to_string()), entry_id: LoggedSessionHistoryEntryId(id.to_string()),
origin: LoggedSessionHistoryOrigin::HumanInput { origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".to_string(), account_id: "account-1".to_string(),
@@ -5030,6 +5030,7 @@ mod tests {
}], }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id, entry_id,
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
+3 -3
View File
@@ -1615,10 +1615,10 @@ where
if ordinary_subjektiv_features_enabled if ordinary_subjektiv_features_enabled
&& lifecycle_enabled && lifecycle_enabled
&& let Some(host) = subjektiv_host.as_ref() && let Some(host) = subjektiv_host.as_ref()
&& host.host.journal_startup_granted() && (host.host.journal_startup_granted() || host.host.journal_compaction_granted())
{ {
worker.install_initial_context_source(Arc::new( worker.install_session_context_source(Arc::new(
crate::feature::builtin::journal::JournalStartupContextSource::new( crate::feature::builtin::journal::JournalContextSource::new(
host.clone(), host.clone(),
worker.prompts(), worker.prompts(),
), ),
+10
View File
@@ -6,6 +6,16 @@
pub mod backend_job_result; pub mod backend_job_result;
pub(crate) mod bash; pub(crate) mod bash;
/// Host metadata participants are independent of model-facing Tool enablement.
/// Disabling a Feature must not preserve stale Session-bound authority evidence.
pub(crate) fn session_metadata_lifecycles()
-> Vec<std::sync::Arc<dyn crate::metadata_lifecycle::SessionMetadataLifecycle>> {
vec![std::sync::Arc::new(
crate::subjektiv_metadata::SessionAttributionMetadata,
)]
}
pub mod chat_invocation; pub mod chat_invocation;
pub mod drive; pub mod drive;
pub mod flow_transition; pub mod flow_transition;
+139 -22
View File
@@ -23,7 +23,7 @@ use crate::hook::{
use crate::subjektiv::{SubjektivHostConnection, SubjektivHostError}; use crate::subjektiv::{SubjektivHostConnection, SubjektivHostError};
use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
use async_trait::async_trait; use async_trait::async_trait;
use serde::Deserialize; use serde::{Deserialize, Serialize};
use serde_json::{Value, json}; use serde_json::{Value, json};
use std::sync::Arc; use std::sync::Arc;
use subjektiv::journal::transport::*; use subjektiv::journal::transport::*;
@@ -32,12 +32,73 @@ use subjektiv::journal::{
MAX_JOURNAL_RESPONSE_BYTES, MAX_JOURNAL_RESPONSE_BYTES,
}; };
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct JournalStartupInput {
pub snapshot: subjektiv::journal::startup::JournalStartupSnapshot,
pub snapshot_sha256: String,
pub prompt_provenance: session_store::PromptRenderProvenance,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
enum JournalReadStatus {
Available,
Absent,
Disabled,
Unavailable,
}
impl JournalReadStatus {
fn as_str(self) -> &'static str {
match self {
Self::Available => "available",
Self::Absent => "absent",
Self::Disabled => "disabled",
Self::Unavailable => "unavailable",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct JournalCompactionInput {
status: JournalReadStatus,
snapshot: Option<subjektiv::journal::startup::JournalStartupSnapshot>,
snapshot_sha256: Option<String>,
prompt_provenance: session_store::PromptRenderProvenance,
}
fn journal_input(
body: String,
payload: session_store::FeatureInputPayload,
kind: &str,
) -> Result<session_store::SystemInputEnvelope, String> {
use session_store::*;
let public = PublicSystemItem {
kind: kind.into(),
body: body.clone(),
presentation: PublicSystemPresentation::Context,
};
let metadata = HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::ReferenceInput,
},
Some(payload),
Some(public),
)
.map_err(|e| e.to_string())?;
SystemInputEnvelope::new(body, metadata).map_err(|e| e.to_string())
}
/// Subjektiv owns startup projection; Core only commits its typed initial items. /// Subjektiv owns startup projection; Core only commits its typed initial items.
pub(crate) struct JournalStartupContextSource { pub(crate) struct JournalContextSource {
host: SubjektivHostConnection, host: SubjektivHostConnection,
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>, prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
} }
impl JournalStartupContextSource { impl JournalContextSource {
pub(crate) fn new( pub(crate) fn new(
host: SubjektivHostConnection, host: SubjektivHostConnection,
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>, prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
@@ -46,9 +107,12 @@ impl JournalStartupContextSource {
} }
} }
#[async_trait] #[async_trait]
impl crate::worker::InitialContextSource for JournalStartupContextSource { impl crate::worker::SessionContextSource for JournalContextSource {
async fn load(&self) -> Result<Vec<session_store::LoggedSystemHistoryEntry>, String> { async fn load(&self) -> Result<Vec<session_store::LoggedSystemHistoryEntry>, String> {
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
if !self.host.host.journal_startup_granted() {
return Ok(Vec::new());
}
let snapshot = self let snapshot = self
.host .host
.journal_startup_context() .journal_startup_context()
@@ -58,8 +122,8 @@ impl crate::worker::InitialContextSource for JournalStartupContextSource {
let empty = snapshot.journals.is_empty() && !snapshot.omitted; let empty = snapshot.journals.is_empty() && !snapshot.omitted;
// Hash the same canonical JSON value retained by SystemItem, so replay // Hash the same canonical JSON value retained by SystemItem, so replay
// can verify provenance without depending on domain struct field order. // can verify provenance without depending on domain struct field order.
let snapshot = serde_json::to_value(snapshot).map_err(|e| e.to_string())?; let snapshot_value = serde_json::to_value(&snapshot).map_err(|e| e.to_string())?;
let snapshot_json = serde_json::to_string(&snapshot).map_err(|e| e.to_string())?; let snapshot_json = serde_json::to_string(&snapshot_value).map_err(|e| e.to_string())?;
if snapshot_json.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES { if snapshot_json.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES {
return Err("Host Journal startup snapshot exceeds independent byte budget".into()); return Err("Host Journal startup snapshot exceeds independent byte budget".into());
} }
@@ -74,9 +138,10 @@ impl crate::worker::InitialContextSource for JournalStartupContextSource {
return Err("Journal startup prompt exceeds rendered byte budget".into()); return Err("Journal startup prompt exceeds rendered byte budget".into());
} }
let projection = prompts.projection(); let projection = prompts.projection();
Ok(vec![session_store::LoggedSystemHistoryEntry { let payload = session_store::FeatureInputPayload::encode(
item: session_store::SystemItem::JournalStartup { "journal:startup",
body, 1,
&JournalStartupInput {
snapshot, snapshot,
snapshot_sha256: Sha256::digest(snapshot_json.as_bytes()) snapshot_sha256: Sha256::digest(snapshot_json.as_bytes())
.iter() .iter()
@@ -89,11 +154,38 @@ impl crate::worker::InitialContextSource for JournalStartupContextSource {
logical_name: "internal.subjektiv_journal_startup".into(), logical_name: "internal.subjektiv_journal_startup".into(),
}, },
}, },
metadata: crate::session_history::metadata( )
session_store::LoggedSessionHistoryOrigin::JournalContext, .map_err(|e| e.to_string())?;
Ok(vec![session_store::LoggedSystemHistoryEntry::from_input(
session_store::LoggedSessionHistoryOrigin::Provided {
producer: "subjektiv.journal".into(),
},
None, None,
), journal_input(body, payload, "journal_startup")?,
}]) )])
}
async fn load_compaction_context(
&self,
session: session_store::SessionId,
) -> Result<Vec<session_store::LoggedSystemHistoryEntry>, String> {
// A changed Session must never reuse the old source's Host ownership.
let connection = if self.host.context.session_id == session.to_string() {
Ok(Some(self.host.clone()))
} else {
Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"context source belongs to a different Session",
))
};
let input = freeze_compaction_journal(connection, &self.prompts.load_full())?;
Ok(vec![session_store::LoggedSystemHistoryEntry::from_input(
session_store::LoggedSessionHistoryOrigin::Provided {
producer: "subjektiv.journal".into(),
},
None,
input,
)])
} }
} }
@@ -103,7 +195,8 @@ impl crate::worker::InitialContextSource for JournalStartupContextSource {
pub(crate) fn freeze_compaction_journal( pub(crate) fn freeze_compaction_journal(
connection: std::io::Result<Option<SubjektivHostConnection>>, connection: std::io::Result<Option<SubjektivHostConnection>>,
prompts: &crate::prompt::catalog::PromptCatalog, prompts: &crate::prompt::catalog::PromptCatalog,
) -> Result<String, crate::prompt::catalog::CatalogError> { ) -> Result<session_store::SystemInputEnvelope, String> {
use JournalReadStatus::*;
let workspace_id = connection.as_ref().ok().and_then(|connection| { let workspace_id = connection.as_ref().ok().and_then(|connection| {
connection connection
.as_ref() .as_ref()
@@ -116,24 +209,25 @@ pub(crate) fn freeze_compaction_journal(
Ok(_) => None, Ok(_) => None,
Err(_) => Some(Err(())), Err(_) => Some(Err(())),
}; };
let snapshot = read.as_ref().and_then(|read| read.as_ref().ok()).cloned();
let (status, json) = match read { let (status, json) = match read {
None => ("disabled", String::new()), None => (Disabled, String::new()),
Some(Ok(snapshot)) => { Some(Ok(snapshot)) => {
let status = if snapshot.journals.is_empty() { let status = if snapshot.journals.is_empty() {
"absent" Absent
} else { } else {
"available" Available
}; };
match serde_json::to_string(&snapshot) { match serde_json::to_string(&snapshot) {
Ok(json) if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES => { Ok(json) if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES => {
(status, json) (status, json)
} }
_ => ("unavailable", String::new()), _ => (Unavailable, String::new()),
} }
} }
Some(Err(())) => ("unavailable", String::new()), Some(Err(())) => (Unavailable, String::new()),
}; };
if status == "unavailable" { if status == Unavailable {
tracing::warn!("Journal compaction snapshot unavailable; continuing without Journal"); tracing::warn!("Journal compaction snapshot unavailable; continuing without Journal");
} }
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
@@ -143,12 +237,35 @@ pub(crate) fn freeze_compaction_journal(
.map(|byte| format!("{byte:02x}")) .map(|byte| format!("{byte:02x}"))
.collect() .collect()
}); });
prompts.subjektiv_journal_compaction( let body = prompts
status, .subjektiv_journal_compaction(
status.as_str(),
&json, &json,
snapshot_sha256.as_deref(), snapshot_sha256.as_deref(),
workspace_id.as_deref(), workspace_id.as_deref(),
) )
.map_err(|e| e.to_string())?;
let payload = session_store::FeatureInputPayload::encode(
"journal:compaction",
1,
&JournalCompactionInput {
status,
prompt_provenance: session_store::PromptRenderProvenance {
workspace_id,
source_digest: prompts.projection().source_digest.clone(),
projection_digest: prompts.projection().catalog_digest.clone(),
logical_name: "internal.subjektiv_journal_compaction".into(),
},
snapshot: if status == Unavailable {
None
} else {
snapshot
},
snapshot_sha256,
},
)
.map_err(|e| e.to_string())?;
journal_input(body, payload, "journal_compaction")
} }
#[derive(Clone)] #[derive(Clone)]
+194 -45
View File
@@ -43,6 +43,80 @@ use crate::worker::{
WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod,
}; };
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct SubjectBehaviorInput {
pub behavior_md: String,
pub prompt_provenance: Option<session_store::PromptRenderProvenance>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct ResidentSummaryInput {
prompt_provenance: Option<session_store::PromptRenderProvenance>,
}
fn resident_input(
body: String,
schema: &str,
payload: &impl serde::Serialize,
kind: &str,
) -> Result<session_store::LoggedSystemHistoryEntry, session_store::InputMetadataError> {
use session_store::*;
let input = SystemInputEnvelope::new(
body.clone(),
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::Retain,
},
Some(FeatureInputPayload::encode(schema, 1, payload)?),
Some(PublicSystemItem {
kind: kind.into(),
body,
presentation: PublicSystemPresentation::Context,
}),
)?,
)?;
Ok(LoggedSystemHistoryEntry::from_input(
LoggedSessionHistoryOrigin::Provided {
producer: "subjektiv.resident-context".into(),
},
None,
input,
))
}
pub(crate) fn subject_behavior_input(
body: String,
behavior_md: String,
prompt_provenance: Option<session_store::PromptRenderProvenance>,
) -> Result<session_store::LoggedSystemHistoryEntry, session_store::InputMetadataError> {
resident_input(
body,
"subject:behavior",
&SubjectBehaviorInput {
behavior_md,
prompt_provenance,
},
"subject_behavior_refresh",
)
}
pub(crate) fn resident_summary_input(
body: String,
prompt_provenance: Option<session_store::PromptRenderProvenance>,
) -> Result<session_store::LoggedSystemHistoryEntry, session_store::InputMetadataError> {
resident_input(
body,
"memory:resident-summary",
&ResidentSummaryInput { prompt_provenance },
"resident_summary_refresh",
)
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct WorkspaceHttpMemoryBackend { pub struct WorkspaceHttpMemoryBackend {
client: Arc<dyn WorkspaceClient>, client: Arc<dyn WorkspaceClient>,
@@ -651,7 +725,7 @@ struct SubjectBehaviorRefreshState {
struct WorkspaceSubjektivResidentSummarySource { struct WorkspaceSubjektivResidentSummarySource {
host: SubjektivHostConnection, host: SubjektivHostConnection,
prompts: Arc<ArcSwap<PromptCatalog>>, prompts: Arc<ArcSwap<PromptCatalog>>,
behavior_state: Mutex<SubjectBehaviorRefreshState>, behavior_state: Arc<Mutex<SubjectBehaviorRefreshState>>,
} }
fn resident_summary_contribution( fn resident_summary_contribution(
@@ -691,6 +765,13 @@ impl SystemPromptContributionSource for WorkspaceResidentSummarySource {
} }
impl WorkspaceSubjektivResidentSummarySource { impl WorkspaceSubjektivResidentSummarySource {
fn confirm_resident_context_behavior(&self, behavior_md: &str) {
self.behavior_state
.lock()
.expect("subject behavior state poisoned")
.applied = Some(behavior_md.to_owned());
}
async fn fetch( async fn fetch(
&self, &self,
) -> Result<server_api::SubjektivResidentContextOutput, SubjektivHostError> { ) -> Result<server_api::SubjektivResidentContextOutput, SubjektivHostError> {
@@ -707,7 +788,7 @@ impl WorkspaceSubjektivResidentSummarySource {
fn render( fn render(
&self, &self,
output: &server_api::SubjektivResidentContextOutput, output: &server_api::SubjektivResidentContextOutput,
) -> Result<String, String> { ) -> Result<(String, session_store::PromptRenderProvenance), String> {
use memory::backend::MemoryResidentSummaryAvailability as Availability; use memory::backend::MemoryResidentSummaryAvailability as Availability;
let availability = match output.memory_surface.availability { let availability = match output.memory_surface.availability {
Availability::Ready => "ready", Availability::Ready => "ready",
@@ -716,53 +797,68 @@ impl WorkspaceSubjektivResidentSummarySource {
Availability::Failed => "failed", Availability::Failed => "failed",
Availability::Unavailable => "unavailable", Availability::Unavailable => "unavailable",
}; };
self.prompts let prompts = self.prompts.load_full();
.load() let body = prompts
.subjektiv_resident_context( .subjektiv_resident_context(
&output.behavior_md, &output.behavior_md,
availability, availability,
output.memory_surface.content.as_deref(), output.memory_surface.content.as_deref(),
) )
.map_err(|error| error.to_string()) .map_err(|error| error.to_string())?;
} let projection = prompts.projection();
Ok((
fn remember_behavior(&self, output: &server_api::SubjektivResidentContextOutput) { body,
let mut state = self session_store::PromptRenderProvenance {
.behavior_state workspace_id: self.host.host.workspace_id(),
.lock() source_digest: projection.source_digest.clone(),
.expect("subject behavior state poisoned"); projection_digest: projection.catalog_digest.clone(),
state.applied = Some(output.behavior_md.clone()); logical_name: "internal.subjektiv_resident_context".into(),
},
))
} }
} }
/// Detect behavior changes against the state reflected in history, not the fetched state. /// Detect behavior changes against the state reflected in history, not the fetched state.
/// ///
/// Initial load builds a prompt contribution; change refresh does not advance applied on fetch/render alone. /// Fetch/render never advances applied state. Initial preparation captures a
/// The interceptor confirms after durably appending a system item, allowing refetch if append fails. /// source-local completion, invoked only after its prompt and active pointer commit.
/// Invalidate applied before rewrite so even identical bodies can be presented in the new history. /// Later refreshes are acknowledged by the interceptor after durable append.
/// Invalidate applied before rewrite so identical bodies can be represented again.
#[async_trait] #[async_trait]
impl SystemPromptContributionSource for WorkspaceSubjektivResidentSummarySource { impl SystemPromptContributionSource for WorkspaceSubjektivResidentSummarySource {
async fn load(&self) -> SystemPromptContribution { async fn load(&self) -> SystemPromptContribution {
self.load_initial().await.0
}
async fn load_initial(
&self,
) -> (
SystemPromptContribution,
Option<crate::worker::PromptContributionCompletion>,
) {
match self.fetch().await { match self.fetch().await {
Ok(output) => match self.render(&output) { Ok(output) => match self.render(&output) {
Ok(rendered) => { Ok((rendered, _)) => {
self.remember_behavior(&output); let state = self.behavior_state.clone();
SystemPromptContribution::Ready(rendered) let behavior = output.behavior_md;
(
SystemPromptContribution::Ready(rendered),
Some(Box::new(move || {
state
.lock()
.expect("subject behavior state poisoned")
.applied = Some(behavior);
})),
)
} }
Err(error) => { Err(error) => {
tracing::error!(%error, "subject resident context render failed"); tracing::error!(%error, "subject resident context render failed");
SystemPromptContribution::Ready( (SystemPromptContribution::Ready("The Host could not render the Subject resident context. Do not treat this as unset behavior or infer replacement behavior from Memory.".into()), None)
"The Host could not render the Subject resident context. Do not treat this as unset behavior or infer replacement behavior from Memory."
.to_string(),
)
} }
}, },
Err(error) => { Err(error) => {
tracing::error!(%error, "subject resident context fetch failed"); tracing::error!(%error, "subject resident context fetch failed");
SystemPromptContribution::Ready( (SystemPromptContribution::Ready("The Host could not fetch the Subject resident context. Do not treat this as unset behavior or infer replacement behavior from Memory.".into()), None)
"The Host could not fetch the Subject resident context. Do not treat this as unset behavior or infer replacement behavior from Memory."
.to_string(),
)
} }
} }
} }
@@ -781,18 +877,31 @@ impl SystemPromptContributionSource for WorkspaceSubjektivResidentSummarySource
if !changed { if !changed {
return Ok(None); return Ok(None);
} }
let rendered = self.render(&output)?; let (rendered, prompt_provenance) = self.render(&output)?;
Ok(Some(ResidentContextRefresh { Ok(Some(ResidentContextRefresh {
body: rendered, body: rendered,
behavior_md: output.behavior_md, behavior_md: output.behavior_md,
prompt_provenance: Some(prompt_provenance),
})) }))
} }
fn confirm_resident_context_behavior(&self, behavior_md: &str) { fn system_item_committed(
self.behavior_state &self,
.lock() _item: &session_store::SystemItem,
.expect("subject behavior state poisoned") metadata: &crate::session_history::SessionHistoryMetadata,
.applied = Some(behavior_md.to_owned()); ) {
if let Some(input) = metadata
.input_metadata
.as_ref()
.and_then(|input| input.feature_payload())
.and_then(|payload| {
payload
.decode::<SubjectBehaviorInput>("subject:behavior", 1)
.ok()
})
{
self.confirm_resident_context_behavior(&input.behavior_md);
}
} }
fn invalidate_resident_context_representation(&self) { fn invalidate_resident_context_representation(&self) {
@@ -944,7 +1053,13 @@ impl Hook<PreLlmRequest> for SubjektivBehaviorRefreshHook {
"Subject behavior refresh requires durable session append authority", "Subject behavior refresh requires durable session append authority",
) )
})?; })?;
system_items.append_subject_behavior_refresh(refresh.body, refresh.behavior_md); let input = subject_behavior_input(
refresh.body,
refresh.behavior_md,
refresh.prompt_provenance,
)
.map_err(|error| HookError::new(HookErrorCategory::Dependency, error.to_string()))?;
system_items.append_input(input);
} }
Ok(HookPreRequestAction::Continue) Ok(HookPreRequestAction::Continue)
} }
@@ -969,16 +1084,18 @@ impl Hook<WorkerRestored> for SubjektivResidentRestoreRefreshHook {
.resident_memory_restore_section(surface_ready, summary) .resident_memory_restore_section(surface_ready, summary)
.map_err(|error| HookError::new(HookErrorCategory::Internal, error.to_string()))?; .map_err(|error| HookError::new(HookErrorCategory::Internal, error.to_string()))?;
let projection = prompts.projection(); let projection = prompts.projection();
context.system_items().append_resident_summary_refresh( let input = resident_summary_input(
body, body,
session_store::PromptRenderProvenance { Some(session_store::PromptRenderProvenance {
workspace_id: self.workspace_id.clone(), workspace_id: self.workspace_id.clone(),
source_digest: projection.source_digest.clone(), source_digest: projection.source_digest.clone(),
projection_digest: projection.catalog_digest.clone(), projection_digest: projection.catalog_digest.clone(),
logical_name: WorkerPrompt::ResidentMemoryRestoreSection.key().to_string(), logical_name: WorkerPrompt::ResidentMemoryRestoreSection.key().to_string(),
}, }),
); )
.map_err(|error| HookError::new(HookErrorCategory::Dependency, error.to_string()))?;
context.system_items().append_input(input);
Ok(()) Ok(())
} }
} }
@@ -1106,7 +1223,7 @@ pub(crate) fn ordinary_subjektiv_resident_summary_source(
Ok(Some(Arc::new(WorkspaceSubjektivResidentSummarySource { Ok(Some(Arc::new(WorkspaceSubjektivResidentSummarySource {
host: host.expect("checked explicit connection"), host: host.expect("checked explicit connection"),
prompts, prompts,
behavior_state: Mutex::new(SubjectBehaviorRefreshState::default()), behavior_state: Arc::new(Mutex::new(SubjectBehaviorRefreshState::default())),
}))) })))
} }
@@ -1723,8 +1840,16 @@ permission = "write"
.unwrap() .unwrap()
.unwrap(); .unwrap();
let initial = source.load().await; let (initial, completion) = source.load_initial().await;
assert!(matches!(initial, SystemPromptContribution::Ready(_))); assert!(matches!(initial, SystemPromptContribution::Ready(_)));
assert!(
source
.load_changed_resident_context()
.await
.unwrap()
.is_some()
);
completion.expect("successful initial contribution must provide completion")();
assert!( assert!(
source source
.load_changed_resident_context() .load_changed_resident_context()
@@ -1741,7 +1866,13 @@ permission = "write"
.expect("changed behavior must produce one refresh"); .expect("changed behavior must produce one refresh");
assert_eq!(refresh.behavior_md, "Ask when uncertain."); assert_eq!(refresh.behavior_md, "Ask when uncertain.");
assert!(refresh.body.contains("Ask when uncertain.")); assert!(refresh.body.contains("Ask when uncertain."));
source.confirm_resident_context_behavior(&refresh.behavior_md); let input = subject_behavior_input(
refresh.body.clone(),
refresh.behavior_md.clone(),
refresh.prompt_provenance.clone(),
)
.unwrap();
source.system_item_committed(&input.item, &input.metadata);
assert!( assert!(
source source
.load_changed_resident_context() .load_changed_resident_context()
@@ -1758,7 +1889,13 @@ permission = "write"
.expect("cleared behavior must produce one refresh"); .expect("cleared behavior must produce one refresh");
assert_eq!(cleared.behavior_md, ""); assert_eq!(cleared.behavior_md, "");
assert!(cleared.body.contains("No user-managed behavior is set")); assert!(cleared.body.contains("No user-managed behavior is set"));
source.confirm_resident_context_behavior(&cleared.behavior_md); let input = subject_behavior_input(
cleared.body.clone(),
cleared.behavior_md.clone(),
cleared.prompt_provenance.clone(),
)
.unwrap();
source.system_item_committed(&input.item, &input.metadata);
assert!( assert!(
source source
.load_changed_resident_context() .load_changed_resident_context()
@@ -1783,7 +1920,7 @@ permission = "write"
// target-side bounded resident request contract. // target-side bounded resident request contract.
assert_eq!( assert_eq!(
client.timeouts.lock().unwrap().as_slice(), client.timeouts.lock().unwrap().as_slice(),
&[RESIDENT_SUMMARY_REQUEST_TIMEOUT; 7] &[RESIDENT_SUMMARY_REQUEST_TIMEOUT; 8]
); );
} }
@@ -1816,7 +1953,13 @@ permission = "write"
.await .await
.unwrap() .unwrap()
.unwrap(); .unwrap();
source.confirm_resident_context_behavior(&first.behavior_md); let input = subject_behavior_input(
first.body.clone(),
first.behavior_md.clone(),
first.prompt_provenance.clone(),
)
.unwrap();
source.system_item_committed(&input.item, &input.metadata);
assert_eq!( assert_eq!(
source source
.load_changed_resident_context() .load_changed_resident_context()
@@ -1825,7 +1968,13 @@ permission = "write"
.unwrap(), .unwrap(),
second second
); );
source.confirm_resident_context_behavior(&second.behavior_md); let input = subject_behavior_input(
second.body.clone(),
second.behavior_md.clone(),
second.prompt_provenance.clone(),
)
.unwrap();
source.system_item_committed(&input.item, &input.metadata);
assert!( assert!(
source source
.load_changed_resident_context() .load_changed_resident_context()
@@ -371,7 +371,7 @@ fn parse_entry_ref(value: &str) -> Result<SessionEntryRef, ToolError> {
fn parse_kind(value: &str) -> Result<ReferenceKind, ToolError> { fn parse_kind(value: &str) -> Result<ReferenceKind, ToolError> {
ReferenceKind::parse(value).ok_or_else(|| { ReferenceKind::parse(value).ok_or_else(|| {
ToolError::InvalidArgument(format!( ToolError::InvalidArgument(format!(
"invalid kind {value:?}; expected user, assistant, or tool" "invalid kind {value:?}; expected user, assistant, system, or tool"
)) ))
}) })
} }
@@ -276,7 +276,9 @@ mod tests {
use super::*; use super::*;
use crate::hook::{PreRequestInfo, SystemItemAppendHandle}; use crate::hook::{PreRequestInfo, SystemItemAppendHandle};
fn pre_request_context(pending: Arc<Mutex<Vec<SystemItem>>>) -> PreRequestContext { fn pre_request_context(
pending: Arc<Mutex<Vec<session_store::LoggedSystemHistoryEntry>>>,
) -> PreRequestContext {
PreRequestContext::new( PreRequestContext::new(
PreRequestInfo { PreRequestInfo {
item_count: 1, item_count: 1,
@@ -315,7 +317,7 @@ mod tests {
let queued = pending.lock().expect("pending queue poisoned"); let queued = pending.lock().expect("pending queue poisoned");
assert_eq!(queued.len(), 1); assert_eq!(queued.len(), 1);
let SystemItem::TaskReminder { body, .. } = &queued[0] else { let SystemItem::TaskReminder { body, .. } = &queued[0].item else {
panic!("unexpected system item: {:?}", queued[0]); panic!("unexpected system item: {:?}", queued[0]);
}; };
assert!(body.starts_with("Current session steps are listed below.")); assert!(body.starts_with("Current session steps are listed below."));
@@ -339,7 +341,7 @@ mod tests {
} }
let queued = pending.lock().expect("pending queue poisoned"); let queued = pending.lock().expect("pending queue poisoned");
let SystemItem::TaskReminder { source, body, .. } = &queued[0] else { let SystemItem::TaskReminder { source, body, .. } = &queued[0].item else {
panic!("unexpected system item: {:?}", queued[0]); panic!("unexpected system item: {:?}", queued[0]);
}; };
assert_eq!(*source, SystemReminderSource::TaskInactivity); assert_eq!(*source, SystemReminderSource::TaskInactivity);
@@ -452,7 +454,7 @@ mod tests {
} }
let queued = pending.lock().expect("pending queue poisoned"); let queued = pending.lock().expect("pending queue poisoned");
assert_eq!(queued.len(), 1); assert_eq!(queued.len(), 1);
let SystemItem::TaskReminder { body, .. } = &queued[0] else { let SystemItem::TaskReminder { body, .. } = &queued[0].item else {
panic!("unexpected system item: {:?}", queued[0]); panic!("unexpected system item: {:?}", queued[0]);
}; };
assert!(body.contains("taskid 1 (completed) done")); assert!(body.contains("taskid 1 (completed) done"));
@@ -813,6 +813,7 @@ mod tests {
_ => return None, _ => return None,
}; };
Some(protocol::SessionSnapshotEntry { Some(protocol::SessionSnapshotEntry {
use_policy: None,
entry_id: format!("fake-{index:08}"), entry_id: format!("fake-{index:08}"),
timestamp: index as u64, timestamp: index as u64,
provenance: protocol::SessionEntryProvenance::LegacyUnknown, provenance: protocol::SessionEntryProvenance::LegacyUnknown,
+39
View File
@@ -39,6 +39,45 @@ const MAX_INVOCATION_DESCRIPTOR_BYTES: usize =
protocol::invocation::MAX_FEATURE_INVOCATION_DESCRIPTOR_BYTES; protocol::invocation::MAX_FEATURE_INVOCATION_DESCRIPTOR_BYTES;
const MAX_COMPLETION_RESPONSE_BYTES: usize = 64 * 1024; const MAX_COMPLETION_RESPONSE_BYTES: usize = 64 * 1024;
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct InvocationResultInput {
pub result: FeatureInvocationResult,
}
pub(crate) fn result_input(
result: FeatureInvocationResult,
body: String,
) -> Result<session_store::LoggedSystemHistoryEntry, session_store::InputMetadataError> {
use session_store::*;
let payload = FeatureInputPayload::encode(
"feature:invocation-result",
1,
&InvocationResultInput { result },
)?;
let metadata = HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::Retain,
},
Some(payload),
Some(PublicSystemItem {
kind: "feature_invocation_result".into(),
body: body.clone(),
presentation: PublicSystemPresentation::Context,
}),
)?;
Ok(LoggedSystemHistoryEntry::from_input(
LoggedSessionHistoryOrigin::Provided {
producer: "feature.invocation".into(),
},
None,
SystemInputEnvelope::new(body, metadata)?,
))
}
pub(crate) fn invocation_result_is_bounded(result: &FeatureInvocationResult) -> bool { pub(crate) fn invocation_result_is_bounded(result: &FeatureInvocationResult) -> bool {
result.message.len() <= MAX_INVOCATION_MESSAGE_BYTES result.message.len() <= MAX_INVOCATION_MESSAGE_BYTES
&& result && result
+37 -43
View File
@@ -25,7 +25,9 @@ use agen::tool::{ToolOutput, ToolResult};
use async_trait::async_trait; use async_trait::async_trait;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use session_store::{SystemItem, SystemReminder}; #[cfg(test)]
use session_store::SystemItem;
use session_store::SystemReminder;
use thiserror::Error; use thiserror::Error;
use crate::SessionEntryRef; use crate::SessionEntryRef;
@@ -254,7 +256,7 @@ impl From<HookTurnEndAction> for TurnEndAction {
// Hook context handles // Hook context handles
// ============================================================================= // =============================================================================
/// Host-created handle for appending approved durable [`SystemItem`] requests. /// Host-created handle for appending approved durable [`session_store::SystemItem`] requests.
/// ///
/// Hook code can use this handle only when the Worker host includes it in an /// Hook code can use this handle only when the Worker host includes it in an
/// event-specific context. The handle queues typed requests; the host drains the /// event-specific context. The handle queues typed requests; the host drains the
@@ -263,11 +265,11 @@ impl From<HookTurnEndAction> for TurnEndAction {
/// raw `agen::Item`, history writer, event sender, `Worker`, `Engine`, or /// raw `agen::Item`, history writer, event sender, `Worker`, `Engine`, or
/// notification buffer. /// notification buffer.
pub struct SystemItemAppendHandle { pub struct SystemItemAppendHandle {
pending: Arc<Mutex<Vec<SystemItem>>>, pending: Arc<Mutex<Vec<session_store::LoggedSystemHistoryEntry>>>,
} }
impl SystemItemAppendHandle { impl SystemItemAppendHandle {
pub(crate) fn new(pending: Arc<Mutex<Vec<SystemItem>>>) -> Self { pub(crate) fn new(pending: Arc<Mutex<Vec<session_store::LoggedSystemHistoryEntry>>>) -> Self {
Self { pending } Self { pending }
} }
@@ -279,53 +281,44 @@ impl SystemItemAppendHandle {
self.pending self.pending
.lock() .lock()
.expect("system-item append queue poisoned") .expect("system-item append queue poisoned")
.push(item); .push(session_store::LoggedSystemHistoryEntry {
item,
metadata: crate::session_history::metadata(
session_store::LoggedSessionHistoryOrigin::BackendInstruction {
operation_id: None,
},
None,
),
});
} }
/// Queue a Subject behavior refresh with its exact represented behavior. The caller receives this /// Queue producer-validated System context. Metadata and body are committed together.
/// narrow operation only at a pre-request boundary, and the host commits it /// The Host exposes this capability only at a pre-request boundary.
/// before exposing it to the model. pub fn append_input(&self, input: session_store::LoggedSystemHistoryEntry) {
pub fn append_subject_behavior_refresh(
&self,
body: impl Into<String>,
behavior_md: impl Into<String>,
) {
self.pending self.pending
.lock() .lock()
.expect("system-item append queue poisoned") .expect("system-item append queue poisoned")
.push(SystemItem::SubjectBehaviorRefresh { .push(input);
body: body.into(),
behavior_md: behavior_md.into(),
prompt_provenance: None,
});
} }
} }
/// Host-created narrow append authority exposed only at a restore lifecycle /// Host-created narrow append authority exposed only at a restore lifecycle
/// boundary. It cannot emit arbitrary system items. /// boundary. No model-facing tool receives this capability.
pub struct RestoreSystemItemAppendHandle { pub struct RestoreSystemItemAppendHandle {
pending: Arc<Mutex<Vec<SystemItem>>>, pending: Arc<Mutex<Vec<session_store::LoggedSystemHistoryEntry>>>,
} }
impl RestoreSystemItemAppendHandle { impl RestoreSystemItemAppendHandle {
pub(crate) fn new(pending: Arc<Mutex<Vec<SystemItem>>>) -> Self { pub(crate) fn new(pending: Arc<Mutex<Vec<session_store::LoggedSystemHistoryEntry>>>) -> Self {
Self { pending } Self { pending }
} }
/// Queue a Feature-rendered resident-summary refresh for durable append at /// Queue producer-validated System context at the Host restore boundary.
/// the restore lifecycle boundary. pub fn append_input(&self, input: session_store::LoggedSystemHistoryEntry) {
pub fn append_resident_summary_refresh(
&self,
body: impl Into<String>,
prompt_provenance: session_store::PromptRenderProvenance,
) {
self.pending self.pending
.lock() .lock()
.expect("restore system-item append queue poisoned") .expect("restore system-item append queue poisoned")
.push(SystemItem::ResidentSummaryRefresh { .push(input);
body: body.into(),
prompt_provenance: Some(prompt_provenance),
});
} }
} }
@@ -983,7 +976,7 @@ mod tests {
let queued = pending.lock().expect("pending queue poisoned"); let queued = pending.lock().expect("pending queue poisoned");
assert_eq!(queued.len(), 1); assert_eq!(queued.len(), 1);
match &queued[0] { match &queued[0].item {
SystemItem::TaskReminder { body, .. } => { SystemItem::TaskReminder { body, .. } => {
assert_eq!(body, "remember tasks"); assert_eq!(body, "remember tasks");
} }
@@ -992,7 +985,7 @@ mod tests {
} }
#[test] #[test]
fn restore_append_handle_queues_only_resident_refresh_items() { fn restore_append_handle_preserves_producer_metadata() {
let pending = Arc::new(Mutex::new(Vec::new())); let pending = Arc::new(Mutex::new(Vec::new()));
let handle = RestoreSystemItemAppendHandle::new(Arc::clone(&pending)); let handle = RestoreSystemItemAppendHandle::new(Arc::clone(&pending));
let provenance = session_store::PromptRenderProvenance { let provenance = session_store::PromptRenderProvenance {
@@ -1003,15 +996,16 @@ mod tests {
logical_name: "internal.resident_memory_restore_section".into(), logical_name: "internal.resident_memory_restore_section".into(),
}; };
handle.append_resident_summary_refresh("current surface", provenance.clone()); let input = crate::feature::builtin::memory::resident_summary_input(
"current surface".into(),
assert!(matches!( Some(provenance),
pending.lock().unwrap().as_slice(), )
[SystemItem::ResidentSummaryRefresh { .unwrap();
body, let expected = serde_json::to_value(&input).unwrap();
prompt_provenance: Some(actual), handle.append_input(input);
}] if body == "current surface" && actual == &provenance let queued = pending.lock().unwrap();
)); assert_eq!(queued.len(), 1);
assert_eq!(serde_json::to_value(&queued[0]).unwrap(), expected);
} }
#[test] #[test]
+1
View File
@@ -1499,6 +1499,7 @@ permission = "write"
is_error: false, is_error: false,
}, },
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId(entry_id.into()), entry_id: session_store::LoggedSessionHistoryEntryId(entry_id.into()),
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown, origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
+154 -36
View File
@@ -216,15 +216,6 @@ impl WorkerInterceptor {
self self
} }
fn confirm_resident_context_item(&self, item: &SystemItem) {
let Some(source) = self.resident_context_source.as_ref() else {
return;
};
if let SystemItem::SubjectBehaviorRefresh { behavior_md, .. } = item {
source.confirm_resident_context_behavior(behavior_md);
}
}
/// Commit each `SystemItem` as its own `LogEntry::AnnotatedSystemItem` /// Commit each `SystemItem` as its own `LogEntry::AnnotatedSystemItem`
/// entry through the attached writer (no-op when no writer is /// entry through the attached writer (no-op when no writer is
/// wired). Sync — writes complete before the matching /// wired). Sync — writes complete before the matching
@@ -243,16 +234,20 @@ impl WorkerInterceptor {
return Ok(()); return Ok(());
}; };
for (item, extensions, history_provenance) in items { for (item, extensions, history_provenance) in items {
let entry = writer.commit_system_item_with_extensions( writer.commit_system_item_with_extensions_then(
item.clone(), item.clone(),
extensions.clone(), extensions.clone(),
history_provenance.clone(), history_provenance.clone(),
)?; &mut |entry| {
self.pending_committed_history self.pending_committed_history
.lock() .lock()
.expect("pending committed history poisoned") .expect("pending committed history poisoned")
.push_back(entry); .push_back(entry.clone());
self.confirm_resident_context_item(item); if let Some(source) = self.resident_context_source.as_ref() {
source.system_item_committed(item, &entry.annotation);
}
},
)?;
} }
Ok(()) Ok(())
} }
@@ -267,6 +262,31 @@ impl WorkerInterceptor {
) )
} }
fn commit_system_entries(
&self,
entries: &[session_store::LoggedSystemHistoryEntry],
) -> Result<(), session_store::StoreError> {
let Some(writer) = &self.log_writer else {
return Ok(());
};
for entry in entries {
writer.commit_annotated_system_item_then(
entry.clone(),
Vec::new(),
&mut |history| {
self.pending_committed_history
.lock()
.expect("pending committed history poisoned")
.push_back(history.clone());
if let Some(source) = &self.resident_context_source {
source.system_item_committed(&entry.item, &history.annotation);
}
},
)?;
}
Ok(())
}
fn current_turn_index(&self) -> usize { fn current_turn_index(&self) -> usize {
self.next_turn_index self.next_turn_index
.load(Ordering::Relaxed) .load(Ordering::Relaxed)
@@ -392,11 +412,6 @@ impl WorkerInterceptor {
} if prompt_provenance.is_none() => { } if prompt_provenance.is_none() => {
*prompt_provenance = Some(provenance("internal.task_reminder")); *prompt_provenance = Some(provenance("internal.task_reminder"));
} }
SystemItem::SubjectBehaviorRefresh {
prompt_provenance, ..
} if prompt_provenance.is_none() => {
*prompt_provenance = Some(provenance("internal.subjektiv_resident_context"));
}
SystemItem::Interrupt { SystemItem::Interrupt {
prompt_provenance, .. prompt_provenance, ..
} if prompt_provenance.is_none() => { } if prompt_provenance.is_none() => {
@@ -622,15 +637,17 @@ impl Interceptor<SessionHistoryMetadata> for WorkerInterceptor {
return Ok(PreRequestAction::Yield); return Ok(PreRequestAction::Yield);
} }
let mut system_items: Vec<SystemItem> = std::mem::take( let mut system_items: Vec<session_store::LoggedSystemHistoryEntry> = std::mem::take(
&mut *pending_hook_system_items &mut *pending_hook_system_items
.lock() .lock()
.expect("pending hook system-item queue poisoned"), .expect("pending hook system-item queue poisoned"),
); );
self.attach_prompt_provenance(&mut system_items); for entry in &mut system_items {
self.attach_prompt_provenance(std::slice::from_mut(&mut entry.item));
}
let mut appended_items: Vec<Item> = system_items let mut appended_items: Vec<Item> = system_items
.iter() .iter()
.map(SystemItem::to_history_item) .map(|entry| entry.item.to_history_item())
.collect(); .collect();
let effective_context = if appended_items.is_empty() { let effective_context = if appended_items.is_empty() {
Cow::Borrowed(context.as_slice()) Cow::Borrowed(context.as_slice())
@@ -644,7 +661,7 @@ impl Interceptor<SessionHistoryMetadata> for WorkerInterceptor {
let compact_decision = let compact_decision =
self.request_compact_decision(current_tokens, effective_context.as_ref()); self.request_compact_decision(current_tokens, effective_context.as_ref());
if !matches!(compact_decision, AutomaticCompactDecision::Continue) { if !matches!(compact_decision, AutomaticCompactDecision::Continue) {
if let Err(error) = self.commit_system_items(&system_items) { if let Err(error) = self.commit_system_entries(&system_items) {
return Ok(PreRequestAction::Cancel(format!( return Ok(PreRequestAction::Cancel(format!(
"session persistence failed: {error}" "session persistence failed: {error}"
))); )));
@@ -688,7 +705,7 @@ impl Interceptor<SessionHistoryMetadata> for WorkerInterceptor {
if appended_items.is_empty() { if appended_items.is_empty() {
return Ok(PreRequestAction::Continue); return Ok(PreRequestAction::Continue);
} }
Ok(match self.commit_system_items(&system_items) { Ok(match self.commit_system_entries(&system_items) {
Ok(()) => PreRequestAction::ContinueWith(appended_items), Ok(()) => PreRequestAction::ContinueWith(appended_items),
Err(error) => PreRequestAction::Cancel(format!("session persistence failed: {error}")), Err(error) => PreRequestAction::Cancel(format!("session persistence failed: {error}")),
}) })
@@ -988,7 +1005,14 @@ mod tests {
input input
.system_items() .system_items()
.expect("test interceptor has durable append authority") .expect("test interceptor has durable append authority")
.append_subject_behavior_refresh("current Subject context", "Ask when uncertain."); .append_input(
crate::feature::builtin::memory::subject_behavior_input(
"current Subject context".into(),
"Ask when uncertain.".into(),
None,
)
.unwrap(),
);
Ok(HookPreRequestAction::Continue) Ok(HookPreRequestAction::Continue)
} }
} }
@@ -1003,8 +1027,26 @@ mod tests {
SystemPromptContribution::Unavailable SystemPromptContribution::Unavailable
} }
fn confirm_resident_context_behavior(&self, behavior_md: &str) { fn system_item_committed(
self.confirmed.lock().unwrap().push(behavior_md.to_owned()); &self,
_item: &SystemItem,
metadata: &crate::session_history::SessionHistoryMetadata,
) {
if let Some(input) = metadata
.input_metadata
.as_ref()
.and_then(|input| input.feature_payload())
.and_then(|payload| {
payload
.decode::<crate::feature::builtin::memory::SubjectBehaviorInput>(
"subject:behavior",
1,
)
.ok()
})
{
self.confirmed.lock().unwrap().push(input.behavior_md);
}
} }
} }
@@ -1548,14 +1590,87 @@ mod tests {
assert_eq!(*confirmed.lock().unwrap(), ["Ask when uncertain."]); assert_eq!(*confirmed.lock().unwrap(), ["Ask when uncertain."]);
assert!(matches!( assert!(matches!(
committed.lock().unwrap().as_slice(), committed.lock().unwrap().as_slice(),
[SystemItem::SubjectBehaviorRefresh { [SystemItem::Input { body }] if body == "current Subject context"
behavior_md,
prompt_provenance: Some(_),
..
}] if behavior_md == "Ask when uncertain."
)); ));
} }
#[test]
fn system_input_completion_reports_only_durable_batch_prefix() {
struct PrefixCommitter(Arc<Mutex<Vec<session_store::LogEntry>>>);
impl SystemItemCommitter for PrefixCommitter {
fn commit_log_entry(
&self,
entry: session_store::LogEntry,
) -> Result<(), session_store::StoreError> {
let mut committed = self.0.lock().unwrap();
if !committed.is_empty() {
return Err(std::io::Error::other("injected second append failure").into());
}
committed.push(entry);
Ok(())
}
}
struct Observer {
saved: Arc<Mutex<Vec<session_store::LogEntry>>>,
receipts: Arc<Mutex<Vec<crate::session_history::SessionHistoryMetadata>>>,
}
#[async_trait]
impl SystemPromptContributionSource for Observer {
async fn load(&self) -> SystemPromptContribution {
SystemPromptContribution::Unavailable
}
fn system_item_committed(
&self,
item: &SystemItem,
metadata: &crate::session_history::SessionHistoryMetadata,
) {
let saved = self.saved.lock().unwrap();
let Some(session_store::LogEntry::AnnotatedSystemItem { entry, .. }) = saved.last()
else {
panic!("completion must follow persistence");
};
assert_eq!(&entry.metadata, metadata);
assert_eq!(entry.item.history_text(), item.history_text());
self.receipts.lock().unwrap().push(metadata.clone());
}
}
let saved = Arc::new(Mutex::new(Vec::new()));
let receipts = Arc::new(Mutex::new(Vec::new()));
let pending = Arc::new(Mutex::new(Vec::new()));
let interceptor = WorkerInterceptor::new(
Arc::new(HookRegistryBuilder::new().build()),
None,
None,
NotifyBuffer::new(),
pending.clone(),
test_prompts(),
Some(Arc::new(PrefixCommitter(saved.clone()))),
)
.with_resident_context_source(Some(Arc::new(Observer {
saved: saved.clone(),
receipts: receipts.clone(),
})));
let items = [
SystemItem::Notification {
message: "first".into(),
body: "first saved input".into(),
prompt_provenance: None,
},
SystemItem::FileAttachment {
path: "test.rs".into(),
body: "second unsaved input".into(),
},
];
assert!(interceptor.commit_system_items(&items).is_err());
assert_eq!(saved.lock().unwrap().len(), 1);
let receipts = receipts.lock().unwrap();
assert_eq!(receipts.len(), 1);
let pending = interceptor.pending_committed_history.lock().unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].annotation, receipts[0]);
assert_eq!(pending[0].item.as_text(), Some("first saved input"));
}
#[test] #[test]
fn behavior_ack_uses_raw_content_including_clear_not_rendered_context() { fn behavior_ack_uses_raw_content_including_clear_not_rendered_context() {
let committed = Arc::new(Mutex::new(Vec::new())); let committed = Arc::new(Mutex::new(Vec::new()));
@@ -1576,12 +1691,15 @@ mod tests {
})), })),
) )
.with_resident_context_source(Some(source)); .with_resident_context_source(Some(source));
let items = ["raw behavior", ""].map(|behavior_md| SystemItem::SubjectBehaviorRefresh { let items = ["raw behavior", ""].map(|behavior_md| {
body: "same rendered context".into(), crate::feature::builtin::memory::subject_behavior_input(
behavior_md: behavior_md.into(), "same rendered context".into(),
prompt_provenance: None, behavior_md.into(),
None,
)
.unwrap()
}); });
interceptor.commit_system_items(&items).unwrap(); interceptor.commit_system_entries(&items).unwrap();
assert_eq!(*confirmed.lock().unwrap(), ["raw behavior", ""]); assert_eq!(*confirmed.lock().unwrap(), ["raw behavior", ""]);
assert_eq!(committed.lock().unwrap().len(), 2); assert_eq!(committed.lock().unwrap().len(), 2);
} }
+72 -14
View File
@@ -90,7 +90,8 @@ pub fn live_log_entry_event(entry: LogEntry) -> Option<Event> {
} }
LogEntry::AnnotatedSystemItem { entry, .. } => { LogEntry::AnnotatedSystemItem { entry, .. } => {
let entry_id = Some(entry.metadata.entry_id.0.clone()); let entry_id = Some(entry.metadata.entry_id.0.clone());
let value = serde_json::to_value(&entry.item).expect("SystemItem is Serialize"); let value = serde_json::to_value(entry.public_item())
.expect("public system input is serializable");
Some(Event::SystemItem { Some(Event::SystemItem {
entry_id, entry_id,
item: value, item: value,
@@ -339,21 +340,13 @@ mod tests {
#[test] #[test]
fn system_item_log_entry_keeps_identity_in_live_event() { fn system_item_log_entry_keeps_identity_in_live_event() {
let entry_id = session_store::LoggedSessionHistoryEntryId::new(); let entry_id = session_store::LoggedSessionHistoryEntryId::new();
let mut input =
crate::feature::builtin::memory::resident_summary_input("refresh".into(), None)
.unwrap();
input.metadata.entry_id = entry_id.clone();
let event = live_log_entry_event(LogEntry::AnnotatedSystemItem { let event = live_log_entry_event(LogEntry::AnnotatedSystemItem {
ts: session_store::segment_log::now_millis(), ts: session_store::segment_log::now_millis(),
entry: session_store::LoggedSystemHistoryEntry { entry: input,
item: session_store::SystemItem::ResidentSummaryRefresh {
body: "refresh".to_string(),
prompt_provenance: None,
},
metadata: session_store::LoggedSessionHistoryMetadata {
entry_id: entry_id.clone(),
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
operation_id: None,
},
derivation: None,
},
},
extensions: Vec::new(), extensions: Vec::new(),
}) })
.expect("SystemItem must be live-relevant"); .expect("SystemItem must be live-relevant");
@@ -370,6 +363,71 @@ mod tests {
} }
} }
#[test]
fn live_and_snapshot_system_projection_exclude_private_feature_fields() {
let provenance = session_store::PromptRenderProvenance {
workspace_id: Some("private-workspace".into()),
source_digest: "private-source".into(),
projection_digest: "private-projection".into(),
logical_name: "private-template".into(),
};
let items = [
crate::feature::builtin::memory::subject_behavior_input(
"displayed context".into(),
"private-raw-behavior".into(),
Some(provenance.clone()),
)
.unwrap(),
crate::feature::builtin::memory::resident_summary_input(
"displayed reference".into(),
Some(provenance),
)
.unwrap(),
];
for input in items {
let entry = LogEntry::AnnotatedSystemItem {
ts: 7,
entry: input,
extensions: vec![],
};
let persisted: LogEntry =
serde_json::from_slice(&serde_json::to_vec(&entry).unwrap()).unwrap();
let snapshot = session_store::public_snapshot::project_current_session_snapshot(&[
persisted.clone(),
]);
let Event::SystemItem {
entry_id,
item: live,
} = live_log_entry_event(persisted).unwrap()
else {
panic!("expected live system input");
};
let snapshot_entry = &snapshot.entries[0];
assert_eq!(entry_id.as_deref(), Some(snapshot_entry.entry_id.as_str()));
let protocol::SessionSnapshotEntryData::SystemItem {
data: Some(saved),
content,
..
} = &snapshot_entry.data
else {
panic!("expected canonical public system input");
};
assert_eq!(&live, saved);
let public: session_store::PublicSystemItem =
serde_json::from_value(live.clone()).unwrap();
assert_eq!(&public.body, content);
assert!(matches!(
public.presentation,
session_store::PublicSystemPresentation::Context
));
let encoded = serde_json::to_string(&snapshot).unwrap();
assert!(!encoded.contains("private-"));
assert!(live.get("prompt_provenance").is_none());
assert!(live.get("snapshot").is_none());
assert!(live.get("behavior_md").is_none());
}
}
#[test] #[test]
fn user_input_log_entry_maps_to_user_message_event() { fn user_input_log_entry_maps_to_user_message_event() {
let segments = vec![protocol::Segment::text("hello from log")]; let segments = vec![protocol::Segment::text("hello from log")];
+2
View File
@@ -16,6 +16,7 @@ pub mod hook;
pub(crate) mod in_flight; pub(crate) mod in_flight;
pub mod ipc; pub mod ipc;
pub mod job; pub mod job;
mod metadata_lifecycle;
pub mod model_client; pub mod model_client;
mod paste_artifact_tool; mod paste_artifact_tool;
pub mod prompt; pub mod prompt;
@@ -27,6 +28,7 @@ mod session_history;
pub mod shared_state; pub mod shared_state;
mod shutdown_after_idle; mod shutdown_after_idle;
mod subject_release; mod subject_release;
mod subjektiv_metadata;
pub use subject_release::SubjectReleaseReservation; pub use subject_release::SubjectReleaseReservation;
pub mod skill; pub mod skill;
pub mod spawn; pub mod spawn;
+69
View File
@@ -0,0 +1,69 @@
//! Atomic metadata transitions with Host-owned Session lifecycle participants.
//!
//! Participants inspect domain evidence inside the metadata update, not after the
//! active pointer has been published. The generic writer never interprets it.
use session_store::worker_metadata::{
WorkerActiveSegmentRef, WorkerMetadata, WorkerMetadataStore, WorkerStoreError,
};
use std::sync::Arc;
pub(crate) trait SessionMetadataLifecycle: Send + Sync {
/// Runs under the metadata update lock. Only compare/transform the supplied
/// value: no I/O, reentrant store calls, lock waits, or Feature refresh.
fn active_changing(&self, metadata: &mut WorkerMetadata, next: Option<&WorkerActiveSegmentRef>);
}
pub(crate) type WorkerMetadataWriter =
Arc<dyn Fn(WorkerMetadata) -> Result<(), WorkerStoreError> + Send + Sync>;
pub(crate) type WorkerMetadataSegmentCas = Arc<
dyn Fn(&str, &WorkerActiveSegmentRef, WorkerActiveSegmentRef) -> Result<bool, WorkerStoreError>
+ Send
+ Sync,
>;
pub(crate) fn worker_metadata_writer_for_store<St>(store: &St) -> WorkerMetadataWriter
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let store = store.clone();
let participants = crate::feature::builtin::session_metadata_lifecycles();
Arc::new(move |next| {
store
.update_by_name(&next.worker_name, |metadata| {
for participant in &participants {
participant.active_changing(metadata, next.active.as_ref());
}
metadata.active = next.active;
metadata.resolved_manifest_snapshot = next.resolved_manifest_snapshot;
if next.workspace_id.is_some() {
metadata.workspace_id = next.workspace_id;
}
if next.workspace_root.is_some() {
metadata.workspace_root = next.workspace_root;
}
})
.map(|_| ())
})
}
pub(crate) fn worker_metadata_segment_cas_for_store<St>(store: &St) -> WorkerMetadataSegmentCas
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let store = store.clone();
let participants = crate::feature::builtin::session_metadata_lifecycles();
Arc::new(move |worker_name, expected, replacement| {
let mut matched = false;
store.update_by_name(worker_name, |metadata| {
if metadata.active.as_ref() == Some(expected) {
for participant in &participants {
participant.active_changing(metadata, Some(&replacement));
}
metadata.active = Some(replacement);
matched = true;
}
})?;
Ok(matched)
})
}
+388 -26
View File
@@ -74,6 +74,7 @@ impl std::fmt::Display for SessionEntryRef {
pub(crate) enum ReferenceKind { pub(crate) enum ReferenceKind {
User, User,
Assistant, Assistant,
System,
Tool, Tool,
} }
@@ -82,6 +83,7 @@ impl ReferenceKind {
match self { match self {
Self::User => "user", Self::User => "user",
Self::Assistant => "assistant", Self::Assistant => "assistant",
Self::System => "system",
Self::Tool => "tool", Self::Tool => "tool",
} }
} }
@@ -90,6 +92,7 @@ impl ReferenceKind {
match value { match value {
"user" => Some(Self::User), "user" => Some(Self::User),
"assistant" | "agent" => Some(Self::Assistant), "assistant" | "agent" => Some(Self::Assistant),
"system" => Some(Self::System),
"tool" => Some(Self::Tool), "tool" => Some(Self::Tool),
_ => None, _ => None,
} }
@@ -131,6 +134,7 @@ pub(crate) struct OverviewItem {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(crate) struct ReferenceEntry { pub(crate) struct ReferenceEntry {
use_policy: Option<protocol::SessionEntryUsePolicy>,
pub id: SessionEntryRef, pub id: SessionEntryRef,
pub origin: SessionEntryProvenance, pub origin: SessionEntryProvenance,
pub entry_range: [u64; 2], pub entry_range: [u64; 2],
@@ -242,6 +246,7 @@ pub(crate) struct SessionEntryEvidence {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct CapturedHistoryEntry { struct CapturedHistoryEntry {
use_policy: Option<protocol::SessionEntryUsePolicy>,
item: Item, item: Item,
entry_id: session_store::LoggedSessionHistoryEntryId, entry_id: session_store::LoggedSessionHistoryEntryId,
origin: SessionEntryProvenance, origin: SessionEntryProvenance,
@@ -302,10 +307,12 @@ impl SessionCapture {
is_error, is_error,
attachments: _, attachments: _,
} => Item::tool_result_item(call_id, summary, content, is_error), } => Item::tool_result_item(call_id, summary, content, is_error),
// Observation deliberately excludes system items and SessionSnapshotEntryData::SystemItem { content, .. } => {
// controller errors from model-visible session evidence. entry.use_policy?;
SessionSnapshotEntryData::SystemItem { .. } Item::system_message(content)
| SessionSnapshotEntryData::Invoke { .. } }
// Controller errors are not model-visible evidence.
SessionSnapshotEntryData::Invoke { .. }
| SessionSnapshotEntryData::Usage { .. } | SessionSnapshotEntryData::Usage { .. }
| SessionSnapshotEntryData::RunStats { .. } | SessionSnapshotEntryData::RunStats { .. }
| SessionSnapshotEntryData::RunYielded { .. } | SessionSnapshotEntryData::RunYielded { .. }
@@ -314,6 +321,7 @@ impl SessionCapture {
| SessionSnapshotEntryData::RunError { .. } => return None, | SessionSnapshotEntryData::RunError { .. } => return None,
}; };
Some(CapturedHistoryEntry { Some(CapturedHistoryEntry {
use_policy: entry.use_policy,
item, item,
entry_id: session_store::LoggedSessionHistoryEntryId(entry.entry_id), entry_id: session_store::LoggedSessionHistoryEntryId(entry.entry_id),
origin: entry.provenance, origin: entry.provenance,
@@ -328,6 +336,7 @@ impl SessionCapture {
.into_iter() .into_iter()
.enumerate() .enumerate()
.map(|(index, item)| CapturedHistoryEntry { .map(|(index, item)| CapturedHistoryEntry {
use_policy: None,
item, item,
entry_id: session_store::LoggedSessionHistoryEntryId(format!("{index:08}")), entry_id: session_store::LoggedSessionHistoryEntryId(format!("{index:08}")),
origin: SessionEntryProvenance::LegacyUnknown, origin: SessionEntryProvenance::LegacyUnknown,
@@ -342,10 +351,32 @@ impl SessionCapture {
) -> Self { ) -> Self {
let entries = entries let entries = entries
.into_iter() .into_iter()
.map(|entry| CapturedHistoryEntry { .map(|entry| {
item: entry.item, let (item, use_policy) = if let Some(input) = &entry.annotation.input_metadata {
let is_system = matches!(
entry.item,
Item::Message {
role: Role::System,
..
}
);
if is_system {
(
Item::system_message(input.public_projection().body),
Some(input.public_system_use_policy()),
)
} else {
(entry.item, Some(input.public_use_policy()))
}
} else {
(entry.item, None)
};
CapturedHistoryEntry {
item,
use_policy,
entry_id: entry.annotation.entry_id, entry_id: entry.annotation.entry_id,
origin: public_provenance(&entry.annotation.origin), origin: public_provenance(&entry.annotation.origin),
}
}) })
.collect(); .collect();
Self::from_captured_entries(segment_id, entries) Self::from_captured_entries(segment_id, entries)
@@ -364,7 +395,8 @@ impl SessionCapture {
let entry_range = [idx as u64, idx as u64]; let entry_range = [idx as u64, idx as u64];
match item { match item {
Item::Message { role, content, .. } => { Item::Message { role, content, .. } => {
let Some(kind) = message_reference_kind(&entry.origin, role) else { let Some(kind) = message_reference_kind(&entry.origin, role, entry.use_policy)
else {
continue; continue;
}; };
let text = content let text = content
@@ -376,6 +408,7 @@ impl SessionCapture {
let summary = truncate_chars(&text, 240); let summary = truncate_chars(&text, 240);
let id = SessionEntryRef::from_history_entry_id(&entry.entry_id); let id = SessionEntryRef::from_history_entry_id(&entry.entry_id);
index.push(ReferenceEntry { index.push(ReferenceEntry {
use_policy: entry.use_policy,
id: id.clone(), id: id.clone(),
origin: entry.origin.clone(), origin: entry.origin.clone(),
entry_range, entry_range,
@@ -386,7 +419,9 @@ impl SessionCapture {
summary: summary.clone(), summary: summary.clone(),
search_text: text.clone(), search_text: text.clone(),
}); });
if matches!(kind, ReferenceKind::User | ReferenceKind::Assistant) { if entry.use_policy.is_none_or(|p| p.observation_readable)
&& matches!(kind, ReferenceKind::User | ReferenceKind::Assistant)
{
overview.push(OverviewItem { overview.push(OverviewItem {
id: id.clone(), id: id.clone(),
origin: entry.origin.clone(), origin: entry.origin.clone(),
@@ -403,6 +438,7 @@ impl SessionCapture {
} => { } => {
let text = format!("{name}\n{arguments}"); let text = format!("{name}\n{arguments}");
index.push(ReferenceEntry { index.push(ReferenceEntry {
use_policy: entry.use_policy,
id: SessionEntryRef::from_history_entry_id(&entry.entry_id), id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
origin: entry.origin.clone(), origin: entry.origin.clone(),
entry_range, entry_range,
@@ -430,6 +466,7 @@ impl SessionCapture {
content.as_deref().unwrap_or_default(), content.as_deref().unwrap_or_default(),
); );
index.push(ReferenceEntry { index.push(ReferenceEntry {
use_policy: entry.use_policy,
id: SessionEntryRef::from_history_entry_id(&entry.entry_id), id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
origin: entry.origin.clone(), origin: entry.origin.clone(),
entry_range, entry_range,
@@ -510,6 +547,9 @@ impl SessionCapture {
let mut hits = Vec::new(); let mut hits = Vec::new();
for entry in &self.index { for entry in &self.index {
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
continue;
}
if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index { if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index {
continue; continue;
} }
@@ -581,6 +621,9 @@ impl SessionCapture {
}; };
for entry in selected { for entry in selected {
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
continue;
}
if entries.len() >= max_items { if entries.len() >= max_items {
truncated = true; truncated = true;
break; break;
@@ -695,21 +738,11 @@ impl SessionCapture {
pub(crate) fn evidence_for(&self, id: &str) -> Option<SessionEntryEvidence> { pub(crate) fn evidence_for(&self, id: &str) -> Option<SessionEntryEvidence> {
let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?; let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?;
let excerpt = self if entry.use_policy.is_some_and(|p| !p.memory_evidence) {
.read( return None;
ReadSelector::Id(id), }
ReadOptions { let item = &self.entries.get(entry.entry_range[0] as usize)?.item;
include_tools: true, let excerpt = render_item(item, entry, ReadDetail::Compact, 2 * 1024);
tool_part: ToolPart::Both,
detail: ReadDetail::Compact,
max_items: 1,
max_bytes: 2 * 1024,
},
)
.entries
.first()
.map(|entry| entry.text.clone())
.unwrap_or_else(|| entry.summary.clone());
Some(SessionEntryEvidence { Some(SessionEntryEvidence {
segment_id: self.segment_id.clone(), segment_id: self.segment_id.clone(),
entry_ref: entry.id.clone(), entry_ref: entry.id.clone(),
@@ -731,9 +764,8 @@ fn public_provenance(origin: &WorkerHistoryProvenance) -> SessionEntryProvenance
} }
WorkerHistoryProvenance::WorkerInput { .. } => SessionEntryProvenance::WorkerInput, WorkerHistoryProvenance::WorkerInput { .. } => SessionEntryProvenance::WorkerInput,
WorkerHistoryProvenance::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction, WorkerHistoryProvenance::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction,
WorkerHistoryProvenance::BackendInstruction { .. } => { WorkerHistoryProvenance::BackendInstruction { .. }
SessionEntryProvenance::BackendInstruction | WorkerHistoryProvenance::Provided { .. } => SessionEntryProvenance::BackendInstruction,
}
WorkerHistoryProvenance::ModelOutput { .. } => SessionEntryProvenance::ModelOutput, WorkerHistoryProvenance::ModelOutput { .. } => SessionEntryProvenance::ModelOutput,
WorkerHistoryProvenance::ToolOutput { .. } => SessionEntryProvenance::ToolOutput, WorkerHistoryProvenance::ToolOutput { .. } => SessionEntryProvenance::ToolOutput,
WorkerHistoryProvenance::CompactionSummary WorkerHistoryProvenance::CompactionSummary
@@ -746,7 +778,15 @@ fn public_provenance(origin: &WorkerHistoryProvenance) -> SessionEntryProvenance
fn message_reference_kind( fn message_reference_kind(
origin: &SessionEntryProvenance, origin: &SessionEntryProvenance,
provider_role: &Role, provider_role: &Role,
use_policy: Option<protocol::SessionEntryUsePolicy>,
) -> Option<ReferenceKind> { ) -> Option<ReferenceKind> {
if use_policy.is_some() {
return match provider_role {
Role::System => Some(ReferenceKind::System),
Role::User => Some(ReferenceKind::User),
Role::Assistant => Some(ReferenceKind::Assistant),
};
}
match origin { match origin {
SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => { SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => {
Some(ReferenceKind::User) Some(ReferenceKind::User)
@@ -950,6 +990,328 @@ mod tests {
} }
} }
#[test]
fn generic_public_input_observation_and_memory_policies_survive_store_restore_independently() {
use session_store::{
FeatureInputPayload, FsStore, HistoryInputMetadata, HistoryInputUsePolicy,
InputCompactionUse, InputEvidenceUse, InputObservationUse, LogEntry,
LoggedSystemHistoryEntry, PublicSystemItem, PublicSystemPresentation, Store,
SystemInputEnvelope,
};
let root = tempfile::tempdir().unwrap();
let store = FsStore::new(root.path()).unwrap();
let session = session_store::new_session_id();
let segment = session_store::new_segment_id();
store
.create_segment(
session,
segment,
&[LogEntry::AnnotatedSegmentStart {
ts: 1,
session_id: session,
system_prompt: None,
config: Default::default(),
history: vec![],
forked_from: None,
compacted_from: None,
}],
)
.unwrap();
let mut inputs = Vec::new();
for (producer, observation, memory_evidence, public) in [
(
"same",
InputObservationUse::Readable,
InputEvidenceUse::Exclude,
true,
),
(
"same",
InputObservationUse::Hidden,
InputEvidenceUse::Include,
true,
),
(
"renamed",
InputObservationUse::Readable,
InputEvidenceUse::Include,
true,
),
(
"renamed",
InputObservationUse::Hidden,
InputEvidenceUse::Exclude,
true,
),
(
"private",
InputObservationUse::Readable,
InputEvidenceUse::Include,
false,
),
] {
let entry = LoggedSystemHistoryEntry::from_input(
WorkerHistoryProvenance::Provided {
producer: producer.into(),
},
None,
SystemInputEnvelope::new(
"private model text",
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence,
observation,
compaction: InputCompactionUse::ReferenceInput,
},
Some(
FeatureInputPayload::encode(
"future:owner",
99,
&serde_json::json!({ "secret": "private metadata" }),
)
.unwrap(),
),
public.then(|| PublicSystemItem {
kind: "supplied-context".into(),
body: "public context: do not extract this".into(),
presentation: PublicSystemPresentation::Context,
}),
)
.unwrap(),
)
.unwrap(),
);
store
.append(
session,
segment,
&LogEntry::AnnotatedSystemItem {
ts: 2,
entry: entry.clone(),
extensions: vec![],
},
)
.unwrap();
inputs.push(entry);
}
let saved = store.read_all(session, segment).unwrap();
let restored =
crate::session_history::restore_history_entries(session, segment, &saved).unwrap();
let snapshot = session_store::public_snapshot::project_session_snapshot(session, &saved);
let snapshot: SessionSnapshot =
serde_json::from_value(serde_json::to_value(snapshot).unwrap()).unwrap();
for capture in [
SessionCapture::from_history_entries(segment.to_string(), restored),
SessionCapture::from_session_snapshot(segment.to_string(), snapshot),
] {
let hits = capture.search(&SearchOptions {
kind: Some(ReferenceKind::System),
..Default::default()
});
assert_eq!(hits.len(), 2);
for (index, input) in inputs.iter().enumerate() {
let reference = SessionEntryRef::from_history_entry_id(&input.metadata.entry_id);
let read =
capture.read(ReadSelector::Id(reference.as_str()), ReadOptions::default());
assert_eq!(!read.entries.is_empty(), matches!(index, 0 | 2));
let evidence = capture.evidence_for(reference.as_str());
assert_eq!(evidence.is_some(), matches!(index, 1 | 2));
for text in read
.entries
.iter()
.map(|entry| entry.text.as_str())
.chain(evidence.iter().map(|entry| entry.excerpt.as_str()))
{
assert!(text.contains("public context: do not extract this"));
assert!(!text.contains("private model text"));
assert!(!text.contains("private metadata"));
assert!(!text.contains("future:owner"));
}
}
}
}
// Real FsStore seed/fork/reopen -> ordinary history replay, public snapshot,
// capture and both evidence consumers; no Feature or Tool execution occurs.
#[test]
fn tool_policy_axes_survive_restore_and_fork_without_exposing_derived_context() {
use session_store::public_material::{
PublicMaterialLimits, PublicMaterialUse, SessionPublicMaterialReader,
};
use session_store::*;
for recording in [InputEvidenceUse::Include, InputEvidenceUse::Exclude] {
for memory in [InputEvidenceUse::Include, InputEvidenceUse::Exclude] {
for observation in [InputObservationUse::Readable, InputObservationUse::Hidden] {
let root = tempfile::tempdir().unwrap();
let store = FsStore::new(root.path()).unwrap();
let session = new_session_id();
let first = new_segment_id();
let fork = new_segment_id();
let items = vec![
Item::tool_call("original", "ArbitraryTool", "{}"),
Item::tool_result("original", "original result"),
Item::tool_call("derived", "ArbitraryTool", "PRIVATE_SYNTHETIC_CONTEXT"),
Item::tool_result("derived", "PRIVATE_SYNTHETIC_RESULT"),
];
let mut history: Vec<_> = items
.iter()
.cloned()
.map(|item| LoggedHistoryEntry {
item: LoggedItem::from(item),
metadata: LoggedSessionHistoryMetadata::legacy_unknown(),
})
.collect();
let sources = history[..2]
.iter()
.map(|entry| entry.metadata.entry_id.clone())
.collect::<Vec<_>>();
for entry in &mut history[..2] {
entry.metadata.input_metadata = Some(
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: recording,
memory_evidence: memory,
observation,
compaction: InputCompactionUse::Retain,
},
None,
None,
)
.unwrap(),
);
}
for entry in &mut history[2..] {
entry.metadata.derivation = Some(LoggedHistoryDerivation {
sources: sources.clone(),
});
entry.metadata.input_metadata = Some(
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::Retain,
},
None,
None,
)
.unwrap(),
);
}
for segment in [first, fork] {
store
.create_segment(
session,
segment,
&[LogEntry::AnnotatedSegmentStart {
ts: 1,
session_id: session,
system_prompt: None,
config: Default::default(),
history: history.clone(),
forked_from: (segment == fork).then_some(SegmentOrigin {
segment_id: first,
at_turn_index: 0,
}),
compacted_from: None,
}],
)
.unwrap();
}
let reopened = FsStore::new(root.path()).unwrap();
let saved = reopened.read_all(session, fork).unwrap();
let restored =
crate::session_history::restore_history_entries(session, fork, &saved)
.unwrap();
for (index, entry) in restored.iter().enumerate() {
assert_eq!(entry.item, items[index]);
assert_eq!(entry.annotation, history[index].metadata);
}
let capture = SessionCapture::from_history_entries(fork.to_string(), restored);
assert!(
matches!(&capture.entries[2].item, Item::ToolCall { call_id, .. } if call_id == "derived")
);
assert!(
matches!(&capture.entries[3].item, Item::ToolResult { call_id, .. } if call_id == "derived")
);
let snapshot = public_snapshot::project_session_snapshot(session, &saved);
assert!(
!serde_json::to_string(&snapshot)
.unwrap()
.contains("PRIVATE_SYNTHETIC")
);
for capture in [
capture,
SessionCapture::from_session_snapshot(fork.to_string(), snapshot),
] {
assert_eq!(
capture
.search(&SearchOptions {
kind: Some(ReferenceKind::Tool),
..Default::default()
})
.len(),
if observation == InputObservationUse::Readable {
2
} else {
0
}
);
for (index, entry) in history.iter().enumerate() {
let reference =
SessionEntryRef::from_history_entry_id(&entry.metadata.entry_id);
assert_eq!(
!capture
.read(
ReadSelector::Id(reference.as_str()),
ReadOptions::default()
)
.entries
.is_empty(),
index < 2 && observation == InputObservationUse::Readable
);
assert_eq!(
capture.evidence_for(reference.as_str()).is_some(),
index < 2 && memory == InputEvidenceUse::Include
);
}
}
for purpose in [
PublicMaterialUse::Recording,
PublicMaterialUse::MemoryEvidence,
] {
let reader = SessionPublicMaterialReader::open_fs_for_use(
root.path(),
session,
fork,
PublicMaterialLimits::default(),
reopened.retention(),
purpose,
)
.unwrap();
let ids = reader
.descriptors()
.map(|entry| entry.identity.entry_ref.clone())
.collect::<Vec<_>>();
let included = match purpose {
PublicMaterialUse::Recording => recording,
PublicMaterialUse::MemoryEvidence => memory,
} == InputEvidenceUse::Include;
assert_eq!(
ids,
sources
.iter()
.filter(|_| included)
.map(|id| format!("E{}", id.0))
.collect::<Vec<_>>()
);
}
}
}
}
}
#[test] #[test]
fn overview_contains_user_and_assistant_only() { fn overview_contains_user_and_assistant_only() {
let view = SessionCapture::new( let view = SessionCapture::new(
+18 -16
View File
@@ -24,6 +24,7 @@ pub(crate) fn test_logged_history_entry(item: impl Into<Item>) -> LoggedHistoryE
LoggedHistoryEntry { LoggedHistoryEntry {
item: session_store::LoggedItem::from(item.into()), item: session_store::LoggedItem::from(item.into()),
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -38,6 +39,7 @@ pub(crate) fn test_logged_system_entry(
session_store::LoggedSystemHistoryEntry { session_store::LoggedSystemHistoryEntry {
item, item,
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(), entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown, origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -58,12 +60,23 @@ pub(crate) fn metadata(
derivation: Option<SessionHistoryDerivation>, derivation: Option<SessionHistoryDerivation>,
) -> SessionHistoryMetadata { ) -> SessionHistoryMetadata {
SessionHistoryMetadata { SessionHistoryMetadata {
input_metadata: None,
entry_id: SessionHistoryEntryId::new(), entry_id: SessionHistoryEntryId::new(),
origin, origin,
derivation, derivation,
} }
} }
pub(crate) fn compaction_use(
metadata: &SessionHistoryMetadata,
) -> session_store::InputCompactionUse {
metadata
.input_metadata
.as_ref()
.map(|input| input.usage().compaction)
.unwrap_or(session_store::InputCompactionUse::Retain)
}
#[cfg(test)] #[cfg(test)]
pub(crate) fn history_entry( pub(crate) fn history_entry(
item: Item, item: Item,
@@ -80,6 +93,7 @@ pub(crate) fn history_entry_with_id(
HistoryEntry::new( HistoryEntry::new(
item, item,
SessionHistoryMetadata { SessionHistoryMetadata {
input_metadata: None,
entry_id, entry_id,
origin, origin,
derivation: None, derivation: None,
@@ -130,22 +144,10 @@ pub(crate) fn restore_history_entries(
} }
*target = from_logged(entry); *target = from_logged(entry);
} }
LogEntry::AnnotatedSystemItem { entry, .. } => { LogEntry::AnnotatedSystemItem { entry, .. } => history.push(HistoryEntry::new(
history.push(HistoryEntry::new(entry.item.to_history_item(), { entry.item.to_history_item(),
let mut metadata = entry.metadata.clone(); entry.metadata.clone(),
// Old typed startup records used generic placeholder origins. )),
// Preserve explicitly supplied metadata on current contributions.
if matches!(
metadata.origin,
WorkerHistoryProvenance::DerivedSummary
| WorkerHistoryProvenance::LegacyUnknown
) && matches!(entry.item, session_store::SystemItem::JournalStartup { .. })
{
metadata.origin = WorkerHistoryProvenance::JournalContext;
}
metadata
}))
}
_ => {} _ => {}
} }
} }
+44
View File
@@ -8,6 +8,50 @@ pub use server_api::{
SkillProjectionStatus, SkillProvenance, SkillResourceRef, SkillSourceKind, SkillProjectionStatus, SkillProvenance, SkillResourceRef, SkillSourceKind,
}; };
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct SkillActivationInput {
pub name: String,
}
pub(crate) fn activation_input(
activation: &SkillActivationResponse,
) -> Result<session_store::LoggedSystemHistoryEntry, session_store::InputMetadataError> {
use session_store::*;
let body = format!(
"Agent Skill `{}` activated from {}.\n\n{}",
activation.name, activation.provenance.id, activation.body
);
let payload = FeatureInputPayload::encode(
"skill:activation",
1,
&SkillActivationInput {
name: activation.name.clone(),
},
)?;
let metadata = HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction: InputCompactionUse::Retain,
},
Some(payload),
Some(PublicSystemItem {
kind: "skill_activation".into(),
body: body.clone(),
presentation: PublicSystemPresentation::Context,
}),
)?;
Ok(LoggedSystemHistoryEntry::from_input(
LoggedSessionHistoryOrigin::Provided {
producer: "skill.activation".into(),
},
None,
SystemInputEnvelope::new(body, metadata)?,
))
}
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum SkillClientError { pub enum SkillClientError {
#[error("workspace client is unavailable: {0}")] #[error("workspace client is unavailable: {0}")]
+1
View File
@@ -77,6 +77,7 @@ mod tests {
.into_iter() .into_iter()
.enumerate() .enumerate()
.map(|(index, value)| protocol::SessionSnapshotEntry { .map(|(index, value)| protocol::SessionSnapshotEntry {
use_policy: None,
entry_id: format!("test-{index}"), entry_id: format!("test-{index}"),
timestamp: index as u64, timestamp: index as u64,
provenance: protocol::SessionEntryProvenance::LegacyUnknown, provenance: protocol::SessionEntryProvenance::LegacyUnknown,
+1
View File
@@ -1358,6 +1358,7 @@ mod tests {
is_error: false, is_error: false,
}, },
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId(entry_id.into()), entry_id: session_store::LoggedSessionHistoryEntryId(entry_id.into()),
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown, origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
+53 -13
View File
@@ -26,6 +26,30 @@ use server_api::*;
use crate::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceServerOperation}; use crate::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceServerOperation};
impl<C: agen::llm_client::client::LlmClient + 'static, St: session_store::Store>
crate::Worker<C, St>
{
/// Bind live Host capability and its context provider, not authority inherited
/// from a previous binding. Ordinary Feature/lifecycle gates also apply to embedders.
pub fn bind_subjektiv_host(&mut self, host: Option<Arc<dyn SubjektivHost>>) {
self.set_subjektiv_host(host);
if self.manifest_lifecycle_features_enabled()
&& self.backend_job().is_none()
&& crate::feature::builtin::memory::ordinary_subjektiv_features_enabled(self.manifest())
&& let Ok(Some(connection)) = self.subjektiv_host_connection()
&& (connection.host.journal_startup_granted()
|| connection.host.journal_compaction_granted())
{
self.install_session_context_source(Arc::new(
crate::feature::builtin::journal::JournalContextSource::new(
connection,
self.prompts(),
),
));
}
}
}
/// Host-issued execution context, never deserialized from model arguments. /// Host-issued execution context, never deserialized from model arguments.
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub struct SubjektivHostContext { pub struct SubjektivHostContext {
@@ -1128,7 +1152,12 @@ enabled = true
let initial = store.read_all(session, segment).unwrap(); let initial = store.read_all(session, segment).unwrap();
let journal = initial.iter().find_map(|entry| match entry { let journal = initial.iter().find_map(|entry| match entry {
LogEntry::AnnotatedSystemItem { entry, .. } LogEntry::AnnotatedSystemItem { entry, .. }
if matches!(entry.item, SystemItem::JournalStartup { .. }) => if entry
.metadata
.input_metadata
.as_ref()
.and_then(|input| input.feature_payload())
.is_some_and(|payload| payload.schema() == "journal:startup") =>
{ {
Some(entry) Some(entry)
} }
@@ -1138,22 +1167,33 @@ enabled = true
let body = journal.map(|journal| { let body = journal.map(|journal| {
assert_eq!( assert_eq!(
journal.metadata.origin, journal.metadata.origin,
LoggedSessionHistoryOrigin::JournalContext LoggedSessionHistoryOrigin::Provided {
producer: "subjektiv.journal".into()
}
); );
let SystemItem::JournalStartup { let SystemItem::Input { body } = &journal.item else {
body,
snapshot: saved,
snapshot_sha256,
prompt_provenance,
} = &journal.item
else {
unreachable!() unreachable!()
}; };
assert_eq!(*saved, serde_json::to_value(&snapshot).unwrap()); let payload = journal
.metadata
.input_metadata
.as_ref()
.unwrap()
.feature_payload()
.unwrap()
.decode::<crate::feature::builtin::journal::JournalStartupInput>(
"journal:startup",
1,
)
.unwrap();
let saved = serde_json::to_value(&payload.snapshot).unwrap();
let snapshot_sha256 = &payload.snapshot_sha256;
let prompt_provenance = &payload.prompt_provenance;
assert_eq!(saved, serde_json::to_value(&snapshot).unwrap());
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
assert_eq!( assert_eq!(
*snapshot_sha256, *snapshot_sha256,
Sha256::digest(serde_json::to_vec(saved).unwrap()) Sha256::digest(serde_json::to_vec(&saved).unwrap())
.iter() .iter()
.map(|byte| format!("{byte:02x}")) .map(|byte| format!("{byte:02x}"))
.collect::<String>() .collect::<String>()
@@ -1166,7 +1206,7 @@ enabled = true
body.trim_end(), body.trim_end(),
format!( format!(
"## Prior Journal context\n\n```json\n{}\n```", "## Prior Journal context\n\n```json\n{}\n```",
serde_json::to_string(saved).unwrap() serde_json::to_string(&saved).unwrap()
) )
); );
body.as_str() body.as_str()
@@ -1235,7 +1275,7 @@ enabled = true
} }
assert_eq!(host.startup_calls.load(Ordering::SeqCst), 1); assert_eq!(host.startup_calls.load(Ordering::SeqCst), 1);
let logs = store.read_all(session, segment).unwrap(); let logs = store.read_all(session, segment).unwrap();
assert_eq!(logs.iter().filter(|entry| matches!(entry, LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, SystemItem::JournalStartup { .. }))).count(), usize::from(contributes_context)); assert_eq!(logs.iter().filter(|entry| matches!(entry, LogEntry::AnnotatedSystemItem { entry, .. } if entry.metadata.input_metadata.as_ref().and_then(|input| input.feature_payload()).is_some_and(|payload| payload.schema() == "journal:startup"))).count(), usize::from(contributes_context));
controller controller
.handle .handle
.send(Method::Shutdown { .send(Method::Shutdown {
+120
View File
@@ -0,0 +1,120 @@
//! Host-owned attribution transitions. Metadata callbacks are synchronous, local
//! comparisons only: no network I/O, Feature lookup, or nested store access.
use crate::metadata_lifecycle::SessionMetadataLifecycle;
use session_store::{
SessionId, SubjektivSessionAttributionState, WorkerActiveSegmentRef, WorkerMetadata,
WorkerMetadataStore,
};
pub(crate) struct SessionAttributionMetadata;
impl SessionMetadataLifecycle for SessionAttributionMetadata {
fn active_changing(
&self,
metadata: &mut WorkerMetadata,
next: Option<&WorkerActiveSegmentRef>,
) {
if metadata.active.as_ref().map(|active| active.session_id)
!= next.map(|active| active.session_id)
{
metadata.subjektiv_session_attribution = None;
}
}
}
pub(crate) fn compare_and_swap_attribution<St: WorkerMetadataStore>(
store: &St,
worker_name: &str,
session_id: SessionId,
expected: Option<&SubjektivSessionAttributionState>,
replacement: SubjektivSessionAttributionState,
) -> Result<(), crate::WorkerError> {
let mut matched = false;
store.update_by_name(worker_name, |metadata| {
if metadata.active.as_ref().map(|active| active.session_id) == Some(session_id)
&& metadata.subjektiv_session_attribution.as_ref() == expected
&& replacement.session_id() == session_id
{
metadata.subjektiv_session_attribution = Some(replacement);
matched = true;
}
})?;
if !matched {
return Err(crate::WorkerError::SubjektivSessionAttribution {
state: "attribution_conflict",
message: "active Session or observed attribution changed during Host confirmation"
.into(),
});
}
Ok(())
}
#[cfg(test)]
mod tests {
//! Connect the production metadata writer/CAS and Host lifecycle to a real
//! filesystem store; no metadata or persistence boundary is substituted.
use super::*;
use crate::metadata_lifecycle::{
worker_metadata_segment_cas_for_store, worker_metadata_writer_for_store,
};
#[test]
fn host_transition_preserves_same_session_and_invalidates_new_session_atomically() {
let dir = tempfile::tempdir().unwrap();
let store = session_store::FsWorkerStore::new(dir.path()).unwrap();
let session = session_store::new_session_id();
let mut metadata = WorkerMetadata::new(
"body",
Some(WorkerActiveSegmentRef::pending_segment(session)),
);
metadata.workspace_id = Some("workspace".into());
metadata.subjektiv_session_attribution =
Some(SubjektivSessionAttributionState::Confirmed {
session_id: session,
subject_id: "subject".into(),
});
store.write(&metadata).unwrap();
let writer = worker_metadata_writer_for_store(&store);
let cas = worker_metadata_segment_cas_for_store(&store);
let segment =
WorkerActiveSegmentRef::active_segment(session, session_store::new_segment_id());
let mut next = WorkerMetadata::new("body", Some(segment.clone()));
writer(next.clone()).unwrap();
let materialized = store.read_by_name("body").unwrap().unwrap();
assert_eq!(
materialized.subjektiv_session_attribution,
metadata.subjektiv_session_attribution
);
assert_eq!(materialized.workspace_id, metadata.workspace_id);
let compacted =
WorkerActiveSegmentRef::active_segment(session, session_store::new_segment_id());
assert!(cas("body", &segment, compacted.clone()).unwrap());
assert_eq!(
store
.read_by_name("body")
.unwrap()
.unwrap()
.subjektiv_session_attribution,
metadata.subjektiv_session_attribution
);
let new_session = WorkerActiveSegmentRef::pending_segment(session_store::new_session_id());
assert!(!cas("body", &segment, new_session.clone()).unwrap());
assert!(cas("body", &compacted, new_session.clone()).unwrap());
let switched = store.read_by_name("body").unwrap().unwrap();
assert_eq!(switched.active, Some(new_session));
assert!(switched.subjektiv_session_attribution.is_none());
assert_eq!(switched.workspace_id, metadata.workspace_id);
// Returning to the old Session cannot recover the invalidated evidence.
next.active = metadata.active;
writer(next).unwrap();
assert!(
store
.read_by_name("body")
.unwrap()
.unwrap()
.subjektiv_session_attribution
.is_none()
);
}
}
+1
View File
@@ -5566,6 +5566,7 @@ mod tests {
entry: session_store::LoggedHistoryEntry { entry: session_store::LoggedHistoryEntry {
item: session_store::LoggedItem::from(saved), item: session_store::LoggedItem::from(saved),
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId::new(), entry_id: session_store::LoggedSessionHistoryEntryId::new(),
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown, origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
File diff suppressed because it is too large Load Diff
+255 -31
View File
@@ -119,6 +119,7 @@ fn annotated(item: Item) -> session_store::LoggedHistoryEntry {
session_store::LoggedHistoryEntry { session_store::LoggedHistoryEntry {
item: session_store::LoggedItem::from(item), item: session_store::LoggedItem::from(item),
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId::new(), entry_id: session_store::LoggedSessionHistoryEntryId::new(),
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown, origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None, derivation: None,
@@ -2212,10 +2213,48 @@ fn summary_entry(
origin: session_store::LoggedSessionHistoryOrigin, origin: session_store::LoggedSessionHistoryOrigin,
) -> session_store::LoggedHistoryEntry { ) -> session_store::LoggedHistoryEntry {
let mut entry = annotated(item); let mut entry = annotated(item);
entry.metadata.input_metadata = matches!(
&entry.item,
session_store::LoggedItem::Message {
role: session_store::LoggedRole::System,
..
}
)
.then(|| {
session_store::HistoryInputMetadata::new(
session_store::HistoryInputUsePolicy {
recording_material: session_store::InputEvidenceUse::Exclude,
memory_evidence: session_store::InputEvidenceUse::Exclude,
observation: session_store::InputObservationUse::Hidden,
compaction: match origin {
session_store::LoggedSessionHistoryOrigin::CompactionSummary => {
session_store::InputCompactionUse::OpeningSummary
}
session_store::LoggedSessionHistoryOrigin::JournalContext => {
session_store::InputCompactionUse::ReferenceInput
}
_ => session_store::InputCompactionUse::Retain,
},
},
None,
None,
)
.unwrap()
});
entry.metadata.origin = origin; entry.metadata.origin = origin;
entry entry
} }
fn compaction_use(
metadata: &session_store::LoggedSessionHistoryMetadata,
) -> session_store::InputCompactionUse {
metadata
.input_metadata
.as_ref()
.map(|input| input.usage().compaction)
.unwrap_or(session_store::InputCompactionUse::Retain)
}
fn long_summary(label: &str) -> String { fn long_summary(label: &str) -> String {
format!( format!(
"## Completed Tasks\n{label}: investigated the transport boundary.\n\n\ "## Completed Tasks\n{label}: investigated the transport boundary.\n\n\
@@ -2351,7 +2390,8 @@ async fn full_initial_summary_survives_small_overview_budgets_and_both_retention
assert_eq!( assert_eq!(
replacement replacement
.iter() .iter()
.filter(|entry| entry.metadata.origin == CompactionSummary) .filter(|entry| compaction_use(&entry.metadata)
== session_store::InputCompactionUse::OpeningSummary)
.count(), .count(),
1 1
); );
@@ -2596,7 +2636,9 @@ async fn three_chained_compactions_transport_each_complete_predecessor_without_r
let seed = replacement_seed(&fixture.worker); let seed = replacement_seed(&fixture.worker);
let inherited = seed let inherited = seed
.iter() .iter()
.filter(|entry| entry.metadata.origin == CompactionSummary) .filter(|entry| {
compaction_use(&entry.metadata) == session_store::InputCompactionUse::OpeningSummary
})
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert_eq!(inherited.len(), 1); assert_eq!(inherited.len(), 1);
assert_eq!( assert_eq!(
@@ -2884,7 +2926,7 @@ impl worker::subjektiv::SubjektivHost for CompactionJournalHost {
} }
fn journal_startup_granted(&self) -> bool { fn journal_startup_granted(&self) -> bool {
true false
} }
fn journal_startup_context( fn journal_startup_context(
@@ -2988,8 +3030,8 @@ fn compaction_snapshot(generation: usize) -> subjektiv::journal::startup::Journa
fn journal_body(worker: &Worker<MockClient, TestStore>) -> String { fn journal_body(worker: &Worker<MockClient, TestStore>) -> String {
let seed = replacement_seed(worker); let seed = replacement_seed(worker);
assert_eq!( assert_eq!(
seed[1].metadata.origin, compaction_use(&seed[1].metadata),
session_store::LoggedSessionHistoryOrigin::JournalContext session_store::InputCompactionUse::ReferenceInput
); );
let item = Item::from(seed[1].item.clone()); let item = Item::from(seed[1].item.clone());
assert!(matches!( assert!(matches!(
@@ -3001,8 +3043,8 @@ fn journal_body(worker: &Worker<MockClient, TestStore>) -> String {
)); ));
assert_eq!( assert_eq!(
seed.iter() seed.iter()
.filter(|entry| entry.metadata.origin .filter(|entry| compaction_use(&entry.metadata)
== session_store::LoggedSessionHistoryOrigin::JournalContext) == session_store::InputCompactionUse::ReferenceInput)
.count(), .count(),
1 1
); );
@@ -3111,20 +3153,34 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
projection.source_digest = "a".repeat(64); projection.source_digest = "a".repeat(64);
let initial_catalog = Arc::new(worker::PromptCatalog::from_projection(projection).unwrap()); let initial_catalog = Arc::new(worker::PromptCatalog::from_projection(projection).unwrap());
fixture.worker.prompts().store(initial_catalog.clone()); fixture.worker.prompts().store(initial_catalog.clone());
let legacy_startup = session_store::LoggedSystemHistoryEntry { // Already migrated generic input with unrelated provenance/payload; Core
item: session_store::SystemItem::JournalStartup { // must select it by explicit policy, not identify a Journal Feature.
body: "OLD_TYPED_STARTUP_JOURNAL".into(), let legacy_startup = session_store::LoggedSystemHistoryEntry::from_input(
snapshot: serde_json::json!({"journals": []}), DerivedSummary,
snapshot_sha256: "legacy-startup-digest".into(), None,
prompt_provenance: session_store::PromptRenderProvenance { session_store::SystemInputEnvelope::new(
workspace_id: None, "OLD_TYPED_STARTUP_JOURNAL",
source_digest: "legacy-source".into(), session_store::HistoryInputMetadata::new(
projection_digest: "legacy-projection".into(), session_store::HistoryInputUsePolicy {
logical_name: "internal.subjektiv_journal_startup".into(), recording_material: session_store::InputEvidenceUse::Exclude,
memory_evidence: session_store::InputEvidenceUse::Exclude,
observation: session_store::InputObservationUse::Hidden,
compaction: session_store::InputCompactionUse::ReferenceInput,
}, },
}, Some(
metadata: summary_entry(Item::system_message(""), DerivedSummary).metadata, session_store::FeatureInputPayload::encode(
}; "renamed-owner:reference",
8,
&serde_json::json!({"private": "retained opaque context"}),
)
.unwrap(),
),
None,
)
.unwrap(),
)
.unwrap(),
);
let legacy_id = legacy_startup.metadata.entry_id.clone(); let legacy_id = legacy_startup.metadata.entry_id.clone();
fixture fixture
.restore_seed_with_system( .restore_seed_with_system(
@@ -3158,7 +3214,7 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
.unwrap() .unwrap()
.annotation .annotation
.origin, .origin,
JournalContext DerivedSummary
); );
let session = fixture.worker.session_id(); let session = fixture.worker.session_id();
let original_segment = fixture.worker.segment_id(); let original_segment = fixture.worker.segment_id();
@@ -3183,8 +3239,8 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
.unwrap() .unwrap()
.annotation .annotation
.origin, .origin,
JournalContext, DerivedSummary,
"legacy startup classification survives a normal turn" "generic source provenance survives a normal turn unchanged"
); );
} }
assert_eq!( assert_eq!(
@@ -3235,7 +3291,7 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
assert_eq!(host.reads.lock().unwrap().len(), index + 1); assert_eq!(host.reads.lock().unwrap().len(), index + 1);
let history = fixture.worker.history(); let history = fixture.worker.history();
let overview = input let overview = input
.split("## Previous Journal reference (historical, not automatically retained)") .split("## Previous reference input (historical, not automatically retained)")
.next() .next()
.unwrap(); .unwrap();
for old in &old_bodies { for old in &old_bodies {
@@ -3328,6 +3384,17 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
.run_text("normal post-compaction turn") .run_text("normal post-compaction turn")
.await .await
.unwrap(); .unwrap();
let frozen_seed = replacement_seed(&fixture.worker)
.into_iter()
.filter(|entry| {
matches!(
compaction_use(&entry.metadata),
session_store::InputCompactionUse::ReferenceInput
| session_store::InputCompactionUse::OpeningSummary
)
})
.collect::<Vec<_>>();
assert_eq!(frozen_seed.len(), 2);
let records = fixture let records = fixture
.worker .worker
.store() .store()
@@ -3367,6 +3434,20 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
.run_text("normal turn after typed history restore") .run_text("normal turn after typed history restore")
.await .await
.unwrap(); .unwrap();
for frozen in frozen_seed {
let replayed = fixture
.worker
.session_history()
.entries()
.iter()
.find(|entry| entry.annotation.entry_id == frozen.metadata.entry_id)
.expect("frozen entry identity survives re-restore");
assert_eq!(replayed.item, Item::from(frozen.item));
assert_eq!(
replayed.annotation, frozen.metadata,
"origin, derivation, public allowlist, policy and private payload survive atomic seed/restore"
);
}
assert_eq!( assert_eq!(
host.reads.lock().unwrap().len(), host.reads.lock().unwrap().len(),
3, 3,
@@ -3437,27 +3518,41 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
) )
.await; .await;
fixture.worker.compact(0).await.unwrap(); fixture.worker.compact(0).await.unwrap();
let input = first_compactor_input(&observed.requests()[1]);
let body = if matches!(case, "disabled" | "no-host") {
assert_eq!(
replacement_seed(&fixture.worker)
.iter()
.filter(|entry| compaction_use(&entry.metadata)
== session_store::InputCompactionUse::ReferenceInput)
.count(),
0
);
assert!(!input.contains("Journal reference frozen at this compaction boundary"));
String::new()
} else {
let body = journal_body(&fixture.worker); let body = journal_body(&fixture.worker);
let expected = match case { let expected = if case == "absent" {
"no-host" | "disabled" => "disabled", "absent"
"absent" => "absent", } else {
_ => "unavailable", "unavailable"
}; };
assert!( assert!(
body.contains(&format!("Journal read status: {expected}.")), body.contains(&format!("Journal read status: {expected}.")),
"{case}" "{case}"
); );
let input = first_compactor_input(&observed.requests()[1]);
assert_frozen_journal_transport(&input, &body); assert_frozen_journal_transport(&input, &body);
assert_journal_provenance( assert_journal_provenance(
&body, &body,
expected_json.as_deref(), expected_json.as_deref(),
&fixture.worker.prompts().load_full(), &fixture.worker.prompts().load_full(),
(case != "no-host").then_some(COMPACTION_HOST_WORKSPACE_ID), Some(COMPACTION_HOST_WORKSPACE_ID),
); );
body
};
assert_full_initial_summary(&input, &inherited); assert_full_initial_summary(&input, &inherited);
assert!(input.contains("UNRECORDED_FALLBACK_GOAL"), "{case}"); assert!(input.contains("UNRECORDED_FALLBACK_GOAL"), "{case}");
assert!(input.contains("## Previous Journal reference (historical, not automatically retained)\nPREVIOUS_JOURNAL_DELEGATED_CONTEXT")); assert!(input.contains("## Previous reference input (historical, not automatically retained)\nPREVIOUS_JOURNAL_DELEGATED_CONTEXT"));
assert!( assert!(
fixture.worker.history().iter().all(|item| !item fixture.worker.history().iter().all(|item| !item
.as_text() .as_text()
@@ -3560,3 +3655,132 @@ async fn journal_snapshot_counts_against_compactor_hard_input_cap_without_mutati
); );
} }
} }
#[tokio::test]
async fn identical_reference_and_retained_inputs_preserve_distinct_metadata_after_compaction() {
use session_store::*;
let client = MockClient::new(vec![
single_text_events("fixture"),
single_text_events("fork checkpoint"),
write_summary_tool_use_events("review-summary", "replacement summary"),
single_text_events("done"),
single_text_events("restore checkpoint"),
]);
let mut fixture = SummaryInputFixture::new(MANUAL_ONLY_MANIFEST_TOML, client).await;
let make = |producer: &str, compaction| {
LoggedSystemHistoryEntry::from_input(
LoggedSessionHistoryOrigin::Provided {
producer: producer.into(),
},
None,
SystemInputEnvelope::new(
"identical model context",
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
compaction,
},
Some(
FeatureInputPayload::encode(
"review:owner",
1,
&serde_json::json!({"producer": producer}),
)
.unwrap(),
),
None,
)
.unwrap(),
)
.unwrap(),
)
.to_history_entry()
};
let historical = make("old-reference", InputCompactionUse::ReferenceInput);
let retained = make("retained-context", InputCompactionUse::Retain);
let repeated = make("second-retained-context", InputCompactionUse::Retain);
assert_ne!(historical.metadata.entry_id, retained.metadata.entry_id);
fixture
.restore_seed(
vec![historical.clone(), retained.clone(), repeated.clone()],
false,
)
.await;
let session = fixture.worker.session_id();
let original = fixture.worker.segment_id();
fixture
.worker
.store()
.append(
session,
original,
&LogEntry::Extension {
ts: 2,
domain: "test.fork_drift".into(),
payload: serde_json::json!({}),
},
)
.unwrap();
fixture.worker.run_text("fork checkpoint").await.unwrap();
let fork = fixture.worker.segment_id();
assert_ne!(fork, original);
let forked = replacement_seed(&fixture.worker);
for expected in [&historical, &retained, &repeated] {
assert!(forked.contains(expected));
}
fixture.worker.compact(1_000_000).await.unwrap();
let seed = replacement_seed(&fixture.worker);
let matching: Vec<_> = seed.iter().filter(|e| e.item == retained.item).collect();
assert_eq!(matching.len(), 2);
assert_eq!(
matching[0].metadata, retained.metadata,
"a retained logical entry must not inherit identity/policy/payload from an equal-body reference that was removed"
);
assert_eq!(matching[1].metadata, repeated.metadata);
let compacted = fixture.worker.segment_id();
let reopened = FsStore::new(fixture._store_dir.path()).unwrap();
let records = reopened.read_all(session, compacted).unwrap();
assert!(
matches!(&records[0], LogEntry::AnnotatedSegmentStart { compacted_from: Some(source), .. } if source.segment_id == fork)
);
fixture.worker.run_text("restore checkpoint").await.unwrap();
let records = fixture.worker.store().read_all(session, compacted).unwrap();
let snapshot = public_snapshot::project_current_session_snapshot(&records);
let checkpoint = snapshot
.entries
.iter()
.rev()
.find(|entry| {
matches!(
entry.data,
protocol::SessionSnapshotEntryData::UserInput { .. }
)
})
.unwrap();
fixture
.worker
.rewind_to(checkpoint.entry_id.clone())
.await
.unwrap();
for expected in [&retained, &repeated] {
assert!(
fixture
.worker
.session_history()
.entries()
.iter()
.any(|entry| entry.annotation == expected.metadata
&& entry.item == Item::from(expected.item.clone()))
);
}
assert!(
!fixture
.worker
.session_history()
.entries()
.iter()
.any(|entry| entry.annotation.entry_id == historical.metadata.entry_id)
);
}
+1 -1
View File
@@ -3091,7 +3091,7 @@ async fn socket_worker_event_turn_ended_while_idle_auto_starts_turn() {
Ok(Some(Event::SystemItem { ref item, .. })) Ok(Some(Event::SystemItem { ref item, .. }))
if item.get("kind").and_then(|k| k.as_str()) == Some("worker_event") if item.get("kind").and_then(|k| k.as_str()) == Some("worker_event")
&& item && item
.pointer("/event/worker_name") .pointer("/presentation/event/worker_name")
.and_then(|v| v.as_str()) == Some("child") => .and_then(|v| v.as_str()) == Some("child") =>
{ {
saw_worker_event_echo = true; saw_worker_event_echo = true;
+15 -5
View File
@@ -14,7 +14,7 @@ use protocol::{
InvocationArgumentDescriptor, InvocationArgumentType, InvocationArgumentValue, InvocationArgumentDescriptor, InvocationArgumentType, InvocationArgumentValue,
InvocationCompletion, InvocationValue, Segment, InvocationCompletion, InvocationValue, Segment,
}; };
use session_store::{CombinedStore, FsStore, FsWorkerStore, LogEntry, SystemItem}; use session_store::{CombinedStore, FsStore, FsWorkerStore, LogEntry};
use worker::feature::{ use worker::feature::{
FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureInvocationContext, FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureInvocationContext,
FeatureInvocationHandler, FeatureInvocationHandlerError, FeatureModule, FeatureRegistryBuilder, FeatureInvocationHandler, FeatureInvocationHandlerError, FeatureModule, FeatureRegistryBuilder,
@@ -224,10 +224,20 @@ fn results(worker: &Worker<Trace, TestStore>) -> Vec<FeatureInvocationResult> {
.0 .0
.into_iter() .into_iter()
.filter_map(|entry| match entry { .filter_map(|entry| match entry {
LogEntry::AnnotatedSystemItem { entry, .. } => match entry.item { LogEntry::AnnotatedSystemItem { entry, .. } => {
SystemItem::FeatureInvocationResult { result, .. } => Some(result), #[derive(serde::Deserialize)]
_ => None, struct SavedResult {
}, result: FeatureInvocationResult,
}
entry
.metadata
.input_metadata
.as_ref()?
.feature_payload()?
.decode::<SavedResult>("feature:invocation-result", 1)
.ok()
.map(|input| input.result)
}
_ => None, _ => None,
}) })
.collect() .collect()
+19 -13
View File
@@ -44704,6 +44704,7 @@ mod tests {
content: vec![LoggedContentPart::Text { text: text.into() }], content: vec![LoggedContentPart::Text { text: text.into() }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.to_string()), entry_id: LoggedSessionHistoryEntryId(id.to_string()),
origin: LoggedSessionHistoryOrigin::HumanInput { origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".to_string(), account_id: "account-1".to_string(),
@@ -44894,6 +44895,7 @@ mod tests {
content: vec![LoggedContentPart::Text { text: text.into() }], content: vec![LoggedContentPart::Text { text: text.into() }],
}, },
metadata: LoggedSessionHistoryMetadata { metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.to_string()), entry_id: LoggedSessionHistoryEntryId(id.to_string()),
origin: LoggedSessionHistoryOrigin::HumanInput { origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".to_string(), account_id: "account-1".to_string(),
@@ -49169,19 +49171,23 @@ mod tests {
let worker_root = root let worker_root = root
.join(".test-embedded-runtime-store/workers") .join(".test-embedded-runtime-store/workers")
.join(&worker.worker_id); .join(&worker.worker_id);
fs::create_dir_all(worker_root.join("session/segments")).unwrap(); use session_store::Store;
fs::write( let store = session_store::WorkerSessionStore::new(worker_root.join("session")).unwrap();
worker_root.join("session/session.json"), let session = session_store::new_session_id();
serde_json::to_vec_pretty(&json!({ let segment = session_store::new_segment_id();
"schema_version": 1, store
"session_id": session_store::new_session_id() .create_segment(
})) session,
.unwrap(), segment,
) &[session_store::LogEntry::AnnotatedSegmentStart {
.unwrap(); ts: 1,
fs::write( session_id: session,
worker_root.join("session/segments/segment-a.jsonl"), system_prompt: Some("backend job retained session evidence".into()),
b"backend job retained session evidence\n", config: Default::default(),
history: vec![],
forked_from: None,
compacted_from: None,
}],
) )
.unwrap(); .unwrap();
} }
@@ -597,30 +597,40 @@ async fn backend_real_runtime_journal_release_and_next_body_persisted_startup()
let segments = session.list_segments(session_id).unwrap(); let segments = session.list_segments(session_id).unwrap();
assert_eq!(segments.len(), 1); assert_eq!(segments.len(), 1);
let entries = session.read_all_read_only(session_id, segments[0]).unwrap(); let entries = session.read_all_read_only(session_id, segments[0]).unwrap();
#[derive(serde::Deserialize)]
struct SavedJournal {
snapshot: serde_json::Value,
snapshot_sha256: String,
prompt_provenance: session_store::PromptRenderProvenance,
}
let startup: Vec<_> = entries let startup: Vec<_> = entries
.iter() .iter()
.enumerate() .enumerate()
.filter_map(|(index, log)| match log { .filter_map(|(index, log)| {
session_store::LogEntry::AnnotatedSystemItem { entry, .. } => match &entry.item { let session_store::LogEntry::AnnotatedSystemItem { entry, .. } = log else {
session_store::SystemItem::JournalStartup { return None;
snapshot, };
snapshot_sha256, let payload = entry
body, .metadata
prompt_provenance, .input_metadata
} => Some((index, snapshot, snapshot_sha256, body, prompt_provenance)), .as_ref()?
_ => None, .feature_payload()?
}, .decode::<SavedJournal>("journal:startup", 1)
_ => None, .ok()?;
Some((index, payload, entry.item.history_text()))
}) })
.collect(); .collect();
assert_eq!(startup.len(), 1, "startup must be durably committed once"); assert_eq!(startup.len(), 1, "startup must be durably committed once");
let (startup_index, snapshot, digest, body, provenance) = startup[0]; let (startup_index, payload, body) = &startup[0];
let snapshot = &payload.snapshot;
let digest = &payload.snapshot_sha256;
let provenance = &payload.prompt_provenance;
let first_input = entries let first_input = entries
.iter() .iter()
.position(|entry| matches!(entry, session_store::LogEntry::AnnotatedUserInput { .. })) .position(|entry| matches!(entry, session_store::LogEntry::AnnotatedUserInput { .. }))
.expect("new body persisted its initial human input"); .expect("new body persisted its initial human input");
assert!( assert!(
startup_index < first_input, *startup_index < first_input,
"startup reference must be persisted before the first human input/model request" "startup reference must be persisted before the first human input/model request"
); );
assert_eq!(snapshot["subject_id"], subject.id); assert_eq!(snapshot["subject_id"], subject.id);
@@ -479,6 +479,7 @@ fn seed_session(
vec![session_store::LoggedHistoryEntry { vec![session_store::LoggedHistoryEntry {
item, item,
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId("read-source-original".into()), entry_id: session_store::LoggedSessionHistoryEntryId("read-source-original".into()),
origin, origin,
derivation: None, derivation: None,
@@ -1707,6 +1708,7 @@ impl SubjectRemovalFixture {
.map(|text| session_store::LoggedHistoryEntry { .map(|text| session_store::LoggedHistoryEntry {
item: message(text), item: message(text),
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId(text.into()), entry_id: session_store::LoggedSessionHistoryEntryId(text.into()),
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None, derivation: None,
@@ -67,6 +67,7 @@ impl Fixture {
content: vec![session_store::LoggedContentPart::Text { text: text.into() }], content: vec![session_store::LoggedContentPart::Text { text: text.into() }],
}, },
metadata: session_store::LoggedSessionHistoryMetadata { metadata: session_store::LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId(format!("original-{index}")), entry_id: session_store::LoggedSessionHistoryEntryId(format!("original-{index}")),
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None, derivation: None,
+3 -3
View File
@@ -70,12 +70,12 @@
] ]
}, },
{ {
"path": "crates/session-store/src/system_item.rs", "path": "crates/session-store/src/legacy_session_log/tests/v5.rs",
"kind": "saved_data", "kind": "saved_data",
"reason": "frozen_subject_behavior_log_reads_old_counter_without_republishing_itの保存済みSystemItem fixture。旧JSONを読んでもexact archived bodyを維持することを証明する。削除すると過去Session log形式の読取回帰を検出できない。", "reason": "旧Subject behavior JSONに保存されていたcounterの実fixture。v1–4専用migration境界で旧綴りを読み、本文を維持して現行payloadへcounterを再公開しないことを検証する。通常v5 decoderは旧Feature kind自体を拒否する。",
"lines": [ "lines": [
{ {
"sha256": "5b5998029cfed30634e8b063b759aefd70e30dab3d02ce9defe9086977fc9811", "sha256": "54c30cb3198eb16e57c7e580fce4dc058f89e793b77a2955be317aa38b490138",
"count": 1 "count": 1
} }
] ]
File diff suppressed because one or more lines are too long
+3 -1
View File
@@ -178,6 +178,8 @@ compaction?: InFlightCompaction | null, };
export type SessionEntryProvenance = "human_input" | "worker_input" | "flow_instruction" | "backend_instruction" | "model_output" | "tool_output" | "derived_summary" | "legacy_unknown"; export type SessionEntryProvenance = "human_input" | "worker_input" | "flow_instruction" | "backend_instruction" | "model_output" | "tool_output" | "derived_summary" | "legacy_unknown";
export type SessionEntryUsePolicy = { observation_readable: boolean, memory_evidence: boolean, };
export type SessionMessageRole = "user" | "assistant"; export type SessionMessageRole = "user" | "assistant";
export type SessionContentPart = { "kind": "text", text: string, } | { "kind": "refusal", refusal: string, }; export type SessionContentPart = { "kind": "text", text: string, } | { "kind": "refusal", refusal: string, };
@@ -206,7 +208,7 @@ entry_id: string,
/** /**
* Timestamp copied from the durable log record that commits this entry. * Timestamp copied from the durable log record that commits this entry.
*/ */
timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array<string>, } & ({ "kind": "user_input", segments: Array<Segment>, } | { "kind": "message", role: SessionMessageRole, content: Array<SessionContentPart>, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array<SessionToolAttachment>, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_yielded", reason: RunYieldReason, active_run_turn_count: number, } | { "kind": "run_resumed", source: RunResumeSource, active_run_turn_count: number, } | { "kind": "invoke", trigger: InvokeKind, } | { "kind": "usage", input_tokens: number, cache_read_input_tokens: number, output_tokens: number, } | { "kind": "run_stats", elapsed_ms: number, requests: number, upload_tokens: number, output_tokens: number, } | { "kind": "run_cancelled" } | { "kind": "run_error", message: string, failure?: RunFailureKind | null, }); timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array<string>, use_policy?: SessionEntryUsePolicy | null, } & ({ "kind": "user_input", segments: Array<Segment>, } | { "kind": "message", role: SessionMessageRole, content: Array<SessionContentPart>, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array<SessionToolAttachment>, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_yielded", reason: RunYieldReason, active_run_turn_count: number, } | { "kind": "run_resumed", source: RunResumeSource, active_run_turn_count: number, } | { "kind": "invoke", trigger: InvokeKind, } | { "kind": "usage", input_tokens: number, cache_read_input_tokens: number, output_tokens: number, } | { "kind": "run_stats", elapsed_ms: number, requests: number, upload_tokens: number, output_tokens: number, } | { "kind": "run_cancelled" } | { "kind": "run_error", message: string, failure?: RunFailureKind | null, });
export type SessionConversationTurn = { turn_id: string, entries: Array<SessionSnapshotEntry>, }; export type SessionConversationTurn = { turn_id: string, entries: Array<SessionSnapshotEntry>, };
@@ -3943,6 +3943,7 @@ Deno.test("live system items reconcile with committed history by entry identity"
const item = { const item = {
kind: "resident_summary_refresh", kind: "resident_summary_refresh",
body: "Resident memory refresh", body: "Resident memory refresh",
presentation: { kind: "context" },
}; };
const event: Event = { const event: Event = {
event: "system_item", event: "system_item",