feat: persist System input policies through public consumers
This commit is contained in:
@@ -982,6 +982,17 @@ pub enum SessionEntryProvenance {
|
||||
LegacyUnknown,
|
||||
}
|
||||
|
||||
/// Public use decisions contain no Feature identity or private payload. Absence
|
||||
/// denotes the older projection contract, not permission to expose System input.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionEntryUsePolicy {
|
||||
pub observation_readable: bool,
|
||||
pub memory_evidence: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
@@ -994,6 +1005,8 @@ pub struct SessionSnapshotEntry {
|
||||
pub provenance: SessionEntryProvenance,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub derived_from: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub use_policy: Option<SessionEntryUsePolicy>,
|
||||
#[serde(flatten)]
|
||||
pub data: SessionSnapshotEntryData,
|
||||
}
|
||||
@@ -2619,6 +2632,7 @@ mod tests {
|
||||
session: SessionSnapshot {
|
||||
pending_submissions: PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: "entry-1".into(),
|
||||
timestamp: 1,
|
||||
provenance: SessionEntryProvenance::HumanInput,
|
||||
|
||||
@@ -361,6 +361,7 @@ impl Store for FsStore {
|
||||
segment_id: SegmentId,
|
||||
entry: &LogEntry,
|
||||
) -> Result<(), StoreError> {
|
||||
entry.validate_system_inputs()?;
|
||||
let _retention = self.retention().lock_live(session_id)?;
|
||||
let line = serde_json::to_string(entry)?;
|
||||
self.append_line(&self.log_path(session_id, segment_id), &line)
|
||||
@@ -448,6 +449,9 @@ impl Store for FsStore {
|
||||
segment_id: SegmentId,
|
||||
entries: &[LogEntry],
|
||||
) -> Result<(), StoreError> {
|
||||
for entry in entries {
|
||||
entry.validate_system_inputs()?;
|
||||
}
|
||||
let _retention = self.retention().lock_live(session_id)?;
|
||||
let path = self.log_path(session_id, segment_id);
|
||||
if let Some(parent) = path.parent() {
|
||||
|
||||
@@ -67,6 +67,10 @@ pub enum LoggedSessionHistoryOrigin {
|
||||
ToolOutput {
|
||||
worker: LoggedWorkerSubject,
|
||||
},
|
||||
/// Descriptive producer identity; use policies live in separate metadata.
|
||||
Provided {
|
||||
producer: String,
|
||||
},
|
||||
/// Host-produced opening context summary; not auto-read or TaskStore material.
|
||||
CompactionSummary,
|
||||
/// Frozen Host Journal reference, never independent observation evidence.
|
||||
@@ -86,14 +90,26 @@ pub struct LoggedSessionHistoryMetadata {
|
||||
pub origin: LoggedSessionHistoryOrigin,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub derivation: Option<LoggedHistoryDerivation>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub system_input: Option<crate::SystemInputMetadata>,
|
||||
}
|
||||
|
||||
impl LoggedSessionHistoryMetadata {
|
||||
fn validate_system_input(&self) -> Result<(), crate::SystemInputError> {
|
||||
if matches!(self.origin, LoggedSessionHistoryOrigin::Provided { .. })
|
||||
&& self.system_input.is_none()
|
||||
{
|
||||
return Err(crate::SystemInputError::MissingMetadata);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn legacy_unknown() -> Self {
|
||||
Self {
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
system_input: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -101,20 +117,140 @@ impl LoggedSessionHistoryMetadata {
|
||||
/// Persisted item and metadata are one value so transforms cannot reorder or
|
||||
/// truncate one without the other.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(try_from = "HistoryEntryWire")]
|
||||
pub struct LoggedHistoryEntry {
|
||||
pub item: LoggedItem,
|
||||
pub metadata: LoggedSessionHistoryMetadata,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct HistoryEntryWire {
|
||||
item: LoggedItem,
|
||||
metadata: LoggedSessionHistoryMetadata,
|
||||
}
|
||||
|
||||
impl TryFrom<HistoryEntryWire> for LoggedHistoryEntry {
|
||||
type Error = crate::SystemInputError;
|
||||
|
||||
fn try_from(wire: HistoryEntryWire) -> Result<Self, Self::Error> {
|
||||
let entry = Self {
|
||||
item: wire.item,
|
||||
metadata: wire.metadata,
|
||||
};
|
||||
entry.validate_system_input()?;
|
||||
Ok(entry)
|
||||
}
|
||||
}
|
||||
|
||||
impl LoggedHistoryEntry {
|
||||
pub fn validate_system_input(&self) -> Result<(), crate::SystemInputError> {
|
||||
self.metadata.validate_system_input()?;
|
||||
if self.metadata.system_input.is_some() {
|
||||
let LoggedItem::Message {
|
||||
role: crate::LoggedRole::System,
|
||||
content,
|
||||
} = &self.item
|
||||
else {
|
||||
return Err(crate::SystemInputError::InconsistentRecord);
|
||||
};
|
||||
let [crate::LoggedContentPart::Text { text }] = content.as_slice() else {
|
||||
return Err(crate::SystemInputError::InconsistentRecord);
|
||||
};
|
||||
if text.len() > crate::system_input::MAX_SYSTEM_INPUT_BYTES {
|
||||
return Err(crate::SystemInputError::ResourceLimit);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed system-item history record. The typed system event remains available
|
||||
/// to client replay while its model-visible projection carries the same stable
|
||||
/// metadata used by live history.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(try_from = "SystemEntryWire")]
|
||||
pub struct LoggedSystemHistoryEntry {
|
||||
pub item: crate::SystemItem,
|
||||
pub metadata: LoggedSessionHistoryMetadata,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SystemEntryWire {
|
||||
item: crate::SystemItem,
|
||||
metadata: LoggedSessionHistoryMetadata,
|
||||
}
|
||||
|
||||
impl TryFrom<SystemEntryWire> for LoggedSystemHistoryEntry {
|
||||
type Error = crate::SystemInputError;
|
||||
|
||||
fn try_from(wire: SystemEntryWire) -> Result<Self, Self::Error> {
|
||||
let entry = Self {
|
||||
item: wire.item,
|
||||
metadata: wire.metadata,
|
||||
};
|
||||
entry.validate_system_input()?;
|
||||
Ok(entry)
|
||||
}
|
||||
}
|
||||
|
||||
impl LoggedSystemHistoryEntry {
|
||||
pub fn from_input(
|
||||
origin: LoggedSessionHistoryOrigin,
|
||||
derivation: Option<LoggedHistoryDerivation>,
|
||||
input: crate::SystemInputEnvelope,
|
||||
) -> Self {
|
||||
let (body, system_input) = input.into_parts();
|
||||
Self {
|
||||
item: crate::SystemItem::Input { body },
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin,
|
||||
derivation,
|
||||
system_input: Some(system_input),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate_system_input(&self) -> Result<(), crate::SystemInputError> {
|
||||
self.metadata.validate_system_input()?;
|
||||
match (&self.item, &self.metadata.system_input) {
|
||||
(crate::SystemItem::Input { body }, Some(_)) => {
|
||||
if body.len() > crate::system_input::MAX_SYSTEM_INPUT_BYTES {
|
||||
return Err(crate::SystemInputError::ResourceLimit);
|
||||
}
|
||||
}
|
||||
(crate::SystemItem::Input { .. }, None) => {
|
||||
return Err(crate::SystemInputError::MissingMetadata);
|
||||
}
|
||||
(_, Some(_)) => return Err(crate::SystemInputError::InconsistentRecord),
|
||||
(_, None) => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn to_history_entry(&self) -> LoggedHistoryEntry {
|
||||
LoggedHistoryEntry {
|
||||
item: self.item.to_history_item().into(),
|
||||
metadata: self.metadata.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn public_item(&self) -> crate::PublicSystemItem {
|
||||
if matches!(self.item, crate::SystemItem::Input { .. }) {
|
||||
return self
|
||||
.metadata
|
||||
.system_input
|
||||
.as_ref()
|
||||
.and_then(crate::SystemInputMetadata::public_item)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.item.public_item());
|
||||
}
|
||||
self.item.public_item()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -132,6 +268,7 @@ mod tests {
|
||||
}],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::HumanInput {
|
||||
account_id: "account-1".into(),
|
||||
@@ -150,6 +287,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_system_records_reject_missing_policy_role_mismatch_and_overlimit_body() {
|
||||
let valid = serde_json::json!({
|
||||
"item": { "kind": "input", "body": "context" },
|
||||
"metadata": {
|
||||
"entry_id": "stable-entry", "origin": { "kind": "provided", "producer": "owner" },
|
||||
"system_input": { "usage": {
|
||||
"recording_material": "exclude", "memory_evidence": "exclude",
|
||||
"observation": "hidden", "compaction": "reference_input"
|
||||
} }
|
||||
}
|
||||
});
|
||||
let restored: LoggedSystemHistoryEntry = serde_json::from_value(valid.clone()).unwrap();
|
||||
let flat = serde_json::to_value(restored.to_history_entry()).unwrap();
|
||||
assert!(serde_json::from_value::<LoggedHistoryEntry>(flat.clone()).is_ok());
|
||||
for generic in [true, false] {
|
||||
let wire = if generic { &valid } else { &flat };
|
||||
let decode = |wire| {
|
||||
if generic {
|
||||
serde_json::from_value::<LoggedSystemHistoryEntry>(wire).is_err()
|
||||
} else {
|
||||
serde_json::from_value::<LoggedHistoryEntry>(wire).is_err()
|
||||
}
|
||||
};
|
||||
let mut missing = wire.clone();
|
||||
missing["metadata"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("system_input");
|
||||
assert!(decode(missing));
|
||||
let mut absent_policy = wire.clone();
|
||||
absent_policy["metadata"]["system_input"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("usage");
|
||||
assert!(decode(absent_policy));
|
||||
let mut invalid_role = wire.clone();
|
||||
invalid_role["item"] =
|
||||
serde_json::to_value(LoggedItem::from(agen::Item::user_message("not system")))
|
||||
.unwrap();
|
||||
assert!(decode(invalid_role));
|
||||
let mut oversized = wire.clone();
|
||||
if generic {
|
||||
oversized["item"]["body"] = "x"
|
||||
.repeat(crate::system_input::MAX_SYSTEM_INPUT_BYTES + 1)
|
||||
.into();
|
||||
} else {
|
||||
oversized["item"]["content"][0]["text"] = "x"
|
||||
.repeat(crate::system_input::MAX_SYSTEM_INPUT_BYTES + 1)
|
||||
.into();
|
||||
}
|
||||
assert!(decode(oversized));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn annotated_segment_start_is_restore_visible_without_projecting_metadata() {
|
||||
let session_id = uuid::Uuid::now_v7();
|
||||
|
||||
@@ -167,6 +167,7 @@ fn legacy_metadata(
|
||||
identity.extend_from_slice(&(line_index as u64).to_be_bytes());
|
||||
identity.extend_from_slice(&(item_index as u64).to_be_bytes());
|
||||
LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(format!(
|
||||
"l-{}",
|
||||
base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, identity)
|
||||
|
||||
@@ -22,6 +22,7 @@ fn seed(
|
||||
content: vec![crate::LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId("summary".into()),
|
||||
origin,
|
||||
derivation: Some(crate::LoggedHistoryDerivation {
|
||||
|
||||
@@ -1296,6 +1296,7 @@ mod tests {
|
||||
LoggedHistoryEntry {
|
||||
item,
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(id.to_owned()),
|
||||
origin,
|
||||
derivation: None,
|
||||
@@ -1537,6 +1538,7 @@ mod tests {
|
||||
prompt_provenance: None,
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId("system-item".into()),
|
||||
origin: LoggedSessionHistoryOrigin::BackendInstruction {
|
||||
operation_id: None,
|
||||
|
||||
@@ -148,15 +148,24 @@ struct SegmentHeader {
|
||||
file_len: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PublicMaterialUse {
|
||||
Recording,
|
||||
MemoryEvidence,
|
||||
}
|
||||
|
||||
/// A fixed material catalog. Opening a new catalog observes newly appended
|
||||
/// records; reading a body through this catalog verifies its saved digest and
|
||||
/// byte length again. Missing/corrected/deleted sources are explicit failures.
|
||||
/// Explicit evidence permission applies to model text, not the private payload;
|
||||
/// Web display and observation instead use the producer's public projection.
|
||||
pub struct SessionPublicMaterialReader {
|
||||
session_id: SessionId,
|
||||
active_segment_id: SegmentId,
|
||||
layout: Layout,
|
||||
limits: PublicMaterialLimits,
|
||||
materials: Vec<Material>,
|
||||
purpose: PublicMaterialUse,
|
||||
retention: Option<crate::retention::SessionRetentionStore>,
|
||||
}
|
||||
|
||||
@@ -180,7 +189,13 @@ impl SessionPublicMaterialReader {
|
||||
active_segment_id: SegmentId,
|
||||
limits: PublicMaterialLimits,
|
||||
) -> Result<Self, PublicMaterialError> {
|
||||
Self::open_worker_guarded(session_root, active_segment_id, limits, None)
|
||||
Self::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment_id,
|
||||
limits,
|
||||
None,
|
||||
PublicMaterialUse::Recording,
|
||||
)
|
||||
}
|
||||
|
||||
/// Uses the Host's aggregate-external control root for the whole scan and every body read.
|
||||
@@ -191,7 +206,30 @@ impl SessionPublicMaterialReader {
|
||||
limits: PublicMaterialLimits,
|
||||
retention: crate::retention::SessionRetentionStore,
|
||||
) -> Result<Self, PublicMaterialError> {
|
||||
Self::open_worker_guarded(session_root, active_segment_id, limits, Some(retention))
|
||||
Self::open_worker_for_use(
|
||||
session_root,
|
||||
active_segment_id,
|
||||
limits,
|
||||
retention,
|
||||
PublicMaterialUse::Recording,
|
||||
)
|
||||
}
|
||||
|
||||
/// Selects the evidence purpose independently of public display and observation.
|
||||
pub fn open_worker_for_use(
|
||||
session_root: &Path,
|
||||
active_segment_id: SegmentId,
|
||||
limits: PublicMaterialLimits,
|
||||
retention: crate::retention::SessionRetentionStore,
|
||||
purpose: PublicMaterialUse,
|
||||
) -> Result<Self, PublicMaterialError> {
|
||||
Self::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment_id,
|
||||
limits,
|
||||
Some(retention),
|
||||
purpose,
|
||||
)
|
||||
}
|
||||
|
||||
fn open_worker_guarded(
|
||||
@@ -199,6 +237,7 @@ impl SessionPublicMaterialReader {
|
||||
active_segment_id: SegmentId,
|
||||
limits: PublicMaterialLimits,
|
||||
retention: Option<crate::retention::SessionRetentionStore>,
|
||||
purpose: PublicMaterialUse,
|
||||
) -> Result<Self, PublicMaterialError> {
|
||||
let store = WorkerSessionStore::open_read_only(session_root)
|
||||
.map_err(|_| PublicMaterialError::Unavailable)?;
|
||||
@@ -212,6 +251,7 @@ impl SessionPublicMaterialReader {
|
||||
Layout::Worker(session_root.into()),
|
||||
limits,
|
||||
retention,
|
||||
purpose,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -237,6 +277,25 @@ impl SessionPublicMaterialReader {
|
||||
active_segment_id: SegmentId,
|
||||
limits: PublicMaterialLimits,
|
||||
retention: crate::retention::SessionRetentionStore,
|
||||
) -> Result<Self, PublicMaterialError> {
|
||||
Self::open_fs_for_use(
|
||||
sessions_root,
|
||||
session_id,
|
||||
active_segment_id,
|
||||
limits,
|
||||
retention,
|
||||
PublicMaterialUse::Recording,
|
||||
)
|
||||
}
|
||||
|
||||
/// Uses retained originals with the independently selected evidence policy.
|
||||
pub fn open_fs_for_use(
|
||||
sessions_root: &Path,
|
||||
session_id: SessionId,
|
||||
active_segment_id: SegmentId,
|
||||
limits: PublicMaterialLimits,
|
||||
retention: crate::retention::SessionRetentionStore,
|
||||
purpose: PublicMaterialUse,
|
||||
) -> Result<Self, PublicMaterialError> {
|
||||
Self::open(
|
||||
session_id,
|
||||
@@ -244,6 +303,7 @@ impl SessionPublicMaterialReader {
|
||||
Layout::Fs(sessions_root.join(session_id.to_string())),
|
||||
limits,
|
||||
Some(retention),
|
||||
purpose,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -263,6 +323,7 @@ impl SessionPublicMaterialReader {
|
||||
layout: Layout,
|
||||
limits: PublicMaterialLimits,
|
||||
retention: Option<crate::retention::SessionRetentionStore>,
|
||||
purpose: PublicMaterialUse,
|
||||
) -> Result<Self, PublicMaterialError> {
|
||||
let _guard = retention
|
||||
.as_ref()
|
||||
@@ -435,23 +496,19 @@ impl SessionPublicMaterialReader {
|
||||
} => turn = *total_turn_count,
|
||||
_ => {}
|
||||
}
|
||||
let system_history;
|
||||
let entries: &[LoggedHistoryEntry] = match &record {
|
||||
LogEntry::AnnotatedSegmentStart { history, .. }
|
||||
| LogEntry::AnnotatedUserInput { history, .. } => history,
|
||||
LogEntry::AnnotatedAssistantItem { entry, .. }
|
||||
| LogEntry::AnnotatedToolResult { entry, .. }
|
||||
| LogEntry::ToolResultCorrected { entry, .. } => std::slice::from_ref(entry),
|
||||
LogEntry::AnnotatedSystemItem { entry, .. } => {
|
||||
system_history = entry.to_history_entry();
|
||||
std::slice::from_ref(&system_history)
|
||||
}
|
||||
_ => &[],
|
||||
};
|
||||
if let LogEntry::AnnotatedSystemItem { entry, .. } = &record {
|
||||
charge(&mut examined, limits.max_entries)?;
|
||||
if entry.metadata.entry_id.0.is_empty()
|
||||
|| !history_seen.insert(entry.metadata.entry_id.0.clone())
|
||||
{
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
history_ids.push(entry.metadata.entry_id.0.clone());
|
||||
}
|
||||
for (history_index, entry) in entries.iter().enumerate() {
|
||||
charge(&mut examined, limits.max_entries)?;
|
||||
let id = &entry.metadata.entry_id.0;
|
||||
@@ -468,7 +525,7 @@ impl SessionPublicMaterialReader {
|
||||
if in_run && ai_materialized(&entry.item) {
|
||||
safe = true;
|
||||
}
|
||||
let Some((sha256, total_bytes)) = canonical_metadata(entry)? else {
|
||||
let Some((sha256, total_bytes)) = canonical_metadata(entry, purpose)? else {
|
||||
continue;
|
||||
};
|
||||
let location = Location {
|
||||
@@ -621,6 +678,7 @@ impl SessionPublicMaterialReader {
|
||||
layout,
|
||||
limits,
|
||||
materials,
|
||||
purpose,
|
||||
retention: retention.clone(),
|
||||
})
|
||||
}
|
||||
@@ -660,6 +718,7 @@ impl SessionPublicMaterialReader {
|
||||
let Some((_, record)) = reader.next()? else {
|
||||
return Err(PublicMaterialError::Missing);
|
||||
};
|
||||
let system_history;
|
||||
let (body, origin) = if let Some(boundary) = run_boundary(&record) {
|
||||
if boundary_reference(
|
||||
&record,
|
||||
@@ -683,12 +742,16 @@ impl SessionPublicMaterialReader {
|
||||
LogEntry::AnnotatedAssistantItem { entry, .. }
|
||||
| LogEntry::AnnotatedToolResult { entry, .. }
|
||||
| LogEntry::ToolResultCorrected { entry, .. } => Some(entry),
|
||||
LogEntry::AnnotatedSystemItem { entry, .. } => {
|
||||
system_history = entry.to_history_entry();
|
||||
Some(&system_history)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
.filter(|e| format!("E{}", e.metadata.entry_id.0) == descriptor.identity.entry_ref)
|
||||
.ok_or(PublicMaterialError::Changed)?;
|
||||
(
|
||||
canonical_body(entry).ok_or(PublicMaterialError::Changed)?,
|
||||
canonical_body(entry, self.purpose).ok_or(PublicMaterialError::Changed)?,
|
||||
PublicMaterialOrigin::History {
|
||||
origin: entry.metadata.origin.clone(),
|
||||
},
|
||||
@@ -828,7 +891,32 @@ fn boundary_reference(
|
||||
.unwrap_or_else(|| format!("El-run-{session}-{segment}-{offset}"))
|
||||
}
|
||||
|
||||
fn canonical_body(entry: &LoggedHistoryEntry) -> Option<CanonicalBody<'_>> {
|
||||
fn canonical_body(
|
||||
entry: &LoggedHistoryEntry,
|
||||
purpose: PublicMaterialUse,
|
||||
) -> Option<CanonicalBody<'_>> {
|
||||
if let Some(input) = &entry.metadata.system_input {
|
||||
let usage = input.usage();
|
||||
let allowed = match purpose {
|
||||
PublicMaterialUse::Recording => usage.recording_material,
|
||||
PublicMaterialUse::MemoryEvidence => usage.memory_evidence,
|
||||
};
|
||||
if allowed == crate::InputEvidenceUse::Exclude {
|
||||
return None;
|
||||
}
|
||||
if let LoggedItem::Message {
|
||||
role: LoggedRole::System,
|
||||
content,
|
||||
} = &entry.item
|
||||
{
|
||||
return Some(CanonicalBody::Message {
|
||||
origin: &entry.metadata.origin,
|
||||
role: LoggedRole::System,
|
||||
content,
|
||||
});
|
||||
}
|
||||
return None;
|
||||
}
|
||||
if matches!(
|
||||
entry.metadata.origin,
|
||||
LoggedSessionHistoryOrigin::DerivedSummary
|
||||
@@ -880,8 +968,9 @@ fn canonical_body(entry: &LoggedHistoryEntry) -> Option<CanonicalBody<'_>> {
|
||||
|
||||
fn canonical_metadata(
|
||||
entry: &LoggedHistoryEntry,
|
||||
purpose: PublicMaterialUse,
|
||||
) -> Result<Option<(String, u64)>, PublicMaterialError> {
|
||||
let Some(body) = canonical_body(entry) else {
|
||||
let Some(body) = canonical_body(entry, purpose) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut sink = BodySink::new(0, 0);
|
||||
|
||||
@@ -9,6 +9,7 @@ fn entry(id: &str, item: LoggedItem) -> LoggedHistoryEntry {
|
||||
LoggedHistoryEntry {
|
||||
item,
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(id.into()),
|
||||
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
@@ -591,6 +592,7 @@ fn reasoning_proves_rollback_safety_but_private_and_derived_items_are_not_materi
|
||||
},
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId("startup-journal".into()),
|
||||
origin: LoggedSessionHistoryOrigin::DerivedSummary,
|
||||
derivation: None,
|
||||
@@ -1243,3 +1245,165 @@ fn compaction_without_exact_derivation_or_with_invalid_turn_is_rejected() {
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_system_material_uses_explicit_evidence_purpose_not_producer_or_body() {
|
||||
use crate::{
|
||||
FeatureInputPayload, InputCompactionUse, InputEvidenceUse, InputObservationUse,
|
||||
LoggedSystemHistoryEntry, SystemInputEnvelope, SystemInputMetadata, SystemInputUsePolicy,
|
||||
};
|
||||
for worker_layout in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let session = crate::new_session_id();
|
||||
let segment = crate::new_segment_id();
|
||||
let fork = crate::new_segment_id();
|
||||
let make = |producer: &str, recording_material, memory_evidence, body: &str| {
|
||||
LoggedSystemHistoryEntry::from_input(
|
||||
LoggedSessionHistoryOrigin::Provided {
|
||||
producer: producer.into(),
|
||||
},
|
||||
None,
|
||||
SystemInputEnvelope::new(
|
||||
body,
|
||||
SystemInputMetadata::new(
|
||||
SystemInputUsePolicy {
|
||||
recording_material,
|
||||
memory_evidence,
|
||||
observation: InputObservationUse::Hidden,
|
||||
compaction: InputCompactionUse::Retain,
|
||||
},
|
||||
Some(
|
||||
FeatureInputPayload::encode(
|
||||
"unknown:future",
|
||||
99,
|
||||
&serde_json::json!({"private": "not-public"}),
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
};
|
||||
let inputs = [
|
||||
make(
|
||||
"same-owner",
|
||||
InputEvidenceUse::Include,
|
||||
InputEvidenceUse::Exclude,
|
||||
"do not extract this; original material",
|
||||
),
|
||||
make(
|
||||
"same-owner",
|
||||
InputEvidenceUse::Exclude,
|
||||
InputEvidenceUse::Include,
|
||||
"memory-only original",
|
||||
),
|
||||
make(
|
||||
"renamed-owner",
|
||||
InputEvidenceUse::Exclude,
|
||||
InputEvidenceUse::Exclude,
|
||||
"reference without warning text",
|
||||
),
|
||||
];
|
||||
let root = if worker_layout {
|
||||
temp.path().join("worker")
|
||||
} else {
|
||||
temp.path().join(session.to_string())
|
||||
};
|
||||
let store: Box<dyn Store> = if worker_layout {
|
||||
Box::new(WorkerSessionStore::new(&root).unwrap())
|
||||
} else {
|
||||
Box::new(FsStore::new(temp.path()).unwrap())
|
||||
};
|
||||
store
|
||||
.create_segment(session, segment, &[seed(session, vec![])])
|
||||
.unwrap();
|
||||
for entry in &inputs {
|
||||
store
|
||||
.append(
|
||||
session,
|
||||
segment,
|
||||
&LogEntry::AnnotatedSystemItem {
|
||||
ts: 2,
|
||||
entry: entry.clone(),
|
||||
extensions: vec![],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
store
|
||||
.append(
|
||||
session,
|
||||
segment,
|
||||
&LogEntry::TurnEnd {
|
||||
ts: 3,
|
||||
turn_count: 1,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let mut fork_seed = seed(
|
||||
session,
|
||||
inputs
|
||||
.iter()
|
||||
.map(LoggedSystemHistoryEntry::to_history_entry)
|
||||
.collect(),
|
||||
);
|
||||
if let LogEntry::AnnotatedSegmentStart { forked_from, .. } = &mut fork_seed {
|
||||
*forked_from = Some(crate::SegmentOrigin {
|
||||
segment_id: segment,
|
||||
at_turn_index: 1,
|
||||
});
|
||||
}
|
||||
store.create_segment(session, fork, &[fork_seed]).unwrap();
|
||||
let open = |active, purpose| {
|
||||
if worker_layout {
|
||||
SessionPublicMaterialReader::open_worker_for_use(
|
||||
&root,
|
||||
active,
|
||||
Default::default(),
|
||||
crate::retention::fs_retention(temp.path()),
|
||||
purpose,
|
||||
)
|
||||
.unwrap()
|
||||
} else {
|
||||
SessionPublicMaterialReader::open_fs_for_use(
|
||||
temp.path(),
|
||||
session,
|
||||
active,
|
||||
Default::default(),
|
||||
crate::retention::fs_retention(temp.path()),
|
||||
purpose,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
for (purpose, selected, expected_text) in [
|
||||
(
|
||||
PublicMaterialUse::Recording,
|
||||
0,
|
||||
"do not extract this; original material",
|
||||
),
|
||||
(PublicMaterialUse::MemoryEvidence, 1, "memory-only original"),
|
||||
] {
|
||||
let original = open(segment, purpose);
|
||||
let inherited = open(fork, purpose);
|
||||
assert_eq!(
|
||||
ids(&original),
|
||||
vec![format!("E{}", inputs[selected].metadata.entry_id.0)]
|
||||
);
|
||||
assert_eq!(ids(&inherited), ids(&original));
|
||||
let descriptor = inherited.descriptors().next().unwrap();
|
||||
let body = inherited
|
||||
.read_content(descriptor, 0, 16 * 1024)
|
||||
.unwrap()
|
||||
.content;
|
||||
let value: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(value["role"], "system");
|
||||
assert_eq!(value["content"][0]["text"], expected_text);
|
||||
assert!(!body.contains("not-public"));
|
||||
assert!(!body.contains("unknown:future"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use thiserror::Error;
|
||||
|
||||
use crate::{
|
||||
LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole,
|
||||
LoggedSessionHistoryOrigin, SegmentId, SegmentOrigin, SessionId, StoreError, SystemItem,
|
||||
LoggedSessionHistoryOrigin, SegmentId, SegmentOrigin, SessionId, StoreError,
|
||||
WorkerAggregateStore, WorkerSessionStore, WorkerStoreError,
|
||||
};
|
||||
|
||||
@@ -1165,7 +1165,7 @@ fn capture_tool_correction(
|
||||
let LogEntry::ToolResultCorrected { entry, .. } = record else {
|
||||
return false;
|
||||
};
|
||||
if let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) {
|
||||
if let Some(data) = project_history_item(entry) {
|
||||
// We read backwards: the first correction seen is the latest one.
|
||||
corrections
|
||||
.entry(entry.metadata.entry_id.0.clone())
|
||||
@@ -1600,12 +1600,12 @@ pub(crate) fn project_session_snapshot_for_segment(
|
||||
} => extend_history(&mut entries, history, Some(segments), *ts),
|
||||
LogEntry::AnnotatedAssistantItem { ts, entry }
|
||||
| LogEntry::AnnotatedToolResult { ts, entry } => {
|
||||
if let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) {
|
||||
if let Some(data) = project_history_item(entry) {
|
||||
entries.push(history_entry(entry, *ts, data));
|
||||
}
|
||||
}
|
||||
LogEntry::ToolResultCorrected { ts, entry } => {
|
||||
if let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) {
|
||||
if let Some(data) = project_history_item(entry) {
|
||||
if let Some(existing) = entries
|
||||
.iter_mut()
|
||||
.find(|existing| existing.entry_id == entry.metadata.entry_id.0)
|
||||
@@ -1618,11 +1618,16 @@ pub(crate) fn project_session_snapshot_for_segment(
|
||||
}
|
||||
}
|
||||
LogEntry::AnnotatedSystemItem { ts, entry, .. } => entries.push(system_entry(
|
||||
&entry.item,
|
||||
&entry.public_item(),
|
||||
entry.metadata.entry_id.0.clone(),
|
||||
*ts,
|
||||
provenance(&entry.metadata.origin),
|
||||
derivation_ids(entry),
|
||||
entry
|
||||
.metadata
|
||||
.system_input
|
||||
.as_ref()
|
||||
.map(crate::SystemInputMetadata::public_use_policy),
|
||||
)),
|
||||
LogEntry::RunYielded {
|
||||
ts,
|
||||
@@ -1775,7 +1780,7 @@ fn extend_history(
|
||||
segments: input_segments.cloned().unwrap_or_default(),
|
||||
}
|
||||
} else {
|
||||
let Some(data) = project_item(&entry.metadata.entry_id.0, &entry.item) else {
|
||||
let Some(data) = project_history_item(entry) else {
|
||||
continue;
|
||||
};
|
||||
data
|
||||
@@ -1790,6 +1795,11 @@ fn history_entry(
|
||||
data: SessionSnapshotEntryData,
|
||||
) -> SessionSnapshotEntry {
|
||||
SessionSnapshotEntry {
|
||||
use_policy: entry
|
||||
.metadata
|
||||
.system_input
|
||||
.as_ref()
|
||||
.map(crate::SystemInputMetadata::public_use_policy),
|
||||
entry_id: entry.metadata.entry_id.0.clone(),
|
||||
timestamp,
|
||||
provenance: provenance(&entry.metadata.origin),
|
||||
@@ -1833,6 +1843,7 @@ fn run_transition_entry(
|
||||
data: SessionSnapshotEntryData,
|
||||
) -> SessionSnapshotEntry {
|
||||
SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: persisted_entry_id
|
||||
.map(|entry_id| entry_id.0.clone())
|
||||
.unwrap_or_else(|| legacy_entry_id(session_key, segment_id, log_index, 0)),
|
||||
@@ -1852,6 +1863,7 @@ fn legacy_entry(
|
||||
data: SessionSnapshotEntryData,
|
||||
) -> SessionSnapshotEntry {
|
||||
SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: legacy_entry_id(session_key, segment_id, log_index, item_index),
|
||||
timestamp,
|
||||
provenance: SessionEntryProvenance::LegacyUnknown,
|
||||
@@ -1884,9 +1896,8 @@ fn provenance(origin: &LoggedSessionHistoryOrigin) -> SessionEntryProvenance {
|
||||
LoggedSessionHistoryOrigin::FlowInstruction { .. } => {
|
||||
SessionEntryProvenance::FlowInstruction
|
||||
}
|
||||
LoggedSessionHistoryOrigin::BackendInstruction { .. } => {
|
||||
SessionEntryProvenance::BackendInstruction
|
||||
}
|
||||
LoggedSessionHistoryOrigin::BackendInstruction { .. }
|
||||
| LoggedSessionHistoryOrigin::Provided { .. } => SessionEntryProvenance::BackendInstruction,
|
||||
LoggedSessionHistoryOrigin::ModelOutput { .. } => SessionEntryProvenance::ModelOutput,
|
||||
LoggedSessionHistoryOrigin::ToolOutput { .. } => SessionEntryProvenance::ToolOutput,
|
||||
LoggedSessionHistoryOrigin::CompactionSummary
|
||||
@@ -1914,6 +1925,18 @@ pub fn session_tool_attachment_id(
|
||||
URL_SAFE_NO_PAD.encode(hash.finalize())
|
||||
}
|
||||
|
||||
fn project_history_item(entry: &LoggedHistoryEntry) -> Option<SessionSnapshotEntryData> {
|
||||
if let Some(input) = &entry.metadata.system_input {
|
||||
let public = input.public_projection();
|
||||
return Some(SessionSnapshotEntryData::SystemItem {
|
||||
item_kind: public.kind.clone(),
|
||||
content: public.body.clone(),
|
||||
data: Some(serde_json::to_value(public).expect("public system input is serializable")),
|
||||
});
|
||||
}
|
||||
project_item(&entry.metadata.entry_id.0, &entry.item)
|
||||
}
|
||||
|
||||
fn project_item(entry_id: &str, item: &LoggedItem) -> Option<SessionSnapshotEntryData> {
|
||||
match item {
|
||||
LoggedItem::Message { role, content } => {
|
||||
@@ -1986,17 +2009,19 @@ fn project_item(entry_id: &str, item: &LoggedItem) -> Option<SessionSnapshotEntr
|
||||
}
|
||||
|
||||
fn system_entry(
|
||||
item: &SystemItem,
|
||||
item: &crate::PublicSystemItem,
|
||||
entry_id: String,
|
||||
timestamp: u64,
|
||||
provenance: SessionEntryProvenance,
|
||||
derived_from: Vec<String>,
|
||||
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||
) -> SessionSnapshotEntry {
|
||||
let public = item.public_item();
|
||||
let public = item.clone();
|
||||
let data = Some(serde_json::to_value(&public).expect("public system input is serializable"));
|
||||
let item_kind = public.kind;
|
||||
let content = public.body;
|
||||
SessionSnapshotEntry {
|
||||
use_policy,
|
||||
entry_id,
|
||||
timestamp,
|
||||
provenance,
|
||||
@@ -2285,6 +2310,7 @@ mod tests {
|
||||
content: vec![LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(entry_id.into()),
|
||||
origin,
|
||||
derivation: None,
|
||||
@@ -3583,6 +3609,7 @@ mod tests {
|
||||
.map(|item| LoggedHistoryEntry {
|
||||
item,
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
@@ -3637,6 +3664,7 @@ mod tests {
|
||||
}],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: user_entry_id,
|
||||
origin: LoggedSessionHistoryOrigin::HumanInput {
|
||||
account_id: "account-1".into(),
|
||||
@@ -3706,6 +3734,7 @@ mod tests {
|
||||
}],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::FlowInstruction {
|
||||
selector: "builtin:coder-review".into(),
|
||||
@@ -3725,6 +3754,7 @@ mod tests {
|
||||
}],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: user_entry_id.clone(),
|
||||
origin: origin.clone(),
|
||||
derivation: Some(LoggedHistoryDerivation {
|
||||
@@ -3755,6 +3785,7 @@ mod tests {
|
||||
fn annotated_projection_preserves_identity_and_provenance() {
|
||||
let session_id = crate::new_session_id();
|
||||
let metadata = LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::ModelOutput {
|
||||
worker: LoggedWorkerSubject {
|
||||
@@ -4317,6 +4348,7 @@ mod tests {
|
||||
is_error: false,
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId("ancestor-image".into()),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
@@ -4411,6 +4443,7 @@ mod tests {
|
||||
is_error: false,
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId("tool-images".into()),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -103,6 +103,7 @@ fn finish(
|
||||
stats.insert(
|
||||
anchor.clone(),
|
||||
SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: format!("run-stats:{anchor}"),
|
||||
timestamp: ended_at,
|
||||
provenance: SessionEntryProvenance::LegacyUnknown,
|
||||
|
||||
@@ -254,6 +254,23 @@ pub struct SegmentOrigin {
|
||||
}
|
||||
|
||||
impl LogEntry {
|
||||
/// Validate generic System records before a Store publishes any bytes.
|
||||
pub fn validate_system_inputs(&self) -> Result<(), crate::SystemInputError> {
|
||||
match self {
|
||||
Self::AnnotatedSegmentStart { history, .. }
|
||||
| Self::AnnotatedUserInput { history, .. } => {
|
||||
for entry in history {
|
||||
entry.validate_system_input()?;
|
||||
}
|
||||
}
|
||||
Self::AnnotatedAssistantItem { entry, .. }
|
||||
| Self::AnnotatedToolResult { entry, .. }
|
||||
| Self::ToolResultCorrected { entry, .. } => entry.validate_system_input()?,
|
||||
Self::AnnotatedSystemItem { entry, .. } => entry.validate_system_input()?,
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Allocate the durable public identity for a newly committed Run
|
||||
/// transition. Calling this repeatedly is idempotent so the exact value
|
||||
/// written to storage is also the value published to live subscribers.
|
||||
@@ -537,6 +554,7 @@ mod tests {
|
||||
LoggedHistoryEntry {
|
||||
item: LoggedItem::from(item),
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -27,6 +27,9 @@ pub enum StoreError {
|
||||
#[error("serialization error: {0}")]
|
||||
Serde(#[from] serde_json::Error),
|
||||
|
||||
#[error("invalid system input: {0}")]
|
||||
InvalidSystemInput(#[from] crate::SystemInputError),
|
||||
|
||||
#[error("segment not found: {0}")]
|
||||
NotFound(SegmentId),
|
||||
|
||||
|
||||
@@ -36,6 +36,10 @@ pub enum SystemInputError {
|
||||
UnsupportedPayload,
|
||||
#[error("system input payload does not match the owner's typed contract")]
|
||||
InvalidTypedPayload,
|
||||
#[error("generic system input is missing explicit metadata")]
|
||||
MissingMetadata,
|
||||
#[error("system input metadata does not match its logical record")]
|
||||
InconsistentRecord,
|
||||
}
|
||||
|
||||
/// One private payload owned by a Feature's versioned typed decoder.
|
||||
@@ -229,6 +233,23 @@ impl SystemInputMetadata {
|
||||
pub fn public_item(&self) -> Option<&PublicSystemItem> {
|
||||
self.public.as_ref()
|
||||
}
|
||||
|
||||
pub fn public_use_policy(&self) -> protocol::SessionEntryUsePolicy {
|
||||
let has_public = self.public.is_some();
|
||||
protocol::SessionEntryUsePolicy {
|
||||
observation_readable: has_public
|
||||
&& self.usage.observation == InputObservationUse::Readable,
|
||||
memory_evidence: has_public && self.usage.memory_evidence == InputEvidenceUse::Include,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn public_projection(&self) -> PublicSystemItem {
|
||||
self.public.clone().unwrap_or_else(|| PublicSystemItem {
|
||||
kind: "input".into(),
|
||||
body: String::new(),
|
||||
presentation: crate::PublicSystemPresentation::Ignored,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// System role is invariant; provenance and use policies are attached data.
|
||||
|
||||
@@ -92,6 +92,9 @@ pub struct PromptRenderProvenance {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum SystemItem {
|
||||
/// Generic System body; private metadata and public presentation are supplied
|
||||
/// independently by the producer in the same annotated history record.
|
||||
Input { body: String },
|
||||
/// One-time derived Journal reference context. The bounded Host snapshot is
|
||||
/// retained as JSON independently of its rendered body and prompt projection.
|
||||
JournalStartup {
|
||||
@@ -227,9 +230,9 @@ impl SystemItem {
|
||||
event: event.clone(),
|
||||
},
|
||||
Self::TaskReminder { .. } => PublicSystemPresentation::TaskReminder,
|
||||
Self::LegacyKnowledgeIgnored { .. } | Self::LegacyIgnored { .. } => {
|
||||
PublicSystemPresentation::Ignored
|
||||
}
|
||||
Self::Input { .. }
|
||||
| Self::LegacyKnowledgeIgnored { .. }
|
||||
| Self::LegacyIgnored { .. } => PublicSystemPresentation::Ignored,
|
||||
Self::JournalStartup { .. }
|
||||
| Self::ResidentSummaryRefresh { .. }
|
||||
| Self::SubjectBehaviorRefresh { .. }
|
||||
@@ -253,6 +256,7 @@ impl SystemItem {
|
||||
/// for this item. Returns the variant's stored `body` verbatim.
|
||||
pub fn history_text(&self) -> String {
|
||||
match self {
|
||||
SystemItem::Input { body } => body.clone(),
|
||||
SystemItem::JournalStartup { body, .. } => body.clone(),
|
||||
SystemItem::Notification { body, .. } => body.clone(),
|
||||
SystemItem::WorkerEvent { body, .. } => body.clone(),
|
||||
@@ -280,6 +284,7 @@ impl SystemItem {
|
||||
/// wire — keep flexible.
|
||||
pub fn kind_label(&self) -> &'static str {
|
||||
match self {
|
||||
SystemItem::Input { .. } => "input",
|
||||
SystemItem::JournalStartup { .. } => "journal_startup",
|
||||
SystemItem::Notification { .. } => "notification",
|
||||
SystemItem::WorkerEvent { .. } => "worker_event",
|
||||
|
||||
@@ -742,6 +742,7 @@ impl Store for WorkerSessionStore {
|
||||
segment_id: SegmentId,
|
||||
entry: &LogEntry,
|
||||
) -> Result<(), StoreError> {
|
||||
entry.validate_system_inputs()?;
|
||||
let _retention = self.lock_publication(session_id)?;
|
||||
self.ensure_session(session_id, true)?;
|
||||
self.append_log_entry(&self.log_path(segment_id), entry)
|
||||
@@ -800,6 +801,9 @@ impl Store for WorkerSessionStore {
|
||||
segment_id: SegmentId,
|
||||
entries: &[LogEntry],
|
||||
) -> Result<(), StoreError> {
|
||||
for entry in entries {
|
||||
entry.validate_system_inputs()?;
|
||||
}
|
||||
let _retention = self.lock_publication(session_id)?;
|
||||
self.ensure_session(session_id, true)?;
|
||||
let mut content = Vec::new();
|
||||
@@ -1430,6 +1434,7 @@ mod tests {
|
||||
LoggedHistoryEntry {
|
||||
item: LoggedItem::from(item),
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -11,6 +11,7 @@ fn annotated(item: Item) -> LoggedHistoryEntry {
|
||||
LoggedHistoryEntry {
|
||||
item: LoggedItem::from(item),
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
@@ -421,3 +422,178 @@ fn lookup_session_of_finds_owning_session() {
|
||||
|
||||
assert_eq!(store.lookup_session_of(segid).unwrap(), Some(sid));
|
||||
}
|
||||
|
||||
// Storage/projection component test: both production layouts, temporary originals,
|
||||
// no Feature, model, runtime or Host authority is substituted into this boundary.
|
||||
#[test]
|
||||
fn generic_system_append_and_fork_preserve_metadata_without_public_payload_leak() {
|
||||
use session_store::{
|
||||
FeatureInputPayload, InputCompactionUse, InputEvidenceUse, InputObservationUse,
|
||||
LoggedHistoryDerivation, LoggedSystemHistoryEntry, PublicSystemItem,
|
||||
PublicSystemPresentation, SystemInputEnvelope, SystemInputMetadata, SystemInputUsePolicy,
|
||||
WorkerSessionStore,
|
||||
};
|
||||
for worker_layout in [false, true] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let sid = new_session_id();
|
||||
let segment = new_segment_id();
|
||||
let fork = new_segment_id();
|
||||
let store: Box<dyn Store> = if worker_layout {
|
||||
Box::new(WorkerSessionStore::new(dir.path()).unwrap())
|
||||
} else {
|
||||
Box::new(FsStore::new(dir.path()).unwrap())
|
||||
};
|
||||
store
|
||||
.create_segment(sid, segment, &[nil_session_start(1, sid)])
|
||||
.unwrap();
|
||||
let entry = LoggedSystemHistoryEntry::from_input(
|
||||
LoggedSessionHistoryOrigin::Provided {
|
||||
producer: "new-feature".into(),
|
||||
},
|
||||
Some(LoggedHistoryDerivation {
|
||||
sources: vec![LoggedSessionHistoryEntryId("source-entry".into())],
|
||||
}),
|
||||
SystemInputEnvelope::new(
|
||||
"private model body",
|
||||
SystemInputMetadata::new(
|
||||
SystemInputUsePolicy {
|
||||
recording_material: InputEvidenceUse::Exclude,
|
||||
memory_evidence: InputEvidenceUse::Exclude,
|
||||
observation: InputObservationUse::Hidden,
|
||||
compaction: InputCompactionUse::ReferenceInput,
|
||||
},
|
||||
Some(
|
||||
FeatureInputPayload::encode(
|
||||
"new-feature:context",
|
||||
101,
|
||||
&serde_json::json!({"private": "secret-detail"}),
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
Some(PublicSystemItem {
|
||||
kind: "context".into(),
|
||||
body: "explicit public body".into(),
|
||||
presentation: PublicSystemPresentation::Context,
|
||||
}),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
store
|
||||
.append(
|
||||
sid,
|
||||
segment,
|
||||
&LogEntry::AnnotatedSystemItem {
|
||||
ts: 2,
|
||||
entry: entry.clone(),
|
||||
extensions: vec![],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let saved = store.read_all(sid, segment).unwrap();
|
||||
let LogEntry::AnnotatedSystemItem {
|
||||
entry: restored, ..
|
||||
} = &saved[1]
|
||||
else {
|
||||
panic!("missing system input")
|
||||
};
|
||||
assert_eq!(restored.to_history_entry(), entry.to_history_entry());
|
||||
assert_eq!(
|
||||
collect_state(&saved).history[0].as_text(),
|
||||
Some("private model body")
|
||||
);
|
||||
let mut fork_seed = nil_session_start(3, sid);
|
||||
if let LogEntry::AnnotatedSegmentStart {
|
||||
history,
|
||||
forked_from,
|
||||
..
|
||||
} = &mut fork_seed
|
||||
{
|
||||
history.push(restored.to_history_entry());
|
||||
*forked_from = Some(session_store::SegmentOrigin {
|
||||
segment_id: segment,
|
||||
at_turn_index: 0,
|
||||
});
|
||||
}
|
||||
store.create_segment(sid, fork, &[fork_seed]).unwrap();
|
||||
let fork_saved = store.read_all(sid, fork).unwrap();
|
||||
let LogEntry::AnnotatedSegmentStart { history, .. } = &fork_saved[0] else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(history, &[entry.to_history_entry()]);
|
||||
let original = session_store::public_snapshot::project_session_snapshot(sid, &saved);
|
||||
let inherited = session_store::public_snapshot::project_session_snapshot(sid, &fork_saved);
|
||||
let public = serde_json::to_string(&original.entries).unwrap();
|
||||
assert!(public.contains("explicit public body"));
|
||||
for private in ["private model body", "secret-detail", "new-feature:context"] {
|
||||
assert!(!public.contains(private), "private field leaked: {private}");
|
||||
}
|
||||
assert_eq!(original.entries.len(), 1);
|
||||
assert_eq!(inherited.entries.len(), 1);
|
||||
assert_eq!(original.entries[0].entry_id, inherited.entries[0].entry_id);
|
||||
assert_eq!(
|
||||
original.entries[0].derived_from,
|
||||
inherited.entries[0].derived_from
|
||||
);
|
||||
assert_eq!(
|
||||
original.entries[0].provenance,
|
||||
inherited.entries[0].provenance
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&original.entries[0].data).unwrap(),
|
||||
serde_json::to_value(&inherited.entries[0].data).unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_generic_system_input_rejects_append_create_and_external_json_without_changing_originals()
|
||||
{
|
||||
use session_store::{StoreError, SystemItem, WorkerSessionStore};
|
||||
for worker_layout in [false, true] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let sid = new_session_id();
|
||||
let segment = new_segment_id();
|
||||
let store: Box<dyn Store> = if worker_layout {
|
||||
Box::new(WorkerSessionStore::new(dir.path()).unwrap())
|
||||
} else {
|
||||
Box::new(FsStore::new(dir.path()).unwrap())
|
||||
};
|
||||
let seed = nil_session_start(1, sid);
|
||||
store
|
||||
.create_segment(sid, segment, std::slice::from_ref(&seed))
|
||||
.unwrap();
|
||||
let path = if worker_layout {
|
||||
dir.path().join("segments").join(format!("{segment}.jsonl"))
|
||||
} else {
|
||||
dir.path()
|
||||
.join(sid.to_string())
|
||||
.join(format!("{segment}.jsonl"))
|
||||
};
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
let invalid = LogEntry::AnnotatedSystemItem {
|
||||
ts: 2,
|
||||
extensions: vec![],
|
||||
entry: session_store::LoggedSystemHistoryEntry {
|
||||
item: SystemItem::Input {
|
||||
body: "no policy".into(),
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata::legacy_unknown(),
|
||||
},
|
||||
};
|
||||
assert!(matches!(
|
||||
store.append(sid, segment, &invalid),
|
||||
Err(StoreError::InvalidSystemInput(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
store.create_segment(sid, segment, &[seed, invalid.clone()]),
|
||||
Err(StoreError::InvalidSystemInput(_))
|
||||
));
|
||||
assert!(
|
||||
serde_json::from_value::<LogEntry>(serde_json::to_value(invalid).unwrap()).is_err()
|
||||
);
|
||||
assert_eq!(std::fs::read(path).unwrap(), before);
|
||||
assert_eq!(store.read_all(sid, segment).unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ fn annotated(items: &[Item]) -> Vec<session_store::LoggedHistoryEntry> {
|
||||
.map(|item| session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::from(item),
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId::new(),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -212,6 +212,7 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry {
|
||||
content: vec![session_store::LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(id.into()),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
|
||||
@@ -1621,6 +1621,7 @@ allow = []
|
||||
let history = |id: &str, item: LoggedItem| LoggedHistoryEntry {
|
||||
item,
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(id.into()),
|
||||
origin: LoggedSessionHistoryOrigin::BackendInstruction { operation_id: None },
|
||||
derivation: None,
|
||||
|
||||
@@ -3567,6 +3567,7 @@ mod completion_flow_tests {
|
||||
session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::from(item),
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId::new(),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
@@ -4324,6 +4325,7 @@ mod completion_flow_tests {
|
||||
}));
|
||||
app.handle_worker_event(Event::SessionEntryCommitted {
|
||||
entry: protocol::SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: "run-failure".into(),
|
||||
timestamp: 2,
|
||||
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
|
||||
|
||||
@@ -35,6 +35,7 @@ fn message(id: &str, role: LoggedRole, text: &str) -> LoggedHistoryEntry {
|
||||
content: vec![LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(id.into()),
|
||||
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
|
||||
@@ -1831,6 +1831,7 @@ mod tests {
|
||||
}],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId("original".into()),
|
||||
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
@@ -1844,6 +1845,7 @@ mod tests {
|
||||
}],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId("summary".into()),
|
||||
origin: LoggedSessionHistoryOrigin::DerivedSummary,
|
||||
derivation: Some(session_store::LoggedHistoryDerivation {
|
||||
|
||||
@@ -12422,6 +12422,7 @@ mod tests {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![protocol::SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: "restored-log-entry".to_owned(),
|
||||
timestamp: 1,
|
||||
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
|
||||
|
||||
@@ -398,6 +398,7 @@ mod tests {
|
||||
content: vec![LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(id.to_string()),
|
||||
origin: LoggedSessionHistoryOrigin::HumanInput {
|
||||
account_id: "account-1".to_string(),
|
||||
|
||||
@@ -5033,6 +5033,7 @@ mod tests {
|
||||
}],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id,
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -371,7 +371,7 @@ fn parse_entry_ref(value: &str) -> Result<SessionEntryRef, ToolError> {
|
||||
fn parse_kind(value: &str) -> Result<ReferenceKind, ToolError> {
|
||||
ReferenceKind::parse(value).ok_or_else(|| {
|
||||
ToolError::InvalidArgument(format!(
|
||||
"invalid kind {value:?}; expected user, assistant, or tool"
|
||||
"invalid kind {value:?}; expected user, assistant, system, or tool"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -813,6 +813,7 @@ mod tests {
|
||||
_ => return None,
|
||||
};
|
||||
Some(protocol::SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: format!("fake-{index:08}"),
|
||||
timestamp: index as u64,
|
||||
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
|
||||
|
||||
@@ -1499,6 +1499,7 @@ permission = "write"
|
||||
is_error: false,
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(entry_id.into()),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -90,7 +90,7 @@ pub fn live_log_entry_event(entry: LogEntry) -> Option<Event> {
|
||||
}
|
||||
LogEntry::AnnotatedSystemItem { entry, .. } => {
|
||||
let entry_id = Some(entry.metadata.entry_id.0.clone());
|
||||
let value = serde_json::to_value(entry.item.public_item())
|
||||
let value = serde_json::to_value(entry.public_item())
|
||||
.expect("public system input is serializable");
|
||||
Some(Event::SystemItem {
|
||||
entry_id,
|
||||
@@ -348,6 +348,7 @@ mod tests {
|
||||
prompt_provenance: None,
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: entry_id.clone(),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
|
||||
operation_id: None,
|
||||
|
||||
@@ -74,6 +74,7 @@ impl std::fmt::Display for SessionEntryRef {
|
||||
pub(crate) enum ReferenceKind {
|
||||
User,
|
||||
Assistant,
|
||||
System,
|
||||
Tool,
|
||||
}
|
||||
|
||||
@@ -82,6 +83,7 @@ impl ReferenceKind {
|
||||
match self {
|
||||
Self::User => "user",
|
||||
Self::Assistant => "assistant",
|
||||
Self::System => "system",
|
||||
Self::Tool => "tool",
|
||||
}
|
||||
}
|
||||
@@ -90,6 +92,7 @@ impl ReferenceKind {
|
||||
match value {
|
||||
"user" => Some(Self::User),
|
||||
"assistant" | "agent" => Some(Self::Assistant),
|
||||
"system" => Some(Self::System),
|
||||
"tool" => Some(Self::Tool),
|
||||
_ => None,
|
||||
}
|
||||
@@ -131,6 +134,7 @@ pub(crate) struct OverviewItem {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ReferenceEntry {
|
||||
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||
pub id: SessionEntryRef,
|
||||
pub origin: SessionEntryProvenance,
|
||||
pub entry_range: [u64; 2],
|
||||
@@ -242,6 +246,7 @@ pub(crate) struct SessionEntryEvidence {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CapturedHistoryEntry {
|
||||
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||
item: Item,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId,
|
||||
origin: SessionEntryProvenance,
|
||||
@@ -302,10 +307,12 @@ impl SessionCapture {
|
||||
is_error,
|
||||
attachments: _,
|
||||
} => Item::tool_result_item(call_id, summary, content, is_error),
|
||||
// Observation deliberately excludes system items and
|
||||
// controller errors from model-visible session evidence.
|
||||
SessionSnapshotEntryData::SystemItem { .. }
|
||||
| SessionSnapshotEntryData::Invoke { .. }
|
||||
SessionSnapshotEntryData::SystemItem { content, .. } => {
|
||||
entry.use_policy?;
|
||||
Item::system_message(content)
|
||||
}
|
||||
// Controller errors are not model-visible evidence.
|
||||
SessionSnapshotEntryData::Invoke { .. }
|
||||
| SessionSnapshotEntryData::Usage { .. }
|
||||
| SessionSnapshotEntryData::RunStats { .. }
|
||||
| SessionSnapshotEntryData::RunYielded { .. }
|
||||
@@ -314,6 +321,7 @@ impl SessionCapture {
|
||||
| SessionSnapshotEntryData::RunError { .. } => return None,
|
||||
};
|
||||
Some(CapturedHistoryEntry {
|
||||
use_policy: entry.use_policy,
|
||||
item,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(entry.entry_id),
|
||||
origin: entry.provenance,
|
||||
@@ -328,6 +336,7 @@ impl SessionCapture {
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, item)| CapturedHistoryEntry {
|
||||
use_policy: None,
|
||||
item,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(format!("{index:08}")),
|
||||
origin: SessionEntryProvenance::LegacyUnknown,
|
||||
@@ -342,10 +351,21 @@ impl SessionCapture {
|
||||
) -> Self {
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.map(|entry| CapturedHistoryEntry {
|
||||
item: entry.item,
|
||||
entry_id: entry.annotation.entry_id,
|
||||
origin: public_provenance(&entry.annotation.origin),
|
||||
.map(|entry| {
|
||||
let (item, use_policy) = if let Some(input) = &entry.annotation.system_input {
|
||||
(
|
||||
Item::system_message(input.public_projection().body),
|
||||
Some(input.public_use_policy()),
|
||||
)
|
||||
} else {
|
||||
(entry.item, None)
|
||||
};
|
||||
CapturedHistoryEntry {
|
||||
item,
|
||||
use_policy,
|
||||
entry_id: entry.annotation.entry_id,
|
||||
origin: public_provenance(&entry.annotation.origin),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Self::from_captured_entries(segment_id, entries)
|
||||
@@ -364,7 +384,8 @@ impl SessionCapture {
|
||||
let entry_range = [idx as u64, idx as u64];
|
||||
match item {
|
||||
Item::Message { role, content, .. } => {
|
||||
let Some(kind) = message_reference_kind(&entry.origin, role) else {
|
||||
let Some(kind) = message_reference_kind(&entry.origin, role, entry.use_policy)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let text = content
|
||||
@@ -376,6 +397,7 @@ impl SessionCapture {
|
||||
let summary = truncate_chars(&text, 240);
|
||||
let id = SessionEntryRef::from_history_entry_id(&entry.entry_id);
|
||||
index.push(ReferenceEntry {
|
||||
use_policy: entry.use_policy,
|
||||
id: id.clone(),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range,
|
||||
@@ -386,7 +408,9 @@ impl SessionCapture {
|
||||
summary: summary.clone(),
|
||||
search_text: text.clone(),
|
||||
});
|
||||
if matches!(kind, ReferenceKind::User | ReferenceKind::Assistant) {
|
||||
if entry.use_policy.is_none_or(|p| p.observation_readable)
|
||||
&& matches!(kind, ReferenceKind::User | ReferenceKind::Assistant)
|
||||
{
|
||||
overview.push(OverviewItem {
|
||||
id: id.clone(),
|
||||
origin: entry.origin.clone(),
|
||||
@@ -403,6 +427,7 @@ impl SessionCapture {
|
||||
} => {
|
||||
let text = format!("{name}\n{arguments}");
|
||||
index.push(ReferenceEntry {
|
||||
use_policy: entry.use_policy,
|
||||
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range,
|
||||
@@ -430,6 +455,7 @@ impl SessionCapture {
|
||||
content.as_deref().unwrap_or_default(),
|
||||
);
|
||||
index.push(ReferenceEntry {
|
||||
use_policy: entry.use_policy,
|
||||
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range,
|
||||
@@ -510,6 +536,9 @@ impl SessionCapture {
|
||||
let mut hits = Vec::new();
|
||||
|
||||
for entry in &self.index {
|
||||
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
|
||||
continue;
|
||||
}
|
||||
if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index {
|
||||
continue;
|
||||
}
|
||||
@@ -581,6 +610,9 @@ impl SessionCapture {
|
||||
};
|
||||
|
||||
for entry in selected {
|
||||
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
|
||||
continue;
|
||||
}
|
||||
if entries.len() >= max_items {
|
||||
truncated = true;
|
||||
break;
|
||||
@@ -695,21 +727,11 @@ impl SessionCapture {
|
||||
|
||||
pub(crate) fn evidence_for(&self, id: &str) -> Option<SessionEntryEvidence> {
|
||||
let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?;
|
||||
let excerpt = self
|
||||
.read(
|
||||
ReadSelector::Id(id),
|
||||
ReadOptions {
|
||||
include_tools: true,
|
||||
tool_part: ToolPart::Both,
|
||||
detail: ReadDetail::Compact,
|
||||
max_items: 1,
|
||||
max_bytes: 2 * 1024,
|
||||
},
|
||||
)
|
||||
.entries
|
||||
.first()
|
||||
.map(|entry| entry.text.clone())
|
||||
.unwrap_or_else(|| entry.summary.clone());
|
||||
if entry.use_policy.is_some_and(|p| !p.memory_evidence) {
|
||||
return None;
|
||||
}
|
||||
let item = &self.entries.get(entry.entry_range[0] as usize)?.item;
|
||||
let excerpt = render_item(item, entry, ReadDetail::Compact, 2 * 1024);
|
||||
Some(SessionEntryEvidence {
|
||||
segment_id: self.segment_id.clone(),
|
||||
entry_ref: entry.id.clone(),
|
||||
@@ -731,9 +753,8 @@ fn public_provenance(origin: &WorkerHistoryProvenance) -> SessionEntryProvenance
|
||||
}
|
||||
WorkerHistoryProvenance::WorkerInput { .. } => SessionEntryProvenance::WorkerInput,
|
||||
WorkerHistoryProvenance::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction,
|
||||
WorkerHistoryProvenance::BackendInstruction { .. } => {
|
||||
SessionEntryProvenance::BackendInstruction
|
||||
}
|
||||
WorkerHistoryProvenance::BackendInstruction { .. }
|
||||
| WorkerHistoryProvenance::Provided { .. } => SessionEntryProvenance::BackendInstruction,
|
||||
WorkerHistoryProvenance::ModelOutput { .. } => SessionEntryProvenance::ModelOutput,
|
||||
WorkerHistoryProvenance::ToolOutput { .. } => SessionEntryProvenance::ToolOutput,
|
||||
WorkerHistoryProvenance::CompactionSummary
|
||||
@@ -746,7 +767,15 @@ fn public_provenance(origin: &WorkerHistoryProvenance) -> SessionEntryProvenance
|
||||
fn message_reference_kind(
|
||||
origin: &SessionEntryProvenance,
|
||||
provider_role: &Role,
|
||||
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||
) -> Option<ReferenceKind> {
|
||||
if use_policy.is_some() {
|
||||
return match provider_role {
|
||||
Role::System => Some(ReferenceKind::System),
|
||||
Role::User => Some(ReferenceKind::User),
|
||||
Role::Assistant => Some(ReferenceKind::Assistant),
|
||||
};
|
||||
}
|
||||
match origin {
|
||||
SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => {
|
||||
Some(ReferenceKind::User)
|
||||
@@ -950,6 +979,148 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_public_input_observation_and_memory_policies_survive_store_restore_independently() {
|
||||
use session_store::{
|
||||
FeatureInputPayload, FsStore, InputCompactionUse, InputEvidenceUse,
|
||||
InputObservationUse, LogEntry, LoggedSystemHistoryEntry, PublicSystemItem,
|
||||
PublicSystemPresentation, Store, SystemInputEnvelope, SystemInputMetadata,
|
||||
SystemInputUsePolicy,
|
||||
};
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let store = FsStore::new(root.path()).unwrap();
|
||||
let session = session_store::new_session_id();
|
||||
let segment = session_store::new_segment_id();
|
||||
store
|
||||
.create_segment(
|
||||
session,
|
||||
segment,
|
||||
&[LogEntry::AnnotatedSegmentStart {
|
||||
ts: 1,
|
||||
session_id: session,
|
||||
system_prompt: None,
|
||||
config: Default::default(),
|
||||
history: vec![],
|
||||
forked_from: None,
|
||||
compacted_from: None,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
let mut inputs = Vec::new();
|
||||
for (producer, observation, memory_evidence, public) in [
|
||||
(
|
||||
"same",
|
||||
InputObservationUse::Readable,
|
||||
InputEvidenceUse::Exclude,
|
||||
true,
|
||||
),
|
||||
(
|
||||
"same",
|
||||
InputObservationUse::Hidden,
|
||||
InputEvidenceUse::Include,
|
||||
true,
|
||||
),
|
||||
(
|
||||
"renamed",
|
||||
InputObservationUse::Readable,
|
||||
InputEvidenceUse::Include,
|
||||
true,
|
||||
),
|
||||
(
|
||||
"renamed",
|
||||
InputObservationUse::Hidden,
|
||||
InputEvidenceUse::Exclude,
|
||||
true,
|
||||
),
|
||||
(
|
||||
"private",
|
||||
InputObservationUse::Readable,
|
||||
InputEvidenceUse::Include,
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let entry = LoggedSystemHistoryEntry::from_input(
|
||||
WorkerHistoryProvenance::Provided {
|
||||
producer: producer.into(),
|
||||
},
|
||||
None,
|
||||
SystemInputEnvelope::new(
|
||||
"private model text",
|
||||
SystemInputMetadata::new(
|
||||
SystemInputUsePolicy {
|
||||
recording_material: InputEvidenceUse::Exclude,
|
||||
memory_evidence,
|
||||
observation,
|
||||
compaction: InputCompactionUse::ReferenceInput,
|
||||
},
|
||||
Some(
|
||||
FeatureInputPayload::encode(
|
||||
"future:owner",
|
||||
99,
|
||||
&serde_json::json!({ "secret": "private metadata" }),
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
public.then(|| PublicSystemItem {
|
||||
kind: "supplied-context".into(),
|
||||
body: "public context: do not extract this".into(),
|
||||
presentation: PublicSystemPresentation::Context,
|
||||
}),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
store
|
||||
.append(
|
||||
session,
|
||||
segment,
|
||||
&LogEntry::AnnotatedSystemItem {
|
||||
ts: 2,
|
||||
entry: entry.clone(),
|
||||
extensions: vec![],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
inputs.push(entry);
|
||||
}
|
||||
let saved = store.read_all(session, segment).unwrap();
|
||||
let restored =
|
||||
crate::session_history::restore_history_entries(session, segment, &saved).unwrap();
|
||||
let snapshot = session_store::public_snapshot::project_session_snapshot(session, &saved);
|
||||
let snapshot: SessionSnapshot =
|
||||
serde_json::from_value(serde_json::to_value(snapshot).unwrap()).unwrap();
|
||||
for capture in [
|
||||
SessionCapture::from_history_entries(segment.to_string(), restored),
|
||||
SessionCapture::from_session_snapshot(segment.to_string(), snapshot),
|
||||
] {
|
||||
let hits = capture.search(&SearchOptions {
|
||||
kind: Some(ReferenceKind::System),
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(hits.len(), 2);
|
||||
for (index, input) in inputs.iter().enumerate() {
|
||||
let reference = SessionEntryRef::from_history_entry_id(&input.metadata.entry_id);
|
||||
let read =
|
||||
capture.read(ReadSelector::Id(reference.as_str()), ReadOptions::default());
|
||||
assert_eq!(!read.entries.is_empty(), matches!(index, 0 | 2));
|
||||
let evidence = capture.evidence_for(reference.as_str());
|
||||
assert_eq!(evidence.is_some(), matches!(index, 1 | 2));
|
||||
for text in read
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| entry.text.as_str())
|
||||
.chain(evidence.iter().map(|entry| entry.excerpt.as_str()))
|
||||
{
|
||||
assert!(text.contains("public context: do not extract this"));
|
||||
assert!(!text.contains("private model text"));
|
||||
assert!(!text.contains("private metadata"));
|
||||
assert!(!text.contains("future:owner"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overview_contains_user_and_assistant_only() {
|
||||
let view = SessionCapture::new(
|
||||
|
||||
@@ -24,6 +24,7 @@ pub(crate) fn test_logged_history_entry(item: impl Into<Item>) -> LoggedHistoryE
|
||||
LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::from(item.into()),
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
@@ -38,6 +39,7 @@ pub(crate) fn test_logged_system_entry(
|
||||
session_store::LoggedSystemHistoryEntry {
|
||||
item,
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId::new(),
|
||||
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
@@ -58,6 +60,7 @@ pub(crate) fn metadata(
|
||||
derivation: Option<SessionHistoryDerivation>,
|
||||
) -> SessionHistoryMetadata {
|
||||
SessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: SessionHistoryEntryId::new(),
|
||||
origin,
|
||||
derivation,
|
||||
@@ -80,6 +83,7 @@ pub(crate) fn history_entry_with_id(
|
||||
HistoryEntry::new(
|
||||
item,
|
||||
SessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id,
|
||||
origin,
|
||||
derivation: None,
|
||||
|
||||
@@ -77,6 +77,7 @@ mod tests {
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| protocol::SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: format!("test-{index}"),
|
||||
timestamp: index as u64,
|
||||
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
|
||||
|
||||
@@ -1338,6 +1338,7 @@ mod tests {
|
||||
is_error: false,
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(entry_id.into()),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -5502,6 +5502,7 @@ mod tests {
|
||||
entry: session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::from(saved),
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId::new(),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -119,6 +119,7 @@ fn annotated(item: Item) -> session_store::LoggedHistoryEntry {
|
||||
session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::from(item),
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId::new(),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
derivation: None,
|
||||
|
||||
@@ -44108,6 +44108,7 @@ mod tests {
|
||||
content: vec![LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(id.to_string()),
|
||||
origin: LoggedSessionHistoryOrigin::HumanInput {
|
||||
account_id: "account-1".to_string(),
|
||||
@@ -44297,6 +44298,7 @@ mod tests {
|
||||
content: vec![LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(id.to_string()),
|
||||
origin: LoggedSessionHistoryOrigin::HumanInput {
|
||||
account_id: "account-1".to_string(),
|
||||
|
||||
@@ -479,6 +479,7 @@ fn seed_session(
|
||||
vec![session_store::LoggedHistoryEntry {
|
||||
item,
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId("read-source-original".into()),
|
||||
origin,
|
||||
derivation: None,
|
||||
@@ -1706,6 +1707,7 @@ impl SubjectRemovalFixture {
|
||||
.map(|text| session_store::LoggedHistoryEntry {
|
||||
item: message(text),
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(text.into()),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
|
||||
@@ -67,6 +67,7 @@ impl Fixture {
|
||||
content: vec![session_store::LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
system_input: None,
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(format!("original-{index}")),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -206,7 +206,7 @@ entry_id: string,
|
||||
/**
|
||||
* Timestamp copied from the durable log record that commits this entry.
|
||||
*/
|
||||
timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array<string>, } & ({ "kind": "user_input", segments: Array<Segment>, } | { "kind": "message", role: SessionMessageRole, content: Array<SessionContentPart>, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array<SessionToolAttachment>, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_yielded", reason: RunYieldReason, active_run_turn_count: number, } | { "kind": "run_resumed", source: RunResumeSource, active_run_turn_count: number, } | { "kind": "invoke", trigger: InvokeKind, } | { "kind": "usage", input_tokens: number, cache_read_input_tokens: number, output_tokens: number, } | { "kind": "run_stats", elapsed_ms: number, requests: number, upload_tokens: number, output_tokens: number, } | { "kind": "run_cancelled" } | { "kind": "run_error", message: string, failure?: RunFailureKind | null, });
|
||||
timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array<string>, use_policy?: SessionEntryUsePolicy | null, } & ({ "kind": "user_input", segments: Array<Segment>, } | { "kind": "message", role: SessionMessageRole, content: Array<SessionContentPart>, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array<SessionToolAttachment>, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_yielded", reason: RunYieldReason, active_run_turn_count: number, } | { "kind": "run_resumed", source: RunResumeSource, active_run_turn_count: number, } | { "kind": "invoke", trigger: InvokeKind, } | { "kind": "usage", input_tokens: number, cache_read_input_tokens: number, output_tokens: number, } | { "kind": "run_stats", elapsed_ms: number, requests: number, upload_tokens: number, output_tokens: number, } | { "kind": "run_cancelled" } | { "kind": "run_error", message: string, failure?: RunFailureKind | null, });
|
||||
|
||||
export type SessionConversationTurn = { turn_id: string, entries: Array<SessionSnapshotEntry>, };
|
||||
|
||||
|
||||
Reference in New Issue
Block a user