Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: preserve role-neutral input policy across compaction and replay

Browse Source
This commit is contained in:
Hare
2026-10-20 04:45:50 +09:00
parent 8f759911ab
commit 3b3381900f
46 changed files with 779 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/session-store/src/fs_store.rs
+2 -2
View File
@@ -361,7 +361,7 @@ impl Store for FsStore {
segment_id: SegmentId,
entry: &LogEntry,
) -> Result<(), StoreError> {
entry.validate_system_inputs()?;
entry.validate_input_metadata()?;
let _retention = self.retention().lock_live(session_id)?;
let line = serde_json::to_string(entry)?;
self.append_line(&self.log_path(session_id, segment_id), &line)
@@ -450,7 +450,7 @@ impl Store for FsStore {
entries: &[LogEntry],
) -> Result<(), StoreError> {
for entry in entries {
entry.validate_system_inputs()?;
entry.validate_input_metadata()?;
}
let _retention = self.retention().lock_live(session_id)?;
let path = self.log_path(session_id, segment_id);
crates/session-store/src/history.rs
+51 -41
View File
@@ -85,21 +85,22 @@ pub struct LoggedHistoryDerivation {
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LoggedSessionHistoryMetadata {
pub entry_id: LoggedSessionHistoryEntryId,
pub origin: LoggedSessionHistoryOrigin,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub derivation: Option<LoggedHistoryDerivation>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub system_input: Option<crate::SystemInputMetadata>,
pub input_metadata: Option<crate::HistoryInputMetadata>,
}
impl LoggedSessionHistoryMetadata {
pub(crate) fn validate_system_input(&self) -> Result<(), crate::SystemInputError> {
pub(crate) fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
if matches!(self.origin, LoggedSessionHistoryOrigin::Provided { .. })
&& self.system_input.is_none()
&& self.input_metadata.is_none()
{
return Err(crate::SystemInputError::MissingMetadata);
return Err(crate::InputMetadataError::MissingMetadata);
}
Ok(())
}
@@ -109,7 +110,7 @@ impl LoggedSessionHistoryMetadata {
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
system_input: None,
input_metadata: None,
}
}
}
@@ -131,34 +132,40 @@ struct HistoryEntryWire {
}
impl TryFrom<HistoryEntryWire> for LoggedHistoryEntry {
type Error = crate::SystemInputError;
type Error = crate::InputMetadataError;
fn try_from(wire: HistoryEntryWire) -> Result<Self, Self::Error> {
let entry = Self {
item: wire.item,
metadata: wire.metadata,
};
entry.validate_system_input()?;
entry.validate_input_metadata()?;
Ok(entry)
}
}
impl LoggedHistoryEntry {
pub fn validate_system_input(&self) -> Result<(), crate::SystemInputError> {
self.metadata.validate_system_input()?;
if self.metadata.system_input.is_some() {
let LoggedItem::Message {
role: crate::LoggedRole::System,
content,
} = &self.item
else {
return Err(crate::SystemInputError::InconsistentRecord);
};
let [crate::LoggedContentPart::Text { text }] = content.as_slice() else {
return Err(crate::SystemInputError::InconsistentRecord);
};
if text.len() > crate::system_input::MAX_SYSTEM_INPUT_BYTES {
return Err(crate::SystemInputError::ResourceLimit);
pub fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
self.metadata.validate_input_metadata()?;
if let Some(input) = &self.metadata.input_metadata {
match &self.item {
LoggedItem::Message {
role: crate::LoggedRole::System,
content,
} => {
let [crate::LoggedContentPart::Text { text }] = content.as_slice() else {
return Err(crate::InputMetadataError::InconsistentRecord);
};
if text.len() > crate::input_metadata::MAX_SYSTEM_INPUT_BYTES {
return Err(crate::InputMetadataError::ResourceLimit);
}
}
// Other logical roles may carry use policy, but cannot masquerade
// as the System-only public/payload extension contract.
_ if input.feature_payload().is_some() || input.public_item().is_some() => {
return Err(crate::InputMetadataError::InconsistentRecord);
}
_ => {}
}
}
Ok(())
@@ -183,14 +190,14 @@ struct SystemEntryWire {
}
impl TryFrom<SystemEntryWire> for LoggedSystemHistoryEntry {
type Error = crate::SystemInputError;
type Error = crate::InputMetadataError;
fn try_from(wire: SystemEntryWire) -> Result<Self, Self::Error> {
let entry = Self {
item: wire.item,
metadata: wire.metadata,
};
entry.validate_system_input()?;
entry.validate_input_metadata()?;
Ok(entry)
}
}
@@ -201,30 +208,30 @@ impl LoggedSystemHistoryEntry {
derivation: Option<LoggedHistoryDerivation>,
input: crate::SystemInputEnvelope,
) -> Self {
let (body, system_input) = input.into_parts();
let (body, input_metadata) = input.into_parts();
Self {
item: crate::SystemItem::Input { body },
metadata: LoggedSessionHistoryMetadata {
entry_id: LoggedSessionHistoryEntryId::new(),
origin,
derivation,
system_input: Some(system_input),
input_metadata: Some(input_metadata),
},
}
}
pub fn validate_system_input(&self) -> Result<(), crate::SystemInputError> {
self.metadata.validate_system_input()?;
match (&self.item, &self.metadata.system_input) {
pub fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
self.metadata.validate_input_metadata()?;
match (&self.item, &self.metadata.input_metadata) {
(crate::SystemItem::Input { body }, Some(_)) => {
if body.len() > crate::system_input::MAX_SYSTEM_INPUT_BYTES {
return Err(crate::SystemInputError::ResourceLimit);
if body.len() > crate::input_metadata::MAX_SYSTEM_INPUT_BYTES {
return Err(crate::InputMetadataError::ResourceLimit);
}
}
(crate::SystemItem::Input { .. }, None) => {
return Err(crate::SystemInputError::MissingMetadata);
return Err(crate::InputMetadataError::MissingMetadata);
}
(_, Some(_)) => return Err(crate::SystemInputError::InconsistentRecord),
(_, Some(_)) => return Err(crate::InputMetadataError::InconsistentRecord),
(_, None) => {}
}
Ok(())
@@ -241,9 +248,9 @@ impl LoggedSystemHistoryEntry {
if matches!(self.item, crate::SystemItem::Input { .. }) {
return self
.metadata
.system_input
.input_metadata
.as_ref()
.and_then(crate::SystemInputMetadata::public_item)
.and_then(crate::HistoryInputMetadata::public_item)
.cloned()
.unwrap_or_else(|| self.item.public_item());
}
@@ -268,7 +275,7 @@ mod tests {
}],
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".into(),
@@ -293,7 +300,7 @@ mod tests {
"item": { "kind": "input", "body": "context" },
"metadata": {
"entry_id": "stable-entry", "origin": { "kind": "provided", "producer": "owner" },
"system_input": { "usage": {
"input_metadata": { "usage": {
"recording_material": "exclude", "memory_evidence": "exclude",
"observation": "hidden", "compaction": "reference_input"
} }
@@ -315,10 +322,10 @@ mod tests {
missing["metadata"]
.as_object_mut()
.unwrap()
.remove("system_input");
.remove("input_metadata");
assert!(decode(missing));
let mut absent_policy = wire.clone();
absent_policy["metadata"]["system_input"]
absent_policy["metadata"]["input_metadata"]
.as_object_mut()
.unwrap()
.remove("usage");
@@ -327,15 +334,18 @@ mod tests {
invalid_role["item"] =
serde_json::to_value(LoggedItem::from(agen::Item::user_message("not system")))
.unwrap();
invalid_role["metadata"]["input_metadata"]["public"] = serde_json::json!({
"kind": "context", "body": "public system projection", "presentation": {"kind": "context"}
});
assert!(decode(invalid_role));
let mut oversized = wire.clone();
if generic {
oversized["item"]["body"] = "x"
.repeat(crate::system_input::MAX_SYSTEM_INPUT_BYTES + 1)
.repeat(crate::input_metadata::MAX_SYSTEM_INPUT_BYTES + 1)
.into();
} else {
oversized["item"]["content"][0]["text"] = "x"
.repeat(crate::system_input::MAX_SYSTEM_INPUT_BYTES + 1)
.repeat(crate::input_metadata::MAX_SYSTEM_INPUT_BYTES + 1)
.into();
}
assert!(decode(oversized));
crates/session-store/src/system_input.rs → crates/session-store/src/input_metadata.rs
+59 -56
View File
@@ -1,4 +1,4 @@
//! Validated, Feature-neutral system-input data.
//! Validated, Feature-neutral history-input policy and System extensions.
//!
//! The enclosing history annotation owns identity, origin and derivation. None of
//! those descriptive facts selects a use policy here. Model text, independent use
@@ -23,22 +23,22 @@ const MAX_JSON_DEPTH: usize = 16;
const MAX_JSON_NODES: usize = 8192;
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum SystemInputError {
#[error("system input payload schema is invalid")]
pub enum InputMetadataError {
#[error("history input payload schema is invalid")]
InvalidSchema,
#[error("system input payload version must be nonzero")]
#[error("history input payload version must be nonzero")]
InvalidVersion,
#[error("system input resource limit exceeded")]
ResourceLimit,
#[error("system input payload could not be serialized")]
#[error("history input payload could not be serialized")]
Serialization,
#[error("system input payload schema or version is unsupported by this consumer")]
#[error("history input payload schema or version is unsupported by this consumer")]
UnsupportedPayload,
#[error("system input payload does not match the owner's typed contract")]
#[error("history input payload does not match the owner's typed contract")]
InvalidTypedPayload,
#[error("generic system input is missing explicit metadata")]
#[error("provided input is missing explicit metadata")]
MissingMetadata,
#[error("system input metadata does not match its logical record")]
#[error("input metadata does not match its logical record")]
InconsistentRecord,
}
@@ -70,7 +70,7 @@ struct PayloadWire {
}
impl TryFrom<PayloadWire> for FeatureInputPayload {
type Error = SystemInputError;
type Error = InputMetadataError;
fn try_from(wire: PayloadWire) -> Result<Self, Self::Error> {
if wire.schema.is_empty()
@@ -80,10 +80,10 @@ impl TryFrom<PayloadWire> for FeatureInputPayload {
byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-_.:/".contains(&byte)
})
{
return Err(SystemInputError::InvalidSchema);
return Err(InputMetadataError::InvalidSchema);
}
if wire.version == 0 {
return Err(SystemInputError::InvalidVersion);
return Err(InputMetadataError::InvalidVersion);
}
validate_json(&wire.value, MAX_FEATURE_PAYLOAD_BYTES)?;
Ok(Self {
@@ -99,11 +99,11 @@ impl FeatureInputPayload {
schema: impl Into<String>,
version: u32,
value: &T,
) -> Result<Self, SystemInputError> {
) -> Result<Self, InputMetadataError> {
PayloadWire {
schema: schema.into(),
version,
value: serde_json::to_value(value).map_err(|_| SystemInputError::Serialization)?,
value: serde_json::to_value(value).map_err(|_| InputMetadataError::Serialization)?,
}
.try_into()
}
@@ -122,12 +122,12 @@ impl FeatureInputPayload {
&self,
schema: &str,
version: u32,
) -> Result<T, SystemInputError> {
) -> Result<T, InputMetadataError> {
if self.schema != schema || self.version != version {
return Err(SystemInputError::UnsupportedPayload);
return Err(InputMetadataError::UnsupportedPayload);
}
serde_json::from_value(self.value.clone())
.map_err(|_| SystemInputError::InvalidTypedPayload)
.map_err(|_| InputMetadataError::InvalidTypedPayload)
}
}
@@ -161,17 +161,20 @@ pub enum InputCompactionUse {
/// Public presentation is deliberately not one of these use decisions.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SystemInputUsePolicy {
pub struct HistoryInputUsePolicy {
pub recording_material: InputEvidenceUse,
pub memory_evidence: InputEvidenceUse,
pub observation: InputObservationUse,
pub compaction: InputCompactionUse,
}
/// Use policy is independent of logical role. Non-System derived Tool context
/// carries policy only; private Feature payload and public presentation belong
/// to the System envelope and are rejected on other roles at the history boundary.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(try_from = "MetadataWire")]
pub struct SystemInputMetadata {
usage: SystemInputUsePolicy,
pub struct HistoryInputMetadata {
usage: HistoryInputUsePolicy,
feature_payload: Option<FeatureInputPayload>,
public: Option<PublicSystemItem>,
}
@@ -179,13 +182,13 @@ pub struct SystemInputMetadata {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct MetadataWire {
usage: SystemInputUsePolicy,
usage: HistoryInputUsePolicy,
feature_payload: Option<FeatureInputPayload>,
public: Option<PublicSystemItem>,
}
impl TryFrom<MetadataWire> for SystemInputMetadata {
type Error = SystemInputError;
impl TryFrom<MetadataWire> for HistoryInputMetadata {
type Error = InputMetadataError;
fn try_from(wire: MetadataWire) -> Result<Self, Self::Error> {
if let Some(public) = &wire.public {
@@ -194,10 +197,10 @@ impl TryFrom<MetadataWire> for SystemInputMetadata {
|| public.kind.len() > MAX_SCHEMA_BYTES
|| public.kind.chars().any(char::is_control)
{
return Err(SystemInputError::InvalidSchema);
return Err(InputMetadataError::InvalidSchema);
}
let value =
serde_json::to_value(public).map_err(|_| SystemInputError::Serialization)?;
serde_json::to_value(public).map_err(|_| InputMetadataError::Serialization)?;
validate_json(&value, MAX_SYSTEM_PUBLIC_BYTES)?;
}
Ok(Self {
@@ -208,12 +211,12 @@ impl TryFrom<MetadataWire> for SystemInputMetadata {
}
}
impl SystemInputMetadata {
impl HistoryInputMetadata {
pub fn new(
usage: SystemInputUsePolicy,
usage: HistoryInputUsePolicy,
feature_payload: Option<FeatureInputPayload>,
public: Option<PublicSystemItem>,
) -> Result<Self, SystemInputError> {
) -> Result<Self, InputMetadataError> {
MetadataWire {
usage,
feature_payload,
@@ -222,7 +225,7 @@ impl SystemInputMetadata {
.try_into()
}
pub fn usage(&self) -> SystemInputUsePolicy {
pub fn usage(&self) -> HistoryInputUsePolicy {
self.usage
}
@@ -258,22 +261,22 @@ impl SystemInputMetadata {
#[serde(try_from = "InputWire")]
pub struct SystemInputEnvelope {
body: String,
metadata: SystemInputMetadata,
metadata: HistoryInputMetadata,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct InputWire {
body: String,
metadata: SystemInputMetadata,
metadata: HistoryInputMetadata,
}
impl TryFrom<InputWire> for SystemInputEnvelope {
type Error = SystemInputError;
type Error = InputMetadataError;
fn try_from(wire: InputWire) -> Result<Self, Self::Error> {
if wire.body.len() > MAX_SYSTEM_INPUT_BYTES {
return Err(SystemInputError::ResourceLimit);
return Err(InputMetadataError::ResourceLimit);
}
Ok(Self {
body: wire.body,
@@ -285,8 +288,8 @@ impl TryFrom<InputWire> for SystemInputEnvelope {
impl SystemInputEnvelope {
pub fn new(
body: impl Into<String>,
metadata: SystemInputMetadata,
) -> Result<Self, SystemInputError> {
metadata: HistoryInputMetadata,
) -> Result<Self, InputMetadataError> {
InputWire {
body: body.into(),
metadata,
@@ -298,7 +301,7 @@ impl SystemInputEnvelope {
&self.body
}
pub fn metadata(&self) -> &SystemInputMetadata {
pub fn metadata(&self) -> &HistoryInputMetadata {
&self.metadata
}
@@ -306,15 +309,15 @@ impl SystemInputEnvelope {
Item::system_message(self.body.clone())
}
pub fn into_parts(self) -> (String, SystemInputMetadata) {
pub fn into_parts(self) -> (String, HistoryInputMetadata) {
(self.body, self.metadata)
}
}
fn validate_json(value: &Value, max_bytes: usize) -> Result<(), SystemInputError> {
fn walk(value: &Value, depth: usize, remaining: &mut usize) -> Result<(), SystemInputError> {
fn validate_json(value: &Value, max_bytes: usize) -> Result<(), InputMetadataError> {
fn walk(value: &Value, depth: usize, remaining: &mut usize) -> Result<(), InputMetadataError> {
if depth > MAX_JSON_DEPTH || *remaining == 0 {
return Err(SystemInputError::ResourceLimit);
return Err(InputMetadataError::ResourceLimit);
}
*remaining -= 1;
match value {
@@ -334,9 +337,9 @@ fn validate_json(value: &Value, max_bytes: usize) -> Result<(), SystemInputError
}
let mut remaining = MAX_JSON_NODES;
walk(value, 0, &mut remaining)?;
let bytes = serde_json::to_vec(value).map_err(|_| SystemInputError::Serialization)?;
let bytes = serde_json::to_vec(value).map_err(|_| InputMetadataError::Serialization)?;
if bytes.len() > max_bytes {
return Err(SystemInputError::ResourceLimit);
return Err(InputMetadataError::ResourceLimit);
}
Ok(())
}
@@ -353,8 +356,8 @@ mod tests {
omitted: bool,
}
fn reference_policy() -> SystemInputUsePolicy {
SystemInputUsePolicy {
fn reference_policy() -> HistoryInputUsePolicy {
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
@@ -375,7 +378,7 @@ mod tests {
.unwrap();
let envelope = SystemInputEnvelope::new(
"private model context",
SystemInputMetadata::new(reference_policy(), Some(payload), None).unwrap(),
HistoryInputMetadata::new(reference_policy(), Some(payload), None).unwrap(),
)
.unwrap();
let restored: SystemInputEnvelope =
@@ -387,7 +390,7 @@ mod tests {
assert_eq!(saved_payload.version(), 99);
assert_eq!(
saved_payload.decode::<OwnerPayload>("known-owner:context", 1),
Err(SystemInputError::UnsupportedPayload)
Err(InputMetadataError::UnsupportedPayload)
);
assert_eq!(
serde_json::to_value(&restored).unwrap(),
@@ -415,7 +418,7 @@ mod tests {
.unwrap();
assert_eq!(
invalid.decode::<OwnerPayload>("owner:context", 1),
Err(SystemInputError::InvalidTypedPayload)
Err(InputMetadataError::InvalidTypedPayload)
);
}
@@ -462,14 +465,14 @@ mod tests {
] {
let mut absent = valid.clone();
absent.as_object_mut().unwrap().remove(field);
assert!(serde_json::from_value::<SystemInputUsePolicy>(absent).is_err());
assert!(serde_json::from_value::<HistoryInputUsePolicy>(absent).is_err());
let mut unknown = valid.clone();
unknown[field] = Value::String("future-allow".into());
assert!(serde_json::from_value::<SystemInputUsePolicy>(unknown).is_err());
assert!(serde_json::from_value::<HistoryInputUsePolicy>(unknown).is_err());
}
let mut extra = valid;
extra["feature_name"] = Value::String("infer-policy-from-me".into());
assert!(serde_json::from_value::<SystemInputUsePolicy>(extra).is_err());
assert!(serde_json::from_value::<HistoryInputUsePolicy>(extra).is_err());
}
#[test]
@@ -483,7 +486,7 @@ mod tests {
body: "explicit public text".into(),
presentation: PublicSystemPresentation::Context,
};
let metadata = SystemInputMetadata::new(
let metadata = HistoryInputMetadata::new(
reference_policy(),
Some(
FeatureInputPayload::encode(
@@ -519,14 +522,14 @@ mod tests {
#[test]
fn use_policies_remain_independent_of_each_other_and_of_public_display() {
let usage = SystemInputUsePolicy {
let usage = HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Include,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Readable,
compaction: InputCompactionUse::Retain,
};
let metadata = SystemInputMetadata::new(usage, None, None).unwrap();
let restored: SystemInputMetadata =
let metadata = HistoryInputMetadata::new(usage, None, None).unwrap();
let restored: HistoryInputMetadata =
serde_json::from_value(serde_json::to_value(&metadata).unwrap()).unwrap();
assert_eq!(restored.usage(), usage);
assert!(restored.public_item().is_none());
@@ -535,7 +538,7 @@ mod tests {
#[test]
fn model_body_and_public_projection_have_independent_resource_limits() {
let metadata = SystemInputMetadata::new(reference_policy(), None, None).unwrap();
let metadata = HistoryInputMetadata::new(reference_policy(), None, None).unwrap();
assert!(
SystemInputEnvelope::new("x".repeat(MAX_SYSTEM_INPUT_BYTES + 1), metadata).is_err()
);
@@ -544,7 +547,7 @@ mod tests {
body: "x".repeat(MAX_SYSTEM_PUBLIC_BYTES),
presentation: PublicSystemPresentation::Context,
};
assert!(SystemInputMetadata::new(reference_policy(), None, Some(public)).is_err());
assert!(HistoryInputMetadata::new(reference_policy(), None, Some(public)).is_err());
let invalid = serde_json::json!({ "body": "small", "metadata": { "public": null, "feature_payload": null } });
assert!(serde_json::from_value::<SystemInputEnvelope>(invalid).is_err());
}
crates/session-store/src/legacy_session_log.rs
+86 -30
View File
@@ -1,4 +1,4 @@
//! Versioned decoder for Session schemas v1-4, including frozen Feature wire.
//! Versioned decoder for Session schemas v1-5, including frozen Feature wire.
//!
//! These types are intentionally private to `session-store`. Current writers,
//! replay, and public projections use [`crate::LogEntry`] exclusively; only the
@@ -93,14 +93,14 @@ impl LegacySystemItem {
fn into_entry(
self,
mut metadata: LoggedSessionHistoryMetadata,
) -> Result<LoggedSystemHistoryEntry, crate::SystemInputError> {
) -> Result<LoggedSystemHistoryEntry, crate::InputMetadataError> {
use crate::InputCompactionUse::{ReferenceInput, Retain};
let item = match self {
Self::Core(item) => item,
Self::Feature(feature) => {
metadata.validate_system_input()?;
if metadata.system_input.is_some() {
return Err(crate::SystemInputError::InconsistentRecord);
metadata.validate_input_metadata()?;
if metadata.input_metadata.is_some() {
return Err(crate::InputMetadataError::InconsistentRecord);
}
let (body, input_metadata) = match feature {
LegacyFeatureSystemItem::JournalStartup { body, payload } => {
@@ -150,12 +150,12 @@ impl LegacySystemItem {
}
};
// Identity/origin/derivation are retained, never re-inferred.
metadata.system_input = Some(input_metadata);
metadata.input_metadata = Some(input_metadata);
SystemItem::Input { body }
}
};
let entry = LoggedSystemHistoryEntry { item, metadata };
entry.validate_system_input()?;
entry.validate_input_metadata()?;
Ok(entry)
}
}
@@ -166,8 +166,8 @@ fn migrated_feature_input<T: Serialize>(
schema: &str,
payload: T,
compaction: crate::InputCompactionUse,
) -> Result<(String, crate::SystemInputMetadata), crate::SystemInputError> {
let metadata = crate::SystemInputMetadata::new(
) -> Result<(String, crate::HistoryInputMetadata), crate::InputMetadataError> {
let metadata = crate::HistoryInputMetadata::new(
migration_policy(compaction),
Some(crate::FeatureInputPayload::encode(schema, 1, &payload)?),
Some(crate::PublicSystemItem {
@@ -207,7 +207,7 @@ enum LegacyAnnotatedOrCurrent {
}
impl LegacyAnnotatedOrCurrent {
fn into_current(self) -> Result<LogEntry, crate::SystemInputError> {
fn into_current(self) -> Result<LogEntry, crate::InputMetadataError> {
match self {
Self::Annotated(LegacyAnnotatedLogEntry::AnnotatedSystemItem {
ts,
@@ -285,16 +285,19 @@ pub(crate) fn decode_entry(
segment_id: SegmentId,
line_index: usize,
) -> Result<LogEntry, serde_json::Error> {
let mut wire: serde_json::Value = serde_json::from_str(line)?;
migrate_legacy_metadata_field(&mut wire)?;
let entry = match schema_version {
1 => match serde_json::from_str::<LegacySessionLogEntryV1>(line)? {
1 => match serde_json::from_value::<LegacySessionLogEntryV1>(wire)? {
LegacySessionLogEntryV1::History(entry) => Entry::History(entry),
LegacySessionLogEntryV1::Current(entry) => Entry::Current(entry),
},
2 => match serde_json::from_str::<LegacySessionLogEntryV2>(line)? {
2 => match serde_json::from_value::<LegacySessionLogEntryV2>(wire)? {
LegacySessionLogEntryV2::History(entry) => Entry::History(entry),
LegacySessionLogEntryV2::Current(entry) => Entry::Current(entry),
},
3 | 4 => Entry::Current(serde_json::from_str(line)?),
3 | 4 => Entry::Current(serde_json::from_value(wire)?),
5 => return serde_json::from_value(wire),
_ => unreachable!("legacy decoder called for unsupported schema {schema_version}"),
};
match entry {
@@ -306,6 +309,53 @@ pub(crate) fn decode_entry(
.map_err(serde::de::Error::custom)
}
// The v5 System-only metadata field was widened to role-neutral input metadata
// in v6. Touch only known history slots, never arbitrary Feature payload objects.
// Both keys are an ambiguous source, not permission to discard either value.
fn migrate_legacy_metadata_field(record: &mut serde_json::Value) -> Result<(), serde_json::Error> {
fn entry(value: &mut serde_json::Value) -> Result<(), serde_json::Error> {
if let Some(metadata) = value
.get_mut("metadata")
.and_then(serde_json::Value::as_object_mut)
{
if let Some(old) = metadata.remove("system_input") {
if metadata.contains_key("input_metadata") {
return Err(serde::de::Error::custom(
"ambiguous legacy input metadata fields",
));
}
metadata.insert("input_metadata".into(), old);
}
}
Ok(())
}
let kind = record
.get("kind")
.and_then(serde_json::Value::as_str)
.unwrap_or_default()
.to_owned();
if matches!(
kind.as_str(),
"annotated_system_item"
| "annotated_assistant_item"
| "annotated_tool_result"
| "tool_result_corrected"
) {
if let Some(value) = record.get_mut("entry") {
entry(value)?;
}
}
if let Some(history) = record
.get_mut("history")
.and_then(serde_json::Value::as_array_mut)
{
for value in history {
entry(value)?;
}
}
Ok(())
}
/// Schema 3 used DerivedSummary for Host compaction seeds. Only the trusted
/// creation boundary may identify one; arbitrary text and later seed items do not.
/// Include already-upgraded entries so an interrupted multi-file migration resumes.
@@ -361,15 +411,15 @@ pub(crate) fn migrate_compaction_origins(
}
}
/// Explicit v1-4 -> v5 boundary. No Feature inference belongs in ordinary
/// Explicit legacy-to-current boundary. No Feature inference belongs in ordinary
/// readers/restoration. Invalid or oversized payloads fail migration; the generic
/// payload/body/public limits are deliberately not relaxed for legacy data.
pub(crate) fn migrate_system_inputs(entry: &mut LogEntry) -> Result<(), crate::SystemInputError> {
entry.validate_system_inputs()?;
pub(crate) fn migrate_system_inputs(entry: &mut LogEntry) -> Result<(), crate::InputMetadataError> {
entry.validate_input_metadata()?;
match entry {
// Feature wire has already been converted by the explicit old-schema
// decoder. Ordinary LogEntry can contain only generic Input/core events.
LogEntry::AnnotatedSystemItem { entry, .. } => entry.validate_system_input()?,
LogEntry::AnnotatedSystemItem { entry, .. } => entry.validate_input_metadata()?,
LogEntry::AnnotatedSegmentStart { history, .. }
| LogEntry::AnnotatedUserInput { history, .. } => {
for entry in history {
@@ -383,11 +433,11 @@ pub(crate) fn migrate_system_inputs(entry: &mut LogEntry) -> Result<(), crate::S
}
_ => {}
}
entry.validate_system_inputs()
entry.validate_input_metadata()
}
fn migration_policy(compaction: crate::InputCompactionUse) -> crate::SystemInputUsePolicy {
crate::SystemInputUsePolicy {
fn migration_policy(compaction: crate::InputCompactionUse) -> crate::HistoryInputUsePolicy {
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
@@ -397,18 +447,22 @@ fn migration_policy(compaction: crate::InputCompactionUse) -> crate::SystemInput
fn migrate_flattened_system_input(
entry: &mut LoggedHistoryEntry,
) -> Result<(), crate::SystemInputError> {
) -> Result<(), crate::InputMetadataError> {
// Check before supplying legacy policy: Provided without metadata is a
// malformed NEW record, not permission to synthesize a producer policy.
entry.validate_system_input()?;
if entry.metadata.system_input.is_none()
&& matches!(
entry.validate_input_metadata()?;
if entry.metadata.input_metadata.is_none()
&& (matches!(
entry.item,
LoggedItem::Message {
role: crate::LoggedRole::System,
..
}
)
) || matches!(
entry.metadata.origin,
LoggedSessionHistoryOrigin::DerivedSummary
| LoggedSessionHistoryOrigin::CompactionSummary
))
{
let compaction = match entry.metadata.origin {
LoggedSessionHistoryOrigin::JournalContext => crate::InputCompactionUse::ReferenceInput,
@@ -417,14 +471,16 @@ fn migrate_flattened_system_input(
}
_ => crate::InputCompactionUse::Retain,
};
// No Feature payload or public projection can be recovered from text.
entry.metadata.system_input = Some(crate::SystemInputMetadata::new(
// Legacy derived context was excluded from independent evidence. Preserve
// that established decision explicitly, including synthetic Tool roles;
// never guess from Tool names or body text. Payload/public stay absent.
entry.metadata.input_metadata = Some(crate::HistoryInputMetadata::new(
migration_policy(compaction),
None,
None,
)?);
}
entry.validate_system_input()
entry.validate_input_metadata()
}
enum Entry {
@@ -442,7 +498,7 @@ fn legacy_metadata(
identity.extend_from_slice(&(line_index as u64).to_be_bytes());
identity.extend_from_slice(&(item_index as u64).to_be_bytes());
LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(format!(
"l-{}",
base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, identity)
@@ -457,7 +513,7 @@ fn canonicalize_history_entry(
segment_id: SegmentId,
line_index: usize,
entry: LegacyHistoryLogEntry,
) -> Result<LogEntry, crate::SystemInputError> {
) -> Result<LogEntry, crate::InputMetadataError> {
Ok(match entry {
LegacyHistoryLogEntry::SegmentStart {
ts,
crates/session-store/src/legacy_session_log/tests.rs
+1 -1
View File
@@ -24,7 +24,7 @@ fn seed(
content: vec![crate::LoggedContentPart::Text { text: text.into() }],
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("summary".into()),
origin,
derivation: Some(crate::LoggedHistoryDerivation {
crates/session-store/src/legacy_session_log/tests/fixtures/v5-role-metadata.jsonl
Vendored
+5
View File
@@ -0,0 +1,5 @@
{"kind":"annotated_system_item","ts":201,"entry":{"item":{"kind":"input","body":"frozen v5 reference"},"metadata":{"entry_id":"reference-v5","origin":{"kind":"provided","producer":"owner-v5"},"system_input":{"usage":{"recording_material":"exclude","memory_evidence":"exclude","observation":"hidden","compaction":"reference_input"},"feature_payload":{"schema":"future:owner","version":8,"value":{"system_input":"opaque owner field must not be renamed"}},"public":null}}}}
{"kind":"annotated_assistant_item","ts":202,"entry":{"item":{"kind":"tool_call","call_id":"original-call","name":"ArbitraryTool","arguments":"{}"},"metadata":{"entry_id":"original-call-entry","origin":{"kind":"model_output","worker":{"worker_id":"body"}}}}}
{"kind":"annotated_tool_result","ts":203,"entry":{"item":{"kind":"tool_result","call_id":"original-call","summary":"original result"},"metadata":{"entry_id":"original-result-entry","origin":{"kind":"tool_output","worker":{"worker_id":"body"}}}}}
{"kind":"annotated_assistant_item","ts":204,"entry":{"item":{"kind":"tool_call","call_id":"derived-call","name":"ArbitraryTool","arguments":"{}"},"metadata":{"entry_id":"derived-call-entry","origin":{"kind":"derived_summary"},"derivation":{"sources":["original-call-entry","original-result-entry"]}}}}
{"kind":"annotated_tool_result","ts":205,"entry":{"item":{"kind":"tool_result","call_id":"derived-call","summary":"derived retrieval result"},"metadata":{"entry_id":"derived-result-entry","origin":{"kind":"derived_summary"},"derivation":{"sources":["original-call-entry","original-result-entry"]}}}}
crates/session-store/src/legacy_session_log/tests/v5.rs
+122 -15
View File
@@ -27,7 +27,7 @@ fn encode(entries: &[LogEntry]) -> Vec<u8> {
fn migrate(entries: &mut [LogEntry]) {
for entry in entries {
migrate_system_inputs(entry).unwrap();
entry.validate_system_inputs().unwrap();
entry.validate_input_metadata().unwrap();
}
}
@@ -100,7 +100,7 @@ fn v4_fixed_wire_preserves_identity_body_payload_public_and_independent_policy()
assert_eq!(entry.metadata.entry_id, old_metadata.entry_id);
assert_eq!(entry.metadata.origin, old_metadata.origin);
assert_eq!(entry.metadata.derivation, old_metadata.derivation);
let metadata = entry.metadata.system_input.as_ref().unwrap();
let metadata = entry.metadata.input_metadata.as_ref().unwrap();
let public = metadata.public_item().unwrap();
assert_eq!(public.kind, old_item["kind"].as_str().unwrap());
assert_eq!(public.body, old_item["body"].as_str().unwrap());
@@ -182,7 +182,7 @@ fn v4_fixed_wire_preserves_identity_body_payload_public_and_independent_policy()
history[index].metadata.derivation,
old[index].metadata.derivation
);
let metadata = history[index].metadata.system_input.as_ref().unwrap();
let metadata = history[index].metadata.input_metadata.as_ref().unwrap();
assert_eq!(metadata.usage(), migration_policy(compaction));
assert!(metadata.public_item().is_none());
assert!(metadata.feature_payload().is_none());
@@ -225,7 +225,7 @@ fn v1_through_v4_worker_migration_reopen_and_mixed_file_retry() {
fs::write(&old_path, source).unwrap();
let store = WorkerSessionStore::new(root.path()).unwrap();
assert_marker(root.path(), 5);
assert_marker(root.path(), 6);
let restored = store.read_all(session, segment).unwrap();
assert_eq!(encode(&restored), new_bytes);
assert_eq!(fs::read(&new_path).unwrap(), new_bytes);
@@ -243,7 +243,7 @@ fn v1_through_v4_worker_migration_reopen_and_mixed_file_retry() {
fn all_segments_preflight_keeps_old_bytes_on_decode_or_conversion_failure() {
let mut oversized: Value = serde_json::from_str(V4.lines().next().unwrap()).unwrap();
oversized["entry"]["item"]["snapshot"] =
json!({"large": "x".repeat(crate::system_input::MAX_FEATURE_PAYLOAD_BYTES + 1)});
json!({"large": "x".repeat(crate::input_metadata::MAX_FEATURE_PAYLOAD_BYTES + 1)});
let mut unknown: Value = serde_json::from_str(V4.lines().next().unwrap()).unwrap();
unknown["entry"]["item"]["kind"] = json!("unsupported_future_feature");
let mut malformed = unknown.clone();
@@ -295,7 +295,7 @@ fn all_segments_preflight_keeps_old_bytes_on_decode_or_conversion_failure() {
#[test]
fn unsupported_schema_preserves_all_old_bytes() {
for version in [0, 6, 99] {
for version in [0, 7, 99] {
let root = tempfile::tempdir().unwrap();
let (_, segment) = write_worker(root.path(), version, V4.as_bytes());
let path = root
@@ -331,13 +331,13 @@ fn standalone_store_uses_same_v1_through_v4_migration_and_reopens() {
let store = FsStore::new(root.path()).unwrap();
let entries = store.read_all(session, segment).unwrap();
for entry in entries {
entry.validate_system_inputs().unwrap();
entry.validate_input_metadata().unwrap();
}
assert_eq!(
fs::read_to_string(dir.join("session-schema.json"))
.unwrap()
.trim(),
"5"
"6"
);
let once = fs::read(&path).unwrap();
drop(store);
@@ -381,7 +381,7 @@ fn compaction_origin_upgrade_is_retained_only_for_pre_v4_sources() {
}
);
assert_eq!(
metadata.system_input.as_ref().unwrap().usage().compaction,
metadata.input_metadata.as_ref().unwrap().usage().compaction,
if version == 3 {
InputCompactionUse::OpeningSummary
} else {
@@ -423,7 +423,7 @@ fn current_v5_rejects_all_legacy_feature_kinds_without_rewriting_sources() {
assert!(serde_json::from_str::<LogEntry>(line).is_err());
let bytes = format!("{line}\n");
let root = tempfile::tempdir().unwrap();
let (session, segment) = write_worker(root.path(), 5, bytes.as_bytes());
let (session, segment) = write_worker(root.path(), 6, bytes.as_bytes());
let path = root
.path()
.join("segments")
@@ -495,7 +495,7 @@ fn v1_v2_feature_wire_uses_the_same_typed_payloads_and_stable_legacy_identity()
else {
panic!()
};
assert_eq!(entry.metadata.system_input, v4.metadata.system_input);
assert_eq!(entry.metadata.input_metadata, v4.metadata.input_metadata);
}
}
}
@@ -555,7 +555,7 @@ fn frozen_behavior_counter_is_accepted_only_at_the_old_schema_boundary() {
assert_eq!(entry.item.history_text(), "exact archived body");
let payload = entry
.metadata
.system_input
.input_metadata
.as_ref()
.unwrap()
.feature_payload()
@@ -573,7 +573,7 @@ fn frozen_behavior_counter_is_accepted_only_at_the_old_schema_boundary() {
}
#[test]
fn current_v5_rejects_generic_input_without_metadata_instead_of_migrating_it() {
fn current_schema_rejects_generic_input_without_metadata_instead_of_migrating_it() {
let mut missing: Value = serde_json::from_str(V4.lines().last().unwrap()).unwrap();
missing["entry"]["metadata"]
.as_object_mut()
@@ -581,7 +581,7 @@ fn current_v5_rejects_generic_input_without_metadata_instead_of_migrating_it() {
.remove("system_input");
let bytes = format!("{}\n", missing);
let root = tempfile::tempdir().unwrap();
let (_, segment) = write_worker(root.path(), 5, bytes.as_bytes());
let (_, segment) = write_worker(root.path(), 6, bytes.as_bytes());
assert!(WorkerSessionStore::new(root.path()).is_err());
assert_eq!(
fs::read(
@@ -592,5 +592,112 @@ fn current_v5_rejects_generic_input_without_metadata_instead_of_migrating_it() {
.unwrap(),
bytes.as_bytes()
);
assert_marker(root.path(), 5);
assert_marker(root.path(), 6);
}
const V5: &str = include_str!("fixtures/v5-role-metadata.jsonl");
#[test]
fn v5_metadata_and_derived_tool_context_upgrade_without_changing_roles_or_original_evidence() {
for standalone in [false, true] {
let root = tempfile::tempdir().unwrap();
let (session, segment) = if standalone {
let session = SessionId::now_v7();
let segment = SegmentId::now_v7();
let dir = root.path().join(session.to_string());
fs::create_dir_all(&dir).unwrap();
fs::write(dir.join("session-schema.json"), b"5").unwrap();
fs::write(dir.join(format!("{segment}.jsonl")), V5).unwrap();
(session, segment)
} else {
write_worker(root.path(), 5, V5.as_bytes())
};
let entries = if standalone {
FsStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
} else {
WorkerSessionStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
};
let originals: Vec<Value> = V5
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(entries.len(), originals.len());
for (index, (entry, old)) in entries.iter().zip(originals).enumerate() {
let wire = serde_json::to_value(entry).unwrap();
assert_eq!(wire["ts"], old["ts"]);
assert_eq!(wire["entry"]["item"], old["entry"]["item"]);
for field in ["entry_id", "origin", "derivation"] {
assert_eq!(
wire["entry"]["metadata"][field],
old["entry"]["metadata"][field]
);
}
let metadata = &wire["entry"]["metadata"];
if index == 0 {
assert_eq!(
metadata["input_metadata"],
old["entry"]["metadata"]["system_input"]
);
assert_eq!(
metadata["input_metadata"]["feature_payload"]["value"]["system_input"],
"opaque owner field must not be renamed"
);
} else if index < 3 {
assert!(
metadata.get("input_metadata").is_none(),
"ordinary model Tool originals must not acquire exclusion policy"
);
} else {
assert_eq!(
metadata["input_metadata"]["usage"],
json!({
"recording_material":"exclude", "memory_evidence":"exclude", "observation":"hidden", "compaction":"retain"
})
);
}
}
let before = encode(&entries);
let reopened = if standalone {
FsStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
} else {
WorkerSessionStore::new(root.path())
.unwrap()
.read_all(session, segment)
.unwrap()
};
assert_eq!(encode(&reopened), before);
}
}
#[test]
fn legacy_metadata_alias_conflict_fails_preflight_without_dropping_either_value() {
let mut bad: Value = serde_json::from_str(V5.lines().next().unwrap()).unwrap();
bad["entry"]["metadata"]["input_metadata"] = bad["entry"]["metadata"]["system_input"].clone();
for version in [4, 5] {
let root = tempfile::tempdir().unwrap();
let (_, segment) = write_worker(root.path(), version, V5.as_bytes());
let path = root
.path()
.join("segments")
.join(format!("{segment}.jsonl"));
let later = root
.path()
.join("segments")
.join(format!("{}.jsonl", SegmentId::from_u128(2)));
let bad_bytes = format!("{bad}\n");
fs::write(&later, &bad_bytes).unwrap();
assert!(WorkerSessionStore::new(root.path()).is_err());
assert_eq!(fs::read(path).unwrap(), V5.as_bytes());
assert_eq!(fs::read(later).unwrap(), bad_bytes.as_bytes());
assert_marker(root.path(), version);
}
}
crates/session-store/src/lib.rs
+5 -5
View File
@@ -41,6 +41,7 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res
pub mod event_trace;
pub mod fs_store;
pub mod history;
pub mod input_metadata;
mod legacy_session_log;
pub mod logged_item;
mod paste_artifact;
@@ -51,7 +52,6 @@ pub mod retention;
pub mod segment;
pub mod segment_log;
pub mod store;
pub mod system_input;
pub mod system_item;
pub mod uploaded_file;
pub mod worker_metadata;
@@ -66,6 +66,10 @@ pub use history::{
LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, LoggedSystemHistoryEntry,
LoggedWorkerSubject,
};
pub use input_metadata::{
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
InputEvidenceUse, InputMetadataError, InputObservationUse, SystemInputEnvelope,
};
pub use logged_item::{LoggedContentPart, LoggedItem, LoggedRole, from_logged, to_logged};
pub use paste_artifact::PasteArtifactLimits;
pub use public_index::{
@@ -98,10 +102,6 @@ pub use segment::{
};
pub use segment_log::{LogEntry, RestoredState, SegmentOrigin, SessionExtension, collect_state};
pub use store::{Store, StoreError};
pub use system_input::{
FeatureInputPayload, InputCompactionUse, InputEvidenceUse, InputObservationUse,
SystemInputEnvelope, SystemInputError, SystemInputMetadata, SystemInputUsePolicy,
};
pub use system_item::{
PromptRenderProvenance, PublicSystemItem, PublicSystemPresentation, SystemItem, SystemReminder,
SystemReminderSource, render_worker_event,
crates/session-store/src/public_index.rs
+2 -2
View File
@@ -1296,7 +1296,7 @@ mod tests {
LoggedHistoryEntry {
item,
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.to_owned()),
origin,
derivation: None,
@@ -1538,7 +1538,7 @@ mod tests {
prompt_provenance: None,
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("system-item".into()),
origin: LoggedSessionHistoryOrigin::BackendInstruction {
operation_id: None,
crates/session-store/src/public_material.rs
+5 -6
View File
@@ -371,11 +371,10 @@ impl SessionPublicMaterialReader {
let source_ids: Option<Vec<String>> = if compact {
let summaries: Vec<_> = history
.iter()
.filter(|e| {
matches!(
e.metadata.origin,
LoggedSessionHistoryOrigin::CompactionSummary
)
.filter(|entry| {
entry.metadata.input_metadata.as_ref().is_some_and(|input| {
input.usage().compaction == crate::InputCompactionUse::OpeningSummary
})
})
.collect();
if summaries.len() != 1 {
@@ -895,7 +894,7 @@ fn canonical_body(
entry: &LoggedHistoryEntry,
purpose: PublicMaterialUse,
) -> Option<CanonicalBody<'_>> {
if let Some(input) = &entry.metadata.system_input {
if let Some(input) = &entry.metadata.input_metadata {
let usage = input.usage();
let allowed = match purpose {
PublicMaterialUse::Recording => usage.recording_material,
crates/session-store/src/public_material/tests.rs
+17 -15
View File
@@ -9,7 +9,7 @@ fn entry(id: &str, item: LoggedItem) -> LoggedHistoryEntry {
LoggedHistoryEntry {
item,
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None,
@@ -31,9 +31,9 @@ fn system_context(
compaction: crate::InputCompactionUse,
) -> LoggedHistoryEntry {
let mut entry = message(id, LoggedRole::System, text);
entry.metadata.system_input = Some(
crate::SystemInputMetadata::new(
crate::SystemInputUsePolicy {
entry.metadata.input_metadata = Some(
crate::HistoryInputMetadata::new(
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
@@ -613,9 +613,9 @@ fn reasoning_proves_rollback_safety_but_private_and_derived_items_are_not_materi
body: "previous Journal must not become new recording material".into(),
},
metadata: LoggedSessionHistoryMetadata {
system_input: Some(
crate::SystemInputMetadata::new(
crate::SystemInputUsePolicy {
input_metadata: Some(
crate::HistoryInputMetadata::new(
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
@@ -812,9 +812,9 @@ fn schema_v3_compaction_migrates_persistently_and_strict_reader_keeps_original_i
panic!()
};
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
summary.metadata.system_input = Some(
crate::SystemInputMetadata::new(
crate::SystemInputUsePolicy {
summary.metadata.input_metadata = Some(
crate::HistoryInputMetadata::new(
crate::HistoryInputUsePolicy {
recording_material: crate::InputEvidenceUse::Exclude,
memory_evidence: crate::InputEvidenceUse::Exclude,
observation: crate::InputObservationUse::Hidden,
@@ -878,7 +878,9 @@ fn midturn_compaction_uses_exact_source_ids_and_preserves_discarded_originals()
"never use as original",
crate::InputCompactionUse::OpeningSummary,
);
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
summary.metadata.origin = LoggedSessionHistoryOrigin::Provided {
producer: "renamed-summary-owner".into(),
};
summary.metadata.derivation = Some(crate::LoggedHistoryDerivation {
sources: vec![
LoggedSessionHistoryEntryId("u".into()),
@@ -1311,8 +1313,8 @@ fn compaction_without_exact_derivation_or_with_invalid_turn_is_rejected() {
#[test]
fn generic_system_material_uses_explicit_evidence_purpose_not_producer_or_body() {
use crate::{
FeatureInputPayload, InputCompactionUse, InputEvidenceUse, InputObservationUse,
LoggedSystemHistoryEntry, SystemInputEnvelope, SystemInputMetadata, SystemInputUsePolicy,
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
InputEvidenceUse, InputObservationUse, LoggedSystemHistoryEntry, SystemInputEnvelope,
};
for worker_layout in [false, true] {
let temp = tempfile::tempdir().unwrap();
@@ -1328,8 +1330,8 @@ fn generic_system_material_uses_explicit_evidence_purpose_not_producer_or_body()
None,
SystemInputEnvelope::new(
format!("private model text: {body}"),
SystemInputMetadata::new(
SystemInputUsePolicy {
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material,
memory_evidence,
observation: InputObservationUse::Hidden,
crates/session-store/src/public_snapshot.rs
+22 -13
View File
@@ -1625,9 +1625,9 @@ pub(crate) fn project_session_snapshot_for_segment(
derivation_ids(entry),
entry
.metadata
.system_input
.input_metadata
.as_ref()
.map(crate::SystemInputMetadata::public_use_policy),
.map(crate::HistoryInputMetadata::public_use_policy),
)),
LogEntry::RunYielded {
ts,
@@ -1797,9 +1797,9 @@ fn history_entry(
SessionSnapshotEntry {
use_policy: entry
.metadata
.system_input
.input_metadata
.as_ref()
.map(crate::SystemInputMetadata::public_use_policy),
.map(crate::HistoryInputMetadata::public_use_policy),
entry_id: entry.metadata.entry_id.0.clone(),
timestamp,
provenance: provenance(&entry.metadata.origin),
@@ -1926,7 +1926,16 @@ pub fn session_tool_attachment_id(
}
fn project_history_item(entry: &LoggedHistoryEntry) -> Option<SessionSnapshotEntryData> {
if let Some(input) = &entry.metadata.system_input {
if let Some(input) = &entry.metadata.input_metadata {
if !matches!(
entry.item,
LoggedItem::Message {
role: LoggedRole::System,
..
}
) {
return None;
}
let public = input.public_projection();
return Some(SessionSnapshotEntryData::SystemItem {
item_kind: public.kind.clone(),
@@ -2310,7 +2319,7 @@ mod tests {
content: vec![LoggedContentPart::Text { text: text.into() }],
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(entry_id.into()),
origin,
derivation: None,
@@ -3609,7 +3618,7 @@ mod tests {
.map(|item| LoggedHistoryEntry {
item,
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
@@ -3664,7 +3673,7 @@ mod tests {
}],
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: user_entry_id,
origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".into(),
@@ -3734,7 +3743,7 @@ mod tests {
}],
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::FlowInstruction {
selector: "builtin:coder-review".into(),
@@ -3754,7 +3763,7 @@ mod tests {
}],
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: user_entry_id.clone(),
origin: origin.clone(),
derivation: Some(LoggedHistoryDerivation {
@@ -3785,7 +3794,7 @@ mod tests {
fn annotated_projection_preserves_identity_and_provenance() {
let session_id = crate::new_session_id();
let metadata = LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::ModelOutput {
worker: LoggedWorkerSubject {
@@ -4348,7 +4357,7 @@ mod tests {
is_error: false,
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("ancestor-image".into()),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
@@ -4443,7 +4452,7 @@ mod tests {
is_error: false,
},
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("tool-images".into()),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
crates/session-store/src/segment_log.rs
+5 -5
View File
@@ -255,18 +255,18 @@ pub struct SegmentOrigin {
impl LogEntry {
/// Validate generic System records before a Store publishes any bytes.
pub fn validate_system_inputs(&self) -> Result<(), crate::SystemInputError> {
pub fn validate_input_metadata(&self) -> Result<(), crate::InputMetadataError> {
match self {
Self::AnnotatedSegmentStart { history, .. }
| Self::AnnotatedUserInput { history, .. } => {
for entry in history {
entry.validate_system_input()?;
entry.validate_input_metadata()?;
}
}
Self::AnnotatedAssistantItem { entry, .. }
| Self::AnnotatedToolResult { entry, .. }
| Self::ToolResultCorrected { entry, .. } => entry.validate_system_input()?,
Self::AnnotatedSystemItem { entry, .. } => entry.validate_system_input()?,
| Self::ToolResultCorrected { entry, .. } => entry.validate_input_metadata()?,
Self::AnnotatedSystemItem { entry, .. } => entry.validate_input_metadata()?,
_ => {}
}
Ok(())
@@ -554,7 +554,7 @@ mod tests {
LoggedHistoryEntry {
item: LoggedItem::from(item),
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
crates/session-store/src/store.rs
+2 -2
View File
@@ -27,8 +27,8 @@ pub enum StoreError {
#[error("serialization error: {0}")]
Serde(#[from] serde_json::Error),
#[error("invalid system input: {0}")]
InvalidSystemInput(#[from] crate::SystemInputError),
#[error("invalid history input metadata: {0}")]
InvalidInputMetadata(#[from] crate::InputMetadataError),
#[error("segment not found: {0}")]
NotFound(SegmentId),
crates/session-store/src/worker_session_store.rs
+16 -13
View File
@@ -36,7 +36,8 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::SystemTime;
const SESSION_SCHEMA_VERSION: u32 = 5;
const SESSION_SCHEMA_VERSION: u32 = 6;
const SYSTEM_INPUT_SESSION_SCHEMA_VERSION: u32 = 5;
const COMPACTION_SESSION_SCHEMA_VERSION: u32 = 4;
const ANNOTATED_SESSION_SCHEMA_VERSION: u32 = 3;
const PREVIOUS_SESSION_SCHEMA_VERSION: u32 = 2;
@@ -225,8 +226,9 @@ impl WorkerSessionStore {
LEGACY_SESSION_SCHEMA_VERSION
| PREVIOUS_SESSION_SCHEMA_VERSION
| ANNOTATED_SESSION_SCHEMA_VERSION
| COMPACTION_SESSION_SCHEMA_VERSION => {
migrate_segment_logs_to_v5(
| COMPACTION_SESSION_SCHEMA_VERSION
| SYSTEM_INPUT_SESSION_SCHEMA_VERSION => {
migrate_segment_logs_to_current(
segment_log_paths(&root)?,
manifest.session_id,
manifest.schema_version,
@@ -753,7 +755,7 @@ impl Store for WorkerSessionStore {
segment_id: SegmentId,
entry: &LogEntry,
) -> Result<(), StoreError> {
entry.validate_system_inputs()?;
entry.validate_input_metadata()?;
let _retention = self.lock_publication(session_id)?;
self.ensure_session(session_id, true)?;
self.append_log_entry(&self.log_path(segment_id), entry)
@@ -813,7 +815,7 @@ impl Store for WorkerSessionStore {
entries: &[LogEntry],
) -> Result<(), StoreError> {
for entry in entries {
entry.validate_system_inputs()?;
entry.validate_input_metadata()?;
}
let _retention = self.lock_publication(session_id)?;
self.ensure_session(session_id, true)?;
@@ -1173,7 +1175,7 @@ fn segment_log_paths_in_directory(
Ok(paths)
}
fn migrate_segment_logs_to_v5(
fn migrate_segment_logs_to_current(
paths: Vec<(SegmentId, PathBuf)>,
session_id: SessionId,
source_schema_version: u32,
@@ -1220,7 +1222,7 @@ fn migrate_segment_logs_to_v5(
crate::legacy_session_log::migrate_compaction_origins(&mut plan.entries, &summaries);
for mut entry in std::mem::take(&mut plan.entries) {
crate::legacy_session_log::migrate_system_inputs(&mut entry)?;
entry.validate_system_inputs()?;
entry.validate_input_metadata()?;
serde_json::to_writer(&mut plan.output, &entry)?;
plan.output.push(b'\n');
}
@@ -1265,8 +1267,9 @@ pub(crate) fn migrate_fs_session(root: &Path, session_id: SessionId) -> Result<(
LEGACY_SESSION_SCHEMA_VERSION
| PREVIOUS_SESSION_SCHEMA_VERSION
| ANNOTATED_SESSION_SCHEMA_VERSION
| COMPACTION_SESSION_SCHEMA_VERSION => {
migrate_segment_logs_to_v5(
| COMPACTION_SESSION_SCHEMA_VERSION
| SYSTEM_INPUT_SESSION_SCHEMA_VERSION => {
migrate_segment_logs_to_current(
segment_log_paths_in_directory(root)?,
session_id,
version,
@@ -1289,7 +1292,7 @@ fn validate_canonical_segment_paths(paths: Vec<(SegmentId, PathBuf)>) -> Result<
for (_, path) in paths {
let entries: Vec<LogEntry> = parse_jsonl(&fs::read(&path)?)?;
for entry in entries {
entry.validate_system_inputs()?;
entry.validate_input_metadata()?;
}
}
Ok(())
@@ -1470,7 +1473,7 @@ mod tests {
LoggedHistoryEntry {
item: LoggedItem::from(item),
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
@@ -2162,7 +2165,7 @@ mod tests {
assert_eq!(fs::read(&marker_path).unwrap(), old_marker);
let updated_first: Vec<LogEntry> = parse_jsonl(&first_published).unwrap();
for entry in &updated_first {
entry.validate_system_inputs().unwrap();
entry.validate_input_metadata().unwrap();
}
assert!(matches!(
&updated_first[0],
@@ -2184,7 +2187,7 @@ mod tests {
assert_eq!(identity_and_body(&first), first_identity);
assert_eq!(identity_and_body(&second), second_identity);
for entry in first.iter().chain(second.iter()) {
entry.validate_system_inputs().unwrap();
entry.validate_input_metadata().unwrap();
}
assert_eq!(fs::read(&first_path).unwrap(), first_published);
let second_published = fs::read(&second_path).unwrap();
crates/session-store/tests/fs_store_test.rs
+8 -9
View File
@@ -11,7 +11,7 @@ fn annotated(item: Item) -> LoggedHistoryEntry {
LoggedHistoryEntry {
item: LoggedItem::from(item),
metadata: LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId::new(),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
@@ -428,10 +428,9 @@ fn lookup_session_of_finds_owning_session() {
#[test]
fn generic_system_append_and_fork_preserve_metadata_without_public_payload_leak() {
use session_store::{
FeatureInputPayload, InputCompactionUse, InputEvidenceUse, InputObservationUse,
LoggedHistoryDerivation, LoggedSystemHistoryEntry, PublicSystemItem,
PublicSystemPresentation, SystemInputEnvelope, SystemInputMetadata, SystemInputUsePolicy,
WorkerSessionStore,
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
InputEvidenceUse, InputObservationUse, LoggedHistoryDerivation, LoggedSystemHistoryEntry,
PublicSystemItem, PublicSystemPresentation, SystemInputEnvelope, WorkerSessionStore,
};
for worker_layout in [false, true] {
let dir = tempfile::tempdir().unwrap();
@@ -455,8 +454,8 @@ fn generic_system_append_and_fork_preserve_metadata_without_public_payload_leak(
}),
SystemInputEnvelope::new(
"private model body",
SystemInputMetadata::new(
SystemInputUsePolicy {
HistoryInputMetadata::new(
HistoryInputUsePolicy {
recording_material: InputEvidenceUse::Exclude,
memory_evidence: InputEvidenceUse::Exclude,
observation: InputObservationUse::Hidden,
@@ -584,11 +583,11 @@ fn invalid_generic_system_input_rejects_append_create_and_external_json_without_
};
assert!(matches!(
store.append(sid, segment, &invalid),
Err(StoreError::InvalidSystemInput(_))
Err(StoreError::InvalidInputMetadata(_))
));
assert!(matches!(
store.create_segment(sid, segment, &[seed, invalid.clone()]),
Err(StoreError::InvalidSystemInput(_))
Err(StoreError::InvalidInputMetadata(_))
));
assert!(
serde_json::from_value::<LogEntry>(serde_json::to_value(invalid).unwrap()).is_err()
crates/session-store/tests/session_test.rs
+1 -1
View File
@@ -24,7 +24,7 @@ fn annotated(items: &[Item]) -> Vec<session_store::LoggedHistoryEntry> {
.map(|item| session_store::LoggedHistoryEntry {
item: session_store::LoggedItem::from(item),
metadata: session_store::LoggedSessionHistoryMetadata {
system_input: None,
input_metadata: None,
entry_id: session_store::LoggedSessionHistoryEntryId::new(),
origin: session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,