fix: keep memory reads globally bounded

This commit is contained in:
2026-10-02 09:52:32 +09:00
parent 860c34b8da
commit 8f8c2676b4
5 changed files with 593 additions and 96 deletions
+9 -2
View File
@@ -4689,6 +4689,11 @@ pub struct SubjektivMemoryQueryResponse {
pub has_more: bool, pub has_more: bool,
} }
/// Maximum pretty-serialized model-visible content for one subject Memory read.
/// This stays below the Worker's generic 64 KiB tool-content ceiling so the
/// result remains valid JSON with explicit continuation coordinates.
pub const SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES: usize = 56 * 1024;
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
pub struct SubjektivMemoryEvidenceCandidate { pub struct SubjektivMemoryEvidenceCandidate {
@@ -4733,8 +4738,10 @@ pub struct SubjektivMemoryReadResponse {
#[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))] #[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))]
pub body_next_byte_offset: Option<usize>, pub body_next_byte_offset: Option<usize>,
pub body_truncated: bool, pub body_truncated: bool,
/// Candidate ids on this evidence page. The corresponding bounded anchors /// Candidate ids on this evidence page. A compatible historical candidate
/// are returned in `source_candidates` with the same order. /// may repeat across pages while its nested anchors advance through the same
/// evidence cursor. Corresponding bounded anchors retain the same order in
/// `source_candidates`.
pub source_candidate_ids: Vec<String>, pub source_candidate_ids: Vec<String>,
pub source_candidates: Vec<SubjektivMemoryEvidenceCandidate>, pub source_candidates: Vec<SubjektivMemoryEvidenceCandidate>,
/// Derivation references on this evidence page. /// Derivation references on this evidence page.
@@ -268,7 +268,17 @@ impl Tool for SubjektivReadTool {
} }
}; };
let value = value.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; let value = value.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
Ok(json_output(summary, &value)?) let output = json_output(summary, &value)?;
if matches!(self.operation, ReadOperation::Read)
&& output.content.as_ref().is_some_and(|content| {
content.len() > server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES
})
{
return Err(ToolError::ExecutionFailed(
"subjektiv Memory read exceeded its model-visible output budget".into(),
));
}
Ok(output)
} }
} }
@@ -976,6 +986,40 @@ mod tests {
} }
} }
#[derive(Debug)]
struct ReadWorkspaceClient(server_api::SubjektivMemoryReadResponse);
impl WorkspaceClient for ReadWorkspaceClient {
fn workspace_id(&self) -> Option<&str> {
Some("workspace-test")
}
fn kind(&self) -> &str {
"read-test"
}
fn is_available(&self) -> bool {
true
}
fn execute(
&self,
request: WorkspaceRequest,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
let request: SubjektivMemoryBackendRequest =
serde_json::from_str(request.body.as_deref().unwrap()).unwrap();
assert!(matches!(
request.operation,
SubjektivMemoryBackendOperation::Read(_)
));
Ok(WorkspaceResponse {
status: 200,
body: serde_json::to_string(&SubjektivMemoryBackendResponse::Read(self.0.clone()))
.unwrap(),
})
}
}
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct RecordingWorkspaceClient { struct RecordingWorkspaceClient {
requests: Mutex<Vec<WorkspaceRequest>>, requests: Mutex<Vec<WorkspaceRequest>>,
@@ -1130,6 +1174,112 @@ mod tests {
assert!(parsed.entry_refs.is_empty()); assert!(parsed.entry_refs.is_empty());
} }
#[tokio::test]
async fn read_tool_preserves_bounded_model_visible_json_with_escape_heavy_content() {
let origin = || memory::schema::EvidenceOrigin {
kind: memory::schema::EvidenceOriginKind::ModelOutput,
account_id: Some("a".repeat(64)),
workspace_id: Some("workspace-test".into()),
runtime_id: Some("r".repeat(64)),
worker_id: Some("w".repeat(64)),
flow_selector: Some("f".repeat(64)),
flow_definition_id: Some("d".repeat(64)),
flow_definition_revision: Some(1),
};
let evidence = (0..MAX_EVIDENCE_REFS)
.map(|index| memory::extract::StagingEvidence {
id: format!("evidence-{index}"),
kind: memory::schema::EvidenceKind::new("model_output"),
entry_range: Some([index as u64, index as u64]),
origin: Some(origin()),
excerpt: Some("\\\u{0000}".repeat(32)),
summary: Some("\\\u{0001}".repeat(32)),
})
.collect::<Vec<_>>();
let source_refs = (0..MAX_EVIDENCE_REFS)
.map(|index| memory::schema::SourceEvidenceRef {
session_id: Some(format!("session-{index}")),
segment_id: Some(format!("segment-{index}")),
entry_range: Some([index as u64, index as u64]),
evidence_id: Some(format!("evidence-{index}")),
origin: Some(origin()),
evidence_kind: Some(memory::schema::EvidenceKind::new("model_output")),
label: Some("\\\u{0002}".repeat(32)),
summary: Some("\\\u{0003}".repeat(32)),
})
.collect::<Vec<_>>();
let response = server_api::SubjektivMemoryReadResponse {
memory_id: "memory-1".into(),
revision: 1,
current_revision: 1,
kind: CandidateKind::Lesson,
state: server_api::SubjektivMemoryState::Active,
claim: "Escape-aware read".into(),
body_md: "\\\u{0000}\u{0001}".repeat(2_000),
why_useful: "Proves the actual ToolOutput remains parseable".into(),
staleness: None,
change_reason: "test".into(),
created_at: "2026-01-01T00:00:00Z".into(),
updated_at: "2026-01-01T00:00:00Z".into(),
body_offset: 0,
body_byte_offset: 0,
body_next_offset: Some(0),
body_next_byte_offset: Some(6_000),
body_truncated: true,
source_candidate_ids: vec!["candidate-1".into()],
source_candidates: vec![server_api::SubjektivMemoryEvidenceCandidate {
candidate_id: "candidate-1".into(),
evidence,
evidence_total: MAX_EVIDENCE_REFS,
evidence_truncated: false,
source_refs,
source_refs_total: MAX_EVIDENCE_REFS,
source_refs_truncated: false,
}],
derived_from: Vec::new(),
evidence_next_cursor: None,
evidence_has_more: false,
};
let mut oversized_response = response.clone();
oversized_response.body_md =
"\\\u{0000}".repeat(server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES);
let tool = SubjektivReadTool {
state: SubjektivMemoryState {
client: Arc::new(ReadWorkspaceClient(response)),
capture: feature().state.capture,
},
operation: ReadOperation::Read,
};
let output = tool
.execute(
r#"{"memory_id":"memory-1"}"#,
ToolExecutionContext::new("call-read", "batch-read", 0),
)
.await
.unwrap();
let content = output.content.unwrap();
assert!(content.len() <= server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES);
let value: serde_json::Value = serde_json::from_str(&content).unwrap();
assert_eq!(value["body_truncated"], true);
assert_eq!(value["body_next_byte_offset"], 6_000);
let oversized_tool = SubjektivReadTool {
state: SubjektivMemoryState {
client: Arc::new(ReadWorkspaceClient(oversized_response)),
capture: feature().state.capture,
},
operation: ReadOperation::Read,
};
let error = oversized_tool
.execute(
r#"{"memory_id":"memory-1"}"#,
ToolExecutionContext::new("call-oversized", "batch-read", 1),
)
.await
.unwrap_err();
assert!(error.to_string().contains("model-visible output budget"));
}
#[tokio::test] #[tokio::test]
async fn backend_conflict_discriminants_survive_as_structured_tool_output() { async fn backend_conflict_discriminants_survive_as_structured_tool_output() {
for code in ["revision_conflict", "stale_cursor"] { for code in ["revision_conflict", "stale_cursor"] {
+388 -76
View File
@@ -16776,8 +16776,8 @@ const SUBJEKTIV_QUERY_MAX_LIMIT: usize = 100;
const SUBJEKTIV_BODY_DEFAULT_LINES: usize = 200; const SUBJEKTIV_BODY_DEFAULT_LINES: usize = 200;
const SUBJEKTIV_BODY_MAX_LINES: usize = 1_000; const SUBJEKTIV_BODY_MAX_LINES: usize = 1_000;
const SUBJEKTIV_BODY_MAX_BYTES: usize = 16 * 1024; const SUBJEKTIV_BODY_MAX_BYTES: usize = 16 * 1024;
const SUBJEKTIV_EVIDENCE_PAGE_SIZE: usize = 1;
const SUBJEKTIV_CANDIDATE_ANCHOR_LIMIT: usize = crate::subjektiv::MAX_STAGING_ANCHORS; const SUBJEKTIV_CANDIDATE_ANCHOR_LIMIT: usize = crate::subjektiv::MAX_STAGING_ANCHORS;
const SUBJEKTIV_EVIDENCE_ANCHOR_PAGE_SIZE: usize = 2;
const SUBJEKTIV_ANCHOR_TEXT_MAX_BYTES: usize = 64; const SUBJEKTIV_ANCHOR_TEXT_MAX_BYTES: usize = 64;
const SUBJEKTIV_NESTED_EVIDENCE_MAX_BYTES: usize = 40 * 1024; const SUBJEKTIV_NESTED_EVIDENCE_MAX_BYTES: usize = 40 * 1024;
@@ -16807,7 +16807,14 @@ struct SubjektivEvidenceCursor {
subject_id: String, subject_id: String,
memory_id: String, memory_id: String,
revision: u64, revision: u64,
/// Candidate/derivation reference offset within the immutable revision.
offset: usize, offset: usize,
/// Nested offsets keep compatible historical candidates with more anchors
/// resumable without admitting new over-limit records.
#[serde(default)]
evidence_offset: usize,
#[serde(default)]
source_ref_offset: usize,
} }
async fn scoped_subjektiv_memory_backend( async fn scoped_subjektiv_memory_backend(
@@ -17038,21 +17045,26 @@ fn subjektiv_memory_read(
.map(|(line, byte)| (Some(line), Some(byte))) .map(|(line, byte)| (Some(line), Some(byte)))
.unwrap_or((None, None)); .unwrap_or((None, None));
let evidence_offset = if let Some(cursor) = input.evidence_cursor { let (evidence_offset, nested_evidence_offset, nested_source_ref_offset) =
let cursor: SubjektivEvidenceCursor = decode_subjektiv_cursor("evidence", &cursor)?; if let Some(cursor) = input.evidence_cursor {
if cursor.subject_id != subject_id let cursor: SubjektivEvidenceCursor = decode_subjektiv_cursor("evidence", &cursor)?;
|| cursor.memory_id != record.id if cursor.subject_id != subject_id
|| cursor.revision != record.revision || cursor.memory_id != record.id
{ || cursor.revision != record.revision
return Err(Error::InvalidInput( {
"subjektiv evidence cursor does not match subject, Memory, or revision".into(), return Err(Error::InvalidInput(
"subjektiv evidence cursor does not match subject, Memory, or revision".into(),
)
.into());
}
(
cursor.offset,
cursor.evidence_offset,
cursor.source_ref_offset,
) )
.into()); } else {
} (0, 0, 0)
cursor.offset };
} else {
0
};
let total_evidence = record let total_evidence = record
.source_candidate_ids .source_candidate_ids
.len() .len()
@@ -17062,61 +17074,89 @@ fn subjektiv_memory_read(
Error::InvalidInput("subjektiv evidence cursor offset is invalid".into()).into(), Error::InvalidInput("subjektiv evidence cursor offset is invalid".into()).into(),
); );
} }
let evidence_end = evidence_offset
.saturating_add(SUBJEKTIV_EVIDENCE_PAGE_SIZE)
.min(total_evidence);
let mut source_candidate_ids = Vec::new(); let mut source_candidate_ids = Vec::new();
let mut source_candidates = Vec::new(); let mut source_candidates = Vec::new();
let mut derived_from = Vec::new(); let mut derived_from = Vec::new();
let mut evidence_budget = SUBJEKTIV_CANDIDATE_ANCHOR_LIMIT; let mut next_evidence_offset = evidence_offset;
let mut source_ref_budget = SUBJEKTIV_CANDIDATE_ANCHOR_LIMIT; let mut next_nested_evidence_offset = 0;
for index in evidence_offset..evidence_end { let mut next_nested_source_ref_offset = 0;
if let Some(candidate_id) = record.source_candidate_ids.get(index) { if let Some(candidate_id) = record.source_candidate_ids.get(evidence_offset) {
let candidate = store let candidate = store
.staging_candidate(subject_id, candidate_id) .staging_candidate(subject_id, candidate_id)
.map_err(subjektiv_store_error)? .map_err(subjektiv_store_error)?
.ok_or_else(|| { .ok_or_else(|| {
Error::Store(format!( Error::Store(format!(
"subjektiv candidate provenance `{candidate_id}` is missing" "subjektiv candidate provenance `{candidate_id}` is missing"
)) ))
})?; })?;
source_candidate_ids.push(candidate_id.clone()); let evidence_total = candidate.evidence.len();
let evidence_total = candidate.evidence.len(); let source_refs_total = candidate.source_refs.len();
let source_refs_total = candidate.source_refs.len(); if nested_evidence_offset > evidence_total || nested_source_ref_offset > source_refs_total {
let evidence_take = evidence_total.min(evidence_budget); return Err(Error::InvalidInput(
let source_refs_take = source_refs_total.min(source_ref_budget); "subjektiv nested evidence cursor offset is invalid".into(),
evidence_budget -= evidence_take; )
source_ref_budget -= source_refs_take; .into());
source_candidates.push(server_api::SubjektivMemoryEvidenceCandidate {
candidate_id: candidate.id,
evidence: candidate
.evidence
.into_iter()
.take(evidence_take)
.map(bounded_subjektiv_staging_evidence)
.collect(),
evidence_total,
evidence_truncated: evidence_total > evidence_take,
source_refs: candidate
.source_refs
.into_iter()
.take(source_refs_take)
.map(bounded_subjektiv_source_ref)
.collect(),
source_refs_total,
source_refs_truncated: source_refs_total > source_refs_take,
});
} else {
let derived_index = index - record.source_candidate_ids.len();
let reference = &record.derived_from[derived_index];
derived_from.push(server_api::SubjektivMemoryRevisionRef {
memory_id: reference.memory_id.clone(),
revision: reference.revision,
});
} }
let evidence_end = nested_evidence_offset
.saturating_add(SUBJEKTIV_EVIDENCE_ANCHOR_PAGE_SIZE)
.min(evidence_total);
let source_ref_end = nested_source_ref_offset
.saturating_add(SUBJEKTIV_EVIDENCE_ANCHOR_PAGE_SIZE)
.min(source_refs_total);
source_candidate_ids.push(candidate_id.clone());
source_candidates.push(server_api::SubjektivMemoryEvidenceCandidate {
candidate_id: candidate.id,
evidence: candidate
.evidence
.into_iter()
.skip(nested_evidence_offset)
.take(evidence_end - nested_evidence_offset)
.map(bounded_subjektiv_staging_evidence)
.collect(),
evidence_total,
evidence_truncated: nested_evidence_offset > 0 || evidence_end < evidence_total,
source_refs: candidate
.source_refs
.into_iter()
.skip(nested_source_ref_offset)
.take(source_ref_end - nested_source_ref_offset)
.map(bounded_subjektiv_source_ref)
.collect(),
source_refs_total,
source_refs_truncated: nested_source_ref_offset > 0
|| source_ref_end < source_refs_total,
});
if evidence_end < evidence_total || source_ref_end < source_refs_total {
next_nested_evidence_offset = evidence_end;
next_nested_source_ref_offset = source_ref_end;
} else {
next_evidence_offset += 1;
}
} else if evidence_offset < total_evidence {
if nested_evidence_offset != 0 || nested_source_ref_offset != 0 {
return Err(Error::InvalidInput(
"subjektiv derivation cursor cannot contain nested offsets".into(),
)
.into());
}
let derived_index = evidence_offset - record.source_candidate_ids.len();
let reference = &record.derived_from[derived_index];
derived_from.push(server_api::SubjektivMemoryRevisionRef {
memory_id: reference.memory_id.clone(),
revision: reference.revision,
});
next_evidence_offset += 1;
} else if nested_evidence_offset != 0 || nested_source_ref_offset != 0 {
return Err(Error::InvalidInput(
"subjektiv evidence cursor points beyond the final reference".into(),
)
.into());
} }
bound_subjektiv_nested_evidence(&mut source_candidates)?; bound_subjektiv_nested_evidence(&source_candidates)?;
let evidence_has_more = evidence_end < total_evidence; let evidence_has_more = next_evidence_offset < total_evidence
|| next_nested_evidence_offset != 0
|| next_nested_source_ref_offset != 0;
let evidence_next_cursor = evidence_has_more let evidence_next_cursor = evidence_has_more
.then(|| { .then(|| {
encode_subjektiv_cursor( encode_subjektiv_cursor(
@@ -17125,13 +17165,15 @@ fn subjektiv_memory_read(
subject_id: subject_id.to_string(), subject_id: subject_id.to_string(),
memory_id: record.id.clone(), memory_id: record.id.clone(),
revision: record.revision, revision: record.revision,
offset: evidence_end, offset: next_evidence_offset,
evidence_offset: next_nested_evidence_offset,
source_ref_offset: next_nested_source_ref_offset,
}, },
) )
}) })
.transpose()?; .transpose()?;
Ok(server_api::SubjektivMemoryReadResponse { let mut response = server_api::SubjektivMemoryReadResponse {
memory_id: record.id, memory_id: record.id,
revision: record.revision, revision: record.revision,
current_revision: current.revision, current_revision: current.revision,
@@ -17154,7 +17196,15 @@ fn subjektiv_memory_read(
derived_from, derived_from,
evidence_next_cursor, evidence_next_cursor,
evidence_has_more, evidence_has_more,
}) };
fit_subjektiv_read_body_to_output_budget(
&mut response,
&lines,
body_offset,
body_byte_offset,
body_limit,
)?;
Ok(response)
} }
fn subjektiv_memory_list_revisions( fn subjektiv_memory_list_revisions(
@@ -17532,6 +17582,76 @@ fn api_proposal_intent(
} }
} }
fn fit_subjektiv_read_body_to_output_budget(
response: &mut server_api::SubjektivMemoryReadResponse,
lines: &[&str],
line_offset: usize,
byte_offset: usize,
line_limit: usize,
) -> ApiResult<()> {
let serialized_len = |response: &server_api::SubjektivMemoryReadResponse| {
serde_json::to_vec_pretty(response)
.map(|value| value.len())
.map_err(|error| Error::Store(format!("encode subjektiv Memory read: {error}")))
};
if serialized_len(response)? <= server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES {
return Ok(());
}
let initial_body_len = response.body_md.len();
let mut lower = 0usize;
let mut upper = initial_body_len;
while lower < upper {
let candidate_limit = lower + (upper - lower).div_ceil(2);
set_subjektiv_read_body_page(
response,
lines,
line_offset,
byte_offset,
line_limit,
candidate_limit,
);
if serialized_len(response)? <= server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES {
lower = candidate_limit;
} else {
upper = candidate_limit - 1;
}
}
set_subjektiv_read_body_page(response, lines, line_offset, byte_offset, line_limit, lower);
let final_len = serialized_len(response)?;
if final_len > server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES {
return Err(Error::Store(format!(
"subjektiv Memory metadata/provenance requires {final_len} bytes and exceeds the {} byte read-output budget",
server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES
))
.into());
}
if initial_body_len > 0 && response.body_md.is_empty() {
return Err(Error::Store(
"subjektiv Memory metadata/provenance leaves no room for resumable body content".into(),
)
.into());
}
Ok(())
}
fn set_subjektiv_read_body_page(
response: &mut server_api::SubjektivMemoryReadResponse,
lines: &[&str],
line_offset: usize,
byte_offset: usize,
line_limit: usize,
byte_limit: usize,
) {
let (body_md, body_next) =
bounded_memory_body_page(lines, line_offset, byte_offset, line_limit, byte_limit);
response.body_md = body_md;
response.body_truncated = body_next.is_some();
(response.body_next_offset, response.body_next_byte_offset) = body_next
.map(|(line, byte)| (Some(line), Some(byte)))
.unwrap_or((None, None));
}
fn bounded_memory_body_page( fn bounded_memory_body_page(
lines: &[&str], lines: &[&str],
line_offset: usize, line_offset: usize,
@@ -17580,7 +17700,7 @@ fn memory_body_lines(body: &str) -> Vec<&str> {
} }
fn bound_subjektiv_nested_evidence( fn bound_subjektiv_nested_evidence(
candidates: &mut [server_api::SubjektivMemoryEvidenceCandidate], candidates: &[server_api::SubjektivMemoryEvidenceCandidate],
) -> ApiResult<()> { ) -> ApiResult<()> {
let serialized = serde_json::to_vec(candidates) let serialized = serde_json::to_vec(candidates)
.map_err(|error| Error::Store(format!("encode subjektiv evidence page: {error}")))?; .map_err(|error| Error::Store(format!("encode subjektiv evidence page: {error}")))?;
@@ -35673,6 +35793,7 @@ mod tests {
nested_candidate, nested_candidate,
)) ))
.unwrap(); .unwrap();
let escape_heavy_body = "\\\u{0000}\u{0001}".repeat(8_000);
let (nested_memory, _) = store let (nested_memory, _) = store
.apply_candidate( .apply_candidate(
&subject.id, &subject.id,
@@ -35681,7 +35802,7 @@ mod tests {
crate::subjektiv::MemoryDraft::active( crate::subjektiv::MemoryDraft::active(
memory::extract::CandidateKind::Lesson, memory::extract::CandidateKind::Lesson,
"Keep nested provenance bounded", "Keep nested provenance bounded",
"bounded body", escape_heavy_body.clone(),
"Prevents unbounded read responses", "Prevents unbounded read responses",
"adopt oversized historical candidate", "adopt oversized historical candidate",
), ),
@@ -35692,7 +35813,7 @@ mod tests {
&store, &store,
&subject.id, &subject.id,
server_api::SubjektivMemoryReadRequest { server_api::SubjektivMemoryReadRequest {
memory_id: nested_memory.id, memory_id: nested_memory.id.clone(),
revision: Some(1), revision: Some(1),
offset: None, offset: None,
byte_offset: None, byte_offset: None,
@@ -35703,12 +35824,15 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(nested_read.source_candidates.len(), 1); assert_eq!(nested_read.source_candidates.len(), 1);
let nested = &nested_read.source_candidates[0]; let nested = &nested_read.source_candidates[0];
assert_eq!(nested.evidence.len(), SUBJEKTIV_CANDIDATE_ANCHOR_LIMIT); assert_eq!(nested.evidence.len(), SUBJEKTIV_EVIDENCE_ANCHOR_PAGE_SIZE);
assert_eq!(nested.evidence_total, nested_count); assert_eq!(nested.evidence_total, nested_count);
assert!(!nested.evidence_truncated); assert!(nested.evidence_truncated);
assert_eq!(nested.source_refs.len(), SUBJEKTIV_CANDIDATE_ANCHOR_LIMIT); assert_eq!(
nested.source_refs.len(),
SUBJEKTIV_EVIDENCE_ANCHOR_PAGE_SIZE
);
assert_eq!(nested.source_refs_total, nested_count); assert_eq!(nested.source_refs_total, nested_count);
assert!(!nested.source_refs_truncated); assert!(nested.source_refs_truncated);
assert!(nested.evidence.iter().all(|item| { assert!(nested.evidence.iter().all(|item| {
item.excerpt item.excerpt
.as_ref() .as_ref()
@@ -35737,7 +35861,195 @@ mod tests {
.len() .len()
<= SUBJEKTIV_NESTED_EVIDENCE_MAX_BYTES <= SUBJEKTIV_NESTED_EVIDENCE_MAX_BYTES
); );
assert!(serde_json::to_vec(&nested_read).unwrap().len() < 64 * 1024); assert!(
serde_json::to_vec_pretty(&nested_read).unwrap().len()
<= server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES
);
assert!(nested_read.body_truncated);
assert!(nested_read.body_next_offset.is_some());
let mut nested_evidence_ids = nested
.evidence
.iter()
.map(|item| item.id.clone())
.collect::<Vec<_>>();
let mut nested_source_sessions = nested
.source_refs
.iter()
.filter_map(|item| item.session_id.clone())
.collect::<Vec<_>>();
let mut nested_cursor = nested_read.evidence_next_cursor.clone();
while let Some(cursor) = nested_cursor {
let page = subjektiv_memory_read(
&store,
&subject.id,
server_api::SubjektivMemoryReadRequest {
memory_id: nested_memory.id.clone(),
revision: Some(1),
offset: None,
byte_offset: None,
limit: None,
evidence_cursor: Some(cursor),
},
)
.unwrap();
let candidate = &page.source_candidates[0];
nested_evidence_ids.extend(candidate.evidence.iter().map(|item| item.id.clone()));
nested_source_sessions.extend(
candidate
.source_refs
.iter()
.filter_map(|item| item.session_id.clone()),
);
nested_cursor = page.evidence_next_cursor;
}
assert_eq!(nested_evidence_ids.len(), nested_count);
assert_eq!(nested_source_sessions.len(), nested_count);
let mut reconstructed_body = nested_read.body_md.clone();
let mut next_line = nested_read.body_next_offset;
let mut next_byte = nested_read.body_next_byte_offset;
while let Some(offset) = next_line {
let page = subjektiv_memory_read(
&store,
&subject.id,
server_api::SubjektivMemoryReadRequest {
memory_id: nested_memory.id.clone(),
revision: Some(1),
offset: Some(offset),
byte_offset: next_byte,
limit: None,
evidence_cursor: None,
},
)
.unwrap();
assert!(
serde_json::to_vec_pretty(&page).unwrap().len()
<= server_api::SUBJEKTIV_MEMORY_READ_MAX_TOOL_CONTENT_BYTES
);
reconstructed_body.push_str(&page.body_md);
next_line = page.body_next_offset;
next_byte = page.body_next_byte_offset;
}
assert_eq!(reconstructed_body, escape_heavy_body);
// Simulate a row accepted before the write-admission anchor cap. The
// compatible schema remains readable, and its nested anchors advance
// through the ordinary evidence cursor instead of becoming orphaned.
let legacy_count = SUBJEKTIV_CANDIDATE_ANCHOR_LIMIT + 7;
let legacy_workspace_id = store.workspace_id().to_string();
let legacy_evidence = (0..legacy_count)
.map(|index| memory::extract::StagingEvidence {
id: format!("legacy-evidence-{index}"),
kind: memory::schema::EvidenceKind::new("model_output"),
entry_range: Some([index as u64, index as u64]),
origin: Some(memory::schema::EvidenceOrigin {
kind: memory::schema::EvidenceOriginKind::ModelOutput,
account_id: None,
workspace_id: Some(legacy_workspace_id.clone()),
runtime_id: None,
worker_id: None,
flow_selector: None,
flow_definition_id: None,
flow_definition_revision: None,
}),
excerpt: Some(format!("legacy excerpt {index}")),
summary: None,
})
.collect::<Vec<_>>();
let legacy_refs = (0..legacy_count)
.map(|index| memory::schema::SourceEvidenceRef {
session_id: Some(format!("legacy-session-{index}")),
segment_id: Some("legacy-segment".to_string()),
entry_range: Some([index as u64, index as u64]),
evidence_id: Some(format!("legacy-evidence-{index}")),
evidence_kind: Some(memory::schema::EvidenceKind::new("model_output")),
origin: Some(memory::schema::EvidenceOrigin {
kind: memory::schema::EvidenceOriginKind::ModelOutput,
account_id: None,
workspace_id: Some(legacy_workspace_id.clone()),
runtime_id: None,
worker_id: None,
flow_selector: None,
flow_definition_id: None,
flow_definition_revision: None,
}),
..Default::default()
})
.collect::<Vec<_>>();
let legacy_candidate = memory::extract::StagingRecord::from_candidate(
"candidate-legacy-over-anchor-limit",
"legacy-over-anchor-limit-dedup",
memory::schema::SourceRef {
segment_id: "legacy-segment".to_string(),
range: [0, legacy_count as u64 - 1],
},
memory::extract::ExtractedCandidate {
kind: memory::extract::CandidateKind::Lesson,
claim: "Preserve compatible historical provenance".to_string(),
why_useful: "Upgrade must not orphan accepted anchors".to_string(),
staleness: None,
evidence_ids: legacy_evidence.iter().map(|item| item.id.clone()).collect(),
},
legacy_evidence,
legacy_refs,
);
let legacy_candidate = store
.stage_legacy_candidate_for_test(crate::subjektiv::SubjectStagingRecord::attach(
&subject.id,
legacy_candidate,
))
.unwrap();
let (legacy_memory, _) = store
.apply_candidate(
&subject.id,
&legacy_candidate.id,
crate::subjektiv::MemoryRevisionTarget::Create,
crate::subjektiv::MemoryDraft::active(
memory::extract::CandidateKind::Lesson,
"Preserve compatible historical provenance",
"legacy body",
"Upgrade must not orphan accepted anchors",
"upgrade fixture",
),
"adopt legacy candidate",
)
.unwrap();
let mut legacy_cursor = None;
let mut retrieved_evidence = Vec::new();
let mut retrieved_sessions = Vec::new();
loop {
let page = subjektiv_memory_read(
&store,
&subject.id,
server_api::SubjektivMemoryReadRequest {
memory_id: legacy_memory.id.clone(),
revision: Some(1),
offset: None,
byte_offset: None,
limit: None,
evidence_cursor: legacy_cursor,
},
)
.unwrap();
let candidate = &page.source_candidates[0];
retrieved_evidence.extend(candidate.evidence.iter().map(|item| item.id.clone()));
retrieved_sessions.extend(
candidate
.source_refs
.iter()
.filter_map(|item| item.session_id.clone()),
);
if !page.evidence_has_more {
break;
}
legacy_cursor = page.evidence_next_cursor;
}
assert_eq!(retrieved_evidence.len(), legacy_count);
assert_eq!(retrieved_sessions.len(), legacy_count);
for index in 0..legacy_count {
assert!(retrieved_evidence.contains(&format!("legacy-evidence-{index}")));
assert!(retrieved_sessions.contains(&format!("legacy-session-{index}")));
}
let revisions = subjektiv_memory_list_revisions( let revisions = subjektiv_memory_list_revisions(
&store, &store,
+30 -6
View File
@@ -899,6 +899,7 @@ impl SubjektivStore {
pub fn stage_candidate(&self, record: SubjectStagingRecord) -> Result<SubjectStagingRecord> { pub fn stage_candidate(&self, record: SubjectStagingRecord) -> Result<SubjectStagingRecord> {
validate_staging_record(self.workspace_id(), &record)?; validate_staging_record(self.workspace_id(), &record)?;
validate_staging_admission(&record)?;
let raw = serde_json::to_string(&record)?; let raw = serde_json::to_string(&record)?;
self.database.try_transaction(|transaction| { self.database.try_transaction(|transaction| {
require_active_subject(transaction, &record.subject_id)?; require_active_subject(transaction, &record.subject_id)?;
@@ -926,6 +927,7 @@ impl SubjektivStore {
} }
attach_session_to_candidate(&mut record, &attribution.session_id)?; attach_session_to_candidate(&mut record, &attribution.session_id)?;
validate_staging_record(self.workspace_id(), &record)?; validate_staging_record(self.workspace_id(), &record)?;
validate_staging_admission(&record)?;
let candidate_raw = serde_json::to_string(&record)?; let candidate_raw = serde_json::to_string(&record)?;
let attribution_raw = serde_json::to_string(&attribution)?; let attribution_raw = serde_json::to_string(&attribution)?;
@@ -938,6 +940,21 @@ impl SubjektivStore {
}) })
} }
/// Inserts a structurally valid record using the pre-admission-limit path so
/// upgrade tests can prove that compatible historical rows stay readable.
#[cfg(test)]
pub(crate) fn stage_legacy_candidate_for_test(
&self,
record: SubjectStagingRecord,
) -> Result<SubjectStagingRecord> {
validate_staging_record(self.workspace_id(), &record)?;
let raw = serde_json::to_string(&record)?;
self.database.try_transaction(|transaction| {
require_active_subject(transaction, &record.subject_id)?;
write_staging_candidate(transaction, self.workspace_id(), record, raw)
})
}
pub fn record_session_attribution( pub fn record_session_attribution(
&self, &self,
attribution: SubjectSessionAttribution, attribution: SubjectSessionAttribution,
@@ -1899,12 +1916,6 @@ fn validate_staging_record(workspace_id: &str, record: &SubjectStagingRecord) ->
)); ));
} }
if record.evidence.len() > MAX_STAGING_ANCHORS || record.source_refs.len() > MAX_STAGING_ANCHORS
{
return Err(SubjektivError::InvalidRecord(format!(
"staging evidence/source_refs are limited to {MAX_STAGING_ANCHORS} items each"
)));
}
let mut evidence_ids = HashSet::new(); let mut evidence_ids = HashSet::new();
for evidence in &record.evidence { for evidence in &record.evidence {
validate_label("evidence id", &evidence.id)?; validate_label("evidence id", &evidence.id)?;
@@ -1967,6 +1978,19 @@ fn validate_staging_record(workspace_id: &str, record: &SubjectStagingRecord) ->
Ok(()) Ok(())
} }
/// Applies the current write-admission contract separately from structural
/// validation. Persisted rows from older compatible implementations remain
/// readable even when a later release tightens admission limits.
fn validate_staging_admission(record: &SubjectStagingRecord) -> Result<()> {
if record.evidence.len() > MAX_STAGING_ANCHORS || record.source_refs.len() > MAX_STAGING_ANCHORS
{
return Err(SubjektivError::InvalidRecord(format!(
"staging evidence/source_refs are limited to {MAX_STAGING_ANCHORS} items each"
)));
}
Ok(())
}
fn validate_revision_proposal_metadata(proposal: &RevisionProposal) -> Result<()> { fn validate_revision_proposal_metadata(proposal: &RevisionProposal) -> Result<()> {
validate_label("revision proposal Memory id", &proposal.memory_id)?; validate_label("revision proposal Memory id", &proposal.memory_id)?;
if proposal.expected_revision == 0 { if proposal.expected_revision == 0 {
+15 -11
View File
@@ -184,17 +184,21 @@ page members.
Read accepts a positive exact revision or resolves the current revision once. A Read accepts a positive exact revision or resolves the current revision once. A
missing historical revision never falls back to current. Markdown pagination is missing historical revision never falls back to current. Markdown pagination is
line-based (default 200, maximum 1000) with an additional 16 KiB UTF-8 response line-based (default 200, maximum 1000) with a preliminary 16 KiB UTF-8 body
budget. It reports `body_truncated`, `body_next_offset`, and cap and a global 56 KiB budget over the exact pretty-serialized model-visible
`body_next_byte_offset`; an oversized single line resumes from the returned byte response. JSON escaping, provenance, and metadata therefore reduce the current
boundary within that same line. Every continuation offset must also supply the exact body page before the Worker content ceiling is reached. It reports
revision returned by the first page, so a current-revision change cannot mix body `body_truncated`, `body_next_offset`, and `body_next_byte_offset`; an oversized
versions. Provenance uses a separate immutable-revision-bound cursor and returns one or escape-heavy single line resumes from the returned UTF-8 byte boundary within
candidate/derivation reference per page. A staged candidate contains at most 10 that same line. Every continuation offset must also supply the exact revision
evidence records and 10 source references, all returned together under a hard 40 KiB returned by the first page, so a current-revision change cannot mix body versions.
serialized budget for the expanded candidate array; displayed anchor text is capped Provenance uses a separate immutable-revision-bound cursor and returns one
at 64 UTF-8 bytes. Every staging path rejects larger anchor sets, so accepted candidate/derivation reference per page. New staged candidates contain at most 10
provenance is never made unreachable. Candidate evidence is the bounded evidence records and 10 source references; read pages project at most two of each,
with displayed anchor text capped at 64 UTF-8 bytes and nested offsets carried by
the ordinary evidence cursor. Compatible candidates persisted before the admission
cap remain readable across as many evidence pages as needed, while every current
staging path rejects new larger anchor sets. Candidate evidence is the bounded
host-resolved anchor saved in staging; raw Session bodies are not copied host-resolved anchor saved in staging; raw Session bodies are not copied
into Memory responses. Resolved, retracted, and historical revisions remain into Memory responses. Resolved, retracted, and historical revisions remain
addressable by ID. addressable by ID.