fix: migrate legacy session history and retained archives
This commit is contained in:
@@ -68,6 +68,24 @@ impl FsStore {
|
||||
) -> Result<Self, StoreError> {
|
||||
let root = root.into();
|
||||
fs::create_dir_all(&root)?;
|
||||
for entry in fs::read_dir(&root)? {
|
||||
let entry = entry?;
|
||||
if entry.file_type()?.is_dir()
|
||||
&& let Some(session_id) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| name.parse().ok())
|
||||
{
|
||||
let _guard = match retention.lock_live(session_id) {
|
||||
Ok(guard) => guard,
|
||||
// Interrupted erasure may leave bytes behind. Never read or
|
||||
// upgrade them, and do not block unrelated live Sessions.
|
||||
Err(StoreError::SessionErased(_)) => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
crate::worker_session_store::migrate_fs_session(&entry.path(), session_id)?;
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
root,
|
||||
append_lock: Arc::new(Mutex::new(())),
|
||||
@@ -434,6 +452,7 @@ impl Store for FsStore {
|
||||
let path = self.log_path(session_id, segment_id);
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
crate::worker_session_store::migrate_fs_session(parent, session_id)?;
|
||||
}
|
||||
let mut content = String::new();
|
||||
for entry in entries {
|
||||
|
||||
@@ -8,6 +8,9 @@ use agen::llm_client::types::RequestConfig;
|
||||
use protocol::Segment;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use crate::{
|
||||
LogEntry, LoggedHistoryEntry, LoggedItem, LoggedSessionHistoryEntryId,
|
||||
LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, LoggedSystemHistoryEntry, SegmentId,
|
||||
@@ -83,6 +86,7 @@ pub(crate) fn decode_entry(
|
||||
LegacySessionLogEntryV2::History(entry) => Entry::History(entry),
|
||||
LegacySessionLogEntryV2::Current(entry) => Entry::Current(entry),
|
||||
},
|
||||
3 => Entry::Current(serde_json::from_str(line)?),
|
||||
_ => unreachable!("legacy decoder called for unsupported schema {schema_version}"),
|
||||
};
|
||||
Ok(match entry {
|
||||
@@ -93,6 +97,61 @@ pub(crate) fn decode_entry(
|
||||
})
|
||||
}
|
||||
|
||||
/// Schema 3 used DerivedSummary for Host compaction seeds. Only the trusted
|
||||
/// creation boundary may identify one; arbitrary text and later seed items do not.
|
||||
/// Include already-upgraded entries so an interrupted multi-file migration resumes.
|
||||
pub(crate) fn compaction_summary_ids<'a>(
|
||||
entries: impl Iterator<Item = &'a LogEntry>,
|
||||
) -> std::collections::HashSet<LoggedSessionHistoryEntryId> {
|
||||
entries
|
||||
.filter_map(|entry| {
|
||||
let LogEntry::AnnotatedSegmentStart {
|
||||
compacted_from: Some(_),
|
||||
forked_from: None,
|
||||
history,
|
||||
..
|
||||
} = entry
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
let first = history.first()?;
|
||||
(matches!(
|
||||
first.metadata.origin,
|
||||
LoggedSessionHistoryOrigin::DerivedSummary
|
||||
| LoggedSessionHistoryOrigin::CompactionSummary
|
||||
) && matches!(
|
||||
first.item,
|
||||
LoggedItem::Message {
|
||||
role: crate::LoggedRole::System,
|
||||
..
|
||||
}
|
||||
) && agen::Item::from(first.item.clone())
|
||||
.as_text()
|
||||
.is_some_and(|text| text.starts_with("[Compacted context summary]\n\n")))
|
||||
.then(|| first.metadata.entry_id.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn migrate_compaction_origins(
|
||||
entries: &mut [LogEntry],
|
||||
summaries: &std::collections::HashSet<LoggedSessionHistoryEntryId>,
|
||||
) {
|
||||
// Only inherited seed copies can share these identities. Appended input is
|
||||
// not a construction boundary and must never be reclassified from its text.
|
||||
for entry in entries {
|
||||
if let LogEntry::AnnotatedSegmentStart { history, .. } = entry {
|
||||
for entry in history {
|
||||
if summaries.contains(&entry.metadata.entry_id)
|
||||
&& entry.metadata.origin == LoggedSessionHistoryOrigin::DerivedSummary
|
||||
{
|
||||
entry.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum Entry {
|
||||
History(LegacyHistoryLogEntry),
|
||||
Current(LogEntry),
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
use super::*;
|
||||
|
||||
fn seed(
|
||||
origin: LoggedSessionHistoryOrigin,
|
||||
compacted: bool,
|
||||
role: crate::LoggedRole,
|
||||
text: &str,
|
||||
) -> LogEntry {
|
||||
LogEntry::AnnotatedSegmentStart {
|
||||
ts: 1,
|
||||
session_id: SessionId::now_v7(),
|
||||
system_prompt: None,
|
||||
config: Default::default(),
|
||||
forked_from: None,
|
||||
compacted_from: compacted.then(|| SegmentOrigin {
|
||||
segment_id: SegmentId::now_v7(),
|
||||
at_turn_index: 0,
|
||||
}),
|
||||
history: vec![LoggedHistoryEntry {
|
||||
item: LoggedItem::Message {
|
||||
role,
|
||||
content: vec![crate::LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
entry_id: LoggedSessionHistoryEntryId("summary".into()),
|
||||
origin,
|
||||
derivation: Some(crate::LoggedHistoryDerivation {
|
||||
sources: vec![LoggedSessionHistoryEntryId("source".into())],
|
||||
}),
|
||||
},
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_trusted_compaction_seed_is_migrated_and_retries_update_inherited_copies() {
|
||||
let text = "[Compacted context summary]\n\nsummary";
|
||||
for (compacted, role, text) in [
|
||||
(false, crate::LoggedRole::System, text),
|
||||
(true, crate::LoggedRole::User, text),
|
||||
(
|
||||
true,
|
||||
crate::LoggedRole::System,
|
||||
"[Session TaskStore snapshot] tasks",
|
||||
),
|
||||
] {
|
||||
let entry = seed(
|
||||
LoggedSessionHistoryOrigin::DerivedSummary,
|
||||
compacted,
|
||||
role,
|
||||
text,
|
||||
);
|
||||
assert!(compaction_summary_ids([&entry].into_iter()).is_empty());
|
||||
}
|
||||
for origin in [
|
||||
LoggedSessionHistoryOrigin::DerivedSummary,
|
||||
LoggedSessionHistoryOrigin::CompactionSummary,
|
||||
] {
|
||||
let creation = seed(origin, true, crate::LoggedRole::System, text);
|
||||
let inherited = seed(
|
||||
LoggedSessionHistoryOrigin::DerivedSummary,
|
||||
false,
|
||||
crate::LoggedRole::System,
|
||||
text,
|
||||
);
|
||||
let LogEntry::AnnotatedSegmentStart { history, .. } = &inherited else {
|
||||
panic!()
|
||||
};
|
||||
let mut pasted = history[0].clone();
|
||||
pasted.metadata.entry_id = LoggedSessionHistoryEntryId("pasted".into());
|
||||
let appended = LogEntry::AnnotatedUserInput {
|
||||
ts: 2,
|
||||
history: vec![pasted.clone()],
|
||||
segments: vec![],
|
||||
extensions: vec![],
|
||||
};
|
||||
let mut entries = vec![creation, inherited, appended];
|
||||
let ids = compaction_summary_ids(entries.iter());
|
||||
migrate_compaction_origins(&mut entries, &ids);
|
||||
for entry in &entries[..2] {
|
||||
let LogEntry::AnnotatedSegmentStart { history, .. } = entry else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(
|
||||
history[0].metadata.origin,
|
||||
LoggedSessionHistoryOrigin::CompactionSummary
|
||||
);
|
||||
assert_eq!(history[0].metadata.entry_id.0, "summary");
|
||||
assert_eq!(
|
||||
history[0].metadata.derivation.as_ref().unwrap().sources[0].0,
|
||||
"source"
|
||||
);
|
||||
}
|
||||
let LogEntry::AnnotatedUserInput { history, .. } = &entries[2] else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(history, &[pasted]);
|
||||
let once = serde_json::to_vec(&entries).unwrap();
|
||||
migrate_compaction_origins(&mut entries, &ids);
|
||||
assert_eq!(serde_json::to_vec(&entries).unwrap(), once);
|
||||
}
|
||||
}
|
||||
@@ -1602,6 +1602,7 @@ mod tests {
|
||||
segment_id(20),
|
||||
"completed",
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: 4,
|
||||
interrupted: false,
|
||||
result: EngineResult::Finished,
|
||||
@@ -1640,7 +1641,10 @@ mod tests {
|
||||
(
|
||||
segment_id(24),
|
||||
"abandoned",
|
||||
LogEntry::PausedTurnAbandoned { ts: 4 },
|
||||
LogEntry::PausedTurnAbandoned {
|
||||
entry_id: None,
|
||||
ts: 4,
|
||||
},
|
||||
),
|
||||
];
|
||||
for (segment_id, text, outcome) in outcomes {
|
||||
|
||||
@@ -814,7 +814,9 @@ fn boundary_reference(
|
||||
offset: u64,
|
||||
) -> String {
|
||||
let id = match record {
|
||||
LogEntry::RunYielded { entry_id, .. }
|
||||
LogEntry::RunCompleted { entry_id, .. }
|
||||
| LogEntry::PausedTurnAbandoned { entry_id, .. }
|
||||
| LogEntry::RunYielded { entry_id, .. }
|
||||
| LogEntry::RunResumed { entry_id, .. }
|
||||
| LogEntry::RunCancelled { entry_id, .. }
|
||||
| LogEntry::RunErrored { entry_id, .. } => entry_id.as_ref(),
|
||||
|
||||
@@ -244,6 +244,7 @@ fn late_run_boundary_is_new_material_without_changing_previously_supplied_bodies
|
||||
),
|
||||
(
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: 5,
|
||||
interrupted: false,
|
||||
result: agen::EngineResult::Finished,
|
||||
@@ -256,6 +257,7 @@ fn late_run_boundary_is_new_material_without_changing_previously_supplied_bodies
|
||||
),
|
||||
(
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: 5,
|
||||
interrupted: true,
|
||||
result: agen::EngineResult::LimitReached,
|
||||
@@ -662,6 +664,143 @@ fn fork_clips_parent_suffix_and_maps_inherited_copy_to_original_segment() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_v3_compaction_migrates_persistently_and_strict_reader_keeps_original_ids() {
|
||||
for standalone in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let session = crate::new_session_id();
|
||||
let root = crate::new_segment_id();
|
||||
let compact = crate::new_segment_id();
|
||||
let mut summary = message(
|
||||
"summary",
|
||||
LoggedRole::System,
|
||||
"[Compacted context summary]\n\nold summary",
|
||||
);
|
||||
summary.metadata.origin = LoggedSessionHistoryOrigin::DerivedSummary;
|
||||
summary.metadata.derivation = Some(crate::LoggedHistoryDerivation {
|
||||
sources: vec![LoggedSessionHistoryEntryId("u".into())],
|
||||
});
|
||||
let mut start = seed(session, vec![summary.clone()]);
|
||||
if let LogEntry::AnnotatedSegmentStart { compacted_from, .. } = &mut start {
|
||||
*compacted_from = Some(SegmentOrigin {
|
||||
segment_id: root,
|
||||
at_turn_index: 0,
|
||||
});
|
||||
}
|
||||
let records = [
|
||||
start,
|
||||
output("new", "new"),
|
||||
cancelled(),
|
||||
LogEntry::RunCompleted {
|
||||
ts: 6,
|
||||
entry_id: None,
|
||||
interrupted: false,
|
||||
result: agen::EngineResult::Finished,
|
||||
active_run_turn_count: None,
|
||||
},
|
||||
LogEntry::PausedTurnAbandoned {
|
||||
ts: 7,
|
||||
entry_id: None,
|
||||
},
|
||||
];
|
||||
let session_root = if standalone {
|
||||
temp.path().join(session.to_string())
|
||||
} else {
|
||||
temp.path().to_path_buf()
|
||||
};
|
||||
let paths = if standalone {
|
||||
session_root.clone()
|
||||
} else {
|
||||
session_root.join("segments")
|
||||
};
|
||||
std::fs::create_dir_all(&paths).unwrap();
|
||||
for (id, entries) in [
|
||||
(
|
||||
root,
|
||||
vec![seed(
|
||||
session,
|
||||
vec![message("u", LoggedRole::User, "request")],
|
||||
)],
|
||||
),
|
||||
(compact, records.to_vec()),
|
||||
] {
|
||||
let bytes: String = entries
|
||||
.iter()
|
||||
.map(|entry| format!("{}\n", serde_json::to_string(entry).unwrap()))
|
||||
.collect();
|
||||
std::fs::write(paths.join(format!("{id}.jsonl")), bytes).unwrap();
|
||||
}
|
||||
if !standalone {
|
||||
std::fs::write(
|
||||
session_root.join("session.json"),
|
||||
serde_json::to_vec(
|
||||
&serde_json::json!({"schema_version": 3, "session_id": session}),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let open = || {
|
||||
if standalone {
|
||||
SessionPublicMaterialReader::open_fs(
|
||||
temp.path(),
|
||||
session,
|
||||
compact,
|
||||
Default::default(),
|
||||
)
|
||||
} else {
|
||||
SessionPublicMaterialReader::open_worker(&session_root, compact, Default::default())
|
||||
}
|
||||
};
|
||||
assert!(
|
||||
open().is_err(),
|
||||
"strict reader must not reinterpret old provenance"
|
||||
);
|
||||
let migrate = || {
|
||||
if standalone {
|
||||
FsStore::new(temp.path()).unwrap();
|
||||
} else {
|
||||
WorkerSessionStore::new(&session_root).unwrap();
|
||||
}
|
||||
};
|
||||
migrate();
|
||||
let reader = open().unwrap();
|
||||
assert_eq!(&ids(&reader)[..2], ["Eu", "Enew"]);
|
||||
let path = paths.join(format!("{compact}.jsonl"));
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
let entries: Vec<LogEntry> = String::from_utf8(bytes.clone())
|
||||
.unwrap()
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str(line).unwrap())
|
||||
.collect();
|
||||
let LogEntry::AnnotatedSegmentStart { history, .. } = &entries[0] else {
|
||||
panic!()
|
||||
};
|
||||
summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary;
|
||||
assert_eq!(history, &[summary]);
|
||||
// Run-transition ids synthesized from byte offsets must be frozen before
|
||||
// the changed summary record shifts every subsequent record offset.
|
||||
let mut old_offset = 0;
|
||||
for (index, original) in records.iter().enumerate() {
|
||||
if index >= 2 {
|
||||
let mut expected = original.clone();
|
||||
expected.ensure_legacy_session_entry_id(session, compact, old_offset);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&entries[index]).unwrap(),
|
||||
serde_json::to_value(expected).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
ids(&reader)[index],
|
||||
format!("El-run-{session}-{compact}-{old_offset}")
|
||||
);
|
||||
}
|
||||
old_offset += serde_json::to_vec(original).unwrap().len() as u64 + 1;
|
||||
}
|
||||
migrate();
|
||||
assert_eq!(std::fs::read(&path).unwrap(), bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn midturn_compaction_uses_exact_source_ids_and_preserves_discarded_originals() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -1705,6 +1705,7 @@ pub(crate) fn project_session_snapshot_for_segment(
|
||||
interrupted,
|
||||
result,
|
||||
active_run_turn_count: persisted_count,
|
||||
..
|
||||
} => {
|
||||
if *interrupted && matches!(result, agen::EngineResult::Yielded) {
|
||||
let count = (*persisted_count)
|
||||
@@ -2038,6 +2039,7 @@ mod tests {
|
||||
|
||||
fn complete(ts: u64) -> LogEntry {
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts,
|
||||
interrupted: false,
|
||||
result: agen::EngineResult::Finished,
|
||||
|
||||
@@ -329,6 +329,7 @@ pub fn save_run_completed(
|
||||
session_id,
|
||||
segment_id,
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: segment_log::now_millis(),
|
||||
interrupted,
|
||||
result,
|
||||
|
||||
@@ -122,6 +122,8 @@ pub enum LogEntry {
|
||||
/// Replay restores both interruption state and any resumable logical-run
|
||||
/// turn budget.
|
||||
RunCompleted {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
entry_id: Option<crate::LoggedSessionHistoryEntryId>,
|
||||
ts: u64,
|
||||
interrupted: bool,
|
||||
result: EngineResult,
|
||||
@@ -194,7 +196,11 @@ pub enum LogEntry {
|
||||
/// A paused interrupted turn was explicitly abandoned without calling
|
||||
/// `run()` or `resume()` again. Replay clears the interrupted marker so
|
||||
/// the restored Worker is idle and future user input starts a normal new turn.
|
||||
PausedTurnAbandoned { ts: u64 },
|
||||
PausedTurnAbandoned {
|
||||
ts: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
entry_id: Option<crate::LoggedSessionHistoryEntryId>,
|
||||
},
|
||||
|
||||
/// `RequestConfig` changed.
|
||||
ConfigChanged { ts: u64, config: RequestConfig },
|
||||
@@ -253,7 +259,9 @@ impl LogEntry {
|
||||
/// written to storage is also the value published to live subscribers.
|
||||
pub fn ensure_session_entry_id(&mut self) {
|
||||
let entry_id = match self {
|
||||
Self::RunYielded { entry_id, .. }
|
||||
Self::RunCompleted { entry_id, .. }
|
||||
| Self::PausedTurnAbandoned { entry_id, .. }
|
||||
| Self::RunYielded { entry_id, .. }
|
||||
| Self::RunResumed { entry_id, .. }
|
||||
| Self::RunCancelled { entry_id, .. }
|
||||
| Self::RunErrored { entry_id, .. } => entry_id,
|
||||
@@ -275,7 +283,9 @@ impl LogEntry {
|
||||
record_start: u64,
|
||||
) {
|
||||
let entry_id = match self {
|
||||
Self::RunYielded { entry_id, .. }
|
||||
Self::RunCompleted { entry_id, .. }
|
||||
| Self::PausedTurnAbandoned { entry_id, .. }
|
||||
| Self::RunYielded { entry_id, .. }
|
||||
| Self::RunResumed { entry_id, .. }
|
||||
| Self::RunCancelled { entry_id, .. }
|
||||
| Self::RunErrored { entry_id, .. } => entry_id,
|
||||
@@ -586,6 +596,7 @@ mod tests {
|
||||
turn_count: 1,
|
||||
},
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: 3200,
|
||||
interrupted: false,
|
||||
result: EngineResult::Finished,
|
||||
@@ -893,12 +904,16 @@ mod tests {
|
||||
compacted_from: None,
|
||||
},
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: 100,
|
||||
interrupted: true,
|
||||
result: EngineResult::Paused,
|
||||
active_run_turn_count: Some(1),
|
||||
},
|
||||
LogEntry::PausedTurnAbandoned { ts: 200 },
|
||||
LogEntry::PausedTurnAbandoned {
|
||||
entry_id: None,
|
||||
ts: 200,
|
||||
},
|
||||
]);
|
||||
assert!(!state.last_run_interrupted);
|
||||
assert_eq!(state.active_run_turn_count, None);
|
||||
@@ -1056,6 +1071,7 @@ mod tests {
|
||||
turn_count: 2,
|
||||
},
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: 300,
|
||||
interrupted: true,
|
||||
result: EngineResult::LimitReached,
|
||||
@@ -1069,13 +1085,16 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn paused_turn_abandoned_entry_round_trip_via_json() {
|
||||
let entry = LogEntry::PausedTurnAbandoned { ts: 12345 };
|
||||
let entry = LogEntry::PausedTurnAbandoned {
|
||||
entry_id: None,
|
||||
ts: 12345,
|
||||
};
|
||||
let json = serde_json::to_string(&entry).unwrap();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed["kind"], "paused_turn_abandoned");
|
||||
let decoded: LogEntry = serde_json::from_str(&json).unwrap();
|
||||
match decoded {
|
||||
LogEntry::PausedTurnAbandoned { ts } => assert_eq!(ts, 12345),
|
||||
LogEntry::PausedTurnAbandoned { ts, .. } => assert_eq!(ts, 12345),
|
||||
other => panic!("expected PausedTurnAbandoned, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,8 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::SystemTime;
|
||||
|
||||
const SESSION_SCHEMA_VERSION: u32 = 3;
|
||||
const SESSION_SCHEMA_VERSION: u32 = 4;
|
||||
const ANNOTATED_SESSION_SCHEMA_VERSION: u32 = 3;
|
||||
const PREVIOUS_SESSION_SCHEMA_VERSION: u32 = 2;
|
||||
const LEGACY_SESSION_SCHEMA_VERSION: u32 = 1;
|
||||
const SESSION_FILE: &str = "session.json";
|
||||
@@ -212,9 +213,11 @@ impl WorkerSessionStore {
|
||||
SESSION_SCHEMA_VERSION => {
|
||||
validate_canonical_segment_logs(&root)?;
|
||||
}
|
||||
PREVIOUS_SESSION_SCHEMA_VERSION | LEGACY_SESSION_SCHEMA_VERSION => {
|
||||
migrate_segment_logs_to_v3(
|
||||
&root,
|
||||
LEGACY_SESSION_SCHEMA_VERSION
|
||||
| PREVIOUS_SESSION_SCHEMA_VERSION
|
||||
| ANNOTATED_SESSION_SCHEMA_VERSION => {
|
||||
migrate_segment_logs_to_v4(
|
||||
segment_log_paths(&root)?,
|
||||
manifest.session_id,
|
||||
manifest.schema_version,
|
||||
)?;
|
||||
@@ -1107,7 +1110,12 @@ fn open_retained_file(path: &Path) -> std::io::Result<File> {
|
||||
}
|
||||
|
||||
fn segment_log_paths(root: &Path) -> Result<Vec<(SegmentId, PathBuf)>, StoreError> {
|
||||
let segments = root.join(SEGMENTS_DIR);
|
||||
segment_log_paths_in_directory(&root.join(SEGMENTS_DIR))
|
||||
}
|
||||
|
||||
fn segment_log_paths_in_directory(
|
||||
segments: &Path,
|
||||
) -> Result<Vec<(SegmentId, PathBuf)>, StoreError> {
|
||||
if !segments.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
@@ -1150,21 +1158,22 @@ fn segment_log_paths(root: &Path) -> Result<Vec<(SegmentId, PathBuf)>, StoreErro
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
fn migrate_segment_logs_to_v3(
|
||||
root: &Path,
|
||||
fn migrate_segment_logs_to_v4(
|
||||
paths: Vec<(SegmentId, PathBuf)>,
|
||||
session_id: SessionId,
|
||||
source_schema_version: u32,
|
||||
) -> Result<(), StoreError> {
|
||||
struct MigrationPlan {
|
||||
path: PathBuf,
|
||||
source: Vec<u8>,
|
||||
entries: Vec<LogEntry>,
|
||||
output: Vec<u8>,
|
||||
}
|
||||
|
||||
// Phase 1 is strictly read-only. Every segment must parse and canonicalize
|
||||
// successfully before the first authoritative byte is replaced.
|
||||
let mut plans = Vec::new();
|
||||
for (segment_id, path) in segment_log_paths(root)? {
|
||||
for (segment_id, path) in paths {
|
||||
let source = fs::read(&path)?;
|
||||
let canonical = parse_legacy_jsonl(source_schema_version, session_id, segment_id, &source)
|
||||
.map_err(|error| StoreError::Corrupt {
|
||||
@@ -1174,17 +1183,25 @@ fn migrate_segment_logs_to_v3(
|
||||
path.display()
|
||||
),
|
||||
})?;
|
||||
let mut output = Vec::new();
|
||||
for entry in canonical {
|
||||
serde_json::to_writer(&mut output, &entry)?;
|
||||
output.push(b'\n');
|
||||
}
|
||||
plans.push(MigrationPlan {
|
||||
path,
|
||||
source,
|
||||
output,
|
||||
entries: canonical,
|
||||
output: Vec::new(),
|
||||
});
|
||||
}
|
||||
// Resolve trusted creation boundaries first, then update inherited copies by
|
||||
// logical identity too (fork/rewind seeds retain the same entry id).
|
||||
let summaries = crate::legacy_session_log::compaction_summary_ids(
|
||||
plans.iter().flat_map(|plan| plan.entries.iter()),
|
||||
);
|
||||
for plan in &mut plans {
|
||||
crate::legacy_session_log::migrate_compaction_origins(&mut plan.entries, &summaries);
|
||||
for entry in std::mem::take(&mut plan.entries) {
|
||||
serde_json::to_writer(&mut plan.output, &entry)?;
|
||||
plan.output.push(b'\n');
|
||||
}
|
||||
}
|
||||
|
||||
// Fence the complete preflight snapshot before starting phase 2. Session
|
||||
// open is the exclusive restore boundary; this additionally fails closed
|
||||
@@ -1207,6 +1224,30 @@ fn migrate_segment_logs_to_v3(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Upgrade the standalone per-Session layout using the same persisted format
|
||||
/// migration as Worker aggregates. Called only at writable Host store startup.
|
||||
pub(crate) fn migrate_fs_session(root: &Path, session_id: SessionId) -> Result<(), StoreError> {
|
||||
let marker = root.join("session-schema.json");
|
||||
let version: u32 = match fs::read(&marker) {
|
||||
Ok(bytes) => serde_json::from_slice(&bytes)?,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
ANNOTATED_SESSION_SCHEMA_VERSION
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
match version {
|
||||
SESSION_SCHEMA_VERSION => Ok(()),
|
||||
ANNOTATED_SESSION_SCHEMA_VERSION => {
|
||||
migrate_segment_logs_to_v4(segment_log_paths_in_directory(root)?, session_id, version)?;
|
||||
atomic_write_json(&marker, &SESSION_SCHEMA_VERSION)
|
||||
}
|
||||
_ => Err(StoreError::Corrupt {
|
||||
line: 0,
|
||||
message: format!("unsupported standalone Session schema {version}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_canonical_segment_logs(root: &Path) -> Result<(), StoreError> {
|
||||
for (_, path) in segment_log_paths(root)? {
|
||||
let _: Vec<LogEntry> = parse_jsonl(&fs::read(&path)?)?;
|
||||
@@ -1223,17 +1264,26 @@ fn parse_legacy_jsonl(
|
||||
let text = std::str::from_utf8(bytes).map_err(|error| {
|
||||
serde_json::Error::io(std::io::Error::new(std::io::ErrorKind::InvalidData, error))
|
||||
})?;
|
||||
text.lines()
|
||||
let mut offset = 0_u64;
|
||||
text.split_inclusive('\n')
|
||||
.enumerate()
|
||||
.filter(|(_, line)| !line.trim().is_empty())
|
||||
.map(|(line_index, line)| {
|
||||
crate::legacy_session_log::decode_entry(
|
||||
.filter_map(|(line_index, line)| {
|
||||
let start = offset;
|
||||
offset += line.len() as u64;
|
||||
(!line.trim().is_empty()).then_some((line_index, line, start))
|
||||
})
|
||||
.map(|(line_index, line, start)| {
|
||||
let mut entry = crate::legacy_session_log::decode_entry(
|
||||
schema_version,
|
||||
line,
|
||||
session_id,
|
||||
segment_id,
|
||||
line_index,
|
||||
)
|
||||
)?;
|
||||
// Byte offsets change during migration. Persist identities previously
|
||||
// synthesized from the OLD offsets before rewriting any record.
|
||||
entry.ensure_legacy_session_entry_id(session_id, segment_id, start);
|
||||
Ok(entry)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ fn round_trip_write_and_read() {
|
||||
turn_count: 1,
|
||||
},
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: None,
|
||||
ts: 3200,
|
||||
interrupted: false,
|
||||
result: EngineResult::Finished,
|
||||
@@ -157,9 +158,10 @@ fn create_segment_replacement_keeps_open_old_log_and_commits_complete_new_log()
|
||||
assert_eq!(reopened.read_entry_count(sid, segid).unwrap(), new.len());
|
||||
assert_eq!(
|
||||
std::fs::read_dir(path.parent().unwrap()).unwrap().count(),
|
||||
1,
|
||||
"successful replacement must not leave temporary files"
|
||||
2,
|
||||
"successful replacement leaves only the segment and Session schema marker"
|
||||
);
|
||||
assert!(path.parent().unwrap().join("session-schema.json").is_file());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user