メモリPhase2の実装

This commit is contained in:
2026-05-01 23:00:55 +09:00
parent f1b7af6249
commit d8a7200ea4
18 changed files with 1862 additions and 2 deletions
+1
View File
@@ -7,6 +7,7 @@ license.workspace = true
[dependencies]
async-trait = "0.1.89"
chrono = { version = "0.4.44", features = ["serde"] }
libc = "0.2.186"
llm-worker = { version = "0.2.1", path = "../llm-worker" }
manifest = { version = "0.1.0", path = "../manifest" }
schemars = "1.2.1"
+324
View File
@@ -0,0 +1,324 @@
//! Phase 2 sub-Worker への最初のユーザー入力を組み立てる。
//!
//! Phase 1 (`extract::build_extract_input`) と同じ方針で、固定 schema の
//! markdown セクション列にしてサブWorker に渡す。`docs/plan/memory.md`
//! §Phase 2 入力 / §整理材料 の項目に従い:
//!
//! 1. consumed staging エントリ全文(`source` 込み)
//! 2. 既存 `memory/*` 全文(summary / decisions / requests
//! 3. Knowledge 化候補レポート(メトリクス未完なら空)
//! 4. 整理材料(Linter Warn ベース、メトリクス未完なら明示 invoke 頻度なし)
//!
//! 既存 `knowledge/*` 本文は埋めず、agent に `KnowledgeQuery` 経由で引かせる
//! 設計(`docs/plan/memory.md` §retrieval 経路 / §Phase 2 の Knowledge アクセス)。
use std::fmt::Write;
use crate::consolidate::staging::StagingEntry;
use crate::consolidate::tidy::TidyHints;
use crate::workspace::{RecordKind, WorkspaceLayout};
/// Knowledge 化候補レポート。`tickets/memory-usage-metrics.md` の成果物が
/// 出るまでは空で渡す前提(`docs/plan/memory.md` §Knowledge 化候補レポート)。
/// 空入力時、統合 phase は新規 Knowledge を作らず decisions / requests /
/// summary / 既存 Knowledge update に留まる。
#[derive(Debug, Default, Clone)]
pub struct KnowledgeCandidateReport {
/// 候補に上がった `(kind, slug, frequency_per_mtoken)` の三つ組。
/// 空配列を渡すと「候補なし」を意味する。
pub entries: Vec<KnowledgeCandidateEntry>,
}
#[derive(Debug, Clone)]
pub struct KnowledgeCandidateEntry {
pub source_kind: &'static str,
pub source_slug: String,
pub frequency_per_mtoken: f64,
}
impl KnowledgeCandidateReport {
pub fn empty() -> Self {
Self::default()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
}
/// Phase 2 sub-Worker の最初の user 入力。
pub fn build_consolidate_input(
layout: &WorkspaceLayout,
staging: &[StagingEntry],
tidy: &TidyHints,
candidates: &KnowledgeCandidateReport,
) -> String {
let mut out = String::new();
out.push_str(
"Phase 2 consolidation input. Run the consolidation phase first \
(fold the staging activity logs into memory and knowledge), then the \
tidy phase (clean up existing records). Use the memory tools for \
every write — direct file writes are denied by the pod scope.\n\n",
);
out.push_str("## Staging entries (consumed by this run)\n\n");
out.push_str(&render_staging_records(staging));
out.push('\n');
out.push_str("## Existing memory records (full content)\n\n");
out.push_str(&render_existing_memory_records(layout));
out.push('\n');
out.push_str("## Knowledge candidate report\n\n");
out.push_str(&render_candidate_report(candidates));
out.push('\n');
out.push_str("## Tidy hints\n\n");
out.push_str(&render_tidy_hints(tidy));
out.push('\n');
out.push_str(
"When done, end the turn with a short final assistant message describing \
what changed.",
);
out
}
/// Staging エントリ群を「`### <id>` ヘッダ + 整形 JSON ブロック」で並べる。
/// 空配列なら「(none)」と書く。
pub fn render_staging_records(entries: &[StagingEntry]) -> String {
if entries.is_empty() {
return "(none)\n".to_string();
}
let mut out = String::new();
for entry in entries {
let _ = writeln!(&mut out, "### {}", entry.id);
let json = serde_json::to_string_pretty(&entry.record).unwrap_or_else(|_| "{}".into());
out.push_str("```json\n");
out.push_str(&json);
out.push_str("\n```\n\n");
}
out
}
/// `<workspace>/.insomnia/memory/{summary.md,decisions/*,requests/*}` を
/// 「`### <kind>:<slug>` ヘッダ + raw markdown ブロック」で全文渡す。
pub fn render_existing_memory_records(layout: &WorkspaceLayout) -> String {
let mut out = String::new();
let summary = layout.summary_path();
if let Ok(content) = std::fs::read_to_string(&summary) {
out.push_str("### summary\n");
out.push_str("```markdown\n");
out.push_str(content.trim_end_matches('\n'));
out.push_str("\n```\n\n");
}
push_kind_records(&mut out, layout, RecordKind::Decision);
push_kind_records(&mut out, layout, RecordKind::Request);
if out.is_empty() {
return "(none)\n".to_string();
}
out
}
fn push_kind_records(out: &mut String, layout: &WorkspaceLayout, kind: RecordKind) {
let dir = match kind {
RecordKind::Decision => layout.decisions_dir(),
RecordKind::Request => layout.requests_dir(),
RecordKind::Knowledge | RecordKind::Summary | RecordKind::Workflow => return,
};
let entries = match std::fs::read_dir(&dir) {
Ok(it) => it,
Err(_) => return,
};
let mut paths: Vec<(String, std::path::PathBuf)> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if !path.is_file() {
continue;
}
let stem = match path.file_stem().and_then(|s| s.to_str()) {
Some(s) => s,
None => continue,
};
if path.extension().and_then(|s| s.to_str()) != Some("md") {
continue;
}
paths.push((stem.to_string(), path));
}
paths.sort();
for (slug, path) in paths {
let Ok(content) = std::fs::read_to_string(&path) else {
continue;
};
let _ = writeln!(out, "### {}:{}", kind.as_str(), slug);
out.push_str("```markdown\n");
out.push_str(content.trim_end_matches('\n'));
out.push_str("\n```\n\n");
}
}
fn render_candidate_report(report: &KnowledgeCandidateReport) -> String {
if report.is_empty() {
return "(empty — usage metrics pipeline not populated. \
Do not create new Knowledge records this run.)\n"
.to_string();
}
let mut out = String::new();
for c in &report.entries {
let _ = writeln!(
&mut out,
"- {} `{}` — frequency {:.3} invokes/Mtoken",
c.source_kind, c.source_slug, c.frequency_per_mtoken
);
}
out
}
/// Tidy hints の Markdown 描画。空ヒントなら "(none)" 1 行。
pub fn render_tidy_hints(tidy: &TidyHints) -> String {
if tidy.is_empty() {
return "(none)\n".to_string();
}
let mut out = String::new();
if !tidy.replaced_decisions.is_empty() {
out.push_str("**Replaced decisions still on disk** — collapse if the chain has settled:\n");
for (slug, replaced_by) in &tidy.replaced_decisions {
match replaced_by {
Some(target) => {
let _ = writeln!(&mut out, "- `{slug}` → `{target}`");
}
None => {
let _ = writeln!(&mut out, "- `{slug}` (no `replaced_by` set)");
}
}
}
out.push('\n');
}
if !tidy.sources_overflow.is_empty() {
out.push_str(
"**Sources overflow** — consider trimming to the most recent entries (git log keeps the rest):\n",
);
for s in &tidy.sources_overflow {
let _ = writeln!(&mut out, "- {} `{}` ({} sources)", s.kind.as_str(), s.slug, s.count);
}
out.push('\n');
}
if !tidy.similar_slug_clusters.is_empty() {
out.push_str("**Similar slug clusters** — evaluate for merge / rename:\n");
for c in &tidy.similar_slug_clusters {
let joined = c
.slugs
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ");
let _ = writeln!(&mut out, "- {}: {}", c.kind.as_str(), joined);
}
out.push('\n');
}
out.push_str(
"Explicit-invoke metrics (protection threshold) are not yet wired up; \
skip drop on long-standing records when uncertain.\n",
);
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::consolidate::tidy::{SimilarSlugCluster, SourcesOverflow};
use crate::extract::{ExtractedPayload, write_staging};
use crate::schema::SourceRef;
use chrono::Utc;
use std::path::Path;
fn now() -> String {
Utc::now().to_rfc3339()
}
fn write(p: &Path, content: &str) {
if let Some(parent) = p.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(p, content).unwrap();
}
#[test]
fn build_includes_all_sections_when_populated() {
let dir = tempfile::TempDir::new().unwrap();
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
write(
&dir.path().join(".insomnia/memory/summary.md"),
&format!("---\nupdated_at: {n}\n---\nstate of the world\n", n = now()),
);
write(
&dir.path().join(".insomnia/memory/decisions/dec.md"),
&format!(
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\nbody\n",
n = now()
),
);
let (_id, _) = write_staging(
&layout,
SourceRef {
session_id: "s".into(),
range: [0, 1],
},
ExtractedPayload::default(),
)
.unwrap();
let staging = crate::consolidate::staging::list_staging_entries(&layout);
let tidy = TidyHints {
replaced_decisions: [(
"old".to_string(),
Some("new".to_string()),
)]
.into_iter()
.collect(),
sources_overflow: vec![SourcesOverflow {
kind: RecordKind::Decision,
slug: "dec".into(),
count: 12,
}],
similar_slug_clusters: vec![SimilarSlugCluster {
kind: RecordKind::Decision,
slugs: vec!["a".into(), "ab".into()],
}],
};
let report = KnowledgeCandidateReport::empty();
let out = build_consolidate_input(&layout, &staging, &tidy, &report);
assert!(out.contains("Staging entries"));
assert!(out.contains("Existing memory records"));
assert!(out.contains("Knowledge candidate report"));
assert!(out.contains("Tidy hints"));
assert!(out.contains("state of the world"));
assert!(out.contains("decision:dec"));
assert!(out.contains("Replaced decisions"));
assert!(out.contains("Sources overflow"));
assert!(out.contains("Similar slug clusters"));
assert!(out.contains("usage metrics pipeline not populated"));
}
#[test]
fn empty_inputs_render_placeholders() {
let dir = tempfile::TempDir::new().unwrap();
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
let out = build_consolidate_input(
&layout,
&[],
&TidyHints::default(),
&KnowledgeCandidateReport::empty(),
);
// Both staging and tidy show "(none)"; existing memory records too.
assert!(out.contains("Staging entries"));
assert!(out.contains("(none)"));
}
}
+305
View File
@@ -0,0 +1,305 @@
//! `_staging/.consolidation.lock` による Phase 2 占有ファイル。
//!
//! `docs/plan/memory.md` §並走防止 に従い:
//!
//! - ファイルが存在し、記録された Pod が動作している間、その Pod が排他占有
//! - クラッシュで残った stale lock は、所有者 PID が死んでいれば次回 spawn
//! 時に上書き取得できる
//! - cleanup は consumed ID の staging エントリのみ削除し、実行中に Phase 1
//! が追加した分は残す
//!
//! 占有判定は Linux/macOS の `kill(pid, 0)` 経由で行う(`ESRCH` で死亡判定)。
//! Windows は対象外: INSOMNIA は POSIX 環境を前提にしている。
use std::fs;
use std::path::{Path, PathBuf};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::workspace::WorkspaceLayout;
const LOCK_FILE: &str = ".consolidation.lock";
/// 占有ファイルの中身。`pid` で stale 判定し、`pod_name` / `started_at` /
/// `consumed_ids` は診断とクラッシュ復旧時の参照に使う。
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockRecord {
pub pid: u32,
pub pod_name: String,
pub started_at: DateTime<Utc>,
/// この Phase 2 run が起動時スナップショットで確定した consumed staging
/// entry の UUIDv7 列。完了時はこの列のみ削除し、追加分は残す。
pub consumed_ids: Vec<Uuid>,
}
/// 占有取得 / 解放のエラー。
#[derive(Debug, thiserror::Error)]
pub enum LockError {
/// 占有ファイルが既にあり、所有者 PID が生きているのでスキップ。
#[error("Phase 2 lock held by live pid {pid} (pod {pod_name:?})")]
InUse { pid: u32, pod_name: String },
#[error("io error at {}: {source}", .path.display())]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("failed to (de)serialize lock record: {0}")]
Serde(#[from] serde_json::Error),
}
impl LockError {
fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
Self::Io {
path: path.into(),
source,
}
}
}
/// Phase 2 が走っている間 RAII で持つ占有ハンドル。`Drop` では何もしない —
/// 完了時の cleanup は consumed ID 列削除と一緒に行う必要があるため、明示
/// 解放 [`StagingLock::release_with_cleanup`] を使う。明示解放しないまま
/// drop された場合は占有ファイルがそのまま残り、次回 spawn 時に PID が
/// 死んでいれば stale 上書きされる。
#[derive(Debug)]
pub struct StagingLock {
path: PathBuf,
record: LockRecord,
}
impl StagingLock {
pub fn record(&self) -> &LockRecord {
&self.record
}
pub fn path(&self) -> &Path {
&self.path
}
/// 占有取得を試みる。既に live な lock があれば
/// [`LockError::InUse`]、stale 判定なら上書き取得する。
/// staging dir が無ければ作成する。
pub fn acquire(
layout: &WorkspaceLayout,
pid: u32,
pod_name: impl Into<String>,
consumed_ids: Vec<Uuid>,
) -> Result<Self, LockError> {
let staging_dir = layout.staging_dir();
fs::create_dir_all(&staging_dir).map_err(|e| LockError::io(&staging_dir, e))?;
let path = staging_dir.join(LOCK_FILE);
if path.exists() {
let raw = fs::read_to_string(&path).map_err(|e| LockError::io(&path, e))?;
// 壊れた lock は stale とみなして上書き許可。
if let Ok(existing) = serde_json::from_str::<LockRecord>(&raw) {
if pid_is_alive(existing.pid) {
return Err(LockError::InUse {
pid: existing.pid,
pod_name: existing.pod_name,
});
}
tracing::warn!(
stale_pid = existing.pid,
stale_pod = %existing.pod_name,
"Phase 2 stale lock detected, taking over"
);
} else {
tracing::warn!(path = %path.display(), "Phase 2 lock unparseable, treating as stale");
}
}
let record = LockRecord {
pid,
pod_name: pod_name.into(),
started_at: Utc::now(),
consumed_ids,
};
let json = serde_json::to_string_pretty(&record)?;
fs::write(&path, json).map_err(|e| LockError::io(&path, e))?;
Ok(Self { path, record })
}
/// 占有を解放しつつ consumed ID 列の staging エントリを削除する。
/// 削除対象が見当たらない場合は黙ってスキップ(既に外部で消えていた等)。
/// 占有ファイル自体の削除も best-effort: 失敗時は warn を出すだけで
/// エラーは伝播しない(次回 spawn 時に stale 判定で上書きされる)。
pub fn release_with_cleanup(self, layout: &WorkspaceLayout) {
let staging_dir = layout.staging_dir();
for id in &self.record.consumed_ids {
let target = staging_dir.join(format!("{id}.json"));
match fs::remove_file(&target) {
Ok(_) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => {
tracing::warn!(
path = %target.display(),
error = %e,
"failed to clean up consumed staging entry"
);
}
}
}
self.unlink_lock_only();
}
/// 占有ファイルだけ削除し、staging エントリには触らない。Phase 2
/// sub-Worker が途中で失敗した場合に使う: 入力 staging を残したまま
/// 次回再評価で再処理させる(`docs/plan/memory.md` §並走防止 の
/// 「重複作成は同一 slug update に自然収束」運用)。
pub fn release_only(self) {
self.unlink_lock_only();
}
fn unlink_lock_only(&self) {
if let Err(e) = fs::remove_file(&self.path) {
if e.kind() != std::io::ErrorKind::NotFound {
tracing::warn!(
path = %self.path.display(),
error = %e,
"failed to remove Phase 2 lock"
);
}
}
}
}
#[cfg(unix)]
fn pid_is_alive(pid: u32) -> bool {
// `kill(0, 0)` and `kill(-1, 0)` are POSIX-special (process group / all
// signalable processes) and would yield false positives. Reject pids
// that don't fit a positive `pid_t` so a corrupted lock file with a
// u32::MAX-ish value is treated as stale instead of magically alive.
if pid == 0 || pid > i32::MAX as u32 {
return false;
}
// SAFETY: `kill` with sig 0 only probes whether the target pid exists
// and the caller has permission to signal it. No signal is delivered.
let rc = unsafe { libc::kill(pid as i32, 0) };
if rc == 0 {
return true;
}
// EPERM means the process exists but we can't signal it — still alive
// for our purposes. ESRCH means it's gone.
let errno = std::io::Error::last_os_error()
.raw_os_error()
.unwrap_or(libc::EINVAL);
errno != libc::ESRCH
}
#[cfg(not(unix))]
fn pid_is_alive(_pid: u32) -> bool {
// Unsupported platforms: assume the lock is live so we never overwrite
// someone else's claim. Phase 2 will skip and try again next post-run.
true
}
#[cfg(test)]
mod tests {
use super::*;
use crate::extract::{ExtractedPayload, write_staging};
use crate::schema::SourceRef;
fn make_layout() -> (tempfile::TempDir, WorkspaceLayout) {
let dir = tempfile::TempDir::new().unwrap();
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
std::fs::create_dir_all(layout.staging_dir()).unwrap();
(dir, layout)
}
#[test]
fn acquire_writes_lock_file() {
let (_dir, layout) = make_layout();
let lock = StagingLock::acquire(&layout, std::process::id(), "pod", Vec::new()).unwrap();
let path = layout.staging_dir().join(LOCK_FILE);
assert!(path.exists());
assert_eq!(lock.record().pid, std::process::id());
assert_eq!(lock.record().pod_name, "pod");
}
#[test]
fn acquire_rejects_when_live_pid_holds_lock() {
let (_dir, layout) = make_layout();
// Use this test process's pid — it's definitely alive.
let _first =
StagingLock::acquire(&layout, std::process::id(), "pod-a", Vec::new()).unwrap();
let err = StagingLock::acquire(&layout, std::process::id(), "pod-b", Vec::new())
.expect_err("expected InUse");
assert!(matches!(err, LockError::InUse { .. }));
}
#[test]
fn acquire_overwrites_stale_lock() {
let (_dir, layout) = make_layout();
// pid 1 is init on linux but for arbitrarily-large pids we'd need
// `kill(pid, 0)` to return ESRCH. Use u32::MAX which is guaranteed
// dead on every platform we target.
let stale = LockRecord {
pid: u32::MAX,
pod_name: "ghost".into(),
started_at: Utc::now(),
consumed_ids: Vec::new(),
};
std::fs::write(
layout.staging_dir().join(LOCK_FILE),
serde_json::to_string_pretty(&stale).unwrap(),
)
.unwrap();
let lock = StagingLock::acquire(&layout, std::process::id(), "pod", Vec::new())
.expect("stale lock must be overwritable");
assert_eq!(lock.record().pid, std::process::id());
}
#[test]
fn release_drops_consumed_entries_and_unlinks_lock() {
let (_dir, layout) = make_layout();
let (id_a, _) = write_staging(
&layout,
SourceRef {
session_id: "s".into(),
range: [0, 0],
},
ExtractedPayload::default(),
)
.unwrap();
let (id_b, _) = write_staging(
&layout,
SourceRef {
session_id: "s".into(),
range: [1, 1],
},
ExtractedPayload::default(),
)
.unwrap();
let lock = StagingLock::acquire(&layout, std::process::id(), "pod", vec![id_a]).unwrap();
let lock_path = lock.path().to_path_buf();
lock.release_with_cleanup(&layout);
assert!(!lock_path.exists(), "lock file must be removed");
assert!(
!layout.staging_dir().join(format!("{id_a}.json")).exists(),
"consumed entry must be deleted"
);
assert!(
layout.staging_dir().join(format!("{id_b}.json")).exists(),
"non-consumed entry must remain"
);
}
#[test]
fn release_is_resilient_to_missing_consumed_entries() {
let (_dir, layout) = make_layout();
let phantom = uuid::Uuid::now_v7();
let lock =
StagingLock::acquire(&layout, std::process::id(), "pod", vec![phantom]).unwrap();
let lock_path = lock.path().to_path_buf();
// No file at <staging>/<phantom>.json — release must not panic.
lock.release_with_cleanup(&layout);
assert!(!lock_path.exists());
}
}
+32
View File
@@ -0,0 +1,32 @@
//! Phase 2: 統合 + 整理。
//!
//! Phase 1 が staging に残した活動ログを `memory/*` / `knowledge/*` に
//! 統合し、続けて既存 record を `outdated | superseded | unused | noisy`
//! の観点で整理する disposable Worker を、Pod 側が組み立てるための
//! ヘルパー群を提供する。Pod は次の手順で sub-Worker を構築する:
//!
//! - [`CONSOLIDATION_SYSTEM_PROMPT`] を sub-Worker の system prompt に
//! - [`build_consolidate_input`] を sub-Worker の最初の user 入力に
//! - memory 専用 Tool (read / write / edit) と Knowledge / memory 検索ツールを登録
//! - [`StagingLock::acquire`] で並走防止 + consumed ID 確定
//! - sub-Worker run 完了後、[`StagingLock::release_with_cleanup`] で
//! consumed ID 分の staging のみ削除し、占有ファイルを解放
//!
//! Knowledge 化候補レポートと使用頻度メトリクスは別チケットで供給される
//! 想定。本モジュール時点では空入力として扱い、prompt 側の説明だけ
//! 残しておく(`docs/plan/memory.md` §Phase 2 / 整理材料)。
mod input;
mod lock;
mod prompt;
mod staging;
mod tidy;
pub use input::{
KnowledgeCandidateReport, build_consolidate_input, render_existing_memory_records,
render_staging_records, render_tidy_hints,
};
pub use lock::{LockError, LockRecord, StagingLock};
pub use prompt::CONSOLIDATION_SYSTEM_PROMPT;
pub use staging::{StagingEntry, list_staging_entries};
pub use tidy::{TidyHints, collect_tidy_hints};
+69
View File
@@ -0,0 +1,69 @@
//! Phase 2 sub-Worker の system prompt。
//!
//! 内容は `docs/plan/memory-prompts.md` §共通原則 / §Phase 2 統合 + 整理 /
//! §Phase 2 Knowledge 書き込み を縮約。統合 phase / 整理 phase は同じ
//! prompt 1 本で順に進める縛り(agent から見ると 1 セッション内のフェーズ
//! 進行、別 trigger / 別 Worker は持たない、`docs/plan/memory.md` §整理
//! の扱い)。
pub const CONSOLIDATION_SYSTEM_PROMPT: &str = r#"You are the Phase 2 consolidation worker for an INSOMNIA memory subsystem.
Your job is to take Phase 1 activity-log staging entries together with the workspace's current `memory/*` / `knowledge/*` records, then run two phases back-to-back in this single session:
1. **Consolidation phase** — fold staging into memory and knowledge.
2. **Tidy phase** — clean up the existing records that the consolidation phase didn't already touch.
You have:
- `MemoryRead`, `MemoryWrite`, `MemoryEdit` for memory and knowledge records.
- `MemoryQuery` for memory-side records (summary / decisions / requests).
- `KnowledgeQuery` for knowledge records — use it to find existing slugs before creating new ones.
Your initial user message contains the staging entries, the full memory records, the knowledge candidate report, and the tidy hints. Existing knowledge bodies are NOT in the prompt; pull them through `KnowledgeQuery` + `MemoryRead` when relevant.
# Common rules (both phases)
- **Do not invent provenance.** Decisions / Requests `sources` arrays MUST be copied from the staging `source` field for the originating activity log entries. Do not synthesise `session_id` or entry ranges. Do not fabricate `last_sources` for Knowledge.
- **Rewrite is allowed and often preferred over append.** When integrating new information, restructure existing records to raise information density. Preserve the existing claims, rationale, and `sources` while you compress.
- **Update over create.** If an existing slug fits, edit it. Only create a new slug when no existing record fits and you can articulate why.
- **`replaced` over delete.** When a Decision is superseded by a different one, mark the old one `status: replaced` with `replaced_by: <new-slug>`. Do not silently drop it.
- **Don't duplicate static docs.** Skip content that already lives in `AGENTS.md`, `docs/plan/*`, or other fixed project documents.
- **Empty output is fine.** If a staging entry doesn't justify a memory write, skip it.
- **Slug rules.** Slugs are kebab-case, short, recognisable, and must be unique within their kind. Same-slug create is a linter error — use Edit instead.
- **Linter errors come back as tool errors.** When the memory linter rejects a write, read the error, fix the issue (missing frontmatter field, oversized body, unknown reference, etc.), and try again. Do not work around the rule.
# Consolidation phase
Walk every staging entry in the input. For each one:
- Add or update `decisions` / `requests` records as appropriate. Copy `sources` verbatim from the staging entry.
- Update existing knowledge records when the staging activity refines them. Use `KnowledgeQuery` to find candidates before creating anything new.
- **Knowledge creation is gated.** Only create a new `knowledge/<slug>.md` when the originating source appears in the supplied "Knowledge candidate report". When the report is empty (the metrics pipeline is still being built), do not create new knowledge — fold the activity into decisions / requests / summary or update existing knowledge instead.
- Rewrite `memory/summary.md` only when needed. Aim for 15k tokens. Preserve the high-level shape (current focus, recent decisions, stable facts) while pruning stale items.
# Tidy phase
Once the consolidation phase is done, evaluate every existing memory and knowledge record against four categories:
- `outdated`: was correct, no longer matches the current implementation / policy / operation.
- `superseded`: another record is now the de-facto authoritative one; this one is mostly redundant.
- `unused`: not wrong, but rarely referenced — noise rather than signal.
- `noisy`: useful content but bad shape (overlap, sources accumulation, fractured slugs that should merge).
A single record may fall into more than one category. Choose one of `drop / merge / split / trim / rewrite`:
- Prefer `merge` and `trim` over `drop` for anything you'd flag as `unused` or `noisy` — git can reverse you, but a confidently-wrong drop hurts discovery.
- `drop` is allowed for `outdated` / `superseded` records you can justify in the diff.
- `replaced` markers (`status: replaced`) and chains pointed at by the tidy hints should be collapsed in this phase.
**Protection threshold.** When the tidy hints include explicit-invoke metrics, records with `frequency >= 1.0 invokes/Mtoken` are off-limits to drop / large compression. The metrics pipeline is not always populated; when the input lacks frequency data, behave conservatively and skip drop on long-standing records.
# Closing the turn
When both phases are done, write a short final assistant message stating:
- which staging entries you folded in (by short summary, not by ID),
- which existing records you touched (slug + operation),
- anything you intentionally left alone and why.
Then end the turn. Do not ask questions — there is no human in the loop for this run.
"#;
+139
View File
@@ -0,0 +1,139 @@
//! `_staging/*.json` を列挙して [`StagingRecord`] に展開する読み込みヘルパー。
//!
//! Phase 2 起動時のスナップショット(consumed ID list 確定)と、整理 phase
//! が終わった後の cleanup の双方で使う。`.consolidation.lock` のような
//! 占有ファイルは UUIDv7 として parse できないので自然に除外される。
//!
//! [`StagingRecord`] のスキーマは Phase 1 が書き出す側 (`crate::extract`)
//! と単一の真実源 — ここでは読み出す側だけを担当する。
use std::path::PathBuf;
use uuid::Uuid;
use crate::extract::StagingRecord;
use crate::workspace::WorkspaceLayout;
/// staging に積まれている 1 件分のエントリ。`id` は UUIDv7 で、ファイル名
/// `<id>.json` を逆引きしたもの。
#[derive(Debug, Clone)]
pub struct StagingEntry {
pub id: Uuid,
pub path: PathBuf,
pub record: StagingRecord,
/// このファイルのバイト長。閾値判定 (`consolidation_threshold_bytes`)
/// に使う。
pub bytes: u64,
}
/// `<staging_dir>/*.json` を読んで UUIDv7 順に並べた [`StagingEntry`]
/// 配列を返す。staging_dir が存在しなければ空配列。読めないファイルや
/// JSON parse 失敗は `tracing::warn!` してスキップ(壊れた個別ファイルが
/// Phase 2 全体を止めないように)。
pub fn list_staging_entries(layout: &WorkspaceLayout) -> Vec<StagingEntry> {
let dir = layout.staging_dir();
let entries = match std::fs::read_dir(&dir) {
Ok(it) => it,
Err(_) => return Vec::new(),
};
let mut out: Vec<StagingEntry> = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if !path.is_file() {
continue;
}
let stem = match path.file_stem().and_then(|s| s.to_str()) {
Some(s) => s,
None => continue,
};
let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
if ext != "json" {
continue;
}
let id = match Uuid::parse_str(stem) {
Ok(u) => u,
Err(_) => continue,
};
let bytes = match std::fs::metadata(&path) {
Ok(m) => m.len(),
Err(_) => 0,
};
let raw = match std::fs::read_to_string(&path) {
Ok(s) => s,
Err(e) => {
tracing::warn!(path = %path.display(), error = %e, "failed to read staging entry");
continue;
}
};
let record = match serde_json::from_str::<StagingRecord>(&raw) {
Ok(r) => r,
Err(e) => {
tracing::warn!(path = %path.display(), error = %e, "failed to parse staging entry");
continue;
}
};
out.push(StagingEntry {
id,
path,
record,
bytes,
});
}
out.sort_by_key(|e| e.id);
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::extract::{ExtractedPayload, write_staging};
use crate::schema::SourceRef;
fn empty_payload() -> ExtractedPayload {
ExtractedPayload::default()
}
fn source(session_id: &str, range: [u64; 2]) -> SourceRef {
SourceRef {
session_id: session_id.into(),
range,
}
}
#[test]
fn lists_in_uuidv7_order() {
let tmp = tempfile::TempDir::new().unwrap();
let layout = WorkspaceLayout::new(tmp.path().to_path_buf());
let (id1, _) = write_staging(&layout, source("s", [0, 1]), empty_payload()).unwrap();
let (id2, _) = write_staging(&layout, source("s", [2, 3]), empty_payload()).unwrap();
let (id3, _) = write_staging(&layout, source("s", [4, 5]), empty_payload()).unwrap();
let entries = list_staging_entries(&layout);
let ids: Vec<Uuid> = entries.iter().map(|e| e.id).collect();
assert_eq!(ids, vec![id1, id2, id3]);
}
#[test]
fn skips_lock_file_and_garbage() {
let tmp = tempfile::TempDir::new().unwrap();
let layout = WorkspaceLayout::new(tmp.path().to_path_buf());
let (_id, _) = write_staging(&layout, source("s", [0, 1]), empty_payload()).unwrap();
// Drop a non-UUID json file and a bare lock file alongside.
std::fs::write(layout.staging_dir().join("not-a-uuid.json"), "{}").unwrap();
std::fs::write(layout.staging_dir().join(".consolidation.lock"), "{}").unwrap();
let entries = list_staging_entries(&layout);
assert_eq!(entries.len(), 1);
}
#[test]
fn missing_dir_returns_empty() {
let tmp = tempfile::TempDir::new().unwrap();
let layout = WorkspaceLayout::new(tmp.path().to_path_buf());
// No staging dir at all.
assert!(list_staging_entries(&layout).is_empty());
}
}
+362
View File
@@ -0,0 +1,362 @@
//! 整理 phase が prompt 入力に乗せる「整理材料」スキャナ。
//!
//! `docs/plan/memory.md` §整理(GC 相当)の扱い と
//! `tickets/memory-phase2-consolidation.md` の整理材料リストに従い、
//! メトリクス未完の現状で機械的に拾えるヒントだけを集める:
//!
//! - `replaced` chain: `status: replaced` の Decision とその `replaced_by`
//! - sources 過多: `sources` / `last_sources` 配列が閾値超過の record
//! - 類似 slug 乱立: 同 kind の slug が Levenshtein 2 以内のクラスター
//!
//! 使用頻度メトリクスベースの保護閾値情報は `tickets/memory-usage-metrics.md`
//! の成果物が出るまで空で渡る。
use std::collections::{BTreeMap, BTreeSet};
use crate::schema::{
DecisionFrontmatter, KnowledgeFrontmatter, RequestFrontmatter, split_frontmatter,
};
use crate::slug::Slug;
use crate::workspace::{RecordKind, WorkspaceLayout};
/// `sources` overflow を flag する閾値。`linter::warnings::SOURCES_OVERFLOW_THRESHOLD`
/// と同値(10)を踏襲する。Linter Warn で sources 過多が検出されるラインと
/// 整理 phase で勧告するラインを揃える狙い。
pub const SOURCES_OVERFLOW_THRESHOLD: usize = 10;
/// 類似 slug クラスタリングの距離。`linter::warnings::SIMILAR_SLUG_DISTANCE`
/// と同値。
pub const SIMILAR_SLUG_DISTANCE: usize = 2;
/// 整理 phase 用の機械集計ヒント。空フィールドは「対象なし」を意味する。
#[derive(Debug, Default, Clone)]
pub struct TidyHints {
/// `status: replaced` で残っている Decision の slug → `replaced_by` map。
/// `replaced_by` が None でも置き換え滞留として列挙する。
pub replaced_decisions: BTreeMap<String, Option<String>>,
/// kind / slug / sources count の三つ組で sources 累積ラインを表す。
pub sources_overflow: Vec<SourcesOverflow>,
/// 同 kind 内で Levenshtein 距離 `<= SIMILAR_SLUG_DISTANCE` のクラスター。
/// クラスター内の slug は sorted。
pub similar_slug_clusters: Vec<SimilarSlugCluster>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourcesOverflow {
pub kind: RecordKind,
pub slug: String,
pub count: usize,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SimilarSlugCluster {
pub kind: RecordKind,
pub slugs: Vec<String>,
}
impl TidyHints {
pub fn is_empty(&self) -> bool {
self.replaced_decisions.is_empty()
&& self.sources_overflow.is_empty()
&& self.similar_slug_clusters.is_empty()
}
}
/// workspace を一通りスキャンして [`TidyHints`] を組み立てる。読めない /
/// parse できない record は黙ってスキップ(Linter は write 経路で守って
/// いるので、ここで顕在化してもどうしようもない)。
pub fn collect_tidy_hints(layout: &WorkspaceLayout) -> TidyHints {
let mut hints = TidyHints::default();
let decisions = read_kind_records(layout, RecordKind::Decision);
let requests = read_kind_records(layout, RecordKind::Request);
let knowledge = read_kind_records(layout, RecordKind::Knowledge);
for (slug, content) in &decisions {
let fm = parse_yaml::<DecisionFrontmatter>(content);
if let Some(fm) = fm.as_ref() {
if matches!(
fm.status,
crate::schema::DecisionStatus::Replaced
) {
hints
.replaced_decisions
.insert(slug.clone(), fm.replaced_by.as_ref().map(|s| s.to_string()));
}
if fm.sources.len() > SOURCES_OVERFLOW_THRESHOLD {
hints.sources_overflow.push(SourcesOverflow {
kind: RecordKind::Decision,
slug: slug.clone(),
count: fm.sources.len(),
});
}
}
}
for (slug, content) in &requests {
if let Some(fm) = parse_yaml::<RequestFrontmatter>(content) {
if fm.sources.len() > SOURCES_OVERFLOW_THRESHOLD {
hints.sources_overflow.push(SourcesOverflow {
kind: RecordKind::Request,
slug: slug.clone(),
count: fm.sources.len(),
});
}
}
}
for (slug, content) in &knowledge {
if let Some(fm) = parse_yaml::<KnowledgeFrontmatter>(content) {
if fm.last_sources.len() > SOURCES_OVERFLOW_THRESHOLD {
hints.sources_overflow.push(SourcesOverflow {
kind: RecordKind::Knowledge,
slug: slug.clone(),
count: fm.last_sources.len(),
});
}
}
}
hints
.sources_overflow
.sort_by(|a, b| (a.kind.as_str(), a.slug.as_str()).cmp(&(b.kind.as_str(), b.slug.as_str())));
let decision_slugs: Vec<&str> = decisions.keys().map(|s| s.as_str()).collect();
let request_slugs: Vec<&str> = requests.keys().map(|s| s.as_str()).collect();
let knowledge_slugs: Vec<&str> = knowledge.keys().map(|s| s.as_str()).collect();
if let Some(c) = cluster_similar(&decision_slugs, RecordKind::Decision) {
hints.similar_slug_clusters.extend(c);
}
if let Some(c) = cluster_similar(&request_slugs, RecordKind::Request) {
hints.similar_slug_clusters.extend(c);
}
if let Some(c) = cluster_similar(&knowledge_slugs, RecordKind::Knowledge) {
hints.similar_slug_clusters.extend(c);
}
hints
.similar_slug_clusters
.sort_by(|a, b| (a.kind.as_str(), &a.slugs).cmp(&(b.kind.as_str(), &b.slugs)));
hints
}
/// `<root>/.insomnia/memory/<kind>/*.md` (Knowledge は
/// `<root>/.insomnia/knowledge/*.md`) を slug ごとに `(slug, full content)`
/// 化して返す。
fn read_kind_records(
layout: &WorkspaceLayout,
kind: RecordKind,
) -> BTreeMap<String, String> {
let dir = match kind {
RecordKind::Decision => layout.decisions_dir(),
RecordKind::Request => layout.requests_dir(),
RecordKind::Knowledge => layout.knowledge_dir(),
RecordKind::Summary | RecordKind::Workflow => return BTreeMap::new(),
};
let mut out: BTreeMap<String, String> = BTreeMap::new();
let entries = match std::fs::read_dir(&dir) {
Ok(it) => it,
Err(_) => return out,
};
for entry in entries.flatten() {
let path = entry.path();
if !path.is_file() {
continue;
}
let stem = match path.file_stem().and_then(|s| s.to_str()) {
Some(s) => s,
None => continue,
};
if path.extension().and_then(|s| s.to_str()) != Some("md") {
continue;
}
if Slug::parse(stem).is_err() {
continue;
}
let content = match std::fs::read_to_string(&path) {
Ok(s) => s,
Err(_) => continue,
};
out.insert(stem.to_string(), content);
}
out
}
fn parse_yaml<F: serde::de::DeserializeOwned>(content: &str) -> Option<F> {
let (yaml, _body) = split_frontmatter(content).ok()?;
serde_yaml::from_str::<F>(yaml).ok()
}
/// Connected-component clustering over the `levenshtein <= SIMILAR_SLUG_DISTANCE`
/// graph among same-kind slugs. Returns each cluster of size >= 2 (singleton
/// clusters are not interesting for the integration phase). Returns `None`
/// when there are no clusters at all.
fn cluster_similar(slugs: &[&str], kind: RecordKind) -> Option<Vec<SimilarSlugCluster>> {
if slugs.len() < 2 {
return None;
}
let n = slugs.len();
let mut parent: Vec<usize> = (0..n).collect();
fn find(parent: &mut [usize], i: usize) -> usize {
if parent[i] == i {
i
} else {
let root = find(parent, parent[i]);
parent[i] = root;
root
}
}
fn union(parent: &mut [usize], a: usize, b: usize) {
let ra = find(parent, a);
let rb = find(parent, b);
if ra != rb {
parent[ra] = rb;
}
}
for i in 0..n {
for j in (i + 1)..n {
if levenshtein(slugs[i], slugs[j]) <= SIMILAR_SLUG_DISTANCE {
union(&mut parent, i, j);
}
}
}
let mut groups: BTreeMap<usize, Vec<String>> = BTreeMap::new();
for i in 0..n {
let root = find(&mut parent, i);
groups.entry(root).or_default().push(slugs[i].to_string());
}
let mut out: Vec<SimilarSlugCluster> = Vec::new();
let mut seen_canonical: BTreeSet<Vec<String>> = BTreeSet::new();
for (_, mut group) in groups {
if group.len() < 2 {
continue;
}
group.sort();
if seen_canonical.insert(group.clone()) {
out.push(SimilarSlugCluster { kind, slugs: group });
}
}
if out.is_empty() { None } else { Some(out) }
}
/// Iterative two-row Levenshtein distance over chars (matches the Linter's
/// implementation; kept private to avoid widening that crate-internal API).
fn levenshtein(a: &str, b: &str) -> usize {
let a: Vec<char> = a.chars().collect();
let b: Vec<char> = b.chars().collect();
if a.is_empty() {
return b.len();
}
if b.is_empty() {
return a.len();
}
let mut prev: Vec<usize> = (0..=b.len()).collect();
let mut curr: Vec<usize> = vec![0; b.len() + 1];
for (i, ca) in a.iter().enumerate() {
curr[0] = i + 1;
for (j, cb) in b.iter().enumerate() {
let cost = if ca == cb { 0 } else { 1 };
curr[j + 1] = (curr[j] + 1).min(prev[j + 1] + 1).min(prev[j] + cost);
}
std::mem::swap(&mut prev, &mut curr);
}
prev[b.len()]
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::Utc;
use std::path::Path;
fn now() -> String {
Utc::now().to_rfc3339()
}
fn write(p: &Path, content: &str) {
if let Some(parent) = p.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(p, content).unwrap();
}
fn workspace() -> (tempfile::TempDir, WorkspaceLayout) {
let dir = tempfile::TempDir::new().unwrap();
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
(dir, layout)
}
#[test]
fn collects_replaced_chain() {
let (dir, layout) = workspace();
write(
&dir.path().join(".insomnia/memory/decisions/replaced.md"),
&format!(
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: replaced\nreplaced_by: winner\n---\n",
n = now()
),
);
write(
&dir.path().join(".insomnia/memory/decisions/winner.md"),
&format!(
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\n",
n = now()
),
);
let hints = collect_tidy_hints(&layout);
assert_eq!(
hints.replaced_decisions.get("replaced").cloned(),
Some(Some("winner".into()))
);
assert!(!hints.replaced_decisions.contains_key("winner"));
}
#[test]
fn flags_sources_overflow() {
let (dir, layout) = workspace();
let many_sources: String = (0..15)
.map(|i| format!(" - session_id: s{i}\n range: [{i}, {i}]\n"))
.collect();
write(
&dir.path().join(".insomnia/memory/decisions/big.md"),
&format!(
"---\ncreated_at: {n}\nupdated_at: {n}\nstatus: open\nsources:\n{m}---\n",
n = now(),
m = many_sources
),
);
let hints = collect_tidy_hints(&layout);
assert_eq!(hints.sources_overflow.len(), 1);
assert_eq!(hints.sources_overflow[0].slug, "big");
assert_eq!(hints.sources_overflow[0].kind, RecordKind::Decision);
assert_eq!(hints.sources_overflow[0].count, 15);
}
#[test]
fn clusters_similar_slugs() {
let (dir, layout) = workspace();
for slug in ["db-pool", "db-pol", "db-pools", "alpha"] {
write(
&dir.path()
.join(format!(".insomnia/memory/decisions/{slug}.md")),
&format!(
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\n",
n = now()
),
);
}
let hints = collect_tidy_hints(&layout);
assert_eq!(hints.similar_slug_clusters.len(), 1);
assert_eq!(
hints.similar_slug_clusters[0].slugs,
vec![
"db-pol".to_string(),
"db-pool".to_string(),
"db-pools".to_string(),
]
);
}
#[test]
fn empty_workspace_yields_empty_hints() {
let (_dir, layout) = workspace();
let hints = collect_tidy_hints(&layout);
assert!(hints.is_empty());
}
}
+1
View File
@@ -6,6 +6,7 @@
//! crate) must not touch these directories — Pod is responsible for
//! denying them at the Scope level when memory is enabled.
pub mod consolidate;
pub mod error;
pub mod extract;
pub mod linter;