メモリPhase2の実装
This commit is contained in:
@@ -7,6 +7,7 @@ license.workspace = true
|
||||
[dependencies]
|
||||
async-trait = "0.1.89"
|
||||
chrono = { version = "0.4.44", features = ["serde"] }
|
||||
libc = "0.2.186"
|
||||
llm-worker = { version = "0.2.1", path = "../llm-worker" }
|
||||
manifest = { version = "0.1.0", path = "../manifest" }
|
||||
schemars = "1.2.1"
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
//! Phase 2 sub-Worker への最初のユーザー入力を組み立てる。
|
||||
//!
|
||||
//! Phase 1 (`extract::build_extract_input`) と同じ方針で、固定 schema の
|
||||
//! markdown セクション列にしてサブWorker に渡す。`docs/plan/memory.md`
|
||||
//! §Phase 2 入力 / §整理材料 の項目に従い:
|
||||
//!
|
||||
//! 1. consumed staging エントリ全文(`source` 込み)
|
||||
//! 2. 既存 `memory/*` 全文(summary / decisions / requests)
|
||||
//! 3. Knowledge 化候補レポート(メトリクス未完なら空)
|
||||
//! 4. 整理材料(Linter Warn ベース、メトリクス未完なら明示 invoke 頻度なし)
|
||||
//!
|
||||
//! 既存 `knowledge/*` 本文は埋めず、agent に `KnowledgeQuery` 経由で引かせる
|
||||
//! 設計(`docs/plan/memory.md` §retrieval 経路 / §Phase 2 の Knowledge アクセス)。
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::consolidate::staging::StagingEntry;
|
||||
use crate::consolidate::tidy::TidyHints;
|
||||
use crate::workspace::{RecordKind, WorkspaceLayout};
|
||||
|
||||
/// Knowledge 化候補レポート。`tickets/memory-usage-metrics.md` の成果物が
|
||||
/// 出るまでは空で渡す前提(`docs/plan/memory.md` §Knowledge 化候補レポート)。
|
||||
/// 空入力時、統合 phase は新規 Knowledge を作らず decisions / requests /
|
||||
/// summary / 既存 Knowledge update に留まる。
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct KnowledgeCandidateReport {
|
||||
/// 候補に上がった `(kind, slug, frequency_per_mtoken)` の三つ組。
|
||||
/// 空配列を渡すと「候補なし」を意味する。
|
||||
pub entries: Vec<KnowledgeCandidateEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct KnowledgeCandidateEntry {
|
||||
pub source_kind: &'static str,
|
||||
pub source_slug: String,
|
||||
pub frequency_per_mtoken: f64,
|
||||
}
|
||||
|
||||
impl KnowledgeCandidateReport {
|
||||
pub fn empty() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Phase 2 sub-Worker の最初の user 入力。
|
||||
pub fn build_consolidate_input(
|
||||
layout: &WorkspaceLayout,
|
||||
staging: &[StagingEntry],
|
||||
tidy: &TidyHints,
|
||||
candidates: &KnowledgeCandidateReport,
|
||||
) -> String {
|
||||
let mut out = String::new();
|
||||
out.push_str(
|
||||
"Phase 2 consolidation input. Run the consolidation phase first \
|
||||
(fold the staging activity logs into memory and knowledge), then the \
|
||||
tidy phase (clean up existing records). Use the memory tools for \
|
||||
every write — direct file writes are denied by the pod scope.\n\n",
|
||||
);
|
||||
|
||||
out.push_str("## Staging entries (consumed by this run)\n\n");
|
||||
out.push_str(&render_staging_records(staging));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Existing memory records (full content)\n\n");
|
||||
out.push_str(&render_existing_memory_records(layout));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Knowledge candidate report\n\n");
|
||||
out.push_str(&render_candidate_report(candidates));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Tidy hints\n\n");
|
||||
out.push_str(&render_tidy_hints(tidy));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str(
|
||||
"When done, end the turn with a short final assistant message describing \
|
||||
what changed.",
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
/// Staging エントリ群を「`### <id>` ヘッダ + 整形 JSON ブロック」で並べる。
|
||||
/// 空配列なら「(none)」と書く。
|
||||
pub fn render_staging_records(entries: &[StagingEntry]) -> String {
|
||||
if entries.is_empty() {
|
||||
return "(none)\n".to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for entry in entries {
|
||||
let _ = writeln!(&mut out, "### {}", entry.id);
|
||||
let json = serde_json::to_string_pretty(&entry.record).unwrap_or_else(|_| "{}".into());
|
||||
out.push_str("```json\n");
|
||||
out.push_str(&json);
|
||||
out.push_str("\n```\n\n");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `<workspace>/.insomnia/memory/{summary.md,decisions/*,requests/*}` を
|
||||
/// 「`### <kind>:<slug>` ヘッダ + raw markdown ブロック」で全文渡す。
|
||||
pub fn render_existing_memory_records(layout: &WorkspaceLayout) -> String {
|
||||
let mut out = String::new();
|
||||
|
||||
let summary = layout.summary_path();
|
||||
if let Ok(content) = std::fs::read_to_string(&summary) {
|
||||
out.push_str("### summary\n");
|
||||
out.push_str("```markdown\n");
|
||||
out.push_str(content.trim_end_matches('\n'));
|
||||
out.push_str("\n```\n\n");
|
||||
}
|
||||
|
||||
push_kind_records(&mut out, layout, RecordKind::Decision);
|
||||
push_kind_records(&mut out, layout, RecordKind::Request);
|
||||
|
||||
if out.is_empty() {
|
||||
return "(none)\n".to_string();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn push_kind_records(out: &mut String, layout: &WorkspaceLayout, kind: RecordKind) {
|
||||
let dir = match kind {
|
||||
RecordKind::Decision => layout.decisions_dir(),
|
||||
RecordKind::Request => layout.requests_dir(),
|
||||
RecordKind::Knowledge | RecordKind::Summary | RecordKind::Workflow => return,
|
||||
};
|
||||
let entries = match std::fs::read_dir(&dir) {
|
||||
Ok(it) => it,
|
||||
Err(_) => return,
|
||||
};
|
||||
let mut paths: Vec<(String, std::path::PathBuf)> = Vec::new();
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
let stem = match path.file_stem().and_then(|s| s.to_str()) {
|
||||
Some(s) => s,
|
||||
None => continue,
|
||||
};
|
||||
if path.extension().and_then(|s| s.to_str()) != Some("md") {
|
||||
continue;
|
||||
}
|
||||
paths.push((stem.to_string(), path));
|
||||
}
|
||||
paths.sort();
|
||||
for (slug, path) in paths {
|
||||
let Ok(content) = std::fs::read_to_string(&path) else {
|
||||
continue;
|
||||
};
|
||||
let _ = writeln!(out, "### {}:{}", kind.as_str(), slug);
|
||||
out.push_str("```markdown\n");
|
||||
out.push_str(content.trim_end_matches('\n'));
|
||||
out.push_str("\n```\n\n");
|
||||
}
|
||||
}
|
||||
|
||||
fn render_candidate_report(report: &KnowledgeCandidateReport) -> String {
|
||||
if report.is_empty() {
|
||||
return "(empty — usage metrics pipeline not populated. \
|
||||
Do not create new Knowledge records this run.)\n"
|
||||
.to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for c in &report.entries {
|
||||
let _ = writeln!(
|
||||
&mut out,
|
||||
"- {} `{}` — frequency {:.3} invokes/Mtoken",
|
||||
c.source_kind, c.source_slug, c.frequency_per_mtoken
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Tidy hints の Markdown 描画。空ヒントなら "(none)" 1 行。
|
||||
pub fn render_tidy_hints(tidy: &TidyHints) -> String {
|
||||
if tidy.is_empty() {
|
||||
return "(none)\n".to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
|
||||
if !tidy.replaced_decisions.is_empty() {
|
||||
out.push_str("**Replaced decisions still on disk** — collapse if the chain has settled:\n");
|
||||
for (slug, replaced_by) in &tidy.replaced_decisions {
|
||||
match replaced_by {
|
||||
Some(target) => {
|
||||
let _ = writeln!(&mut out, "- `{slug}` → `{target}`");
|
||||
}
|
||||
None => {
|
||||
let _ = writeln!(&mut out, "- `{slug}` (no `replaced_by` set)");
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
if !tidy.sources_overflow.is_empty() {
|
||||
out.push_str(
|
||||
"**Sources overflow** — consider trimming to the most recent entries (git log keeps the rest):\n",
|
||||
);
|
||||
for s in &tidy.sources_overflow {
|
||||
let _ = writeln!(&mut out, "- {} `{}` ({} sources)", s.kind.as_str(), s.slug, s.count);
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
if !tidy.similar_slug_clusters.is_empty() {
|
||||
out.push_str("**Similar slug clusters** — evaluate for merge / rename:\n");
|
||||
for c in &tidy.similar_slug_clusters {
|
||||
let joined = c
|
||||
.slugs
|
||||
.iter()
|
||||
.map(|s| format!("`{s}`"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let _ = writeln!(&mut out, "- {}: {}", c.kind.as_str(), joined);
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
out.push_str(
|
||||
"Explicit-invoke metrics (protection threshold) are not yet wired up; \
|
||||
skip drop on long-standing records when uncertain.\n",
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::consolidate::tidy::{SimilarSlugCluster, SourcesOverflow};
|
||||
use crate::extract::{ExtractedPayload, write_staging};
|
||||
use crate::schema::SourceRef;
|
||||
use chrono::Utc;
|
||||
use std::path::Path;
|
||||
|
||||
fn now() -> String {
|
||||
Utc::now().to_rfc3339()
|
||||
}
|
||||
|
||||
fn write(p: &Path, content: &str) {
|
||||
if let Some(parent) = p.parent() {
|
||||
std::fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
std::fs::write(p, content).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_includes_all_sections_when_populated() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
|
||||
write(
|
||||
&dir.path().join(".insomnia/memory/summary.md"),
|
||||
&format!("---\nupdated_at: {n}\n---\nstate of the world\n", n = now()),
|
||||
);
|
||||
write(
|
||||
&dir.path().join(".insomnia/memory/decisions/dec.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\nbody\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
let (_id, _) = write_staging(
|
||||
&layout,
|
||||
SourceRef {
|
||||
session_id: "s".into(),
|
||||
range: [0, 1],
|
||||
},
|
||||
ExtractedPayload::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let staging = crate::consolidate::staging::list_staging_entries(&layout);
|
||||
let tidy = TidyHints {
|
||||
replaced_decisions: [(
|
||||
"old".to_string(),
|
||||
Some("new".to_string()),
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
sources_overflow: vec![SourcesOverflow {
|
||||
kind: RecordKind::Decision,
|
||||
slug: "dec".into(),
|
||||
count: 12,
|
||||
}],
|
||||
similar_slug_clusters: vec![SimilarSlugCluster {
|
||||
kind: RecordKind::Decision,
|
||||
slugs: vec!["a".into(), "ab".into()],
|
||||
}],
|
||||
};
|
||||
let report = KnowledgeCandidateReport::empty();
|
||||
|
||||
let out = build_consolidate_input(&layout, &staging, &tidy, &report);
|
||||
assert!(out.contains("Staging entries"));
|
||||
assert!(out.contains("Existing memory records"));
|
||||
assert!(out.contains("Knowledge candidate report"));
|
||||
assert!(out.contains("Tidy hints"));
|
||||
assert!(out.contains("state of the world"));
|
||||
assert!(out.contains("decision:dec"));
|
||||
assert!(out.contains("Replaced decisions"));
|
||||
assert!(out.contains("Sources overflow"));
|
||||
assert!(out.contains("Similar slug clusters"));
|
||||
assert!(out.contains("usage metrics pipeline not populated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_inputs_render_placeholders() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
let out = build_consolidate_input(
|
||||
&layout,
|
||||
&[],
|
||||
&TidyHints::default(),
|
||||
&KnowledgeCandidateReport::empty(),
|
||||
);
|
||||
// Both staging and tidy show "(none)"; existing memory records too.
|
||||
assert!(out.contains("Staging entries"));
|
||||
assert!(out.contains("(none)"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
//! `_staging/.consolidation.lock` による Phase 2 占有ファイル。
|
||||
//!
|
||||
//! `docs/plan/memory.md` §並走防止 に従い:
|
||||
//!
|
||||
//! - ファイルが存在し、記録された Pod が動作している間、その Pod が排他占有
|
||||
//! - クラッシュで残った stale lock は、所有者 PID が死んでいれば次回 spawn
|
||||
//! 時に上書き取得できる
|
||||
//! - cleanup は consumed ID の staging エントリのみ削除し、実行中に Phase 1
|
||||
//! が追加した分は残す
|
||||
//!
|
||||
//! 占有判定は Linux/macOS の `kill(pid, 0)` 経由で行う(`ESRCH` で死亡判定)。
|
||||
//! Windows は対象外: INSOMNIA は POSIX 環境を前提にしている。
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::workspace::WorkspaceLayout;
|
||||
|
||||
const LOCK_FILE: &str = ".consolidation.lock";
|
||||
|
||||
/// 占有ファイルの中身。`pid` で stale 判定し、`pod_name` / `started_at` /
|
||||
/// `consumed_ids` は診断とクラッシュ復旧時の参照に使う。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LockRecord {
|
||||
pub pid: u32,
|
||||
pub pod_name: String,
|
||||
pub started_at: DateTime<Utc>,
|
||||
/// この Phase 2 run が起動時スナップショットで確定した consumed staging
|
||||
/// entry の UUIDv7 列。完了時はこの列のみ削除し、追加分は残す。
|
||||
pub consumed_ids: Vec<Uuid>,
|
||||
}
|
||||
|
||||
/// 占有取得 / 解放のエラー。
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LockError {
|
||||
/// 占有ファイルが既にあり、所有者 PID が生きているのでスキップ。
|
||||
#[error("Phase 2 lock held by live pid {pid} (pod {pod_name:?})")]
|
||||
InUse { pid: u32, pod_name: String },
|
||||
#[error("io error at {}: {source}", .path.display())]
|
||||
Io {
|
||||
path: PathBuf,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("failed to (de)serialize lock record: {0}")]
|
||||
Serde(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
impl LockError {
|
||||
fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
|
||||
Self::Io {
|
||||
path: path.into(),
|
||||
source,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Phase 2 が走っている間 RAII で持つ占有ハンドル。`Drop` では何もしない —
|
||||
/// 完了時の cleanup は consumed ID 列削除と一緒に行う必要があるため、明示
|
||||
/// 解放 [`StagingLock::release_with_cleanup`] を使う。明示解放しないまま
|
||||
/// drop された場合は占有ファイルがそのまま残り、次回 spawn 時に PID が
|
||||
/// 死んでいれば stale 上書きされる。
|
||||
#[derive(Debug)]
|
||||
pub struct StagingLock {
|
||||
path: PathBuf,
|
||||
record: LockRecord,
|
||||
}
|
||||
|
||||
impl StagingLock {
|
||||
pub fn record(&self) -> &LockRecord {
|
||||
&self.record
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// 占有取得を試みる。既に live な lock があれば
|
||||
/// [`LockError::InUse`]、stale 判定なら上書き取得する。
|
||||
/// staging dir が無ければ作成する。
|
||||
pub fn acquire(
|
||||
layout: &WorkspaceLayout,
|
||||
pid: u32,
|
||||
pod_name: impl Into<String>,
|
||||
consumed_ids: Vec<Uuid>,
|
||||
) -> Result<Self, LockError> {
|
||||
let staging_dir = layout.staging_dir();
|
||||
fs::create_dir_all(&staging_dir).map_err(|e| LockError::io(&staging_dir, e))?;
|
||||
let path = staging_dir.join(LOCK_FILE);
|
||||
|
||||
if path.exists() {
|
||||
let raw = fs::read_to_string(&path).map_err(|e| LockError::io(&path, e))?;
|
||||
// 壊れた lock は stale とみなして上書き許可。
|
||||
if let Ok(existing) = serde_json::from_str::<LockRecord>(&raw) {
|
||||
if pid_is_alive(existing.pid) {
|
||||
return Err(LockError::InUse {
|
||||
pid: existing.pid,
|
||||
pod_name: existing.pod_name,
|
||||
});
|
||||
}
|
||||
tracing::warn!(
|
||||
stale_pid = existing.pid,
|
||||
stale_pod = %existing.pod_name,
|
||||
"Phase 2 stale lock detected, taking over"
|
||||
);
|
||||
} else {
|
||||
tracing::warn!(path = %path.display(), "Phase 2 lock unparseable, treating as stale");
|
||||
}
|
||||
}
|
||||
|
||||
let record = LockRecord {
|
||||
pid,
|
||||
pod_name: pod_name.into(),
|
||||
started_at: Utc::now(),
|
||||
consumed_ids,
|
||||
};
|
||||
let json = serde_json::to_string_pretty(&record)?;
|
||||
fs::write(&path, json).map_err(|e| LockError::io(&path, e))?;
|
||||
Ok(Self { path, record })
|
||||
}
|
||||
|
||||
/// 占有を解放しつつ consumed ID 列の staging エントリを削除する。
|
||||
/// 削除対象が見当たらない場合は黙ってスキップ(既に外部で消えていた等)。
|
||||
/// 占有ファイル自体の削除も best-effort: 失敗時は warn を出すだけで
|
||||
/// エラーは伝播しない(次回 spawn 時に stale 判定で上書きされる)。
|
||||
pub fn release_with_cleanup(self, layout: &WorkspaceLayout) {
|
||||
let staging_dir = layout.staging_dir();
|
||||
for id in &self.record.consumed_ids {
|
||||
let target = staging_dir.join(format!("{id}.json"));
|
||||
match fs::remove_file(&target) {
|
||||
Ok(_) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
path = %target.display(),
|
||||
error = %e,
|
||||
"failed to clean up consumed staging entry"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.unlink_lock_only();
|
||||
}
|
||||
|
||||
/// 占有ファイルだけ削除し、staging エントリには触らない。Phase 2
|
||||
/// sub-Worker が途中で失敗した場合に使う: 入力 staging を残したまま
|
||||
/// 次回再評価で再処理させる(`docs/plan/memory.md` §並走防止 の
|
||||
/// 「重複作成は同一 slug update に自然収束」運用)。
|
||||
pub fn release_only(self) {
|
||||
self.unlink_lock_only();
|
||||
}
|
||||
|
||||
fn unlink_lock_only(&self) {
|
||||
if let Err(e) = fs::remove_file(&self.path) {
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
tracing::warn!(
|
||||
path = %self.path.display(),
|
||||
error = %e,
|
||||
"failed to remove Phase 2 lock"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn pid_is_alive(pid: u32) -> bool {
|
||||
// `kill(0, 0)` and `kill(-1, 0)` are POSIX-special (process group / all
|
||||
// signalable processes) and would yield false positives. Reject pids
|
||||
// that don't fit a positive `pid_t` so a corrupted lock file with a
|
||||
// u32::MAX-ish value is treated as stale instead of magically alive.
|
||||
if pid == 0 || pid > i32::MAX as u32 {
|
||||
return false;
|
||||
}
|
||||
// SAFETY: `kill` with sig 0 only probes whether the target pid exists
|
||||
// and the caller has permission to signal it. No signal is delivered.
|
||||
let rc = unsafe { libc::kill(pid as i32, 0) };
|
||||
if rc == 0 {
|
||||
return true;
|
||||
}
|
||||
// EPERM means the process exists but we can't signal it — still alive
|
||||
// for our purposes. ESRCH means it's gone.
|
||||
let errno = std::io::Error::last_os_error()
|
||||
.raw_os_error()
|
||||
.unwrap_or(libc::EINVAL);
|
||||
errno != libc::ESRCH
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn pid_is_alive(_pid: u32) -> bool {
|
||||
// Unsupported platforms: assume the lock is live so we never overwrite
|
||||
// someone else's claim. Phase 2 will skip and try again next post-run.
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::extract::{ExtractedPayload, write_staging};
|
||||
use crate::schema::SourceRef;
|
||||
|
||||
fn make_layout() -> (tempfile::TempDir, WorkspaceLayout) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
std::fs::create_dir_all(layout.staging_dir()).unwrap();
|
||||
(dir, layout)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_writes_lock_file() {
|
||||
let (_dir, layout) = make_layout();
|
||||
let lock = StagingLock::acquire(&layout, std::process::id(), "pod", Vec::new()).unwrap();
|
||||
let path = layout.staging_dir().join(LOCK_FILE);
|
||||
assert!(path.exists());
|
||||
assert_eq!(lock.record().pid, std::process::id());
|
||||
assert_eq!(lock.record().pod_name, "pod");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_rejects_when_live_pid_holds_lock() {
|
||||
let (_dir, layout) = make_layout();
|
||||
// Use this test process's pid — it's definitely alive.
|
||||
let _first =
|
||||
StagingLock::acquire(&layout, std::process::id(), "pod-a", Vec::new()).unwrap();
|
||||
let err = StagingLock::acquire(&layout, std::process::id(), "pod-b", Vec::new())
|
||||
.expect_err("expected InUse");
|
||||
assert!(matches!(err, LockError::InUse { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_overwrites_stale_lock() {
|
||||
let (_dir, layout) = make_layout();
|
||||
// pid 1 is init on linux but for arbitrarily-large pids we'd need
|
||||
// `kill(pid, 0)` to return ESRCH. Use u32::MAX which is guaranteed
|
||||
// dead on every platform we target.
|
||||
let stale = LockRecord {
|
||||
pid: u32::MAX,
|
||||
pod_name: "ghost".into(),
|
||||
started_at: Utc::now(),
|
||||
consumed_ids: Vec::new(),
|
||||
};
|
||||
std::fs::write(
|
||||
layout.staging_dir().join(LOCK_FILE),
|
||||
serde_json::to_string_pretty(&stale).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let lock = StagingLock::acquire(&layout, std::process::id(), "pod", Vec::new())
|
||||
.expect("stale lock must be overwritable");
|
||||
assert_eq!(lock.record().pid, std::process::id());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_drops_consumed_entries_and_unlinks_lock() {
|
||||
let (_dir, layout) = make_layout();
|
||||
let (id_a, _) = write_staging(
|
||||
&layout,
|
||||
SourceRef {
|
||||
session_id: "s".into(),
|
||||
range: [0, 0],
|
||||
},
|
||||
ExtractedPayload::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let (id_b, _) = write_staging(
|
||||
&layout,
|
||||
SourceRef {
|
||||
session_id: "s".into(),
|
||||
range: [1, 1],
|
||||
},
|
||||
ExtractedPayload::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let lock = StagingLock::acquire(&layout, std::process::id(), "pod", vec![id_a]).unwrap();
|
||||
let lock_path = lock.path().to_path_buf();
|
||||
lock.release_with_cleanup(&layout);
|
||||
|
||||
assert!(!lock_path.exists(), "lock file must be removed");
|
||||
assert!(
|
||||
!layout.staging_dir().join(format!("{id_a}.json")).exists(),
|
||||
"consumed entry must be deleted"
|
||||
);
|
||||
assert!(
|
||||
layout.staging_dir().join(format!("{id_b}.json")).exists(),
|
||||
"non-consumed entry must remain"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_is_resilient_to_missing_consumed_entries() {
|
||||
let (_dir, layout) = make_layout();
|
||||
let phantom = uuid::Uuid::now_v7();
|
||||
let lock =
|
||||
StagingLock::acquire(&layout, std::process::id(), "pod", vec![phantom]).unwrap();
|
||||
let lock_path = lock.path().to_path_buf();
|
||||
// No file at <staging>/<phantom>.json — release must not panic.
|
||||
lock.release_with_cleanup(&layout);
|
||||
assert!(!lock_path.exists());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Phase 2: 統合 + 整理。
|
||||
//!
|
||||
//! Phase 1 が staging に残した活動ログを `memory/*` / `knowledge/*` に
|
||||
//! 統合し、続けて既存 record を `outdated | superseded | unused | noisy`
|
||||
//! の観点で整理する disposable Worker を、Pod 側が組み立てるための
|
||||
//! ヘルパー群を提供する。Pod は次の手順で sub-Worker を構築する:
|
||||
//!
|
||||
//! - [`CONSOLIDATION_SYSTEM_PROMPT`] を sub-Worker の system prompt に
|
||||
//! - [`build_consolidate_input`] を sub-Worker の最初の user 入力に
|
||||
//! - memory 専用 Tool (read / write / edit) と Knowledge / memory 検索ツールを登録
|
||||
//! - [`StagingLock::acquire`] で並走防止 + consumed ID 確定
|
||||
//! - sub-Worker run 完了後、[`StagingLock::release_with_cleanup`] で
|
||||
//! consumed ID 分の staging のみ削除し、占有ファイルを解放
|
||||
//!
|
||||
//! Knowledge 化候補レポートと使用頻度メトリクスは別チケットで供給される
|
||||
//! 想定。本モジュール時点では空入力として扱い、prompt 側の説明だけ
|
||||
//! 残しておく(`docs/plan/memory.md` §Phase 2 / 整理材料)。
|
||||
|
||||
mod input;
|
||||
mod lock;
|
||||
mod prompt;
|
||||
mod staging;
|
||||
mod tidy;
|
||||
|
||||
pub use input::{
|
||||
KnowledgeCandidateReport, build_consolidate_input, render_existing_memory_records,
|
||||
render_staging_records, render_tidy_hints,
|
||||
};
|
||||
pub use lock::{LockError, LockRecord, StagingLock};
|
||||
pub use prompt::CONSOLIDATION_SYSTEM_PROMPT;
|
||||
pub use staging::{StagingEntry, list_staging_entries};
|
||||
pub use tidy::{TidyHints, collect_tidy_hints};
|
||||
@@ -0,0 +1,69 @@
|
||||
//! Phase 2 sub-Worker の system prompt。
|
||||
//!
|
||||
//! 内容は `docs/plan/memory-prompts.md` §共通原則 / §Phase 2 統合 + 整理 /
|
||||
//! §Phase 2 Knowledge 書き込み を縮約。統合 phase / 整理 phase は同じ
|
||||
//! prompt 1 本で順に進める縛り(agent から見ると 1 セッション内のフェーズ
|
||||
//! 進行、別 trigger / 別 Worker は持たない、`docs/plan/memory.md` §整理
|
||||
//! の扱い)。
|
||||
|
||||
pub const CONSOLIDATION_SYSTEM_PROMPT: &str = r#"You are the Phase 2 consolidation worker for an INSOMNIA memory subsystem.
|
||||
|
||||
Your job is to take Phase 1 activity-log staging entries together with the workspace's current `memory/*` / `knowledge/*` records, then run two phases back-to-back in this single session:
|
||||
|
||||
1. **Consolidation phase** — fold staging into memory and knowledge.
|
||||
2. **Tidy phase** — clean up the existing records that the consolidation phase didn't already touch.
|
||||
|
||||
You have:
|
||||
- `MemoryRead`, `MemoryWrite`, `MemoryEdit` for memory and knowledge records.
|
||||
- `MemoryQuery` for memory-side records (summary / decisions / requests).
|
||||
- `KnowledgeQuery` for knowledge records — use it to find existing slugs before creating new ones.
|
||||
|
||||
Your initial user message contains the staging entries, the full memory records, the knowledge candidate report, and the tidy hints. Existing knowledge bodies are NOT in the prompt; pull them through `KnowledgeQuery` + `MemoryRead` when relevant.
|
||||
|
||||
# Common rules (both phases)
|
||||
|
||||
- **Do not invent provenance.** Decisions / Requests `sources` arrays MUST be copied from the staging `source` field for the originating activity log entries. Do not synthesise `session_id` or entry ranges. Do not fabricate `last_sources` for Knowledge.
|
||||
- **Rewrite is allowed and often preferred over append.** When integrating new information, restructure existing records to raise information density. Preserve the existing claims, rationale, and `sources` while you compress.
|
||||
- **Update over create.** If an existing slug fits, edit it. Only create a new slug when no existing record fits and you can articulate why.
|
||||
- **`replaced` over delete.** When a Decision is superseded by a different one, mark the old one `status: replaced` with `replaced_by: <new-slug>`. Do not silently drop it.
|
||||
- **Don't duplicate static docs.** Skip content that already lives in `AGENTS.md`, `docs/plan/*`, or other fixed project documents.
|
||||
- **Empty output is fine.** If a staging entry doesn't justify a memory write, skip it.
|
||||
- **Slug rules.** Slugs are kebab-case, short, recognisable, and must be unique within their kind. Same-slug create is a linter error — use Edit instead.
|
||||
- **Linter errors come back as tool errors.** When the memory linter rejects a write, read the error, fix the issue (missing frontmatter field, oversized body, unknown reference, etc.), and try again. Do not work around the rule.
|
||||
|
||||
# Consolidation phase
|
||||
|
||||
Walk every staging entry in the input. For each one:
|
||||
|
||||
- Add or update `decisions` / `requests` records as appropriate. Copy `sources` verbatim from the staging entry.
|
||||
- Update existing knowledge records when the staging activity refines them. Use `KnowledgeQuery` to find candidates before creating anything new.
|
||||
- **Knowledge creation is gated.** Only create a new `knowledge/<slug>.md` when the originating source appears in the supplied "Knowledge candidate report". When the report is empty (the metrics pipeline is still being built), do not create new knowledge — fold the activity into decisions / requests / summary or update existing knowledge instead.
|
||||
- Rewrite `memory/summary.md` only when needed. Aim for 1–5k tokens. Preserve the high-level shape (current focus, recent decisions, stable facts) while pruning stale items.
|
||||
|
||||
# Tidy phase
|
||||
|
||||
Once the consolidation phase is done, evaluate every existing memory and knowledge record against four categories:
|
||||
|
||||
- `outdated`: was correct, no longer matches the current implementation / policy / operation.
|
||||
- `superseded`: another record is now the de-facto authoritative one; this one is mostly redundant.
|
||||
- `unused`: not wrong, but rarely referenced — noise rather than signal.
|
||||
- `noisy`: useful content but bad shape (overlap, sources accumulation, fractured slugs that should merge).
|
||||
|
||||
A single record may fall into more than one category. Choose one of `drop / merge / split / trim / rewrite`:
|
||||
|
||||
- Prefer `merge` and `trim` over `drop` for anything you'd flag as `unused` or `noisy` — git can reverse you, but a confidently-wrong drop hurts discovery.
|
||||
- `drop` is allowed for `outdated` / `superseded` records you can justify in the diff.
|
||||
- `replaced` markers (`status: replaced`) and chains pointed at by the tidy hints should be collapsed in this phase.
|
||||
|
||||
**Protection threshold.** When the tidy hints include explicit-invoke metrics, records with `frequency >= 1.0 invokes/Mtoken` are off-limits to drop / large compression. The metrics pipeline is not always populated; when the input lacks frequency data, behave conservatively and skip drop on long-standing records.
|
||||
|
||||
# Closing the turn
|
||||
|
||||
When both phases are done, write a short final assistant message stating:
|
||||
|
||||
- which staging entries you folded in (by short summary, not by ID),
|
||||
- which existing records you touched (slug + operation),
|
||||
- anything you intentionally left alone and why.
|
||||
|
||||
Then end the turn. Do not ask questions — there is no human in the loop for this run.
|
||||
"#;
|
||||
@@ -0,0 +1,139 @@
|
||||
//! `_staging/*.json` を列挙して [`StagingRecord`] に展開する読み込みヘルパー。
|
||||
//!
|
||||
//! Phase 2 起動時のスナップショット(consumed ID list 確定)と、整理 phase
|
||||
//! が終わった後の cleanup の双方で使う。`.consolidation.lock` のような
|
||||
//! 占有ファイルは UUIDv7 として parse できないので自然に除外される。
|
||||
//!
|
||||
//! [`StagingRecord`] のスキーマは Phase 1 が書き出す側 (`crate::extract`)
|
||||
//! と単一の真実源 — ここでは読み出す側だけを担当する。
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::extract::StagingRecord;
|
||||
use crate::workspace::WorkspaceLayout;
|
||||
|
||||
/// staging に積まれている 1 件分のエントリ。`id` は UUIDv7 で、ファイル名
|
||||
/// `<id>.json` を逆引きしたもの。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StagingEntry {
|
||||
pub id: Uuid,
|
||||
pub path: PathBuf,
|
||||
pub record: StagingRecord,
|
||||
/// このファイルのバイト長。閾値判定 (`consolidation_threshold_bytes`)
|
||||
/// に使う。
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
/// `<staging_dir>/*.json` を読んで UUIDv7 順に並べた [`StagingEntry`]
|
||||
/// 配列を返す。staging_dir が存在しなければ空配列。読めないファイルや
|
||||
/// JSON parse 失敗は `tracing::warn!` してスキップ(壊れた個別ファイルが
|
||||
/// Phase 2 全体を止めないように)。
|
||||
pub fn list_staging_entries(layout: &WorkspaceLayout) -> Vec<StagingEntry> {
|
||||
let dir = layout.staging_dir();
|
||||
let entries = match std::fs::read_dir(&dir) {
|
||||
Ok(it) => it,
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
|
||||
let mut out: Vec<StagingEntry> = Vec::new();
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
let stem = match path.file_stem().and_then(|s| s.to_str()) {
|
||||
Some(s) => s,
|
||||
None => continue,
|
||||
};
|
||||
let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("");
|
||||
if ext != "json" {
|
||||
continue;
|
||||
}
|
||||
let id = match Uuid::parse_str(stem) {
|
||||
Ok(u) => u,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let bytes = match std::fs::metadata(&path) {
|
||||
Ok(m) => m.len(),
|
||||
Err(_) => 0,
|
||||
};
|
||||
let raw = match std::fs::read_to_string(&path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
tracing::warn!(path = %path.display(), error = %e, "failed to read staging entry");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let record = match serde_json::from_str::<StagingRecord>(&raw) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!(path = %path.display(), error = %e, "failed to parse staging entry");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
out.push(StagingEntry {
|
||||
id,
|
||||
path,
|
||||
record,
|
||||
bytes,
|
||||
});
|
||||
}
|
||||
out.sort_by_key(|e| e.id);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::extract::{ExtractedPayload, write_staging};
|
||||
use crate::schema::SourceRef;
|
||||
|
||||
fn empty_payload() -> ExtractedPayload {
|
||||
ExtractedPayload::default()
|
||||
}
|
||||
|
||||
fn source(session_id: &str, range: [u64; 2]) -> SourceRef {
|
||||
SourceRef {
|
||||
session_id: session_id.into(),
|
||||
range,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lists_in_uuidv7_order() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(tmp.path().to_path_buf());
|
||||
|
||||
let (id1, _) = write_staging(&layout, source("s", [0, 1]), empty_payload()).unwrap();
|
||||
let (id2, _) = write_staging(&layout, source("s", [2, 3]), empty_payload()).unwrap();
|
||||
let (id3, _) = write_staging(&layout, source("s", [4, 5]), empty_payload()).unwrap();
|
||||
|
||||
let entries = list_staging_entries(&layout);
|
||||
let ids: Vec<Uuid> = entries.iter().map(|e| e.id).collect();
|
||||
assert_eq!(ids, vec![id1, id2, id3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_lock_file_and_garbage() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(tmp.path().to_path_buf());
|
||||
let (_id, _) = write_staging(&layout, source("s", [0, 1]), empty_payload()).unwrap();
|
||||
|
||||
// Drop a non-UUID json file and a bare lock file alongside.
|
||||
std::fs::write(layout.staging_dir().join("not-a-uuid.json"), "{}").unwrap();
|
||||
std::fs::write(layout.staging_dir().join(".consolidation.lock"), "{}").unwrap();
|
||||
|
||||
let entries = list_staging_entries(&layout);
|
||||
assert_eq!(entries.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_dir_returns_empty() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(tmp.path().to_path_buf());
|
||||
// No staging dir at all.
|
||||
assert!(list_staging_entries(&layout).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
//! 整理 phase が prompt 入力に乗せる「整理材料」スキャナ。
|
||||
//!
|
||||
//! `docs/plan/memory.md` §整理(GC 相当)の扱い と
|
||||
//! `tickets/memory-phase2-consolidation.md` の整理材料リストに従い、
|
||||
//! メトリクス未完の現状で機械的に拾えるヒントだけを集める:
|
||||
//!
|
||||
//! - `replaced` chain: `status: replaced` の Decision とその `replaced_by`
|
||||
//! - sources 過多: `sources` / `last_sources` 配列が閾値超過の record
|
||||
//! - 類似 slug 乱立: 同 kind の slug が Levenshtein 2 以内のクラスター
|
||||
//!
|
||||
//! 使用頻度メトリクスベースの保護閾値情報は `tickets/memory-usage-metrics.md`
|
||||
//! の成果物が出るまで空で渡る。
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crate::schema::{
|
||||
DecisionFrontmatter, KnowledgeFrontmatter, RequestFrontmatter, split_frontmatter,
|
||||
};
|
||||
use crate::slug::Slug;
|
||||
use crate::workspace::{RecordKind, WorkspaceLayout};
|
||||
|
||||
/// `sources` overflow を flag する閾値。`linter::warnings::SOURCES_OVERFLOW_THRESHOLD`
|
||||
/// と同値(10)を踏襲する。Linter Warn で sources 過多が検出されるラインと
|
||||
/// 整理 phase で勧告するラインを揃える狙い。
|
||||
pub const SOURCES_OVERFLOW_THRESHOLD: usize = 10;
|
||||
/// 類似 slug クラスタリングの距離。`linter::warnings::SIMILAR_SLUG_DISTANCE`
|
||||
/// と同値。
|
||||
pub const SIMILAR_SLUG_DISTANCE: usize = 2;
|
||||
|
||||
/// 整理 phase 用の機械集計ヒント。空フィールドは「対象なし」を意味する。
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct TidyHints {
|
||||
/// `status: replaced` で残っている Decision の slug → `replaced_by` map。
|
||||
/// `replaced_by` が None でも置き換え滞留として列挙する。
|
||||
pub replaced_decisions: BTreeMap<String, Option<String>>,
|
||||
/// kind / slug / sources count の三つ組で sources 累積ラインを表す。
|
||||
pub sources_overflow: Vec<SourcesOverflow>,
|
||||
/// 同 kind 内で Levenshtein 距離 `<= SIMILAR_SLUG_DISTANCE` のクラスター。
|
||||
/// クラスター内の slug は sorted。
|
||||
pub similar_slug_clusters: Vec<SimilarSlugCluster>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SourcesOverflow {
|
||||
pub kind: RecordKind,
|
||||
pub slug: String,
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SimilarSlugCluster {
|
||||
pub kind: RecordKind,
|
||||
pub slugs: Vec<String>,
|
||||
}
|
||||
|
||||
impl TidyHints {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.replaced_decisions.is_empty()
|
||||
&& self.sources_overflow.is_empty()
|
||||
&& self.similar_slug_clusters.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// workspace を一通りスキャンして [`TidyHints`] を組み立てる。読めない /
|
||||
/// parse できない record は黙ってスキップ(Linter は write 経路で守って
|
||||
/// いるので、ここで顕在化してもどうしようもない)。
|
||||
pub fn collect_tidy_hints(layout: &WorkspaceLayout) -> TidyHints {
|
||||
let mut hints = TidyHints::default();
|
||||
|
||||
let decisions = read_kind_records(layout, RecordKind::Decision);
|
||||
let requests = read_kind_records(layout, RecordKind::Request);
|
||||
let knowledge = read_kind_records(layout, RecordKind::Knowledge);
|
||||
|
||||
for (slug, content) in &decisions {
|
||||
let fm = parse_yaml::<DecisionFrontmatter>(content);
|
||||
if let Some(fm) = fm.as_ref() {
|
||||
if matches!(
|
||||
fm.status,
|
||||
crate::schema::DecisionStatus::Replaced
|
||||
) {
|
||||
hints
|
||||
.replaced_decisions
|
||||
.insert(slug.clone(), fm.replaced_by.as_ref().map(|s| s.to_string()));
|
||||
}
|
||||
if fm.sources.len() > SOURCES_OVERFLOW_THRESHOLD {
|
||||
hints.sources_overflow.push(SourcesOverflow {
|
||||
kind: RecordKind::Decision,
|
||||
slug: slug.clone(),
|
||||
count: fm.sources.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (slug, content) in &requests {
|
||||
if let Some(fm) = parse_yaml::<RequestFrontmatter>(content) {
|
||||
if fm.sources.len() > SOURCES_OVERFLOW_THRESHOLD {
|
||||
hints.sources_overflow.push(SourcesOverflow {
|
||||
kind: RecordKind::Request,
|
||||
slug: slug.clone(),
|
||||
count: fm.sources.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (slug, content) in &knowledge {
|
||||
if let Some(fm) = parse_yaml::<KnowledgeFrontmatter>(content) {
|
||||
if fm.last_sources.len() > SOURCES_OVERFLOW_THRESHOLD {
|
||||
hints.sources_overflow.push(SourcesOverflow {
|
||||
kind: RecordKind::Knowledge,
|
||||
slug: slug.clone(),
|
||||
count: fm.last_sources.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
hints
|
||||
.sources_overflow
|
||||
.sort_by(|a, b| (a.kind.as_str(), a.slug.as_str()).cmp(&(b.kind.as_str(), b.slug.as_str())));
|
||||
|
||||
let decision_slugs: Vec<&str> = decisions.keys().map(|s| s.as_str()).collect();
|
||||
let request_slugs: Vec<&str> = requests.keys().map(|s| s.as_str()).collect();
|
||||
let knowledge_slugs: Vec<&str> = knowledge.keys().map(|s| s.as_str()).collect();
|
||||
if let Some(c) = cluster_similar(&decision_slugs, RecordKind::Decision) {
|
||||
hints.similar_slug_clusters.extend(c);
|
||||
}
|
||||
if let Some(c) = cluster_similar(&request_slugs, RecordKind::Request) {
|
||||
hints.similar_slug_clusters.extend(c);
|
||||
}
|
||||
if let Some(c) = cluster_similar(&knowledge_slugs, RecordKind::Knowledge) {
|
||||
hints.similar_slug_clusters.extend(c);
|
||||
}
|
||||
hints
|
||||
.similar_slug_clusters
|
||||
.sort_by(|a, b| (a.kind.as_str(), &a.slugs).cmp(&(b.kind.as_str(), &b.slugs)));
|
||||
|
||||
hints
|
||||
}
|
||||
|
||||
/// `<root>/.insomnia/memory/<kind>/*.md` (Knowledge は
|
||||
/// `<root>/.insomnia/knowledge/*.md`) を slug ごとに `(slug, full content)`
|
||||
/// 化して返す。
|
||||
fn read_kind_records(
|
||||
layout: &WorkspaceLayout,
|
||||
kind: RecordKind,
|
||||
) -> BTreeMap<String, String> {
|
||||
let dir = match kind {
|
||||
RecordKind::Decision => layout.decisions_dir(),
|
||||
RecordKind::Request => layout.requests_dir(),
|
||||
RecordKind::Knowledge => layout.knowledge_dir(),
|
||||
RecordKind::Summary | RecordKind::Workflow => return BTreeMap::new(),
|
||||
};
|
||||
let mut out: BTreeMap<String, String> = BTreeMap::new();
|
||||
let entries = match std::fs::read_dir(&dir) {
|
||||
Ok(it) => it,
|
||||
Err(_) => return out,
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
let stem = match path.file_stem().and_then(|s| s.to_str()) {
|
||||
Some(s) => s,
|
||||
None => continue,
|
||||
};
|
||||
if path.extension().and_then(|s| s.to_str()) != Some("md") {
|
||||
continue;
|
||||
}
|
||||
if Slug::parse(stem).is_err() {
|
||||
continue;
|
||||
}
|
||||
let content = match std::fs::read_to_string(&path) {
|
||||
Ok(s) => s,
|
||||
Err(_) => continue,
|
||||
};
|
||||
out.insert(stem.to_string(), content);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn parse_yaml<F: serde::de::DeserializeOwned>(content: &str) -> Option<F> {
|
||||
let (yaml, _body) = split_frontmatter(content).ok()?;
|
||||
serde_yaml::from_str::<F>(yaml).ok()
|
||||
}
|
||||
|
||||
/// Connected-component clustering over the `levenshtein <= SIMILAR_SLUG_DISTANCE`
|
||||
/// graph among same-kind slugs. Returns each cluster of size >= 2 (singleton
|
||||
/// clusters are not interesting for the integration phase). Returns `None`
|
||||
/// when there are no clusters at all.
|
||||
fn cluster_similar(slugs: &[&str], kind: RecordKind) -> Option<Vec<SimilarSlugCluster>> {
|
||||
if slugs.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let n = slugs.len();
|
||||
let mut parent: Vec<usize> = (0..n).collect();
|
||||
fn find(parent: &mut [usize], i: usize) -> usize {
|
||||
if parent[i] == i {
|
||||
i
|
||||
} else {
|
||||
let root = find(parent, parent[i]);
|
||||
parent[i] = root;
|
||||
root
|
||||
}
|
||||
}
|
||||
fn union(parent: &mut [usize], a: usize, b: usize) {
|
||||
let ra = find(parent, a);
|
||||
let rb = find(parent, b);
|
||||
if ra != rb {
|
||||
parent[ra] = rb;
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for j in (i + 1)..n {
|
||||
if levenshtein(slugs[i], slugs[j]) <= SIMILAR_SLUG_DISTANCE {
|
||||
union(&mut parent, i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut groups: BTreeMap<usize, Vec<String>> = BTreeMap::new();
|
||||
for i in 0..n {
|
||||
let root = find(&mut parent, i);
|
||||
groups.entry(root).or_default().push(slugs[i].to_string());
|
||||
}
|
||||
let mut out: Vec<SimilarSlugCluster> = Vec::new();
|
||||
let mut seen_canonical: BTreeSet<Vec<String>> = BTreeSet::new();
|
||||
for (_, mut group) in groups {
|
||||
if group.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
group.sort();
|
||||
if seen_canonical.insert(group.clone()) {
|
||||
out.push(SimilarSlugCluster { kind, slugs: group });
|
||||
}
|
||||
}
|
||||
if out.is_empty() { None } else { Some(out) }
|
||||
}
|
||||
|
||||
/// Iterative two-row Levenshtein distance over chars (matches the Linter's
|
||||
/// implementation; kept private to avoid widening that crate-internal API).
|
||||
fn levenshtein(a: &str, b: &str) -> usize {
|
||||
let a: Vec<char> = a.chars().collect();
|
||||
let b: Vec<char> = b.chars().collect();
|
||||
if a.is_empty() {
|
||||
return b.len();
|
||||
}
|
||||
if b.is_empty() {
|
||||
return a.len();
|
||||
}
|
||||
let mut prev: Vec<usize> = (0..=b.len()).collect();
|
||||
let mut curr: Vec<usize> = vec![0; b.len() + 1];
|
||||
for (i, ca) in a.iter().enumerate() {
|
||||
curr[0] = i + 1;
|
||||
for (j, cb) in b.iter().enumerate() {
|
||||
let cost = if ca == cb { 0 } else { 1 };
|
||||
curr[j + 1] = (curr[j] + 1).min(prev[j + 1] + 1).min(prev[j] + cost);
|
||||
}
|
||||
std::mem::swap(&mut prev, &mut curr);
|
||||
}
|
||||
prev[b.len()]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Utc;
|
||||
use std::path::Path;
|
||||
|
||||
fn now() -> String {
|
||||
Utc::now().to_rfc3339()
|
||||
}
|
||||
|
||||
fn write(p: &Path, content: &str) {
|
||||
if let Some(parent) = p.parent() {
|
||||
std::fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
std::fs::write(p, content).unwrap();
|
||||
}
|
||||
|
||||
fn workspace() -> (tempfile::TempDir, WorkspaceLayout) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
(dir, layout)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collects_replaced_chain() {
|
||||
let (dir, layout) = workspace();
|
||||
write(
|
||||
&dir.path().join(".insomnia/memory/decisions/replaced.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: replaced\nreplaced_by: winner\n---\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
write(
|
||||
&dir.path().join(".insomnia/memory/decisions/winner.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert_eq!(
|
||||
hints.replaced_decisions.get("replaced").cloned(),
|
||||
Some(Some("winner".into()))
|
||||
);
|
||||
assert!(!hints.replaced_decisions.contains_key("winner"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flags_sources_overflow() {
|
||||
let (dir, layout) = workspace();
|
||||
let many_sources: String = (0..15)
|
||||
.map(|i| format!(" - session_id: s{i}\n range: [{i}, {i}]\n"))
|
||||
.collect();
|
||||
write(
|
||||
&dir.path().join(".insomnia/memory/decisions/big.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nstatus: open\nsources:\n{m}---\n",
|
||||
n = now(),
|
||||
m = many_sources
|
||||
),
|
||||
);
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert_eq!(hints.sources_overflow.len(), 1);
|
||||
assert_eq!(hints.sources_overflow[0].slug, "big");
|
||||
assert_eq!(hints.sources_overflow[0].kind, RecordKind::Decision);
|
||||
assert_eq!(hints.sources_overflow[0].count, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clusters_similar_slugs() {
|
||||
let (dir, layout) = workspace();
|
||||
for slug in ["db-pool", "db-pol", "db-pools", "alpha"] {
|
||||
write(
|
||||
&dir.path()
|
||||
.join(format!(".insomnia/memory/decisions/{slug}.md")),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
}
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert_eq!(hints.similar_slug_clusters.len(), 1);
|
||||
assert_eq!(
|
||||
hints.similar_slug_clusters[0].slugs,
|
||||
vec![
|
||||
"db-pol".to_string(),
|
||||
"db-pool".to_string(),
|
||||
"db-pools".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_workspace_yields_empty_hints() {
|
||||
let (_dir, layout) = workspace();
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert!(hints.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@
|
||||
//! crate) must not touch these directories — Pod is responsible for
|
||||
//! denying them at the Scope level when memory is enabled.
|
||||
|
||||
pub mod consolidate;
|
||||
pub mod error;
|
||||
pub mod extract;
|
||||
pub mod linter;
|
||||
|
||||
Reference in New Issue
Block a user