memory: consolidate staging through queue tools
This commit is contained in:
@@ -7,11 +7,16 @@
|
||||
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use chrono::Utc;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::audit::{AuditEvent, append_audit_event};
|
||||
use crate::consolidate::list_staging_entries_snapshot;
|
||||
use crate::extract::{
|
||||
ExtractedCandidate, ExtractedPayload, StagingEvidence, write_staging, write_staging_candidate,
|
||||
};
|
||||
@@ -31,6 +36,9 @@ pub enum MemoryBackendOperation {
|
||||
AppendAudit(MemoryAppendAuditOperation),
|
||||
StageCandidate(MemoryStageCandidateOperation),
|
||||
StageExtracted(MemoryStageExtractedOperation),
|
||||
StagingList(MemoryStagingListOperation),
|
||||
StagingRead(MemoryStagingReadOperation),
|
||||
StagingClose(MemoryStagingCloseOperation),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -127,6 +135,84 @@ pub struct MemoryStageExtractedOperation {
|
||||
pub payload: ExtractedPayload,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct MemoryStagingListOperation {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub limit: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct MemoryStagingReadOperation {
|
||||
pub candidate_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct MemoryStagingCloseOperation {
|
||||
pub candidate_id: String,
|
||||
pub action: MemoryStagingCloseAction,
|
||||
pub reason: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub affected_memory: Vec<MemoryStagingAffectedMemory>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MemoryStagingCloseAction {
|
||||
Applied,
|
||||
Discarded,
|
||||
Invalid,
|
||||
Duplicate,
|
||||
AlreadyCovered,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct MemoryStagingAffectedMemory {
|
||||
pub kind: MemoryToolKind,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub slug: Option<String>,
|
||||
pub operation: MemoryStagingAffectedMemoryOperation,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MemoryStagingAffectedMemoryOperation {
|
||||
Read,
|
||||
Write,
|
||||
Edit,
|
||||
Delete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryConsolidateStagingOperation {
|
||||
#[serde(default)]
|
||||
pub force: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub threshold_files: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub threshold_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryConsolidationOutput {
|
||||
pub status: String,
|
||||
pub summary: String,
|
||||
pub candidate_count: usize,
|
||||
pub total_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct MemoryStagingCloseDispositionRecord {
|
||||
schema_version: u32,
|
||||
candidate_id: String,
|
||||
staging_path: String,
|
||||
recorded_at: String,
|
||||
action: MemoryStagingCloseAction,
|
||||
reason: String,
|
||||
affected_memory: Vec<MemoryStagingAffectedMemory>,
|
||||
}
|
||||
|
||||
const STAGING_RESOLUTIONS_FILE: &str = "_resolutions.jsonl";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MemoryBackendAckOutput {
|
||||
pub summary: String,
|
||||
@@ -194,6 +280,15 @@ pub fn execute_memory_backend_operation(
|
||||
},
|
||||
))
|
||||
}
|
||||
MemoryBackendOperation::StagingList(operation) => {
|
||||
execute_staging_list(layout, operation).map(MemoryBackendOperationResult::ToolOutput)
|
||||
}
|
||||
MemoryBackendOperation::StagingRead(operation) => {
|
||||
execute_staging_read(layout, operation).map(MemoryBackendOperationResult::ToolOutput)
|
||||
}
|
||||
MemoryBackendOperation::StagingClose(operation) => {
|
||||
execute_staging_close(layout, operation).map(MemoryBackendOperationResult::ToolOutput)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -391,6 +486,109 @@ fn execute_delete(
|
||||
})
|
||||
}
|
||||
|
||||
fn execute_staging_list(
|
||||
layout: &WorkspaceLayout,
|
||||
operation: MemoryStagingListOperation,
|
||||
) -> io::Result<MemoryToolOutput> {
|
||||
let limit = operation.limit.unwrap_or(20).min(100);
|
||||
let snapshot = list_staging_entries_snapshot(layout);
|
||||
let total = snapshot.entries.len();
|
||||
let invalid_count = snapshot.invalid_count;
|
||||
let records = snapshot
|
||||
.entries
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|entry| {
|
||||
serde_json::json!({
|
||||
"candidate_id": entry.id.to_string(),
|
||||
"bytes": entry.bytes,
|
||||
"path": entry.path.display().to_string(),
|
||||
"source": entry.record.source,
|
||||
"kind": entry.record.kind,
|
||||
"claim": entry.record.claim,
|
||||
"why_useful": entry.record.why_useful,
|
||||
"staleness": entry.record.staleness,
|
||||
"evidence_count": entry.record.evidence.len(),
|
||||
"source_ref_count": entry.record.source_refs.len(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Ok(MemoryToolOutput {
|
||||
summary: format!(
|
||||
"Listed {} of {total} staging candidate(s); invalid_count={invalid_count}",
|
||||
records.len()
|
||||
),
|
||||
content: Some(serde_json::to_string_pretty(&records).map_err(io::Error::other)?),
|
||||
})
|
||||
}
|
||||
|
||||
fn execute_staging_read(
|
||||
layout: &WorkspaceLayout,
|
||||
operation: MemoryStagingReadOperation,
|
||||
) -> io::Result<MemoryToolOutput> {
|
||||
let entry = find_staging_entry(layout, &operation.candidate_id)?;
|
||||
Ok(MemoryToolOutput {
|
||||
summary: format!("Read staging candidate {}", entry.id),
|
||||
content: Some(serde_json::to_string_pretty(&entry.record).map_err(io::Error::other)?),
|
||||
})
|
||||
}
|
||||
|
||||
fn execute_staging_close(
|
||||
layout: &WorkspaceLayout,
|
||||
operation: MemoryStagingCloseOperation,
|
||||
) -> io::Result<MemoryToolOutput> {
|
||||
if operation.reason.trim().is_empty() {
|
||||
return Err(invalid_input("reason is required"));
|
||||
}
|
||||
validate_affected_memory(&operation.affected_memory)?;
|
||||
let entry = find_staging_entry(layout, &operation.candidate_id)?;
|
||||
let disposition = MemoryStagingCloseDispositionRecord {
|
||||
schema_version: 1,
|
||||
candidate_id: entry.id.to_string(),
|
||||
staging_path: entry.path.display().to_string(),
|
||||
recorded_at: Utc::now().to_rfc3339(),
|
||||
action: operation.action,
|
||||
reason: operation.reason,
|
||||
affected_memory: operation.affected_memory,
|
||||
};
|
||||
let resolutions_path = layout.memory_dir().join(STAGING_RESOLUTIONS_FILE);
|
||||
if let Some(parent) = resolutions_path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut line = serde_json::to_string(&disposition).map_err(io::Error::other)?;
|
||||
line.push('\n');
|
||||
fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&resolutions_path)?
|
||||
.write_all(line.as_bytes())?;
|
||||
fs::remove_file(&entry.path)?;
|
||||
Ok(MemoryToolOutput {
|
||||
summary: format!("Closed staging candidate {}", entry.id),
|
||||
content: Some(serde_json::to_string_pretty(&disposition).map_err(io::Error::other)?),
|
||||
})
|
||||
}
|
||||
|
||||
fn find_staging_entry(
|
||||
layout: &WorkspaceLayout,
|
||||
candidate_id: &str,
|
||||
) -> io::Result<crate::consolidate::StagingEntry> {
|
||||
let candidate_id = Uuid::parse_str(candidate_id)
|
||||
.map_err(|err| invalid_input(format!("invalid candidate_id: {err}")))?;
|
||||
list_staging_entries_snapshot(layout)
|
||||
.entries
|
||||
.into_iter()
|
||||
.find(|entry| entry.id == candidate_id)
|
||||
.ok_or_else(|| invalid_input(format!("staging candidate not found: {candidate_id}")))
|
||||
}
|
||||
|
||||
fn validate_affected_memory(records: &[MemoryStagingAffectedMemory]) -> io::Result<()> {
|
||||
for record in records {
|
||||
validate_slug_rules(record.kind, record.slug.as_deref())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn memory_path(
|
||||
layout: &WorkspaceLayout,
|
||||
kind: MemoryToolKind,
|
||||
@@ -538,3 +736,84 @@ fn tool_output_from_string(value: String) -> MemoryToolOutput {
|
||||
fn invalid_input(message: impl Into<String>) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, message.into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::extract::{CandidateKind, ExtractedCandidate};
|
||||
|
||||
#[test]
|
||||
fn staging_list_read_close_records_reason_and_deletes_candidate() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let layout = WorkspaceLayout::resolve(&manifest::MemoryConfig::default(), temp.path());
|
||||
let source = SourceRef {
|
||||
segment_id: "segment-1".into(),
|
||||
range: [0, 1],
|
||||
};
|
||||
let payload = ExtractedPayload {
|
||||
candidates: vec![ExtractedCandidate {
|
||||
kind: CandidateKind::Preference,
|
||||
claim: "User prefers short reviews".into(),
|
||||
why_useful: "Review style preference".into(),
|
||||
staleness: None,
|
||||
evidence_ids: Vec::new(),
|
||||
}],
|
||||
};
|
||||
let result = execute_memory_backend_operation(
|
||||
&layout,
|
||||
MemoryBackendOperation::StageExtracted(MemoryStageExtractedOperation {
|
||||
source,
|
||||
payload,
|
||||
}),
|
||||
)
|
||||
.unwrap();
|
||||
let candidate_id = match result {
|
||||
MemoryBackendOperationResult::StagingWritten(output) => output.staging_ids[0].clone(),
|
||||
_ => panic!("expected staging write output"),
|
||||
};
|
||||
|
||||
let list = execute_memory_backend_operation(
|
||||
&layout,
|
||||
MemoryBackendOperation::StagingList(MemoryStagingListOperation { limit: Some(10) }),
|
||||
)
|
||||
.unwrap();
|
||||
let MemoryBackendOperationResult::ToolOutput(list) = list else {
|
||||
panic!("expected list tool output")
|
||||
};
|
||||
assert!(list.content.unwrap().contains(&candidate_id));
|
||||
|
||||
let read = execute_memory_backend_operation(
|
||||
&layout,
|
||||
MemoryBackendOperation::StagingRead(MemoryStagingReadOperation {
|
||||
candidate_id: candidate_id.clone(),
|
||||
}),
|
||||
)
|
||||
.unwrap();
|
||||
let MemoryBackendOperationResult::ToolOutput(read) = read else {
|
||||
panic!("expected read tool output")
|
||||
};
|
||||
assert!(read.content.unwrap().contains("short reviews"));
|
||||
|
||||
execute_memory_backend_operation(
|
||||
&layout,
|
||||
MemoryBackendOperation::StagingClose(MemoryStagingCloseOperation {
|
||||
candidate_id: candidate_id.clone(),
|
||||
action: MemoryStagingCloseAction::Applied,
|
||||
reason: "Merged into durable request memory.".into(),
|
||||
affected_memory: vec![MemoryStagingAffectedMemory {
|
||||
kind: MemoryToolKind::Request,
|
||||
slug: Some("review-preferences".into()),
|
||||
operation: MemoryStagingAffectedMemoryOperation::Edit,
|
||||
}],
|
||||
}),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let snapshot = list_staging_entries_snapshot(&layout);
|
||||
assert!(snapshot.entries.is_empty());
|
||||
let resolutions =
|
||||
fs::read_to_string(layout.memory_dir().join(STAGING_RESOLUTIONS_FILE)).unwrap();
|
||||
assert!(resolutions.contains(&candidate_id));
|
||||
assert!(resolutions.contains("Merged into durable request memory."));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,356 +0,0 @@
|
||||
//! consolidation sub-Engine への最初のユーザー入力を組み立てる。
|
||||
//!
|
||||
//! extract (`extract::build_extract_input`) と同じ方針で、固定 schema の
|
||||
//! markdown セクション列にしてサブEngine に渡す。`docs/plan/memory.md`
|
||||
//! §Consolidation 入力 / §整理材料 の項目に従い:
|
||||
//!
|
||||
//! 1. consumed staging エントリ全文（`source` 込み）
|
||||
//! 2. 既存 `memory/*` 全文（summary / decisions / requests）
|
||||
//! 3. Usage evidence report（明示使用回数 + resident exposure cost）
|
||||
//! 4. 整理材料（Linter Warn ベース、hard protection 判定はしない）
|
||||
//!
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::consolidate::staging::StagingEntry;
|
||||
use crate::consolidate::tidy::TidyHints;
|
||||
use crate::usage::UsageReport;
|
||||
use crate::workspace::{RecordKind, WorkspaceLayout};
|
||||
|
||||
/// consolidation sub-Engine の最初の user 入力。
|
||||
pub fn build_consolidate_input(
|
||||
layout: &WorkspaceLayout,
|
||||
staging: &[StagingEntry],
|
||||
tidy: &TidyHints,
|
||||
usage_report: &UsageReport,
|
||||
) -> String {
|
||||
let mut out = String::new();
|
||||
out.push_str(
|
||||
"consolidation input. Run the integration step first \
|
||||
(fold the staging activity logs into memory), then the \
|
||||
tidy step (clean up existing records). Use the memory tools for \
|
||||
every write — direct file writes are denied by the worker scope.\n\n",
|
||||
);
|
||||
|
||||
out.push_str("## Staging entries (consumed by this run)\n\n");
|
||||
out.push_str(&render_staging_records(staging));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Existing memory records (full content)\n\n");
|
||||
out.push_str(&render_existing_memory_records(layout));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Usage evidence report\n\n");
|
||||
out.push_str(&render_usage_report(usage_report));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Tidy hints\n\n");
|
||||
out.push_str(&render_tidy_hints(tidy));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str(
|
||||
"When done, end the turn with a short final assistant message describing \
|
||||
what changed.",
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
/// Staging エントリ群を「`### <id>` ヘッダ + 整形 JSON ブロック」で並べる。
|
||||
/// 空配列なら「(none)」と書く。
|
||||
pub fn render_staging_records(entries: &[StagingEntry]) -> String {
|
||||
if entries.is_empty() {
|
||||
return "(none)\n".to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for entry in entries {
|
||||
let _ = writeln!(&mut out, "### {}", entry.id);
|
||||
let json = serde_json::to_string_pretty(&entry.record).unwrap_or_else(|_| "{}".into());
|
||||
out.push_str("```json\n");
|
||||
out.push_str(&json);
|
||||
out.push_str("\n```\n\n");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `<workspace>/.yoi/memory/{summary.md,decisions/*,requests/*}` を
|
||||
/// 「`### <kind>:<slug>` ヘッダ + raw markdown ブロック」で全文渡す。
|
||||
pub fn render_existing_memory_records(layout: &WorkspaceLayout) -> String {
|
||||
let mut out = String::new();
|
||||
|
||||
let summary = layout.summary_path();
|
||||
if let Ok(content) = std::fs::read_to_string(&summary) {
|
||||
out.push_str("### summary\n");
|
||||
out.push_str("```markdown\n");
|
||||
out.push_str(content.trim_end_matches('\n'));
|
||||
out.push_str("\n```\n\n");
|
||||
}
|
||||
|
||||
push_kind_records(&mut out, layout, RecordKind::Decision);
|
||||
push_kind_records(&mut out, layout, RecordKind::Request);
|
||||
|
||||
if out.is_empty() {
|
||||
return "(none)\n".to_string();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn push_kind_records(out: &mut String, layout: &WorkspaceLayout, kind: RecordKind) {
|
||||
let dir = match kind {
|
||||
RecordKind::Decision => layout.decisions_dir(),
|
||||
RecordKind::Request => layout.requests_dir(),
|
||||
RecordKind::Summary => return,
|
||||
};
|
||||
let entries = match std::fs::read_dir(&dir) {
|
||||
Ok(it) => it,
|
||||
Err(_) => return,
|
||||
};
|
||||
let mut paths: Vec<(String, std::path::PathBuf)> = Vec::new();
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
let stem = match path.file_stem().and_then(|s| s.to_str()) {
|
||||
Some(s) => s,
|
||||
None => continue,
|
||||
};
|
||||
if path.extension().and_then(|s| s.to_str()) != Some("md") {
|
||||
continue;
|
||||
}
|
||||
paths.push((stem.to_string(), path));
|
||||
}
|
||||
paths.sort();
|
||||
for (slug, path) in paths {
|
||||
let Ok(content) = std::fs::read_to_string(&path) else {
|
||||
continue;
|
||||
};
|
||||
let _ = writeln!(out, "### {}:{}", kind.as_str(), slug);
|
||||
out.push_str("```markdown\n");
|
||||
out.push_str(content.trim_end_matches('\n'));
|
||||
out.push_str("\n```\n\n");
|
||||
}
|
||||
}
|
||||
|
||||
fn render_usage_report(report: &UsageReport) -> String {
|
||||
if report.is_empty() {
|
||||
return "(empty — no explicit memory usage events recorded yet. \
|
||||
Treat this as lack of evidence, not proof that records are unused.)\n"
|
||||
.to_string();
|
||||
}
|
||||
let json = serde_json::to_string_pretty(report).unwrap_or_else(|_| "{}".to_string());
|
||||
format!(
|
||||
"This report is evidence only. Do not make hard tidy-protection decisions from it alone.\n\n```json\n{json}\n```\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// Tidy hints の Markdown 描画。空ヒントなら "(none)" 1 行。
|
||||
pub fn render_tidy_hints(tidy: &TidyHints) -> String {
|
||||
if tidy.is_empty() {
|
||||
return "(none)\n".to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
|
||||
if !tidy.replaced_decisions.is_empty() {
|
||||
out.push_str("**Replaced decisions still on disk** — collapse if the chain has settled:\n");
|
||||
for (slug, replaced_by) in &tidy.replaced_decisions {
|
||||
match replaced_by {
|
||||
Some(target) => {
|
||||
let _ = writeln!(&mut out, "- `{slug}` → `{target}`");
|
||||
}
|
||||
None => {
|
||||
let _ = writeln!(&mut out, "- `{slug}` (no `replaced_by` set)");
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
if !tidy.sources_overflow.is_empty() {
|
||||
out.push_str(
|
||||
"**Sources overflow** — consider trimming to the most recent entries (git log keeps the rest):\n",
|
||||
);
|
||||
for s in &tidy.sources_overflow {
|
||||
let _ = writeln!(
|
||||
&mut out,
|
||||
"- {} `{}` ({} sources)",
|
||||
s.kind.as_str(),
|
||||
s.slug,
|
||||
s.count
|
||||
);
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
if !tidy.similar_slug_clusters.is_empty() {
|
||||
out.push_str("**Similar slug clusters** — evaluate for merge / rename:\n");
|
||||
for c in &tidy.similar_slug_clusters {
|
||||
let joined = c
|
||||
.slugs
|
||||
.iter()
|
||||
.map(|s| format!("`{s}`"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let _ = writeln!(&mut out, "- {}: {}", c.kind.as_str(), joined);
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
out.push_str(
|
||||
"Use the Usage evidence report as soft context only; \
|
||||
require an explicit reason before deleting or heavily compressing records with recent use.\n",
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::consolidate::tidy::{SimilarSlugCluster, SourcesOverflow};
|
||||
use crate::extract::{
|
||||
CandidateKind, ExtractedCandidate, ExtractedPayload, STAGING_SCHEMA_VERSION,
|
||||
StagingEvidence, StagingRecord, write_staging,
|
||||
};
|
||||
use crate::schema::{EvidenceKind, SourceEvidenceRef, SourceRef};
|
||||
use chrono::Utc;
|
||||
use std::path::Path;
|
||||
|
||||
fn now() -> String {
|
||||
Utc::now().to_rfc3339()
|
||||
}
|
||||
|
||||
fn write(p: &Path, content: &str) {
|
||||
if let Some(parent) = p.parent() {
|
||||
std::fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
std::fs::write(p, content).unwrap();
|
||||
}
|
||||
|
||||
fn candidate_payload(kind: CandidateKind, claim: &str) -> ExtractedPayload {
|
||||
ExtractedPayload {
|
||||
candidates: vec![ExtractedCandidate {
|
||||
kind,
|
||||
claim: claim.into(),
|
||||
why_useful: "useful for consolidation".into(),
|
||||
staleness: None,
|
||||
evidence_ids: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_includes_all_sections_when_populated() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
|
||||
write(
|
||||
&dir.path().join(".yoi/memory/summary.md"),
|
||||
&format!("---\nupdated_at: {n}\n---\nstate of the world\n", n = now()),
|
||||
);
|
||||
write(
|
||||
&dir.path().join(".yoi/memory/decisions/dec.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\nbody\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
let _written = write_staging(
|
||||
&layout,
|
||||
SourceRef {
|
||||
segment_id: "s".into(),
|
||||
range: [0, 1],
|
||||
},
|
||||
candidate_payload(CandidateKind::Preference, "Prefer concise tickets"),
|
||||
)
|
||||
.unwrap();
|
||||
let staging = crate::consolidate::staging::list_staging_entries(&layout);
|
||||
let tidy = TidyHints {
|
||||
replaced_decisions: [("old".to_string(), Some("new".to_string()))]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
sources_overflow: vec![SourcesOverflow {
|
||||
kind: RecordKind::Decision,
|
||||
slug: "dec".into(),
|
||||
count: 12,
|
||||
}],
|
||||
similar_slug_clusters: vec![SimilarSlugCluster {
|
||||
kind: RecordKind::Decision,
|
||||
slugs: vec!["a".into(), "ab".into()],
|
||||
}],
|
||||
};
|
||||
let report = UsageReport::empty();
|
||||
|
||||
let out = build_consolidate_input(&layout, &staging, &tidy, &report);
|
||||
assert!(out.contains("Staging entries"));
|
||||
assert!(out.contains("Existing memory records"));
|
||||
assert!(out.contains("Usage evidence report"));
|
||||
assert!(out.contains("Tidy hints"));
|
||||
assert!(out.contains("state of the world"));
|
||||
assert!(out.contains("decision:dec"));
|
||||
assert!(out.contains("Replaced decisions"));
|
||||
assert!(out.contains("Sources overflow"));
|
||||
assert!(out.contains("Similar slug clusters"));
|
||||
assert!(out.contains("no explicit memory usage events"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn staging_render_preserves_entry_source_refs() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
std::fs::create_dir_all(layout.staging_dir()).unwrap();
|
||||
let id = uuid::Uuid::now_v7();
|
||||
let record = StagingRecord {
|
||||
schema_version: STAGING_SCHEMA_VERSION,
|
||||
id: id.to_string(),
|
||||
extract_run_id: "run-1".into(),
|
||||
source: SourceRef {
|
||||
segment_id: "segment-record".into(),
|
||||
range: [0, 10],
|
||||
},
|
||||
kind: CandidateKind::Decision,
|
||||
claim: "Keep flat staging records".into(),
|
||||
why_useful: "consolidation input is lossless JSON".into(),
|
||||
staleness: None,
|
||||
evidence: vec![StagingEvidence {
|
||||
id: "ev-1".into(),
|
||||
kind: EvidenceKind::new(EvidenceKind::MESSAGE),
|
||||
entry_range: Some([3, 4]),
|
||||
excerpt: Some("bounded excerpt".into()),
|
||||
summary: Some("bounded summary".into()),
|
||||
}],
|
||||
source_refs: vec![SourceEvidenceRef {
|
||||
session_id: Some("session-1".into()),
|
||||
segment_id: Some("segment-1".into()),
|
||||
entry_range: Some([3, 4]),
|
||||
evidence_id: Some("ev-1".into()),
|
||||
evidence_kind: Some(EvidenceKind::new(EvidenceKind::MESSAGE)),
|
||||
label: Some("user message".into()),
|
||||
summary: Some("bounded summary".into()),
|
||||
}],
|
||||
};
|
||||
std::fs::write(
|
||||
layout.staging_dir().join(format!("{id}.json")),
|
||||
serde_json::to_string_pretty(&record).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let staging = crate::consolidate::staging::list_staging_entries(&layout);
|
||||
|
||||
let out = render_staging_records(&staging);
|
||||
|
||||
assert!(out.contains("source_refs"));
|
||||
assert!(out.contains("session-1"));
|
||||
assert!(out.contains("entry_range"));
|
||||
assert!(out.contains("ev-1"));
|
||||
assert!(out.contains("message"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_inputs_render_placeholders() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
let out =
|
||||
build_consolidate_input(&layout, &[], &TidyHints::default(), &UsageReport::empty());
|
||||
// Both staging and tidy show "(none)"; existing memory records too.
|
||||
assert!(out.contains("Staging entries"));
|
||||
assert!(out.contains("(none)"));
|
||||
}
|
||||
}
|
||||
@@ -1,321 +0,0 @@
|
||||
//! `_staging/.consolidation.lock` による consolidation 占有ファイル。
|
||||
//!
|
||||
//! `docs/plan/memory.md` §並走防止 に従い:
|
||||
//!
|
||||
//! - ファイルが存在し、記録された Worker が動作している間、その Worker が排他占有
|
||||
//! - クラッシュで残った stale lock は、所有者 PID が死んでいれば次回 spawn
|
||||
//! 時に上書き取得できる
|
||||
//! - cleanup は consumed ID の staging エントリのみ削除し、実行中に extract
|
||||
//! が追加した分は残す
|
||||
//!
|
||||
//! 占有判定は Linux/macOS の `kill(pid, 0)` 経由で行う（`ESRCH` で死亡判定）。
|
||||
//! Windows は対象外: Yoi は POSIX 環境を前提にしている。
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::workspace::WorkspaceLayout;
|
||||
|
||||
const LOCK_FILE: &str = ".consolidation.lock";
|
||||
|
||||
/// 占有ファイルの中身。`pid` で stale 判定し、`worker_name` / `started_at` /
|
||||
/// `consumed_ids` は診断とクラッシュ復旧時の参照に使う。
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LockRecord {
|
||||
pub pid: u32,
|
||||
pub worker_name: String,
|
||||
pub started_at: DateTime<Utc>,
|
||||
/// この consolidation run が起動時スナップショットで確定した consumed staging
|
||||
/// entry の UUIDv7 列。完了時はこの列のみ削除し、追加分は残す。
|
||||
pub consumed_ids: Vec<Uuid>,
|
||||
}
|
||||
|
||||
/// 占有取得 / 解放のエラー。
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LockError {
|
||||
/// 占有ファイルが既にあり、所有者 PID が生きているのでスキップ。
|
||||
#[error("consolidation lock held by live pid {pid} (worker {worker_name:?})")]
|
||||
InUse { pid: u32, worker_name: String },
|
||||
#[error("io error at {}: {source}", .path.display())]
|
||||
Io {
|
||||
path: PathBuf,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("failed to (de)serialize lock record: {0}")]
|
||||
Serde(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
impl LockError {
|
||||
fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
|
||||
Self::Io {
|
||||
path: path.into(),
|
||||
source,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// consolidation が走っている間 RAII で持つ占有ハンドル。`Drop` では何もしない —
|
||||
/// 完了時の cleanup は consumed ID 列削除と一緒に行う必要があるため、明示
|
||||
/// 解放 [`StagingLock::release_with_cleanup`] を使う。明示解放しないまま
|
||||
/// drop された場合は占有ファイルがそのまま残り、次回 spawn 時に PID が
|
||||
/// 死んでいれば stale 上書きされる。
|
||||
#[derive(Debug)]
|
||||
pub struct StagingLock {
|
||||
path: PathBuf,
|
||||
record: LockRecord,
|
||||
}
|
||||
|
||||
impl StagingLock {
|
||||
pub fn record(&self) -> &LockRecord {
|
||||
&self.record
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// 占有取得を試みる。既に live な lock があれば
|
||||
/// [`LockError::InUse`]、stale 判定なら上書き取得する。
|
||||
/// staging dir が無ければ作成する。
|
||||
pub fn acquire(
|
||||
layout: &WorkspaceLayout,
|
||||
pid: u32,
|
||||
worker_name: impl Into<String>,
|
||||
consumed_ids: Vec<Uuid>,
|
||||
) -> Result<Self, LockError> {
|
||||
let staging_dir = layout.staging_dir();
|
||||
fs::create_dir_all(&staging_dir).map_err(|e| LockError::io(&staging_dir, e))?;
|
||||
let path = staging_dir.join(LOCK_FILE);
|
||||
|
||||
if path.exists() {
|
||||
let raw = fs::read_to_string(&path).map_err(|e| LockError::io(&path, e))?;
|
||||
// 壊れた lock は stale とみなして上書き許可。
|
||||
if let Ok(existing) = serde_json::from_str::<LockRecord>(&raw) {
|
||||
if pid_is_alive(existing.pid) {
|
||||
return Err(LockError::InUse {
|
||||
pid: existing.pid,
|
||||
worker_name: existing.worker_name,
|
||||
});
|
||||
}
|
||||
tracing::warn!(
|
||||
stale_pid = existing.pid,
|
||||
stale_pod = %existing.worker_name,
|
||||
"consolidation stale lock detected, taking over"
|
||||
);
|
||||
} else {
|
||||
tracing::warn!(path = %path.display(), "consolidation lock unparseable, treating as stale");
|
||||
}
|
||||
}
|
||||
|
||||
let record = LockRecord {
|
||||
pid,
|
||||
worker_name: worker_name.into(),
|
||||
started_at: Utc::now(),
|
||||
consumed_ids,
|
||||
};
|
||||
let json = serde_json::to_string_pretty(&record)?;
|
||||
fs::write(&path, json).map_err(|e| LockError::io(&path, e))?;
|
||||
Ok(Self { path, record })
|
||||
}
|
||||
|
||||
/// 占有を解放しつつ consumed ID 列の staging エントリを削除する。
|
||||
/// 削除対象が見当たらない場合は黙ってスキップ（既に外部で消えていた等）。
|
||||
/// 占有ファイル自体の削除も best-effort: 失敗時は warn を出すだけで
|
||||
/// エラーは伝播しない（次回 spawn 時に stale 判定で上書きされる）。
|
||||
pub fn release_with_cleanup(self, layout: &WorkspaceLayout) {
|
||||
let staging_dir = layout.staging_dir();
|
||||
for id in &self.record.consumed_ids {
|
||||
let target = staging_dir.join(format!("{id}.json"));
|
||||
match fs::remove_file(&target) {
|
||||
Ok(_) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
path = %target.display(),
|
||||
error = %e,
|
||||
"failed to clean up consumed staging entry"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.unlink_lock_only();
|
||||
}
|
||||
|
||||
/// 占有ファイルだけ削除し、staging エントリには触らない。consolidation
|
||||
/// sub-Engine が途中で失敗した場合に使う: 入力 staging を残したまま
|
||||
/// 次回再評価で再処理させる（`docs/plan/memory.md` §並走防止 の
|
||||
/// 「重複作成は同一 slug update に自然収束」運用）。
|
||||
pub fn release_only(self) {
|
||||
self.unlink_lock_only();
|
||||
}
|
||||
|
||||
fn unlink_lock_only(&self) {
|
||||
if let Err(e) = fs::remove_file(&self.path) {
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
tracing::warn!(
|
||||
path = %self.path.display(),
|
||||
error = %e,
|
||||
"failed to remove consolidation lock"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn pid_is_alive(pid: u32) -> bool {
|
||||
// `kill(0, 0)` and `kill(-1, 0)` are POSIX-special (process group / all
|
||||
// signalable processes) and would yield false positives. Reject pids
|
||||
// that don't fit a positive `pid_t` so a corrupted lock file with a
|
||||
// u32::MAX-ish value is treated as stale instead of magically alive.
|
||||
if pid == 0 || pid > i32::MAX as u32 {
|
||||
return false;
|
||||
}
|
||||
// SAFETY: `kill` with sig 0 only probes whether the target pid exists
|
||||
// and the caller has permission to signal it. No signal is delivered.
|
||||
let rc = unsafe { libc::kill(pid as i32, 0) };
|
||||
if rc == 0 {
|
||||
return true;
|
||||
}
|
||||
// EPERM means the process exists but we can't signal it — still alive
|
||||
// for our purposes. ESRCH means it's gone.
|
||||
let errno = std::io::Error::last_os_error()
|
||||
.raw_os_error()
|
||||
.unwrap_or(libc::EINVAL);
|
||||
errno != libc::ESRCH
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn pid_is_alive(_pid: u32) -> bool {
|
||||
// Unsupported platforms: assume the lock is live so we never overwrite
|
||||
// someone else's claim. consolidation will skip and try again next post-run.
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::extract::{CandidateKind, ExtractedCandidate, ExtractedPayload, write_staging};
|
||||
use crate::schema::SourceRef;
|
||||
|
||||
fn make_layout() -> (tempfile::TempDir, WorkspaceLayout) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
std::fs::create_dir_all(layout.staging_dir()).unwrap();
|
||||
(dir, layout)
|
||||
}
|
||||
|
||||
fn candidate_payload(claim: &str) -> ExtractedPayload {
|
||||
ExtractedPayload {
|
||||
candidates: vec![ExtractedCandidate {
|
||||
kind: CandidateKind::Lesson,
|
||||
claim: claim.into(),
|
||||
why_useful: "useful for test".into(),
|
||||
staleness: None,
|
||||
evidence_ids: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_writes_lock_file() {
|
||||
let (_dir, layout) = make_layout();
|
||||
let lock = StagingLock::acquire(&layout, std::process::id(), "worker", Vec::new()).unwrap();
|
||||
let path = layout.staging_dir().join(LOCK_FILE);
|
||||
assert!(path.exists());
|
||||
assert_eq!(lock.record().pid, std::process::id());
|
||||
assert_eq!(lock.record().worker_name, "worker");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_rejects_when_live_pid_holds_lock() {
|
||||
let (_dir, layout) = make_layout();
|
||||
// Use this test process's pid — it's definitely alive.
|
||||
let _first =
|
||||
StagingLock::acquire(&layout, std::process::id(), "worker-a", Vec::new()).unwrap();
|
||||
let err = StagingLock::acquire(&layout, std::process::id(), "worker-b", Vec::new())
|
||||
.expect_err("expected InUse");
|
||||
assert!(matches!(err, LockError::InUse { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_overwrites_stale_lock() {
|
||||
let (_dir, layout) = make_layout();
|
||||
// pid 1 is init on linux but for arbitrarily-large pids we'd need
|
||||
// `kill(pid, 0)` to return ESRCH. Use u32::MAX which is guaranteed
|
||||
// dead on every platform we target.
|
||||
let stale = LockRecord {
|
||||
pid: u32::MAX,
|
||||
worker_name: "ghost".into(),
|
||||
started_at: Utc::now(),
|
||||
consumed_ids: Vec::new(),
|
||||
};
|
||||
std::fs::write(
|
||||
layout.staging_dir().join(LOCK_FILE),
|
||||
serde_json::to_string_pretty(&stale).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let lock = StagingLock::acquire(&layout, std::process::id(), "worker", Vec::new())
|
||||
.expect("stale lock must be overwritable");
|
||||
assert_eq!(lock.record().pid, std::process::id());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_drops_consumed_entries_and_unlinks_lock() {
|
||||
let (_dir, layout) = make_layout();
|
||||
let id_a = write_staging(
|
||||
&layout,
|
||||
SourceRef {
|
||||
segment_id: "s".into(),
|
||||
range: [0, 0],
|
||||
},
|
||||
candidate_payload("a"),
|
||||
)
|
||||
.unwrap()
|
||||
.remove(0)
|
||||
.id;
|
||||
let id_b = write_staging(
|
||||
&layout,
|
||||
SourceRef {
|
||||
segment_id: "s".into(),
|
||||
range: [1, 1],
|
||||
},
|
||||
candidate_payload("b"),
|
||||
)
|
||||
.unwrap()
|
||||
.remove(0)
|
||||
.id;
|
||||
|
||||
let lock = StagingLock::acquire(&layout, std::process::id(), "worker", vec![id_a]).unwrap();
|
||||
let lock_path = lock.path().to_path_buf();
|
||||
lock.release_with_cleanup(&layout);
|
||||
|
||||
assert!(!lock_path.exists(), "lock file must be removed");
|
||||
assert!(
|
||||
!layout.staging_dir().join(format!("{id_a}.json")).exists(),
|
||||
"consumed entry must be deleted"
|
||||
);
|
||||
assert!(
|
||||
layout.staging_dir().join(format!("{id_b}.json")).exists(),
|
||||
"non-consumed entry must remain"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_is_resilient_to_missing_consumed_entries() {
|
||||
let (_dir, layout) = make_layout();
|
||||
let phantom = uuid::Uuid::now_v7();
|
||||
let lock =
|
||||
StagingLock::acquire(&layout, std::process::id(), "worker", vec![phantom]).unwrap();
|
||||
let lock_path = lock.path().to_path_buf();
|
||||
// No file at <staging>/<phantom>.json — release must not panic.
|
||||
lock.release_with_cleanup(&layout);
|
||||
assert!(!lock_path.exists());
|
||||
}
|
||||
}
|
||||
@@ -1,32 +1,12 @@
|
||||
//! consolidation: 統合 + 整理。
|
||||
//! Memory staging queue helpers.
|
||||
//!
|
||||
//! extract が staging に残した活動ログを `memory/*` に
|
||||
//! 統合し、続けて既存 record を `outdated | superseded | unused | noisy`
|
||||
//! の観点で整理する disposable Engine を、Worker 側が組み立てるための
|
||||
//! ヘルパー群を提供する。Worker は次の手順で sub-Engine を構築する:
|
||||
//!
|
||||
//! - [`build_consolidate_input`] を sub-Engine の最初の user 入力に
|
||||
//! - memory 専用 Tool (read / write / edit) と memory 検索ツールを登録
|
||||
//! - [`StagingLock::acquire`] で並走防止 + consumed ID 確定
|
||||
//! - sub-Engine run 完了後、[`StagingLock::release_with_cleanup`] で
|
||||
//! consumed ID 分の staging のみ削除し、占有ファイルを解放
|
||||
//!
|
||||
//! system prompt は Worker の `PromptCatalog`
|
||||
//! (`WorkerPrompt::MemoryConsolidationSystem`) で管理される。Usage report は
|
||||
//! 判断材料として渡すだけで、ここでは protection の hard decision はしない
|
||||
//! （`docs/plan/memory.md` §Consolidation / 整理材料）。
|
||||
//! Staging candidates are consumed by the backend-managed memory-consolidation
|
||||
//! Worker through MemoryStaging tools. This module only exposes bounded staging
|
||||
//! listing/read support; consolidation decisions are recorded by the backend close
|
||||
//! operation before a staging file is deleted.
|
||||
|
||||
mod input;
|
||||
mod lock;
|
||||
mod staging;
|
||||
mod tidy;
|
||||
|
||||
pub use input::{
|
||||
build_consolidate_input, render_existing_memory_records, render_staging_records,
|
||||
render_tidy_hints,
|
||||
};
|
||||
pub use lock::{LockError, LockRecord, StagingLock};
|
||||
pub use staging::{
|
||||
StagingEntriesSnapshot, StagingEntry, list_staging_entries, list_staging_entries_snapshot,
|
||||
};
|
||||
pub use tidy::{TidyHints, collect_tidy_hints};
|
||||
|
||||
@@ -1,335 +0,0 @@
|
||||
//! 整理 step が prompt 入力に乗せる「整理材料」スキャナ。
|
||||
//!
|
||||
//! `docs/plan/memory.md` §整理（GC 相当）の扱い と
|
||||
//! `tickets/memory-consolidation.md` の整理材料リストに従い、
|
||||
//! メトリクス未完の現状で機械的に拾えるヒントだけを集める:
|
||||
//!
|
||||
//! - `replaced` chain: `status: replaced` の Decision とその `replaced_by`
|
||||
//! - sources 過多: `sources` / `last_sources` 配列が閾値超過の record
|
||||
//! - 類似 slug 乱立: 同 kind の slug が Levenshtein 2 以内のクラスター
|
||||
//!
|
||||
//! 使用頻度メトリクスベースの保護閾値情報は `tickets/memory-usage-metrics.md`
|
||||
//! の成果物が出るまで空で渡る。
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crate::Slug;
|
||||
use crate::schema::{DecisionFrontmatter, RequestFrontmatter, split_frontmatter};
|
||||
use crate::workspace::{RecordKind, WorkspaceLayout};
|
||||
|
||||
/// `sources` overflow を flag する閾値。`linter::warnings::SOURCES_OVERFLOW_THRESHOLD`
|
||||
/// と同値（10）を踏襲する。Linter Warn で sources 過多が検出されるラインと
|
||||
/// 整理 step で勧告するラインを揃える狙い。
|
||||
pub const SOURCES_OVERFLOW_THRESHOLD: usize = 10;
|
||||
/// 類似 slug クラスタリングの距離。`linter::warnings::SIMILAR_SLUG_DISTANCE`
|
||||
/// と同値。
|
||||
pub const SIMILAR_SLUG_DISTANCE: usize = 2;
|
||||
|
||||
/// 整理 step 用の機械集計ヒント。空フィールドは「対象なし」を意味する。
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct TidyHints {
|
||||
/// `status: replaced` で残っている Decision の slug → `replaced_by` map。
|
||||
/// `replaced_by` が None でも置き換え滞留として列挙する。
|
||||
pub replaced_decisions: BTreeMap<String, Option<String>>,
|
||||
/// kind / slug / sources count の三つ組で sources 累積ラインを表す。
|
||||
pub sources_overflow: Vec<SourcesOverflow>,
|
||||
/// 同 kind 内で Levenshtein 距離 `<= SIMILAR_SLUG_DISTANCE` のクラスター。
|
||||
/// クラスター内の slug は sorted。
|
||||
pub similar_slug_clusters: Vec<SimilarSlugCluster>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SourcesOverflow {
|
||||
pub kind: RecordKind,
|
||||
pub slug: String,
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SimilarSlugCluster {
|
||||
pub kind: RecordKind,
|
||||
pub slugs: Vec<String>,
|
||||
}
|
||||
|
||||
impl TidyHints {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.replaced_decisions.is_empty()
|
||||
&& self.sources_overflow.is_empty()
|
||||
&& self.similar_slug_clusters.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// workspace を一通りスキャンして [`TidyHints`] を組み立てる。読めない /
|
||||
/// parse できない record は黙ってスキップ（Linter は write 経路で守って
|
||||
/// いるので、ここで顕在化してもどうしようもない）。
|
||||
pub fn collect_tidy_hints(layout: &WorkspaceLayout) -> TidyHints {
|
||||
let mut hints = TidyHints::default();
|
||||
|
||||
let decisions = read_kind_records(layout, RecordKind::Decision);
|
||||
let requests = read_kind_records(layout, RecordKind::Request);
|
||||
|
||||
for (slug, content) in &decisions {
|
||||
let fm = parse_yaml::<DecisionFrontmatter>(content);
|
||||
if let Some(fm) = fm.as_ref() {
|
||||
if matches!(fm.status, crate::schema::DecisionStatus::Replaced) {
|
||||
hints
|
||||
.replaced_decisions
|
||||
.insert(slug.clone(), fm.replaced_by.as_ref().map(|s| s.to_string()));
|
||||
}
|
||||
if fm.sources.len() > SOURCES_OVERFLOW_THRESHOLD {
|
||||
hints.sources_overflow.push(SourcesOverflow {
|
||||
kind: RecordKind::Decision,
|
||||
slug: slug.clone(),
|
||||
count: fm.sources.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (slug, content) in &requests {
|
||||
if let Some(fm) = parse_yaml::<RequestFrontmatter>(content) {
|
||||
if fm.sources.len() > SOURCES_OVERFLOW_THRESHOLD {
|
||||
hints.sources_overflow.push(SourcesOverflow {
|
||||
kind: RecordKind::Request,
|
||||
slug: slug.clone(),
|
||||
count: fm.sources.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
hints.sources_overflow.sort_by(|a, b| {
|
||||
(a.kind.as_str(), a.slug.as_str()).cmp(&(b.kind.as_str(), b.slug.as_str()))
|
||||
});
|
||||
|
||||
let decision_slugs: Vec<&str> = decisions.keys().map(|s| s.as_str()).collect();
|
||||
let request_slugs: Vec<&str> = requests.keys().map(|s| s.as_str()).collect();
|
||||
if let Some(c) = cluster_similar(&decision_slugs, RecordKind::Decision) {
|
||||
hints.similar_slug_clusters.extend(c);
|
||||
}
|
||||
if let Some(c) = cluster_similar(&request_slugs, RecordKind::Request) {
|
||||
hints.similar_slug_clusters.extend(c);
|
||||
}
|
||||
hints
|
||||
.similar_slug_clusters
|
||||
.sort_by(|a, b| (a.kind.as_str(), &a.slugs).cmp(&(b.kind.as_str(), &b.slugs)));
|
||||
|
||||
hints
|
||||
}
|
||||
|
||||
/// `<root>/.yoi/memory/<kind>/*.md` を slug ごとに `(slug, full content)`
|
||||
/// 化して返す。
|
||||
fn read_kind_records(layout: &WorkspaceLayout, kind: RecordKind) -> BTreeMap<String, String> {
|
||||
let dir = match kind {
|
||||
RecordKind::Decision => layout.decisions_dir(),
|
||||
RecordKind::Request => layout.requests_dir(),
|
||||
RecordKind::Summary => return BTreeMap::new(),
|
||||
};
|
||||
let mut out: BTreeMap<String, String> = BTreeMap::new();
|
||||
let entries = match std::fs::read_dir(&dir) {
|
||||
Ok(it) => it,
|
||||
Err(_) => return out,
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
let stem = match path.file_stem().and_then(|s| s.to_str()) {
|
||||
Some(s) => s,
|
||||
None => continue,
|
||||
};
|
||||
if path.extension().and_then(|s| s.to_str()) != Some("md") {
|
||||
continue;
|
||||
}
|
||||
if Slug::parse(stem).is_err() {
|
||||
continue;
|
||||
}
|
||||
let content = match std::fs::read_to_string(&path) {
|
||||
Ok(s) => s,
|
||||
Err(_) => continue,
|
||||
};
|
||||
out.insert(stem.to_string(), content);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn parse_yaml<F: serde::de::DeserializeOwned>(content: &str) -> Option<F> {
|
||||
let (yaml, _body) = split_frontmatter(content).ok()?;
|
||||
serde_yaml::from_str::<F>(yaml).ok()
|
||||
}
|
||||
|
||||
/// Connected-component clustering over the `levenshtein <= SIMILAR_SLUG_DISTANCE`
|
||||
/// graph among same-kind slugs. Returns each cluster of size >= 2 (singleton
|
||||
/// clusters are not interesting for the integration step). Returns `None`
|
||||
/// when there are no clusters at all.
|
||||
fn cluster_similar(slugs: &[&str], kind: RecordKind) -> Option<Vec<SimilarSlugCluster>> {
|
||||
if slugs.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let n = slugs.len();
|
||||
let mut parent: Vec<usize> = (0..n).collect();
|
||||
fn find(parent: &mut [usize], i: usize) -> usize {
|
||||
if parent[i] == i {
|
||||
i
|
||||
} else {
|
||||
let root = find(parent, parent[i]);
|
||||
parent[i] = root;
|
||||
root
|
||||
}
|
||||
}
|
||||
fn union(parent: &mut [usize], a: usize, b: usize) {
|
||||
let ra = find(parent, a);
|
||||
let rb = find(parent, b);
|
||||
if ra != rb {
|
||||
parent[ra] = rb;
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for j in (i + 1)..n {
|
||||
if levenshtein(slugs[i], slugs[j]) <= SIMILAR_SLUG_DISTANCE {
|
||||
union(&mut parent, i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut groups: BTreeMap<usize, Vec<String>> = BTreeMap::new();
|
||||
for i in 0..n {
|
||||
let root = find(&mut parent, i);
|
||||
groups.entry(root).or_default().push(slugs[i].to_string());
|
||||
}
|
||||
let mut out: Vec<SimilarSlugCluster> = Vec::new();
|
||||
let mut seen_canonical: BTreeSet<Vec<String>> = BTreeSet::new();
|
||||
for (_, mut group) in groups {
|
||||
if group.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
group.sort();
|
||||
if seen_canonical.insert(group.clone()) {
|
||||
out.push(SimilarSlugCluster { kind, slugs: group });
|
||||
}
|
||||
}
|
||||
if out.is_empty() { None } else { Some(out) }
|
||||
}
|
||||
|
||||
/// Iterative two-row Levenshtein distance over chars (matches the Linter's
|
||||
/// implementation; kept private to avoid widening that crate-internal API).
|
||||
fn levenshtein(a: &str, b: &str) -> usize {
|
||||
let a: Vec<char> = a.chars().collect();
|
||||
let b: Vec<char> = b.chars().collect();
|
||||
if a.is_empty() {
|
||||
return b.len();
|
||||
}
|
||||
if b.is_empty() {
|
||||
return a.len();
|
||||
}
|
||||
let mut prev: Vec<usize> = (0..=b.len()).collect();
|
||||
let mut curr: Vec<usize> = vec![0; b.len() + 1];
|
||||
for (i, ca) in a.iter().enumerate() {
|
||||
curr[0] = i + 1;
|
||||
for (j, cb) in b.iter().enumerate() {
|
||||
let cost = if ca == cb { 0 } else { 1 };
|
||||
curr[j + 1] = (curr[j] + 1).min(prev[j + 1] + 1).min(prev[j] + cost);
|
||||
}
|
||||
std::mem::swap(&mut prev, &mut curr);
|
||||
}
|
||||
prev[b.len()]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Utc;
|
||||
use std::path::Path;
|
||||
|
||||
fn now() -> String {
|
||||
Utc::now().to_rfc3339()
|
||||
}
|
||||
|
||||
fn write(p: &Path, content: &str) {
|
||||
if let Some(parent) = p.parent() {
|
||||
std::fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
std::fs::write(p, content).unwrap();
|
||||
}
|
||||
|
||||
fn workspace() -> (tempfile::TempDir, WorkspaceLayout) {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
(dir, layout)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collects_replaced_chain() {
|
||||
let (dir, layout) = workspace();
|
||||
write(
|
||||
&dir.path().join(".yoi/memory/decisions/replaced.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: replaced\nreplaced_by: winner\n---\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
write(
|
||||
&dir.path().join(".yoi/memory/decisions/winner.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert_eq!(
|
||||
hints.replaced_decisions.get("replaced").cloned(),
|
||||
Some(Some("winner".into()))
|
||||
);
|
||||
assert!(!hints.replaced_decisions.contains_key("winner"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flags_sources_overflow() {
|
||||
let (dir, layout) = workspace();
|
||||
let many_sources: String = (0..15)
|
||||
.map(|i| format!(" - segment_id: s{i}\n range: [{i}, {i}]\n"))
|
||||
.collect();
|
||||
write(
|
||||
&dir.path().join(".yoi/memory/decisions/big.md"),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nstatus: open\nsources:\n{m}---\n",
|
||||
n = now(),
|
||||
m = many_sources
|
||||
),
|
||||
);
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert_eq!(hints.sources_overflow.len(), 1);
|
||||
assert_eq!(hints.sources_overflow[0].slug, "big");
|
||||
assert_eq!(hints.sources_overflow[0].kind, RecordKind::Decision);
|
||||
assert_eq!(hints.sources_overflow[0].count, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clusters_similar_slugs() {
|
||||
let (dir, layout) = workspace();
|
||||
for slug in ["db-pool", "db-pol", "db-pools", "alpha"] {
|
||||
write(
|
||||
&dir.path().join(format!(".yoi/memory/decisions/{slug}.md")),
|
||||
&format!(
|
||||
"---\ncreated_at: {n}\nupdated_at: {n}\nsources: []\nstatus: open\n---\n",
|
||||
n = now()
|
||||
),
|
||||
);
|
||||
}
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert_eq!(hints.similar_slug_clusters.len(), 1);
|
||||
assert_eq!(
|
||||
hints.similar_slug_clusters[0].slugs,
|
||||
vec![
|
||||
"db-pol".to_string(),
|
||||
"db-pool".to_string(),
|
||||
"db-pools".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_workspace_yields_empty_hints() {
|
||||
let (_dir, layout) = workspace();
|
||||
let hints = collect_tidy_hints(&layout);
|
||||
assert!(hints.is_empty());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user