merge: memory usage metrics
This commit is contained in:
commit
3354c41e66
1
TODO.md
1
TODO.md
|
|
@ -25,7 +25,6 @@
|
|||
- Manifest: Tool Output / File Upload 上限の分離とデフォルト緩和 → [tickets/manifest-output-upload-limits.md](tickets/manifest-output-upload-limits.md)
|
||||
- Prune: 保護境界を turn 数から末尾 token budget に置き換え → [tickets/prune-token-budget.md](tickets/prune-token-budget.md)
|
||||
- メモリ機構
|
||||
- 明示使用回数ログ → [tickets/memory-usage-metrics.md](tickets/memory-usage-metrics.md)
|
||||
- extract / consolidation 監査ログ → [tickets/memory-audit-log.md](tickets/memory-audit-log.md)
|
||||
- セッション内 Task ツールの注意機構（無アクティビティで `<system-reminder>` ナッジ） → [tickets/session-todo-reminder.md](tickets/session-todo-reminder.md)
|
||||
- ワークスペースのメモリーをLintするヘッドレスCLI
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
//!
|
||||
//! 1. consumed staging エントリ全文（`source` 込み）
|
||||
//! 2. 既存 `memory/*` 全文（summary / decisions / requests）
|
||||
//! 3. Knowledge 化候補レポート（メトリクス未完なら空）
|
||||
//! 4. 整理材料（Linter Warn ベース、メトリクス未完なら明示 invoke 頻度なし）
|
||||
//! 3. Usage evidence report（明示使用回数 + resident exposure cost）
|
||||
//! 4. 整理材料（Linter Warn ベース、hard protection 判定はしない）
|
||||
//!
|
||||
//! 既存 `knowledge/*` 本文は埋めず、agent に `KnowledgeQuery` 経由で引かせる
|
||||
//! 設計（`docs/plan/memory.md` §retrieval 経路 / §Consolidation の Knowledge アクセス）。
|
||||
|
|
@ -16,41 +16,15 @@ use std::fmt::Write;
|
|||
|
||||
use crate::consolidate::staging::StagingEntry;
|
||||
use crate::consolidate::tidy::TidyHints;
|
||||
use crate::usage::UsageReport;
|
||||
use crate::workspace::{RecordKind, WorkspaceLayout};
|
||||
|
||||
/// Knowledge 化候補レポート。`tickets/memory-usage-metrics.md` の成果物が
|
||||
/// 出るまでは空で渡す前提（`docs/plan/memory.md` §Knowledge 化候補レポート）。
|
||||
/// 空入力時、統合 step は新規 Knowledge を作らず decisions / requests /
|
||||
/// summary / 既存 Knowledge update に留まる。
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct KnowledgeCandidateReport {
|
||||
/// 候補に上がった `(kind, slug, frequency_per_mtoken)` の三つ組。
|
||||
/// 空配列を渡すと「候補なし」を意味する。
|
||||
pub entries: Vec<KnowledgeCandidateEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct KnowledgeCandidateEntry {
|
||||
pub source_kind: &'static str,
|
||||
pub source_slug: String,
|
||||
pub frequency_per_mtoken: f64,
|
||||
}
|
||||
|
||||
impl KnowledgeCandidateReport {
|
||||
pub fn empty() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// consolidation sub-Worker の最初の user 入力。
|
||||
pub fn build_consolidate_input(
|
||||
layout: &WorkspaceLayout,
|
||||
staging: &[StagingEntry],
|
||||
tidy: &TidyHints,
|
||||
candidates: &KnowledgeCandidateReport,
|
||||
usage_report: &UsageReport,
|
||||
) -> String {
|
||||
let mut out = String::new();
|
||||
out.push_str(
|
||||
|
|
@ -68,8 +42,8 @@ pub fn build_consolidate_input(
|
|||
out.push_str(&render_existing_memory_records(layout));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Knowledge candidate report\n\n");
|
||||
out.push_str(&render_candidate_report(candidates));
|
||||
out.push_str("## Usage evidence report\n\n");
|
||||
out.push_str(&render_usage_report(usage_report));
|
||||
out.push('\n');
|
||||
|
||||
out.push_str("## Tidy hints\n\n");
|
||||
|
|
@ -159,21 +133,16 @@ fn push_kind_records(out: &mut String, layout: &WorkspaceLayout, kind: RecordKin
|
|||
}
|
||||
}
|
||||
|
||||
fn render_candidate_report(report: &KnowledgeCandidateReport) -> String {
|
||||
fn render_usage_report(report: &UsageReport) -> String {
|
||||
if report.is_empty() {
|
||||
return "(empty — usage metrics pipeline not populated. \
|
||||
Do not create new Knowledge records this run.)\n"
|
||||
return "(empty — no explicit memory/knowledge usage events recorded yet. \
|
||||
Treat this as lack of evidence, not proof that records are unused.)\n"
|
||||
.to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
for c in &report.entries {
|
||||
let _ = writeln!(
|
||||
&mut out,
|
||||
"- {} `{}` — frequency {:.3} invokes/Mtoken",
|
||||
c.source_kind, c.source_slug, c.frequency_per_mtoken
|
||||
);
|
||||
}
|
||||
out
|
||||
let json = serde_json::to_string_pretty(report).unwrap_or_else(|_| "{}".to_string());
|
||||
format!(
|
||||
"This report is evidence only. Do not make hard Knowledge-creation or tidy-protection decisions from it alone.\n\n```json\n{json}\n```\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// Tidy hints の Markdown 描画。空ヒントなら "(none)" 1 行。
|
||||
|
|
@ -229,8 +198,8 @@ pub fn render_tidy_hints(tidy: &TidyHints) -> String {
|
|||
}
|
||||
|
||||
out.push_str(
|
||||
"Explicit-invoke metrics (protection threshold) are not yet wired up; \
|
||||
skip drop on long-standing records when uncertain.\n",
|
||||
"Use the Usage evidence report as soft context only; \
|
||||
require an explicit reason before deleting or heavily compressing records with recent use.\n",
|
||||
);
|
||||
out
|
||||
}
|
||||
|
|
@ -295,31 +264,27 @@ mod tests {
|
|||
slugs: vec!["a".into(), "ab".into()],
|
||||
}],
|
||||
};
|
||||
let report = KnowledgeCandidateReport::empty();
|
||||
let report = UsageReport::empty();
|
||||
|
||||
let out = build_consolidate_input(&layout, &staging, &tidy, &report);
|
||||
assert!(out.contains("Staging entries"));
|
||||
assert!(out.contains("Existing memory records"));
|
||||
assert!(out.contains("Knowledge candidate report"));
|
||||
assert!(out.contains("Usage evidence report"));
|
||||
assert!(out.contains("Tidy hints"));
|
||||
assert!(out.contains("state of the world"));
|
||||
assert!(out.contains("decision:dec"));
|
||||
assert!(out.contains("Replaced decisions"));
|
||||
assert!(out.contains("Sources overflow"));
|
||||
assert!(out.contains("Similar slug clusters"));
|
||||
assert!(out.contains("usage metrics pipeline not populated"));
|
||||
assert!(out.contains("no explicit memory/knowledge usage events"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_inputs_render_placeholders() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
let out = build_consolidate_input(
|
||||
&layout,
|
||||
&[],
|
||||
&TidyHints::default(),
|
||||
&KnowledgeCandidateReport::empty(),
|
||||
);
|
||||
let out =
|
||||
build_consolidate_input(&layout, &[], &TidyHints::default(), &UsageReport::empty());
|
||||
// Both staging and tidy show "(none)"; existing memory records too.
|
||||
assert!(out.contains("Staging entries"));
|
||||
assert!(out.contains("(none)"));
|
||||
|
|
|
|||
|
|
@ -12,9 +12,8 @@
|
|||
//! consumed ID 分の staging のみ削除し、占有ファイルを解放
|
||||
//!
|
||||
//! system prompt は Pod の `PromptCatalog`
|
||||
//! (`PodPrompt::MemoryConsolidationSystem`) で管理される。Knowledge 化候補
|
||||
//! レポートと使用頻度メトリクスは別チケットで供給される想定。本モジュール
|
||||
//! 時点では空入力として扱い、prompt 側の説明だけ残しておく
|
||||
//! (`PodPrompt::MemoryConsolidationSystem`) で管理される。Usage report は
|
||||
//! 判断材料として渡すだけで、ここでは Knowledge 化や protection の hard decision はしない
|
||||
//! （`docs/plan/memory.md` §Consolidation / 整理材料）。
|
||||
|
||||
mod input;
|
||||
|
|
@ -23,8 +22,8 @@ mod staging;
|
|||
mod tidy;
|
||||
|
||||
pub use input::{
|
||||
KnowledgeCandidateReport, build_consolidate_input, render_existing_memory_records,
|
||||
render_staging_records, render_tidy_hints,
|
||||
build_consolidate_input, render_existing_memory_records, render_staging_records,
|
||||
render_tidy_hints,
|
||||
};
|
||||
pub use lock::{LockError, LockRecord, StagingLock};
|
||||
pub use staging::{StagingEntry, list_staging_entries};
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ pub mod scope;
|
|||
pub mod skill;
|
||||
pub mod slug;
|
||||
pub mod tool;
|
||||
pub mod usage;
|
||||
pub mod workflow;
|
||||
pub mod workspace;
|
||||
|
||||
|
|
@ -24,8 +25,15 @@ pub use extract::ExtractPointerPayload;
|
|||
pub use linter::{LintReport, Linter};
|
||||
pub use resident::{ResidentKnowledgeEntry, collect_resident_knowledge};
|
||||
pub use scope::deny_write_rules;
|
||||
pub use skill::{SKILL_FILENAME, SkillParseError, SkillRecord, load_skills_from_dir, parse_skill_md};
|
||||
pub use skill::{
|
||||
SKILL_FILENAME, SkillParseError, SkillRecord, load_skills_from_dir, parse_skill_md,
|
||||
};
|
||||
pub use slug::Slug;
|
||||
pub use usage::{
|
||||
UsageEvent, UsageEventKind, UsageRecordSnapshot, UsageReport, UsageReportRecord, UsageSource,
|
||||
append_resident_exposure_event, append_usage_event, append_use_event, build_usage_report,
|
||||
snapshot_record_from_bytes, snapshot_record_from_layout,
|
||||
};
|
||||
pub use workflow::{
|
||||
ResidentWorkflowEntry, ShadowedSkill, WORKFLOW_DESCRIPTION_HARD_CAP, WorkflowLoadError,
|
||||
WorkflowRecord, WorkflowRegistry, WorkflowSource, load_workflows,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ use crate::workspace::{RecordKind, WorkspaceLayout};
|
|||
|
||||
pub use edit::edit_tool;
|
||||
pub use query::{QueryConfig, knowledge_query_tool, memory_query_tool};
|
||||
pub use read::read_tool;
|
||||
pub use read::{read_tool, read_tool_with_usage};
|
||||
pub use write::write_tool;
|
||||
|
||||
/// Kinds the memory tools accept as input. `Workflow` is intentionally
|
||||
|
|
|
|||
|
|
@ -568,6 +568,29 @@ mod tests {
|
|||
assert!(records.is_empty(), "got records: {:?}", out.content);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_hits_do_not_log_usage() {
|
||||
let (dir, layout) = setup();
|
||||
write_decision(dir.path(), "alpha", "needle line\n");
|
||||
write_knowledge(
|
||||
dir.path(),
|
||||
"policy",
|
||||
"policy",
|
||||
"needle desc",
|
||||
"needle body\n",
|
||||
);
|
||||
|
||||
let (_, memory_tool) = memory_query_tool(layout.clone(), QueryConfig::default())();
|
||||
let (_, knowledge_tool) = knowledge_query_tool(layout.clone(), QueryConfig::default())();
|
||||
let inp = serde_json::json!({ "query": "needle" });
|
||||
memory_tool.execute(&inp.to_string()).await.unwrap();
|
||||
knowledge_tool.execute(&inp.to_string()).await.unwrap();
|
||||
|
||||
let report = crate::usage::build_usage_report(&layout).unwrap();
|
||||
assert!(report.records.is_empty());
|
||||
assert!(!layout.usage_events_path().exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn memory_query_respects_result_limit() {
|
||||
let (dir, layout) = setup();
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use llm_worker::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
|
|||
use serde::Deserialize;
|
||||
|
||||
use crate::tool::MemoryToolKind;
|
||||
use crate::usage::{self, UsageSource};
|
||||
use crate::workspace::WorkspaceLayout;
|
||||
|
||||
const DESCRIPTION: &str = "Read a memory or knowledge record by `kind` + `slug`. \
|
||||
|
|
@ -38,6 +39,7 @@ struct ReadParams {
|
|||
|
||||
struct ReadTool {
|
||||
layout: WorkspaceLayout,
|
||||
usage_session_id: Option<String>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
@ -58,6 +60,22 @@ impl Tool for ReadTool {
|
|||
})?;
|
||||
|
||||
let text = String::from_utf8_lossy(&bytes).into_owned();
|
||||
if let Some(session_id) = self.usage_session_id.as_deref() {
|
||||
let usage_slug = params.slug.as_deref().unwrap_or("summary");
|
||||
let snapshot = usage::snapshot_record_from_bytes(
|
||||
params.kind.record_kind(),
|
||||
usage_slug.to_string(),
|
||||
&bytes,
|
||||
);
|
||||
if let Err(err) = usage::append_use_event(
|
||||
&self.layout,
|
||||
session_id.to_string(),
|
||||
UsageSource::MemoryRead,
|
||||
vec![snapshot],
|
||||
) {
|
||||
tracing::warn!(error = %err, "failed to append MemoryRead usage event");
|
||||
}
|
||||
}
|
||||
let offset = params.offset.unwrap_or(0);
|
||||
let limit = params.limit.unwrap_or(DEFAULT_LIMIT).max(1);
|
||||
let rendered = render_numbered(&text, offset, limit);
|
||||
|
|
@ -117,6 +135,17 @@ fn render_numbered(text: &str, offset: usize, limit: usize) -> Rendered {
|
|||
}
|
||||
|
||||
pub fn read_tool(layout: WorkspaceLayout) -> ToolDefinition {
|
||||
read_tool_inner(layout, None)
|
||||
}
|
||||
|
||||
pub fn read_tool_with_usage(
|
||||
layout: WorkspaceLayout,
|
||||
session_id: impl Into<String>,
|
||||
) -> ToolDefinition {
|
||||
read_tool_inner(layout, Some(session_id.into()))
|
||||
}
|
||||
|
||||
fn read_tool_inner(layout: WorkspaceLayout, usage_session_id: Option<String>) -> ToolDefinition {
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(ReadParams);
|
||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||
|
|
@ -125,6 +154,7 @@ pub fn read_tool(layout: WorkspaceLayout) -> ToolDefinition {
|
|||
.input_schema(schema_value);
|
||||
let tool: Arc<dyn Tool> = Arc::new(ReadTool {
|
||||
layout: layout.clone(),
|
||||
usage_session_id: usage_session_id.clone(),
|
||||
});
|
||||
(meta, tool)
|
||||
})
|
||||
|
|
@ -209,6 +239,27 @@ mod tests {
|
|||
assert!(out.content.unwrap().contains("k"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_logs_explicit_use_when_usage_session_is_set() {
|
||||
let (dir, layout) = setup();
|
||||
let path = dir.path().join(".insomnia/memory/decisions/foo.md");
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(&path, "alpha\n").unwrap();
|
||||
|
||||
let (_, tool) = read_tool_with_usage(layout.clone(), "session-1")();
|
||||
let inp = serde_json::json!({ "kind": "decision", "slug": "foo" });
|
||||
tool.execute(&inp.to_string()).await.unwrap();
|
||||
|
||||
let report = usage::build_usage_report(&layout).unwrap();
|
||||
assert_eq!(report.records.len(), 1);
|
||||
let record = &report.records[0];
|
||||
assert_eq!(record.kind, "decision");
|
||||
assert_eq!(record.slug, "foo");
|
||||
assert_eq!(record.use_count, 1);
|
||||
assert_eq!(record.source_breakdown["MemoryRead"], 1);
|
||||
assert_eq!(record.resident_exposure_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_file_returns_execution_failed() {
|
||||
let (_dir, layout) = setup();
|
||||
|
|
|
|||
383
crates/memory/src/usage.rs
Normal file
383
crates/memory/src/usage.rs
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
//! Workspace-local usage event log for memory / knowledge / workflow records.
|
||||
//!
|
||||
//! The log is append-only JSONL under the workspace's `.insomnia/` tree. It is
|
||||
//! intentionally evidence-only: aggregation reports explicit context reads and
|
||||
//! resident exposure cost telemetry, but it does not classify records as
|
||||
//! Knowledge candidates or tidy-protected records.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::{self, BufRead, Write};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::workspace::{RecordKind, WorkspaceLayout};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageEventKind {
|
||||
Use,
|
||||
ResidentExposure,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
pub enum UsageSource {
|
||||
MemoryRead,
|
||||
KnowledgeRef,
|
||||
WorkflowInvoke,
|
||||
ResidentInjection,
|
||||
}
|
||||
|
||||
impl UsageSource {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::MemoryRead => "MemoryRead",
|
||||
Self::KnowledgeRef => "KnowledgeRef",
|
||||
Self::WorkflowInvoke => "WorkflowInvoke",
|
||||
Self::ResidentInjection => "ResidentInjection",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageRecordSnapshot {
|
||||
pub kind: String,
|
||||
pub slug: String,
|
||||
pub file_bytes: u64,
|
||||
pub file_tokens_estimate: u64,
|
||||
}
|
||||
|
||||
impl UsageRecordSnapshot {
|
||||
pub fn from_bytes(kind: RecordKind, slug: impl Into<String>, bytes: &[u8]) -> Self {
|
||||
Self {
|
||||
kind: kind.as_str().to_string(),
|
||||
slug: slug.into(),
|
||||
file_bytes: bytes.len() as u64,
|
||||
file_tokens_estimate: estimate_tokens(bytes.len()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageEvent {
|
||||
pub id: Uuid,
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
pub session_id: String,
|
||||
pub event: UsageEventKind,
|
||||
pub source: UsageSource,
|
||||
pub records: Vec<UsageRecordSnapshot>,
|
||||
}
|
||||
|
||||
impl UsageEvent {
|
||||
pub fn new(
|
||||
session_id: impl Into<String>,
|
||||
event: UsageEventKind,
|
||||
source: UsageSource,
|
||||
records: Vec<UsageRecordSnapshot>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: Uuid::now_v7(),
|
||||
occurred_at: Utc::now(),
|
||||
session_id: session_id.into(),
|
||||
event,
|
||||
source,
|
||||
records,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReport {
|
||||
pub records: Vec<UsageReportRecord>,
|
||||
}
|
||||
|
||||
impl UsageReport {
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
records: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.records.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UsageReportRecord {
|
||||
pub kind: String,
|
||||
pub slug: String,
|
||||
pub use_count: u64,
|
||||
pub last_used_at: Option<DateTime<Utc>>,
|
||||
pub source_breakdown: BTreeMap<String, u64>,
|
||||
pub resident_exposure_count: u64,
|
||||
pub estimated_tokens_per_injection: u64,
|
||||
pub estimated_total_resident_exposure_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Accumulator {
|
||||
kind: String,
|
||||
slug: String,
|
||||
use_count: u64,
|
||||
last_used_at: Option<DateTime<Utc>>,
|
||||
source_breakdown: BTreeMap<String, u64>,
|
||||
resident_exposure_count: u64,
|
||||
last_resident_tokens: u64,
|
||||
}
|
||||
|
||||
/// Append one usage event to the workspace-local JSONL log.
|
||||
pub fn append_usage_event(layout: &WorkspaceLayout, event: &UsageEvent) -> io::Result<()> {
|
||||
let path = layout.usage_events_path();
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let line = serde_json::to_string(event)
|
||||
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
|
||||
let mut file = OpenOptions::new().create(true).append(true).open(path)?;
|
||||
writeln!(file, "{line}")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Convenience for a successful explicit record read.
|
||||
pub fn append_use_event(
|
||||
layout: &WorkspaceLayout,
|
||||
session_id: impl Into<String>,
|
||||
source: UsageSource,
|
||||
records: Vec<UsageRecordSnapshot>,
|
||||
) -> io::Result<()> {
|
||||
if records.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
append_usage_event(
|
||||
layout,
|
||||
&UsageEvent::new(session_id, UsageEventKind::Use, source, records),
|
||||
)
|
||||
}
|
||||
|
||||
/// Convenience for resident model-invocation exposure cost telemetry.
|
||||
pub fn append_resident_exposure_event(
|
||||
layout: &WorkspaceLayout,
|
||||
session_id: impl Into<String>,
|
||||
records: Vec<UsageRecordSnapshot>,
|
||||
) -> io::Result<()> {
|
||||
if records.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
append_usage_event(
|
||||
layout,
|
||||
&UsageEvent::new(
|
||||
session_id,
|
||||
UsageEventKind::ResidentExposure,
|
||||
UsageSource::ResidentInjection,
|
||||
records,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Read a record from the workspace and build the snapshot stored in usage
|
||||
/// events. `slug` should be `"summary"` for [`RecordKind::Summary`].
|
||||
pub fn snapshot_record_from_layout(
|
||||
layout: &WorkspaceLayout,
|
||||
kind: RecordKind,
|
||||
slug: &str,
|
||||
) -> io::Result<UsageRecordSnapshot> {
|
||||
let path = record_path(layout, kind, slug)?;
|
||||
let bytes = fs::read(path)?;
|
||||
Ok(UsageRecordSnapshot::from_bytes(
|
||||
kind,
|
||||
slug.to_string(),
|
||||
&bytes,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn snapshot_record_from_bytes(
|
||||
kind: RecordKind,
|
||||
slug: impl Into<String>,
|
||||
bytes: &[u8],
|
||||
) -> UsageRecordSnapshot {
|
||||
UsageRecordSnapshot::from_bytes(kind, slug, bytes)
|
||||
}
|
||||
|
||||
fn record_path(
|
||||
layout: &WorkspaceLayout,
|
||||
kind: RecordKind,
|
||||
slug: &str,
|
||||
) -> io::Result<std::path::PathBuf> {
|
||||
match kind {
|
||||
RecordKind::Summary => Ok(layout.summary_path()),
|
||||
RecordKind::Decision => {
|
||||
let slug = crate::Slug::parse(slug.to_string()).map_err(invalid_slug_error)?;
|
||||
Ok(layout.decision_path(&slug))
|
||||
}
|
||||
RecordKind::Request => {
|
||||
let slug = crate::Slug::parse(slug.to_string()).map_err(invalid_slug_error)?;
|
||||
Ok(layout.request_path(&slug))
|
||||
}
|
||||
RecordKind::Workflow => {
|
||||
let slug = crate::Slug::parse(slug.to_string()).map_err(invalid_slug_error)?;
|
||||
Ok(layout.workflow_path(&slug))
|
||||
}
|
||||
RecordKind::Knowledge => {
|
||||
let slug = crate::Slug::parse(slug.to_string()).map_err(invalid_slug_error)?;
|
||||
Ok(layout.knowledge_path(&slug))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_slug_error(err: crate::LintError) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, err)
|
||||
}
|
||||
|
||||
/// Aggregate the append-only usage log into per-record evidence.
|
||||
pub fn build_usage_report(layout: &WorkspaceLayout) -> io::Result<UsageReport> {
|
||||
let path = layout.usage_events_path();
|
||||
if !path.exists() {
|
||||
return Ok(UsageReport::empty());
|
||||
}
|
||||
|
||||
let file = fs::File::open(path)?;
|
||||
let reader = io::BufReader::new(file);
|
||||
let mut acc: HashMap<(String, String), Accumulator> = HashMap::new();
|
||||
|
||||
for line in reader.lines() {
|
||||
let line = line?;
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let event: UsageEvent = match serde_json::from_str(&line) {
|
||||
Ok(event) => event,
|
||||
Err(err) => {
|
||||
tracing::warn!(error = %err, "Skipping malformed memory usage event log line");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for record in event.records {
|
||||
let key = (record.kind.clone(), record.slug.clone());
|
||||
let entry = acc.entry(key).or_insert_with(|| Accumulator {
|
||||
kind: record.kind.clone(),
|
||||
slug: record.slug.clone(),
|
||||
..Accumulator::default()
|
||||
});
|
||||
match event.event {
|
||||
UsageEventKind::Use => {
|
||||
entry.use_count += 1;
|
||||
let source = event.source.as_str().to_string();
|
||||
*entry.source_breakdown.entry(source).or_insert(0) += 1;
|
||||
entry.last_used_at = Some(
|
||||
entry
|
||||
.last_used_at
|
||||
.map(|prev| prev.max(event.occurred_at))
|
||||
.unwrap_or(event.occurred_at),
|
||||
);
|
||||
}
|
||||
UsageEventKind::ResidentExposure => {
|
||||
entry.resident_exposure_count += 1;
|
||||
entry.last_resident_tokens = record.file_tokens_estimate;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut records: Vec<UsageReportRecord> = acc
|
||||
.into_values()
|
||||
.map(|a| UsageReportRecord {
|
||||
kind: a.kind,
|
||||
slug: a.slug,
|
||||
use_count: a.use_count,
|
||||
last_used_at: a.last_used_at,
|
||||
source_breakdown: a.source_breakdown,
|
||||
resident_exposure_count: a.resident_exposure_count,
|
||||
estimated_tokens_per_injection: a.last_resident_tokens,
|
||||
estimated_total_resident_exposure_tokens: a
|
||||
.last_resident_tokens
|
||||
.saturating_mul(a.resident_exposure_count),
|
||||
})
|
||||
.collect();
|
||||
records.sort_by(|a, b| (&a.kind, &a.slug).cmp(&(&b.kind, &b.slug)));
|
||||
Ok(UsageReport { records })
|
||||
}
|
||||
|
||||
fn estimate_tokens(bytes: usize) -> u64 {
|
||||
(bytes as u64).div_ceil(4)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn setup() -> (TempDir, WorkspaceLayout) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let layout = WorkspaceLayout::new(dir.path().to_path_buf());
|
||||
(dir, layout)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregates_use_and_resident_exposure_separately() {
|
||||
let (_dir, layout) = setup();
|
||||
let decision = snapshot_record_from_bytes(RecordKind::Decision, "alpha", b"abcd");
|
||||
let knowledge = snapshot_record_from_bytes(RecordKind::Knowledge, "policy", b"abcdefgh");
|
||||
|
||||
append_use_event(
|
||||
&layout,
|
||||
"session-a",
|
||||
UsageSource::MemoryRead,
|
||||
vec![decision.clone()],
|
||||
)
|
||||
.unwrap();
|
||||
append_use_event(
|
||||
&layout,
|
||||
"session-a",
|
||||
UsageSource::KnowledgeRef,
|
||||
vec![knowledge.clone()],
|
||||
)
|
||||
.unwrap();
|
||||
append_use_event(
|
||||
&layout,
|
||||
"session-b",
|
||||
UsageSource::KnowledgeRef,
|
||||
vec![knowledge.clone()],
|
||||
)
|
||||
.unwrap();
|
||||
append_resident_exposure_event(&layout, "session-b", vec![knowledge]).unwrap();
|
||||
|
||||
let report = build_usage_report(&layout).unwrap();
|
||||
let decision = report
|
||||
.records
|
||||
.iter()
|
||||
.find(|r| r.kind == "decision" && r.slug == "alpha")
|
||||
.unwrap();
|
||||
assert_eq!(decision.use_count, 1);
|
||||
assert_eq!(decision.source_breakdown["MemoryRead"], 1);
|
||||
assert_eq!(decision.resident_exposure_count, 0);
|
||||
assert!(decision.last_used_at.is_some());
|
||||
|
||||
let knowledge = report
|
||||
.records
|
||||
.iter()
|
||||
.find(|r| r.kind == "knowledge" && r.slug == "policy")
|
||||
.unwrap();
|
||||
assert_eq!(knowledge.use_count, 2);
|
||||
assert_eq!(knowledge.source_breakdown["KnowledgeRef"], 2);
|
||||
assert_eq!(knowledge.resident_exposure_count, 1);
|
||||
assert_eq!(knowledge.estimated_tokens_per_injection, 2);
|
||||
assert_eq!(knowledge.estimated_total_resident_exposure_tokens, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resident_only_record_does_not_increment_use_count() {
|
||||
let (_dir, layout) = setup();
|
||||
let snapshot = snapshot_record_from_bytes(RecordKind::Knowledge, "policy", b"abcdefgh");
|
||||
append_resident_exposure_event(&layout, "session", vec![snapshot]).unwrap();
|
||||
|
||||
let report = build_usage_report(&layout).unwrap();
|
||||
let record = &report.records[0];
|
||||
assert_eq!(record.use_count, 0);
|
||||
assert!(record.last_used_at.is_none());
|
||||
assert!(record.source_breakdown.is_empty());
|
||||
assert_eq!(record.resident_exposure_count, 1);
|
||||
}
|
||||
}
|
||||
|
|
@ -33,6 +33,8 @@ const SUMMARY_FILE: &str = "summary.md";
|
|||
const DECISIONS_DIR: &str = "decisions";
|
||||
const REQUESTS_DIR: &str = "requests";
|
||||
const STAGING_DIR: &str = "_staging";
|
||||
const USAGE_DIR: &str = "_usage";
|
||||
const USAGE_EVENTS_FILE: &str = "events.jsonl";
|
||||
|
||||
/// What kind of record a path under the memory tree represents.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
|
@ -126,6 +128,14 @@ impl WorkspaceLayout {
|
|||
self.memory_dir().join(STAGING_DIR)
|
||||
}
|
||||
|
||||
pub fn usage_dir(&self) -> PathBuf {
|
||||
self.memory_dir().join(USAGE_DIR)
|
||||
}
|
||||
|
||||
pub fn usage_events_path(&self) -> PathBuf {
|
||||
self.usage_dir().join(USAGE_EVENTS_FILE)
|
||||
}
|
||||
|
||||
pub fn decision_path(&self, slug: &Slug) -> PathBuf {
|
||||
self.decisions_dir().join(format!("{slug}.md"))
|
||||
}
|
||||
|
|
@ -145,7 +155,7 @@ impl WorkspaceLayout {
|
|||
/// Classify a path under the memory tree. Returns `None` if the
|
||||
/// path is not under `.insomnia/memory/`, `.insomnia/knowledge/`,
|
||||
/// or `.insomnia/workflow/` of this workspace, or if it lives in
|
||||
/// `_staging/` (which is opaque to the linter).
|
||||
/// `_staging/` / `_usage/` (opaque subsystem-owned trees).
|
||||
///
|
||||
/// On a conventional path that's *almost* a record but malformed
|
||||
/// (e.g. `.insomnia/memory/decisions/Foo.md` with an invalid slug),
|
||||
|
|
@ -182,8 +192,8 @@ impl WorkspaceLayout {
|
|||
slug: None,
|
||||
}));
|
||||
}
|
||||
if first == STAGING_DIR {
|
||||
// Linter opts out of `_staging/`; extract handles its schema.
|
||||
if first == STAGING_DIR || first == USAGE_DIR {
|
||||
// Linter opts out of subsystem-owned opaque trees.
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
|
|
@ -295,6 +305,14 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_tree_is_opaque_to_classifier() {
|
||||
let cp = layout()
|
||||
.classify(&PathBuf::from("/ws/.insomnia/memory/_usage/events.jsonl"))
|
||||
.unwrap();
|
||||
assert!(cp.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outside_returns_none() {
|
||||
assert!(
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ impl PodController {
|
|||
// `PodFsView` to the shared state once the latter exists.
|
||||
let fs_for_view: tools::ScopedFs;
|
||||
let task_store = pod.task_store();
|
||||
let session_id_for_usage = pod.session_id().to_string();
|
||||
|
||||
let scope_change_sink = pod.scope_change_sink();
|
||||
|
||||
|
|
@ -334,7 +335,10 @@ impl PodController {
|
|||
if let Some(mem) = memory_config.as_ref() {
|
||||
let layout = memory::WorkspaceLayout::resolve(mem, &pwd_for_tools);
|
||||
let query_cfg = memory::tool::QueryConfig::from(mem);
|
||||
worker.register_tool(memory::tool::read_tool(layout.clone()));
|
||||
worker.register_tool(memory::tool::read_tool_with_usage(
|
||||
layout.clone(),
|
||||
session_id_for_usage.clone(),
|
||||
));
|
||||
worker.register_tool(memory::tool::write_tool(layout.clone()));
|
||||
worker.register_tool(memory::tool::edit_tool(layout.clone()));
|
||||
worker.register_tool(memory::tool::memory_query_tool(layout.clone(), query_cfg));
|
||||
|
|
|
|||
|
|
@ -828,17 +828,19 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
return Ok(());
|
||||
};
|
||||
let alerter = self.alerter.clone();
|
||||
let worker = self.worker.as_mut().expect("worker present");
|
||||
// Materialise any pending tool factories so the template sees the
|
||||
// full list of tool names. Redundant with the flush inside
|
||||
// `Worker::lock()`; safe because `flush_pending` is idempotent.
|
||||
worker.tool_server_handle().flush_pending();
|
||||
let tool_names: Vec<String> = worker
|
||||
.tool_server_handle()
|
||||
.tool_definitions_sorted()
|
||||
.into_iter()
|
||||
.map(|d| d.name)
|
||||
.collect();
|
||||
let tool_names: Vec<String> = {
|
||||
let worker = self.worker.as_mut().expect("worker present");
|
||||
// Materialise any pending tool factories so the template sees the
|
||||
// full list of tool names. Redundant with the flush inside
|
||||
// `Worker::lock()`; safe because `flush_pending` is idempotent.
|
||||
worker.tool_server_handle().flush_pending();
|
||||
worker
|
||||
.tool_server_handle()
|
||||
.tool_definitions_sorted()
|
||||
.into_iter()
|
||||
.map(|d| d.name)
|
||||
.collect()
|
||||
};
|
||||
let agents_md_read = read_agents_md(&self.pwd);
|
||||
for warning in agents_md_read.warnings {
|
||||
if let Some(n) = alerter.as_ref() {
|
||||
|
|
@ -875,6 +877,8 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
} else {
|
||||
None
|
||||
};
|
||||
let resident_exposure_snapshots =
|
||||
self.resident_exposure_snapshots(&resident, &resident_workflows);
|
||||
let scope_snapshot = self.scope.snapshot();
|
||||
let ctx = SystemPromptContext {
|
||||
now: chrono::Utc::now(),
|
||||
|
|
@ -889,7 +893,11 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
let rendered = template
|
||||
.render(&ctx)
|
||||
.map_err(|source| PodError::SystemPromptRender { source })?;
|
||||
worker.set_system_prompt(rendered);
|
||||
self.worker
|
||||
.as_mut()
|
||||
.expect("worker present")
|
||||
.set_system_prompt(rendered);
|
||||
self.append_resident_exposure_event(resident_exposure_snapshots);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -979,11 +987,13 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
}
|
||||
self.user_segments.push(input.clone());
|
||||
|
||||
// Resolve `@<path>` refs and `/<slug>` workflow invocations to
|
||||
// system messages stashed for the PodInterceptor to attach right
|
||||
// after the user message. File failures are non-fatal alerts; explicit
|
||||
// workflow invocation failures abort before the Worker sees the turn.
|
||||
// Resolve `@<path>` refs, `#<slug>` Knowledge refs, and `/<slug>`
|
||||
// workflow invocations to system messages stashed for the
|
||||
// PodInterceptor to attach right after the user message. File and
|
||||
// Knowledge failures are non-fatal alerts; explicit workflow invocation
|
||||
// failures abort before the Worker sees the turn.
|
||||
let mut attachments = self.resolve_file_refs(&input);
|
||||
attachments.extend(self.resolve_knowledge_refs(&input));
|
||||
attachments.extend(self.resolve_workflow_invocations(&input)?);
|
||||
if !attachments.is_empty() {
|
||||
*self
|
||||
|
|
@ -1034,6 +1044,127 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
out
|
||||
}
|
||||
|
||||
fn resolve_knowledge_refs(&self, segments: &[Segment]) -> Vec<Item> {
|
||||
let Some(layout) = self.memory_layout.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
for seg in segments {
|
||||
let Segment::KnowledgeRef { slug } = seg else {
|
||||
continue;
|
||||
};
|
||||
let parsed = match memory::Slug::parse(slug.clone()) {
|
||||
Ok(slug) => slug,
|
||||
Err(e) => {
|
||||
self.alert(
|
||||
AlertLevel::Warn,
|
||||
AlertSource::Pod,
|
||||
format!("knowledge ref #{slug} has invalid slug: {e}"),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let path = layout.knowledge_path(&parsed);
|
||||
let bytes = match std::fs::read(&path) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
self.alert(
|
||||
AlertLevel::Warn,
|
||||
AlertSource::Pod,
|
||||
format!("knowledge ref #{slug} could not be read: {e}"),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let raw = String::from_utf8_lossy(&bytes).into_owned();
|
||||
let body = match memory::schema::split_frontmatter(&raw) {
|
||||
Ok((_yaml, body)) => body,
|
||||
Err(e) => {
|
||||
self.alert(
|
||||
AlertLevel::Warn,
|
||||
AlertSource::Pod,
|
||||
format!("knowledge ref #{slug} has invalid frontmatter: {e}"),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let snapshot = memory::snapshot_record_from_bytes(
|
||||
memory::workspace::RecordKind::Knowledge,
|
||||
slug.clone(),
|
||||
&bytes,
|
||||
);
|
||||
self.append_memory_use_event(memory::UsageSource::KnowledgeRef, vec![snapshot]);
|
||||
out.push(Item::system_message(format!(
|
||||
"[Knowledge #{}]\n{}",
|
||||
slug,
|
||||
body.trim_end()
|
||||
)));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn resident_exposure_snapshots(
|
||||
&self,
|
||||
knowledge: &[memory::ResidentKnowledgeEntry],
|
||||
workflows: &[memory::ResidentWorkflowEntry],
|
||||
) -> Vec<memory::UsageRecordSnapshot> {
|
||||
let Some(layout) = self.memory_layout.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut snapshots = Vec::new();
|
||||
for entry in knowledge {
|
||||
match memory::snapshot_record_from_layout(
|
||||
layout,
|
||||
memory::workspace::RecordKind::Knowledge,
|
||||
&entry.slug,
|
||||
) {
|
||||
Ok(snapshot) => snapshots.push(snapshot),
|
||||
Err(err) => {
|
||||
warn!(knowledge = %entry.slug, error = %err, "failed to snapshot resident knowledge exposure")
|
||||
}
|
||||
}
|
||||
}
|
||||
for entry in workflows {
|
||||
match memory::snapshot_record_from_layout(
|
||||
layout,
|
||||
memory::workspace::RecordKind::Workflow,
|
||||
&entry.slug,
|
||||
) {
|
||||
Ok(snapshot) => snapshots.push(snapshot),
|
||||
Err(err) => {
|
||||
warn!(workflow = %entry.slug, error = %err, "failed to snapshot resident workflow exposure")
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshots
|
||||
}
|
||||
|
||||
fn append_memory_use_event(
|
||||
&self,
|
||||
source: memory::UsageSource,
|
||||
records: Vec<memory::UsageRecordSnapshot>,
|
||||
) {
|
||||
let Some(layout) = self.memory_layout.as_ref() else {
|
||||
return;
|
||||
};
|
||||
if let Err(err) =
|
||||
memory::append_use_event(layout, self.session_id.to_string(), source, records)
|
||||
{
|
||||
warn!(error = %err, "failed to append memory usage event");
|
||||
}
|
||||
}
|
||||
|
||||
fn append_resident_exposure_event(&self, records: Vec<memory::UsageRecordSnapshot>) {
|
||||
let Some(layout) = self.memory_layout.as_ref() else {
|
||||
return;
|
||||
};
|
||||
if let Err(err) =
|
||||
memory::append_resident_exposure_event(layout, self.session_id.to_string(), records)
|
||||
{
|
||||
warn!(error = %err, "failed to append resident exposure event");
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_workflow_invocations(
|
||||
&self,
|
||||
segments: &[Segment],
|
||||
|
|
@ -1057,6 +1188,21 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
layout,
|
||||
slug,
|
||||
)?;
|
||||
match memory::snapshot_record_from_layout(
|
||||
layout,
|
||||
memory::workspace::RecordKind::Workflow,
|
||||
slug,
|
||||
) {
|
||||
Ok(snapshot) => {
|
||||
self.append_memory_use_event(
|
||||
memory::UsageSource::WorkflowInvoke,
|
||||
vec![snapshot],
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(workflow = %slug, error = %err, "failed to snapshot workflow usage");
|
||||
}
|
||||
}
|
||||
out.extend(items);
|
||||
}
|
||||
Ok(out)
|
||||
|
|
@ -1103,14 +1249,16 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
match seg {
|
||||
Segment::Text { .. } | Segment::Paste { .. } | Segment::FileRef { .. } => {}
|
||||
Segment::KnowledgeRef { slug } => {
|
||||
self.alert(
|
||||
AlertLevel::Warn,
|
||||
AlertSource::Pod,
|
||||
format!(
|
||||
"knowledge ref #{slug} cannot be resolved \
|
||||
(resolver not yet implemented); passed to LLM as placeholder"
|
||||
),
|
||||
);
|
||||
if self.memory_layout.is_none() {
|
||||
self.alert(
|
||||
AlertLevel::Warn,
|
||||
AlertSource::Pod,
|
||||
format!(
|
||||
"knowledge ref #{slug} cannot be resolved \
|
||||
because memory is disabled; passed to LLM as placeholder"
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
Segment::WorkflowInvoke { .. } => {}
|
||||
Segment::Unknown => {
|
||||
|
|
@ -2139,7 +2287,10 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
// のKnowledgeアクセス (agent pulls knowledge through the search
|
||||
// tool instead of via system-prompt residency).
|
||||
let query_cfg = memory::tool::QueryConfig::from(memory_cfg);
|
||||
worker.register_tool(memory::tool::read_tool(layout.clone()));
|
||||
worker.register_tool(memory::tool::read_tool_with_usage(
|
||||
layout.clone(),
|
||||
self.session_id.to_string(),
|
||||
));
|
||||
worker.register_tool(memory::tool::write_tool(layout.clone()));
|
||||
worker.register_tool(memory::tool::edit_tool(layout.clone()));
|
||||
worker.register_tool(memory::tool::memory_query_tool(layout.clone(), query_cfg));
|
||||
|
|
@ -2149,9 +2300,15 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
|
|||
));
|
||||
|
||||
let tidy = consolidate::collect_tidy_hints(&layout);
|
||||
let candidates = consolidate::KnowledgeCandidateReport::empty();
|
||||
let usage_report = match memory::build_usage_report(&layout) {
|
||||
Ok(report) => report,
|
||||
Err(err) => {
|
||||
warn!(error = %err, "failed to build memory usage report for consolidation");
|
||||
memory::UsageReport::empty()
|
||||
}
|
||||
};
|
||||
let input_text =
|
||||
consolidate::build_consolidate_input(&layout, &entries, &tidy, &candidates);
|
||||
consolidate::build_consolidate_input(&layout, &entries, &tidy, &usage_report);
|
||||
|
||||
let run_result = worker.run(input_text).await;
|
||||
match run_result {
|
||||
|
|
|
|||
|
|
@ -1,140 +0,0 @@
|
|||
# メモリ機構: 明示使用回数ログ
|
||||
|
||||
## 背景
|
||||
|
||||
memory / Knowledge の改善で必要なのは、「どの record が実際に context に読まれたか」を後から確認できる観測ログである。
|
||||
|
||||
検索結果に出たことや session あたりの浮上率は、session の目的や query の広さに強く依存し、record が読まれたことも意味しない。`n/Mtoken` のような線形スコアも、Knowledge の価値や使用実態を表す軸としては不安定である。
|
||||
|
||||
本チケットでは、判断ロジックではなく **明示使用回数の計測** を実装する。Knowledge 化、整理時の保護、`model_invokation` ON/OFF の判断は、このログを材料に consolidation / Doctor / prompt-eval 側で行う。
|
||||
|
||||
## 方針
|
||||
|
||||
- workspace 側に append-only な usage event log を残す
|
||||
- staging や session-local state と独立させ、session データ喪失で統計が消えないようにする
|
||||
- 主指標は record が明示的に context に読まれた回数に限定する
|
||||
- 検索 hit / query result surface は主指標にしない。初期実装では記録しなくてよい
|
||||
- session 浮上率や `n/Mtoken` は採用しない
|
||||
- `model_invokation` 常駐注入は「使用回数」には含めず、resident exposure cost として別口で記録する
|
||||
|
||||
## 使用回数として数える event
|
||||
|
||||
### `use`
|
||||
|
||||
record が明示的に context に取り込まれたことを表す。
|
||||
|
||||
対象:
|
||||
|
||||
- `MemoryRead`
|
||||
- `#<slug>` 完全一致参照
|
||||
- `/workflow` で workflow 自体が呼び出された場合の workflow record
|
||||
- 将来追加される明示的な Knowledge / Workflow read 経路
|
||||
|
||||
補足:
|
||||
|
||||
- `MemoryQuery` / `KnowledgeQuery` の検索結果に出ただけでは count しない
|
||||
- LLM が検索後に自分で `MemoryRead` した場合は、その read を count する
|
||||
- 同一 session 内での連続参照 coalescing は初期実装では必須にしない。必要になった時に report 側で補助的に扱う
|
||||
|
||||
### `resident_exposure`
|
||||
|
||||
`model_invokation: true` 等により常駐注入されたことを表す。これは使用回数ではなく cost 観測である。
|
||||
|
||||
対象:
|
||||
|
||||
- resident knowledge injection
|
||||
|
||||
用途:
|
||||
|
||||
- record token size / injection count / 推定 total exposure cost の把握
|
||||
- 明示使用が少ない高コスト常駐 record の発見
|
||||
- usefulness の分子には含めない
|
||||
|
||||
## event schema
|
||||
|
||||
具体形式は実装で決定するが、最低限以下を保持する。
|
||||
|
||||
```text
|
||||
id: uuid
|
||||
occurred_at
|
||||
session_id
|
||||
event: use | resident_exposure
|
||||
source: MemoryRead | KnowledgeRef | WorkflowInvoke | ResidentInjection
|
||||
records[]:
|
||||
kind
|
||||
slug
|
||||
file_bytes
|
||||
file_tokens_estimate
|
||||
```
|
||||
|
||||
`input_tokens` や prompt occupancy は、取得できるなら補助メタデータとして残してよい。ただし集計・判定の主軸にしない。
|
||||
|
||||
## report
|
||||
|
||||
consolidation / Doctor / prompt-eval から読める集計 API を提供する。
|
||||
|
||||
record ごとに最低限以下を出す。
|
||||
|
||||
```text
|
||||
kind
|
||||
slug
|
||||
use_count
|
||||
last_used_at
|
||||
source_breakdown
|
||||
resident_exposure_count
|
||||
estimated_tokens_per_injection
|
||||
estimated_total_resident_exposure_tokens
|
||||
```
|
||||
|
||||
この report は hard decision ではなく、判断材料である。
|
||||
|
||||
- Knowledge 化候補: `decision` / `request` のうち、明示使用回数が多いものを確認対象にする
|
||||
- tidy protection: 明示使用された record は削除 / 大幅圧縮時に追加確認する
|
||||
- resident 見直し: resident exposure cost が高く、明示使用が少ない record を確認対象にする
|
||||
|
||||
## 現 worktree 実装の扱い
|
||||
|
||||
`.worktree/memory-usage-metrics` の実装から残してよいもの:
|
||||
|
||||
- workspace-local append-only JSONL event log
|
||||
- `MemoryRead` hook
|
||||
- `#<slug>` / `/workflow` 経路の hook
|
||||
- resident injection を明示使用と別イベントにする配線
|
||||
- record snapshot に file bytes / token estimate を持つこと
|
||||
- report API の土台
|
||||
|
||||
作り替えるもの:
|
||||
|
||||
- `ExplicitInvoke` という単一分類
|
||||
- `MemoryQuery` / `KnowledgeQuery` hit を主指標として記録する設計
|
||||
- `frequency_per_mtoken`
|
||||
- cumulative input token を前提にした window 集計
|
||||
- session 浮上率や複数 session 要件を hard metric にする設計
|
||||
- `usage_candidate_frequency_threshold` / `usage_protection_frequency_threshold` 系の設定
|
||||
- frequency threshold による Knowledge 化候補 / protection 判定
|
||||
|
||||
## 範囲外
|
||||
|
||||
- Doctor / prompt-eval の実装本体（`tickets/prompt-eval-metrics.md`）
|
||||
- Knowledge の自動作成
|
||||
- `model_invokation` ON/OFF の完全自動切替
|
||||
- query hit の計測
|
||||
- session 浮上率の算出
|
||||
- record が model output に実際に使われたかの自動判定
|
||||
|
||||
## 完了条件
|
||||
|
||||
- 明示使用 event が workspace 側に append-only に積まれる
|
||||
- `MemoryRead` / `#<slug>` / `/workflow` 経路が使用回数として count される
|
||||
- `MemoryQuery` / `KnowledgeQuery` の検索 hit が使用回数に含まれていない
|
||||
- resident exposure が使用回数とは別口で記録される
|
||||
- record ごとの `use_count` / `last_used_at` / `source_breakdown` / resident cost を集計できる
|
||||
- consolidation / Doctor 側から report API を呼べる
|
||||
- `n/Mtoken` や session 浮上率を Knowledge 化候補や tidy protection の主判断として使っていない
|
||||
|
||||
## 参照
|
||||
|
||||
- `docs/plan/memory.md` §使用頻度メトリクス / §判断ルール / §retrieval 経路
|
||||
- `tickets/memory-search-tools.md`（hook 挿入点）
|
||||
- `tickets/memory-consolidation.md`（統合 / 整理 両 step の消費者）
|
||||
- `tickets/prompt-eval-metrics.md`（Doctor / prompt-eval 側の事後評価）
|
||||
Loading…
Reference in New Issue
Block a user