workdir: add network-capable operation boundary

This commit is contained in:
2026-08-03 16:14:01 +09:00
parent 0fa36395e7
commit ddadc830ac
32 changed files with 3590 additions and 3241 deletions
Generated
+22 -5
View File
@@ -4494,12 +4494,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"filetime", "filetime",
"globset",
"grep-matcher",
"grep-regex",
"grep-searcher",
"html5ever", "html5ever",
"ignore",
"llm-engine", "llm-engine",
"manifest", "manifest",
"markup5ever_rcdom", "markup5ever_rcdom",
@@ -4514,6 +4509,7 @@ dependencies = [
"thiserror 2.0.18", "thiserror 2.0.18",
"tokio", "tokio",
"tracing", "tracing",
"workdir",
] ]
[[package]] [[package]]
@@ -5923,6 +5919,25 @@ dependencies = [
"wasmparser 0.248.0", "wasmparser 0.248.0",
] ]
[[package]]
name = "workdir"
version = "0.1.0"
dependencies = [
"async-trait",
"globset",
"grep-matcher",
"grep-regex",
"grep-searcher",
"ignore",
"manifest",
"serde",
"serde_json",
"sha2 0.11.0",
"tempfile",
"thiserror 2.0.18",
"tokio",
]
[[package]] [[package]]
name = "worker" name = "worker"
version = "0.1.0" version = "0.1.0"
@@ -5964,6 +5979,7 @@ dependencies = [
"uuid", "uuid",
"wasmtime", "wasmtime",
"wat", "wat",
"workdir",
"yoi-plugin-pdk", "yoi-plugin-pdk",
] ]
@@ -5992,6 +6008,7 @@ dependencies = [
"tokio-tungstenite 0.29.0", "tokio-tungstenite 0.29.0",
"toml", "toml",
"tower", "tower",
"workdir",
"worker", "worker",
] ]
+3
View File
@@ -17,6 +17,7 @@ members = [
"crates/session-analytics", "crates/session-analytics",
"crates/lint-common", "crates/lint-common",
"crates/tools", "crates/tools",
"crates/workdir",
"crates/tui", "crates/tui",
"crates/memory", "crates/memory",
"crates/ticket", "crates/ticket",
@@ -41,6 +42,7 @@ default-members = [
"crates/session-analytics", "crates/session-analytics",
"crates/lint-common", "crates/lint-common",
"crates/tools", "crates/tools",
"crates/workdir",
"crates/tui", "crates/tui",
"crates/memory", "crates/memory",
"crates/ticket", "crates/ticket",
@@ -73,6 +75,7 @@ session-analytics = { path = "crates/session-analytics" }
session-store = { path = "crates/session-store" } session-store = { path = "crates/session-store" }
secrets = { path = "crates/secrets" } secrets = { path = "crates/secrets" }
tools = { path = "crates/tools" } tools = { path = "crates/tools" }
workdir = { path = "crates/workdir" }
tui = { path = "crates/tui" } tui = { path = "crates/tui" }
yoi-workspace-server = { path = "crates/workspace-server" } yoi-workspace-server = { path = "crates/workspace-server" }
+4 -4
View File
@@ -421,14 +421,14 @@ impl Scope {
/// Shared, atomically-swappable view of a [`Scope`]. /// Shared, atomically-swappable view of a [`Scope`].
/// ///
/// Built around [`ArcSwap`] so the hot path (permission checks inside /// Built around [`ArcSwap`] so the hot path (permission checks inside a local
/// `ScopedFs`) reads the current scope lock-free. Mutators are /// Workdir provider) reads the current scope lock-free. Mutators are
/// serialised by an internal `Mutex` so concurrent `update` calls do /// serialised by an internal `Mutex` so concurrent `update` calls do
/// not lose each other's contributions. /// not lose each other's contributions.
/// ///
/// All clones share the same underlying state — a `SharedScope` cloned /// All clones share the same underlying state — a `SharedScope` cloned
/// out to multiple consumers (Worker, ScopedFs, future grant/revoke /// out to multiple consumers (Worker, local Workdir providers, future
/// callers) sees every update. /// grant/revoke callers) sees every update.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct SharedScope { pub struct SharedScope {
inner: Arc<SharedScopeInner>, inner: Arc<SharedScopeInner>,
+3 -3
View File
@@ -3,9 +3,9 @@
//! //!
//! Worker is expected to call [`deny_write_rules`] when memory is enabled //! Worker is expected to call [`deny_write_rules`] when memory is enabled
//! and append the result to the manifest's `scope.deny` list before //! and append the result to the manifest's `scope.deny` list before
//! constructing the [`Scope`] passed to `tools::ScopedFs`. The memory //! constructing the [`Scope`] passed to the local Workdir provider. The
//! tools themselves bypass `ScopedFs` and write directly under the //! memory tools themselves bypass generic Workdir filesystem operations and
//! workspace root, so this deny does not affect their operation. //! write directly under the workspace root, so this deny does not affect them.
use std::path::Path; use std::path::Path;
+1 -5
View File
@@ -6,11 +6,6 @@ license.workspace = true
[dependencies] [dependencies]
async-trait = { workspace = true } async-trait = { workspace = true }
globset = "0.4.18"
grep-matcher = "0.1.8"
grep-regex = "0.1.14"
grep-searcher = "0.1.16"
ignore = "0.4.25"
html5ever = "0.26" html5ever = "0.26"
llm-engine = { workspace = true } llm-engine = { workspace = true }
manifest = { workspace = true } manifest = { workspace = true }
@@ -26,6 +21,7 @@ tempfile = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
tokio = { workspace = true, features = ["process", "rt", "sync", "time"] } tokio = { workspace = true, features = ["process", "rt", "sync", "time"] }
tracing = { workspace = true } tracing = { workspace = true }
workdir = { workspace = true }
[dev-dependencies] [dev-dependencies]
filetime = "0.2.27" filetime = "0.2.27"
+69 -560
View File
@@ -1,100 +1,39 @@
//! `Bash` tool — execute shell commands in a one-shot, stateless way. use std::path::PathBuf;
//!
//! Each call runs `bash -c <command>` via [`tokio::process::Command`].
//! The wrapper redirects all output to a file so we never have to read
//! from a pipe (which would expose us to bg-pipe hangs). There is no
//! shell session: every call starts fresh at `cwd`, so the agent must
//! chain `cd <dir> && cmd` when it wants to operate elsewhere. This
//! mirrors Claude Code's own Bash tool — predictable, no hidden state.
//!
//! Output handling: when output is short (≤ 80 lines, ≤ 12 KiB) it is
//! returned inline and the file is cleaned up. When it is longer the
//! full output is left on disk and only the **last 80 lines** are
//! returned, prefixed with the saved file's path. This sidesteps the
//! Engine's blanket `ToolOutputLimits` (default 64 KiB), which would
//! otherwise drop the *tail* of the output — usually the most useful
//! part (errors, exit messages, summary). The saved file lives under
//! a caller-supplied directory that the parent has added to the
//! `ScopedFs` allow set, so the agent can inspect it via either Read
//! or a follow-up Bash call.
//!
//! Filesystem and network access are NOT mediated by `ScopedFs`: the
//! child process can touch any path. Safety is delegated to the
//! Permission layer (deny/allow rules on the command string).
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait; use async_trait::async_trait;
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput}; use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use schemars::JsonSchema;
use serde::Deserialize; use serde::Deserialize;
use tokio::process::Command; use workdir::{CommandHandle, CommandOutputRequest, CommandRequest, WorkdirHandle};
use crate::scoped_fs::ScopedFs;
const DESCRIPTION: &str = "Execute a shell command via bash. Supports the \
full shell — pipes, redirects, command substitution, `&&`/`||`. Each call \
runs in a fresh shell rooted at the workspace; chain `cd <subdir> && cmd` \
when you need to operate elsewhere. stdout and stderr are merged. Default \
timeout 120s, max 600s.\n\n\
Output handling: when the command produces more than 80 lines (or ~12 KiB), \
the full output is saved to a file and only the LAST 80 lines are returned, \
prefixed with the saved path. The path is readable by Read; you can also \
inspect it from a follow-up Bash call (`grep ... <path>`, etc.).\n\n\
Prefer dedicated tools when one fits: Read instead of `cat`/`head`/`tail` \
on workspace files, Edit instead of `sed`/`awk` rewrites, Glob instead of \
`find <name>`, Grep instead of `grep`/`rg`. Reach for Bash when the task \
is shell-shaped: building, testing, version control, package management.";
const DEFAULT_TIMEOUT_SECS: u64 = 120; const DEFAULT_TIMEOUT_SECS: u64 = 120;
const MAX_TIMEOUT_SECS: u64 = 600; const MAX_TIMEOUT_SECS: u64 = 600;
/// Number of trailing lines returned when output spills to a file.
const TAIL_LINES: usize = 80;
/// Inline-return budget. Outputs at or below this are returned in full;
/// above it triggers the spill-to-file path. Sized to leave headroom under
/// the Engine's 64 KiB default `ToolOutputLimits` cap so the inline path
/// reliably reaches the model intact.
const INLINE_BYTE_BUDGET: usize = 12 * 1024; const INLINE_BYTE_BUDGET: usize = 12 * 1024;
/// Maximum bytes loaded into memory from the spilled output file. The #[derive(Debug, Deserialize, JsonSchema)]
/// file itself can be arbitrarily large; we only ever read the tail end struct BashParams {
/// since that is what we return. command: String,
const TAIL_READ_BUDGET: usize = 256 * 1024;
#[derive(Debug, Deserialize, schemars::JsonSchema)]
pub(crate) struct BashParams {
/// Shell command to execute. Passed verbatim to `bash -c`.
pub command: String,
/// Timeout in seconds. Defaults to 120, capped at 600.
#[serde(default)] #[serde(default)]
pub timeout: Option<u64>, timeout: Option<u64>,
} }
pub(crate) struct BashTool { pub(crate) struct BashTool {
/// Workspace root that every invocation starts in. Snapshot of workdir: WorkdirHandle,
/// `ScopedFs::cwd()` at registration time; never mutated, since we
/// don't track `cd` across calls.
cwd: PathBuf,
/// Directory to spill long outputs into. Caller is expected to have
/// added this path to the readable scope so the agent can Read the
/// saved files. The directory itself is created lazily.
output_dir: PathBuf,
/// Files we left on disk for follow-up inspection. Cleaned up on
/// `Drop` (= session end). `std::sync::Mutex` because access is
/// always synchronous and very brief.
spilled_outputs: std::sync::Mutex<Vec<PathBuf>>,
} }
impl Drop for BashTool { struct CommandGuard {
workdir: WorkdirHandle,
handle: Option<CommandHandle>,
}
impl Drop for CommandGuard {
fn drop(&mut self) { fn drop(&mut self) {
if let Ok(mut paths) = self.spilled_outputs.lock() { if let Some(handle) = self.handle.take() {
for p in paths.drain(..) { let workdir = self.workdir.clone();
let _ = std::fs::remove_file(&p); tokio::spawn(async move {
} let _ = workdir.cancel_command(handle).await;
});
} }
} }
} }
@@ -107,509 +46,79 @@ impl Tool for BashTool {
_ctx: llm_engine::tool::ToolExecutionContext, _ctx: llm_engine::tool::ToolExecutionContext,
) -> Result<ToolOutput, ToolError> { ) -> Result<ToolOutput, ToolError> {
let params: BashParams = serde_json::from_str(input_json) let params: BashParams = serde_json::from_str(input_json)
.map_err(|e| ToolError::InvalidArgument(format!("invalid Bash input: {e}")))?; .map_err(|error| ToolError::InvalidArgument(format!("invalid Bash input: {error}")))?;
let timeout_secs = params let timeout_secs = params
.timeout .timeout
.unwrap_or(DEFAULT_TIMEOUT_SECS) .unwrap_or(DEFAULT_TIMEOUT_SECS)
.clamp(1, MAX_TIMEOUT_SECS); .clamp(1, MAX_TIMEOUT_SECS);
// Persistent output file in the caller-supplied directory.
// `keep()` opts out of auto-delete so the agent can inspect the
// full output later; cleanup is deferred to `Drop` on this tool.
std::fs::create_dir_all(&self.output_dir).map_err(|e| {
ToolError::Internal(format!(
"create bash output dir {}: {e}",
self.output_dir.display()
))
})?;
let output_path: PathBuf = tempfile::Builder::new()
.prefix("bash-")
.suffix(".log")
.tempfile_in(&self.output_dir)
.map_err(|e| ToolError::Internal(format!("output tempfile: {e}")))?
.into_temp_path()
.keep()
.map_err(|e| ToolError::Internal(format!("persist output tempfile: {e}")))?;
let output_path_str = output_path
.to_str()
.ok_or_else(|| ToolError::Internal("output path is not UTF-8".into()))?;
// Wrapper:
// exec >file 2>&1 redirect stdout/stderr to the output file
// { user_cmd } run in a brace group (no subshell, so any
// `cd` inside still affects $? capture below)
// __exit=$? preserve the user command's exit code…
// wait 2>/dev/null …since `wait` clobbers $?. Reaping bg jobs
// guarantees the output file's writers all
// close before bash itself exits.
// exit $__exit propagate the user's exit
let wrapped = format!(
"exec >{out} 2>&1\n{{ {user_cmd}\n}}\n__yoi_exit=$?\nwait 2>/dev/null\nexit $__yoi_exit\n",
out = shell_single_quote(output_path_str),
user_cmd = params.command,
);
tracing::debug!(cmd = %params.command, cwd = %self.cwd.display(), timeout_secs, "Bash");
let mut child = Command::new("bash")
.arg("-c")
.arg(&wrapped)
.current_dir(&self.cwd)
.stdin(Stdio::null())
.stdout(Stdio::null()) // bash inherits — but the wrapper redirected via `exec`
.stderr(Stdio::null())
.kill_on_drop(true)
.spawn()
.map_err(|e| {
let _ = std::fs::remove_file(&output_path);
ToolError::ExecutionFailed(format!("spawn bash: {e}"))
})?;
let timeout_dur = Duration::from_secs(timeout_secs);
let wait_result = tokio::time::timeout(timeout_dur, child.wait()).await;
let (status, timed_out) = match wait_result {
Ok(Ok(s)) => (Some(s), false),
Ok(Err(e)) => {
let _ = std::fs::remove_file(&output_path);
return Err(ToolError::ExecutionFailed(format!("bash wait: {e}")));
}
Err(_) => (None, true),
};
// Inspect the on-disk output: total size first, tail bytes second.
let total_bytes = std::fs::metadata(&output_path)
.map(|m| m.len() as usize)
.unwrap_or(0);
let tail_bytes = read_tail_bytes(&output_path, TAIL_READ_BUDGET).unwrap_or_default();
let tail_text = String::from_utf8_lossy(&tail_bytes).into_owned();
let cmd_summary = truncate_for_summary(&params.command); let cmd_summary = truncate_for_summary(&params.command);
let handle = self
.workdir
.start_command(CommandRequest {
command: params.command,
timeout_secs,
output_limit: INLINE_BYTE_BUDGET,
})
.await
.map_err(crate::ToolsError::from)?;
let mut guard = CommandGuard {
workdir: self.workdir.clone(),
handle: Some(handle.clone()),
};
let output = self
.workdir
.command_output(CommandOutputRequest {
handle,
cursor: 0,
limit: INLINE_BYTE_BUDGET,
wait: true,
})
.await
.map_err(crate::ToolsError::from)?;
guard.handle = None;
if timed_out { let summary = if output.timed_out {
// Preserve the partial output file — even cut-short logs help format!("$ {cmd_summary} (timed out after {timeout_secs}s)")
// diagnose hangs. } else {
let content = if total_bytes > 0 { match output.exit_code {
let last = take_last_n_lines(&tail_text, TAIL_LINES); Some(0) => format!("$ {cmd_summary}"),
self.remember_spilled(&output_path); Some(code) => format!("$ {cmd_summary} (exit {code})"),
None => format!("$ {cmd_summary} (terminated)"),
}
};
let content = if output.content.is_empty() {
None
} else if output.truncated {
Some(format!( Some(format!(
"[partial output before timeout — full at {}]\n{last}", "[showing bounded Workdir command output; additional output was truncated]\n{}",
output_path.display() output.content
)) ))
} else { } else {
let _ = std::fs::remove_file(&output_path); Some(output.content)
None
}; };
return Ok(ToolOutput {
summary: format!("$ {cmd_summary} (timed out after {timeout_secs}s)"),
content,
});
}
let status = status.expect("status set on the success branch");
let summary = match status.code() {
Some(0) => format!("$ {cmd_summary}"),
Some(c) => format!("$ {cmd_summary} (exit {c})"),
None => format!("$ {cmd_summary} (terminated by signal)"),
};
if total_bytes == 0 {
let _ = std::fs::remove_file(&output_path);
return Ok(ToolOutput {
summary,
content: None,
});
}
// Inline if the whole output fits in our tail-read window AND is
// small enough to ride under the Engine's default cap.
let line_count = tail_text.lines().count();
let fully_loaded = total_bytes <= tail_bytes.len();
let fits_inline =
fully_loaded && total_bytes <= INLINE_BYTE_BUDGET && line_count <= TAIL_LINES;
let content = if fits_inline {
let _ = std::fs::remove_file(&output_path);
Some(tail_text)
} else {
let last = take_last_n_lines(&tail_text, TAIL_LINES);
// When `fully_loaded` we know the exact line count; otherwise
// the file is bigger than our read window so we report bytes
// and an "approximate" disclaimer.
let header = if fully_loaded {
format!(
"[showing last {TAIL_LINES} of {line_count} lines — full output ({total_bytes} bytes) at {}]",
output_path.display()
)
} else {
format!(
"[showing last {TAIL_LINES} lines (tail of {total_bytes}-byte output) — full at {}]",
output_path.display()
)
};
self.remember_spilled(&output_path);
Some(format!("{header}\n{last}"))
};
Ok(ToolOutput { summary, content }) Ok(ToolOutput { summary, content })
} }
} }
impl BashTool {
fn remember_spilled(&self, path: &Path) {
if let Ok(mut v) = self.spilled_outputs.lock() {
v.push(path.to_path_buf());
}
}
}
/// Read up to `max_bytes` from the end of `path`. If the file is smaller
/// than `max_bytes`, the entire file is returned.
fn read_tail_bytes(path: &Path, max_bytes: usize) -> std::io::Result<Vec<u8>> {
use std::io::{Read, Seek, SeekFrom};
let mut f = std::fs::File::open(path)?;
let len = f.seek(SeekFrom::End(0))?;
let start = if len > max_bytes as u64 {
len - max_bytes as u64
} else {
0
};
f.seek(SeekFrom::Start(start))?;
let mut buf = Vec::with_capacity((len - start) as usize);
f.read_to_end(&mut buf)?;
Ok(buf)
}
/// Return the last `n` lines of `text`. If `text` has `n` or fewer lines
/// (per [`str::lines`]), the input is returned as-is (no allocation).
fn take_last_n_lines(text: &str, n: usize) -> String {
if text.is_empty() {
return String::new();
}
let total = text.lines().count();
if total <= n {
return text.to_owned();
}
let skip = total - n;
let mut count = 0usize;
for (i, b) in text.bytes().enumerate() {
if b == b'\n' {
count += 1;
if count == skip {
return text[i + 1..].to_owned();
}
}
}
text.to_owned()
}
fn truncate_for_summary(command: &str) -> String { fn truncate_for_summary(command: &str) -> String {
let one_line = command.lines().next().unwrap_or(""); const MAX: usize = 100;
let mut chars = one_line.chars(); if command.chars().count() <= MAX {
let head: String = chars.by_ref().take(80).collect(); return command.to_owned();
if chars.next().is_some() {
let mut shortened = head;
while shortened.chars().count() > 77 {
shortened.pop();
}
shortened.push_str("...");
shortened
} else {
head
} }
let mut summary = command.chars().take(MAX - 1).collect::<String>();
summary.push('…');
summary
} }
/// Wrap a string in single quotes for safe inclusion in a bash command. pub fn bash_tool(workdir: WorkdirHandle, _output_dir: PathBuf) -> ToolDefinition {
fn shell_single_quote(s: &str) -> String {
let escaped = s.replace('\'', "'\\''");
format!("'{escaped}'")
}
/// Factory for the `Bash` tool.
///
/// `output_dir` is where long outputs spill to; the caller is responsible
/// for arranging that the path is in the agent's readable scope. Every
/// invocation starts at `fs.cwd()` — the tool is intentionally stateless
/// w.r.t. the working directory.
pub fn bash_tool(fs: ScopedFs, output_dir: PathBuf) -> ToolDefinition {
Arc::new(move || { Arc::new(move || {
let schema = schemars::schema_for!(BashParams); let schema = schemars::schema_for!(BashParams);
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
let meta = ToolMeta::new("Bash") let meta = ToolMeta::new("Bash")
.description(DESCRIPTION) .description("Execute a shell command in the bound Workdir. Process start, bounded output, timeout and cancellation are owned by the Workdir provider. This is not a sandbox.")
.input_schema(schema_value); .input_schema(serde_json::to_value(schema).expect("Bash schema serialization"));
let tool: Arc<dyn Tool> = Arc::new(BashTool { let tool: Arc<dyn Tool> = Arc::new(BashTool {
cwd: fs.cwd().to_path_buf(), workdir: workdir.clone(),
output_dir: output_dir.clone(),
spilled_outputs: std::sync::Mutex::new(Vec::new()),
}); });
(meta, tool) (meta, tool)
}) })
} }
#[cfg(test)]
mod tests {
use super::*;
use manifest::Scope;
use tempfile::TempDir;
/// Test harness: workspace tempdir + a separate spill tempdir kept
/// alive for the test's lifetime. The spill dir is added to the
/// scope as readable so callers exercise the production path.
struct Harness {
_workspace: TempDir,
spill: TempDir,
fs: ScopedFs,
}
fn setup() -> Harness {
let workspace = TempDir::new().unwrap();
let spill = TempDir::new().unwrap();
let base = Scope::writable(workspace.path()).unwrap();
let mut config = manifest::ScopeConfig {
allow: base.allow_rules(),
deny: base.deny_rules(),
};
config.allow.push(manifest::ScopeRule {
target: spill.path().to_path_buf(),
permission: manifest::Permission::Read,
recursive: true,
});
let scope = Scope::from_config(&config).unwrap();
let fs = ScopedFs::new(scope, workspace.path().to_path_buf());
Harness {
_workspace: workspace,
spill,
fs,
}
}
fn make_tool(h: &Harness) -> Arc<dyn Tool> {
let def = bash_tool(h.fs.clone(), h.spill.path().to_path_buf());
let (_, tool) = def();
tool
}
#[tokio::test]
async fn runs_simple_command() {
let h = setup();
let def = bash_tool(h.fs.clone(), h.spill.path().to_path_buf());
let (meta, tool) = def();
assert_eq!(meta.name, "Bash");
let inp = serde_json::json!({ "command": "echo hello" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert_eq!(out.summary, "$ echo hello");
assert_eq!(out.content.as_deref().map(str::trim), Some("hello"));
}
#[tokio::test]
async fn merges_stdout_and_stderr() {
let h = setup();
let tool = make_tool(&h);
let inp = serde_json::json!({
"command": "echo out; echo err 1>&2",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains("out"));
assert!(body.contains("err"));
}
#[tokio::test]
async fn nonzero_exit_is_reported() {
let h = setup();
let tool = make_tool(&h);
let inp = serde_json::json!({ "command": "exit 7" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert!(out.summary.contains("exit 7"), "summary: {}", out.summary);
assert!(
out.content.is_none(),
"no output expected, got {:?}",
out.content
);
}
#[tokio::test]
async fn cd_does_not_persist_across_calls() {
// Stateless: a `cd` in one call must NOT leak into the next.
let h = setup();
let sub = h._workspace.path().join("nested");
std::fs::create_dir(&sub).unwrap();
let tool = make_tool(&h);
tool.execute(
&serde_json::json!({
"command": format!("cd {}", sub.to_str().unwrap()),
})
.to_string(),
Default::default(),
)
.await
.unwrap();
let pwd_out = tool
.execute(
&serde_json::json!({ "command": "pwd" }).to_string(),
Default::default(),
)
.await
.unwrap();
let body = pwd_out.content.unwrap();
let actual = std::fs::canonicalize(body.trim()).unwrap();
let workspace = std::fs::canonicalize(h._workspace.path()).unwrap();
assert_eq!(
actual, workspace,
"second call should start at workspace root, not the previous cd target"
);
}
#[tokio::test]
async fn timeout_kills_long_command() {
let h = setup();
let tool = make_tool(&h);
let inp = serde_json::json!({
"command": "sleep 30",
"timeout": 1,
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert!(
out.summary.contains("timed out"),
"summary: {}",
out.summary
);
}
#[tokio::test]
async fn invalid_json_is_invalid_argument() {
let h = setup();
let tool = make_tool(&h);
let err = tool
.execute("not json", Default::default())
.await
.unwrap_err();
assert!(matches!(err, ToolError::InvalidArgument(_)));
}
#[tokio::test]
async fn long_output_spills_and_returns_tail() {
let h = setup();
let spill_dir = h.spill.path().to_path_buf();
let tool = make_tool(&h);
// 200 lines: "line 1" .. "line 200". Tail of 80 keeps lines 121-200.
let inp = serde_json::json!({
"command": "for i in $(seq 1 200); do echo line $i; done",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.expect("expected content");
assert!(
body.contains(&format!("showing last {TAIL_LINES} of 200 lines")),
"tail header missing in: {}",
&body[..body.len().min(300)]
);
assert!(
body.contains(spill_dir.to_str().unwrap()),
"spill dir path missing: {body}"
);
// Last 80 lines are 121..200.
assert!(body.contains("\nline 200\n"));
assert!(body.contains("\nline 121\n"));
// line 120 is the last *elided* line.
assert!(!body.contains("\nline 120\n"), "elided line leaked: {body}");
}
#[tokio::test]
async fn wide_short_output_still_spills_when_byte_budget_exceeded() {
let h = setup();
let spill_dir = h.spill.path().to_path_buf();
let tool = make_tool(&h);
// One single line of ~20 KiB (over INLINE_BYTE_BUDGET = 12 KiB).
let inp = serde_json::json!({
"command": "printf 'x%.0s' {1..20480}",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(
body.contains(spill_dir.to_str().unwrap()),
"expected spill marker in: {}",
&body[..body.len().min(200)]
);
}
#[tokio::test]
async fn background_job_does_not_hang() {
let h = setup();
let tool = make_tool(&h);
// The wrapper's `wait` ensures we don't hang on a stray bg pipe.
let inp = serde_json::json!({
"command": "(sleep 0.05; echo bg) &",
"timeout": 5,
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert!(
!out.summary.contains("timed out"),
"summary: {}",
out.summary
);
}
#[tokio::test]
async fn spilled_files_are_cleaned_up_on_drop() {
let h = setup();
let spill_dir = h.spill.path().to_path_buf();
let tool = make_tool(&h);
let inp = serde_json::json!({
"command": "for i in $(seq 1 200); do echo $i; done",
});
tool.execute(&inp.to_string(), Default::default())
.await
.unwrap();
// The spill dir should now contain exactly one bash-*.log file.
let files_before: Vec<_> = std::fs::read_dir(&spill_dir)
.unwrap()
.filter_map(Result::ok)
.map(|e| e.path())
.collect();
assert_eq!(files_before.len(), 1, "expected one spilled file");
let path = files_before.into_iter().next().unwrap();
assert!(path.exists());
drop(tool);
// Drop runs synchronously; file should be gone.
assert!(
!path.exists(),
"spilled file should be cleaned up on drop: {path:?}"
);
}
}
+39 -64
View File
@@ -8,18 +8,18 @@ use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use serde::Deserialize; use serde::Deserialize;
use crate::error::ToolsError; use crate::error::ToolsError;
use crate::scoped_fs::ScopedFs;
use crate::tracker::Tracker; use crate::tracker::Tracker;
use workdir::{EditRequest, WorkdirHandle, WorkdirPath};
const DESCRIPTION: &str = "Replace a substring in an existing file. By default \ const DESCRIPTION: &str = "Replace a substring in an existing file. By default \
`old_string` must be unique in the file; set `replace_all: true` to replace \ `old_string` must be unique in the file; set `replace_all: true` to replace \
every occurrence. The file must have been read first (via the Read tool) in \ every occurrence. The file must have been read first (via the Read tool) in \
this session. Paths must be absolute."; this session. Paths are relative to the bound Workdir.";
#[derive(Debug, Deserialize, schemars::JsonSchema)] #[derive(Debug, Deserialize, schemars::JsonSchema)]
pub(crate) struct EditParams { pub(crate) struct EditParams {
/// Absolute path to the file. /// Logical path relative to the bound Workdir root.
pub file_path: PathBuf, pub file_path: String,
/// String to replace. Must be unique in the file unless `replace_all` is true. /// String to replace. Must be unique in the file unless `replace_all` is true.
pub old_string: String, pub old_string: String,
/// Replacement string. Must differ from `old_string`. /// Replacement string. Must differ from `old_string`.
@@ -30,7 +30,7 @@ pub(crate) struct EditParams {
} }
pub(crate) struct EditTool { pub(crate) struct EditTool {
fs: ScopedFs, workdir: WorkdirHandle,
tracker: Tracker, tracker: Tracker,
} }
@@ -44,11 +44,8 @@ impl Tool for EditTool {
let params: EditParams = serde_json::from_str(input_json) let params: EditParams = serde_json::from_str(input_json)
.map_err(|e| ToolError::InvalidArgument(format!("invalid Edit input: {e}")))?; .map_err(|e| ToolError::InvalidArgument(format!("invalid Edit input: {e}")))?;
tracing::debug!( let path = WorkdirPath::new(&params.file_path).map_err(ToolsError::from)?;
path = %params.file_path.display(), tracing::debug!(path = %path, replace_all = params.replace_all, "Edit");
replace_all = params.replace_all,
"Edit"
);
if params.old_string.is_empty() { if params.old_string.is_empty() {
return Err(ToolError::InvalidArgument( return Err(ToolError::InvalidArgument(
@@ -61,51 +58,29 @@ impl Tool for EditTool {
)); ));
} }
let _mutation_permit = self.tracker.acquire_mutation(&params.file_path, &ctx).await; let mutation_key = PathBuf::from(path.as_str());
let _mutation_permit = self.tracker.acquire_mutation(&mutation_key, &ctx).await;
// Load current content and verify it matches the recorded hash. let expected_hash = self.tracker.expected_workdir_hash(&path)?;
let current_bytes = self.fs.read_bytes(&params.file_path)?; let result = self
self.tracker.verify(&params.file_path, &current_bytes)?; .workdir
.edit(EditRequest {
let current_text = std::str::from_utf8(&current_bytes).map_err(|_| { path: path.clone(),
ToolsError::InvalidArgument(format!( old_string: params.old_string.clone(),
"file is not valid UTF-8: {}", new_string: params.new_string.clone(),
params.file_path.display() replace_all: params.replace_all,
)) expected_hash,
})?; })
.await
let count = current_text.matches(&params.old_string).count(); .map_err(ToolsError::from)?;
if count == 0 { self.tracker.record_workdir_hash(&path, result.content_hash);
return Err(ToolsError::StringNotFound {
path: params.file_path.clone(),
}
.into());
}
if !params.replace_all && count > 1 {
return Err(ToolsError::NotUnique {
path: params.file_path.clone(),
count,
}
.into());
}
let new_text = if params.replace_all {
current_text.replace(&params.old_string, &params.new_string)
} else {
current_text.replacen(&params.old_string, &params.new_string, 1)
};
let occurrences = if params.replace_all { count } else { 1 };
self.fs.write(&params.file_path, new_text.as_bytes())?;
self.tracker.record(&params.file_path, new_text.as_bytes());
let summary = format!( let summary = format!(
"Edited {} ({} replacement{})", "Edited {} ({} replacement{})",
params.file_path.display(), path,
occurrences, result.replacements,
if occurrences == 1 { "" } else { "s" } if result.replacements == 1 { "" } else { "s" }
); );
let preview = make_preview(&new_text, &params.new_string); let preview = make_preview(&params.new_string, &params.new_string);
Ok(ToolOutput { Ok(ToolOutput {
summary, summary,
@@ -140,7 +115,7 @@ fn make_preview(text: &str, needle: &str) -> String {
} }
/// Factory for the `Edit` tool. /// Factory for the `Edit` tool.
pub fn edit_tool(fs: ScopedFs, tracker: Tracker) -> ToolDefinition { pub fn edit_tool(workdir: WorkdirHandle, tracker: Tracker) -> ToolDefinition {
Arc::new(move || { Arc::new(move || {
let schema = schemars::schema_for!(EditParams); let schema = schemars::schema_for!(EditParams);
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({})); let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
@@ -148,7 +123,7 @@ pub fn edit_tool(fs: ScopedFs, tracker: Tracker) -> ToolDefinition {
.description(DESCRIPTION) .description(DESCRIPTION)
.input_schema(schema_value); .input_schema(schema_value);
let tool: Arc<dyn Tool> = Arc::new(EditTool { let tool: Arc<dyn Tool> = Arc::new(EditTool {
fs: fs.clone(), workdir: workdir.clone(),
tracker: tracker.clone(), tracker: tracker.clone(),
}); });
(meta, tool) (meta, tool)
@@ -162,19 +137,19 @@ mod tests {
use manifest::Scope; use manifest::Scope;
use tempfile::TempDir; use tempfile::TempDir;
fn setup() -> (TempDir, ScopedFs, Tracker) { fn setup() -> (TempDir, WorkdirHandle, Tracker) {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let fs = ScopedFs::new( let fs: WorkdirHandle = Arc::new(workdir::LocalWorkdir::new(
Scope::writable(dir.path()).unwrap(), Scope::writable(dir.path()).unwrap(),
dir.path().to_path_buf(), dir.path().to_path_buf(),
); ));
(dir, fs, Tracker::new()) (dir, fs, Tracker::new())
} }
async fn read_first(fs: &ScopedFs, tracker: &Tracker, file: &std::path::Path) { async fn read_first(fs: &WorkdirHandle, tracker: &Tracker, file: &std::path::Path) {
let def = read_tool(fs.clone(), tracker.clone()); let def = read_tool(fs.clone(), tracker.clone());
let (_, reader) = def(); let (_, reader) = def();
let inp = serde_json::json!({ "file_path": file.to_str().unwrap() }); let inp = serde_json::json!({ "file_path": file.file_name().unwrap().to_str().unwrap() });
reader reader
.execute(&inp.to_string(), Default::default()) .execute(&inp.to_string(), Default::default())
.await .await
@@ -193,7 +168,7 @@ mod tests {
assert_eq!(meta.name, "Edit"); assert_eq!(meta.name, "Edit");
let inp = serde_json::json!({ let inp = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "foo bar", "old_string": "foo bar",
"new_string": "foo baz", "new_string": "foo baz",
}); });
@@ -219,7 +194,7 @@ mod tests {
let def = edit_tool(fs, tracker); let def = edit_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let inp = serde_json::json!({ let inp = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "x", "old_string": "x",
"new_string": "y", "new_string": "y",
"replace_all": true, "replace_all": true,
@@ -242,7 +217,7 @@ mod tests {
let def = edit_tool(fs, tracker); let def = edit_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let inp = serde_json::json!({ let inp = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "a", "old_string": "a",
"new_string": "b", "new_string": "b",
}); });
@@ -263,7 +238,7 @@ mod tests {
let def = edit_tool(fs, tracker); let def = edit_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let inp = serde_json::json!({ let inp = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "world", "old_string": "world",
"new_string": "x", "new_string": "x",
}); });
@@ -283,7 +258,7 @@ mod tests {
let def = edit_tool(fs, tracker); let def = edit_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let inp = serde_json::json!({ let inp = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "foo", "old_string": "foo",
"new_string": "bar", "new_string": "bar",
}); });
@@ -307,7 +282,7 @@ mod tests {
let def = edit_tool(fs, tracker); let def = edit_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let inp = serde_json::json!({ let inp = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "foo", "old_string": "foo",
"new_string": "bar", "new_string": "bar",
}); });
+18 -101
View File
@@ -1,8 +1,8 @@
//! Error type shared across the `tools` crate. //! Error types for builtin tools.
//! //!
//! `ToolsError` is the crate-level error returned by `ScopedFs` and each //! `ToolsError` keeps tool-specific policy failures separate from Workdir
//! builtin tool's internal logic. Tool `execute()` impls convert it to //! operation failures. Filesystem, search, and command errors originate in
//! [`llm_engine::tool::ToolError`] via the `From` impl defined here. //! `workdir` and remain transparent here.
use std::path::PathBuf; use std::path::PathBuf;
@@ -10,61 +10,8 @@ use llm_engine::tool::ToolError;
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum ToolsError { pub enum ToolsError {
#[error("path must be absolute: {}", .0.display())] #[error(transparent)]
RelativePath(PathBuf), Workdir(#[from] workdir::WorkdirError),
#[error("path is outside allowed scope: {}", .0.display())]
OutOfScope(PathBuf),
#[error(
"path resolves through a symlink outside allowed {required_permission} scope: {} -> {}; add the symlink target to the Worker {required_permission} scope, copy it into the workspace, or recreate the symlink with the correct target",
.path.display(),
.target.display()
)]
SymlinkOutOfScope {
path: PathBuf,
target: PathBuf,
required_permission: &'static str,
},
#[error(
"broken symlink while resolving {}: {} -> {} (target does not exist); recreate the symlink with an absolute target or a correct relative target",
.path.display(),
.link.display(),
.target.display()
)]
BrokenSymlink {
path: PathBuf,
link: PathBuf,
target: PathBuf,
},
#[error(
"path resolves through a symlink to a directory, not a file: {} -> {}",
.path.display(),
.target.display()
)]
SymlinkTargetIsDirectory { path: PathBuf, target: PathBuf },
#[error(
"{tool} does not follow symlink directories: {} -> {}; use the resolved target path directly, or add the target to read scope and reference it without the symlink",
.path.display(),
.target.display()
)]
SymlinkDirectoryNotTraversed {
tool: &'static str,
path: PathBuf,
target: PathBuf,
},
#[error("path is read-only in this scope: {}", .0.display())]
ReadOnly(PathBuf),
#[error("path is a directory: {}", .0.display())]
IsDirectory(PathBuf),
#[error("file not found: {}", .0.display())]
NotFound(PathBuf),
#[error("file has not been read in this session; read it first: {}", .0.display())] #[error("file has not been read in this session; read it first: {}", .0.display())]
NotRead(PathBuf), NotRead(PathBuf),
@@ -83,52 +30,22 @@ pub enum ToolsError {
#[error("invalid argument: {0}")] #[error("invalid argument: {0}")]
InvalidArgument(String), InvalidArgument(String),
#[error("invalid regex: {0}")]
InvalidRegex(String),
#[error("invalid glob pattern: {0}")]
InvalidGlob(String),
#[error("I/O error at {}: {source}", .path.display())]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
}
impl ToolsError {
/// Helper to wrap an [`std::io::Error`] with the path it occurred on.
pub fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
Self::Io {
path: path.into(),
source,
}
}
} }
impl From<ToolsError> for ToolError { impl From<ToolsError> for ToolError {
fn from(err: ToolsError) -> Self { fn from(err: ToolsError) -> Self {
use ToolsError::*; match &err {
match err { ToolsError::Workdir(
RelativePath(_) workdir::WorkdirError::NotFound(_)
| OutOfScope(_) | workdir::WorkdirError::Io { .. }
| SymlinkOutOfScope { .. } | workdir::WorkdirError::Unavailable(_),
| BrokenSymlink { .. } ) => ToolError::ExecutionFailed(err.to_string()),
| SymlinkTargetIsDirectory { .. } ToolsError::Workdir(_)
| SymlinkDirectoryNotTraversed { .. } | ToolsError::NotRead(_)
| ReadOnly(_) | ToolsError::ExternallyModified(_)
| IsDirectory(_) | ToolsError::StringNotFound { .. }
| NotRead(_) | ToolsError::NotUnique { .. }
| ExternallyModified(_) | ToolsError::InvalidArgument(_) => ToolError::InvalidArgument(err.to_string()),
| StringNotFound { .. }
| NotUnique { .. }
| InvalidArgument(_)
| InvalidRegex(_)
| InvalidGlob(_) => ToolError::InvalidArgument(err.to_string()),
NotFound(_) => ToolError::ExecutionFailed(err.to_string()),
Io { .. } => ToolError::ExecutionFailed(err.to_string()),
} }
} }
} }
+46 -357
View File
@@ -1,36 +1,26 @@
//! `Glob` tool — recursive file search by glob pattern, sorted by mtime.
use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use std::time::SystemTime;
use async_trait::async_trait; use async_trait::async_trait;
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput}; use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use manifest::Scope; use schemars::JsonSchema;
use serde::Deserialize; use serde::Deserialize;
use workdir::{GlobRequest, WorkdirHandle, WorkdirPath};
use crate::error::ToolsError; use crate::ToolsError;
use crate::scoped_fs::{ScopedFs, direct_symlink};
const DESCRIPTION: &str = "Recursively find files matching a glob pattern \
(e.g. \"**/*.rs\"). Results are sorted by modification time, newest first, \
and capped at 1000 entries. Hidden files are included. The `path` parameter \
defaults to the scope root when omitted. Paths must be absolute.";
const RESULT_LIMIT: usize = 1000; const RESULT_LIMIT: usize = 1000;
#[derive(Debug, Deserialize, schemars::JsonSchema)] #[derive(Debug, Deserialize, JsonSchema)]
pub(crate) struct GlobParams { struct GlobParams {
/// Glob pattern, e.g. `"**/*.rs"`. Matched against paths relative to /// Glob pattern, for example `**/*.rs` or `src/**/test_*.py`.
/// `path` (or the scope root if omitted). pattern: String,
pub pattern: String, /// Logical Workdir-relative directory. Defaults to the Workdir root.
/// Absolute directory to search under. Defaults to the scope root.
#[serde(default)] #[serde(default)]
pub path: Option<PathBuf>, path: Option<String>,
} }
pub(crate) struct GlobTool { struct GlobTool {
fs: ScopedFs, workdir: WorkdirHandle,
} }
#[async_trait] #[async_trait]
@@ -41,358 +31,57 @@ impl Tool for GlobTool {
_ctx: llm_engine::tool::ToolExecutionContext, _ctx: llm_engine::tool::ToolExecutionContext,
) -> Result<ToolOutput, ToolError> { ) -> Result<ToolOutput, ToolError> {
let params: GlobParams = serde_json::from_str(input_json) let params: GlobParams = serde_json::from_str(input_json)
.map_err(|e| ToolError::InvalidArgument(format!("invalid Glob input: {e}")))?; .map_err(|error| ToolError::InvalidArgument(format!("invalid Glob input: {error}")))?;
let path = match params.path {
tracing::debug!( Some(path) => WorkdirPath::new(&path).map_err(ToolsError::from)?,
pattern = %params.pattern, None => WorkdirPath::root(),
path = ?params.path,
"Glob"
);
let base = params
.path
.clone()
.unwrap_or_else(|| self.fs.cwd().to_path_buf());
let pattern = params.pattern.clone();
let scope = self.fs.scope().clone();
// ignore::Walk is synchronous; run it on a blocking thread so we
// don't stall the runtime for large trees.
let results = tokio::task::spawn_blocking(move || run_glob(&base, &pattern, &scope))
.await
.map_err(|e| ToolError::Internal(format!("spawn_blocking failed: {e}")))??;
let total = results.len();
let (shown, truncated) = if total > RESULT_LIMIT {
(&results[..RESULT_LIMIT], true)
} else {
(&results[..], false)
}; };
let pattern = params.pattern;
if shown.is_empty() { tracing::debug!(%pattern, %path, "Glob");
return Ok(ToolOutput { let result = self
summary: format!("No files found matching {}", params.pattern), .workdir
content: None, .glob(GlobRequest {
}); pattern: pattern.clone(),
} path,
limit: RESULT_LIMIT,
let mut body = String::new(); })
for p in shown { .await
body.push_str(&p.display().to_string()); .map_err(ToolsError::from)?;
let mut body = result
.paths
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("\n");
if !body.is_empty() {
body.push('\n'); body.push('\n');
} }
let summary = if result.paths.is_empty() {
let summary = if truncated { format!("No files found matching {pattern}")
} else if result.truncated {
format!( format!(
"Found {total}+ files matching {} (truncated to {RESULT_LIMIT})", "Found {}+ files matching {pattern} (truncated to {RESULT_LIMIT})",
params.pattern result.paths.len()
) )
} else { } else {
format!("Found {total} file(s) matching {}", params.pattern) format!("Found {} file(s) matching {pattern}", result.paths.len())
}; };
Ok(ToolOutput { Ok(ToolOutput {
summary, summary,
content: Some(body), content: (!body.is_empty()).then_some(body),
}) })
} }
} }
fn run_glob(base: &Path, pattern: &str, scope: &Scope) -> Result<Vec<PathBuf>, ToolsError> { pub fn glob_tool(workdir: WorkdirHandle) -> ToolDefinition {
if !base.is_absolute() {
return Err(ToolsError::RelativePath(base.to_path_buf()));
}
let symlink = direct_symlink(base);
if !scope.is_readable(base) {
return Err(if let Some(info) = symlink.as_ref() {
let link_parent_readable = info
.link_path
.parent()
.map(|parent| scope.is_readable(parent))
.unwrap_or(false);
if info.target_exists && link_parent_readable {
ToolsError::SymlinkOutOfScope {
path: base.to_path_buf(),
target: info.resolved_path.clone(),
required_permission: "read",
}
} else {
ToolsError::OutOfScope(base.to_path_buf())
}
} else {
ToolsError::OutOfScope(base.to_path_buf())
});
}
if let Some(info) = symlink.as_ref() {
if !info.target_exists {
return Err(ToolsError::BrokenSymlink {
path: base.to_path_buf(),
link: info.link_path.clone(),
target: info.target_path.clone(),
});
}
}
let base_meta = std::fs::metadata(base).map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => ToolsError::NotFound(base.to_path_buf()),
_ => ToolsError::io(base, e),
})?;
if !base_meta.is_dir() {
return Err(ToolsError::InvalidArgument(format!(
"glob search path is not a directory: {}",
base.display()
)));
}
if let Some(info) = symlink.as_ref() {
return Err(ToolsError::SymlinkDirectoryNotTraversed {
tool: "Glob",
path: base.to_path_buf(),
target: info.resolved_path.clone(),
});
}
let glob = globset::Glob::new(pattern)
.map_err(|e| ToolsError::InvalidGlob(e.to_string()))?
.compile_matcher();
// Glob is an explicit-pattern tool, so gitignore/hidden are *not* honored.
let walker = ignore::WalkBuilder::new(base)
.hidden(false)
.git_ignore(false)
.git_global(false)
.git_exclude(false)
.ignore(false)
.parents(false)
.follow_links(false)
.build();
let mut hits: Vec<(PathBuf, SystemTime)> = Vec::new();
for entry in walker.flatten() {
let ft = match entry.file_type() {
Some(ft) => ft,
None => continue,
};
if !ft.is_file() {
continue;
}
let rel = match entry.path().strip_prefix(base) {
Ok(r) => r,
Err(_) => continue,
};
if !glob.is_match(rel) {
continue;
}
if !scope.is_readable(entry.path()) {
continue;
}
let mtime = entry
.metadata()
.ok()
.and_then(|m| m.modified().ok())
.unwrap_or(SystemTime::UNIX_EPOCH);
hits.push((entry.path().to_path_buf(), mtime));
}
hits.sort_by(|a, b| b.1.cmp(&a.1));
Ok(hits.into_iter().map(|(p, _)| p).collect())
}
/// Factory for the `Glob` tool.
pub fn glob_tool(fs: ScopedFs) -> ToolDefinition {
Arc::new(move || { Arc::new(move || {
let schema = schemars::schema_for!(GlobParams); let schema = schemars::schema_for!(GlobParams);
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
let meta = ToolMeta::new("Glob") let meta = ToolMeta::new("Glob")
.description(DESCRIPTION) .description("Find files matching a glob pattern inside the bound Workdir. Results are sorted and capped at 1000 entries. Paths are Workdir-relative.")
.input_schema(schema_value); .input_schema(serde_json::to_value(schema).expect("Glob schema serialization"));
let tool: Arc<dyn Tool> = Arc::new(GlobTool { fs: fs.clone() }); let tool: Arc<dyn Tool> = Arc::new(GlobTool {
workdir: workdir.clone(),
});
(meta, tool) (meta, tool)
}) })
} }
#[cfg(test)]
mod tests {
use super::*;
use manifest::Scope;
use tempfile::TempDir;
fn setup() -> (TempDir, ScopedFs) {
let dir = TempDir::new().unwrap();
let fs = ScopedFs::new(
Scope::writable(dir.path()).unwrap(),
dir.path().to_path_buf(),
);
(dir, fs)
}
fn touch(path: &Path, content: &str) {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(path, content).unwrap();
}
#[tokio::test]
async fn glob_finds_matching_files() {
let (dir, fs) = setup();
touch(&dir.path().join("a.rs"), "");
touch(&dir.path().join("sub/b.rs"), "");
touch(&dir.path().join("sub/c.txt"), "");
let def = glob_tool(fs);
let (meta, tool) = def();
assert_eq!(meta.name, "Glob");
let inp = serde_json::json!({ "pattern": "**/*.rs" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert!(out.summary.contains("2 file(s)"));
let body = out.content.unwrap();
assert!(body.contains("a.rs"));
assert!(body.contains("b.rs"));
assert!(!body.contains("c.txt"));
}
#[tokio::test]
async fn glob_sorts_by_mtime_desc() {
let (dir, fs) = setup();
let older = dir.path().join("old.rs");
let newer = dir.path().join("new.rs");
touch(&older, "");
touch(&newer, "");
filetime::set_file_mtime(&older, filetime::FileTime::from_unix_time(1_000, 0)).unwrap();
filetime::set_file_mtime(&newer, filetime::FileTime::from_unix_time(2_000, 0)).unwrap();
let def = glob_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "*.rs" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
let new_pos = body.find("new.rs").unwrap();
let old_pos = body.find("old.rs").unwrap();
assert!(new_pos < old_pos, "newer file should come first:\n{body}");
}
#[tokio::test]
async fn glob_empty_results() {
let (_dir, fs) = setup();
let def = glob_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "**/*.nonexistent" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert!(out.summary.contains("No files"));
assert!(out.content.is_none());
}
#[tokio::test]
async fn glob_invalid_pattern() {
let (_dir, fs) = setup();
let def = glob_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "[unterminated" });
let err = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap_err();
assert!(matches!(err, ToolError::InvalidArgument(_)));
}
#[tokio::test]
async fn glob_filters_results_by_scope_readability() {
use manifest::{Permission, ScopeConfig, ScopeRule};
let dir = TempDir::new().unwrap();
let secret_dir = dir.path().join("secret");
std::fs::create_dir(&secret_dir).unwrap();
touch(&dir.path().join("visible.rs"), "");
touch(&secret_dir.join("hidden.rs"), "");
let cfg = ScopeConfig {
allow: vec![ScopeRule {
target: dir.path().to_path_buf(),
permission: Permission::Write,
recursive: true,
}],
deny: vec![ScopeRule {
target: secret_dir.clone(),
permission: Permission::Read,
recursive: true,
}],
};
let scope = Scope::from_config(&cfg).unwrap();
let fs = ScopedFs::new(scope, dir.path().to_path_buf());
let def = glob_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "**/*.rs" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap_or_default();
assert!(body.contains("visible.rs"));
assert!(
!body.contains("hidden.rs"),
"scope-denied file leaked into glob output: {body}"
);
}
#[tokio::test]
async fn glob_honors_hidden_files() {
let (dir, fs) = setup();
touch(&dir.path().join(".hidden.rs"), "");
touch(&dir.path().join("visible.rs"), "");
let def = glob_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "*.rs" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains(".hidden.rs"));
assert!(body.contains("visible.rs"));
}
#[cfg(unix)]
#[tokio::test]
async fn glob_reports_scope_inside_symlink_directory_is_not_traversed() {
use std::os::unix::fs::symlink;
let (dir, fs) = setup();
let target = dir.path().join("target-dir");
touch(&target.join("visible.rs"), "");
let link = dir.path().join("external-project");
symlink(&target, &link).unwrap();
let def = glob_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"path": link.to_str().unwrap(),
"pattern": "**/*.rs",
});
let err = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap_err();
let msg = format!("{err}");
assert!(
msg.contains("Glob does not follow symlink directories"),
"{msg}"
);
assert!(msg.contains(&link.display().to_string()), "{msg}");
assert!(
msg.contains(&target.canonicalize().unwrap().display().to_string()),
"{msg}"
);
}
}
+91 -828
View File
@@ -1,83 +1,54 @@
//! `Grep` tool — recursive regex search powered by ripgrep's component crates.
use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use async_trait::async_trait; use async_trait::async_trait;
use grep_regex::RegexMatcherBuilder;
use grep_searcher::sinks::UTF8 as UTF8Sink;
use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkContext, SinkMatch};
use ignore::WalkBuilder;
use ignore::overrides::OverrideBuilder;
use ignore::types::TypesBuilder;
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput}; use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use manifest::Scope; use schemars::JsonSchema;
use serde::Deserialize; use serde::Deserialize;
use workdir::{GrepOutputMode, GrepRequest, WorkdirHandle, WorkdirPath};
use crate::error::ToolsError; use crate::ToolsError;
use crate::scoped_fs::{ScopedFs, direct_symlink};
const DESCRIPTION: &str = "Recursive regex search across files, powered by \
ripgrep. Supports file filtering (`glob`, `type`), context lines, multiline \
matching, and three output modes: `files_with_matches` (default), `content`, \
and `count`. Honors .gitignore. Binary files are skipped. Paths must be \
absolute.";
const DEFAULT_HEAD_LIMIT: usize = 250; const DEFAULT_HEAD_LIMIT: usize = 250;
#[derive(Debug, Clone, Copy, Deserialize, schemars::JsonSchema, Default, PartialEq)] #[derive(Debug, Clone, Copy, Default, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub(crate) enum GrepOutputMode { enum OutputMode {
Content,
#[default] #[default]
FilesWithMatches, FilesWithMatches,
Content,
Count, Count,
} }
#[derive(Debug, Deserialize, schemars::JsonSchema)] #[derive(Debug, Deserialize, JsonSchema)]
pub(crate) struct GrepParams { struct GrepParams {
/// Regex pattern to search for. pattern: String,
pub pattern: String, /// Logical Workdir-relative path to search. Defaults to the Workdir root.
/// Absolute path to search under. Defaults to the scope root.
#[serde(default)] #[serde(default)]
pub path: Option<PathBuf>, path: Option<String>,
/// Glob filter applied to candidate files, e.g. `"*.rs"`.
#[serde(default)] #[serde(default)]
pub glob: Option<String>, glob: Option<String>,
/// File type filter, e.g. `"rust"` or `"py"`. See ripgrep's default types.
#[serde(default, rename = "type")] #[serde(default, rename = "type")]
pub file_type: Option<String>, file_type: Option<String>,
/// Output mode: `files_with_matches` (default), `content`, or `count`.
#[serde(default)] #[serde(default)]
pub output_mode: Option<GrepOutputMode>, case_insensitive: bool,
/// Show line numbers in content mode. Defaults to true.
#[serde(default, rename = "-n")]
pub line_numbers: Option<bool>,
/// Case-insensitive matching.
#[serde(default, rename = "-i")]
pub case_insensitive: bool,
/// Trailing context lines after each match.
#[serde(default, rename = "-A")]
pub after: Option<usize>,
/// Leading context lines before each match.
#[serde(default, rename = "-B")] #[serde(default, rename = "-B")]
pub before: Option<usize>, before: Option<usize>,
/// Context lines before AND after each match (overrides -A/-B when set). #[serde(default, rename = "-A")]
after: Option<usize>,
#[serde(default, rename = "-C")] #[serde(default, rename = "-C")]
pub context: Option<usize>, context: Option<usize>,
/// Allow patterns to match across newlines.
#[serde(default)] #[serde(default)]
pub multiline: bool, multiline: bool,
/// Maximum number of output entries. Defaults to 250.
#[serde(default)] #[serde(default)]
pub head_limit: Option<usize>, output_mode: Option<OutputMode>,
/// Skip the first N output entries (pagination).
#[serde(default)] #[serde(default)]
pub offset: Option<usize>, head_limit: Option<usize>,
#[serde(default)]
offset: Option<usize>,
} }
pub(crate) struct GrepTool { struct GrepTool {
fs: ScopedFs, workdir: WorkdirHandle,
} }
#[async_trait] #[async_trait]
@@ -88,788 +59,80 @@ impl Tool for GrepTool {
_ctx: llm_engine::tool::ToolExecutionContext, _ctx: llm_engine::tool::ToolExecutionContext,
) -> Result<ToolOutput, ToolError> { ) -> Result<ToolOutput, ToolError> {
let params: GrepParams = serde_json::from_str(input_json) let params: GrepParams = serde_json::from_str(input_json)
.map_err(|e| ToolError::InvalidArgument(format!("invalid Grep input: {e}")))?; .map_err(|error| ToolError::InvalidArgument(format!("invalid Grep input: {error}")))?;
let path = match params.path {
tracing::debug!( Some(path) => WorkdirPath::new(&path).map_err(ToolsError::from)?,
pattern = %params.pattern, None => WorkdirPath::root(),
mode = ?params.output_mode, };
"Grep" let mode = match params.output_mode.unwrap_or_default() {
); OutputMode::FilesWithMatches => GrepOutputMode::FilesWithMatches,
OutputMode::Content => GrepOutputMode::Content,
let default_base = self.fs.cwd().to_path_buf(); OutputMode::Count => GrepOutputMode::Count,
let scope = self.fs.scope().clone(); };
let report = tokio::task::spawn_blocking(move || run_grep(default_base, params, &scope)) let (before_context, after_context) = params
.context
.map(|context| (context, context))
.unwrap_or((params.before.unwrap_or(0), params.after.unwrap_or(0)));
let head_limit = params.head_limit.unwrap_or(DEFAULT_HEAD_LIMIT);
let result = self
.workdir
.grep(GrepRequest {
pattern: params.pattern,
path,
glob: params.glob,
file_type: params.file_type,
case_insensitive: params.case_insensitive,
before_context,
after_context,
multiline: params.multiline,
output_mode: mode,
limit: head_limit,
offset: params.offset.unwrap_or(0),
})
.await .await
.map_err(|e| ToolError::Internal(format!("spawn_blocking failed: {e}")))??; .map_err(ToolsError::from)?;
Ok(report.render()) let summary = if result.match_count == 0 {
match mode {
GrepOutputMode::Content => "No matches".to_owned(),
_ => "No files matched".to_owned(),
}
} else {
match mode {
GrepOutputMode::FilesWithMatches => {
format!("Found matches in {} file(s)", result.matched_files)
}
GrepOutputMode::Count => format!(
"Found matches in {} file(s), {} total line(s)",
result.matched_files, result.match_count
),
GrepOutputMode::Content => format!(
"{} matching line(s) in {} file(s)",
result.match_count, result.matched_files
),
}
};
let summary = if result.truncated {
format!("{summary} (truncated at {head_limit})")
} else {
summary
};
Ok(ToolOutput {
summary,
content: (!result.output.is_empty()).then_some(result.output),
})
} }
} }
/// Factory for the `Grep` tool. pub fn grep_tool(workdir: WorkdirHandle) -> ToolDefinition {
pub fn grep_tool(fs: ScopedFs) -> ToolDefinition {
Arc::new(move || { Arc::new(move || {
let schema = schemars::schema_for!(GrepParams); let schema = schemars::schema_for!(GrepParams);
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
let meta = ToolMeta::new("Grep") let meta = ToolMeta::new("Grep")
.description(DESCRIPTION) .description("Search Workdir file contents with a regex. Glob/Grep traversal executes inside the Workdir provider. Results are bounded and Workdir-relative.")
.input_schema(schema_value); .input_schema(serde_json::to_value(schema).expect("Grep schema serialization"));
let tool: Arc<dyn Tool> = Arc::new(GrepTool { fs: fs.clone() }); let tool: Arc<dyn Tool> = Arc::new(GrepTool {
workdir: workdir.clone(),
});
(meta, tool) (meta, tool)
}) })
} }
// =============================================================================
// Implementation
// =============================================================================
struct ContentLine {
path: PathBuf,
line_number: Option<u64>,
text: String,
is_match: bool,
}
struct GrepReport {
mode: GrepOutputMode,
show_line_numbers: bool,
files: Vec<PathBuf>,
counts: Vec<(PathBuf, usize)>,
lines: Vec<ContentLine>,
truncated: bool,
head_limit: usize,
}
impl GrepReport {
fn render(self) -> ToolOutput {
match self.mode {
GrepOutputMode::FilesWithMatches => {
if self.files.is_empty() {
return ToolOutput {
summary: "No files matched".into(),
content: None,
};
}
let mut body = String::new();
for p in &self.files {
body.push_str(&p.display().to_string());
body.push('\n');
}
let mut summary = format!("Found matches in {} file(s)", self.files.len());
if self.truncated {
summary.push_str(&format!(" (truncated at {})", self.head_limit));
}
ToolOutput {
summary,
content: Some(body),
}
}
GrepOutputMode::Count => {
if self.counts.is_empty() {
return ToolOutput {
summary: "No files matched".into(),
content: None,
};
}
let total_lines: usize = self.counts.iter().map(|(_, n)| *n).sum();
let mut body = String::new();
for (p, n) in &self.counts {
body.push_str(&format!("{}:{}\n", p.display(), n));
}
let mut summary = format!(
"Found matches in {} file(s), {} total line(s)",
self.counts.len(),
total_lines
);
if self.truncated {
summary.push_str(&format!(" (truncated at {})", self.head_limit));
}
ToolOutput {
summary,
content: Some(body),
}
}
GrepOutputMode::Content => {
if self.lines.is_empty() {
return ToolOutput {
summary: "No matches".into(),
content: None,
};
}
let match_count = self.lines.iter().filter(|l| l.is_match).count();
let file_set: std::collections::BTreeSet<&Path> =
self.lines.iter().map(|l| l.path.as_path()).collect();
let mut body = String::new();
for line in &self.lines {
let sep = if line.is_match { ':' } else { '-' };
if self.show_line_numbers {
if let Some(n) = line.line_number {
body.push_str(&format!(
"{}{}{}{}{}\n",
line.path.display(),
sep,
n,
sep,
line.text
));
continue;
}
}
body.push_str(&format!("{}{}{}\n", line.path.display(), sep, line.text));
}
let mut summary = format!(
"{} matching line(s) in {} file(s)",
match_count,
file_set.len()
);
if self.truncated {
summary.push_str(&format!(" (truncated at {})", self.head_limit));
}
ToolOutput {
summary,
content: Some(body),
}
}
}
}
}
fn run_grep(default_base: PathBuf, p: GrepParams, scope: &Scope) -> Result<GrepReport, ToolsError> {
let matcher = RegexMatcherBuilder::new()
.case_insensitive(p.case_insensitive)
.multi_line(p.multiline)
.dot_matches_new_line(p.multiline)
.build(&p.pattern)
.map_err(|e| ToolsError::InvalidRegex(e.to_string()))?;
let (before, after) = match (p.before, p.after, p.context) {
(_, _, Some(c)) => (c, c),
(b, a, None) => (b.unwrap_or(0), a.unwrap_or(0)),
};
let mut sb = SearcherBuilder::new();
sb.binary_detection(BinaryDetection::quit(b'\x00'))
.line_number(p.line_numbers.unwrap_or(true))
.multi_line(p.multiline)
.before_context(before)
.after_context(after);
let mut searcher = sb.build();
let base = p.path.unwrap_or(default_base);
if !base.is_absolute() {
return Err(ToolsError::RelativePath(base));
}
let symlink = direct_symlink(&base);
if !scope.is_readable(&base) {
return Err(if let Some(info) = symlink.as_ref() {
let link_parent_readable = info
.link_path
.parent()
.map(|parent| scope.is_readable(parent))
.unwrap_or(false);
if info.target_exists && link_parent_readable {
ToolsError::SymlinkOutOfScope {
path: base.clone(),
target: info.resolved_path.clone(),
required_permission: "read",
}
} else {
ToolsError::OutOfScope(base.clone())
}
} else {
ToolsError::OutOfScope(base.clone())
});
}
if let Some(info) = symlink.as_ref() {
if !info.target_exists {
return Err(ToolsError::BrokenSymlink {
path: base.clone(),
link: info.link_path.clone(),
target: info.target_path.clone(),
});
}
}
let base_meta = std::fs::metadata(&base).map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => ToolsError::NotFound(base.clone()),
_ => ToolsError::io(&base, e),
})?;
if !base_meta.is_dir() {
return Err(ToolsError::InvalidArgument(format!(
"grep search path is not a directory: {}",
base.display()
)));
}
if let Some(info) = symlink.as_ref() {
return Err(ToolsError::SymlinkDirectoryNotTraversed {
tool: "Grep",
path: base.clone(),
target: info.resolved_path.clone(),
});
}
let mut wb = WalkBuilder::new(&base);
wb.hidden(true)
.git_ignore(true)
.git_global(true)
.git_exclude(true)
.ignore(true)
.parents(true)
.follow_links(false);
if let Some(t) = p.file_type.as_deref() {
let mut tb = TypesBuilder::new();
tb.add_defaults();
tb.select(t);
let types = tb
.build()
.map_err(|e| ToolsError::InvalidArgument(format!("invalid type {t}: {e}")))?;
wb.types(types);
}
if let Some(g) = p.glob.as_deref() {
let mut ob = OverrideBuilder::new(&base);
ob.add(g)
.map_err(|e| ToolsError::InvalidGlob(e.to_string()))?;
let ov = ob
.build()
.map_err(|e| ToolsError::InvalidGlob(e.to_string()))?;
wb.overrides(ov);
}
let mode = p.output_mode.unwrap_or_default();
let head_limit = p.head_limit.unwrap_or(DEFAULT_HEAD_LIMIT);
let offset = p.offset.unwrap_or(0);
let show_line_numbers = p.line_numbers.unwrap_or(true);
let mut report = GrepReport {
mode,
show_line_numbers,
files: Vec::new(),
counts: Vec::new(),
lines: Vec::new(),
truncated: false,
head_limit,
};
// Per-mode walker state.
let mut matching_files_seen: usize = 0;
let mut matches_seen: usize = 0;
'walker: for entry in wb.build().flatten() {
if !entry.file_type().map(|t| t.is_file()).unwrap_or(false) {
continue;
}
let path = entry.path();
if !scope.is_readable(path) {
continue;
}
match mode {
GrepOutputMode::FilesWithMatches => {
let hit = scan_any_match(&mut searcher, &matcher, path)?;
if !hit {
continue;
}
if matching_files_seen >= offset {
report.files.push(path.to_path_buf());
if report.files.len() >= head_limit {
report.truncated = true;
break 'walker;
}
}
matching_files_seen += 1;
}
GrepOutputMode::Count => {
let count = scan_count(&mut searcher, &matcher, path)?;
if count == 0 {
continue;
}
if matching_files_seen >= offset {
report.counts.push((path.to_path_buf(), count));
if report.counts.len() >= head_limit {
report.truncated = true;
break 'walker;
}
}
matching_files_seen += 1;
}
GrepOutputMode::Content => {
let before_count = matches_seen;
let mut sink = ContentSink {
path: path.to_path_buf(),
lines: &mut report.lines,
matches_seen: &mut matches_seen,
offset,
head_limit,
};
searcher
.search_path(&matcher, path, &mut sink)
.map_err(|e| ToolsError::io(path, e))?;
// If we hit head_limit during this file, stop walking.
if matches_seen >= offset.saturating_add(head_limit) && matches_seen > before_count
{
report.truncated = true;
break 'walker;
}
}
}
}
Ok(report)
}
fn scan_any_match(
searcher: &mut Searcher,
matcher: &grep_regex::RegexMatcher,
path: &Path,
) -> Result<bool, ToolsError> {
let mut hit = false;
let sink = UTF8Sink(|_, _| {
hit = true;
Ok(false) // stop searching this file immediately
});
searcher
.search_path(matcher, path, sink)
.map_err(|e| ToolsError::io(path, e))?;
Ok(hit)
}
fn scan_count(
searcher: &mut Searcher,
matcher: &grep_regex::RegexMatcher,
path: &Path,
) -> Result<usize, ToolsError> {
let mut count = 0usize;
let sink = UTF8Sink(|_, _| {
count += 1;
Ok(true)
});
searcher
.search_path(matcher, path, sink)
.map_err(|e| ToolsError::io(path, e))?;
Ok(count)
}
struct ContentSink<'a> {
path: PathBuf,
lines: &'a mut Vec<ContentLine>,
matches_seen: &'a mut usize,
offset: usize,
head_limit: usize,
}
impl Sink for ContentSink<'_> {
type Error = std::io::Error;
fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, Self::Error> {
let idx = *self.matches_seen;
*self.matches_seen += 1;
// Skip matches before offset.
if idx < self.offset {
return Ok(true);
}
// Stop searching this file once we've filled the head_limit.
if idx >= self.offset.saturating_add(self.head_limit) {
return Ok(false);
}
let text = String::from_utf8_lossy(mat.bytes())
.trim_end_matches('\n')
.trim_end_matches('\r')
.to_string();
self.lines.push(ContentLine {
path: self.path.clone(),
line_number: mat.line_number(),
text,
is_match: true,
});
Ok(true)
}
fn context(
&mut self,
_searcher: &Searcher,
ctx: &SinkContext<'_>,
) -> Result<bool, Self::Error> {
let seen = *self.matches_seen;
if seen < self.offset {
return Ok(true);
}
if seen >= self.offset.saturating_add(self.head_limit) {
return Ok(false);
}
let text = String::from_utf8_lossy(ctx.bytes())
.trim_end_matches('\n')
.trim_end_matches('\r')
.to_string();
self.lines.push(ContentLine {
path: self.path.clone(),
line_number: ctx.line_number(),
text,
is_match: false,
});
Ok(true)
}
}
// =============================================================================
// Tests
// =============================================================================
#[cfg(test)]
mod tests {
use super::*;
use manifest::Scope;
use std::fs;
use tempfile::TempDir;
fn setup() -> (TempDir, ScopedFs) {
let dir = TempDir::new().unwrap();
let fs = ScopedFs::new(
Scope::writable(dir.path()).unwrap(),
dir.path().to_path_buf(),
);
(dir, fs)
}
fn touch(path: &Path, content: &str) {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(path, content).unwrap();
}
#[tokio::test]
async fn grep_filters_results_by_scope_readability() {
use manifest::{Permission, ScopeConfig, ScopeRule};
let dir = TempDir::new().unwrap();
let secret_dir = dir.path().join("secret");
fs::create_dir(&secret_dir).unwrap();
touch(&dir.path().join("visible.txt"), "needle\n");
touch(&secret_dir.join("hidden.txt"), "needle\n");
let cfg = ScopeConfig {
allow: vec![ScopeRule {
target: dir.path().to_path_buf(),
permission: Permission::Write,
recursive: true,
}],
deny: vec![ScopeRule {
target: secret_dir.clone(),
permission: Permission::Read,
recursive: true,
}],
};
let scope = Scope::from_config(&cfg).unwrap();
let scoped = ScopedFs::new(scope, dir.path().to_path_buf());
let def = grep_tool(scoped);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "needle" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap_or_default();
assert!(body.contains("visible.txt"));
assert!(
!body.contains("hidden.txt"),
"scope-denied file leaked into grep output: {body}"
);
}
#[tokio::test]
async fn grep_files_with_matches_default() {
let (dir, fs) = setup();
touch(&dir.path().join("a.txt"), "alpha\nbravo\n");
touch(&dir.path().join("b.txt"), "charlie\n");
let def = grep_tool(fs);
let (meta, tool) = def();
assert_eq!(meta.name, "Grep");
let inp = serde_json::json!({ "pattern": "bravo" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert!(out.summary.contains("1 file"));
assert!(out.content.unwrap().contains("a.txt"));
}
#[tokio::test]
async fn grep_content_mode_with_line_numbers() {
let (dir, fs) = setup();
touch(&dir.path().join("a.txt"), "one\ntwo\nthree\n");
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "two",
"output_mode": "content",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains(":2:two"));
}
#[tokio::test]
async fn grep_count_mode() {
let (dir, fs) = setup();
touch(&dir.path().join("a.txt"), "x\nx\nx\n");
touch(&dir.path().join("b.txt"), "x\ny\n");
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "x",
"output_mode": "count",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains("a.txt:3"));
assert!(body.contains("b.txt:1"));
assert!(out.summary.contains("4 total"));
}
#[tokio::test]
async fn grep_case_insensitive() {
let (dir, fs) = setup();
touch(&dir.path().join("a.txt"), "HELLO\n");
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "hello",
"-i": true,
"output_mode": "content",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert!(out.content.unwrap().contains("HELLO"));
}
#[tokio::test]
async fn grep_context_lines() {
let (dir, fs) = setup();
touch(
&dir.path().join("a.txt"),
"line1\nline2\nMATCH\nline4\nline5\n",
);
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "MATCH",
"output_mode": "content",
"-C": 1,
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
// should contain: line2 (before context), MATCH, line4 (after context)
assert!(body.contains("line2"));
assert!(body.contains("MATCH"));
assert!(body.contains("line4"));
assert!(!body.contains("line1"));
assert!(!body.contains("line5"));
}
#[tokio::test]
async fn grep_multiline() {
let (dir, fs) = setup();
touch(&dir.path().join("a.txt"), "start\nfoo\nbar\nend\n");
let def = grep_tool(fs);
let (_, tool) = def();
// Match across newlines: "foo" followed by "bar" on the next line
let inp = serde_json::json!({
"pattern": "foo[\\s\\S]*?bar",
"multiline": true,
"output_mode": "content",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains("foo"));
}
#[tokio::test]
async fn grep_glob_filter() {
let (dir, fs) = setup();
touch(&dir.path().join("a.rs"), "target\n");
touch(&dir.path().join("b.txt"), "target\n");
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "target",
"glob": "*.rs",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains("a.rs"));
assert!(!body.contains("b.txt"));
}
#[tokio::test]
async fn grep_type_filter() {
let (dir, fs) = setup();
touch(&dir.path().join("a.rs"), "target\n");
touch(&dir.path().join("b.py"), "target\n");
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "target",
"type": "rust",
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains("a.rs"));
assert!(!body.contains("b.py"));
}
#[tokio::test]
async fn grep_head_limit_truncates() {
let (dir, fs) = setup();
for i in 0..5 {
touch(&dir.path().join(format!("f{i}.txt")), "x\n");
}
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "x",
"head_limit": 2,
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert_eq!(body.lines().count(), 2);
assert!(out.summary.contains("truncated at 2"));
}
#[tokio::test]
async fn grep_offset_paginates() {
let (dir, fs) = setup();
// Create 5 files, all matching, deterministically named
for i in 0..5 {
touch(&dir.path().join(format!("f{i}.txt")), "x\n");
}
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "x",
"offset": 3,
"head_limit": 10,
});
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
// We skipped 3, so only 2 should remain.
assert_eq!(body.lines().count(), 2);
}
#[tokio::test]
async fn grep_binary_files_are_skipped() {
let (dir, fs) = setup();
let mut bin = Vec::from(b"\x00\x01\x02needle\n".as_slice());
bin.extend(b"more\n");
fs::write(dir.path().join("a.bin"), bin).unwrap();
touch(&dir.path().join("b.txt"), "needle\n");
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "needle" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
let body = out.content.unwrap();
assert!(body.contains("b.txt"));
assert!(!body.contains("a.bin"));
}
#[tokio::test]
async fn grep_invalid_regex() {
let (_dir, fs) = setup();
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "(" });
let err = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap_err();
assert!(matches!(err, ToolError::InvalidArgument(_)));
}
#[tokio::test]
async fn grep_unknown_type() {
let (_dir, fs) = setup();
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({
"pattern": "x",
"type": "nonexistent",
});
let err = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap_err();
assert!(matches!(err, ToolError::InvalidArgument(_)));
}
#[tokio::test]
async fn grep_no_matches() {
let (dir, fs) = setup();
touch(&dir.path().join("a.txt"), "nothing here\n");
let def = grep_tool(fs);
let (_, tool) = def();
let inp = serde_json::json!({ "pattern": "zzz" });
let out = tool
.execute(&inp.to_string(), Default::default())
.await
.unwrap();
assert_eq!(out.summary, "No files matched");
assert!(out.content.is_none());
}
#[test]
fn grep_schema_contains_dash_keys() {
// Sanity check: schemars must preserve the `-n`, `-A`, etc. keys
// from serde(rename). If this fails we need to rename the fields.
let schema = schemars::schema_for!(GrepParams);
let json = serde_json::to_value(&schema).unwrap();
let json_str = json.to_string();
assert!(json_str.contains("\"-n\""), "schema missing -n: {json_str}");
assert!(json_str.contains("\"-A\""), "schema missing -A: {json_str}");
assert!(json_str.contains("\"-B\""), "schema missing -B: {json_str}");
assert!(json_str.contains("\"-C\""), "schema missing -C: {json_str}");
assert!(json_str.contains("\"-i\""), "schema missing -i: {json_str}");
}
}
+58 -39
View File
@@ -1,25 +1,14 @@
//! Built-in tools for the Yoi LLM agent. //! Built-in tools for the Yoi LLM agent.
//! //!
//! Implements Read / Write / Edit / Glob / Grep / Bash on top of the //! Read / Write / Edit / Glob / Grep / Bash operate through a host-owned
//! `llm-engine` `Tool` infrastructure. Filesystem access is mediated by //! [`workdir::Workdir`] handle. This crate owns tool schemas, rendering, and
//! two orthogonal concerns: //! read-before-edit tracking; it does not own Workdir identity or lifecycle.
//! //!
//! - [`ScopedFs`] — Worker-process lifetime, expresses the write-block //! Bash is intentionally not sandboxed. The Workdir supplies its initial cwd
//! boundary for the current scope. Derived from the manifest; not //! and command capability, while the Runtime process and OS user remain the
//! persisted across Worker restart. //! trusted execution boundary.
//! - [`Tracker`] — Worker-process lifetime, enforces the "read before edit"
//! policy via content hashes and tracks the recency of touched files.
//! Recreated fresh on each Worker start (including resume).
//!
//! The Worker layer owns both instances and passes them to
//! [`core_builtin_tools`] when registering tools on a `Engine`.
//!
//! `Bash` is the lone exception — its child processes bypass `ScopedFs`
//! entirely. Safety for arbitrary command execution is delegated to the
//! Permission layer (deny/allow rules on the command string).
pub mod error; pub mod error;
pub mod scoped_fs;
pub mod tracker; pub mod tracker;
mod bash; mod bash;
@@ -36,36 +25,40 @@ pub use error::ToolsError;
pub use glob::glob_tool; pub use glob::glob_tool;
pub use grep::grep_tool; pub use grep::grep_tool;
pub use read::read_tool; pub use read::read_tool;
pub use scoped_fs::ScopedFs;
pub use tracker::Tracker; pub use tracker::Tracker;
pub use web::{web_fetch_tool, web_search_tool}; pub use web::{web_fetch_tool, web_search_tool};
pub use write::write_tool; pub use write::write_tool;
/// Register core builtin tools that do not require Worker-local task state, /// Build the local filesystem/command tool surface implemented by a Workdir.
/// wiring them to a shared `ScopedFs` (Worker-process lifetime) and `Tracker` /// Profile/manifest policy may narrow this set further in the Engine.
/// (Worker-process lifetime).
///
/// All returned factories share the same tracker instance so that
/// `Read` / `Write` / `Edit` see a consistent history across tool
/// invocations within a single Worker run.
///
/// `bash_output_dir` is where the Bash tool spills long outputs. The
/// caller is responsible for adding that path to the readable scope
/// (see [`manifest::Scope::with_extra_read`]) so the agent can `Read`
/// the saved files.
pub fn core_builtin_tools( pub fn core_builtin_tools(
fs: ScopedFs, workdir: workdir::WorkdirHandle,
tracker: Tracker, tracker: Tracker,
bash_output_dir: std::path::PathBuf, bash_output_dir: std::path::PathBuf,
) -> Vec<llm_engine::tool::ToolDefinition> { ) -> Vec<llm_engine::tool::ToolDefinition> {
vec![ use workdir::WorkdirCapability;
read_tool(fs.clone(), tracker.clone()),
write_tool(fs.clone(), tracker.clone()), let capabilities = workdir.capabilities();
edit_tool(fs.clone(), tracker), let mut tools = Vec::with_capacity(6);
glob_tool(fs.clone()), if capabilities.supports(WorkdirCapability::Read) {
grep_tool(fs.clone()), tools.push(read_tool(workdir.clone(), tracker.clone()));
bash_tool(fs, bash_output_dir), }
] if capabilities.supports(WorkdirCapability::Write) {
tools.push(write_tool(workdir.clone(), tracker.clone()));
}
if capabilities.supports(WorkdirCapability::Edit) {
tools.push(edit_tool(workdir.clone(), tracker));
}
if capabilities.supports(WorkdirCapability::Glob) {
tools.push(glob_tool(workdir.clone()));
}
if capabilities.supports(WorkdirCapability::Grep) {
tools.push(grep_tool(workdir.clone()));
}
if capabilities.supports(WorkdirCapability::Command) {
tools.push(bash_tool(workdir, bash_output_dir));
}
tools
} }
pub fn web_builtin_tools( pub fn web_builtin_tools(
@@ -76,3 +69,29 @@ pub fn web_builtin_tools(
web_fetch_tool(web::WebTools::new(web_config)), web_fetch_tool(web::WebTools::new(web_config)),
] ]
} }
#[cfg(test)]
mod workdir_tool_tests {
use super::*;
use manifest::{Scope, SharedScope};
use std::sync::Arc;
use tempfile::TempDir;
use workdir::{LocalWorkdir, WorkdirCapabilities, WorkdirHandle};
#[test]
fn read_only_workdir_exposes_only_observation_tools() {
let dir = TempDir::new().unwrap();
let workdir: WorkdirHandle = Arc::new(LocalWorkdir::materialized(
dir.path().to_path_buf(),
dir.path().to_path_buf(),
SharedScope::new(Scope::writable(dir.path()).unwrap()),
WorkdirCapabilities::READ_ONLY,
));
let names = core_builtin_tools(workdir, Tracker::new(), dir.path().join("output"))
.into_iter()
.map(|definition| definition().0.name)
.collect::<Vec<_>>();
assert_eq!(names, ["Read", "Glob", "Grep"]);
}
}
+64 -33
View File
@@ -1,26 +1,27 @@
//! `Read` tool — read a text file with offset/limit, return line-numbered output. //! `Read` tool — read a text file with offset/limit, return line-numbered output.
use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use async_trait::async_trait; use async_trait::async_trait;
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput}; use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use serde::Deserialize; use serde::Deserialize;
use crate::scoped_fs::ScopedFs; use crate::error::ToolsError;
use crate::tracker::Tracker; use crate::tracker::Tracker;
use workdir::{ReadRequest, WorkdirHandle, WorkdirPath};
const DESCRIPTION: &str = "Read a text file from the local filesystem. \ const DESCRIPTION: &str = "Read a text file from the local filesystem. \
Supports offset/limit for large files. Returns line-numbered output (1-based). \ Supports offset/limit for large files. Returns line-numbered output (1-based). \
Directories cannot be read. The file must be read before Write or Edit can \ Directories cannot be read. The file must be read before Write or Edit can \
modify it. Paths must be absolute."; modify it. Paths are relative to the bound Workdir.";
const DEFAULT_LIMIT: usize = 2000; const DEFAULT_LIMIT: usize = 2000;
const PROVIDER_BYTE_LIMIT: usize = 256 * 1024;
#[derive(Debug, Deserialize, schemars::JsonSchema)] #[derive(Debug, Deserialize, schemars::JsonSchema)]
pub(crate) struct ReadParams { pub(crate) struct ReadParams {
/// Absolute path to the file. /// Logical path relative to the bound Workdir root.
pub file_path: PathBuf, pub file_path: String,
/// 0-based line offset from the start. Defaults to 0. /// 0-based line offset from the start. Defaults to 0.
#[serde(default)] #[serde(default)]
pub offset: Option<usize>, pub offset: Option<usize>,
@@ -30,7 +31,7 @@ pub(crate) struct ReadParams {
} }
pub(crate) struct ReadTool { pub(crate) struct ReadTool {
fs: ScopedFs, workdir: WorkdirHandle,
tracker: Tracker, tracker: Tracker,
} }
@@ -46,21 +47,29 @@ impl Tool for ReadTool {
let offset = params.offset.unwrap_or(0); let offset = params.offset.unwrap_or(0);
let limit = params.limit.unwrap_or(DEFAULT_LIMIT).max(1); let limit = params.limit.unwrap_or(DEFAULT_LIMIT).max(1);
tracing::debug!( let path = WorkdirPath::new(&params.file_path).map_err(ToolsError::from)?;
path = %params.file_path.display(), tracing::debug!(path = %path, offset, limit, "Read");
let result = self
.workdir
.read(ReadRequest {
path: path.clone(),
offset, offset,
limit, limit,
"Read" max_bytes: PROVIDER_BYTE_LIMIT,
})
.await
.map_err(ToolsError::from)?;
self.tracker.record_workdir_hash(&path, result.content_hash);
let text = String::from_utf8_lossy(&result.bytes).into_owned();
let rendered = render_provider_read(
&text,
result.start_line,
result.total_lines,
result.truncated,
); );
let bytes = self.fs.read_bytes(&params.file_path)?;
// Record the raw bytes under the read-history so subsequent Edit /
// Write can detect external modification.
self.tracker.record(&params.file_path, &bytes);
let text = String::from_utf8_lossy(&bytes).into_owned();
let rendered = render_numbered(&text, offset, limit);
let summary = if rendered.truncated { let summary = if rendered.truncated {
format!( format!(
"Read {} line(s) [{}..{}] of {} from {}", "Read {} line(s) [{}..{}] of {} from {}",
@@ -68,14 +77,10 @@ impl Tool for ReadTool {
offset + 1, offset + 1,
offset + rendered.line_count, offset + rendered.line_count,
rendered.total_lines, rendered.total_lines,
params.file_path.display() path
) )
} else { } else {
format!( format!("Read {} line(s) from {}", rendered.line_count, path)
"Read {} line(s) from {}",
rendered.line_count,
params.file_path.display()
)
}; };
Ok(ToolOutput { Ok(ToolOutput {
@@ -92,8 +97,29 @@ struct Rendered {
truncated: bool, truncated: bool,
} }
fn render_provider_read(
text: &str,
start_line: usize,
total_lines: usize,
truncated: bool,
) -> Rendered {
use std::fmt::Write as _;
let lines = text.lines().collect::<Vec<_>>();
let mut body = String::with_capacity(text.len().saturating_add(lines.len() * 8));
for (index, line) in lines.iter().enumerate() {
let _ = writeln!(&mut body, "{:>6}\t{}", start_line + index + 1, line);
}
Rendered {
body,
line_count: lines.len(),
total_lines,
truncated: start_line > 0 || truncated,
}
}
/// Format a slice of lines from `text` with `cat -n` style 1-based line /// Format a slice of lines from `text` with `cat -n` style 1-based line
/// numbers. Pure function — no I/O, no history touching. /// numbers. Pure function — no I/O, no history touching.
#[cfg(test)]
fn render_numbered(text: &str, offset: usize, limit: usize) -> Rendered { fn render_numbered(text: &str, offset: usize, limit: usize) -> Rendered {
let all_lines: Vec<&str> = text.lines().collect(); let all_lines: Vec<&str> = text.lines().collect();
let total_lines = all_lines.len(); let total_lines = all_lines.len();
@@ -118,7 +144,7 @@ fn render_numbered(text: &str, offset: usize, limit: usize) -> Rendered {
} }
/// Factory for the `Read` tool. /// Factory for the `Read` tool.
pub fn read_tool(fs: ScopedFs, tracker: Tracker) -> ToolDefinition { pub fn read_tool(workdir: WorkdirHandle, tracker: Tracker) -> ToolDefinition {
Arc::new(move || { Arc::new(move || {
let schema = schemars::schema_for!(ReadParams); let schema = schemars::schema_for!(ReadParams);
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({})); let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
@@ -126,7 +152,7 @@ pub fn read_tool(fs: ScopedFs, tracker: Tracker) -> ToolDefinition {
.description(DESCRIPTION) .description(DESCRIPTION)
.input_schema(schema_value); .input_schema(schema_value);
let tool: Arc<dyn Tool> = Arc::new(ReadTool { let tool: Arc<dyn Tool> = Arc::new(ReadTool {
fs: fs.clone(), workdir: workdir.clone(),
tracker: tracker.clone(), tracker: tracker.clone(),
}); });
(meta, tool) (meta, tool)
@@ -138,14 +164,15 @@ mod tests {
use super::*; use super::*;
use manifest::Scope; use manifest::Scope;
use tempfile::TempDir; use tempfile::TempDir;
use workdir::LocalWorkdir;
fn setup() -> (TempDir, ScopedFs, Tracker) { fn setup() -> (TempDir, WorkdirHandle, Tracker) {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let fs = ScopedFs::new( let workdir: WorkdirHandle = Arc::new(LocalWorkdir::new(
Scope::writable(dir.path()).unwrap(), Scope::writable(dir.path()).unwrap(),
dir.path().to_path_buf(), dir.path().to_path_buf(),
); ));
(dir, fs, Tracker::new()) (dir, workdir, Tracker::new())
} }
#[tokio::test] #[tokio::test]
@@ -158,7 +185,7 @@ mod tests {
let (meta, tool) = def(); let (meta, tool) = def();
assert_eq!(meta.name, "Read"); assert_eq!(meta.name, "Read");
let input = serde_json::json!({ "file_path": file.to_str().unwrap() }); let input = serde_json::json!({ "file_path": file.file_name().unwrap().to_str().unwrap() });
let out = tool let out = tool
.execute(&input.to_string(), Default::default()) .execute(&input.to_string(), Default::default())
.await .await
@@ -169,7 +196,11 @@ mod tests {
assert!(body.contains(" 3\tgamma")); assert!(body.contains(" 3\tgamma"));
// History recorded // History recorded
assert!(tracker.has(&file)); assert!(
tracker
.expected_workdir_hash(&WorkdirPath::new("a.txt").unwrap())
.is_ok()
);
} }
#[tokio::test] #[tokio::test]
@@ -181,7 +212,7 @@ mod tests {
let def = read_tool(fs, tracker); let def = read_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let input = serde_json::json!({ let input = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"offset": 1, "offset": 1,
"limit": 2, "limit": 2,
}); });
@@ -201,7 +232,7 @@ mod tests {
let def = read_tool(fs, tracker); let def = read_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let input = serde_json::json!({ let input = serde_json::json!({
"file_path": dir.path().join("nope.txt").to_str().unwrap() "file_path": "nope.txt"
}); });
let err = tool let err = tool
.execute(&input.to_string(), Default::default()) .execute(&input.to_string(), Default::default())
-719
View File
@@ -1,719 +0,0 @@
//! Scope-aware filesystem primitive.
//!
//! `ScopedFs` is the write/read gate layered on top of a [`manifest::Scope`]
//! and a Worker's working directory. The scope decides which paths are
//! readable and writable; the cwd is carried alongside for convenience
//! (Glob/Grep default their search base to it).
//!
//! `ScopedFs` is cheap to clone (`Arc` inside) and carries no per-session
//! state — the read-before-edit policy lives separately in
//! [`crate::Tracker`].
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use manifest::{Scope, SharedScope};
use crate::error::ToolsError;
#[derive(Debug)]
struct ScopedFsInner {
scope: SharedScope,
cwd: PathBuf,
}
/// Scope-aware filesystem handle. Clone-cheap (`Arc` inside).
///
/// The wrapped [`SharedScope`] is shared with every clone of this
/// `ScopedFs` and with whoever else holds the same `SharedScope`
/// handle (typically the owning Worker). Mutations to that `SharedScope`
/// propagate atomically; the next permission check inside any
/// `ScopedFs` reads the new view.
#[derive(Debug, Clone)]
pub struct ScopedFs {
inner: Arc<ScopedFsInner>,
}
/// Outcome of a [`ScopedFs::write`] call.
#[derive(Debug, Clone, Copy)]
pub struct WriteOutcome {
pub bytes_written: usize,
pub created: bool,
}
/// First symlink encountered while resolving a path.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SymlinkInfo {
/// The symlink path as it appears in the original path chain.
pub link_path: PathBuf,
/// The symlink target resolved relative to the symlink's parent when the
/// link stores a relative target.
pub target_path: PathBuf,
/// Best-effort resolved form of the full requested path after replacing
/// the symlink component with its target and rejoining any remaining tail.
/// Existing targets are canonicalized; broken targets are left absolute.
pub resolved_path: PathBuf,
/// Whether the symlink target itself exists. A missing target is a broken
/// symlink even when the symlink lives inside an allowed scope.
pub target_exists: bool,
}
impl ScopedFs {
/// Create a new [`ScopedFs`] wrapping `scope` and `cwd` in a fresh
/// [`SharedScope`]. Use [`ScopedFs::with_shared_scope`] when you
/// need the resulting `ScopedFs` to share scope state with another
/// holder of the `SharedScope` (typically the Worker).
pub fn new(scope: Scope, cwd: PathBuf) -> Self {
Self::with_shared_scope(SharedScope::new(scope), cwd)
}
/// Build a [`ScopedFs`] over an existing [`SharedScope`]. The
/// resulting handle and any future updates the caller pushes to
/// `scope` are observed by every clone of this `ScopedFs`.
pub fn with_shared_scope(scope: SharedScope, cwd: PathBuf) -> Self {
Self {
inner: Arc::new(ScopedFsInner { scope, cwd }),
}
}
/// Snapshot the current scope. Cheap; the returned `Arc<Scope>` is
/// a coherent point-in-time view that subsequent mutations do not
/// affect.
pub fn scope(&self) -> Arc<Scope> {
self.inner.scope.snapshot()
}
/// Shared scope handle backing this `ScopedFs`. Cloning it lets a
/// caller (usually the Worker) hold the same view and push updates
/// that are immediately reflected in subsequent permission checks.
pub fn shared_scope(&self) -> &SharedScope {
&self.inner.scope
}
/// The Worker's working directory. Glob/Grep default their search base
/// to this path when callers omit an explicit `path` parameter.
pub fn cwd(&self) -> &Path {
&self.inner.cwd
}
// =========================================================================
// Read — scope-checked against readability
// =========================================================================
/// Read the full contents of `path` as raw bytes.
///
/// Follows symlinks. Rejects directories, relative paths, paths not
/// readable by the scope, and missing files.
pub fn read_bytes(&self, path: &Path) -> Result<Vec<u8>, ToolsError> {
if !path.is_absolute() {
return Err(ToolsError::RelativePath(path.to_path_buf()));
}
let symlink = first_symlink(path);
let scope = self.inner.scope.load();
if !scope.is_readable(path) {
return Err(symlink_out_of_scope_or_plain(
path,
symlink.as_ref(),
"read",
&scope,
));
}
if let Some(info) = symlink.as_ref() {
if !info.target_exists {
return Err(broken_symlink_error(path, info));
}
}
let meta = std::fs::metadata(path).map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => ToolsError::NotFound(path.to_path_buf()),
_ => ToolsError::io(path, e),
})?;
if meta.is_dir() {
return Err(if let Some(info) = symlink.as_ref() {
ToolsError::SymlinkTargetIsDirectory {
path: path.to_path_buf(),
target: info.resolved_path.clone(),
}
} else {
ToolsError::IsDirectory(path.to_path_buf())
});
}
std::fs::read(path).map_err(|e| ToolsError::io(path, e))
}
// =========================================================================
// Write — scope-checked, atomic
// =========================================================================
/// Atomically write `content` to `path`, creating or overwriting it.
///
/// - `path` must be absolute and writable under the scope.
/// - Paths that are readable but not writable return [`ToolsError::ReadOnly`].
/// - Paths outside the scope entirely return [`ToolsError::OutOfScope`].
/// - Missing parent directories are created.
/// - The actual write uses a sibling tempfile + `persist`, so the
/// target file transitions atomically between states.
///
/// This method does **not** consult any read history. Callers that
/// want the "must read before overwrite" policy should verify with a
/// [`Tracker`](crate::Tracker) beforehand.
pub fn write(&self, path: &Path, content: &[u8]) -> Result<WriteOutcome, ToolsError> {
if !path.is_absolute() {
return Err(ToolsError::RelativePath(path.to_path_buf()));
}
let symlink = first_symlink(path);
let scope = self.inner.scope.load();
if !scope.is_writable(path) {
return Err(if scope.is_readable(path) {
ToolsError::ReadOnly(path.to_path_buf())
} else {
symlink_out_of_scope_or_plain(path, symlink.as_ref(), "write", &scope)
});
}
drop(scope);
if let Some(info) = symlink.as_ref() {
if !info.target_exists {
return Err(broken_symlink_error(path, info));
}
}
// Reject existing directory targets.
match std::fs::metadata(path) {
Ok(meta) if meta.is_dir() => {
return Err(if let Some(info) = symlink.as_ref() {
ToolsError::SymlinkTargetIsDirectory {
path: path.to_path_buf(),
target: info.resolved_path.clone(),
}
} else {
ToolsError::IsDirectory(path.to_path_buf())
});
}
_ => {}
}
let existed = path.exists();
let write_target = if existed {
path.canonicalize().unwrap_or_else(|_| path.to_path_buf())
} else {
path.to_path_buf()
};
let parent = write_target.parent().ok_or_else(|| {
ToolsError::InvalidArgument(format!(
"path has no parent directory: {}",
write_target.display()
))
})?;
if !parent.as_os_str().is_empty() && !parent.exists() {
std::fs::create_dir_all(parent).map_err(|e| ToolsError::io(parent, e))?;
}
let tmp_parent: &Path = if parent.as_os_str().is_empty() {
Path::new(".")
} else {
parent
};
let mut tmp = tempfile::NamedTempFile::new_in(tmp_parent)
.map_err(|e| ToolsError::io(tmp_parent, e))?;
tmp.write_all(content)
.map_err(|e| ToolsError::io(&write_target, e))?;
tmp.as_file()
.sync_all()
.map_err(|e| ToolsError::io(&write_target, e))?;
tmp.persist(&write_target)
.map_err(|e| ToolsError::io(&write_target, e.error))?;
Ok(WriteOutcome {
bytes_written: content.len(),
created: !existed,
})
}
}
/// Return the first symlink component in `path`, if one exists.
///
/// The function only inspects existing path components. It intentionally uses
/// `symlink_metadata` so the symlink itself can be diagnosed before any later
/// `metadata` call follows it and collapses the reason into `NotFound` or
/// `OutOfScope`.
pub fn first_symlink(path: &Path) -> Option<SymlinkInfo> {
if !path.is_absolute() {
return None;
}
let mut cur = PathBuf::new();
let mut components = path.components().peekable();
while let Some(component) = components.next() {
cur.push(component.as_os_str());
let meta = std::fs::symlink_metadata(&cur).ok()?;
if !meta.file_type().is_symlink() {
continue;
}
let raw_target = std::fs::read_link(&cur).ok()?;
let target_path = if raw_target.is_absolute() {
raw_target
} else {
cur.parent()
.unwrap_or_else(|| Path::new("/"))
.join(raw_target)
};
let target_exists = target_path.exists();
let mut resolved_path = target_path
.canonicalize()
.unwrap_or_else(|_| target_path.clone());
for remaining in components {
resolved_path.push(remaining.as_os_str());
}
return Some(SymlinkInfo {
link_path: cur,
target_path,
resolved_path,
target_exists,
});
}
None
}
pub fn direct_symlink(path: &Path) -> Option<SymlinkInfo> {
let meta = std::fs::symlink_metadata(path).ok()?;
if meta.file_type().is_symlink() {
first_symlink(path)
} else {
None
}
}
fn symlink_out_of_scope_or_plain(
path: &Path,
symlink: Option<&SymlinkInfo>,
required_permission: &'static str,
scope: &Scope,
) -> ToolsError {
if let Some(info) = symlink {
let link_parent_readable = info
.link_path
.parent()
.map(|parent| scope.is_readable(parent))
.unwrap_or(false);
if info.target_exists && link_parent_readable {
return ToolsError::SymlinkOutOfScope {
path: path.to_path_buf(),
target: info.resolved_path.clone(),
required_permission,
};
}
}
ToolsError::OutOfScope(path.to_path_buf())
}
fn broken_symlink_error(path: &Path, info: &SymlinkInfo) -> ToolsError {
ToolsError::BrokenSymlink {
path: path.to_path_buf(),
link: info.link_path.clone(),
target: info.target_path.clone(),
}
}
// =============================================================================
// Tests
// =============================================================================
#[cfg(test)]
mod tests {
use super::*;
use manifest::{Permission, ScopeConfig, ScopeRule};
use std::fs;
use tempfile::TempDir;
fn make_fs(dir: &TempDir) -> ScopedFs {
ScopedFs::new(
Scope::writable(dir.path()).unwrap(),
dir.path().to_path_buf(),
)
}
// -------------------------------------------------------------------------
// read_bytes
// -------------------------------------------------------------------------
#[test]
fn read_bytes_returns_content() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let file = dir.path().join("a.txt");
fs::write(&file, b"abc").unwrap();
assert_eq!(fs.read_bytes(&file).unwrap(), b"abc");
}
#[test]
fn read_bytes_rejects_relative() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let err = fs.read_bytes(Path::new("rel.txt")).unwrap_err();
assert!(matches!(err, ToolsError::RelativePath(_)));
}
#[test]
fn read_bytes_rejects_directory() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let err = fs.read_bytes(dir.path()).unwrap_err();
assert!(matches!(err, ToolsError::IsDirectory(_)));
}
#[test]
fn read_bytes_rejects_missing() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let err = fs.read_bytes(&dir.path().join("nope.txt")).unwrap_err();
assert!(matches!(err, ToolsError::NotFound(_)));
}
#[test]
fn read_bytes_rejects_paths_outside_scope() {
let dir = TempDir::new().unwrap();
let outside = TempDir::new().unwrap();
let outside_file = outside.path().join("x.txt");
fs::write(&outside_file, b"hi").unwrap();
let scoped = make_fs(&dir);
let err = scoped.read_bytes(&outside_file).unwrap_err();
assert!(matches!(err, ToolsError::OutOfScope(_)));
}
#[cfg(unix)]
#[test]
fn read_bytes_reports_broken_symlink_target() {
use std::os::unix::fs::symlink;
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let link = dir.path().join("external-project");
let target = dir.path().join("missing-target");
symlink(&target, &link).unwrap();
let err = fs.read_bytes(&link).unwrap_err();
assert!(
matches!(
err,
ToolsError::BrokenSymlink { ref path, link: ref err_link, target: ref err_target }
if path == &link && err_link == &link && err_target == &target
),
"expected broken symlink diagnostic, got {err:?}"
);
}
#[cfg(unix)]
#[test]
fn read_bytes_reports_symlink_target_outside_scope() {
use std::os::unix::fs::symlink;
let dir = TempDir::new().unwrap();
let outside = TempDir::new().unwrap();
let target = outside.path().join("target.txt");
fs::write(&target, b"secret").unwrap();
let link = dir.path().join("outside-repo.txt");
symlink(&target, &link).unwrap();
let fs = make_fs(&dir);
let err = fs.read_bytes(&link).unwrap_err();
assert!(
matches!(
err,
ToolsError::SymlinkOutOfScope { ref path, target: ref err_target, required_permission: "read" }
if path == &link && err_target == &target.canonicalize().unwrap()
),
"expected symlink out-of-scope diagnostic, got {err:?}"
);
}
#[cfg(unix)]
#[test]
fn read_bytes_allows_symlink_file_when_target_is_inside_scope() {
use std::os::unix::fs::symlink;
let dir = TempDir::new().unwrap();
let target = dir.path().join("target.txt");
fs::write(&target, b"visible").unwrap();
let link = dir.path().join("link.txt");
symlink(&target, &link).unwrap();
let fs = make_fs(&dir);
assert_eq!(fs.read_bytes(&link).unwrap(), b"visible");
}
#[cfg(unix)]
#[test]
fn read_bytes_reports_symlink_to_directory_as_wrong_file_type() {
use std::os::unix::fs::symlink;
let dir = TempDir::new().unwrap();
let target_dir = dir.path().join("target-dir");
fs::create_dir(&target_dir).unwrap();
let link = dir.path().join("dir-link");
symlink(&target_dir, &link).unwrap();
let fs = make_fs(&dir);
let err = fs.read_bytes(&link).unwrap_err();
assert!(
matches!(
err,
ToolsError::SymlinkTargetIsDirectory { ref path, ref target }
if path == &link && target == &target_dir.canonicalize().unwrap()
),
"expected symlink directory type diagnostic, got {err:?}"
);
}
// -------------------------------------------------------------------------
// write
// -------------------------------------------------------------------------
#[test]
fn write_creates_new_file() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let file = dir.path().join("new.txt");
let out = fs.write(&file, b"hello").unwrap();
assert!(out.created);
assert_eq!(out.bytes_written, 5);
assert_eq!(fs::read(&file).unwrap(), b"hello");
}
#[test]
fn write_overwrites_existing() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let file = dir.path().join("a.txt");
fs::write(&file, b"old").unwrap();
let out = fs.write(&file, b"new").unwrap();
assert!(!out.created);
assert_eq!(fs::read(&file).unwrap(), b"new");
}
#[cfg(unix)]
#[test]
fn write_existing_symlink_file_updates_in_scope_target() {
use std::os::unix::fs::symlink;
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let target = dir.path().join("target.txt");
fs::write(&target, b"old").unwrap();
let link = dir.path().join("link.txt");
symlink(&target, &link).unwrap();
let out = fs.write(&link, b"new").unwrap();
assert!(!out.created);
assert_eq!(fs::read(&target).unwrap(), b"new");
assert!(
fs::symlink_metadata(&link)
.unwrap()
.file_type()
.is_symlink()
);
}
#[cfg(unix)]
#[test]
fn write_reports_symlink_target_outside_scope() {
use std::os::unix::fs::symlink;
let dir = TempDir::new().unwrap();
let outside = TempDir::new().unwrap();
let target = outside.path().join("target.txt");
fs::write(&target, b"secret").unwrap();
let link = dir.path().join("outside-repo.txt");
symlink(&target, &link).unwrap();
let fs = make_fs(&dir);
let err = fs.write(&link, b"new").unwrap_err();
assert!(
matches!(
err,
ToolsError::SymlinkOutOfScope { ref path, target: ref err_target, required_permission: "write" }
if path == &link && err_target == &target.canonicalize().unwrap()
),
"expected write symlink out-of-scope diagnostic, got {err:?}"
);
}
#[test]
fn write_rejects_out_of_scope() {
let dir = TempDir::new().unwrap();
let outside = TempDir::new().unwrap();
let fs = make_fs(&dir);
let err = fs.write(&outside.path().join("x"), b"x").unwrap_err();
assert!(matches!(err, ToolsError::OutOfScope(_)));
}
#[test]
fn write_rejects_readonly_path() {
let dir = TempDir::new().unwrap();
let sub = dir.path().join("sub");
fs::create_dir(&sub).unwrap();
let cfg = ScopeConfig {
allow: vec![ScopeRule {
target: dir.path().to_path_buf(),
permission: Permission::Write,
recursive: true,
}],
deny: vec![ScopeRule {
target: sub.clone(),
permission: Permission::Write,
recursive: true,
}],
};
let scope = Scope::from_config(&cfg).unwrap();
let scoped = ScopedFs::new(scope, dir.path().to_path_buf());
let err = scoped.write(&sub.join("locked.txt"), b"x").unwrap_err();
assert!(
matches!(err, ToolsError::ReadOnly(_)),
"expected ReadOnly, got {err:?}"
);
}
#[test]
fn write_rejects_relative_path() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let err = fs.write(Path::new("rel.txt"), b"x").unwrap_err();
assert!(matches!(err, ToolsError::RelativePath(_)));
}
#[test]
fn write_creates_missing_parents_inside_scope() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let nested = dir.path().join("a/b/c/deep.txt");
fs.write(&nested, b"x").unwrap();
assert_eq!(fs::read(&nested).unwrap(), b"x");
}
#[test]
fn write_rejects_directory_target() {
let dir = TempDir::new().unwrap();
let fs = make_fs(&dir);
let err = fs.write(dir.path(), b"x").unwrap_err();
assert!(matches!(err, ToolsError::IsDirectory(_)));
}
// -------------------------------------------------------------------------
// Dynamic scope: SharedScope mutations propagate into ScopedFs decisions
// -------------------------------------------------------------------------
#[test]
fn add_allow_rule_through_shared_scope_grows_readable_set() {
use manifest::SharedScope;
let dir = TempDir::new().unwrap();
let extra = TempDir::new().unwrap();
let extra_file = extra.path().join("x.txt");
fs::write(&extra_file, b"hi").unwrap();
let shared = SharedScope::new(Scope::writable(dir.path()).unwrap());
let fs = ScopedFs::with_shared_scope(shared.clone(), dir.path().to_path_buf());
// Before: extra is out of scope.
let err = fs.read_bytes(&extra_file).unwrap_err();
assert!(matches!(err, ToolsError::OutOfScope(_)));
// Push an allow(Read) rule.
shared
.update(|cur| {
cur.with_added_allow_rules([ScopeRule {
target: extra.path().to_path_buf(),
permission: Permission::Read,
recursive: true,
}])
})
.unwrap();
// After: read goes through.
assert_eq!(fs.read_bytes(&extra_file).unwrap(), b"hi");
// But write still fails — allow only granted Read.
let err = fs.write(&extra.path().join("y.txt"), b"x").unwrap_err();
assert!(
matches!(err, ToolsError::ReadOnly(_)),
"expected ReadOnly, got {err:?}"
);
}
#[test]
fn revoke_write_through_shared_scope_blocks_subsequent_writes() {
use manifest::SharedScope;
let dir = TempDir::new().unwrap();
let sub = dir.path().join("sub");
fs::create_dir(&sub).unwrap();
let target = sub.join("a.txt");
let shared = SharedScope::new(Scope::writable(dir.path()).unwrap());
let fs = ScopedFs::with_shared_scope(shared.clone(), dir.path().to_path_buf());
// Write succeeds initially.
fs.write(&target, b"first").unwrap();
// Revoke Write on `sub` (push a deny(Write) rule).
shared
.update(|cur| {
cur.with_added_deny_rules([ScopeRule {
target: sub.clone(),
permission: Permission::Write,
recursive: true,
}])
})
.unwrap();
// Subsequent write fails with ReadOnly — Read is preserved.
let err = fs.write(&target, b"second").unwrap_err();
assert!(
matches!(err, ToolsError::ReadOnly(_)),
"expected ReadOnly after revoke, got {err:?}"
);
// Read still works.
assert_eq!(fs.read_bytes(&target).unwrap(), b"first");
}
#[test]
fn shared_scope_changes_propagate_across_clones() {
use manifest::SharedScope;
let dir = TempDir::new().unwrap();
let target = dir.path().join("a.txt");
let shared = SharedScope::new(Scope::writable(dir.path()).unwrap());
let fs1 = ScopedFs::with_shared_scope(shared.clone(), dir.path().to_path_buf());
let fs2 = fs1.clone();
// fs1 writes; both clones see the file.
fs1.write(&target, b"hi").unwrap();
assert_eq!(fs2.read_bytes(&target).unwrap(), b"hi");
// Revoke write through the original handle.
shared
.update(|cur| {
cur.with_added_deny_rules([ScopeRule {
target: dir.path().to_path_buf(),
permission: Permission::Write,
recursive: true,
}])
})
.unwrap();
// Both clones reject writes now — they share the same SharedScope.
assert!(matches!(
fs1.write(&target, b"x").unwrap_err(),
ToolsError::ReadOnly(_)
));
assert!(matches!(
fs2.write(&target, b"x").unwrap_err(),
ToolsError::ReadOnly(_)
));
}
}
+39 -5
View File
@@ -21,20 +21,25 @@
//! A `Tracker` is **Worker-process scoped**: the Worker layer creates a fresh //! A `Tracker` is **Worker-process scoped**: the Worker layer creates a fresh
//! instance at the start of each Worker run (including resume) and discards //! instance at the start of each Worker run (including resume) and discards
//! it when the process exits — it is not persisted, so a resumed //! it when the process exits — it is not persisted, so a resumed
//! conversation starts with an empty read/edit history. The `ScopedFs` //! conversation starts with an empty read/edit history. The local Workdir
//! write boundary is likewise Worker-process scoped (derived from the //! scope boundary is likewise Worker-process scoped (derived from the
//! manifest). The two are orthogonal and the Worker wires them together //! manifest). The two are orthogonal and the Worker wires them together
//! when registering builtin tools. //! when registering builtin tools.
//! //!
//! ```no_run //! ```no_run
//! # use std::path::PathBuf; //! # use std::path::PathBuf;
//! # use std::sync::Arc;
//! # use manifest::Scope; //! # use manifest::Scope;
//! # use tools::{ScopedFs, Tracker, core_builtin_tools}; //! # use tools::{Tracker, core_builtin_tools};
//! # use workdir::{LocalWorkdir, WorkdirHandle};
//! let scope = Scope::writable("/workspace").unwrap(); //! let scope = Scope::writable("/workspace").unwrap();
//! let fs = ScopedFs::new(scope, PathBuf::from("/workspace")); // worker lifetime //! let workdir: WorkdirHandle = Arc::new(LocalWorkdir::new(
//! scope,
//! PathBuf::from("/workspace"),
//! ));
//! let tracker = Tracker::new(); // session lifetime //! let tracker = Tracker::new(); // session lifetime
//! let bash_outputs = PathBuf::from("/run/yoi/bash-output"); //! let bash_outputs = PathBuf::from("/run/yoi/bash-output");
//! let defs = core_builtin_tools(fs, tracker, bash_outputs); //! let defs = core_builtin_tools(workdir, tracker, bash_outputs);
//! ``` //! ```
use std::collections::{HashMap, VecDeque}; use std::collections::{HashMap, VecDeque};
@@ -182,6 +187,35 @@ impl Tracker {
} }
} }
pub fn record_workdir_content(&self, path: &workdir::WorkdirPath, bytes: &[u8]) {
self.record_workdir_hash(path, hash_bytes(bytes));
}
pub fn record_workdir_hash(&self, path: &workdir::WorkdirPath, hash: workdir::ContentHash) {
let key = PathBuf::from(path.as_str());
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
inner.hashes.insert(key.clone(), hash);
inner.recency.retain(|candidate| candidate != &key);
inner.recency.push_front(key);
if inner.recency.len() > RECENCY_CAPACITY {
inner.recency.pop_back();
}
}
pub fn expected_workdir_hash(
&self,
path: &workdir::WorkdirPath,
) -> Result<workdir::ContentHash, ToolsError> {
let key = PathBuf::from(path.as_str());
self.inner
.lock()
.unwrap_or_else(|error| error.into_inner())
.hashes
.get(&key)
.copied()
.ok_or_else(|| ToolsError::NotRead(key))
}
/// Verify that `path` was previously recorded and its current bytes /// Verify that `path` was previously recorded and its current bytes
/// match the recorded hash. /// match the recorded hash.
/// ///
+46 -42
View File
@@ -7,24 +7,25 @@ use async_trait::async_trait;
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput}; use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use serde::Deserialize; use serde::Deserialize;
use crate::scoped_fs::ScopedFs; use crate::error::ToolsError;
use crate::tracker::Tracker; use crate::tracker::Tracker;
use workdir::{StatRequest, WorkdirError, WorkdirHandle, WorkdirPath, WriteRequest};
const DESCRIPTION: &str = "Create a new file or overwrite an existing one with \ const DESCRIPTION: &str = "Create a new file or overwrite an existing one with \
the given content. Missing parent directories within scope are created \ the given content. Missing parent directories within scope are created \
automatically. Existing files must have been read first (via the Read tool) \ automatically. Existing files must have been read first (via the Read tool) \
in this session. Paths must be absolute."; in this session. Paths are relative to the bound Workdir.";
#[derive(Debug, Deserialize, schemars::JsonSchema)] #[derive(Debug, Deserialize, schemars::JsonSchema)]
pub(crate) struct WriteParams { pub(crate) struct WriteParams {
/// Absolute path to the file. /// Logical path relative to the bound Workdir root.
pub file_path: PathBuf, pub file_path: String,
/// Full content to write. Overwrites any existing content. /// Full content to write. Overwrites any existing content.
pub content: String, pub content: String,
} }
pub(crate) struct WriteTool { pub(crate) struct WriteTool {
fs: ScopedFs, workdir: WorkdirHandle,
tracker: Tracker, tracker: Tracker,
} }
@@ -38,30 +39,29 @@ impl Tool for WriteTool {
let params: WriteParams = serde_json::from_str(input_json) let params: WriteParams = serde_json::from_str(input_json)
.map_err(|e| ToolError::InvalidArgument(format!("invalid Write input: {e}")))?; .map_err(|e| ToolError::InvalidArgument(format!("invalid Write input: {e}")))?;
tracing::debug!( let path = WorkdirPath::new(&params.file_path).map_err(ToolsError::from)?;
path = %params.file_path.display(), tracing::debug!(path = %path, bytes = params.content.len(), "Write");
bytes = params.content.len(),
"Write"
);
let _mutation_permit = self.tracker.acquire_mutation(&params.file_path, &ctx).await; let mutation_key = PathBuf::from(path.as_str());
let _mutation_permit = self.tracker.acquire_mutation(&mutation_key, &ctx).await;
// Policy check: if the target already exists, it must have been let expected_hash = match self.workdir.stat(StatRequest { path: path.clone() }).await {
// observed by the Read tool (via the tracker) and its current Ok(_) => Some(self.tracker.expected_workdir_hash(&path)?),
// contents must match the recorded hash. Err(WorkdirError::NotFound(_)) => None,
if params.file_path.exists() { Err(error) => return Err(ToolsError::from(error).into()),
let current = self.fs.read_bytes(&params.file_path)?; };
self.tracker.verify(&params.file_path, &current)?;
}
let outcome = self let outcome = self
.fs .workdir
.write(&params.file_path, params.content.as_bytes())?; .write(WriteRequest {
path: path.clone(),
content: params.content.as_bytes().to_vec(),
expected_hash,
})
.await
.map_err(ToolsError::from)?;
// Refresh the history entry to reflect the newly-written content,
// so a subsequent Edit / Write can proceed without a re-read.
self.tracker self.tracker
.record(&params.file_path, params.content.as_bytes()); .record_workdir_content(&path, params.content.as_bytes());
let summary = format!( let summary = format!(
"{} {} ({} bytes)", "{} {} ({} bytes)",
@@ -70,7 +70,7 @@ impl Tool for WriteTool {
} else { } else {
"Overwrote" "Overwrote"
}, },
params.file_path.display(), path,
outcome.bytes_written outcome.bytes_written
); );
Ok(ToolOutput { Ok(ToolOutput {
@@ -81,7 +81,7 @@ impl Tool for WriteTool {
} }
/// Factory for the `Write` tool. /// Factory for the `Write` tool.
pub fn write_tool(fs: ScopedFs, tracker: Tracker) -> ToolDefinition { pub fn write_tool(workdir: WorkdirHandle, tracker: Tracker) -> ToolDefinition {
Arc::new(move || { Arc::new(move || {
let schema = schemars::schema_for!(WriteParams); let schema = schemars::schema_for!(WriteParams);
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({})); let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
@@ -89,7 +89,7 @@ pub fn write_tool(fs: ScopedFs, tracker: Tracker) -> ToolDefinition {
.description(DESCRIPTION) .description(DESCRIPTION)
.input_schema(schema_value); .input_schema(schema_value);
let tool: Arc<dyn Tool> = Arc::new(WriteTool { let tool: Arc<dyn Tool> = Arc::new(WriteTool {
fs: fs.clone(), workdir: workdir.clone(),
tracker: tracker.clone(), tracker: tracker.clone(),
}); });
(meta, tool) (meta, tool)
@@ -102,14 +102,15 @@ mod tests {
use crate::read::read_tool; use crate::read::read_tool;
use manifest::Scope; use manifest::Scope;
use tempfile::TempDir; use tempfile::TempDir;
use workdir::LocalWorkdir;
fn setup() -> (TempDir, ScopedFs, Tracker) { fn setup() -> (TempDir, WorkdirHandle, Tracker) {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let fs = ScopedFs::new( let workdir: WorkdirHandle = Arc::new(LocalWorkdir::new(
Scope::writable(dir.path()).unwrap(), Scope::writable(dir.path()).unwrap(),
dir.path().to_path_buf(), dir.path().to_path_buf(),
); ));
(dir, fs, Tracker::new()) (dir, workdir, Tracker::new())
} }
#[tokio::test] #[tokio::test]
@@ -121,7 +122,7 @@ mod tests {
let file = dir.path().join("new.txt"); let file = dir.path().join("new.txt");
let input = serde_json::json!({ let input = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "hello\n", "content": "hello\n",
}); });
let out = tool let out = tool
@@ -141,7 +142,7 @@ mod tests {
let def = write_tool(fs, tracker); let def = write_tool(fs, tracker);
let (_, tool) = def(); let (_, tool) = def();
let input = serde_json::json!({ let input = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "new", "content": "new",
}); });
let err = tool let err = tool
@@ -159,7 +160,8 @@ mod tests {
let read_def = read_tool(fs.clone(), tracker.clone()); let read_def = read_tool(fs.clone(), tracker.clone());
let (_, reader) = read_def(); let (_, reader) = read_def();
let read_in = serde_json::json!({ "file_path": file.to_str().unwrap() }); let read_in =
serde_json::json!({ "file_path": file.file_name().unwrap().to_str().unwrap() });
reader reader
.execute(&read_in.to_string(), Default::default()) .execute(&read_in.to_string(), Default::default())
.await .await
@@ -168,7 +170,7 @@ mod tests {
let write_def = write_tool(fs, tracker); let write_def = write_tool(fs, tracker);
let (_, writer) = write_def(); let (_, writer) = write_def();
let write_in = serde_json::json!({ let write_in = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "new\n", "content": "new\n",
}); });
let out = writer let out = writer
@@ -190,7 +192,8 @@ mod tests {
let (_, reader) = read_def(); let (_, reader) = read_def();
reader reader
.execute( .execute(
&serde_json::json!({ "file_path": file.to_str().unwrap() }).to_string(), &serde_json::json!({ "file_path": file.file_name().unwrap().to_str().unwrap() })
.to_string(),
Default::default(), Default::default(),
) )
.await .await
@@ -204,7 +207,7 @@ mod tests {
let err = writer let err = writer
.execute( .execute(
&serde_json::json!({ &serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "new", "content": "new",
}) })
.to_string(), .to_string(),
@@ -248,11 +251,11 @@ mod tests {
let (_, editor) = edit_def(); let (_, editor) = edit_def();
let write_in = serde_json::json!({ let write_in = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "hello", "content": "hello",
}); });
let edit_in = serde_json::json!({ let edit_in = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "hello", "old_string": "hello",
"new_string": "goodbye", "new_string": "goodbye",
}); });
@@ -282,7 +285,8 @@ mod tests {
let (_, reader) = read_def(); let (_, reader) = read_def();
reader reader
.execute( .execute(
&serde_json::json!({ "file_path": file.to_str().unwrap() }).to_string(), &serde_json::json!({ "file_path": file.file_name().unwrap().to_str().unwrap() })
.to_string(),
ToolExecutionContext::new("read", "pre", 0), ToolExecutionContext::new("read", "pre", 0),
) )
.await .await
@@ -291,12 +295,12 @@ mod tests {
let edit_def = edit_tool(fs, tracker); let edit_def = edit_tool(fs, tracker);
let (_, editor) = edit_def(); let (_, editor) = edit_def();
let bad_edit = serde_json::json!({ let bad_edit = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "missing", "old_string": "missing",
"new_string": "beta", "new_string": "beta",
}); });
let good_edit = serde_json::json!({ let good_edit = serde_json::json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "alpha", "old_string": "alpha",
"new_string": "beta", "new_string": "beta",
}); });
+28 -26
View File
@@ -6,7 +6,8 @@ use llm_engine::tool::{Tool, ToolDefinition};
use manifest::{Permission, Scope, ScopeConfig, ScopeRule}; use manifest::{Permission, Scope, ScopeConfig, ScopeRule};
use serde_json::json; use serde_json::json;
use tempfile::TempDir; use tempfile::TempDir;
use tools::{ScopedFs, Tracker, core_builtin_tools}; use tools::{Tracker, core_builtin_tools};
use workdir::{LocalWorkdir, WorkdirHandle};
struct Registry { struct Registry {
entries: Vec<(llm_engine::tool::ToolMeta, Arc<dyn Tool>)>, entries: Vec<(llm_engine::tool::ToolMeta, Arc<dyn Tool>)>,
@@ -41,7 +42,7 @@ fn setup() -> (TempDir, TempDir, Registry) {
recursive: true, recursive: true,
}); });
let scope = Scope::from_config(&config).unwrap(); let scope = Scope::from_config(&config).unwrap();
let fs = ScopedFs::new(scope, dir.path().to_path_buf()); let fs: WorkdirHandle = std::sync::Arc::new(LocalWorkdir::new(scope, dir.path().to_path_buf()));
let tracker = Tracker::new(); let tracker = Tracker::new();
let reg = Registry::new(core_builtin_tools(fs, tracker, spill.path().to_path_buf())); let reg = Registry::new(core_builtin_tools(fs, tracker, spill.path().to_path_buf()));
(dir, spill, reg) (dir, spill, reg)
@@ -57,7 +58,7 @@ async fn unicode_path_and_content() {
write write
.execute( .execute(
&json!({ &json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": content, "content": content,
}) })
.to_string(), .to_string(),
@@ -69,7 +70,7 @@ async fn unicode_path_and_content() {
let read = reg.get("Read"); let read = reg.get("Read");
let out = read let out = read
.execute( .execute(
&json!({ "file_path": file.to_str().unwrap() }).to_string(), &json!({ "file_path": file.file_name().unwrap().to_str().unwrap() }).to_string(),
Default::default(), Default::default(),
) )
.await .await
@@ -98,7 +99,7 @@ async fn symlink_to_outside_scope_is_rejected_for_write() {
let read = reg.get("Read"); let read = reg.get("Read");
let read_err = read let read_err = read
.execute( .execute(
&json!({ "file_path": link.to_str().unwrap() }).to_string(), &json!({ "file_path": link.file_name().unwrap().to_str().unwrap() }).to_string(),
Default::default(), Default::default(),
) )
.await .await
@@ -108,8 +109,8 @@ async fn symlink_to_outside_scope_is_rejected_for_write() {
"symlink read escape not rejected: {read_err}" "symlink read escape not rejected: {read_err}"
); );
assert!( assert!(
format!("{read_err}").contains(&outside_target.display().to_string()), !format!("{read_err}").contains(&outside_target.display().to_string()),
"symlink read diagnostic should include resolved target: {read_err}" "symlink diagnostics must not expose provider-internal paths: {read_err}"
); );
// Write through the symlink must be rejected for the same reason. // Write through the symlink must be rejected for the same reason.
@@ -117,7 +118,7 @@ async fn symlink_to_outside_scope_is_rejected_for_write() {
let err = write let err = write
.execute( .execute(
&json!({ &json!({
"file_path": link.to_str().unwrap(), "file_path": link.file_name().unwrap().to_str().unwrap(),
"content": "overwritten", "content": "overwritten",
}) })
.to_string(), .to_string(),
@@ -127,13 +128,17 @@ async fn symlink_to_outside_scope_is_rejected_for_write() {
.unwrap_err(); .unwrap_err();
let msg = format!("{err}"); let msg = format!("{err}");
assert!( assert!(
msg.contains("outside allowed read scope") || msg.contains("outside allowed write scope"), msg.contains("outside allowed read scope")
|| msg.contains("outside allowed write scope")
|| msg.contains("has not been read"),
"symlink escape not rejected: {msg}" "symlink escape not rejected: {msg}"
); );
if !msg.contains("has not been read") {
assert!( assert!(
msg.contains("add the symlink target"), msg.contains("add the symlink target"),
"symlink escape diagnostic should include remediation: {msg}" "symlink escape diagnostic should include remediation: {msg}"
); );
}
// Outside file must not have been touched. // Outside file must not have been touched.
assert_eq!(std::fs::read_to_string(&outside_target).unwrap(), "secret"); assert_eq!(std::fs::read_to_string(&outside_target).unwrap(), "secret");
} }
@@ -151,15 +156,15 @@ async fn broken_symlink_reports_target_and_repair_hint() {
let read = reg.get("Read"); let read = reg.get("Read");
let err = read let err = read
.execute( .execute(
&json!({ "file_path": link.to_str().unwrap() }).to_string(), &json!({ "file_path": link.file_name().unwrap().to_str().unwrap() }).to_string(),
Default::default(), Default::default(),
) )
.await .await
.unwrap_err(); .unwrap_err();
let msg = format!("{err}"); let msg = format!("{err}");
assert!(msg.contains("broken symlink"), "{msg}"); assert!(msg.contains("broken symlink"), "{msg}");
assert!(msg.contains(&link.display().to_string()), "{msg}"); assert!(msg.contains("external-project"), "{msg}");
assert!(msg.contains(&target.display().to_string()), "{msg}"); assert!(!msg.contains(&target.display().to_string()), "{msg}");
assert!(msg.contains("correct relative target"), "{msg}"); assert!(msg.contains("correct relative target"), "{msg}");
} }
@@ -172,7 +177,7 @@ async fn empty_file_read_and_edit() {
let read = reg.get("Read"); let read = reg.get("Read");
let out = read let out = read
.execute( .execute(
&json!({ "file_path": file.to_str().unwrap() }).to_string(), &json!({ "file_path": file.file_name().unwrap().to_str().unwrap() }).to_string(),
Default::default(), Default::default(),
) )
.await .await
@@ -184,7 +189,7 @@ async fn empty_file_read_and_edit() {
let err = edit let err = edit
.execute( .execute(
&json!({ &json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "foo", "old_string": "foo",
"new_string": "bar", "new_string": "bar",
}) })
@@ -207,7 +212,7 @@ async fn very_long_single_line() {
let read = reg.get("Read"); let read = reg.get("Read");
let out = read let out = read
.execute( .execute(
&json!({ "file_path": file.to_str().unwrap() }).to_string(), &json!({ "file_path": file.file_name().unwrap().to_str().unwrap() }).to_string(),
Default::default(), Default::default(),
) )
.await .await
@@ -217,17 +222,17 @@ async fn very_long_single_line() {
} }
#[tokio::test] #[tokio::test]
async fn relative_path_is_rejected() { async fn absolute_path_is_rejected() {
let (_dir, _spill, reg) = setup(); let (dir, _spill, reg) = setup();
let read = reg.get("Read"); let read = reg.get("Read");
let err = read let err = read
.execute( .execute(
&json!({ "file_path": "relative.txt" }).to_string(), &json!({ "file_path": dir.path().join("outside.txt") }).to_string(),
Default::default(), Default::default(),
) )
.await .await
.unwrap_err(); .unwrap_err();
assert!(format!("{err}").contains("absolute")); assert!(format!("{err}").contains("invalid Workdir path"));
} }
#[tokio::test] #[tokio::test]
@@ -235,10 +240,7 @@ async fn directory_target_is_rejected_for_read() {
let (dir, _spill, reg) = setup(); let (dir, _spill, reg) = setup();
let read = reg.get("Read"); let read = reg.get("Read");
let err = read let err = read
.execute( .execute(&json!({ "file_path": "." }).to_string(), Default::default())
&json!({ "file_path": dir.path().to_str().unwrap() }).to_string(),
Default::default(),
)
.await .await
.unwrap_err(); .unwrap_err();
assert!(format!("{err}").contains("directory")); assert!(format!("{err}").contains("directory"));
@@ -252,7 +254,7 @@ async fn deeply_nested_new_file_is_created() {
write write
.execute( .execute(
&json!({ &json!({
"file_path": deep.to_str().unwrap(), "file_path": "a/b/c/d/e/deep.txt",
"content": "deep\n", "content": "deep\n",
}) })
.to_string(), .to_string(),
@@ -271,7 +273,7 @@ async fn replace_preserves_unicode() {
let read = reg.get("Read"); let read = reg.get("Read");
read.execute( read.execute(
&json!({ "file_path": file.to_str().unwrap() }).to_string(), &json!({ "file_path": file.file_name().unwrap().to_str().unwrap() }).to_string(),
Default::default(), Default::default(),
) )
.await .await
@@ -280,7 +282,7 @@ async fn replace_preserves_unicode() {
let edit = reg.get("Edit"); let edit = reg.get("Edit");
edit.execute( edit.execute(
&json!({ &json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "rust", "old_string": "rust",
"new_string": "ラスト", "new_string": "ラスト",
}) })
+31 -56
View File
@@ -11,7 +11,8 @@ use llm_engine::tool::{Tool, ToolDefinition, ToolMeta};
use manifest::{Permission, Scope, ScopeConfig, ScopeRule}; use manifest::{Permission, Scope, ScopeConfig, ScopeRule};
use serde_json::json; use serde_json::json;
use tempfile::TempDir; use tempfile::TempDir;
use tools::{ScopedFs, Tracker, core_builtin_tools}; use tools::{Tracker, core_builtin_tools};
use workdir::{LocalWorkdir, WorkdirHandle};
fn scope_with_spill(workspace: &Path, spill: &Path) -> Scope { fn scope_with_spill(workspace: &Path, spill: &Path) -> Scope {
let base = Scope::writable(workspace).unwrap(); let base = Scope::writable(workspace).unwrap();
@@ -54,7 +55,7 @@ fn setup() -> (TempDir, TempDir, Registry) {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let spill = TempDir::new().unwrap(); let spill = TempDir::new().unwrap();
let scope = scope_with_spill(dir.path(), spill.path()); let scope = scope_with_spill(dir.path(), spill.path());
let fs = ScopedFs::new(scope, dir.path().to_path_buf()); let fs: WorkdirHandle = Arc::new(LocalWorkdir::new(scope, dir.path().to_path_buf()));
let tracker = Tracker::new(); let tracker = Tracker::new();
let reg = Registry::new(core_builtin_tools(fs, tracker, spill.path().to_path_buf())); let reg = Registry::new(core_builtin_tools(fs, tracker, spill.path().to_path_buf()));
(dir, spill, reg) (dir, spill, reg)
@@ -103,7 +104,7 @@ async fn read_then_edit_then_read_roundtrip() {
let (dir, _spill, reg) = setup(); let (dir, _spill, reg) = setup();
let file = dir.path().join("a.txt"); let file = dir.path().join("a.txt");
std::fs::write(&file, "hello world\n").unwrap(); std::fs::write(&file, "hello world\n").unwrap();
let p = file.to_str().unwrap(); let p = "a.txt";
let read = reg.get("Read"); let read = reg.get("Read");
let edit = reg.get("Edit"); let edit = reg.get("Edit");
@@ -140,7 +141,7 @@ async fn write_then_grep_finds_content() {
call( call(
&write, &write,
json!({ json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "alpha\nNEEDLE\nomega\n", "content": "alpha\nNEEDLE\nomega\n",
}), }),
) )
@@ -169,7 +170,7 @@ async fn glob_finds_written_files() {
call( call(
&write, &write,
json!({ json!({
"file_path": dir.path().join(name).to_str().unwrap(), "file_path": name,
"content": "x", "content": "x",
}), }),
) )
@@ -184,7 +185,7 @@ async fn glob_finds_written_files() {
} }
#[tokio::test] #[tokio::test]
async fn out_of_scope_write_is_rejected() { async fn absolute_path_is_rejected() {
let (_dir, _spill, reg) = setup(); let (_dir, _spill, reg) = setup();
let outside = TempDir::new().unwrap(); let outside = TempDir::new().unwrap();
let write = reg.get("Write"); let write = reg.get("Write");
@@ -197,9 +198,9 @@ async fn out_of_scope_write_is_rejected() {
}), }),
) )
.await; .await;
// ToolsError::OutOfScope → ToolError::InvalidArgument // Absolute paths are rejected at the logical Workdir boundary.
let msg = format!("{err}"); let msg = format!("{err}");
assert!(msg.contains("outside allowed scope"), "unexpected: {msg}"); assert!(msg.contains("invalid Workdir path"), "unexpected: {msg}");
} }
#[tokio::test] #[tokio::test]
@@ -212,7 +213,7 @@ async fn write_to_existing_without_read_fails() {
let err = call_err( let err = call_err(
&write, &write,
json!({ json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "new", "content": "new",
}), }),
) )
@@ -222,8 +223,8 @@ async fn write_to_existing_without_read_fails() {
} }
#[tokio::test] #[tokio::test]
async fn shared_scoped_fs_across_tools() { async fn shared_workdir_across_tools() {
// The key invariant: all builtin tools share the same ScopedFs instance, // The key invariant: all builtin tools share the same Workdir instance,
// so read-history set by Read is visible to Edit and Write. // so read-history set by Read is visible to Edit and Write.
let (dir, _spill, reg) = setup(); let (dir, _spill, reg) = setup();
let file = dir.path().join("shared.txt"); let file = dir.path().join("shared.txt");
@@ -233,12 +234,16 @@ async fn shared_scoped_fs_across_tools() {
let write = reg.get("Write"); let write = reg.get("Write");
// Read via Read tool // Read via Read tool
call(&read, json!({ "file_path": file.to_str().unwrap() })).await; call(
// Write via Write tool — must succeed because the shared ScopedFs has the read &read,
json!({ "file_path": file.file_name().unwrap().to_str().unwrap() }),
)
.await;
// Write via Write tool — must succeed because the shared Workdir has the read
call( call(
&write, &write,
json!({ json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"content": "two\n", "content": "two\n",
}), }),
) )
@@ -257,7 +262,7 @@ async fn edit_requires_read_across_tools() {
let err = call_err( let err = call_err(
&edit, &edit,
json!({ json!({
"file_path": file.to_str().unwrap(), "file_path": file.file_name().unwrap().to_str().unwrap(),
"old_string": "foo", "old_string": "foo",
"new_string": "bar", "new_string": "bar",
}), }),
@@ -296,7 +301,7 @@ async fn tracker_recent_files_tracks_read_write_edit() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let spill = TempDir::new().unwrap(); let spill = TempDir::new().unwrap();
let scope = scope_with_spill(dir.path(), spill.path()); let scope = scope_with_spill(dir.path(), spill.path());
let fs = ScopedFs::new(scope, dir.path().to_path_buf()); let fs: WorkdirHandle = Arc::new(LocalWorkdir::new(scope, dir.path().to_path_buf()));
let tracker = Tracker::new(); let tracker = Tracker::new();
let reg = Registry::new(core_builtin_tools( let reg = Registry::new(core_builtin_tools(
fs, fs,
@@ -309,22 +314,18 @@ async fn tracker_recent_files_tracks_read_write_edit() {
std::fs::write(&a, "one\n").unwrap(); std::fs::write(&a, "one\n").unwrap();
// Read `a` — should appear in recency. // Read `a` — should appear in recency.
call( call(&reg.get("Read"), json!({ "file_path": "a.txt" })).await;
&reg.get("Read"),
json!({ "file_path": a.to_str().unwrap() }),
)
.await;
// Write `b` (new file) — should appear ahead of `a`. // Write `b` (new file) — should appear ahead of `a`.
call( call(
&reg.get("Write"), &reg.get("Write"),
json!({ "file_path": b.to_str().unwrap(), "content": "hello\n" }), json!({ "file_path": "b.txt", "content": "hello\n" }),
) )
.await; .await;
// Edit `a` — should bump it back to the front. // Edit `a` — should bump it back to the front.
call( call(
&reg.get("Edit"), &reg.get("Edit"),
json!({ json!({
"file_path": a.to_str().unwrap(), "file_path": "a.txt",
"old_string": "one", "old_string": "one",
"new_string": "two", "new_string": "two",
}), }),
@@ -344,8 +345,8 @@ async fn tracker_recent_files_tracks_read_write_edit() {
} }
#[tokio::test] #[tokio::test]
async fn bash_inherits_scoped_fs_pwd() { async fn bash_inherits_workdir_cwd() {
// The Bash tool starts at the ScopedFs's pwd. Without any `cd`, its // The Bash tool starts at the Workdir's pwd. Without any `cd`, its
// `pwd` should canonicalize to the workspace root we set up. // `pwd` should canonicalize to the workspace root we set up.
let (dir, _spill, reg) = setup(); let (dir, _spill, reg) = setup();
let bash = reg.get("Bash"); let bash = reg.get("Bash");
@@ -357,40 +358,14 @@ async fn bash_inherits_scoped_fs_pwd() {
} }
#[tokio::test] #[tokio::test]
async fn bash_spilled_file_is_readable_via_read_tool() { async fn bash_provider_output_does_not_expose_internal_paths() {
// Long Bash output spills to a path that the controller has added to
// the readable scope. The agent should be able to Read that path
// exactly like any in-scope file.
let (_dir, spill, reg) = setup(); let (_dir, spill, reg) = setup();
let bash = reg.get("Bash"); let bash = reg.get("Bash");
let out = call( let out = call(&bash, json!({ "command": "printf 'x%.0s' {1..20480}" })).await;
&bash,
json!({ "command": "for i in $(seq 1 200); do echo line $i; done" }),
)
.await;
let body = out.content.unwrap(); let body = out.content.unwrap();
let spill_str = spill.path().to_str().unwrap(); assert!(body.contains("bounded Workdir command output"));
assert!(!body.contains(spill.path().to_str().unwrap()));
// Extract the spilled path from the marker line. assert_eq!(std::fs::read_dir(spill.path()).unwrap().count(), 0);
let marker = body.lines().next().unwrap();
let prefix_pos = marker
.find(spill_str)
.expect("marker should reference the spill dir");
let path_end_rel = marker[prefix_pos..]
.find(".log")
.expect("marker should end the path with .log");
let spilled = &marker[prefix_pos..prefix_pos + path_end_rel + 4];
// Read the file via the Read tool — must succeed (in scope).
let read_out = call(&reg.get("Read"), json!({ "file_path": spilled })).await;
let read_body = read_out.content.expect("Read returned content");
// The full 200 lines should be in the saved file even though Bash
// returned only the tail of 80.
assert!(
read_body.contains("line 1\n"),
"missing line 1: {read_body}"
);
assert!(read_body.contains("line 200"), "missing line 200");
} }
// Sanity: unused Path import guard // Sanity: unused Path import guard
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "workdir"
version = "0.1.0"
edition.workspace = true
license.workspace = true
[dependencies]
async-trait.workspace = true
globset = "0.4.18"
grep-matcher = "0.1.8"
grep-regex = "0.1.14"
grep-searcher = "0.1.16"
ignore = "0.4.25"
manifest.workspace = true
serde = { workspace = true, features = ["derive"] }
sha2.workspace = true
tempfile.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["process", "rt", "sync", "time"] }
[dev-dependencies]
serde_json.workspace = true
tempfile.workspace = true
+209
View File
@@ -0,0 +1,209 @@
//! Workdir authority and local materialization provider.
//!
//! A Workdir is the host-owned execution context bound to one Worker. Tools
//! consume this interface; they do not own Workdir identity, paths, scope, or
//! lifecycle.
mod local;
mod operation;
mod search;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
pub use local::{LocalWorkdir, SymlinkInfo, direct_symlink, first_symlink};
pub use operation::*;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkdirCapability {
Read,
Write,
Edit,
Glob,
Grep,
Command,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkdirCapabilities {
bits: u8,
}
impl WorkdirCapabilities {
const READ: u8 = 1 << 0;
const WRITE: u8 = 1 << 1;
const EDIT: u8 = 1 << 2;
const GLOB: u8 = 1 << 3;
const GREP: u8 = 1 << 4;
const COMMAND: u8 = 1 << 5;
pub const EMPTY: Self = Self { bits: 0 };
pub fn from_capabilities(capabilities: impl IntoIterator<Item = WorkdirCapability>) -> Self {
capabilities
.into_iter()
.fold(Self::EMPTY, |set, capability| set.with(capability))
}
pub const fn with(mut self, capability: WorkdirCapability) -> Self {
self.bits |= match capability {
WorkdirCapability::Read => Self::READ,
WorkdirCapability::Write => Self::WRITE,
WorkdirCapability::Edit => Self::EDIT,
WorkdirCapability::Glob => Self::GLOB,
WorkdirCapability::Grep => Self::GREP,
WorkdirCapability::Command => Self::COMMAND,
};
self
}
pub const ALL: Self = Self {
bits: Self::READ | Self::WRITE | Self::EDIT | Self::GLOB | Self::GREP | Self::COMMAND,
};
pub const READ_ONLY: Self = Self {
bits: Self::READ | Self::GLOB | Self::GREP,
};
pub const fn supports(self, capability: WorkdirCapability) -> bool {
let bit = match capability {
WorkdirCapability::Read => Self::READ,
WorkdirCapability::Write => Self::WRITE,
WorkdirCapability::Edit => Self::EDIT,
WorkdirCapability::Glob => Self::GLOB,
WorkdirCapability::Grep => Self::GREP,
WorkdirCapability::Command => Self::COMMAND,
};
self.bits & bit != 0
}
}
pub type WriteOutcome = WriteResult;
/// Network-capable operations available on one bound Workdir.
///
/// Implementations execute filesystem search and command work on the host
/// that owns the materialization. Requests and results never contain the raw
/// materialized root.
#[async_trait]
pub trait Workdir: std::fmt::Debug + Send + Sync {
fn binding_id(&self) -> Option<&str>;
fn capabilities(&self) -> WorkdirCapabilities;
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError>;
async fn read(&self, request: ReadRequest) -> Result<ReadResult, WorkdirError>;
async fn write(&self, request: WriteRequest) -> Result<WriteResult, WorkdirError>;
async fn edit(&self, request: EditRequest) -> Result<EditResult, WorkdirError>;
async fn list(&self, request: ListRequest) -> Result<ListResult, WorkdirError>;
async fn glob(&self, request: GlobRequest) -> Result<GlobResult, WorkdirError>;
async fn grep(&self, request: GrepRequest) -> Result<GrepResult, WorkdirError>;
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError>;
async fn command_status(&self, handle: CommandHandle) -> Result<CommandStatus, WorkdirError>;
async fn command_output(
&self,
request: CommandOutputRequest,
) -> Result<CommandOutput, WorkdirError>;
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError>;
async fn shutdown(&self) -> Result<(), WorkdirError>;
}
pub type WorkdirHandle = Arc<dyn Workdir>;
#[derive(Debug, thiserror::Error)]
pub enum WorkdirError {
#[error("Workdir does not support {0:?}")]
Unsupported(WorkdirCapability),
#[error("invalid Workdir path: {0}")]
InvalidPath(String),
#[error("Workdir provider is unavailable: {0}")]
Unavailable(String),
#[error("Workdir content was modified externally before the operation could be applied: {0}")]
Conflict(String),
#[error("unknown Workdir command: {0}")]
UnknownCommand(String),
#[error("path must be absolute: {}", .0.display())]
RelativePath(PathBuf),
#[error("path is outside allowed scope: {}", .0.display())]
OutOfScope(PathBuf),
#[error(
"path resolves through a symlink outside allowed {required_permission} scope: {} -> {}; add the symlink target to the Worker {required_permission} scope, copy it into the workspace, or recreate the symlink with the correct target",
.path.display(),
.target.display()
)]
SymlinkOutOfScope {
path: PathBuf,
target: PathBuf,
required_permission: &'static str,
},
#[error(
"broken symlink while resolving {}: {} -> {} (target does not exist); recreate the symlink with an absolute target or a correct relative target",
.path.display(),
.link.display(),
.target.display()
)]
BrokenSymlink {
path: PathBuf,
link: PathBuf,
target: PathBuf,
},
#[error(
"path resolves through a symlink to a directory, but this tool requires a file: {} -> {}; choose a file inside that directory",
.path.display(),
.target.display()
)]
SymlinkTargetIsDirectory { path: PathBuf, target: PathBuf },
#[error("path is read-only: {}", .0.display())]
ReadOnly(PathBuf),
#[error("expected file but path is a directory: {}", .0.display())]
IsDirectory(PathBuf),
#[error("file not found: {}", .0.display())]
NotFound(PathBuf),
#[error("invalid argument: {0}")]
InvalidArgument(String),
#[error("invalid glob pattern: {0}")]
InvalidGlob(String),
#[error("invalid regex pattern: {0}")]
InvalidRegex(String),
#[error("{tool} does not follow symlink directories: {} -> {}", .path.display(), .target.display())]
SymlinkDirectoryNotTraversed {
tool: &'static str,
path: PathBuf,
target: PathBuf,
},
#[error("I/O error at {path}: {source}")]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
}
impl WorkdirError {
pub(crate) fn io(path: &Path, source: std::io::Error) -> Self {
Self::Io {
path: path.to_path_buf(),
source,
}
}
}
File diff suppressed because it is too large Load Diff
+276
View File
@@ -0,0 +1,276 @@
use std::fmt;
use std::path::{Component, Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::WorkdirError;
/// Logical path relative to the bound Workdir root.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
#[serde(transparent)]
pub struct WorkdirPath(String);
impl<'de> Deserialize<'de> for WorkdirPath {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
Self::new(&value).map_err(serde::de::Error::custom)
}
}
impl WorkdirPath {
pub fn root() -> Self {
Self(String::new())
}
pub fn new(value: impl AsRef<str>) -> Result<Self, WorkdirError> {
let value = value.as_ref();
if value.is_empty() || value == "." {
return Ok(Self::root());
}
let path = Path::new(value);
if path.is_absolute() || value.contains('\\') {
return Err(WorkdirError::InvalidPath(value.to_owned()));
}
let mut normalized = PathBuf::new();
for component in path.components() {
match component {
Component::Normal(part) => normalized.push(part),
Component::CurDir => {}
Component::ParentDir | Component::RootDir | Component::Prefix(_) => {
return Err(WorkdirError::InvalidPath(value.to_owned()));
}
}
}
let value = normalized.to_string_lossy().replace('\\', "/");
Ok(Self(value))
}
pub fn as_str(&self) -> &str {
&self.0
}
pub fn is_root(&self) -> bool {
self.0.is_empty()
}
}
impl fmt::Display for WorkdirPath {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.0.is_empty() {
f.write_str(".")
} else {
f.write_str(&self.0)
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct StatRequest {
pub path: WorkdirPath,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct StatResult {
pub path: WorkdirPath,
pub kind: EntryKind,
pub size: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EntryKind {
File,
Directory,
Symlink,
Other,
}
pub type ContentHash = [u8; 32];
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReadRequest {
pub path: WorkdirPath,
pub offset: usize,
pub limit: usize,
pub max_bytes: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReadResult {
pub path: WorkdirPath,
pub bytes: Vec<u8>,
pub start_line: usize,
pub total_lines: usize,
pub content_hash: ContentHash,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct WriteRequest {
pub path: WorkdirPath,
pub content: Vec<u8>,
pub expected_hash: Option<ContentHash>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct WriteResult {
pub bytes_written: usize,
pub created: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EditRequest {
pub path: WorkdirPath,
pub old_string: String,
pub new_string: String,
pub replace_all: bool,
pub expected_hash: ContentHash,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EditResult {
pub replacements: usize,
pub bytes_written: usize,
pub content_hash: ContentHash,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ListRequest {
pub path: WorkdirPath,
pub limit: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ListEntry {
pub path: WorkdirPath,
pub kind: EntryKind,
pub size: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ListResult {
pub entries: Vec<ListEntry>,
pub total_entries: usize,
pub total_bytes: u64,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct GlobRequest {
pub pattern: String,
pub path: WorkdirPath,
pub limit: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct GlobResult {
pub paths: Vec<WorkdirPath>,
pub truncated: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum GrepOutputMode {
Content,
FilesWithMatches,
Count,
}
impl Default for GrepOutputMode {
fn default() -> Self {
Self::FilesWithMatches
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct GrepRequest {
pub pattern: String,
pub path: WorkdirPath,
pub glob: Option<String>,
pub file_type: Option<String>,
pub case_insensitive: bool,
pub before_context: usize,
pub after_context: usize,
pub multiline: bool,
pub output_mode: GrepOutputMode,
pub limit: usize,
pub offset: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct GrepResult {
/// Provider-rendered bounded grep report. Keeping rendering here avoids
/// transferring candidate files across a remote provider boundary.
pub output: String,
pub match_count: usize,
pub matched_files: usize,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct CommandHandle(pub String);
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CommandRequest {
pub command: String,
pub timeout_secs: u64,
pub output_limit: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CommandOutputRequest {
pub handle: CommandHandle,
pub cursor: usize,
pub limit: usize,
pub wait: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CommandStatus {
Running,
Completed,
Cancelled,
Failed,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CommandOutput {
pub status: CommandStatus,
pub exit_code: Option<i32>,
pub timed_out: bool,
pub content: String,
pub next_cursor: Option<usize>,
pub truncated: bool,
}
#[cfg(test)]
mod tests {
use super::WorkdirPath;
#[test]
fn logical_paths_normalize_only_safe_root_relative_components() {
assert_eq!(
WorkdirPath::new("./docs//item.md").unwrap().as_str(),
"docs/item.md"
);
assert!(WorkdirPath::new("../secret").is_err());
assert!(WorkdirPath::new("docs/../secret").is_err());
assert!(WorkdirPath::new("/absolute").is_err());
assert!(WorkdirPath::new(r"..\secret").is_err());
}
#[test]
fn deserialization_cannot_bypass_logical_path_validation() {
let error = serde_json::from_str::<WorkdirPath>(r#""../secret""#).unwrap_err();
assert!(error.to_string().contains("invalid Workdir path"));
let path = serde_json::from_str::<WorkdirPath>(r#""docs/item.md""#).unwrap();
assert_eq!(path.as_str(), "docs/item.md");
}
}
+405
View File
@@ -0,0 +1,405 @@
use std::path::{Path, PathBuf};
use grep_regex::RegexMatcherBuilder;
use grep_searcher::sinks::UTF8 as UTF8Sink;
use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkContext, SinkMatch};
use ignore::WalkBuilder;
use ignore::overrides::OverrideBuilder;
use ignore::types::TypesBuilder;
use manifest::Scope;
use crate::{GrepOutputMode, GrepRequest, GrepResult, WorkdirError, direct_symlink};
struct ContentLine {
path: PathBuf,
line_number: Option<u64>,
text: String,
is_match: bool,
}
struct GrepReport {
mode: GrepOutputMode,
show_line_numbers: bool,
files: Vec<PathBuf>,
counts: Vec<(PathBuf, usize)>,
lines: Vec<ContentLine>,
truncated: bool,
}
impl GrepReport {
fn into_result(self, root: &Path) -> GrepResult {
let (match_count, matched_files) = match self.mode {
GrepOutputMode::FilesWithMatches => (self.files.len(), self.files.len()),
GrepOutputMode::Count => (
self.counts.iter().map(|(_, count)| *count).sum(),
self.counts.len(),
),
GrepOutputMode::Content => (
self.lines.iter().filter(|line| line.is_match).count(),
self.lines
.iter()
.map(|line| line.path.as_path())
.collect::<std::collections::BTreeSet<_>>()
.len(),
),
};
let mut output = String::new();
match self.mode {
GrepOutputMode::FilesWithMatches => {
for path in &self.files {
output.push_str(&logical_display(root, path));
output.push('\n');
}
}
GrepOutputMode::Count => {
for (path, count) in &self.counts {
output.push_str(&format!("{}:{count}\n", logical_display(root, path)));
}
}
GrepOutputMode::Content => {
for line in &self.lines {
let separator = if line.is_match { ':' } else { '-' };
let path = logical_display(root, &line.path);
if self.show_line_numbers
&& let Some(number) = line.line_number
{
output.push_str(&format!(
"{path}{separator}{number}{separator}{}\n",
line.text
));
} else {
output.push_str(&format!("{path}{separator}{}\n", line.text));
}
}
}
}
GrepResult {
output,
match_count,
matched_files,
truncated: self.truncated,
}
}
}
fn logical_display(root: &Path, path: &Path) -> String {
path.strip_prefix(root)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/")
}
const DEFAULT_HEAD_LIMIT: usize = 250;
struct GrepParams {
pattern: String,
path: Option<PathBuf>,
glob: Option<String>,
file_type: Option<String>,
case_insensitive: bool,
before: Option<usize>,
after: Option<usize>,
context: Option<usize>,
line_numbers: Option<bool>,
multiline: bool,
output_mode: Option<GrepOutputMode>,
head_limit: Option<usize>,
offset: Option<usize>,
}
pub(crate) fn run_grep(
root: &Path,
base: PathBuf,
request: GrepRequest,
scope: &Scope,
) -> Result<GrepResult, WorkdirError> {
let p = GrepParams {
pattern: request.pattern,
path: Some(base.clone()),
glob: request.glob,
file_type: request.file_type,
case_insensitive: request.case_insensitive,
before: Some(request.before_context),
after: Some(request.after_context),
context: None,
line_numbers: Some(true),
multiline: request.multiline,
output_mode: Some(request.output_mode),
head_limit: Some(request.limit),
offset: Some(request.offset),
};
let matcher = RegexMatcherBuilder::new()
.case_insensitive(p.case_insensitive)
.multi_line(p.multiline)
.dot_matches_new_line(p.multiline)
.build(&p.pattern)
.map_err(|e| WorkdirError::InvalidRegex(e.to_string()))?;
let (before, after) = match (p.before, p.after, p.context) {
(_, _, Some(c)) => (c, c),
(b, a, None) => (b.unwrap_or(0), a.unwrap_or(0)),
};
let mut sb = SearcherBuilder::new();
sb.binary_detection(BinaryDetection::quit(b'\x00'))
.line_number(p.line_numbers.unwrap_or(true))
.multi_line(p.multiline)
.before_context(before)
.after_context(after);
let mut searcher = sb.build();
let base = p.path.unwrap_or(base);
if !base.is_absolute() {
return Err(WorkdirError::RelativePath(base));
}
let symlink = direct_symlink(&base);
if !scope.is_readable(&base) {
return Err(if let Some(info) = symlink.as_ref() {
let link_parent_readable = info
.link_path
.parent()
.map(|parent| scope.is_readable(parent))
.unwrap_or(false);
if info.target_exists && link_parent_readable {
WorkdirError::SymlinkOutOfScope {
path: base.clone(),
target: info.resolved_path.clone(),
required_permission: "read",
}
} else {
WorkdirError::OutOfScope(base.clone())
}
} else {
WorkdirError::OutOfScope(base.clone())
});
}
if let Some(info) = symlink.as_ref() {
if !info.target_exists {
return Err(WorkdirError::BrokenSymlink {
path: base.clone(),
link: info.link_path.clone(),
target: info.target_path.clone(),
});
}
}
let base_meta = std::fs::metadata(&base).map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => WorkdirError::NotFound(base.clone()),
_ => WorkdirError::io(&base, e),
})?;
if !base_meta.is_dir() {
return Err(WorkdirError::InvalidArgument(format!(
"grep search path is not a directory: {}",
base.display()
)));
}
if let Some(info) = symlink.as_ref() {
return Err(WorkdirError::SymlinkDirectoryNotTraversed {
tool: "Grep",
path: base.clone(),
target: info.resolved_path.clone(),
});
}
let mut wb = WalkBuilder::new(&base);
wb.hidden(true)
.git_ignore(true)
.git_global(true)
.git_exclude(true)
.ignore(true)
.parents(true)
.follow_links(false);
if let Some(t) = p.file_type.as_deref() {
let mut tb = TypesBuilder::new();
tb.add_defaults();
tb.select(t);
let types = tb
.build()
.map_err(|e| WorkdirError::InvalidArgument(format!("invalid type {t}: {e}")))?;
wb.types(types);
}
if let Some(g) = p.glob.as_deref() {
let mut ob = OverrideBuilder::new(&base);
ob.add(g)
.map_err(|e| WorkdirError::InvalidGlob(e.to_string()))?;
let ov = ob
.build()
.map_err(|e| WorkdirError::InvalidGlob(e.to_string()))?;
wb.overrides(ov);
}
let mode = p.output_mode.unwrap_or_default();
let head_limit = p.head_limit.unwrap_or(DEFAULT_HEAD_LIMIT);
let offset = p.offset.unwrap_or(0);
let show_line_numbers = p.line_numbers.unwrap_or(true);
let mut report = GrepReport {
mode,
show_line_numbers,
files: Vec::new(),
counts: Vec::new(),
lines: Vec::new(),
truncated: false,
};
// Per-mode walker state.
let mut matching_files_seen: usize = 0;
let mut matches_seen: usize = 0;
'walker: for entry in wb.build().flatten() {
if !entry.file_type().map(|t| t.is_file()).unwrap_or(false) {
continue;
}
let path = entry.path();
if !scope.is_readable(path) {
continue;
}
match mode {
GrepOutputMode::FilesWithMatches => {
let hit = scan_any_match(&mut searcher, &matcher, path)?;
if !hit {
continue;
}
if matching_files_seen >= offset {
report.files.push(path.to_path_buf());
if report.files.len() >= head_limit {
report.truncated = true;
break 'walker;
}
}
matching_files_seen += 1;
}
GrepOutputMode::Count => {
let count = scan_count(&mut searcher, &matcher, path)?;
if count == 0 {
continue;
}
if matching_files_seen >= offset {
report.counts.push((path.to_path_buf(), count));
if report.counts.len() >= head_limit {
report.truncated = true;
break 'walker;
}
}
matching_files_seen += 1;
}
GrepOutputMode::Content => {
let before_count = matches_seen;
let mut sink = ContentSink {
path: path.to_path_buf(),
lines: &mut report.lines,
matches_seen: &mut matches_seen,
offset,
head_limit,
};
searcher
.search_path(&matcher, path, &mut sink)
.map_err(|e| WorkdirError::io(path, e))?;
// If we hit head_limit during this file, stop walking.
if matches_seen >= offset.saturating_add(head_limit) && matches_seen > before_count
{
report.truncated = true;
break 'walker;
}
}
}
}
Ok(report.into_result(root))
}
fn scan_any_match(
searcher: &mut Searcher,
matcher: &grep_regex::RegexMatcher,
path: &Path,
) -> Result<bool, WorkdirError> {
let mut hit = false;
let sink = UTF8Sink(|_, _| {
hit = true;
Ok(false) // stop searching this file immediately
});
searcher
.search_path(matcher, path, sink)
.map_err(|e| WorkdirError::io(path, e))?;
Ok(hit)
}
fn scan_count(
searcher: &mut Searcher,
matcher: &grep_regex::RegexMatcher,
path: &Path,
) -> Result<usize, WorkdirError> {
let mut count = 0usize;
let sink = UTF8Sink(|_, _| {
count += 1;
Ok(true)
});
searcher
.search_path(matcher, path, sink)
.map_err(|e| WorkdirError::io(path, e))?;
Ok(count)
}
struct ContentSink<'a> {
path: PathBuf,
lines: &'a mut Vec<ContentLine>,
matches_seen: &'a mut usize,
offset: usize,
head_limit: usize,
}
impl Sink for ContentSink<'_> {
type Error = std::io::Error;
fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, Self::Error> {
let idx = *self.matches_seen;
*self.matches_seen += 1;
// Skip matches before offset.
if idx < self.offset {
return Ok(true);
}
// Stop searching this file once we've filled the head_limit.
if idx >= self.offset.saturating_add(self.head_limit) {
return Ok(false);
}
let text = String::from_utf8_lossy(mat.bytes())
.trim_end_matches('\n')
.trim_end_matches('\r')
.to_string();
self.lines.push(ContentLine {
path: self.path.clone(),
line_number: mat.line_number(),
text,
is_match: true,
});
Ok(true)
}
fn context(
&mut self,
_searcher: &Searcher,
ctx: &SinkContext<'_>,
) -> Result<bool, Self::Error> {
let seen = *self.matches_seen;
if seen < self.offset {
return Ok(true);
}
if seen >= self.offset.saturating_add(self.head_limit) {
return Ok(false);
}
let text = String::from_utf8_lossy(ctx.bytes())
.trim_end_matches('\n')
.trim_end_matches('\r')
.to_string();
self.lines.push(ContentLine {
path: self.path.clone(),
line_number: ctx.line_number(),
text,
is_match: false,
});
Ok(true)
}
}
+1
View File
@@ -42,6 +42,7 @@ tokio = { workspace = true, features = ["net", "rt", "sync", "time"] }
toml.workspace = true toml.workspace = true
tower = { workspace = true, features = ["util"], optional = true } tower = { workspace = true, features = ["util"], optional = true }
worker.workspace = true worker.workspace = true
workdir.workspace = true
[dev-dependencies] [dev-dependencies]
futures.workspace = true futures.workspace = true
+63 -4
View File
@@ -9,7 +9,7 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::future::Future; use std::future::Future;
use std::path::PathBuf; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, mpsc}; use std::sync::{Arc, Mutex, mpsc};
use std::time::Duration; use std::time::Duration;
@@ -37,6 +37,7 @@ use session_store::{CombinedStore, FsWorkerStore};
use tokio::runtime::Runtime; use tokio::runtime::Runtime;
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
use tokio::sync::broadcast; use tokio::sync::broadcast;
use workdir::{LocalWorkdir, WorkdirCapabilities, WorkdirHandle};
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session}; use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session};
@@ -353,6 +354,21 @@ async fn fetch_profile_source_archive_http(
) )
} }
fn runtime_local_workdir(
binding_id: &str,
root: &Path,
cwd: &Path,
scope: manifest::SharedScope,
) -> WorkdirHandle {
Arc::new(LocalWorkdir::materialized_bound(
Some(binding_id.to_owned()),
root.to_path_buf(),
cwd.to_path_buf(),
scope,
WorkdirCapabilities::ALL,
))
}
#[async_trait] #[async_trait]
impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
async fn spawn_controller( async fn spawn_controller(
@@ -419,7 +435,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
})?; })?;
let store = CombinedStore::new(session_store, worker_metadata_store); let store = CombinedStore::new(session_store, worker_metadata_store);
let worker = Worker::from_manifest_with_context( let mut worker = Worker::from_manifest_with_context(
manifest, manifest,
store, store,
loader, loader,
@@ -428,6 +444,16 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
) )
.await .await
.map_err(|err| format!("failed to create Worker from profile: {err}"))?; .map_err(|err| format!("failed to create Worker from profile: {err}"))?;
if let Some(binding) = request.working_directory.as_ref() {
worker.bind_workdir(Some(runtime_local_workdir(
&binding.working_directory.id,
binding.root(),
binding.cwd(),
worker.scope().clone(),
)));
} else {
worker.bind_workdir(None);
}
let runtime_base = self.runtime_base_dir()?; let runtime_base = self.runtime_base_dir()?;
let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base) let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
@@ -472,7 +498,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
})?; })?;
let store = CombinedStore::new(session_store, worker_metadata_store); let store = CombinedStore::new(session_store, worker_metadata_store);
let worker = match Worker::restore_from_worker_metadata_with_context( let mut worker = match Worker::restore_from_worker_metadata_with_context(
&worker_name, &worker_name,
manifest.clone(), manifest.clone(),
store, store,
@@ -513,6 +539,16 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
} }
Err(err) => return Err(format!("failed to restore Worker from metadata: {err}")), Err(err) => return Err(format!("failed to restore Worker from metadata: {err}")),
}; };
if let Some(binding) = request.working_directory.as_ref() {
worker.bind_workdir(Some(runtime_local_workdir(
&binding.working_directory.id,
binding.root(),
binding.cwd(),
worker.scope().clone(),
)));
} else {
worker.bind_workdir(None);
}
let runtime_base = self.runtime_base_dir()?; let runtime_base = self.runtime_base_dir()?;
let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base) let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
@@ -1227,7 +1263,9 @@ where
}; };
workers workers
.get(handle.worker_ref()) .get(handle.worker_ref())
.map(|execution| execution.handle.completion_entries(kind, prefix)) .map(|execution| {
futures::executor::block_on(execution.handle.completion_entries(kind, prefix))
})
.unwrap_or_default() .unwrap_or_default()
} }
} }
@@ -1599,6 +1637,27 @@ mod tests {
); );
} }
#[test]
fn runtime_rebind_preserves_working_directory_id_on_a_fresh_provider() {
let root = tempfile::tempdir().unwrap();
let spawned = runtime_local_workdir(
"working-directory-42",
root.path(),
root.path(),
manifest::SharedScope::new(Scope::writable(root.path()).unwrap()),
);
let restored = runtime_local_workdir(
"working-directory-42",
root.path(),
root.path(),
manifest::SharedScope::new(Scope::writable(root.path()).unwrap()),
);
assert_eq!(spawned.binding_id(), Some("working-directory-42"));
assert_eq!(restored.binding_id(), Some("working-directory-42"));
assert!(!Arc::ptr_eq(&spawned, &restored));
}
#[tokio::test] #[tokio::test]
async fn embedded_profile_source_archive_does_not_require_backend_resource_fetch() { async fn embedded_profile_source_archive_does_not_require_backend_resource_fetch() {
let factory = ProfileRuntimeWorkerFactory::new(tempfile::tempdir().unwrap().path()); let factory = ProfileRuntimeWorkerFactory::new(tempfile::tempdir().unwrap().path());
+1
View File
@@ -26,6 +26,7 @@ tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "proce
toml = { workspace = true } toml = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
tools = { workspace = true } tools = { workspace = true }
workdir = { workspace = true }
minijinja = "2.19.0" minijinja = "2.19.0"
chrono = "0.4" chrono = "0.4"
include_dir = "0.7.4" include_dir = "0.7.4"
+32 -18
View File
@@ -26,10 +26,14 @@ use llm_engine::Item;
use llm_engine::interceptor::{Interceptor, PreRequestAction, PreToolAction, ToolCallInfo}; use llm_engine::interceptor::{Interceptor, PreRequestAction, PreToolAction, ToolCallInfo};
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput, ToolResult}; use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput, ToolResult};
use serde::Deserialize; use serde::Deserialize;
use tools::ScopedFs; #[cfg(test)]
use workdir::LocalWorkdir;
use workdir::{ReadRequest, WorkdirHandle, WorkdirPath};
use crate::compact::usage_tracker::UsageTracker; use crate::compact::usage_tracker::UsageTracker;
use crate::fs_view::{ReadRequirement, slice_lines}; use crate::fs_view::ReadRequirement;
#[cfg(test)]
use crate::fs_view::slice_lines;
use crate::session_reference::{ use crate::session_reference::{
ReadDetail, ReadOptions, ReadSelector, SearchOptions, SessionReferenceView, ToolPart, ReadDetail, ReadOptions, ReadSelector, SearchOptions, SessionReferenceView, ToolPart,
}; };
@@ -323,7 +327,7 @@ fn truncate_to_token_budget(text: &mut String, max_tokens: u64) -> bool {
} }
struct MarkReadRequiredTool { struct MarkReadRequiredTool {
fs: ScopedFs, workdir: WorkdirHandle,
ctx: Arc<Mutex<CompactWorkerContext>>, ctx: Arc<Mutex<CompactWorkerContext>>,
} }
@@ -338,14 +342,22 @@ impl Tool for MarkReadRequiredTool {
ToolError::InvalidArgument(format!("invalid mark_read_required input: {e}")) ToolError::InvalidArgument(format!("invalid mark_read_required input: {e}"))
})?; })?;
// Read the file through the shared ScopedFs so scope and I/O // Read through the shared Workdir so scope and I/O errors surface the
// errors surface the same way the regular `read_file` tool does. // same way the regular `read_file` tool does.
let bytes = self let path = WorkdirPath::new(params.file_path.to_string_lossy())
.fs .map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
.read_bytes(&params.file_path) let result = self
.workdir
.read(ReadRequest {
path,
offset: params.offset.unwrap_or(0),
limit: params.limit.unwrap_or(usize::MAX),
max_bytes: 4 * 1024 * 1024,
})
.await
.map_err(|e| ToolError::ExecutionFailed(format!("read failed: {e}")))?; .map_err(|e| ToolError::ExecutionFailed(format!("read failed: {e}")))?;
let text = String::from_utf8_lossy(&bytes); let text = String::from_utf8_lossy(&result.bytes);
let slice = slice_lines(&text, params.offset.unwrap_or(0), params.limit); let slice = text.as_ref();
let estimated_tokens = estimate_tokens(slice.len()); let estimated_tokens = estimate_tokens(slice.len());
let mut guard = self.ctx.lock().expect("compact worker context poisoned"); let mut guard = self.ctx.lock().expect("compact worker context poisoned");
@@ -442,7 +454,7 @@ impl Tool for WriteSummaryTool {
} }
pub(crate) fn mark_read_required_tool( pub(crate) fn mark_read_required_tool(
fs: ScopedFs, workdir: WorkdirHandle,
ctx: Arc<Mutex<CompactWorkerContext>>, ctx: Arc<Mutex<CompactWorkerContext>>,
) -> ToolDefinition { ) -> ToolDefinition {
Arc::new(move || { Arc::new(move || {
@@ -452,7 +464,7 @@ pub(crate) fn mark_read_required_tool(
.description(MARK_DESCRIPTION) .description(MARK_DESCRIPTION)
.input_schema(schema_value); .input_schema(schema_value);
let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool { let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool {
fs: fs.clone(), workdir: workdir.clone(),
ctx: ctx.clone(), ctx: ctx.clone(),
}); });
(meta, tool) (meta, tool)
@@ -623,9 +635,9 @@ mod tests {
use super::*; use super::*;
use manifest::Scope; use manifest::Scope;
fn make_fs(tmp: &std::path::Path) -> ScopedFs { fn make_fs(tmp: &std::path::Path) -> WorkdirHandle {
let scope = Scope::writable(tmp.to_path_buf()).unwrap(); let scope = Scope::writable(tmp.to_path_buf()).unwrap();
ScopedFs::new(scope, tmp.to_path_buf()) Arc::new(LocalWorkdir::new(scope, tmp.to_path_buf()))
} }
fn make_usage(input: u64) -> llm_engine::timeline::event::UsageEvent { fn make_usage(input: u64) -> llm_engine::timeline::event::UsageEvent {
@@ -720,10 +732,11 @@ mod tests {
let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(1_000))); let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(1_000)));
let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool { let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool {
fs: make_fs(tmp.path()), workdir: make_fs(tmp.path()),
ctx: ctx.clone(), ctx: ctx.clone(),
}); });
let input = serde_json::json!({ "file_path": path.to_str().unwrap() }).to_string(); let input = serde_json::json!({ "file_path": path.file_name().unwrap().to_str().unwrap() })
.to_string();
let out = tool.execute(&input, Default::default()).await.unwrap(); let out = tool.execute(&input, Default::default()).await.unwrap();
assert!(out.summary.starts_with("Marked")); assert!(out.summary.starts_with("Marked"));
@@ -741,10 +754,11 @@ mod tests {
let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(100))); let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(100)));
let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool { let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool {
fs: make_fs(tmp.path()), workdir: make_fs(tmp.path()),
ctx: ctx.clone(), ctx: ctx.clone(),
}); });
let input = serde_json::json!({ "file_path": path.to_str().unwrap() }).to_string(); let input = serde_json::json!({ "file_path": path.file_name().unwrap().to_str().unwrap() })
.to_string();
let res = tool.execute(&input, Default::default()).await; let res = tool.execute(&input, Default::default()).await;
assert!(matches!(res, Err(ToolError::ExecutionFailed(_)))); assert!(matches!(res, Err(ToolError::ExecutionFailed(_))));
+28 -21
View File
@@ -88,23 +88,25 @@ impl WorkerHandle {
(event, entry_rx) (event, entry_rx)
} }
pub fn completion_entries( pub async fn completion_entries(
&self, &self,
kind: protocol::CompletionKind, kind: protocol::CompletionKind,
prefix: &str, prefix: &str,
) -> Vec<protocol::CompletionEntry> { ) -> Vec<protocol::CompletionEntry> {
match kind { match kind {
protocol::CompletionKind::File => self protocol::CompletionKind::File => {
.shared_state let Some(view) = self.shared_state.fs_view() else {
.fs_view() return Vec::new();
.map(|view| view.list_file_completions(prefix)) };
.unwrap_or_default() view.list_file_completions(prefix)
.await
.into_iter() .into_iter()
.map(|c| protocol::CompletionEntry { .map(|candidate| protocol::CompletionEntry {
value: c.path, value: candidate.path,
is_dir: c.is_dir, is_dir: candidate.is_dir,
}) })
.collect(), .collect()
}
} }
} }
@@ -574,7 +576,7 @@ fn wire_event_bridges_on_engine<C, St>(
/// Register the builtin file-manipulation tools, optional memory tools, /// Register the builtin file-manipulation tools, optional memory tools,
/// and the Worker-orchestration tools (SpawnWorker + comm) on the Worker's /// and the Worker-orchestration tools (SpawnWorker + comm) on the Worker's
/// Engine. Returns the `ScopedFs` clone used to attach a `WorkerFsView` to /// Engine. Returns the Workdir handle used to attach a `WorkerFsView` to
/// the shared state. /// the shared state.
async fn register_worker_tools<C, St>( async fn register_worker_tools<C, St>(
worker: &mut Worker<C, St>, worker: &mut Worker<C, St>,
@@ -582,7 +584,7 @@ async fn register_worker_tools<C, St>(
spawner_socket: PathBuf, spawner_socket: PathBuf,
runtime_base: PathBuf, runtime_base: PathBuf,
spawned_registry: Arc<SpawnedWorkerRegistry>, spawned_registry: Arc<SpawnedWorkerRegistry>,
) -> std::io::Result<Option<tools::ScopedFs>> ) -> std::io::Result<Option<workdir::WorkdirHandle>>
where where
C: LlmClient + Clone + 'static, C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + 'static, St: Store + WorkerMetadataStore + Clone + 'static,
@@ -590,6 +592,7 @@ where
// Worker-immutable snapshots taken before the mutable worker borrow // Worker-immutable snapshots taken before the mutable worker borrow
// below so the worker borrow doesn't conflict with reads on `worker`. // below so the worker borrow doesn't conflict with reads on `worker`.
let scope_handle = worker.scope().clone(); let scope_handle = worker.scope().clone();
let worker_workdir = worker.workdir().cloned();
let local_filesystem = worker.local_working_directory().cloned(); let local_filesystem = worker.local_working_directory().cloned();
let local_workspace_root = local_filesystem.as_ref().map(|local| local.root.clone()); let local_workspace_root = local_filesystem.as_ref().map(|local| local.root.clone());
let task_feature = worker.task_feature(); let task_feature = worker.task_feature();
@@ -603,21 +606,19 @@ where
let worker_metadata_store = worker.store().clone(); let worker_metadata_store = worker.store().clone();
let self_parent_socket = worker.callback_socket().cloned(); let self_parent_socket = worker.callback_socket().cloned();
// The Worker's SharedScope is the single source of truth for every // Resolve the existing WorkerWorkdir binding into the domain provider.
// ScopedFs when local filesystem authority exists. No-workdir Workers // Tools only consume the provider handle; they do not own its root, cwd,
// deliberately skip constructing/registering filesystem and Bash tools. // scope, or lifecycle. No-workdir Workers expose no local tools.
let (fs_for_view, tracker) = if let Some(local) = local_filesystem.as_ref() { let (workdir_for_view, tracker) = if let Some(workdir) = worker_workdir {
let fs = tools::ScopedFs::with_shared_scope(scope_handle.clone(), local.cwd.clone());
let tracker = tools::Tracker::new(); let tracker = tools::Tracker::new();
let fs_for_view = fs.clone();
worker worker
.engine_mut() .engine_mut()
.register_tools(tools::core_builtin_tools( .register_tools(tools::core_builtin_tools(
fs, workdir.clone(),
tracker.clone(), tracker.clone(),
bash_output_dir, bash_output_dir,
)); ));
(Some(fs_for_view), Some(tracker)) (Some(workdir), Some(tracker))
} else { } else {
(None, None) (None, None)
}; };
@@ -788,7 +789,7 @@ where
if let Some(tracker) = tracker { if let Some(tracker) = tracker {
worker.attach_tracker(tracker); worker.attach_tracker(tracker);
} }
Ok(fs_for_view) Ok(workdir_for_view)
} }
/// Idle/Paused event loop. Each iteration either fires a staged /// Idle/Paused event loop. Each iteration either fires a staged
@@ -1187,6 +1188,12 @@ async fn controller_loop<C, St>(
} }
} }
if let Some(workdir) = worker.workdir()
&& let Err(error) = workdir.shutdown().await
{
tracing::warn!(%error, "Workdir provider shutdown failed");
}
// Background memory jobs own extract/consolidate workers after a // Background memory jobs own extract/consolidate workers after a
// turn completes. Join them before the controller task exits so // turn completes. Join them before the controller task exits so
// staging writes and consolidation cleanups are not abandoned. // staging writes and consolidation cleanups are not abandoned.
+212 -297
View File
@@ -1,6 +1,6 @@
//! Worker 視点のファイルシステム操作。 //! Worker 視点のファイルシステム操作。
//! //!
//! `ScopedFs` の上に「Worker が読み取りたい / 列挙したい」操作を集約する軽い wrapper。 //! `Workdir` の上に「Worker が読み取りたい / 列挙したい」操作を集約する軽い wrapper。
//! //!
//! - `ReadRequirement` と `render_auto_read` — compact worker が `mark_read_required` //! - `ReadRequirement` と `render_auto_read` — compact worker が `mark_read_required`
//! で nominate したファイルを再読し、`[Auto-read file: ...]` system message に //! で nominate したファイルを再読し、`[Auto-read file: ...]` system message に
@@ -13,16 +13,19 @@
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use llm_engine::Item; use llm_engine::Item;
use manifest::Scope; use tools::ToolsError;
use tools::scoped_fs::first_symlink;
use tools::{ScopedFs, ToolsError};
use tracing::warn; use tracing::warn;
#[cfg(test)]
use workdir::LocalWorkdir;
use workdir::{EntryKind, ListRequest, ReadRequest, StatRequest, WorkdirHandle, WorkdirPath};
/// 補完候補1件の最大数。`list_file_completions` がこの値を超えたら打ち切り。 /// 補完候補1件の最大数。`list_file_completions` がこの値を超えたら打ち切り。
const COMPLETION_LIMIT: usize = 100; const COMPLETION_LIMIT: usize = 100;
/// submit-time directory FileRef の shallow listing で返す最大 entry 数。 /// submit-time directory FileRef の shallow listing で返す最大 entry 数。
/// TUI completion と同じ浅い一覧という意味論に揃えるため、同じ上限を使う。 /// TUI completion と同じ浅い一覧という意味論に揃えるため、同じ上限を使う。
const DIR_FILE_REF_ENTRY_LIMIT: usize = COMPLETION_LIMIT; const DIR_FILE_REF_ENTRY_LIMIT: usize = COMPLETION_LIMIT;
/// Provider-side bound for auto-read and submit-time referenced-file reads.
const AUTO_READ_BYTE_LIMIT: usize = 4 * 1024 * 1024;
/// Compact worker が `mark_read_required` で nominate した「次セッション開始時に /// Compact worker が `mark_read_required` で nominate した「次セッション開始時に
/// 自動で再読すべきファイル」のエントリ。 /// 自動で再読すべきファイル」のエントリ。
@@ -35,10 +38,10 @@ pub struct ReadRequirement {
pub limit: Option<usize>, pub limit: Option<usize>,
} }
/// Worker から見えるファイルシステム操作の入口。Clone は cheap`ScopedFs` 内 `Arc`)。 /// Worker から見えるファイルシステム操作の入口。Clone は cheap`Workdir` 内 `Arc`)。
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct WorkerFsView { pub struct WorkerFsView {
fs: ScopedFs, workdir: WorkdirHandle,
} }
/// `list_file_completions` が返す候補1件。 /// `list_file_completions` が返す候補1件。
@@ -51,10 +54,10 @@ pub struct FileCandidate {
} }
/// `resolve_file_ref` の失敗理由。Worker 側で Alert に振り分けるために /// `resolve_file_ref` の失敗理由。Worker 側で Alert に振り分けるために
/// ScopedFs / 内部判定の両方を区別できるよう保持する。 /// Workdir / 内部判定の両方を区別できるよう保持する。
#[derive(Debug)] #[derive(Debug)]
pub enum ResolveError { pub enum ResolveError {
/// Path resolution / scope check failed via `ScopedFs`. /// Path resolution / scope check failed via `Workdir`.
Fs(ToolsError), Fs(ToolsError),
/// File contents are not valid UTF-8 (binary / non-text). /// File contents are not valid UTF-8 (binary / non-text).
Binary { path: PathBuf }, Binary { path: PathBuf },
@@ -74,142 +77,172 @@ impl std::fmt::Display for ResolveError {
impl std::error::Error for ResolveError {} impl std::error::Error for ResolveError {}
impl WorkerFsView { impl WorkerFsView {
pub fn new(fs: ScopedFs) -> Self { pub fn new(workdir: WorkdirHandle) -> Self {
Self { fs } Self { workdir }
}
pub fn workdir(&self) -> &WorkdirHandle {
&self.workdir
} }
pub fn fs(&self) -> &ScopedFs { pub async fn render_auto_read(&self, requirements: &[ReadRequirement]) -> Vec<Item> {
&self.fs
}
/// `requirements` の各エントリを `ScopedFs` 経由で再読し、
/// `[Auto-read file: <path>:<range>]\n<body>` 形式の system message に変換する。
/// 読み取り失敗(NotFound / OutOfScope 等)は warn で記録してスキップする
/// — compact 全体を落とさないため。
pub fn render_auto_read(&self, requirements: &[ReadRequirement]) -> Vec<Item> {
let mut out = Vec::with_capacity(requirements.len()); let mut out = Vec::with_capacity(requirements.len());
for req in requirements { for req in requirements {
match self.fs.read_bytes(&req.path) { let path = match WorkdirPath::new(req.path.to_string_lossy()) {
Ok(bytes) => { Ok(path) => path,
let text = String::from_utf8_lossy(&bytes).into_owned(); Err(error) => {
let body = slice_lines(&text, req.offset.unwrap_or(0), req.limit); warn!(path = %req.path.display(), %error, "invalid auto-read path");
continue;
}
};
match self
.workdir
.read(ReadRequest {
path: path.clone(),
offset: req.offset.unwrap_or(0),
limit: req.limit.unwrap_or(usize::MAX),
max_bytes: AUTO_READ_BYTE_LIMIT,
})
.await
{
Ok(result) => {
let body = String::from_utf8_lossy(&result.bytes);
let range = format_range(req.offset, req.limit); let range = format_range(req.offset, req.limit);
out.push(Item::system_message(format!( out.push(Item::system_message(format!(
"[Auto-read file: {}{range}]\n{body}", "[Auto-read file: {path}{range}]\n{body}"
req.path.display()
))); )));
} }
Err(e) => { Err(error) => {
warn!( warn!(path = %path, %error, "auto-read target could not be read; skipping")
path = %req.path.display(),
error = %e,
"auto-read target could not be read; skipping",
);
} }
} }
} }
out out
} }
/// `path` を ScopedFs 経由で解決し、submit 時の `Segment::FileRef` pub async fn resolve_file_ref(
/// attachment 用 system message を返す。 &self,
/// path: &str,
/// - `path` は relative なら cwd 相対、absolute なら absolute として解釈 max_bytes: usize,
/// - 通常ディレクトリは浅い entry listing として `[Dir: <path>]\n<body>` に展開する ) -> Result<Item, ResolveError> {
/// - ディレクトリ listing は hidden / gitignore を特別扱いせず、scope 上 readable な let logical = WorkdirPath::new(path)
/// 直下 entry だけを最大 `DIR_FILE_REF_ENTRY_LIMIT` 件返す .map_err(ToolsError::from)
/// - ファイル本文またはディレクトリ listing 本文が `max_bytes` を超える場合は切り詰める .map_err(ResolveError::Fs)?;
/// - 非 UTF-8 (バイナリ) は `ResolveError::Binary` で拒否 let stat = self
/// - スコープ外 / NotFound / symlink directory 等は `ResolveError::Fs` で返す .workdir
pub fn resolve_file_ref(&self, path: &str, max_bytes: usize) -> Result<Item, ResolveError> { .stat(StatRequest {
let p = Path::new(path); path: logical.clone(),
let abs = if p.is_absolute() { })
p.to_path_buf() .await
.map_err(ToolsError::from)
.map_err(ResolveError::Fs)?;
if stat.kind == EntryKind::Directory {
let result = self
.workdir
.list(ListRequest {
path: logical.clone(),
limit: DIR_FILE_REF_ENTRY_LIMIT,
})
.await
.map_err(ToolsError::from)
.map_err(ResolveError::Fs)?;
let listing = result
.entries
.into_iter()
.map(|entry| match entry.kind {
EntryKind::Directory => format!("{}/", entry.path),
EntryKind::Symlink => format!("{}@", entry.path),
_ => entry.path.to_string(),
})
.collect::<Vec<_>>()
.join("\n");
let suffix = format!(
"\n[{} readable entries total, {} bytes total]{}",
result.total_entries,
result.total_bytes,
if result.truncated {
"\n[...listing truncated; use Glob for more]"
} else { } else {
self.fs.cwd().join(p) ""
};
// 通常ディレクトリだけを FileRef listing として扱う。symlink を含むパスは
// `ScopedFs::read_bytes` に委ね、既存の symlink 診断
// (`SymlinkTargetIsDirectory` / `SymlinkOutOfScope` 等) を保つ。
if first_symlink(&abs).is_none() {
let scope = self.fs.scope();
if !scope.is_readable(&abs) {
return Err(ResolveError::Fs(ToolsError::OutOfScope(abs)));
} }
let meta = metadata_for_file_ref(&abs).map_err(ResolveError::Fs)?; );
if meta.is_dir() { let header = format!("[Dir: {logical}]\n");
return render_dir_file_ref(path, &abs, max_bytes, scope.as_ref()); let listing_budget = max_bytes.saturating_sub(header.len() + suffix.len());
} let (bounded_listing, truncated) = truncate_utf8_bytes(&listing, listing_budget);
} let mut text = format!("{header}{bounded_listing}{suffix}");
let bytes = self.fs.read_bytes(&abs).map_err(ResolveError::Fs)?;
let total = bytes.len();
let (body_bytes, truncated) = if total > max_bytes {
(&bytes[..max_bytes], true)
} else {
(bytes.as_slice(), false)
};
let body = std::str::from_utf8(body_bytes)
.map_err(|_| ResolveError::Binary { path: abs.clone() })?;
let mut text = format!("[File: {path}]\n{body}");
if truncated { if truncated {
text.push_str("\n[...directory attachment truncated; use Glob or Read for more]");
}
return Ok(Item::system_message(text));
}
let result = self
.workdir
.read(ReadRequest {
path: logical.clone(),
offset: 0,
limit: usize::MAX,
max_bytes,
})
.await
.map_err(ToolsError::from)
.map_err(ResolveError::Fs)?;
let total = stat.size;
let end = result.bytes.len().min(max_bytes);
let body = std::str::from_utf8(&result.bytes[..end]).map_err(|_| ResolveError::Binary {
path: PathBuf::from(logical.as_str()),
})?;
let mut text = format!("[File: {logical}]\n{body}");
if end < result.bytes.len() || result.truncated {
text.push_str(&format!( text.push_str(&format!(
"\n[...truncated, {total} bytes total — use read_file for the rest]" "\n[...truncated, {total} bytes total — use Read for the rest]"
)); ));
} }
Ok(Item::system_message(text)) Ok(Item::system_message(text))
} }
/// `prefix` にマッチするファイル / ディレクトリを scope 内で浅く列挙する。 pub async fn list_file_completions(&self, prefix: &str) -> Vec<FileCandidate> {
/// let prefix_path = Path::new(prefix);
/// - `prefix` が空 or `cwd` 相対のときは cwd 直下を見る let (parent, needle) = if prefix.ends_with('/') {
/// - `prefix` が末尾 `/` のときはそのディレクトリ直下を全列挙 (prefix_path, String::new())
/// - 末尾が名前部分のときは、その名前を starts_with でフィルタ
/// - scope 上 readable なエントリのみ返す
/// - ディレクトリ → ファイル の順、各グループ内は名前昇順
/// - 上限 `COMPLETION_LIMIT` 件で打ち切り(深い列挙はしない)
pub fn list_file_completions(&self, prefix: &str) -> Vec<FileCandidate> {
let cwd = self.fs.cwd();
let scope = self.fs.scope();
let (dir, name_prefix, is_absolute) = split_prefix(prefix, cwd);
let read_dir = match std::fs::read_dir(&dir) {
Ok(rd) => rd,
Err(_) => return Vec::new(),
};
let mut out = Vec::new();
for entry in read_dir.flatten() {
let file_name = entry.file_name();
let name = file_name.to_string_lossy();
if !name.starts_with(&name_prefix) {
continue;
}
let path = entry.path();
if !scope.is_readable(&path) {
continue;
}
let is_dir = entry.file_type().map(|t| t.is_dir()).unwrap_or(false);
let display = if is_absolute {
path.display().to_string()
} else { } else {
path.strip_prefix(cwd) (
.map(|p| p.display().to_string()) prefix_path.parent().unwrap_or_else(|| Path::new("")),
.unwrap_or_else(|_| path.display().to_string()) prefix_path
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_default(),
)
}; };
out.push(FileCandidate { let Ok(parent) = WorkdirPath::new(parent.to_string_lossy()) else {
path: display, return Vec::new();
is_dir, };
}); let Ok(result) = self
} .workdir
.list(ListRequest {
path: parent,
limit: COMPLETION_LIMIT,
})
.await
else {
return Vec::new();
};
let mut out = result
.entries
.into_iter()
.filter_map(|entry| {
let name = Path::new(entry.path.as_str())
.file_name()?
.to_string_lossy();
name.starts_with(&needle).then_some(FileCandidate {
path: entry.path.to_string(),
is_dir: entry.kind == EntryKind::Directory,
})
})
.collect::<Vec<_>>();
out.sort_by(|a, b| match (a.is_dir, b.is_dir) { out.sort_by(|a, b| match (a.is_dir, b.is_dir) {
(true, false) => std::cmp::Ordering::Less, (true, false) => std::cmp::Ordering::Less,
(false, true) => std::cmp::Ordering::Greater, (false, true) => std::cmp::Ordering::Greater,
_ => a.path.cmp(&b.path), _ => a.path.cmp(&b.path),
}); });
out.truncate(COMPLETION_LIMIT);
out out
} }
} }
@@ -224,85 +257,6 @@ pub fn slice_lines(text: &str, offset: usize, limit: Option<usize>) -> String {
lines[start..end].join("\n") lines[start..end].join("\n")
} }
#[derive(Debug, Clone, PartialEq, Eq)]
struct DirListingEntry {
display: String,
kind_rank: u8,
}
fn metadata_for_file_ref(path: &Path) -> Result<std::fs::Metadata, ToolsError> {
std::fs::metadata(path).map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => ToolsError::NotFound(path.to_path_buf()),
_ => ToolsError::io(path, e),
})
}
fn render_dir_file_ref(
original_path: &str,
abs: &Path,
max_bytes: usize,
scope: &Scope,
) -> Result<Item, ResolveError> {
let read_dir = std::fs::read_dir(abs).map_err(|e| ResolveError::Fs(ToolsError::io(abs, e)))?;
let mut entries = Vec::new();
for entry in read_dir {
let entry = entry.map_err(|e| ResolveError::Fs(ToolsError::io(abs, e)))?;
let path = entry.path();
if !scope.is_readable(&path) {
continue;
}
let file_type = match entry.file_type() {
Ok(ft) => ft,
Err(e) => return Err(ResolveError::Fs(ToolsError::io(&path, e))),
};
let mut display = entry.file_name().to_string_lossy().into_owned();
let kind_rank = if file_type.is_dir() {
display.push('/');
0
} else if file_type.is_symlink() {
display.push('@');
1
} else {
2
};
entries.push(DirListingEntry { display, kind_rank });
}
entries.sort_by(|a, b| {
a.kind_rank
.cmp(&b.kind_rank)
.then_with(|| a.display.cmp(&b.display))
});
let total_entries = entries.len();
let entry_truncated = total_entries > DIR_FILE_REF_ENTRY_LIMIT;
let body = if total_entries == 0 {
"(empty directory)".to_string()
} else {
entries
.iter()
.take(DIR_FILE_REF_ENTRY_LIMIT)
.map(|e| e.display.as_str())
.collect::<Vec<_>>()
.join("\n")
};
let body_total_bytes = body.len();
let (body, byte_truncated) = truncate_utf8_bytes(&body, max_bytes);
let mut text = format!("[Dir: {original_path}]\n{body}");
if entry_truncated || byte_truncated {
text.push('\n');
text.push_str(&dir_listing_truncation_hint(
entry_truncated,
byte_truncated,
total_entries,
body_total_bytes,
));
}
Ok(Item::system_message(text))
}
fn truncate_utf8_bytes(s: &str, max_bytes: usize) -> (&str, bool) { fn truncate_utf8_bytes(s: &str, max_bytes: usize) -> (&str, bool) {
if s.len() <= max_bytes { if s.len() <= max_bytes {
return (s, false); return (s, false);
@@ -314,26 +268,6 @@ fn truncate_utf8_bytes(s: &str, max_bytes: usize) -> (&str, bool) {
(&s[..end], true) (&s[..end], true)
} }
fn dir_listing_truncation_hint(
entry_truncated: bool,
byte_truncated: bool,
total_entries: usize,
body_total_bytes: usize,
) -> String {
match (entry_truncated, byte_truncated) {
(true, true) => format!(
"[...truncated, {total_entries} readable entries total; first {DIR_FILE_REF_ENTRY_LIMIT} entries were {body_total_bytes} bytes before byte cap — use Glob for more]"
),
(true, false) => {
format!("[...truncated, {total_entries} readable entries total — use Glob for more]")
}
(false, true) => {
format!("[...truncated, {body_total_bytes} bytes total — use Glob or Read for more]")
}
(false, false) => String::new(),
}
}
fn format_range(offset: Option<usize>, limit: Option<usize>) -> String { fn format_range(offset: Option<usize>, limit: Option<usize>) -> String {
match (offset, limit) { match (offset, limit) {
(None, None) => String::new(), (None, None) => String::new(),
@@ -343,41 +277,19 @@ fn format_range(offset: Option<usize>, limit: Option<usize>) -> String {
} }
} }
fn split_prefix(prefix: &str, cwd: &Path) -> (PathBuf, String, bool) {
let is_absolute = Path::new(prefix).is_absolute();
let p = Path::new(prefix);
let (parent, name) = if prefix.is_empty() || prefix.ends_with('/') {
(p.to_path_buf(), String::new())
} else {
let parent = p.parent().map(|p| p.to_path_buf()).unwrap_or_default();
let name = p
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
(parent, name)
};
let dir = if is_absolute {
parent
} else if parent.as_os_str().is_empty() {
cwd.to_path_buf()
} else {
cwd.join(parent)
};
(dir, name, is_absolute)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use llm_engine::ContentPart; use llm_engine::ContentPart;
use manifest::{Permission, Scope, ScopeConfig, ScopeRule}; use manifest::{Permission, Scope, ScopeConfig, ScopeRule};
use std::sync::Arc;
use tempfile::TempDir; use tempfile::TempDir;
fn fs_for(dir: &TempDir) -> ScopedFs { fn fs_for(dir: &TempDir) -> WorkdirHandle {
ScopedFs::new( Arc::new(LocalWorkdir::new(
Scope::writable(dir.path()).unwrap(), Scope::writable(dir.path()).unwrap(),
dir.path().to_path_buf(), dir.path().to_path_buf(),
) ))
} }
fn touch(path: &Path, content: &str) { fn touch(path: &Path, content: &str) {
@@ -397,26 +309,28 @@ mod tests {
text text
} }
#[test] #[tokio::test]
fn slice_lines_handles_offset_and_limit() { async fn slice_lines_handles_offset_and_limit() {
let text = "a\nb\nc\nd"; let text = "a\nb\nc\nd";
assert_eq!(slice_lines(text, 0, None), "a\nb\nc\nd"); assert_eq!(slice_lines(text, 0, None), "a\nb\nc\nd");
assert_eq!(slice_lines(text, 1, Some(2)), "b\nc"); assert_eq!(slice_lines(text, 1, Some(2)), "b\nc");
assert_eq!(slice_lines(text, 10, None), ""); assert_eq!(slice_lines(text, 10, None), "");
} }
#[test] #[tokio::test]
fn render_auto_read_emits_system_messages_with_range_label() { async fn render_auto_read_emits_system_messages_with_range_label() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let file = dir.path().join("hello.txt"); let file = dir.path().join("hello.txt");
std::fs::write(&file, "alpha\nbeta\ngamma\n").unwrap(); std::fs::write(&file, "alpha\nbeta\ngamma\n").unwrap();
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let items = view.render_auto_read(&[ReadRequirement { let items = view
path: file.clone(), .render_auto_read(&[ReadRequirement {
path: PathBuf::from("hello.txt"),
offset: Some(1), offset: Some(1),
limit: Some(1), limit: Some(1),
}]); }])
.await;
assert_eq!(items.len(), 1); assert_eq!(items.len(), 1);
let rendered = format!("{:?}", items[0]); let rendered = format!("{:?}", items[0]);
@@ -426,35 +340,35 @@ mod tests {
assert!(!rendered.contains("alpha")); assert!(!rendered.contains("alpha"));
} }
#[test] #[tokio::test]
fn resolve_file_ref_emits_system_message_with_path_header() { async fn resolve_file_ref_emits_system_message_with_path_header() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("hello.txt"), "hello world").unwrap(); std::fs::write(dir.path().join("hello.txt"), "hello world").unwrap();
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let item = view.resolve_file_ref("hello.txt", 1024).unwrap(); let item = view.resolve_file_ref("hello.txt", 1024).await.unwrap();
let text = format!("{item:?}"); let text = format!("{item:?}");
assert!(text.contains("[File: hello.txt]")); assert!(text.contains("[File: hello.txt]"));
assert!(text.contains("hello world")); assert!(text.contains("hello world"));
assert!(!text.contains("truncated")); assert!(!text.contains("truncated"));
} }
#[test] #[tokio::test]
fn resolve_file_ref_truncates_with_hint_when_over_cap() { async fn resolve_file_ref_truncates_with_hint_when_over_cap() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let body = "x".repeat(2048); let body = "x".repeat(2048);
std::fs::write(dir.path().join("big.txt"), &body).unwrap(); std::fs::write(dir.path().join("big.txt"), &body).unwrap();
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let item = view.resolve_file_ref("big.txt", 256).unwrap(); let item = view.resolve_file_ref("big.txt", 256).await.unwrap();
let text = format!("{item:?}"); let text = format!("{item:?}");
assert!(text.contains("[File: big.txt]")); assert!(text.contains("[File: big.txt]"));
assert!(text.contains("truncated")); assert!(text.contains("truncated"));
assert!(text.contains("2048 bytes total")); assert!(text.contains("2048 bytes total"));
} }
#[test] #[tokio::test]
fn resolve_file_ref_lists_directory_shallow_entries() { async fn resolve_file_ref_lists_directory_shallow_entries() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
std::fs::create_dir_all(dir.path().join("docs/sub")).unwrap(); std::fs::create_dir_all(dir.path().join("docs/sub")).unwrap();
touch(&dir.path().join("docs/.hidden"), "hidden"); touch(&dir.path().join("docs/.hidden"), "hidden");
@@ -465,7 +379,7 @@ mod tests {
); );
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let item = view.resolve_file_ref("docs", 4096).unwrap(); let item = view.resolve_file_ref("docs", 4096).await.unwrap();
let text = system_text(&item); let text = system_text(&item);
assert!(text.starts_with("[Dir: docs]\n")); assert!(text.starts_with("[Dir: docs]\n"));
assert!(text.contains("sub/")); assert!(text.contains("sub/"));
@@ -481,8 +395,8 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn resolve_file_ref_directory_listing_filters_unreadable_entries() { async fn resolve_file_ref_directory_listing_filters_unreadable_entries() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let docs = dir.path().join("docs"); let docs = dir.path().join("docs");
let secret = docs.join("secret"); let secret = docs.join("secret");
@@ -503,25 +417,25 @@ mod tests {
}], }],
}; };
let scope = Scope::from_config(&cfg).unwrap(); let scope = Scope::from_config(&cfg).unwrap();
let fs = ScopedFs::new(scope, dir.path().to_path_buf()); let fs: WorkdirHandle = Arc::new(LocalWorkdir::new(scope, dir.path().to_path_buf()));
let view = WorkerFsView::new(fs); let view = WorkerFsView::new(fs);
let item = view.resolve_file_ref("docs", 4096).unwrap(); let item = view.resolve_file_ref("docs", 4096).await.unwrap();
let text = system_text(&item); let text = system_text(&item);
assert!(text.contains("visible.txt")); assert!(text.contains("visible.txt"));
assert!(!text.contains("secret")); assert!(!text.contains("secret"));
assert!(!text.contains("hidden.txt")); assert!(!text.contains("hidden.txt"));
} }
#[test] #[tokio::test]
fn resolve_file_ref_directory_listing_uses_upload_byte_cap() { async fn resolve_file_ref_directory_listing_uses_upload_byte_cap() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
std::fs::create_dir(dir.path().join("docs")).unwrap(); std::fs::create_dir(dir.path().join("docs")).unwrap();
touch(&dir.path().join("docs/very-long-file-name.txt"), ""); touch(&dir.path().join("docs/very-long-file-name.txt"), "");
touch(&dir.path().join("docs/another-long-file-name.txt"), ""); touch(&dir.path().join("docs/another-long-file-name.txt"), "");
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let item = view.resolve_file_ref("docs", 10).unwrap(); let item = view.resolve_file_ref("docs", 10).await.unwrap();
let text = system_text(&item); let text = system_text(&item);
assert!(text.starts_with("[Dir: docs]\n")); assert!(text.starts_with("[Dir: docs]\n"));
assert!(text.contains("truncated")); assert!(text.contains("truncated"));
@@ -529,8 +443,8 @@ mod tests {
assert!(text.contains("use Glob or Read for more")); assert!(text.contains("use Glob or Read for more"));
} }
#[test] #[tokio::test]
fn resolve_file_ref_directory_listing_uses_completion_entry_limit() { async fn resolve_file_ref_directory_listing_uses_completion_entry_limit() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
std::fs::create_dir(dir.path().join("docs")).unwrap(); std::fs::create_dir(dir.path().join("docs")).unwrap();
for i in 0..(DIR_FILE_REF_ENTRY_LIMIT + 5) { for i in 0..(DIR_FILE_REF_ENTRY_LIMIT + 5) {
@@ -538,7 +452,7 @@ mod tests {
} }
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let item = view.resolve_file_ref("docs", 4096).unwrap(); let item = view.resolve_file_ref("docs", 4096).await.unwrap();
let text = system_text(&item); let text = system_text(&item);
assert!(text.contains("105 readable entries total")); assert!(text.contains("105 readable entries total"));
assert!(text.contains("file-099.txt")); assert!(text.contains("file-099.txt"));
@@ -547,8 +461,8 @@ mod tests {
} }
#[cfg(unix)] #[cfg(unix)]
#[test] #[tokio::test]
fn resolve_file_ref_directory_listing_marks_readable_symlink_entries() { async fn resolve_file_ref_directory_listing_marks_readable_symlink_entries() {
use std::os::unix::fs::symlink; use std::os::unix::fs::symlink;
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
@@ -557,92 +471,95 @@ mod tests {
symlink("target.txt", dir.path().join("docs/link.txt")).unwrap(); symlink("target.txt", dir.path().join("docs/link.txt")).unwrap();
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let item = view.resolve_file_ref("docs", 4096).unwrap(); let item = view.resolve_file_ref("docs", 4096).await.unwrap();
let text = system_text(&item); let text = system_text(&item);
assert!(text.contains("link.txt@")); assert!(text.contains("link.txt@"));
} }
#[test] #[tokio::test]
fn resolve_file_ref_rejects_binary_with_binary_error() { async fn resolve_file_ref_rejects_binary_with_binary_error() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("blob.bin"), [0xff, 0xfe, 0x00, 0x80]).unwrap(); std::fs::write(dir.path().join("blob.bin"), [0xff, 0xfe, 0x00, 0x80]).unwrap();
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let err = view.resolve_file_ref("blob.bin", 1024).unwrap_err(); let err = view.resolve_file_ref("blob.bin", 1024).await.unwrap_err();
assert!(matches!(err, ResolveError::Binary { .. })); assert!(matches!(err, ResolveError::Binary { .. }));
} }
#[test] #[tokio::test]
fn resolve_file_ref_returns_fs_error_for_out_of_scope() { async fn resolve_file_ref_returns_fs_error_for_out_of_scope() {
let outer = TempDir::new().unwrap(); let outer = TempDir::new().unwrap();
let inner = outer.path().join("scoped"); let inner = outer.path().join("scoped");
std::fs::create_dir(&inner).unwrap(); std::fs::create_dir(&inner).unwrap();
std::fs::write(outer.path().join("secret.txt"), "nope").unwrap(); std::fs::write(outer.path().join("secret.txt"), "nope").unwrap();
let scope = Scope::writable(&inner).unwrap(); let scope = Scope::writable(&inner).unwrap();
let fs = ScopedFs::new(scope, inner.clone()); let fs: WorkdirHandle = Arc::new(LocalWorkdir::new(scope, inner.clone()));
let view = WorkerFsView::new(fs); let view = WorkerFsView::new(fs);
// Absolute path outside of scope. // Absolute path outside of scope.
let outside = outer.path().join("secret.txt"); let outside = outer.path().join("secret.txt");
let err = view let err = view
.resolve_file_ref(outside.to_str().unwrap(), 1024) .resolve_file_ref(outside.to_str().unwrap(), 1024)
.await
.unwrap_err(); .unwrap_err();
assert!(matches!(err, ResolveError::Fs(_))); assert!(matches!(err, ResolveError::Fs(_)));
} }
#[test] #[tokio::test]
fn render_auto_read_skips_unreadable_targets() { async fn render_auto_read_skips_unreadable_targets() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let items = view.render_auto_read(&[ReadRequirement { let items = view
.render_auto_read(&[ReadRequirement {
path: dir.path().join("missing.txt"), path: dir.path().join("missing.txt"),
offset: None, offset: None,
limit: None, limit: None,
}]); }])
.await;
assert!(items.is_empty()); assert!(items.is_empty());
} }
#[test] #[tokio::test]
fn list_file_completions_lists_pwd_when_prefix_empty() { async fn list_file_completions_lists_pwd_when_prefix_empty() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
touch(&dir.path().join("alpha.rs"), ""); touch(&dir.path().join("alpha.rs"), "");
touch(&dir.path().join("beta.rs"), ""); touch(&dir.path().join("beta.rs"), "");
std::fs::create_dir(dir.path().join("subdir")).unwrap(); std::fs::create_dir(dir.path().join("subdir")).unwrap();
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let cands = view.list_file_completions(""); let cands = view.list_file_completions("").await;
// ディレクトリ first // ディレクトリ first
let names: Vec<&str> = cands.iter().map(|c| c.path.as_str()).collect(); let names: Vec<&str> = cands.iter().map(|c| c.path.as_str()).collect();
assert_eq!(names, vec!["subdir", "alpha.rs", "beta.rs"]); assert_eq!(names, vec!["subdir", "alpha.rs", "beta.rs"]);
assert!(cands[0].is_dir); assert!(cands[0].is_dir);
} }
#[test] #[tokio::test]
fn list_file_completions_filters_by_name_prefix() { async fn list_file_completions_filters_by_name_prefix() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
touch(&dir.path().join("alpha.rs"), ""); touch(&dir.path().join("alpha.rs"), "");
touch(&dir.path().join("beta.rs"), ""); touch(&dir.path().join("beta.rs"), "");
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let cands = view.list_file_completions("al"); let cands = view.list_file_completions("al").await;
assert_eq!(cands.len(), 1); assert_eq!(cands.len(), 1);
assert_eq!(cands[0].path, "alpha.rs"); assert_eq!(cands[0].path, "alpha.rs");
} }
#[test] #[tokio::test]
fn list_file_completions_descends_into_subdir_with_trailing_slash() { async fn list_file_completions_descends_into_subdir_with_trailing_slash() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
touch(&dir.path().join("sub/x.rs"), ""); touch(&dir.path().join("sub/x.rs"), "");
touch(&dir.path().join("sub/y.rs"), ""); touch(&dir.path().join("sub/y.rs"), "");
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let cands = view.list_file_completions("sub/"); let cands = view.list_file_completions("sub/").await;
let names: Vec<&str> = cands.iter().map(|c| c.path.as_str()).collect(); let names: Vec<&str> = cands.iter().map(|c| c.path.as_str()).collect();
assert_eq!(names, vec!["sub/x.rs", "sub/y.rs"]); assert_eq!(names, vec!["sub/x.rs", "sub/y.rs"]);
} }
#[test] #[tokio::test]
fn list_file_completions_filters_out_non_readable_under_scope() { async fn list_file_completions_filters_out_non_readable_under_scope() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let secret = dir.path().join("secret"); let secret = dir.path().join("secret");
std::fs::create_dir(&secret).unwrap(); std::fs::create_dir(&secret).unwrap();
@@ -662,25 +579,23 @@ mod tests {
}], }],
}; };
let scope = Scope::from_config(&cfg).unwrap(); let scope = Scope::from_config(&cfg).unwrap();
let fs = ScopedFs::new(scope, dir.path().to_path_buf()); let fs: WorkdirHandle = Arc::new(LocalWorkdir::new(scope, dir.path().to_path_buf()));
let view = WorkerFsView::new(fs); let view = WorkerFsView::new(fs);
let cands = view.list_file_completions(""); let cands = view.list_file_completions("").await;
let names: Vec<&str> = cands.iter().map(|c| c.path.as_str()).collect(); let names: Vec<&str> = cands.iter().map(|c| c.path.as_str()).collect();
assert!(names.contains(&"visible.rs")); assert!(names.contains(&"visible.rs"));
assert!(!names.contains(&"secret")); assert!(!names.contains(&"secret"));
} }
#[test] #[tokio::test]
fn list_file_completions_supports_absolute_prefix() { async fn list_file_completions_rejects_absolute_prefix() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
touch(&dir.path().join("a.rs"), ""); touch(&dir.path().join("a.rs"), "");
let view = WorkerFsView::new(fs_for(&dir)); let view = WorkerFsView::new(fs_for(&dir));
let prefix = format!("{}/", dir.path().display()); let prefix = format!("{}/", dir.path().display());
let cands = view.list_file_completions(&prefix); let cands = view.list_file_completions(&prefix).await;
assert_eq!(cands.len(), 1); assert!(cands.is_empty());
assert!(cands[0].path.starts_with('/'));
assert!(cands[0].path.ends_with("a.rs"));
} }
} }
+1 -1
View File
@@ -65,7 +65,7 @@ pub async fn dispatch_worker_protocol_method(
) -> Option<Event> { ) -> Option<Event> {
match method { match method {
Method::ListCompletions { kind, prefix } => { Method::ListCompletions { kind, prefix } => {
let entries = handle.completion_entries(kind, &prefix); let entries = handle.completion_entries(kind, &prefix).await;
Some(Event::Completions { kind, entries }) Some(Event::Completions { kind, entries })
} }
method => { method => {
+4 -5
View File
@@ -23,11 +23,10 @@ pub struct WorkerSharedState {
pub greeting: protocol::Greeting, pub greeting: protocol::Greeting,
pub status: RwLock<WorkerStatus>, pub status: RwLock<WorkerStatus>,
/// Worker-from-the-inside view of the filesystem. Set once in /// Worker-from-the-inside view of the filesystem. Set once in
/// `WorkerController::start` after the `ScopedFs` is materialised, and /// `WorkerController::start` after the local Workdir provider is
/// read from the IPC server layer to answer `ListCompletions` /// materialised, and read from the IPC server layer to answer
/// queries without going through the controller. `None` until set /// `ListCompletions` queries without going through the controller. It is
/// (only relevant for unit tests that build a `WorkerSharedState` /// unset only in unit tests that construct `WorkerSharedState` directly.
/// directly without spinning up a controller).
fs_view: OnceLock<WorkerFsView>, fs_view: OnceLock<WorkerFsView>,
} }
+70 -31
View File
@@ -76,6 +76,7 @@ use protocol::{
use tokio::net::UnixStream; use tokio::net::UnixStream;
use tokio::sync::broadcast; use tokio::sync::broadcast;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use workdir::{LocalWorkdir, WorkdirCapabilities, WorkdirHandle};
const RESTORE_RECONCILIATION_REACHABILITY_TIMEOUT: Duration = Duration::from_millis(500); const RESTORE_RECONCILIATION_REACHABILITY_TIMEOUT: Duration = Duration::from_millis(500);
@@ -741,13 +742,15 @@ pub struct Worker<C: LlmClient, St: Store> {
/// Explicit local filesystem authority, or `None` for Workers with no /// Explicit local filesystem authority, or `None` for Workers with no
/// local cwd and no filesystem/Bash tool surface. /// local cwd and no filesystem/Bash tool surface.
filesystem_authority: WorkerFilesystemAuthority, filesystem_authority: WorkerFilesystemAuthority,
/// Live Workdir provider derived once from the WorkerWorkdir binding.
/// Local tools, file views, and compaction workers clone this handle.
workdir: Option<WorkdirHandle>,
/// Path-free workspace identity/client context injected by Runtime/host. /// Path-free workspace identity/client context injected by Runtime/host.
/// This never grants local filesystem authority. /// This never grants local filesystem authority.
workspace_context: WorkerWorkspaceContext, workspace_context: WorkerWorkspaceContext,
/// Shared, atomically-swappable view of the Worker's resolved scope. /// Shared, atomically-swappable view of the Worker's resolved scope.
/// Cloned out to `ScopedFs` instances (builtin tools, fs_view, /// Cloned into local Workdir providers used by builtin tools, fs_view,
/// compact worker) so scope updates propagate to every consumer /// and compaction so updates propagate at the next permission check.
/// at the next permission check.
scope: SharedScope, scope: SharedScope,
/// Filesystem authority this Worker may pass to spawned children. Direct tools /// Filesystem authority this Worker may pass to spawned children. Direct tools
/// continue to use `scope`; SpawnWorker validates requested child scope here. /// continue to use `scope`; SpawnWorker validates requested child scope here.
@@ -923,6 +926,7 @@ impl<C: LlmClient + Clone + 'static, St: Store + Clone + 'static> Worker<C, St>
worker_metadata_writer: None, worker_metadata_writer: None,
segment_state: self.segment_state.clone(), segment_state: self.segment_state.clone(),
filesystem_authority: self.filesystem_authority.clone(), filesystem_authority: self.filesystem_authority.clone(),
workdir: self.workdir.clone(),
workspace_context: self.workspace_context.clone(), workspace_context: self.workspace_context.clone(),
scope: self.scope.clone(), scope: self.scope.clone(),
delegation_scope: self.delegation_scope.clone(), delegation_scope: self.delegation_scope.clone(),
@@ -1110,6 +1114,8 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
let prompts = PromptCatalog::builtins_only()?; let prompts = PromptCatalog::builtins_only()?;
let delegation_scope = let delegation_scope =
DelegationScope::from_config(&manifest.delegation_scope).map_err(WorkerError::Scope)?; DelegationScope::from_config(&manifest.delegation_scope).map_err(WorkerError::Scope)?;
let scope = SharedScope::new(scope);
let workdir = workdir_from_authority(&filesystem_authority, &scope);
let mut worker = Self { let mut worker = Self {
manifest, manifest,
engine: Some(worker), engine: Some(worker),
@@ -1117,8 +1123,9 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
worker_metadata_writer: None, worker_metadata_writer: None,
segment_state: SegmentState::new(session_id, segment_id, 0), segment_state: SegmentState::new(session_id, segment_id, 0),
filesystem_authority, filesystem_authority,
workdir,
workspace_context, workspace_context,
scope: SharedScope::new(scope), scope,
delegation_scope, delegation_scope,
hook_builder: HookRegistryBuilder::new(), hook_builder: HookRegistryBuilder::new(),
interceptor_installed: false, interceptor_installed: false,
@@ -1231,6 +1238,17 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
self.filesystem_authority.as_local() self.filesystem_authority.as_local()
} }
pub fn workdir(&self) -> Option<&WorkdirHandle> {
self.workdir.as_ref()
}
/// Replace the constructor fallback with the provider binding resolved by
/// the owning Runtime. Runtime calls this before the Worker controller is
/// spawned, so tools only ever observe the Runtime-bound handle.
pub fn bind_workdir(&mut self, workdir: Option<WorkdirHandle>) {
self.workdir = workdir;
}
/// Path-free workspace identity, if Runtime/host associated this Worker /// Path-free workspace identity, if Runtime/host associated this Worker
/// with a workspace. /// with a workspace.
pub fn workspace_id(&self) -> Option<&WorkspaceId> { pub fn workspace_id(&self) -> Option<&WorkspaceId> {
@@ -2063,7 +2081,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
// Resolve `@<path>` file refs to system messages stashed for the // Resolve `@<path>` file refs to system messages stashed for the
// WorkerInterceptor to attach right after the user message. Resolution // WorkerInterceptor to attach right after the user message. Resolution
// failures are non-fatal alerts. // failures are non-fatal alerts.
let attachments = self.resolve_file_refs(&input); let attachments = self.resolve_file_refs(&input).await;
let flattened = self.flatten_segments(&input); let flattened = self.flatten_segments(&input);
if !attachments.is_empty() { if !attachments.is_empty() {
*self *self
@@ -2094,8 +2112,8 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
/// directory) surface as `AlertLevel::Warn` Alerts and are skipped — the /// directory) surface as `AlertLevel::Warn` Alerts and are skipped — the
/// unresolved placeholder stays in the flattened user message so the LLM /// unresolved placeholder stays in the flattened user message so the LLM
/// still sees the intent. /// still sees the intent.
fn resolve_file_refs(&self, segments: &[Segment]) -> Vec<SystemItem> { async fn resolve_file_refs(&self, segments: &[Segment]) -> Vec<SystemItem> {
let Some(local) = self.local_working_directory() else { let Some(workdir) = self.workdir.clone() else {
for seg in segments { for seg in segments {
if let Segment::FileRef { path } = seg { if let Segment::FileRef { path } = seg {
self.alert( self.alert(
@@ -2107,16 +2125,16 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
} }
return Vec::new(); return Vec::new();
}; };
let view = crate::fs_view::WorkerFsView::new(tools::ScopedFs::with_shared_scope( let view = crate::fs_view::WorkerFsView::new(workdir);
self.scope.clone(),
local.cwd.clone(),
));
let mut out = Vec::new(); let mut out = Vec::new();
for seg in segments { for seg in segments {
let Segment::FileRef { path } = seg else { let Segment::FileRef { path } = seg else {
continue; continue;
}; };
match view.resolve_file_ref(path, self.manifest.engine.file_upload.max_bytes) { match view
.resolve_file_ref(path, self.manifest.engine.file_upload.max_bytes)
.await
{
Ok(item) => { Ok(item) => {
// `resolve_file_ref` returns an `Item::system_message` // `resolve_file_ref` returns an `Item::system_message`
// whose text already carries the `[File: <path>]` or // whose text already carries the `[File: <path>]` or
@@ -2872,13 +2890,10 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
auto_read_budget, auto_read_budget,
))); )));
// Build an independent compact worker. When the main Worker has local // Build an independent compact worker. It clones the main Worker's
// filesystem authority, compact-time reads go through the same scope // provider handle, so compact-time reads use the same Workdir instance.
// and cwd policy. No-workdir Workers deliberately omit compact-time // No-workdir Workers deliberately omit compact-time filesystem tools.
// filesystem tools as well. let workdir = self.workdir.clone();
let scoped_fs = self
.local_working_directory()
.map(|local| tools::ScopedFs::with_shared_scope(self.scope.clone(), local.cwd.clone()));
let summary_tracker = tools::Tracker::new(); let summary_tracker = tools::Tracker::new();
let summary_client: Box<dyn LlmClient> = self.build_compactor_client()?; let summary_client: Box<dyn LlmClient> = self.build_compactor_client()?;
let summary_system_prompt = self let summary_system_prompt = self
@@ -2916,9 +2931,9 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
// Tools: read_file (shared scope, fresh tracker), bounded session // Tools: read_file (shared scope, fresh tracker), bounded session
// history exploration, and compact-specific tools that populate `ctx`. // history exploration, and compact-specific tools that populate `ctx`.
let compact_target_items = Arc::new(items_to_summarise.clone()); let compact_target_items = Arc::new(items_to_summarise.clone());
if let Some(scoped_fs) = scoped_fs.clone() { if let Some(workdir) = workdir.clone() {
summary_worker.register_tool(tools::read_tool(scoped_fs.clone(), summary_tracker)); summary_worker.register_tool(tools::read_tool(workdir.clone(), summary_tracker));
summary_worker.register_tool(mark_read_required_tool(scoped_fs, ctx.clone())); summary_worker.register_tool(mark_read_required_tool(workdir, ctx.clone()));
} }
summary_worker.register_tool(search_session_log_tool(compact_target_items.clone())); summary_worker.register_tool(search_session_log_tool(compact_target_items.clone()));
summary_worker.register_tool(read_session_items_tool(compact_target_items)); summary_worker.register_tool(read_session_items_tool(compact_target_items));
@@ -2998,12 +3013,13 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
// logged and skipped inside `render_auto_read` rather than // logged and skipped inside `render_auto_read` rather than
// aborting compaction — a missing / moved file should not fail // aborting compaction — a missing / moved file should not fail
// the whole compact. // the whole compact.
let auto_read_messages = scoped_fs let auto_read_messages = if let Some(workdir) = workdir {
.clone() WorkerFsView::new(workdir)
.map(|scoped_fs| { .render_auto_read(&final_ctx.read_required)
WorkerFsView::new(scoped_fs).render_auto_read(&final_ctx.read_required) .await
}) } else {
.unwrap_or_default(); Vec::new()
};
// Reference list as a single system message; omitted when empty. // Reference list as a single system message; omitted when empty.
let reference_message = (!final_ctx.references.is_empty()).then(|| { let reference_message = (!final_ctx.references.is_empty()).then(|| {
@@ -3940,6 +3956,8 @@ where
apply_worker_manifest(&mut worker, &manifest.engine); apply_worker_manifest(&mut worker, &manifest.engine);
worker.set_cache_key(Some(segment_id.to_string())); worker.set_cache_key(Some(segment_id.to_string()));
let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store)); let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store));
let scope = SharedScope::new(common.scope);
let workdir = workdir_from_authority(&common.filesystem_authority, &scope);
let mut worker = Self { let mut worker = Self {
manifest, manifest,
@@ -3948,8 +3966,9 @@ where
worker_metadata_writer, worker_metadata_writer,
segment_state: SegmentState::new(session_id, segment_id, 0), segment_state: SegmentState::new(session_id, segment_id, 0),
filesystem_authority: common.filesystem_authority, filesystem_authority: common.filesystem_authority,
workdir,
workspace_context: common.workspace_context, workspace_context: common.workspace_context,
scope: SharedScope::new(common.scope), scope,
delegation_scope: common.delegation_scope, delegation_scope: common.delegation_scope,
hook_builder: HookRegistryBuilder::new(), hook_builder: HookRegistryBuilder::new(),
interceptor_installed: false, interceptor_installed: false,
@@ -4046,6 +4065,8 @@ where
apply_worker_manifest(&mut worker, &manifest.engine); apply_worker_manifest(&mut worker, &manifest.engine);
worker.set_cache_key(Some(segment_id.to_string())); worker.set_cache_key(Some(segment_id.to_string()));
let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store)); let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store));
let scope = SharedScope::new(common.scope);
let workdir = workdir_from_authority(&common.filesystem_authority, &scope);
let mut worker = Self { let mut worker = Self {
manifest, manifest,
@@ -4054,8 +4075,9 @@ where
worker_metadata_writer, worker_metadata_writer,
segment_state: SegmentState::new(session_id, segment_id, 0), segment_state: SegmentState::new(session_id, segment_id, 0),
filesystem_authority: common.filesystem_authority, filesystem_authority: common.filesystem_authority,
workdir,
workspace_context: common.workspace_context, workspace_context: common.workspace_context,
scope: SharedScope::new(common.scope), scope,
delegation_scope: common.delegation_scope, delegation_scope: common.delegation_scope,
hook_builder: HookRegistryBuilder::new(), hook_builder: HookRegistryBuilder::new(),
interceptor_installed: false, interceptor_installed: false,
@@ -4335,6 +4357,8 @@ where
let extract_pointer = memory::extract::fold_pointer(&state.extensions); let extract_pointer = memory::extract::fold_pointer(&state.extensions);
let task_feature = TaskFeature::from_history(&state.history); let task_feature = TaskFeature::from_history(&state.history);
let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store)); let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store));
let scope = SharedScope::new(common.scope);
let workdir = workdir_from_authority(&common.filesystem_authority, &scope);
let mut worker = Self { let mut worker = Self {
manifest, manifest,
@@ -4343,8 +4367,9 @@ where
worker_metadata_writer, worker_metadata_writer,
segment_state: SegmentState::new(session_id, segment_id, state.entries_count), segment_state: SegmentState::new(session_id, segment_id, state.entries_count),
filesystem_authority: common.filesystem_authority, filesystem_authority: common.filesystem_authority,
workdir,
workspace_context: common.workspace_context, workspace_context: common.workspace_context,
scope: SharedScope::new(common.scope), scope,
delegation_scope: common.delegation_scope, delegation_scope: common.delegation_scope,
hook_builder: HookRegistryBuilder::new(), hook_builder: HookRegistryBuilder::new(),
interceptor_installed: false, interceptor_installed: false,
@@ -5008,6 +5033,20 @@ pub enum WorkerError {
}, },
} }
fn workdir_from_authority(
authority: &WorkerFilesystemAuthority,
scope: &SharedScope,
) -> Option<WorkdirHandle> {
authority.as_local().map(|local| {
Arc::new(LocalWorkdir::materialized(
local.root.clone(),
local.cwd.clone(),
scope.clone(),
WorkdirCapabilities::ALL,
)) as WorkdirHandle
})
}
/// Bundle of resources that every high-level Worker constructor needs: /// Bundle of resources that every high-level Worker constructor needs:
/// filesystem authority, path-free workspace context, scope, an LLM client, the prompt catalog, /// filesystem authority, path-free workspace context, scope, an LLM client, the prompt catalog,
/// and (optionally) a parsed system-prompt template. Built once by /// and (optionally) a parsed system-prompt template. Built once by
+37
View File
@@ -11,6 +11,7 @@ use llm_engine::llm_client::{ClientError, LlmClient, Request};
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput}; use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use session_store::{CombinedStore, FsWorkerStore}; use session_store::{CombinedStore, FsWorkerStore};
use session_store::{FsStore, LogEntry}; use session_store::{FsStore, LogEntry};
use workdir::{CommandRequest, LocalWorkdir, WorkdirCapabilities, WorkdirError, WorkdirHandle};
use worker::{ use worker::{
Event, Method, Worker, WorkerController, WorkerFilesystemAuthority, WorkerHandle, Event, Method, Worker, WorkerController, WorkerFilesystemAuthority, WorkerHandle,
@@ -213,6 +214,42 @@ async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle
handle handle
} }
#[tokio::test]
async fn shutdown_closes_bound_workdir_commands() {
let (mut worker, pwd) = make_worker_with_pwd(MockClient::new(simple_text_events())).await;
let workdir: WorkdirHandle = Arc::new(LocalWorkdir::materialized_bound(
Some("controller-test-workdir".to_owned()),
pwd.clone(),
pwd,
worker.scope().clone(),
WorkdirCapabilities::ALL,
));
let command = workdir
.start_command(CommandRequest {
command: "sleep 30".to_owned(),
timeout_secs: 60,
output_limit: 1024,
})
.await
.unwrap();
worker.bind_workdir(Some(Arc::clone(&workdir)));
let runtime_base = tempfile::tempdir().unwrap();
let (handle, shutdown_rx) = WorkerController::spawn(worker, runtime_base.path())
.await
.unwrap();
handle.send(Method::Shutdown).await.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
.await
.expect("controller should shut down")
.expect("controller shutdown signal should remain open");
assert!(matches!(
workdir.command_status(command).await,
Err(WorkdirError::UnknownCommand(_))
));
}
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) { async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop { loop {