Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: add client-hosted external workdirs

Browse Source
This commit is contained in:
Hare
2026-09-22 06:49:15 +09:00
parent dc135ee1de
commit ce9534f3a2
45 changed files with 6024 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/fs-operation/Cargo.toml
+1
View File
@@ -10,6 +10,7 @@ grep-matcher = "0.1.8"
grep-regex = "0.1.14"
grep-searcher = "0.1.16"
ignore = "0.4.25"
libc = "0.2"
serde = { workspace = true, features = ["derive"] }
sha2 = "0.10.9"
tempfile.workspace = true
crates/fs-operation/src/glob.rs
+25 -2
View File
@@ -26,15 +26,31 @@ pub fn run_glob(
.map_err(|error| FsError::InvalidGlob(error.to_string()))?
.compile_matcher();
let mut matches = Vec::new();
let mut retained_path_bytes = 0_usize;
let mut provider_truncated = false;
let mut walker = WalkBuilder::new(base);
walker.hidden(false).follow_links(false);
let mut visited = 0_usize;
for entry in walker.build().flatten() {
visited = visited.saturating_add(1);
if visited > crate::MAX_TRAVERSAL_ENTRIES {
return Err(FsError::InvalidArgument(format!(
"glob traversal exceeds provider limit {}",
crate::MAX_TRAVERSAL_ENTRIES
)));
}
let path = entry.path();
let readable = resolve_access_path(path)
.is_ok_and(|resolved| access.is_readable_paths(path, &resolved));
if !path.is_file() || !readable {
continue;
}
let Ok(file) = access.open_read_file(path, path) else {
continue;
};
if !file.metadata().is_ok_and(|metadata| metadata.is_file()) {
continue;
}
let relative = path.strip_prefix(base).unwrap_or(path);
if !matcher.is_match(relative) {
continue;
@@ -42,10 +58,17 @@ pub fn run_glob(
let logical = path.strip_prefix(root).map_err(|_| {
FsError::InvalidArgument("provider returned a path outside its root".to_string())
})?;
matches.push(FsPath::new(logical.to_string_lossy())?);
let result_path = FsPath::new(logical.to_string_lossy())?;
let retained = result_path.as_str().len().saturating_add(16);
if retained_path_bytes.saturating_add(retained) > crate::MAX_RESULT_PATH_BYTES {
provider_truncated = true;
break;
}
retained_path_bytes = retained_path_bytes.saturating_add(retained);
matches.push(result_path);
}
matches.sort_by(|left, right| left.as_str().cmp(right.as_str()));
let truncated = matches.len() > request.limit;
let truncated = provider_truncated || matches.len() > request.limit;
matches.truncate(request.limit);
Ok(GlobResult {
paths: matches,
crates/fs-operation/src/lib.rs
+312 -1
View File
@@ -14,15 +14,167 @@ use std::path::{Path, PathBuf};
use thiserror::Error;
pub use glob::run_glob;
pub use local::{resolve_access_path, run_edit, run_list, run_read, run_stat, run_write};
pub use local::{
resolve_access_path, run_edit, run_list, run_read, run_read_bounded, run_stat, run_write,
};
pub use operation::*;
pub use search::run_grep;
/// Provider-side ceiling for directory/tree traversal, independent of result truncation.
pub const MAX_TRAVERSAL_ENTRIES: usize = 100_000;
/// Provider-side ceiling for aggregate retained result paths per operation.
pub const MAX_RESULT_PATH_BYTES: usize = 1024 * 1024;
/// Provider-side ceiling for aggregate grep source bytes per operation.
pub const MAX_GREP_SOURCE_BYTES: u64 = 64 * 1024 * 1024;
/// Open `path` beneath `root` without following any symbolic link. Linux uses
/// `openat2` so resolution and open are one kernel-enforced operation. Other
/// platforms fail closed rather than silently weakening an External grant.
#[cfg(target_os = "linux")]
pub fn open_root_no_symlinks(root: &Path) -> std::io::Result<std::fs::File> {
use std::ffi::CString;
use std::os::fd::{FromRawFd, RawFd};
use std::os::unix::ffi::OsStrExt;
let root = CString::new(root.as_os_str().as_bytes())
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "root contains NUL"))?;
// SAFETY: `root` is a valid C string and the returned descriptor is checked
// before ownership is transferred to File.
let fd: RawFd = unsafe {
libc::open(
root.as_ptr(),
libc::O_PATH | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
)
};
if fd < 0 {
return Err(std::io::Error::last_os_error());
}
// SAFETY: fd is a new owned descriptor returned by open.
Ok(unsafe { std::fs::File::from_raw_fd(fd) })
}
#[cfg(not(target_os = "linux"))]
pub fn open_root_no_symlinks(_root: &Path) -> std::io::Result<std::fs::File> {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"root-confined no-symlink open is unavailable on this platform",
))
}
#[cfg(target_os = "linux")]
pub fn open_beneath_no_symlinks_at(
root: &std::fs::File,
relative: &Path,
) -> std::io::Result<std::fs::File> {
use std::ffi::CString;
use std::os::fd::{AsRawFd, FromRawFd, RawFd};
use std::os::unix::ffi::OsStrExt;
#[repr(C)]
struct OpenHow {
flags: u64,
mode: u64,
resolve: u64,
}
const RESOLVE_NO_XDEV: u64 = 0x01;
const RESOLVE_NO_SYMLINKS: u64 = 0x04;
const RESOLVE_BENEATH: u64 = 0x08;
if relative.is_absolute() {
return Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"path is outside provider root",
));
}
let relative = if relative.as_os_str().is_empty() {
CString::new(".").expect("static path")
} else {
CString::new(relative.as_os_str().as_bytes()).map_err(|_| {
std::io::Error::new(std::io::ErrorKind::InvalidInput, "path contains NUL")
})?
};
let how = OpenHow {
flags: (libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK) as u64,
mode: 0,
resolve: RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS | RESOLVE_NO_XDEV,
};
// SAFETY: `how` and the path remain alive for the syscall duration, and the
// borrowed root descriptor remains valid for the call.
let fd = unsafe {
libc::syscall(
libc::SYS_openat2,
root.as_raw_fd(),
relative.as_ptr(),
&how as *const OpenHow,
std::mem::size_of::<OpenHow>(),
) as RawFd
};
if fd < 0 {
return Err(std::io::Error::last_os_error());
}
// SAFETY: fd is a new owned descriptor returned by openat2.
Ok(unsafe { std::fs::File::from_raw_fd(fd) })
}
#[cfg(not(target_os = "linux"))]
pub fn open_beneath_no_symlinks_at(
_root: &std::fs::File,
_relative: &Path,
) -> std::io::Result<std::fs::File> {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"root-confined no-symlink open is unavailable on this platform",
))
}
#[cfg(target_os = "linux")]
pub fn open_beneath_no_symlinks(root: &Path, path: &Path) -> std::io::Result<std::fs::File> {
let relative = path.strip_prefix(root).map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"path is outside provider root",
)
})?;
let root = open_root_no_symlinks(root)?;
open_beneath_no_symlinks_at(&root, relative)
}
#[cfg(not(target_os = "linux"))]
pub fn open_beneath_no_symlinks(_root: &Path, _path: &Path) -> std::io::Result<std::fs::File> {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"root-confined no-symlink open is unavailable on this platform",
))
}
/// Provider-owned access policy used by local filesystem operations.
pub trait FsAccessPolicy: Send + Sync {
fn is_readable(&self, path: &Path) -> bool;
fn is_writable(&self, path: &Path) -> bool;
/// Open an already-authorized readable file. Capability providers override
/// this to bind path resolution and open into one root-confined operation.
fn open_read_file(&self, _logical: &Path, resolved: &Path) -> std::io::Result<std::fs::File> {
std::fs::File::open(resolved)
}
/// Obtain metadata for an already-authorized path. Capability providers
/// override this to prevent a path swap between authorization and stat.
fn read_metadata(
&self,
logical: &Path,
_resolved: &Path,
) -> std::io::Result<std::fs::Metadata> {
std::fs::symlink_metadata(logical)
}
/// Open an already-authorized directory for bounded enumeration.
/// Capability providers override this to bind traversal to a confined
/// directory descriptor rather than reopening a mutable path.
fn open_read_dir(&self, _logical: &Path, resolved: &Path) -> std::io::Result<std::fs::ReadDir> {
std::fs::read_dir(resolved)
}
/// Authorize both the Workdir-visible path and its provider-resolved
/// target. Implementations that do not distinguish symbolic-link identity
/// retain resolved-target semantics through the defaults.
@@ -201,6 +353,165 @@ mod tests {
assert!(serde_json::from_str::<FsPath>(r#""../secret""#).is_err());
}
#[cfg(target_os = "linux")]
#[test]
fn root_confined_open_rejects_a_symlink_swap_between_authorization_and_open() {
use std::os::unix::fs::symlink;
use std::sync::atomic::{AtomicBool, Ordering};
struct SwapAccess {
root: PathBuf,
selected: PathBuf,
outside: PathBuf,
swapped: AtomicBool,
}
impl FsAccessPolicy for SwapAccess {
fn is_readable(&self, path: &Path) -> bool {
path.starts_with(&self.root)
}
fn is_writable(&self, _path: &Path) -> bool {
false
}
fn is_readable_paths(&self, _logical: &Path, _resolved: &Path) -> bool {
if !self.swapped.swap(true, Ordering::SeqCst) {
std::fs::remove_file(&self.selected).unwrap();
symlink(&self.outside, &self.selected).unwrap();
}
true
}
fn open_read_file(
&self,
_logical: &Path,
resolved: &Path,
) -> std::io::Result<std::fs::File> {
open_beneath_no_symlinks(&self.root, resolved)
}
}
let root_dir = tempfile::tempdir().unwrap();
let outside_dir = tempfile::tempdir().unwrap();
let root = root_dir.path().canonicalize().unwrap();
let outside = outside_dir.path().join("secret.txt");
let selected = root.join("selected.txt");
std::fs::write(&selected, "safe").unwrap();
std::fs::write(&outside, "secret").unwrap();
let access = SwapAccess {
root: root.clone(),
selected,
outside,
swapped: AtomicBool::new(false),
};
let error = run_read_bounded(
&root,
ReadRequest {
path: FsPath::new("selected.txt").unwrap(),
offset: 0,
limit: 10,
max_bytes: 1024,
},
&access,
BoundedReadLimits::EXTERNAL_DEFAULT,
)
.expect_err("openat2 must reject the swapped symbolic link");
assert!(matches!(error, FsError::Io { .. }));
}
#[cfg(target_os = "linux")]
#[test]
fn root_confined_stat_and_list_reject_swaps_after_authorization() {
use std::os::unix::fs::symlink;
use std::sync::atomic::{AtomicBool, Ordering};
struct SwapAccess {
root: PathBuf,
selected: PathBuf,
outside: PathBuf,
directory: bool,
swapped: AtomicBool,
}
impl FsAccessPolicy for SwapAccess {
fn is_readable(&self, path: &Path) -> bool {
path.starts_with(&self.root)
}
fn is_writable(&self, _path: &Path) -> bool {
false
}
fn is_readable_paths(&self, logical: &Path, _resolved: &Path) -> bool {
if logical == self.selected && !self.swapped.swap(true, Ordering::SeqCst) {
if self.directory {
std::fs::remove_dir(&self.selected).unwrap();
} else {
std::fs::remove_file(&self.selected).unwrap();
}
symlink(&self.outside, &self.selected).unwrap();
}
true
}
fn read_metadata(
&self,
_logical: &Path,
resolved: &Path,
) -> std::io::Result<std::fs::Metadata> {
open_beneath_no_symlinks(&self.root, resolved)?.metadata()
}
fn open_read_dir(
&self,
_logical: &Path,
resolved: &Path,
) -> std::io::Result<std::fs::ReadDir> {
use std::os::fd::AsRawFd;
let directory = open_beneath_no_symlinks(&self.root, resolved)?;
std::fs::read_dir(format!("/proc/self/fd/{}", directory.as_raw_fd()))
}
}
let root_dir = tempfile::tempdir().unwrap();
let outside_dir = tempfile::tempdir().unwrap();
let root = root_dir.path().canonicalize().unwrap();
let selected_file = root.join("selected.txt");
let outside_file = outside_dir.path().join("secret.txt");
std::fs::write(&selected_file, "safe").unwrap();
std::fs::write(&outside_file, "secret").unwrap();
let stat_error = run_stat(
&root,
StatRequest {
path: FsPath::new("selected.txt").unwrap(),
},
&SwapAccess {
root: root.clone(),
selected: selected_file,
outside: outside_file,
directory: false,
swapped: AtomicBool::new(false),
},
)
.expect_err("confined stat must reject a swapped symlink");
assert!(matches!(stat_error, FsError::Io { .. }));
let selected_dir = root.join("selected-dir");
let outside_content = outside_dir.path().join("outside-dir");
std::fs::create_dir(&selected_dir).unwrap();
std::fs::create_dir(&outside_content).unwrap();
std::fs::write(outside_content.join("secret.txt"), "secret").unwrap();
let list_error = run_list(
&root,
ListRequest {
path: FsPath::new("selected-dir").unwrap(),
limit: 10,
},
&SwapAccess {
root: root.clone(),
selected: selected_dir,
outside: outside_content,
directory: true,
swapped: AtomicBool::new(false),
},
)
.expect_err("confined list must reject a swapped symlink");
assert!(matches!(list_error, FsError::Io { .. }));
}
#[test]
fn direct_operations_cover_stat_read_write_edit_and_list() {
let temp = tempfile::tempdir().unwrap();
crates/fs-operation/src/local.rs
+214 -43
View File
@@ -1,16 +1,18 @@
use std::ffi::OsString;
use std::fs;
use std::io::Write;
use std::io::{BufReader, Read, Write};
use std::path::{Path, PathBuf};
use sha2::{Digest, Sha256};
use crate::{
ContentHash, EditRequest, EditResult, EntryKind, FsAccessPolicy, FsError, FsPath, ListEntry,
ListRequest, ListResult, ReadRequest, ReadResult, StatRequest, StatResult, WriteRequest,
WriteResult, direct_symlink,
BoundedReadLimits, ContentHash, EditRequest, EditResult, EntryKind, FsAccessPolicy, FsError,
FsPath, ListEntry, ListRequest, ListResult, ReadRequest, ReadResult, StatRequest, StatResult,
WriteRequest, WriteResult, direct_symlink,
};
const READ_BUFFER_BYTES: usize = 16 * 1024;
/// Execute stat while keeping host paths inside the provider boundary.
pub fn run_stat(
root: &Path,
@@ -23,7 +25,9 @@ pub fn run_stat(
if !access.is_readable_paths(&path, &resolved) {
return Err(FsError::OutOfScope(PathBuf::from(logical.as_str())));
}
let metadata = fs::symlink_metadata(&path).map_err(|error| map_io(&logical, error))?;
let metadata = access
.read_metadata(&path, &resolved)
.map_err(|error| map_io(&logical, error))?;
let kind = if metadata.file_type().is_symlink() {
EntryKind::Symlink
} else if metadata.is_file() {
@@ -44,20 +48,110 @@ pub fn run_read(
root: &Path,
request: ReadRequest,
access: &dyn FsAccessPolicy,
) -> Result<ReadResult, FsError> {
run_read_with_limits(root, request, access, None)
}
/// Execute a read while enforcing provider-owned source and response bounds.
///
/// The source is streamed through a fixed-size buffer. Files larger than the
/// configured source bound fail before their content is read, and growth after
/// the metadata check is fenced while streaming.
pub fn run_read_bounded(
root: &Path,
request: ReadRequest,
access: &dyn FsAccessPolicy,
limits: BoundedReadLimits,
) -> Result<ReadResult, FsError> {
run_read_with_limits(root, request, access, Some(limits))
}
fn run_read_with_limits(
root: &Path,
request: ReadRequest,
access: &dyn FsAccessPolicy,
limits: Option<BoundedReadLimits>,
) -> Result<ReadResult, FsError> {
let logical = request.path;
let path = resolve(root, &logical)?;
let path = require_access(&path, &logical, access, false, false)?;
let metadata = fs::metadata(&path).map_err(|error| map_io(&logical, error))?;
let target = require_access(&path, &logical, access, false, false)?;
let file = access
.open_read_file(&path, &target)
.map_err(|error| map_io(&logical, error))?;
let metadata = file.metadata().map_err(|error| map_io(&logical, error))?;
if metadata.is_dir() {
return Err(FsError::IsDirectory(PathBuf::from(logical.as_str())));
}
let bytes = fs::read(&path).map_err(|error| map_io(&logical, error))?;
let content_hash = hash_bytes(&bytes);
let lines = bytes
.split_inclusive(|byte| *byte == b'\n')
.collect::<Vec<_>>();
let total_lines = lines.len();
if !metadata.is_file() {
return Err(FsError::InvalidArgument(
"read source must be a regular file".to_string(),
));
}
if let Some(limits) = limits
&& metadata.len() > limits.max_source_bytes
{
return Err(FsError::InvalidArgument(format!(
"read source size {} exceeds provider limit {}",
metadata.len(),
limits.max_source_bytes
)));
}
let response_limit = limits
.map(|limits| request.max_bytes.min(limits.max_response_bytes))
.unwrap_or(request.max_bytes);
let retained_limit = response_limit.saturating_add(4);
let selected_end = request.offset.saturating_add(request.limit);
let mut selected = Vec::with_capacity(retained_limit.min(READ_BUFFER_BYTES));
let mut selected_bytes_seen = 0usize;
let mut current_line = 0usize;
let mut source_bytes_seen = 0u64;
let mut last_byte = None;
let mut content_hasher = Sha256::new();
let mut reader = BufReader::with_capacity(READ_BUFFER_BYTES, file);
let mut buffer = [0u8; READ_BUFFER_BYTES];
loop {
let read_limit = limits
.map(|limits| {
limits
.max_source_bytes
.saturating_sub(source_bytes_seen)
.saturating_add(1)
.min(READ_BUFFER_BYTES as u64) as usize
})
.unwrap_or(READ_BUFFER_BYTES);
let read = reader
.read(&mut buffer[..read_limit])
.map_err(|error| map_io(&logical, error))?;
if read == 0 {
break;
}
source_bytes_seen = source_bytes_seen.saturating_add(read as u64);
if let Some(limits) = limits
&& source_bytes_seen > limits.max_source_bytes
{
return Err(FsError::InvalidArgument(format!(
"read source exceeds provider limit {}",
limits.max_source_bytes
)));
}
content_hasher.update(&buffer[..read]);
for byte in &buffer[..read] {
if current_line >= request.offset && current_line < selected_end {
selected_bytes_seen = selected_bytes_seen.saturating_add(1);
if selected.len() < retained_limit {
selected.push(*byte);
}
}
if *byte == b'\n' {
current_line = current_line.saturating_add(1);
}
last_byte = Some(*byte);
}
}
let total_lines =
current_line.saturating_add(usize::from(last_byte.is_some_and(|byte| byte != b'\n')));
if request.offset > total_lines && request.offset != 0 {
return Err(FsError::InvalidArgument(format!(
"offset {} exceeds file length {total_lines}",
@@ -68,16 +162,14 @@ pub fn run_read(
.offset
.saturating_add(request.limit)
.min(total_lines);
let mut selected = lines[request.offset.min(total_lines)..end]
.iter()
.flat_map(|line| line.iter().copied())
.collect::<Vec<_>>();
let byte_truncated = selected.len() > request.max_bytes;
let byte_truncated = selected_bytes_seen > response_limit;
if byte_truncated {
let mut byte_end = request.max_bytes;
if let Ok(text) = std::str::from_utf8(&selected) {
while byte_end > 0 && !text.is_char_boundary(byte_end) {
byte_end -= 1;
let mut byte_end = response_limit.min(selected.len());
while byte_end > 0 {
match std::str::from_utf8(&selected[..byte_end]) {
Ok(_) => break,
Err(error) if error.error_len().is_none() => byte_end -= 1,
Err(_) => break,
}
}
selected.truncate(byte_end);
@@ -87,7 +179,7 @@ pub fn run_read(
bytes: selected,
start_line: request.offset,
total_lines,
content_hash,
content_hash: content_hasher.finalize().into(),
truncated: end < total_lines || byte_truncated,
})
}
@@ -173,35 +265,39 @@ pub fn run_list(
let path = resolve(root, &logical)?;
let logical_base = path.clone();
let path = require_access(&path, &logical, access, false, true)?;
let metadata = fs::metadata(&path).map_err(|error| map_io(&logical, error))?;
let metadata = access
.open_read_file(&logical_base, &path)
.and_then(|directory| directory.metadata())
.map_err(|error| map_io(&logical, error))?;
if !metadata.is_dir() {
return Err(FsError::NotDirectory(PathBuf::from(logical.as_str())));
}
let mut entries = Vec::new();
let read_dir = fs::read_dir(&path).map_err(|error| map_io(&logical, error))?;
for entry in read_dir {
let mut retained_path_bytes = 0_usize;
let mut provider_truncated = false;
let read_dir = access
.open_read_dir(&logical_base, &path)
.map_err(|error| map_io(&logical, error))?;
for (entry_index, entry) in read_dir.enumerate() {
if entry_index >= crate::MAX_TRAVERSAL_ENTRIES {
return Err(FsError::InvalidArgument(format!(
"directory traversal exceeds provider limit {}",
crate::MAX_TRAVERSAL_ENTRIES
)));
}
let entry = entry.map_err(|error| map_io(&logical, error))?;
let absolute = entry.path();
let relative_to_base = absolute.strip_prefix(&path).map_err(|_| {
FsError::InvalidArgument("provider returned a path outside its list base".to_string())
})?;
let logical_absolute = logical_base.join(relative_to_base);
let resolved = match resolve_access_path(&absolute) {
let logical_absolute = logical_base.join(entry.file_name());
let resolved = match resolve_access_path(&logical_absolute) {
Ok(resolved) => resolved,
Err(_) => continue,
};
if !access.is_readable_paths(&logical_absolute, &resolved) {
continue;
}
let link_metadata =
fs::symlink_metadata(&absolute).map_err(|error| map_io(&logical, error))?;
let is_symlink = link_metadata.file_type().is_symlink();
let metadata = if is_symlink {
link_metadata
} else {
entry.metadata().map_err(|error| map_io(&logical, error))?
};
let kind = if is_symlink {
let metadata = access
.read_metadata(&logical_absolute, &resolved)
.map_err(|error| map_io(&logical, error))?;
let kind = if metadata.file_type().is_symlink() {
EntryKind::Symlink
} else if metadata.is_file() {
EntryKind::File
@@ -213,8 +309,15 @@ pub fn run_list(
let relative = logical_absolute.strip_prefix(root).map_err(|_| {
FsError::InvalidArgument("provider returned a path outside its root".to_string())
})?;
let result_path = FsPath::new(relative.to_string_lossy())?;
let retained = result_path.as_str().len().saturating_add(64);
if retained_path_bytes.saturating_add(retained) > crate::MAX_RESULT_PATH_BYTES {
provider_truncated = true;
break;
}
retained_path_bytes = retained_path_bytes.saturating_add(retained);
entries.push(ListEntry {
path: FsPath::new(relative.to_string_lossy())?,
path: result_path,
kind,
size: metadata.len(),
});
@@ -228,7 +331,7 @@ pub fn run_list(
});
let total_entries = entries.len();
let total_bytes = entries.iter().map(|entry| entry.size).sum();
let truncated = entries.len() > request.limit;
let truncated = provider_truncated || entries.len() > request.limit;
entries.truncate(request.limit);
Ok(ListResult {
entries,
@@ -366,3 +469,71 @@ fn map_io(logical: &FsPath, error: std::io::Error) -> FsError {
},
}
}
#[cfg(test)]
mod bounded_read_tests {
use std::path::{Path, PathBuf};
use super::*;
struct RootAccess(PathBuf);
impl FsAccessPolicy for RootAccess {
fn is_readable(&self, path: &Path) -> bool {
path.starts_with(&self.0)
}
fn is_writable(&self, _path: &Path) -> bool {
false
}
}
#[test]
fn bounded_read_rejects_an_oversized_source_before_loading_content() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().canonicalize().unwrap();
let source = root.join("large.log");
let file = fs::File::create(&source).unwrap();
file.set_len(4097).unwrap();
let error = run_read_bounded(
&root,
ReadRequest {
path: FsPath::new("large.log").unwrap(),
offset: 0,
limit: 10,
max_bytes: usize::MAX,
},
&RootAccess(root.clone()),
BoundedReadLimits::new(4096, 1024).unwrap(),
)
.unwrap_err();
assert!(matches!(error, FsError::InvalidArgument(_)));
assert!(error.to_string().contains("exceeds provider limit 4096"));
}
#[test]
fn bounded_read_caps_retained_response_independently_of_request() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().canonicalize().unwrap();
fs::write(root.join("events.log"), "alpha\nbeta\ngamma\n").unwrap();
let result = run_read_bounded(
&root,
ReadRequest {
path: FsPath::new("events.log").unwrap(),
offset: 0,
limit: 10,
max_bytes: usize::MAX,
},
&RootAccess(root.clone()),
BoundedReadLimits::new(1024, 5).unwrap(),
)
.unwrap();
assert_eq!(result.bytes, b"alpha");
assert_eq!(result.total_lines, 3);
assert!(result.truncated);
}
}
crates/fs-operation/src/operation.rs
+79
View File
@@ -5,6 +5,71 @@ use serde::{Deserialize, Serialize};
use crate::FsError;
/// Absolute provider-side bounds for one filesystem read.
///
/// `max_source_bytes` limits how much of the source the provider will inspect
/// while computing line and content metadata. `max_response_bytes` limits the
/// bytes retained for the response independently of the caller's requested
/// `ReadRequest::max_bytes`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct BoundedReadLimits {
pub max_source_bytes: u64,
pub max_response_bytes: usize,
}
impl BoundedReadLimits {
/// Conservative defaults for a client-hosted, read-only provider.
pub const EXTERNAL_DEFAULT: Self = Self {
max_source_bytes: 64 * 1024 * 1024,
max_response_bytes: 1024 * 1024,
};
pub fn new(max_source_bytes: u64, max_response_bytes: usize) -> Result<Self, FsError> {
let limits = Self {
max_source_bytes,
max_response_bytes,
};
limits.validate()?;
Ok(limits)
}
pub fn validate(self) -> Result<(), FsError> {
if self.max_source_bytes == 0 {
return Err(FsError::InvalidArgument(
"max_source_bytes must be greater than zero".to_string(),
));
}
if self.max_response_bytes == 0 {
return Err(FsError::InvalidArgument(
"max_response_bytes must be greater than zero".to_string(),
));
}
if self.max_response_bytes as u128 > self.max_source_bytes as u128 {
return Err(FsError::InvalidArgument(
"max_response_bytes must not exceed max_source_bytes".to_string(),
));
}
Ok(())
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct BoundedReadLimitsWire {
max_source_bytes: u64,
max_response_bytes: usize,
}
impl<'de> Deserialize<'de> for BoundedReadLimits {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let wire = BoundedReadLimitsWire::deserialize(deserializer)?;
Self::new(wire.max_source_bytes, wire.max_response_bytes).map_err(serde::de::Error::custom)
}
}
/// Scope-checked filesystem path. Relative paths resolve below the bound
/// Workdir root; absolute paths require an explicit matching scope rule.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
@@ -89,11 +154,13 @@ impl fmt::Display for FsPath {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StatRequest {
pub path: FsPath,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StatResult {
pub path: FsPath,
pub kind: EntryKind,
@@ -112,6 +179,7 @@ pub enum EntryKind {
pub type ContentHash = [u8; 32];
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReadRequest {
pub path: FsPath,
pub offset: usize,
@@ -120,6 +188,7 @@ pub struct ReadRequest {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReadResult {
pub path: FsPath,
pub bytes: Vec<u8>,
@@ -130,6 +199,7 @@ pub struct ReadResult {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WriteRequest {
pub path: FsPath,
pub content: Vec<u8>,
@@ -143,6 +213,7 @@ pub struct WriteResult {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct EditRequest {
pub path: FsPath,
pub old_string: String,
@@ -152,6 +223,7 @@ pub struct EditRequest {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct EditResult {
pub replacements: usize,
pub bytes_written: usize,
@@ -159,12 +231,14 @@ pub struct EditResult {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ListRequest {
pub path: FsPath,
pub limit: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ListEntry {
pub path: FsPath,
pub kind: EntryKind,
@@ -172,6 +246,7 @@ pub struct ListEntry {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ListResult {
pub entries: Vec<ListEntry>,
pub total_entries: usize,
@@ -180,6 +255,7 @@ pub struct ListResult {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GlobRequest {
pub pattern: String,
pub path: FsPath,
@@ -187,6 +263,7 @@ pub struct GlobRequest {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GlobResult {
pub paths: Vec<FsPath>,
pub truncated: bool,
@@ -207,6 +284,7 @@ impl Default for GrepOutputMode {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GrepRequest {
pub pattern: String,
pub path: FsPath,
@@ -222,6 +300,7 @@ pub struct GrepRequest {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GrepResult {
/// Provider-rendered bounded grep report. Keeping rendering here avoids
/// transferring candidate files across a remote provider boundary.
crates/fs-operation/src/search.rs
+158 -5
View File
@@ -1,5 +1,6 @@
use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::io::Read;
use std::path::{Path, PathBuf};
use crate::FsAccessPolicy;
@@ -14,6 +15,35 @@ use crate::{
FsError, GrepOutputMode, GrepRequest, GrepResult, direct_symlink, resolve_access_path,
};
struct SourceBoundedReader<'a, R> {
inner: R,
remaining: &'a mut u64,
}
impl<R: Read> Read for SourceBoundedReader<'_, R> {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
if buffer.is_empty() {
return Ok(0);
}
if *self.remaining == 0 {
let mut probe = [0_u8; 1];
return match self.inner.read(&mut probe)? {
0 => Ok(0),
_ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"grep source exceeded provider byte limit",
)),
};
}
let limit = usize::try_from(*self.remaining)
.unwrap_or(usize::MAX)
.min(buffer.len());
let read = self.inner.read(&mut buffer[..limit])?;
*self.remaining = (*self.remaining).saturating_sub(read as u64);
Ok(read)
}
}
struct ContentLine {
path: PathBuf,
line_number: Option<u64>,
@@ -27,9 +57,25 @@ struct GrepReport {
files: Vec<PathBuf>,
counts: Vec<(PathBuf, usize)>,
lines: Vec<ContentLine>,
retained_bytes: usize,
truncated: bool,
}
const MAX_GREP_RESULT_RETAINED_BYTES: usize = 512 * 1024;
fn reserve_report_bytes(report: &mut GrepReport, bytes: usize) -> bool {
let Some(next) = report.retained_bytes.checked_add(bytes) else {
report.truncated = true;
return false;
};
if next > MAX_GREP_RESULT_RETAINED_BYTES {
report.truncated = true;
return false;
}
report.retained_bytes = next;
true
}
impl GrepReport {
fn into_result(self, root: &Path) -> GrepResult {
let (match_count, matched_files) = match self.mode {
@@ -281,12 +327,14 @@ pub fn run_grep(
files: Vec::new(),
counts: Vec::new(),
lines: Vec::new(),
retained_bytes: 0,
truncated: false,
};
let mut matching_files_seen = 0;
let mut matches_seen = 0;
if base_meta.is_file() {
let mut source_bytes_remaining = crate::MAX_GREP_SOURCE_BYTES;
if direct_file_selected(&base, overrides.as_ref(), types.as_ref()) {
scan_path(
&mut searcher,
@@ -298,6 +346,8 @@ pub fn run_grep(
&mut matches_seen,
offset,
head_limit,
&mut source_bytes_remaining,
access,
)?;
}
return Ok(report.into_result(root));
@@ -319,7 +369,16 @@ pub fn run_grep(
walker.overrides(overrides);
}
let mut visited = 0_usize;
let mut source_bytes_remaining = crate::MAX_GREP_SOURCE_BYTES;
for entry in walker.build().flatten() {
visited = visited.saturating_add(1);
if visited > crate::MAX_TRAVERSAL_ENTRIES {
return Err(FsError::InvalidArgument(format!(
"grep traversal exceeds provider limit {}",
crate::MAX_TRAVERSAL_ENTRIES
)));
}
if !entry
.file_type()
.map(|kind| kind.is_file())
@@ -343,6 +402,8 @@ pub fn run_grep(
&mut matches_seen,
offset,
head_limit,
&mut source_bytes_remaining,
access,
)? {
break;
}
@@ -362,13 +423,38 @@ fn scan_path(
matches_seen: &mut usize,
offset: usize,
head_limit: usize,
source_bytes_remaining: &mut u64,
access: &dyn FsAccessPolicy,
) -> Result<bool, FsError> {
let reader = access
.open_read_file(path, path)
.map_err(|error| FsError::io(path, error))?;
let metadata = reader
.metadata()
.map_err(|error| FsError::io(path, error))?;
if !metadata.is_file() {
return Err(FsError::InvalidArgument(
"grep source must be a regular file".to_string(),
));
}
if metadata.len() > *source_bytes_remaining {
return Err(FsError::InvalidArgument(
"grep sources exceed provider byte limit".to_string(),
));
}
let mut reader = SourceBoundedReader {
inner: reader,
remaining: source_bytes_remaining,
};
match mode {
GrepOutputMode::FilesWithMatches => {
if !scan_any_match(searcher, matcher, path)? {
if !scan_any_match(searcher, matcher, &mut reader, path)? {
return Ok(false);
}
if *matching_files_seen >= offset {
if !reserve_report_bytes(report, path.to_string_lossy().len().saturating_add(1)) {
return Ok(true);
}
report.files.push(path.to_path_buf());
if report.files.len() >= head_limit {
report.truncated = true;
@@ -378,11 +464,14 @@ fn scan_path(
*matching_files_seen += 1;
}
GrepOutputMode::Count => {
let count = scan_count(searcher, matcher, path)?;
let count = scan_count(searcher, matcher, &mut reader, path)?;
if count == 0 {
return Ok(false);
}
if *matching_files_seen >= offset {
if !reserve_report_bytes(report, path.to_string_lossy().len().saturating_add(32)) {
return Ok(true);
}
report.counts.push((path.to_path_buf(), count));
if report.counts.len() >= head_limit {
report.truncated = true;
@@ -396,13 +485,18 @@ fn scan_path(
let mut sink = ContentSink {
path: path.to_path_buf(),
lines: &mut report.lines,
retained_bytes: &mut report.retained_bytes,
truncated: &mut report.truncated,
matches_seen,
offset,
head_limit,
};
searcher
.search_path(matcher, path, &mut sink)
.search_reader(matcher, &mut reader, &mut sink)
.map_err(|error| FsError::io(path, error))?;
if report.truncated {
return Ok(true);
}
if *matches_seen >= offset.saturating_add(head_limit) && *matches_seen > before_count {
report.truncated = true;
return Ok(true);
@@ -415,6 +509,7 @@ fn scan_path(
fn scan_any_match(
searcher: &mut Searcher,
matcher: &grep_regex::RegexMatcher,
reader: &mut dyn Read,
path: &Path,
) -> Result<bool, FsError> {
let mut hit = false;
@@ -423,7 +518,7 @@ fn scan_any_match(
Ok(false) // stop searching this file immediately
});
searcher
.search_path(matcher, path, sink)
.search_reader(matcher, reader, sink)
.map_err(|e| FsError::io(path, e))?;
Ok(hit)
}
@@ -431,6 +526,7 @@ fn scan_any_match(
fn scan_count(
searcher: &mut Searcher,
matcher: &grep_regex::RegexMatcher,
reader: &mut dyn Read,
path: &Path,
) -> Result<usize, FsError> {
let mut count = 0usize;
@@ -439,7 +535,7 @@ fn scan_count(
Ok(true)
});
searcher
.search_path(matcher, path, sink)
.search_reader(matcher, reader, sink)
.map_err(|e| FsError::io(path, e))?;
Ok(count)
}
@@ -447,11 +543,34 @@ fn scan_count(
struct ContentSink<'a> {
path: PathBuf,
lines: &'a mut Vec<ContentLine>,
retained_bytes: &'a mut usize,
truncated: &'a mut bool,
matches_seen: &'a mut usize,
offset: usize,
head_limit: usize,
}
impl ContentSink<'_> {
fn reserve(&mut self, content_bytes: usize) -> bool {
let retained = self
.path
.to_string_lossy()
.len()
.saturating_add(content_bytes)
.saturating_add(64);
let Some(next) = (*self.retained_bytes).checked_add(retained) else {
*self.truncated = true;
return false;
};
if next > MAX_GREP_RESULT_RETAINED_BYTES {
*self.truncated = true;
return false;
}
*self.retained_bytes = next;
true
}
}
impl Sink for ContentSink<'_> {
type Error = std::io::Error;
@@ -468,6 +587,9 @@ impl Sink for ContentSink<'_> {
return Ok(false);
}
if !self.reserve(mat.bytes().len()) {
return Ok(false);
}
let text = String::from_utf8_lossy(mat.bytes())
.trim_end_matches('\n')
.trim_end_matches('\r')
@@ -493,6 +615,9 @@ impl Sink for ContentSink<'_> {
if seen >= self.offset.saturating_add(self.head_limit) {
return Ok(false);
}
if !self.reserve(ctx.bytes().len()) {
return Ok(false);
}
let text = String::from_utf8_lossy(ctx.bytes())
.trim_end_matches('\n')
.trim_end_matches('\r')
@@ -506,3 +631,31 @@ impl Sink for ContentSink<'_> {
Ok(true)
}
}
#[cfg(test)]
mod bounded_reader_tests {
use std::io::{Cursor, Read as _};
use super::SourceBoundedReader;
#[test]
fn source_budget_is_shared_across_candidate_files() {
let mut remaining = 3_u64;
let mut first = SourceBoundedReader {
inner: Cursor::new(b"ab"),
remaining: &mut remaining,
};
let mut output = Vec::new();
first.read_to_end(&mut output).unwrap();
drop(first);
assert_eq!(remaining, 1);
let mut second = SourceBoundedReader {
inner: Cursor::new(b"cd"),
remaining: &mut remaining,
};
let error = second.read_to_end(&mut output).unwrap_err();
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(output, b"abc");
}
}