feat: project attached checkouts with scoped native AI operations

This commit is contained in:
2026-10-06 03:37:07 +09:00
parent 3deef49897
commit 149853bb0e
43 changed files with 8706 additions and 346 deletions
+518
View File
@@ -0,0 +1,518 @@
//! Identity-fenced access adapter for the existing read/edit/write engine.
//!
//! Descriptor-rooted resolution is Linux-only and symlinks/mount crossings fail
//! closed. The final compare and atomic rename are NOT a kernel CAS against an
//! arbitrary external editor: external editors must cooperate with provider
//! serialization. A hostile rename in that last syscall gap is not excluded.
use crate::{AtomicWriteMode, FsAccessPolicy};
use std::fs::{File, Metadata};
use std::io::{Seek, SeekFrom};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
/// Identity and state, deliberately not just a content digest. atime is omitted
/// so reading does not invalidate an observation. No host paths are encoded.
pub fn identity_validator(metadata: &Metadata) -> std::io::Result<Vec<u8>> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let mut bytes = b"yoi-checkout-v1".to_vec();
for value in [
metadata.dev(),
metadata.ino(),
metadata.len(),
metadata.mode() as u64,
metadata.mtime() as u64,
metadata.mtime_nsec() as u64,
metadata.ctime() as u64,
metadata.ctime_nsec() as u64,
] {
bytes.extend_from_slice(&value.to_le_bytes());
}
Ok(bytes)
}
#[cfg(not(unix))]
{
let _ = metadata;
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"identity validators unavailable",
))
}
}
/// One pinned target and its save parent. Retained through the shared operation.
pub struct CheckedTarget<'a> {
policy: &'a dyn FsAccessPolicy,
root: File,
relative: PathBuf,
logical: PathBuf,
file: File,
parent: Option<File>,
validator: Vec<u8>,
effects_possible: AtomicBool,
}
impl<'a> CheckedTarget<'a> {
pub fn pin(
root: &File,
relative: &Path,
logical: &Path,
policy: &'a dyn FsAccessPolicy,
write: bool,
) -> std::io::Result<Self> {
policy.check_cancelled()?;
if !(if write {
policy.is_writable_paths(logical, logical)
} else {
policy.is_readable_paths(logical, logical)
}) {
return Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"checkout target denied",
));
}
let parent = if relative.as_os_str().is_empty() {
None
} else {
Some(crate::open_beneath_no_symlinks_at(
root,
relative.parent().unwrap_or(Path::new("")),
)?)
};
let file = if let Some(parent) = &parent {
crate::open_beneath_no_symlinks_at(parent, Path::new(relative.file_name().unwrap()))?
} else {
crate::open_beneath_no_symlinks_at(root, Path::new(""))?
};
let metadata = file.metadata()?;
if !metadata.is_file() && !metadata.is_dir() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"checkout requires regular file or directory",
));
}
let result = Self {
policy,
root: root.try_clone()?,
relative: relative.to_path_buf(),
logical: logical.to_path_buf(),
file,
parent,
validator: identity_validator(&metadata)?,
effects_possible: AtomicBool::new(false),
};
result.confirm()?;
Ok(result)
}
pub fn validator(&self) -> &[u8] {
&self.validator
}
pub fn metadata(&self) -> std::io::Result<Metadata> {
self.file.metadata()
}
pub fn effects_possible(&self) -> bool {
self.effects_possible.load(Ordering::Acquire)
}
pub fn confirm(&self) -> std::io::Result<()> {
self.policy.check_cancelled()?;
if let Some(parent) = &self.parent {
let named_parent = crate::open_beneath_no_symlinks_at(
&self.root,
self.relative.parent().unwrap_or(Path::new("")),
)?;
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let pinned_metadata = parent.metadata()?;
let named_metadata = named_parent.metadata()?;
if (pinned_metadata.dev(), pinned_metadata.ino())
!= (named_metadata.dev(), named_metadata.ino())
{
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"checkout parent identity changed",
));
}
}
}
let named = crate::open_beneath_no_symlinks_at(&self.root, &self.relative)?;
if identity_validator(&named.metadata()?)? != self.validator
|| identity_validator(&self.file.metadata()?)? != self.validator
{
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"checkout observation changed",
));
}
Ok(())
}
/// Refresh a directory observation after Create, retaining the original
/// pinned target/root/parent handles. A replacement named directory cannot
/// supply the validator for effects performed in the original directory.
pub fn refresh_directory_observation(&mut self) -> std::io::Result<()> {
let metadata = self.file.metadata()?;
if !metadata.is_dir() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"Create target must be directory",
));
}
self.validator = identity_validator(&metadata)?;
self.confirm()
}
/// Adapter for no-clobber creation beneath a pinned directory. Both target
/// and destination authorization remain in the provider's normal policy.
pub fn create_access<'b>(&'b self, suffix: PathBuf, logical: PathBuf) -> CheckedCreate<'b, 'a> {
CheckedCreate {
target: self,
suffix,
logical,
}
}
}
impl FsAccessPolicy for CheckedTarget<'_> {
fn is_readable(&self, p: &Path) -> bool {
self.policy.is_readable(p)
}
fn is_writable(&self, p: &Path) -> bool {
self.policy.is_writable(p)
}
fn is_readable_paths(&self, p: &Path, r: &Path) -> bool {
p == self.logical && r == p && self.policy.is_readable_paths(p, r)
}
fn is_writable_paths(&self, p: &Path, r: &Path) -> bool {
p == self.logical && r == p && self.policy.is_writable_paths(p, r)
}
fn check_cancelled(&self) -> std::io::Result<()> {
self.policy.check_cancelled()
}
fn resolve_access_path(&self, p: &Path) -> std::io::Result<PathBuf> {
Ok(p.to_path_buf())
}
fn read_metadata(&self, _: &Path, _: &Path) -> std::io::Result<Metadata> {
self.file.metadata()
}
fn open_read_file(&self, _: &Path, _: &Path) -> std::io::Result<File> {
self.confirm()?;
let mut file = self.file.try_clone()?;
file.seek(SeekFrom::Start(0))?;
Ok(file)
}
fn max_write_bytes(&self) -> Option<usize> {
Some(
self.policy
.max_write_bytes()
.unwrap_or(8 * 1024 * 1024)
.min(8 * 1024 * 1024),
)
}
fn max_edit_replacements(&self) -> Option<usize> {
Some(
self.policy
.max_edit_replacements()
.unwrap_or(10_000)
.min(10_000),
)
}
fn atomic_write_file(
&self,
_: &Path,
_: &Path,
content: &[u8],
mode: AtomicWriteMode,
) -> std::io::Result<()> {
self.confirm()?;
let parent = self.parent.as_ref().ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::InvalidInput, "cannot save root")
})?;
crate::atomic_write_beneath_no_symlinks_at(
parent,
Path::new(self.relative.file_name().unwrap()),
content,
mode,
|| {
self.confirm()?;
// From here rename/encoding/post-observation failures have an unknown outcome.
self.effects_possible.store(true, Ordering::Release);
Ok(())
},
)
}
}
pub struct CheckedCreate<'b, 'a> {
target: &'b CheckedTarget<'a>,
suffix: PathBuf,
logical: PathBuf,
}
impl FsAccessPolicy for CheckedCreate<'_, '_> {
fn is_readable(&self, p: &Path) -> bool {
self.target.policy.is_readable(p)
}
fn is_writable(&self, p: &Path) -> bool {
p == self.logical && self.target.policy.is_writable_paths(p, p)
}
fn resolve_access_path(&self, p: &Path) -> std::io::Result<PathBuf> {
Ok(p.to_path_buf())
}
fn check_cancelled(&self) -> std::io::Result<()> {
self.target.policy.check_cancelled()
}
fn max_write_bytes(&self) -> Option<usize> {
self.target.max_write_bytes()
}
fn read_metadata(&self, _: &Path, _: &Path) -> std::io::Result<Metadata> {
match crate::open_beneath_no_symlinks_at(&self.target.file, &self.suffix) {
Ok(_) => Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"Create destination exists",
)),
Err(e) => Err(e),
}
}
fn atomic_write_file(
&self,
_: &Path,
_: &Path,
content: &[u8],
mode: AtomicWriteMode,
) -> std::io::Result<()> {
self.target.confirm()?;
if mode != AtomicWriteMode::CreateNew || !self.target.file.metadata()?.is_dir() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"Create requires directory",
));
}
// Creating parents must not turn a narrow destination grant into
// permission to populate ungranted intermediate directories.
let mut parent = self.target.logical.clone();
let mut components = self.suffix.components().peekable();
while let Some(component) = components.next() {
parent.push(component.as_os_str());
if components.peek().is_some()
&& !self.target.policy.is_writable_paths(&parent, &parent)
{
return Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"Create parent denied",
));
}
}
// Nested mkdir may have effects before failure; never auto-retry.
self.target.effects_possible.store(true, Ordering::Release);
crate::atomic_write_beneath_no_symlinks_at(
&self.target.file,
&self.suffix,
content,
mode,
|| self.target.policy.check_cancelled(),
)
}
}
#[cfg(all(test, target_os = "linux"))]
mod tests {
use super::*;
use crate::{FsError, FsPath, ReadRequest, WriteRequest};
use sha2::{Digest, Sha256};
use std::sync::atomic::AtomicUsize;
struct Access {
path: PathBuf,
counter: AtomicUsize,
trigger: usize,
replacement: bool,
}
impl FsAccessPolicy for Access {
fn is_readable(&self, _: &Path) -> bool {
true
}
fn is_writable(&self, _: &Path) -> bool {
true
}
fn check_cancelled(&self) -> std::io::Result<()> {
if self.counter.fetch_add(1, Ordering::SeqCst) + 1 == self.trigger {
if self.replacement {
std::fs::rename(&self.path, self.path.with_extension("old"))?;
std::fs::write(&self.path, "replacement")?;
} else {
std::fs::write(&self.path, "updated")?;
}
}
Ok(())
}
}
#[test]
fn checked_save_boundary_rejects_name_substitution_without_mutating_replacement() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("a");
std::fs::write(&path, "original").unwrap();
let policy = Access {
path: path.clone(),
counter: AtomicUsize::new(0),
trigger: 4,
replacement: true,
};
let handle = crate::open_root_no_symlinks(root.path()).unwrap();
let checked = CheckedTarget::pin(&handle, Path::new("a"), &path, &policy, true).unwrap();
policy.counter.store(0, Ordering::SeqCst);
let error = crate::run_write(
root.path(),
WriteRequest {
path: FsPath::new("a").unwrap(),
content: b"worker".to_vec(),
expected_hash: Some(Sha256::digest(b"original").into()),
},
&checked,
)
.unwrap_err();
assert!(
matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::WouldBlock)
);
assert!(!checked.effects_possible());
assert_eq!(std::fs::read_to_string(&path).unwrap(), "replacement");
assert_eq!(
std::fs::read_to_string(path.with_extension("old")).unwrap(),
"original"
);
}
#[test]
fn checked_read_midread_name_replacement_never_pairs_old_bytes_with_new_validator() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("a");
std::fs::write(&path, "original").unwrap();
let policy = Access {
path: path.clone(),
counter: AtomicUsize::new(0),
trigger: 3,
replacement: true,
};
let handle = crate::open_root_no_symlinks(root.path()).unwrap();
let checked = CheckedTarget::pin(&handle, Path::new("a"), &path, &policy, false).unwrap();
let validator = checked.validator().to_vec();
policy.counter.store(0, Ordering::SeqCst);
let result = crate::run_read(
root.path(),
ReadRequest {
path: FsPath::new("a").unwrap(),
offset: 0,
limit: 20,
max_bytes: 1024,
},
&checked,
);
match result {
Err(FsError::Conflict(_)) => {}
Ok(read) => {
// Same pinned inode supplies bytes and hash. Execute's final
// named-target check must reject, not refresh to replacement.
assert_eq!(read.bytes, b"original");
let expected_hash: crate::ContentHash = Sha256::digest(b"original").into();
assert_eq!(read.content_hash, expected_hash);
assert_eq!(
checked.confirm().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
}
other => panic!("unexpected Read result: {other:?}"),
}
assert_eq!(checked.validator(), validator);
assert_ne!(
identity_validator(&std::fs::metadata(&path).unwrap()).unwrap(),
validator
);
assert_eq!(std::fs::read_to_string(&path).unwrap(), "replacement");
}
#[test]
fn create_post_parent_observation_retains_bound_directory_and_rejects_substitution() {
let root = tempfile::tempdir().unwrap();
let directory = root.path().join("d");
std::fs::create_dir(&directory).unwrap();
let policy = Access {
path: directory.clone(),
counter: AtomicUsize::new(0),
trigger: usize::MAX,
replacement: false,
};
let handle = crate::open_root_no_symlinks(root.path()).unwrap();
let mut checked =
CheckedTarget::pin(&handle, Path::new("d"), &directory, &policy, true).unwrap();
let initial = checked.validator().to_vec();
let create = checked.create_access(PathBuf::from("new"), directory.join("new"));
crate::run_write(
root.path(),
WriteRequest {
path: FsPath::new("d/new").unwrap(),
content: b"new".to_vec(),
expected_hash: None,
},
&create,
)
.unwrap();
checked.refresh_directory_observation().unwrap();
assert_ne!(initial, checked.validator());
assert_eq!(
checked.validator(),
identity_validator(&std::fs::metadata(&directory).unwrap()).unwrap()
);
std::fs::rename(&directory, root.path().join("old")).unwrap();
std::fs::create_dir(&directory).unwrap();
assert_eq!(
checked.refresh_directory_observation().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
assert!(checked.effects_possible());
}
#[test]
fn checked_parent_swap_is_rejected_even_with_a_hardlinked_same_inode_target() {
let root = tempfile::tempdir().unwrap();
let directory = root.path().join("d");
std::fs::create_dir(&directory).unwrap();
let path = directory.join("a");
std::fs::write(&path, "original").unwrap();
let policy = Access {
path: path.clone(),
counter: AtomicUsize::new(0),
trigger: usize::MAX,
replacement: false,
};
let handle = crate::open_root_no_symlinks(root.path()).unwrap();
let checked = CheckedTarget::pin(&handle, Path::new("d/a"), &path, &policy, true).unwrap();
std::fs::rename(&directory, root.path().join("old")).unwrap();
std::fs::create_dir(&directory).unwrap();
std::fs::hard_link(root.path().join("old/a"), &path).unwrap();
assert_eq!(
checked.confirm().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
assert!(!checked.effects_possible());
}
#[test]
fn shared_text_read_rejects_mid_read_metadata_change() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("a");
std::fs::write(&path, "original").unwrap();
let policy = Access {
path,
counter: AtomicUsize::new(0),
trigger: 2,
replacement: false,
};
let error = crate::run_read(
root.path(),
ReadRequest {
path: FsPath::new("a").unwrap(),
offset: 0,
limit: 20,
max_bytes: 1024,
},
&policy,
)
.unwrap_err();
assert!(matches!(error, FsError::Conflict(_)));
}
}
+21 -26
View File
@@ -44,6 +44,10 @@ pub fn run_glob(
.as_ref()
.map(|traversal| traversal.path())
.unwrap_or(base);
let logical_walk_root = traversal
.as_ref()
.and_then(|t| t.logical_root())
.unwrap_or(root);
let mut walker = WalkBuilder::new(walker_root);
walker.hidden(false).follow_links(false);
if traversal.is_some() {
@@ -56,22 +60,15 @@ pub fn run_glob(
.git_exclude(false)
.ignore(false)
.parents(false);
let search_relative = base.strip_prefix(root).map_err(|_| {
FsError::InvalidArgument("glob base is outside its provider root".to_string())
})?;
let search_relative = search_relative.to_path_buf();
let filter_root = walker_root.to_path_buf();
walker.filter_entry(move |entry| {
entry
.path()
.strip_prefix(&filter_root)
.is_ok_and(|relative| {
relative.starts_with(&search_relative) || search_relative.starts_with(relative)
})
});
}
let mut visited = 0_usize;
for entry in walker.build().flatten() {
let walk = if traversal.is_some() {
crate::walk::Walk::descriptor(logical_walk_root, access)
} else {
crate::walk::Walk::Plain(walker.build())
};
for entry in walk {
let entry = entry?;
access.check_cancelled().map_err(|source| FsError::Io {
path: PathBuf::from(request.path.as_str()),
source,
@@ -83,17 +80,10 @@ pub fn run_glob(
crate::MAX_TRAVERSAL_ENTRIES
)));
}
let walked_path = entry.path();
let path = if traversal.is_some() {
let relative = walked_path.strip_prefix(walker_root).map_err(|_| {
FsError::InvalidArgument(
"descriptor traversal returned a path outside its root".to_string(),
)
})?;
root.join(relative)
} else {
walked_path.to_path_buf()
};
let path = entry.path;
if path.strip_prefix(root).unwrap_or(&path).to_str().is_none() {
continue;
}
let resolved = match access.resolve_access_path(&path) {
Ok(resolved) if access.is_readable_paths(&path, &resolved) => resolved,
_ => continue,
@@ -111,7 +101,12 @@ pub fn run_glob(
let logical = path.strip_prefix(root).map_err(|_| {
FsError::InvalidArgument("provider returned a path outside its root".to_string())
})?;
let result_path = FsPath::new(logical.to_string_lossy())?;
let Some(logical) = logical.to_str() else {
continue;
};
let Ok(result_path) = FsPath::new(logical) else {
continue;
};
let retained = result_path.as_str().len().saturating_add(16);
if retained_path_bytes.saturating_add(retained) > crate::MAX_RESULT_PATH_BYTES {
provider_truncated = true;
+30
View File
@@ -4,10 +4,15 @@
//! transport, or LLM Tool implementations. Paths are logical and root-relative;
//! providers supply host roots and access policy.
mod checked;
pub use checked::{CheckedTarget, identity_validator};
mod glob;
mod local;
mod operation;
mod search;
#[cfg(test)]
mod traversal_tests;
mod walk;
use std::path::{Path, PathBuf};
@@ -272,6 +277,7 @@ pub fn atomic_write_beneath_no_symlinks_at(
#[derive(Debug)]
pub struct FsTraversalRoot {
path: PathBuf,
logical_root: Option<PathBuf>,
_directory: std::fs::File,
}
@@ -279,20 +285,44 @@ impl FsTraversalRoot {
pub fn new(path: PathBuf, directory: std::fs::File) -> Self {
Self {
path,
logical_root: None,
_directory: directory,
}
}
/// Map walked paths to this logical anchor rather than the provider root.
pub fn new_at(path: PathBuf, directory: std::fs::File, logical_root: PathBuf) -> Self {
Self {
path,
logical_root: Some(logical_root),
_directory: directory,
}
}
pub fn logical_root(&self) -> Option<&Path> {
self.logical_root.as_deref()
}
pub fn path(&self) -> &Path {
&self.path
}
}
/// Owned predicate used before walker descent. Paths are provider-logical;
/// true directories require enumeration authority, other entries require read.
pub type FsTraversalFilter = std::sync::Arc<dyn Fn(&Path, bool) -> bool + Send + Sync>;
/// Provider-owned access policy used by local filesystem operations.
pub trait FsAccessPolicy: Send + Sync {
fn is_readable(&self, path: &Path) -> bool;
fn is_writable(&self, path: &Path) -> bool;
fn can_enumerate_directory(&self, logical: &Path, resolved: &Path) -> bool {
self.is_readable_paths(logical, resolved)
}
/// Owned because the shared walker requires a 'static entry predicate.
fn traversal_filter(&self) -> Option<FsTraversalFilter> {
None
}
/// Cooperatively stop provider-owned filesystem work. Implementations may
/// use this for per-operation cancellation and deadlines.
fn check_cancelled(&self) -> std::io::Result<()> {
+30 -1
View File
@@ -162,6 +162,19 @@ fn run_read_with_limits(
}
}
#[cfg(unix)]
if crate::identity_validator(&metadata).map_err(|e| map_io(&logical, e))?
!= crate::identity_validator(
&reader
.get_ref()
.metadata()
.map_err(|e| map_io(&logical, e))?,
)
.map_err(|e| map_io(&logical, e))?
{
return Err(FsError::Conflict(logical.as_str().to_string()));
}
let total_lines =
current_line.saturating_add(usize::from(last_byte.is_some_and(|byte| byte != b'\n')));
if request.offset > total_lines && request.offset != 0 {
@@ -473,6 +486,14 @@ pub fn run_list(
if !metadata.is_dir() {
return Err(FsError::NotDirectory(PathBuf::from(logical.as_str())));
}
if !access.can_enumerate_directory(&logical_base, &path) {
return Ok(ListResult {
entries: Vec::new(),
total_entries: 0,
total_bytes: 0,
truncated: false,
});
}
let mut entries = Vec::new();
let mut retained_path_bytes = 0_usize;
let mut provider_truncated = false;
@@ -490,6 +511,9 @@ pub fn run_list(
)));
}
let entry = entry.map_err(|error| map_io(&logical, error))?;
if entry.file_name().to_str().is_none() {
continue;
}
let logical_absolute = logical_base.join(entry.file_name());
let resolved = match access.resolve_access_path(&logical_absolute) {
Ok(resolved) => resolved,
@@ -513,7 +537,12 @@ pub fn run_list(
let relative = logical_absolute.strip_prefix(root).map_err(|_| {
FsError::InvalidArgument("provider returned a path outside its root".to_string())
})?;
let result_path = FsPath::new(relative.to_string_lossy())?;
let Some(relative) = relative.to_str() else {
continue;
};
let Ok(result_path) = FsPath::new(relative) else {
continue;
};
let retained = result_path.as_str().len().saturating_add(64);
if retained_path_bytes.saturating_add(retained) > crate::MAX_RESULT_PATH_BYTES {
provider_truncated = true;
+4
View File
@@ -353,6 +353,10 @@ pub struct GrepRequest {
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct GrepResult {
/// Exact typed provider paths represented in this returned report, not all
/// candidate/matched source files. Older providers may omit this field.
#[serde(default)]
pub paths: Vec<FsPath>,
/// Provider-rendered bounded grep report. Keeping rendering here avoids
/// transferring candidate files across a remote provider boundary.
pub output: String,
+65 -37
View File
@@ -79,6 +79,28 @@ fn reserve_report_bytes(report: &mut GrepReport, bytes: usize) -> bool {
impl GrepReport {
fn into_result(self, root: &Path) -> GrepResult {
// Build Object paths from retained provider identities, never by parsing
// rendered text. Offset, limit and truncation already selected this set.
let paths = match self.mode {
GrepOutputMode::FilesWithMatches => self
.files
.iter()
.filter_map(|path| logical_path(root, path))
.collect(),
GrepOutputMode::Count => self
.counts
.iter()
.filter_map(|(path, _)| logical_path(root, path))
.collect(),
GrepOutputMode::Content => self
.lines
.iter()
.map(|line| line.path.as_path())
.collect::<std::collections::BTreeSet<_>>()
.into_iter()
.filter_map(|path| logical_path(root, path))
.collect(),
};
let (match_count, matched_files) = match self.mode {
GrepOutputMode::FilesWithMatches => (self.files.len(), self.files.len()),
GrepOutputMode::Count => (
@@ -116,6 +138,7 @@ impl GrepReport {
}
}
GrepResult {
paths,
output,
match_count,
matched_files,
@@ -166,11 +189,15 @@ fn render_content_lines(root: &Path, lines: &[ContentLine], show_line_numbers: b
output
}
fn logical_path(root: &Path, path: &Path) -> Option<crate::FsPath> {
crate::FsPath::new_scoped(path.strip_prefix(root).unwrap_or(path).to_str()?).ok()
}
fn logical_display(root: &Path, path: &Path) -> String {
path.strip_prefix(root)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/")
logical_path(root, path)
.expect("only representable provider paths enter a report")
.as_str()
.to_string()
}
const DEFAULT_HEAD_LIMIT: usize = 250;
@@ -464,6 +491,9 @@ pub fn run_grep(
retained_bytes: 0,
truncated: false,
};
if head_limit == 0 {
return Ok(report.into_result(root));
}
let mut matching_files_seen = 0;
let mut matches_seen = 0;
@@ -474,6 +504,7 @@ pub fn run_grep(
&mut searcher,
&matcher,
&base,
root,
mode,
&mut report,
&mut matching_files_seen,
@@ -494,6 +525,10 @@ pub fn run_grep(
.as_ref()
.map(|traversal| traversal.path())
.unwrap_or(&base);
let logical_walk_root = traversal
.as_ref()
.and_then(|t| t.logical_root())
.unwrap_or(root);
let mut walker = WalkBuilder::new(walker_root);
if traversal.is_some() {
walker
@@ -506,19 +541,6 @@ pub fn run_grep(
.ignore(false)
.parents(false)
.follow_links(false);
let search_relative = base.strip_prefix(root).map_err(|_| {
FsError::InvalidArgument("grep base is outside its provider root".to_string())
})?;
let search_relative = search_relative.to_path_buf();
let filter_root = walker_root.to_path_buf();
walker.filter_entry(move |entry| {
entry
.path()
.strip_prefix(&filter_root)
.is_ok_and(|relative| {
relative.starts_with(&search_relative) || search_relative.starts_with(relative)
})
});
} else {
walker
.hidden(true)
@@ -563,7 +585,13 @@ pub fn run_grep(
)?;
}
}
for entry in walker.build().flatten() {
let walk = if traversal.is_some() {
crate::walk::Walk::descriptor(logical_walk_root, access)
} else {
crate::walk::Walk::Plain(walker.build())
};
for entry in walk {
let entry = entry?;
access
.check_cancelled()
.map_err(|error| FsError::io(&base, error))?;
@@ -574,18 +602,11 @@ pub fn run_grep(
crate::MAX_TRAVERSAL_ENTRIES
)));
}
let walked_path = entry.path();
let path = if traversal.is_some() {
let relative = walked_path.strip_prefix(walker_root).map_err(|_| {
FsError::InvalidArgument(
"descriptor traversal returned a path outside its root".to_string(),
)
})?;
root.join(relative)
} else {
walked_path.to_path_buf()
};
let file_type = entry.file_type();
let path = entry.path;
if logical_path(root, &path).is_none() {
continue;
}
let file_type = entry.kind;
if traversal.is_some() && file_type.as_ref().is_some_and(|kind| kind.is_dir()) {
descriptor_ignores.load_directory(&path, &base, &mut source_bytes_remaining, access)?;
continue;
@@ -599,10 +620,12 @@ pub fn run_grep(
"descriptor traversal returned a path outside its search base".to_string(),
)
})?;
if relative
.components()
.any(|component| component.as_os_str().to_string_lossy().starts_with('.'))
|| descriptor_ignores.is_ignored(root, &path)
if relative.components().any(|component| {
component
.as_os_str()
.to_str()
.is_some_and(|name| name.starts_with('.'))
}) || descriptor_ignores.is_ignored(root, &path)
|| !direct_file_selected(&path, overrides.as_ref(), types.as_ref())
{
continue;
@@ -618,6 +641,7 @@ pub fn run_grep(
&mut searcher,
&matcher,
&path,
root,
mode,
&mut report,
&mut matching_files_seen,
@@ -639,6 +663,7 @@ fn scan_path(
searcher: &mut Searcher,
matcher: &grep_regex::RegexMatcher,
path: &Path,
root: &Path,
mode: GrepOutputMode,
report: &mut GrepReport,
matching_files_seen: &mut usize,
@@ -648,6 +673,9 @@ fn scan_path(
source_bytes_remaining: &mut u64,
access: &dyn FsAccessPolicy,
) -> Result<bool, FsError> {
if logical_path(root, path).is_none() {
return Ok(false);
}
let resolved = access
.resolve_access_path(path)
.map_err(|error| FsError::io(path, error))?;
@@ -678,7 +706,7 @@ fn scan_path(
return Ok(false);
}
if *matching_files_seen >= offset {
if !reserve_report_bytes(report, path.to_string_lossy().len().saturating_add(1)) {
if !reserve_report_bytes(report, path.as_os_str().len().saturating_add(1)) {
return Ok(true);
}
report.files.push(path.to_path_buf());
@@ -695,7 +723,7 @@ fn scan_path(
return Ok(false);
}
if *matching_files_seen >= offset {
if !reserve_report_bytes(report, path.to_string_lossy().len().saturating_add(32)) {
if !reserve_report_bytes(report, path.as_os_str().len().saturating_add(32)) {
return Ok(true);
}
report.counts.push((path.to_path_buf(), count));
@@ -780,7 +808,7 @@ impl ContentSink<'_> {
fn reserve(&mut self, content_bytes: usize) -> bool {
let retained = self
.path
.to_string_lossy()
.as_os_str()
.len()
.saturating_add(content_bytes)
.saturating_add(64);
+108
View File
@@ -0,0 +1,108 @@
#![cfg(target_os = "linux")]
use crate::*;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
struct Filtered {
root: PathBuf,
seen: Arc<Mutex<Vec<PathBuf>>>,
}
impl FsAccessPolicy for Filtered {
fn is_readable(&self, path: &Path) -> bool {
path.starts_with(&self.root) && !path.starts_with(self.root.join("search/denied"))
}
fn is_writable(&self, _: &Path) -> bool {
false
}
fn open_traversal_root(
&self,
_: &Path,
resolved: &Path,
) -> std::io::Result<Option<FsTraversalRoot>> {
use std::os::fd::AsRawFd;
let file = std::fs::File::open(resolved)?;
let walk = PathBuf::from(format!("/proc/self/fd/{}", file.as_raw_fd()));
Ok(Some(FsTraversalRoot::new_at(
walk,
file,
resolved.to_path_buf(),
)))
}
fn traversal_filter(&self) -> Option<FsTraversalFilter> {
let denied = self.root.join("search/denied");
let seen = self.seen.clone();
Some(Arc::new(move |path, _| {
seen.lock().unwrap().push(path.to_path_buf());
!path.starts_with(&denied)
}))
}
fn open_read_file(&self, _: &Path, resolved: &Path) -> std::io::Result<std::fs::File> {
assert!(
!resolved.starts_with(self.root.join("search/denied")),
"denied source opened"
);
std::fs::File::open(resolved)
}
}
#[test]
fn descriptor_search_anchor_and_predicate_prune_before_denied_directory_descent() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("search/denied/deep")).unwrap();
std::fs::write(root.path().join("search/allowed.txt"), "needle allowed").unwrap();
std::fs::write(
root.path().join("search/denied/deep/secret.txt"),
"needle secret",
)
.unwrap();
std::fs::write(root.path().join("outside.txt"), "needle outside").unwrap();
let policy = Filtered {
root: root.path().to_path_buf(),
seen: Arc::new(Mutex::new(Vec::new())),
};
let glob = run_glob(
root.path(),
&root.path().join("search"),
GlobRequest {
path: FsPath::new("search").unwrap(),
pattern: "**/*.txt".into(),
limit: 100,
},
&policy,
)
.unwrap();
assert_eq!(glob.paths, vec![FsPath::new("search/allowed.txt").unwrap()]);
let grep = run_grep(
root.path(),
root.path().join("search"),
GrepRequest {
path: FsPath::new("search").unwrap(),
pattern: "needle".into(),
glob: None,
file_type: None,
case_insensitive: false,
before_context: 0,
after_context: 0,
multiline: false,
output_mode: GrepOutputMode::Content,
offset: 0,
limit: 100,
},
&policy,
)
.unwrap();
assert_eq!(grep.paths, vec![FsPath::new("search/allowed.txt").unwrap()]);
let seen = policy.seen.lock().unwrap();
assert!(
seen.iter()
.any(|path| path == &root.path().join("search/denied"))
);
assert!(
!seen
.iter()
.any(|path| path.starts_with(root.path().join("search/denied/deep")))
);
assert!(
!seen
.iter()
.any(|path| path == &root.path().join("outside.txt"))
);
}
+226
View File
@@ -0,0 +1,226 @@
//! Bounded descriptor-provider walk shared by Glob and Grep. Unlike a
//! WalkBuilder predicate, this iterator counts even rejected entries and checks
//! cancellation while scanning them. Denied directories are never enumerated.
use crate::{FsAccessPolicy, FsError, FsTraversalFilter};
use std::path::{Path, PathBuf};
pub(crate) struct Entry {
pub path: PathBuf,
pub kind: Option<std::fs::FileType>,
}
pub(crate) enum Walk<'a> {
Plain(ignore::Walk),
Descriptor(DescriptorWalk<'a>),
}
impl<'a> Walk<'a> {
pub fn descriptor(root: &Path, access: &'a dyn FsAccessPolicy) -> Self {
Self::Descriptor(DescriptorWalk {
access,
filter: access.traversal_filter(),
pending: Some((root.to_path_buf(), 0)),
root: Some(root.to_path_buf()),
stack: Vec::new(),
visited: 0,
failed: false,
})
}
}
impl Iterator for Walk<'_> {
type Item = Result<Entry, FsError>;
fn next(&mut self) -> Option<Self::Item> {
match self {
Self::Plain(walk) => loop {
match walk.next()? {
Ok(entry) => {
return Some(Ok(Entry {
path: entry.path().to_path_buf(),
kind: entry.file_type(),
}));
}
Err(_) => continue,
}
},
Self::Descriptor(walk) => walk.next(),
}
}
}
pub(crate) struct DescriptorWalk<'a> {
access: &'a dyn FsAccessPolicy,
filter: Option<FsTraversalFilter>,
root: Option<PathBuf>,
pending: Option<(PathBuf, usize)>,
stack: Vec<(PathBuf, usize, std::fs::ReadDir)>,
visited: usize,
failed: bool,
}
impl DescriptorWalk<'_> {
fn next_entry(&mut self) -> Result<Option<Entry>, FsError> {
if let Some(root) = self.root.take() {
self.access
.check_cancelled()
.map_err(|e| FsError::io(&root, e))?;
let resolved = self
.access
.resolve_access_path(&root)
.map_err(|e| FsError::io(&root, e))?;
let metadata = self
.access
.read_metadata(&root, &resolved)
.map_err(|e| FsError::io(&root, e))?;
return Ok(Some(Entry {
path: root,
kind: Some(metadata.file_type()),
}));
}
loop {
if let Some((path, depth)) = self.pending.take() {
self.access
.check_cancelled()
.map_err(|e| FsError::io(&path, e))?;
let selected = self
.filter
.as_ref()
.is_none_or(|filter| filter(&path, true));
let resolved = self
.access
.resolve_access_path(&path)
.map_err(|e| FsError::io(&path, e))?;
if selected && self.access.can_enumerate_directory(&path, &resolved) {
if depth >= 128 {
return Err(FsError::InvalidArgument(
"descriptor traversal depth exceeds provider limit 128".into(),
));
}
let entries = self
.access
.open_read_dir(&path, &resolved)
.map_err(|e| FsError::io(&path, e))?;
self.stack.push((path, depth, entries));
}
}
let Some((parent, depth, entries)) = self.stack.last_mut() else {
return Ok(None);
};
self.access
.check_cancelled()
.map_err(|e| FsError::io(&*parent, e))?;
let Some(entry) = entries.next() else {
self.stack.pop();
continue;
};
self.visited = self.visited.saturating_add(1);
if self.visited > crate::MAX_TRAVERSAL_ENTRIES {
return Err(FsError::InvalidArgument(
"descriptor traversal exceeds provider entry limit".into(),
));
}
let entry = entry.map_err(|e| FsError::io(&*parent, e))?;
if entry.file_name().to_str().is_none() {
continue;
}
let path = parent.join(entry.file_name());
let kind = entry.file_type().map_err(|e| FsError::io(&path, e))?;
let selected = self
.filter
.as_ref()
.is_none_or(|filter| filter(&path, kind.is_dir()));
if !selected {
continue;
}
let resolved = match self.access.resolve_access_path(&path) {
Ok(path) => path,
Err(_) => continue,
};
if !self.access.is_readable_paths(&path, &resolved) {
continue;
}
if kind.is_dir() {
self.pending = Some((path.clone(), depth.saturating_add(1)));
}
return Ok(Some(Entry {
path,
kind: Some(kind),
}));
}
}
}
impl Iterator for DescriptorWalk<'_> {
type Item = Result<Entry, FsError>;
fn next(&mut self) -> Option<Self::Item> {
if self.failed {
return None;
}
match self.next_entry() {
Ok(entry) => entry.map(Ok),
Err(error) => {
self.failed = true;
Some(Err(error))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
struct RejectFiles {
calls: AtomicUsize,
cancel_at: usize,
}
impl FsAccessPolicy for RejectFiles {
fn is_readable(&self, _: &Path) -> bool {
true
}
fn is_writable(&self, _: &Path) -> bool {
false
}
fn traversal_filter(&self) -> Option<FsTraversalFilter> {
Some(Arc::new(|_, directory| directory))
}
fn check_cancelled(&self) -> std::io::Result<()> {
if self.calls.fetch_add(1, Ordering::Relaxed) >= self.cancel_at {
Err(std::io::Error::from(std::io::ErrorKind::Interrupted))
} else {
Ok(())
}
}
}
#[test]
fn descriptor_walk_counts_rejected_entries_toward_the_entry_budget() {
let root = tempfile::tempdir().unwrap();
std::fs::write(root.path().join("rejected"), "must not be opened").unwrap();
let policy = RejectFiles {
calls: AtomicUsize::new(0),
cancel_at: usize::MAX,
};
let Walk::Descriptor(mut walk) = Walk::descriptor(root.path(), &policy) else {
panic!()
};
assert!(walk.next().unwrap().is_ok());
walk.visited = crate::MAX_TRAVERSAL_ENTRIES;
assert!(matches!(
walk.next().unwrap(),
Err(FsError::InvalidArgument(_))
));
}
#[test]
fn descriptor_walk_checks_cancellation_while_scanning_only_rejected_entries() {
let root = tempfile::tempdir().unwrap();
for index in 0..20 {
std::fs::write(root.path().join(index.to_string()), "must not be opened").unwrap();
}
let policy = RejectFiles {
calls: AtomicUsize::new(0),
cancel_at: 4,
};
let mut walk = Walk::descriptor(root.path(), &policy);
assert!(walk.next().unwrap().is_ok());
assert!(
matches!(walk.next().unwrap(), Err(FsError::Io { source, .. }) if source.kind() == std::io::ErrorKind::Interrupted)
);
}
}