fix(checkouts): preserve scoped descriptor ignore rules

T-699 rev20, MR 01a10d66-db02-70c2-b152-197aca92d6b7 seq3 native_glob_ignore_regression. Load authorized root/nested ignore sources through FsAccessPolicy descriptor opens, retain only bounded ancestor matchers, preserve source precedence/negation/anchors and prune ignored directories before enumeration. Expanded parity also exposed Grep source-class precedence and ignored-directory rescue errors; use the shared matcher without relaxing Grep's existing ignore-open/Git-marker provider contract. No ambient global/linked-worktree administration pointer traversal or unsafe WalkBuilder ignore loading.

Add 7 fs boundary tests, 4 actual Local/scoped provider parity tests and 1 native WIP search/link regression. Current corrected source passes cargo check --workspace --all-targets; cargo test -p fs-operation -p workdir -p tools -p worker (fs37 Workdir125 Tools72+10+29+1doc Worker780+109integration); cargo test -p worker-runtime -p yoi-workspace-server (Runtime279+17+17 Backend576+11); fmt/diff. Logs target/validation/T-699/rereview-{check,core,hosts}.log. Earlier intermediate parity/WIP assertion failures retained, not passing evidence. No target/ref integration, RO source changes, environment update or completion.
This commit is contained in:
2026-10-06 04:24:35 +09:00
parent b952e56bc2
commit 7a96c42130
10 changed files with 1196 additions and 206 deletions
@@ -0,0 +1,294 @@
//! Ignore rules loaded only through the provider's authorized descriptor opens.
//! Never use GitignoreBuilder::add/build_global: those reopen host pathnames.
use crate::{FsAccessPolicy, FsError};
use ignore::gitignore::{Gitignore, GitignoreBuilder};
use std::io::{BufRead, BufReader, Read};
use std::path::{Path, PathBuf};
pub(crate) const MAX_IGNORE_SOURCE_BYTES: u64 = 8 * 1024 * 1024;
const MAX_IGNORE_FILE_BYTES: u64 = 1024 * 1024;
const MAX_IGNORE_LINE_BYTES: u64 = 16 * 1024;
pub(crate) struct SourceBoundedReader<'a, R> {
pub inner: R,
pub remaining: &'a mut u64,
pub access: &'a dyn FsAccessPolicy,
}
impl<R: Read> Read for SourceBoundedReader<'_, R> {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
self.access.check_cancelled()?;
if buffer.is_empty() {
return Ok(0);
}
if *self.remaining == 0 {
let mut probe = [0_u8; 1];
return match self.inner.read(&mut probe)? {
0 => Ok(0),
_ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"search source exceeded provider byte limit",
)),
};
}
let limit = usize::try_from(*self.remaining)
.unwrap_or(usize::MAX)
.min(buffer.len());
let read = self.inner.read(&mut buffer[..limit])?;
*self.remaining = (*self.remaining).saturating_sub(read as u64);
Ok(read)
}
}
/// Shared Glob/Grep grammar and provider authorization. Glob, like WalkBuilder,
/// ignores unavailable configuration files; Grep retains its existing explicit
/// error on a rejected ignore-file open. Cancellation/bounds always fail closed.
pub(crate) fn add_checked_ignore_file(
builder: &mut GitignoreBuilder,
path: &Path,
remaining: &mut u64,
access: &dyn FsAccessPolicy,
skip_unavailable: bool,
) -> Result<bool, FsError> {
access.check_cancelled().map_err(|e| FsError::io(path, e))?;
let resolved = match access.resolve_access_path(path) {
Ok(path) => path,
Err(error) if skip_unavailable && error.kind() != std::io::ErrorKind::Interrupted => {
return Ok(false);
}
Err(error) => return Err(FsError::io(path, error)),
};
if !access.is_readable_paths(path, &resolved) {
return Ok(false);
}
let file = match access.open_read_file(path, &resolved) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(error) if skip_unavailable && error.kind() != std::io::ErrorKind::Interrupted => {
return Ok(false);
}
Err(error) => return Err(FsError::io(path, error)),
};
let metadata = file.metadata().map_err(|e| FsError::io(path, e))?;
if !metadata.is_file() {
return Ok(false);
}
if metadata.len() > MAX_IGNORE_FILE_BYTES {
return Err(FsError::InvalidArgument(
"ignore file exceeds provider limit 1 MiB".into(),
));
}
let bounded = SourceBoundedReader {
inner: file,
remaining,
access,
};
// Take also fences files that grow after metadata. Each line is bounded
// before allocation, and every underlying read checks cancellation.
let mut reader = BufReader::new(bounded.take(MAX_IGNORE_FILE_BYTES + 1));
let mut line = Vec::new();
let mut bytes = 0_u64;
let mut added = false;
loop {
line.clear();
// BufRead::read_line/read_until retry Interrupted internally. Explicit
// fill_buf prevents a cooperative cancellation from becoming a retry
// loop, and caps allocation before copying even a growing long line.
loop {
access.check_cancelled().map_err(|e| FsError::io(path, e))?;
let buffer = reader.fill_buf().map_err(|e| FsError::io(path, e))?;
if buffer.is_empty() {
break;
}
let count = buffer
.iter()
.position(|byte| *byte == b'\n')
.map_or(buffer.len(), |index| index + 1);
if line.len().saturating_add(count) as u64 > MAX_IGNORE_LINE_BYTES {
return Err(FsError::InvalidArgument(
"ignore line exceeds provider limit 16 KiB".into(),
));
}
let finished = buffer[count - 1] == b'\n';
line.extend_from_slice(&buffer[..count]);
reader.consume(count);
if finished {
break;
}
}
if line.is_empty() {
break;
}
bytes = bytes.saturating_add(line.len() as u64);
if bytes > MAX_IGNORE_FILE_BYTES {
return Err(FsError::InvalidArgument(
"ignore file exceeds provider limit 1 MiB".into(),
));
}
let line = std::str::from_utf8(&line)
.map_err(|_| FsError::InvalidArgument("ignore file must be UTF-8".into()))?
.trim_end_matches(['\r', '\n']);
// One invalid glob must not discard other valid rules.
if builder.add_line(Some(path.to_path_buf()), line).is_ok() {
added = true;
}
}
Ok(added)
}
struct DirectoryRules {
path: PathBuf,
ignore: Gitignore,
git_ignore: Gitignore,
exclude: Gitignore,
has_git: bool,
}
/// Request-local ancestor stack, not a file-tree cache. Only the current
/// directory's ancestry is retained; source bytes and depth bound its matchers.
pub(crate) struct DescriptorIgnores {
directories: Vec<DirectoryRules>,
remaining: u64,
require_git: bool,
skip_unavailable: bool,
}
impl DescriptorIgnores {
pub fn new(root: &Path, base: &Path, access: &dyn FsAccessPolicy) -> Result<Self, FsError> {
Self::with_options(root, base, access, true, true)
}
pub fn grep(root: &Path, base: &Path, access: &dyn FsAccessPolicy) -> Result<Self, FsError> {
// Preserve the existing descriptor Grep provider contract: gitignore
// sources apply without a .git marker and rejected opens fail closed.
Self::with_options(root, base, access, false, false)
}
fn with_options(
root: &Path,
base: &Path,
access: &dyn FsAccessPolicy,
require_git: bool,
skip_unavailable: bool,
) -> Result<Self, FsError> {
let relative = base.strip_prefix(root).map_err(|_| {
FsError::InvalidArgument("glob base is outside its provider root".into())
})?;
if relative.components().count() >= 128 {
return Err(FsError::InvalidArgument(
"ignore ancestry exceeds provider depth limit 128".into(),
));
}
let mut this = Self {
directories: Vec::new(),
remaining: MAX_IGNORE_SOURCE_BYTES,
require_git,
skip_unavailable,
};
let mut directory = root.to_path_buf();
for component in relative.components() {
this.load_directory(&directory, access)?;
directory.push(component);
}
Ok(this)
}
pub fn load_directory(
&mut self,
directory: &Path,
access: &dyn FsAccessPolicy,
) -> Result<(), FsError> {
self.directories
.retain(|rules| directory.starts_with(&rules.path));
if self
.directories
.last()
.is_some_and(|rules| rules.path == directory)
{
return Ok(());
}
if self.directories.len() >= 128 {
return Err(FsError::InvalidArgument(
"ignore ancestry exceeds provider depth limit 128".into(),
));
}
access
.check_cancelled()
.map_err(|e| FsError::io(directory, e))?;
let marker = directory.join(".git");
let git_kind = access
.resolve_access_path(&marker)
.ok()
.and_then(|resolved| {
if !access.is_readable_paths(&marker, &resolved) {
return None;
}
access
.read_metadata(&marker, &resolved)
.ok()
.map(|metadata| metadata.file_type())
});
let has_git = git_kind.is_some_and(|kind| kind.is_dir() || kind.is_file());
let mut load = |name: &str| -> Result<Gitignore, FsError> {
let mut builder = GitignoreBuilder::new(directory);
add_checked_ignore_file(
&mut builder,
&directory.join(name),
&mut self.remaining,
access,
self.skip_unavailable,
)?;
builder.build().map_err(|_| {
FsError::InvalidArgument("ignore patterns could not be compiled".into())
})
};
let ignore = load(".ignore")?;
let git_ignore = load(".gitignore")?;
// A linked-worktree .git file activates repository rules but is not a
// directory. Never chase its arbitrary host gitdir/commondir pointers.
let exclude = if git_kind.is_some_and(|kind| kind.is_dir()) {
load(".git/info/exclude")?
} else {
Gitignore::empty()
};
self.directories.push(DirectoryRules {
path: directory.to_path_buf(),
ignore,
git_ignore,
exclude,
has_git,
});
Ok(())
}
pub fn is_ignored(&mut self, path: &Path, is_dir: bool) -> bool {
self.directories
.retain(|rules| path.starts_with(&rules.path));
// .ignore has priority over all .gitignore/exclude rules. Within each
// class the nearest directory's last matching rule wins. Match entries,
// not any parents: ignored directories are pruned by the walker, while
// an explicitly selected ignored search root remains traversable.
for rules in self.directories.iter().rev() {
let matched = rules.ignore.matched(path, is_dir);
if !matched.is_none() {
return matched.is_ignore();
}
}
if self.require_git && !self.directories.iter().any(|rules| rules.has_git) {
return false;
}
let mut exclude = None;
for rules in self.directories.iter().rev() {
let matched = rules.git_ignore.matched(path, is_dir);
if !matched.is_none() {
return matched.is_ignore();
}
if exclude.is_none() {
let matched = rules.exclude.matched(path, is_dir);
if !matched.is_none() {
exclude = Some(matched.is_ignore());
}
}
if rules.has_git {
break;
}
}
exclude.unwrap_or(false)
}
}
+27 -16
View File
@@ -48,26 +48,24 @@ pub fn run_glob(
.as_ref()
.and_then(|t| t.logical_root())
.unwrap_or(root);
let mut walker = WalkBuilder::new(walker_root);
walker.hidden(false).follow_links(false);
if traversal.is_some() {
// Descriptor providers must never let WalkBuilder open pathname-based
// ignore files, which may be symlinked outside the pinned root. Glob's
// explicit pattern remains the complete selection contract.
walker
.git_ignore(false)
.git_global(false)
.git_exclude(false)
.ignore(false)
.parents(false);
}
let mut descriptor_ignores = traversal
.as_ref()
.map(|_| crate::descriptor_ignore::DescriptorIgnores::new(root, logical_walk_root, access))
.transpose()?;
let mut visited = 0_usize;
let walk = if traversal.is_some() {
let mut walk = if traversal.is_some() {
// Configuration is read via authorized provider opens, never by
// WalkBuilder's host-path ignore discovery.
crate::walk::Walk::descriptor(logical_walk_root, access)
} else {
crate::walk::Walk::Plain(walker.build())
crate::walk::Walk::Plain(
WalkBuilder::new(walker_root)
.hidden(false)
.follow_links(false)
.build(),
)
};
for entry in walk {
while let Some(entry) = walk.next() {
let entry = entry?;
access.check_cancelled().map_err(|source| FsError::Io {
path: PathBuf::from(request.path.as_str()),
@@ -88,6 +86,19 @@ pub fn run_glob(
Ok(resolved) if access.is_readable_paths(&path, &resolved) => resolved,
_ => continue,
};
if let Some(ignores) = descriptor_ignores.as_mut() {
let is_dir = entry.kind.is_some_and(|kind| kind.is_dir());
if path != logical_walk_root && ignores.is_ignored(&path, is_dir) {
if is_dir {
walk.skip_current_dir();
}
continue;
}
if is_dir {
ignores.load_directory(&path, access)?;
continue;
}
}
let Ok(file) = access.open_read_file(&path, &resolved) else {
continue;
};
+326
View File
@@ -0,0 +1,326 @@
#![cfg(target_os = "linux")]
use crate::*;
use std::path::{Path, PathBuf};
use std::sync::{
Mutex,
atomic::{AtomicUsize, Ordering},
};
struct Access {
root: PathBuf,
denied: Option<PathBuf>,
opened: Mutex<Vec<PathBuf>>,
enumerated: Mutex<Vec<PathBuf>>,
checks: AtomicUsize,
cancel_at: usize,
}
impl Access {
fn new(root: &Path) -> Self {
Self {
root: root.into(),
denied: None,
opened: Mutex::new(Vec::new()),
enumerated: Mutex::new(Vec::new()),
checks: AtomicUsize::new(0),
cancel_at: usize::MAX,
}
}
}
impl FsAccessPolicy for Access {
fn is_readable(&self, path: &Path) -> bool {
path.starts_with(&self.root) && self.denied.as_ref().is_none_or(|denied| path != denied)
}
fn is_writable(&self, _: &Path) -> bool {
false
}
fn resolve_access_path(&self, path: &Path) -> std::io::Result<PathBuf> {
Ok(path.into())
}
fn check_cancelled(&self) -> std::io::Result<()> {
if self.checks.fetch_add(1, Ordering::Relaxed) >= self.cancel_at {
Err(std::io::ErrorKind::Interrupted.into())
} else {
Ok(())
}
}
fn open_read_file(&self, logical: &Path, resolved: &Path) -> std::io::Result<std::fs::File> {
assert!(
self.is_readable(logical),
"denied ignore/source file opened"
);
self.opened.lock().unwrap().push(logical.into());
open_beneath_no_symlinks(&self.root, resolved)
}
fn read_metadata(&self, _: &Path, resolved: &Path) -> std::io::Result<std::fs::Metadata> {
open_beneath_no_symlinks(&self.root, resolved)?.metadata()
}
fn open_traversal_root(
&self,
_: &Path,
resolved: &Path,
) -> std::io::Result<Option<FsTraversalRoot>> {
use std::os::fd::AsRawFd;
let directory = open_beneath_no_symlinks(&self.root, resolved)?;
let walk = PathBuf::from(format!("/proc/self/fd/{}", directory.as_raw_fd()));
Ok(Some(FsTraversalRoot::new_at(
walk,
directory,
resolved.into(),
)))
}
fn open_read_dir(&self, logical: &Path, resolved: &Path) -> std::io::Result<std::fs::ReadDir> {
use std::os::fd::AsRawFd;
self.enumerated.lock().unwrap().push(logical.into());
let directory = open_beneath_no_symlinks(&self.root, resolved)?;
std::fs::read_dir(format!("/proc/self/fd/{}", directory.as_raw_fd()))
}
}
fn glob(root: &Path, access: &dyn FsAccessPolicy) -> Result<GlobResult, FsError> {
run_glob(
root,
root,
GlobRequest {
path: FsPath::root(),
pattern: "**/*.txt".into(),
limit: 100,
},
access,
)
}
#[test]
fn descriptor_glob_prunes_ignored_directories_before_enumeration_and_loading_rules() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("ignored/deep")).unwrap();
std::fs::write(root.path().join(".ignore"), "ignored/\n").unwrap();
// This rejected directory must never even attempt to load ignore files.
std::fs::File::create(root.path().join("ignored/.ignore"))
.unwrap()
.set_len(2 * 1024 * 1024)
.unwrap();
std::fs::write(root.path().join("ignored/deep/secret.txt"), "secret").unwrap();
std::fs::write(root.path().join("visible.txt"), "visible").unwrap();
let access = Access::new(root.path());
assert_eq!(
glob(root.path(), &access).unwrap().paths,
vec![FsPath::new("visible.txt").unwrap()]
);
assert_eq!(
*access.enumerated.lock().unwrap(),
vec![root.path().to_path_buf()]
);
assert!(
!access
.opened
.lock()
.unwrap()
.iter()
.any(|path| path.starts_with(root.path().join("ignored")))
);
}
#[test]
fn descriptor_glob_denied_and_symlinked_ignore_files_cannot_change_selection() {
let root = tempfile::tempdir().unwrap();
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("rules"), "visible.txt\n").unwrap();
std::fs::create_dir(root.path().join(".git")).unwrap();
std::os::unix::fs::symlink(outside.path().join("rules"), root.path().join(".gitignore"))
.unwrap();
std::fs::write(root.path().join(".ignore"), "visible.txt\n").unwrap();
std::fs::write(root.path().join("visible.txt"), "visible").unwrap();
let mut access = Access::new(root.path());
access.denied = Some(root.path().join(".ignore"));
assert_eq!(
glob(root.path(), &access).unwrap().paths,
vec![FsPath::new("visible.txt").unwrap()]
);
assert!(
!access
.opened
.lock()
.unwrap()
.contains(&root.path().join(".ignore"))
);
}
#[test]
fn descriptor_ignore_loading_checks_cancellation_without_retrying_interrupted() {
let root = tempfile::tempdir().unwrap();
std::fs::write(root.path().join(".ignore"), "ignored.txt\n".repeat(1000)).unwrap();
let mut access = Access::new(root.path());
// First check is before resolution, second at the first fill_buf.
access.cancel_at = 1;
let error = crate::descriptor_ignore::add_checked_ignore_file(
&mut ignore::gitignore::GitignoreBuilder::new(root.path()),
&root.path().join(".ignore"),
&mut 1024_000,
&access,
true,
)
.unwrap_err();
assert!(
matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::Interrupted)
);
assert_eq!(access.checks.load(Ordering::Relaxed), 2);
}
#[test]
fn descriptor_ignore_loading_bounds_file_line_and_aggregate_bytes() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join(".ignore");
let access = Access::new(root.path());
std::fs::File::create(&path)
.unwrap()
.set_len(1024 * 1024 + 1)
.unwrap();
assert!(matches!(
glob(root.path(), &access),
Err(FsError::InvalidArgument(_))
));
std::fs::write(&path, "x".repeat(16 * 1024 + 1)).unwrap();
assert!(matches!(
glob(root.path(), &access),
Err(FsError::InvalidArgument(_))
));
std::fs::write(&path, "ignored.txt\n").unwrap();
let error = crate::descriptor_ignore::add_checked_ignore_file(
&mut ignore::gitignore::GitignoreBuilder::new(root.path()),
&path,
&mut 3,
&access,
true,
)
.unwrap_err();
assert!(
matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::InvalidData)
);
}
#[test]
fn descriptor_ignore_ancestry_is_bounded_without_enumerating_a_tree() {
let root = tempfile::tempdir().unwrap();
let access = Access::new(root.path());
let base = (0..128).fold(root.path().to_path_buf(), |path, _| path.join("d"));
assert!(matches!(
crate::descriptor_ignore::DescriptorIgnores::new(root.path(), &base, &access),
Err(FsError::InvalidArgument(_))
));
assert!(access.opened.lock().unwrap().is_empty());
}
struct Plain(PathBuf);
impl FsAccessPolicy for Plain {
fn is_readable(&self, path: &Path) -> bool {
path.starts_with(&self.0)
}
fn is_writable(&self, _: &Path) -> bool {
false
}
}
#[test]
fn descriptor_glob_matches_plain_git_markers_exclude_and_nested_repository_boundaries() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("nested/.git/info")).unwrap();
std::fs::create_dir_all(root.path().join(".git/info")).unwrap();
std::fs::write(
root.path().join(".gitignore"),
"*.outer.txt\n!include.txt\n",
)
.unwrap();
std::fs::write(root.path().join(".ignore"), "always.txt\n").unwrap();
std::fs::write(
root.path().join(".git/info/exclude"),
"excluded.txt\ninclude.txt\n",
)
.unwrap();
std::fs::write(
root.path().join("nested/.git/info/exclude"),
"/excluded.txt\n",
)
.unwrap();
for name in [
"a.outer.txt",
"excluded.txt",
"include.txt",
"always.txt",
"nested/a.outer.txt",
"nested/excluded.txt",
"nested/always.txt",
] {
std::fs::write(root.path().join(name), "data").unwrap();
}
let native = glob(root.path(), &Access::new(root.path())).unwrap();
let ordinary = glob(root.path(), &Plain(root.path().into())).unwrap();
assert_eq!(native, ordinary);
assert_eq!(
native.paths,
vec![
FsPath::new("include.txt").unwrap(),
FsPath::new("nested/a.outer.txt").unwrap()
]
);
}
#[test]
fn descriptor_glob_requires_authorized_git_marker_for_gitignore_not_for_ignore() {
let root = tempfile::tempdir().unwrap();
std::fs::write(root.path().join(".gitignore"), "gitignored.txt\n").unwrap();
std::fs::write(root.path().join(".ignore"), "ignored.txt\n").unwrap();
for name in ["gitignored.txt", "ignored.txt", "visible.txt"] {
std::fs::write(root.path().join(name), "data").unwrap();
}
let native = glob(root.path(), &Access::new(root.path())).unwrap();
// Ambient ancestors may themselves be repositories (e.g. the harness's
// TMPDIR). Their markers/ignore files are outside this provider root and
// cannot activate Git rules or be inspected by descriptor search.
assert_eq!(native.paths.len(), 2);
std::fs::create_dir(root.path().join(".git")).unwrap();
let native = glob(root.path(), &Access::new(root.path())).unwrap();
assert_eq!(
native,
glob(root.path(), &Plain(root.path().into())).unwrap()
);
assert_eq!(native.paths, vec![FsPath::new("visible.txt").unwrap()]);
let mut denied = Access::new(root.path());
denied.denied = Some(root.path().join(".git"));
assert_eq!(glob(root.path(), &denied).unwrap().paths.len(), 2);
std::fs::remove_dir(root.path().join(".git")).unwrap();
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("exclude"), "visible.txt\n").unwrap();
std::fs::write(
root.path().join(".git"),
format!("gitdir: {}\n", outside.path().display()),
)
.unwrap();
// Real managed Workdirs use a .git file. Activate root .gitignore without
// failing on .git/info/exclude or following external administration paths.
let access = Access::new(root.path());
assert_eq!(
glob(root.path(), &access).unwrap().paths,
vec![FsPath::new("visible.txt").unwrap()]
);
std::fs::write(root.path().join("visible.txt"), "needle\n").unwrap();
let grep = run_grep(
root.path(),
root.path().into(),
GrepRequest {
path: FsPath::root(),
pattern: "needle".into(),
glob: None,
file_type: None,
case_insensitive: false,
before_context: 0,
after_context: 0,
multiline: false,
output_mode: GrepOutputMode::FilesWithMatches,
offset: 0,
limit: 100,
},
&access,
)
.unwrap();
assert_eq!(grep.paths, vec![FsPath::new("visible.txt").unwrap()]);
assert!(
!access
.opened
.lock()
.unwrap()
.iter()
.any(|path| path.starts_with(outside.path()))
);
}
+3
View File
@@ -6,7 +6,10 @@
mod checked;
pub use checked::{CheckedTarget, identity_validator};
mod descriptor_ignore;
mod glob;
#[cfg(test)]
mod ignore_tests;
mod local;
mod operation;
mod search;
+32 -189
View File
@@ -1,50 +1,20 @@
use std::collections::{BTreeMap, HashSet};
use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::io::{BufRead, BufReader, Read};
use std::io::Read;
use std::path::{Path, PathBuf};
use crate::descriptor_ignore::{DescriptorIgnores, SourceBoundedReader};
use crate::FsAccessPolicy;
use grep_regex::RegexMatcherBuilder;
use grep_searcher::sinks::UTF8 as UTF8Sink;
use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkContext, SinkMatch};
use ignore::WalkBuilder;
use ignore::gitignore::{Gitignore, GitignoreBuilder};
use ignore::overrides::{Override, OverrideBuilder};
use ignore::types::{Types, TypesBuilder};
use crate::{FsError, GrepOutputMode, GrepRequest, GrepResult, direct_symlink};
struct SourceBoundedReader<'a, R> {
inner: R,
remaining: &'a mut u64,
access: &'a dyn FsAccessPolicy,
}
impl<R: Read> Read for SourceBoundedReader<'_, R> {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
self.access.check_cancelled()?;
if buffer.is_empty() {
return Ok(0);
}
if *self.remaining == 0 {
let mut probe = [0_u8; 1];
return match self.inner.read(&mut probe)? {
0 => Ok(0),
_ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"grep source exceeded provider byte limit",
)),
};
}
let limit = usize::try_from(*self.remaining)
.unwrap_or(usize::MAX)
.min(buffer.len());
let read = self.inner.read(&mut buffer[..limit])?;
*self.remaining = (*self.remaining).saturating_sub(read as u64);
Ok(read)
}
}
struct ContentLine {
path: PathBuf,
line_number: Option<u64>,
@@ -234,119 +204,6 @@ fn direct_file_selected(path: &Path, overrides: Option<&Override>, types: Option
&& !types.is_some_and(|filter| filter.matched(path, false).is_ignore())
}
#[derive(Default)]
struct DescriptorIgnoreMatchers {
loaded_directories: HashSet<PathBuf>,
by_directory: BTreeMap<PathBuf, Gitignore>,
}
impl DescriptorIgnoreMatchers {
fn load_directory(
&mut self,
directory: &Path,
explicit_base: &Path,
source_bytes_remaining: &mut u64,
access: &dyn FsAccessPolicy,
) -> Result<(), FsError> {
if !self.loaded_directories.insert(directory.to_path_buf()) {
return Ok(());
}
let mut builder = GitignoreBuilder::new(directory);
let mut has_patterns = false;
for name in [".gitignore", ".ignore"] {
let path = directory.join(name);
let resolved = access
.resolve_access_path(&path)
.map_err(|error| FsError::io(&path, error))?;
if !access.is_readable_paths(&path, &resolved) {
continue;
}
let file = match access.open_read_file(&path, &resolved) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(FsError::io(&path, error)),
};
if !file
.metadata()
.map_err(|error| FsError::io(&path, error))?
.is_file()
{
continue;
}
let bounded = SourceBoundedReader {
inner: file,
remaining: source_bytes_remaining,
access,
};
let mut reader = BufReader::new(bounded);
let mut line = String::new();
let mut line_number = 0_u64;
loop {
line.clear();
let read = reader
.read_line(&mut line)
.map_err(|error| FsError::io(&path, error))?;
if read == 0 {
break;
}
line_number = line_number.saturating_add(1);
if line.len() > 1024 * 1024 {
return Err(FsError::InvalidArgument(format!(
"ignore pattern line {line_number} exceeds provider limit"
)));
}
let line = line.trim_end_matches(['\r', '\n']);
// Match the ignore crate's partial-error behavior: one invalid
// pattern does not discard the remaining valid lines.
if builder.add_line(Some(path.clone()), line).is_ok() {
has_patterns = true;
}
}
}
if !has_patterns {
return Ok(());
}
let matcher = builder.build().map_err(|_| {
FsError::InvalidArgument("ignore patterns could not be compiled".to_string())
})?;
// An explicitly selected search root is traversed even when a strict
// ancestor ignores that directory, matching path-backed Grep behavior.
if directory != explicit_base
&& explicit_base.starts_with(directory)
&& matcher
.matched_path_or_any_parents(explicit_base, true)
.is_ignore()
{
return Ok(());
}
self.by_directory.insert(directory.to_path_buf(), matcher);
Ok(())
}
fn is_ignored(&self, root: &Path, path: &Path) -> bool {
let mut directories = path
.parent()
.into_iter()
.flat_map(Path::ancestors)
.take_while(|directory| directory.starts_with(root))
.collect::<Vec<_>>();
directories.reverse();
let mut ignored = false;
for directory in directories {
let Some(matcher) = self.by_directory.get(directory) else {
continue;
};
let matched = matcher.matched_path_or_any_parents(path, false);
if matched.is_ignore() {
ignored = true;
} else if matched.is_whitelist() {
ignored = false;
}
}
ignored
}
}
struct GrepParams {
pattern: String,
path: Option<PathBuf>,
@@ -560,37 +417,16 @@ pub fn run_grep(
let mut visited = 0_usize;
let mut source_bytes_remaining = crate::MAX_GREP_SOURCE_BYTES;
let mut descriptor_ignores = DescriptorIgnoreMatchers::default();
if traversal.is_some() {
let mut directory = root.to_path_buf();
descriptor_ignores.load_directory(
&directory,
&base,
&mut source_bytes_remaining,
access,
)?;
for component in base
.strip_prefix(root)
.map_err(|_| {
FsError::InvalidArgument("grep base is outside its provider root".to_string())
})?
.components()
{
directory.push(component);
descriptor_ignores.load_directory(
&directory,
&base,
&mut source_bytes_remaining,
access,
)?;
}
}
let walk = if traversal.is_some() {
let mut descriptor_ignores = traversal
.as_ref()
.map(|_| DescriptorIgnores::grep(root, logical_walk_root, access))
.transpose()?;
let mut walk = if traversal.is_some() {
crate::walk::Walk::descriptor(logical_walk_root, access)
} else {
crate::walk::Walk::Plain(walker.build())
};
for entry in walk {
while let Some(entry) = walk.next() {
let entry = entry?;
access
.check_cancelled()
@@ -607,29 +443,36 @@ pub fn run_grep(
continue;
}
let file_type = entry.kind;
if traversal.is_some() && file_type.as_ref().is_some_and(|kind| kind.is_dir()) {
descriptor_ignores.load_directory(&path, &base, &mut source_bytes_remaining, access)?;
continue;
}
if !file_type.map(|kind| kind.is_file()).unwrap_or(false) {
continue;
}
if traversal.is_some() {
if let Some(ignores) = descriptor_ignores.as_mut() {
let relative = path.strip_prefix(&base).map_err(|_| {
FsError::InvalidArgument(
"descriptor traversal returned a path outside its search base".to_string(),
)
})?;
if relative.components().any(|component| {
component
.as_os_str()
.to_str()
.is_some_and(|name| name.starts_with('.'))
}) || descriptor_ignores.is_ignored(root, &path)
|| !direct_file_selected(&path, overrides.as_ref(), types.as_ref())
let is_dir = file_type.is_some_and(|kind| kind.is_dir());
if path != logical_walk_root
&& (relative.components().any(|component| {
component
.as_os_str()
.to_str()
.is_some_and(|name| name.starts_with('.'))
}) || ignores.is_ignored(&path, is_dir))
{
if is_dir {
walk.skip_current_dir();
}
continue;
}
if is_dir {
ignores.load_directory(&path, access)?;
continue;
}
if !direct_file_selected(&path, overrides.as_ref(), types.as_ref()) {
continue;
}
}
if !file_type.map(|kind| kind.is_file()).unwrap_or(false) {
continue;
}
let readable = access
.resolve_access_path(&path)
+7
View File
@@ -13,6 +13,13 @@ pub(crate) enum Walk<'a> {
Descriptor(DescriptorWalk<'a>),
}
impl<'a> Walk<'a> {
/// A rejected directory has been yielded but not enumerated yet. Keep
/// ignore pruning inside the same bounded descriptor walk as scope pruning.
pub fn skip_current_dir(&mut self) {
if let Self::Descriptor(walk) = self {
walk.pending = None;
}
}
pub fn descriptor(root: &Path, access: &'a dyn FsAccessPolicy) -> Self {
Self::Descriptor(DescriptorWalk {
access,