fix(checkouts): preserve scoped descriptor ignore rules
T-699 rev20, MR 01a10d66-db02-70c2-b152-197aca92d6b7 seq3 native_glob_ignore_regression. Load authorized root/nested ignore sources through FsAccessPolicy descriptor opens, retain only bounded ancestor matchers, preserve source precedence/negation/anchors and prune ignored directories before enumeration. Expanded parity also exposed Grep source-class precedence and ignored-directory rescue errors; use the shared matcher without relaxing Grep's existing ignore-open/Git-marker provider contract. No ambient global/linked-worktree administration pointer traversal or unsafe WalkBuilder ignore loading.
Add 7 fs boundary tests, 4 actual Local/scoped provider parity tests and 1 native WIP search/link regression. Current corrected source passes cargo check --workspace --all-targets; cargo test -p fs-operation -p workdir -p tools -p worker (fs37 Workdir125 Tools72+10+29+1doc Worker780+109integration); cargo test -p worker-runtime -p yoi-workspace-server (Runtime279+17+17 Backend576+11); fmt/diff. Logs target/validation/T-699/rereview-{check,core,hosts}.log. Earlier intermediate parity/WIP assertion failures retained, not passing evidence. No target/ref integration, RO source changes, environment update or completion.
This commit is contained in:
@@ -0,0 +1,294 @@
|
||||
//! Ignore rules loaded only through the provider's authorized descriptor opens.
|
||||
//! Never use GitignoreBuilder::add/build_global: those reopen host pathnames.
|
||||
use crate::{FsAccessPolicy, FsError};
|
||||
use ignore::gitignore::{Gitignore, GitignoreBuilder};
|
||||
use std::io::{BufRead, BufReader, Read};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub(crate) const MAX_IGNORE_SOURCE_BYTES: u64 = 8 * 1024 * 1024;
|
||||
const MAX_IGNORE_FILE_BYTES: u64 = 1024 * 1024;
|
||||
const MAX_IGNORE_LINE_BYTES: u64 = 16 * 1024;
|
||||
|
||||
pub(crate) struct SourceBoundedReader<'a, R> {
|
||||
pub inner: R,
|
||||
pub remaining: &'a mut u64,
|
||||
pub access: &'a dyn FsAccessPolicy,
|
||||
}
|
||||
impl<R: Read> Read for SourceBoundedReader<'_, R> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
self.access.check_cancelled()?;
|
||||
if buffer.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
if *self.remaining == 0 {
|
||||
let mut probe = [0_u8; 1];
|
||||
return match self.inner.read(&mut probe)? {
|
||||
0 => Ok(0),
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"search source exceeded provider byte limit",
|
||||
)),
|
||||
};
|
||||
}
|
||||
let limit = usize::try_from(*self.remaining)
|
||||
.unwrap_or(usize::MAX)
|
||||
.min(buffer.len());
|
||||
let read = self.inner.read(&mut buffer[..limit])?;
|
||||
*self.remaining = (*self.remaining).saturating_sub(read as u64);
|
||||
Ok(read)
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared Glob/Grep grammar and provider authorization. Glob, like WalkBuilder,
|
||||
/// ignores unavailable configuration files; Grep retains its existing explicit
|
||||
/// error on a rejected ignore-file open. Cancellation/bounds always fail closed.
|
||||
pub(crate) fn add_checked_ignore_file(
|
||||
builder: &mut GitignoreBuilder,
|
||||
path: &Path,
|
||||
remaining: &mut u64,
|
||||
access: &dyn FsAccessPolicy,
|
||||
skip_unavailable: bool,
|
||||
) -> Result<bool, FsError> {
|
||||
access.check_cancelled().map_err(|e| FsError::io(path, e))?;
|
||||
let resolved = match access.resolve_access_path(path) {
|
||||
Ok(path) => path,
|
||||
Err(error) if skip_unavailable && error.kind() != std::io::ErrorKind::Interrupted => {
|
||||
return Ok(false);
|
||||
}
|
||||
Err(error) => return Err(FsError::io(path, error)),
|
||||
};
|
||||
if !access.is_readable_paths(path, &resolved) {
|
||||
return Ok(false);
|
||||
}
|
||||
let file = match access.open_read_file(path, &resolved) {
|
||||
Ok(file) => file,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) if skip_unavailable && error.kind() != std::io::ErrorKind::Interrupted => {
|
||||
return Ok(false);
|
||||
}
|
||||
Err(error) => return Err(FsError::io(path, error)),
|
||||
};
|
||||
let metadata = file.metadata().map_err(|e| FsError::io(path, e))?;
|
||||
if !metadata.is_file() {
|
||||
return Ok(false);
|
||||
}
|
||||
if metadata.len() > MAX_IGNORE_FILE_BYTES {
|
||||
return Err(FsError::InvalidArgument(
|
||||
"ignore file exceeds provider limit 1 MiB".into(),
|
||||
));
|
||||
}
|
||||
let bounded = SourceBoundedReader {
|
||||
inner: file,
|
||||
remaining,
|
||||
access,
|
||||
};
|
||||
// Take also fences files that grow after metadata. Each line is bounded
|
||||
// before allocation, and every underlying read checks cancellation.
|
||||
let mut reader = BufReader::new(bounded.take(MAX_IGNORE_FILE_BYTES + 1));
|
||||
let mut line = Vec::new();
|
||||
let mut bytes = 0_u64;
|
||||
let mut added = false;
|
||||
loop {
|
||||
line.clear();
|
||||
// BufRead::read_line/read_until retry Interrupted internally. Explicit
|
||||
// fill_buf prevents a cooperative cancellation from becoming a retry
|
||||
// loop, and caps allocation before copying even a growing long line.
|
||||
loop {
|
||||
access.check_cancelled().map_err(|e| FsError::io(path, e))?;
|
||||
let buffer = reader.fill_buf().map_err(|e| FsError::io(path, e))?;
|
||||
if buffer.is_empty() {
|
||||
break;
|
||||
}
|
||||
let count = buffer
|
||||
.iter()
|
||||
.position(|byte| *byte == b'\n')
|
||||
.map_or(buffer.len(), |index| index + 1);
|
||||
if line.len().saturating_add(count) as u64 > MAX_IGNORE_LINE_BYTES {
|
||||
return Err(FsError::InvalidArgument(
|
||||
"ignore line exceeds provider limit 16 KiB".into(),
|
||||
));
|
||||
}
|
||||
let finished = buffer[count - 1] == b'\n';
|
||||
line.extend_from_slice(&buffer[..count]);
|
||||
reader.consume(count);
|
||||
if finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if line.is_empty() {
|
||||
break;
|
||||
}
|
||||
bytes = bytes.saturating_add(line.len() as u64);
|
||||
if bytes > MAX_IGNORE_FILE_BYTES {
|
||||
return Err(FsError::InvalidArgument(
|
||||
"ignore file exceeds provider limit 1 MiB".into(),
|
||||
));
|
||||
}
|
||||
let line = std::str::from_utf8(&line)
|
||||
.map_err(|_| FsError::InvalidArgument("ignore file must be UTF-8".into()))?
|
||||
.trim_end_matches(['\r', '\n']);
|
||||
// One invalid glob must not discard other valid rules.
|
||||
if builder.add_line(Some(path.to_path_buf()), line).is_ok() {
|
||||
added = true;
|
||||
}
|
||||
}
|
||||
Ok(added)
|
||||
}
|
||||
|
||||
struct DirectoryRules {
|
||||
path: PathBuf,
|
||||
ignore: Gitignore,
|
||||
git_ignore: Gitignore,
|
||||
exclude: Gitignore,
|
||||
has_git: bool,
|
||||
}
|
||||
|
||||
/// Request-local ancestor stack, not a file-tree cache. Only the current
|
||||
/// directory's ancestry is retained; source bytes and depth bound its matchers.
|
||||
pub(crate) struct DescriptorIgnores {
|
||||
directories: Vec<DirectoryRules>,
|
||||
remaining: u64,
|
||||
require_git: bool,
|
||||
skip_unavailable: bool,
|
||||
}
|
||||
impl DescriptorIgnores {
|
||||
pub fn new(root: &Path, base: &Path, access: &dyn FsAccessPolicy) -> Result<Self, FsError> {
|
||||
Self::with_options(root, base, access, true, true)
|
||||
}
|
||||
pub fn grep(root: &Path, base: &Path, access: &dyn FsAccessPolicy) -> Result<Self, FsError> {
|
||||
// Preserve the existing descriptor Grep provider contract: gitignore
|
||||
// sources apply without a .git marker and rejected opens fail closed.
|
||||
Self::with_options(root, base, access, false, false)
|
||||
}
|
||||
fn with_options(
|
||||
root: &Path,
|
||||
base: &Path,
|
||||
access: &dyn FsAccessPolicy,
|
||||
require_git: bool,
|
||||
skip_unavailable: bool,
|
||||
) -> Result<Self, FsError> {
|
||||
let relative = base.strip_prefix(root).map_err(|_| {
|
||||
FsError::InvalidArgument("glob base is outside its provider root".into())
|
||||
})?;
|
||||
if relative.components().count() >= 128 {
|
||||
return Err(FsError::InvalidArgument(
|
||||
"ignore ancestry exceeds provider depth limit 128".into(),
|
||||
));
|
||||
}
|
||||
let mut this = Self {
|
||||
directories: Vec::new(),
|
||||
remaining: MAX_IGNORE_SOURCE_BYTES,
|
||||
require_git,
|
||||
skip_unavailable,
|
||||
};
|
||||
let mut directory = root.to_path_buf();
|
||||
for component in relative.components() {
|
||||
this.load_directory(&directory, access)?;
|
||||
directory.push(component);
|
||||
}
|
||||
Ok(this)
|
||||
}
|
||||
|
||||
pub fn load_directory(
|
||||
&mut self,
|
||||
directory: &Path,
|
||||
access: &dyn FsAccessPolicy,
|
||||
) -> Result<(), FsError> {
|
||||
self.directories
|
||||
.retain(|rules| directory.starts_with(&rules.path));
|
||||
if self
|
||||
.directories
|
||||
.last()
|
||||
.is_some_and(|rules| rules.path == directory)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if self.directories.len() >= 128 {
|
||||
return Err(FsError::InvalidArgument(
|
||||
"ignore ancestry exceeds provider depth limit 128".into(),
|
||||
));
|
||||
}
|
||||
access
|
||||
.check_cancelled()
|
||||
.map_err(|e| FsError::io(directory, e))?;
|
||||
let marker = directory.join(".git");
|
||||
let git_kind = access
|
||||
.resolve_access_path(&marker)
|
||||
.ok()
|
||||
.and_then(|resolved| {
|
||||
if !access.is_readable_paths(&marker, &resolved) {
|
||||
return None;
|
||||
}
|
||||
access
|
||||
.read_metadata(&marker, &resolved)
|
||||
.ok()
|
||||
.map(|metadata| metadata.file_type())
|
||||
});
|
||||
let has_git = git_kind.is_some_and(|kind| kind.is_dir() || kind.is_file());
|
||||
let mut load = |name: &str| -> Result<Gitignore, FsError> {
|
||||
let mut builder = GitignoreBuilder::new(directory);
|
||||
add_checked_ignore_file(
|
||||
&mut builder,
|
||||
&directory.join(name),
|
||||
&mut self.remaining,
|
||||
access,
|
||||
self.skip_unavailable,
|
||||
)?;
|
||||
builder.build().map_err(|_| {
|
||||
FsError::InvalidArgument("ignore patterns could not be compiled".into())
|
||||
})
|
||||
};
|
||||
let ignore = load(".ignore")?;
|
||||
let git_ignore = load(".gitignore")?;
|
||||
// A linked-worktree .git file activates repository rules but is not a
|
||||
// directory. Never chase its arbitrary host gitdir/commondir pointers.
|
||||
let exclude = if git_kind.is_some_and(|kind| kind.is_dir()) {
|
||||
load(".git/info/exclude")?
|
||||
} else {
|
||||
Gitignore::empty()
|
||||
};
|
||||
self.directories.push(DirectoryRules {
|
||||
path: directory.to_path_buf(),
|
||||
ignore,
|
||||
git_ignore,
|
||||
exclude,
|
||||
has_git,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_ignored(&mut self, path: &Path, is_dir: bool) -> bool {
|
||||
self.directories
|
||||
.retain(|rules| path.starts_with(&rules.path));
|
||||
// .ignore has priority over all .gitignore/exclude rules. Within each
|
||||
// class the nearest directory's last matching rule wins. Match entries,
|
||||
// not any parents: ignored directories are pruned by the walker, while
|
||||
// an explicitly selected ignored search root remains traversable.
|
||||
for rules in self.directories.iter().rev() {
|
||||
let matched = rules.ignore.matched(path, is_dir);
|
||||
if !matched.is_none() {
|
||||
return matched.is_ignore();
|
||||
}
|
||||
}
|
||||
if self.require_git && !self.directories.iter().any(|rules| rules.has_git) {
|
||||
return false;
|
||||
}
|
||||
let mut exclude = None;
|
||||
for rules in self.directories.iter().rev() {
|
||||
let matched = rules.git_ignore.matched(path, is_dir);
|
||||
if !matched.is_none() {
|
||||
return matched.is_ignore();
|
||||
}
|
||||
if exclude.is_none() {
|
||||
let matched = rules.exclude.matched(path, is_dir);
|
||||
if !matched.is_none() {
|
||||
exclude = Some(matched.is_ignore());
|
||||
}
|
||||
}
|
||||
if rules.has_git {
|
||||
break;
|
||||
}
|
||||
}
|
||||
exclude.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
@@ -48,26 +48,24 @@ pub fn run_glob(
|
||||
.as_ref()
|
||||
.and_then(|t| t.logical_root())
|
||||
.unwrap_or(root);
|
||||
let mut walker = WalkBuilder::new(walker_root);
|
||||
walker.hidden(false).follow_links(false);
|
||||
if traversal.is_some() {
|
||||
// Descriptor providers must never let WalkBuilder open pathname-based
|
||||
// ignore files, which may be symlinked outside the pinned root. Glob's
|
||||
// explicit pattern remains the complete selection contract.
|
||||
walker
|
||||
.git_ignore(false)
|
||||
.git_global(false)
|
||||
.git_exclude(false)
|
||||
.ignore(false)
|
||||
.parents(false);
|
||||
}
|
||||
let mut descriptor_ignores = traversal
|
||||
.as_ref()
|
||||
.map(|_| crate::descriptor_ignore::DescriptorIgnores::new(root, logical_walk_root, access))
|
||||
.transpose()?;
|
||||
let mut visited = 0_usize;
|
||||
let walk = if traversal.is_some() {
|
||||
let mut walk = if traversal.is_some() {
|
||||
// Configuration is read via authorized provider opens, never by
|
||||
// WalkBuilder's host-path ignore discovery.
|
||||
crate::walk::Walk::descriptor(logical_walk_root, access)
|
||||
} else {
|
||||
crate::walk::Walk::Plain(walker.build())
|
||||
crate::walk::Walk::Plain(
|
||||
WalkBuilder::new(walker_root)
|
||||
.hidden(false)
|
||||
.follow_links(false)
|
||||
.build(),
|
||||
)
|
||||
};
|
||||
for entry in walk {
|
||||
while let Some(entry) = walk.next() {
|
||||
let entry = entry?;
|
||||
access.check_cancelled().map_err(|source| FsError::Io {
|
||||
path: PathBuf::from(request.path.as_str()),
|
||||
@@ -88,6 +86,19 @@ pub fn run_glob(
|
||||
Ok(resolved) if access.is_readable_paths(&path, &resolved) => resolved,
|
||||
_ => continue,
|
||||
};
|
||||
if let Some(ignores) = descriptor_ignores.as_mut() {
|
||||
let is_dir = entry.kind.is_some_and(|kind| kind.is_dir());
|
||||
if path != logical_walk_root && ignores.is_ignored(&path, is_dir) {
|
||||
if is_dir {
|
||||
walk.skip_current_dir();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_dir {
|
||||
ignores.load_directory(&path, access)?;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let Ok(file) = access.open_read_file(&path, &resolved) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
#![cfg(target_os = "linux")]
|
||||
use crate::*;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{
|
||||
Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
struct Access {
|
||||
root: PathBuf,
|
||||
denied: Option<PathBuf>,
|
||||
opened: Mutex<Vec<PathBuf>>,
|
||||
enumerated: Mutex<Vec<PathBuf>>,
|
||||
checks: AtomicUsize,
|
||||
cancel_at: usize,
|
||||
}
|
||||
impl Access {
|
||||
fn new(root: &Path) -> Self {
|
||||
Self {
|
||||
root: root.into(),
|
||||
denied: None,
|
||||
opened: Mutex::new(Vec::new()),
|
||||
enumerated: Mutex::new(Vec::new()),
|
||||
checks: AtomicUsize::new(0),
|
||||
cancel_at: usize::MAX,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl FsAccessPolicy for Access {
|
||||
fn is_readable(&self, path: &Path) -> bool {
|
||||
path.starts_with(&self.root) && self.denied.as_ref().is_none_or(|denied| path != denied)
|
||||
}
|
||||
fn is_writable(&self, _: &Path) -> bool {
|
||||
false
|
||||
}
|
||||
fn resolve_access_path(&self, path: &Path) -> std::io::Result<PathBuf> {
|
||||
Ok(path.into())
|
||||
}
|
||||
fn check_cancelled(&self) -> std::io::Result<()> {
|
||||
if self.checks.fetch_add(1, Ordering::Relaxed) >= self.cancel_at {
|
||||
Err(std::io::ErrorKind::Interrupted.into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn open_read_file(&self, logical: &Path, resolved: &Path) -> std::io::Result<std::fs::File> {
|
||||
assert!(
|
||||
self.is_readable(logical),
|
||||
"denied ignore/source file opened"
|
||||
);
|
||||
self.opened.lock().unwrap().push(logical.into());
|
||||
open_beneath_no_symlinks(&self.root, resolved)
|
||||
}
|
||||
fn read_metadata(&self, _: &Path, resolved: &Path) -> std::io::Result<std::fs::Metadata> {
|
||||
open_beneath_no_symlinks(&self.root, resolved)?.metadata()
|
||||
}
|
||||
fn open_traversal_root(
|
||||
&self,
|
||||
_: &Path,
|
||||
resolved: &Path,
|
||||
) -> std::io::Result<Option<FsTraversalRoot>> {
|
||||
use std::os::fd::AsRawFd;
|
||||
let directory = open_beneath_no_symlinks(&self.root, resolved)?;
|
||||
let walk = PathBuf::from(format!("/proc/self/fd/{}", directory.as_raw_fd()));
|
||||
Ok(Some(FsTraversalRoot::new_at(
|
||||
walk,
|
||||
directory,
|
||||
resolved.into(),
|
||||
)))
|
||||
}
|
||||
fn open_read_dir(&self, logical: &Path, resolved: &Path) -> std::io::Result<std::fs::ReadDir> {
|
||||
use std::os::fd::AsRawFd;
|
||||
self.enumerated.lock().unwrap().push(logical.into());
|
||||
let directory = open_beneath_no_symlinks(&self.root, resolved)?;
|
||||
std::fs::read_dir(format!("/proc/self/fd/{}", directory.as_raw_fd()))
|
||||
}
|
||||
}
|
||||
fn glob(root: &Path, access: &dyn FsAccessPolicy) -> Result<GlobResult, FsError> {
|
||||
run_glob(
|
||||
root,
|
||||
root,
|
||||
GlobRequest {
|
||||
path: FsPath::root(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
},
|
||||
access,
|
||||
)
|
||||
}
|
||||
#[test]
|
||||
fn descriptor_glob_prunes_ignored_directories_before_enumeration_and_loading_rules() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(root.path().join("ignored/deep")).unwrap();
|
||||
std::fs::write(root.path().join(".ignore"), "ignored/\n").unwrap();
|
||||
// This rejected directory must never even attempt to load ignore files.
|
||||
std::fs::File::create(root.path().join("ignored/.ignore"))
|
||||
.unwrap()
|
||||
.set_len(2 * 1024 * 1024)
|
||||
.unwrap();
|
||||
std::fs::write(root.path().join("ignored/deep/secret.txt"), "secret").unwrap();
|
||||
std::fs::write(root.path().join("visible.txt"), "visible").unwrap();
|
||||
let access = Access::new(root.path());
|
||||
assert_eq!(
|
||||
glob(root.path(), &access).unwrap().paths,
|
||||
vec![FsPath::new("visible.txt").unwrap()]
|
||||
);
|
||||
assert_eq!(
|
||||
*access.enumerated.lock().unwrap(),
|
||||
vec![root.path().to_path_buf()]
|
||||
);
|
||||
assert!(
|
||||
!access
|
||||
.opened
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|path| path.starts_with(root.path().join("ignored")))
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn descriptor_glob_denied_and_symlinked_ignore_files_cannot_change_selection() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let outside = tempfile::tempdir().unwrap();
|
||||
std::fs::write(outside.path().join("rules"), "visible.txt\n").unwrap();
|
||||
std::fs::create_dir(root.path().join(".git")).unwrap();
|
||||
std::os::unix::fs::symlink(outside.path().join("rules"), root.path().join(".gitignore"))
|
||||
.unwrap();
|
||||
std::fs::write(root.path().join(".ignore"), "visible.txt\n").unwrap();
|
||||
std::fs::write(root.path().join("visible.txt"), "visible").unwrap();
|
||||
let mut access = Access::new(root.path());
|
||||
access.denied = Some(root.path().join(".ignore"));
|
||||
assert_eq!(
|
||||
glob(root.path(), &access).unwrap().paths,
|
||||
vec![FsPath::new("visible.txt").unwrap()]
|
||||
);
|
||||
assert!(
|
||||
!access
|
||||
.opened
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains(&root.path().join(".ignore"))
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn descriptor_ignore_loading_checks_cancellation_without_retrying_interrupted() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::write(root.path().join(".ignore"), "ignored.txt\n".repeat(1000)).unwrap();
|
||||
let mut access = Access::new(root.path());
|
||||
// First check is before resolution, second at the first fill_buf.
|
||||
access.cancel_at = 1;
|
||||
let error = crate::descriptor_ignore::add_checked_ignore_file(
|
||||
&mut ignore::gitignore::GitignoreBuilder::new(root.path()),
|
||||
&root.path().join(".ignore"),
|
||||
&mut 1024_000,
|
||||
&access,
|
||||
true,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::Interrupted)
|
||||
);
|
||||
assert_eq!(access.checks.load(Ordering::Relaxed), 2);
|
||||
}
|
||||
#[test]
|
||||
fn descriptor_ignore_loading_bounds_file_line_and_aggregate_bytes() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let path = root.path().join(".ignore");
|
||||
let access = Access::new(root.path());
|
||||
std::fs::File::create(&path)
|
||||
.unwrap()
|
||||
.set_len(1024 * 1024 + 1)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
glob(root.path(), &access),
|
||||
Err(FsError::InvalidArgument(_))
|
||||
));
|
||||
std::fs::write(&path, "x".repeat(16 * 1024 + 1)).unwrap();
|
||||
assert!(matches!(
|
||||
glob(root.path(), &access),
|
||||
Err(FsError::InvalidArgument(_))
|
||||
));
|
||||
std::fs::write(&path, "ignored.txt\n").unwrap();
|
||||
let error = crate::descriptor_ignore::add_checked_ignore_file(
|
||||
&mut ignore::gitignore::GitignoreBuilder::new(root.path()),
|
||||
&path,
|
||||
&mut 3,
|
||||
&access,
|
||||
true,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::InvalidData)
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn descriptor_ignore_ancestry_is_bounded_without_enumerating_a_tree() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let access = Access::new(root.path());
|
||||
let base = (0..128).fold(root.path().to_path_buf(), |path, _| path.join("d"));
|
||||
assert!(matches!(
|
||||
crate::descriptor_ignore::DescriptorIgnores::new(root.path(), &base, &access),
|
||||
Err(FsError::InvalidArgument(_))
|
||||
));
|
||||
assert!(access.opened.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
struct Plain(PathBuf);
|
||||
impl FsAccessPolicy for Plain {
|
||||
fn is_readable(&self, path: &Path) -> bool {
|
||||
path.starts_with(&self.0)
|
||||
}
|
||||
fn is_writable(&self, _: &Path) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn descriptor_glob_matches_plain_git_markers_exclude_and_nested_repository_boundaries() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(root.path().join("nested/.git/info")).unwrap();
|
||||
std::fs::create_dir_all(root.path().join(".git/info")).unwrap();
|
||||
std::fs::write(
|
||||
root.path().join(".gitignore"),
|
||||
"*.outer.txt\n!include.txt\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(root.path().join(".ignore"), "always.txt\n").unwrap();
|
||||
std::fs::write(
|
||||
root.path().join(".git/info/exclude"),
|
||||
"excluded.txt\ninclude.txt\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
root.path().join("nested/.git/info/exclude"),
|
||||
"/excluded.txt\n",
|
||||
)
|
||||
.unwrap();
|
||||
for name in [
|
||||
"a.outer.txt",
|
||||
"excluded.txt",
|
||||
"include.txt",
|
||||
"always.txt",
|
||||
"nested/a.outer.txt",
|
||||
"nested/excluded.txt",
|
||||
"nested/always.txt",
|
||||
] {
|
||||
std::fs::write(root.path().join(name), "data").unwrap();
|
||||
}
|
||||
let native = glob(root.path(), &Access::new(root.path())).unwrap();
|
||||
let ordinary = glob(root.path(), &Plain(root.path().into())).unwrap();
|
||||
assert_eq!(native, ordinary);
|
||||
assert_eq!(
|
||||
native.paths,
|
||||
vec![
|
||||
FsPath::new("include.txt").unwrap(),
|
||||
FsPath::new("nested/a.outer.txt").unwrap()
|
||||
]
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn descriptor_glob_requires_authorized_git_marker_for_gitignore_not_for_ignore() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::write(root.path().join(".gitignore"), "gitignored.txt\n").unwrap();
|
||||
std::fs::write(root.path().join(".ignore"), "ignored.txt\n").unwrap();
|
||||
for name in ["gitignored.txt", "ignored.txt", "visible.txt"] {
|
||||
std::fs::write(root.path().join(name), "data").unwrap();
|
||||
}
|
||||
let native = glob(root.path(), &Access::new(root.path())).unwrap();
|
||||
// Ambient ancestors may themselves be repositories (e.g. the harness's
|
||||
// TMPDIR). Their markers/ignore files are outside this provider root and
|
||||
// cannot activate Git rules or be inspected by descriptor search.
|
||||
assert_eq!(native.paths.len(), 2);
|
||||
std::fs::create_dir(root.path().join(".git")).unwrap();
|
||||
let native = glob(root.path(), &Access::new(root.path())).unwrap();
|
||||
assert_eq!(
|
||||
native,
|
||||
glob(root.path(), &Plain(root.path().into())).unwrap()
|
||||
);
|
||||
assert_eq!(native.paths, vec![FsPath::new("visible.txt").unwrap()]);
|
||||
let mut denied = Access::new(root.path());
|
||||
denied.denied = Some(root.path().join(".git"));
|
||||
assert_eq!(glob(root.path(), &denied).unwrap().paths.len(), 2);
|
||||
std::fs::remove_dir(root.path().join(".git")).unwrap();
|
||||
let outside = tempfile::tempdir().unwrap();
|
||||
std::fs::write(outside.path().join("exclude"), "visible.txt\n").unwrap();
|
||||
std::fs::write(
|
||||
root.path().join(".git"),
|
||||
format!("gitdir: {}\n", outside.path().display()),
|
||||
)
|
||||
.unwrap();
|
||||
// Real managed Workdirs use a .git file. Activate root .gitignore without
|
||||
// failing on .git/info/exclude or following external administration paths.
|
||||
let access = Access::new(root.path());
|
||||
assert_eq!(
|
||||
glob(root.path(), &access).unwrap().paths,
|
||||
vec![FsPath::new("visible.txt").unwrap()]
|
||||
);
|
||||
std::fs::write(root.path().join("visible.txt"), "needle\n").unwrap();
|
||||
let grep = run_grep(
|
||||
root.path(),
|
||||
root.path().into(),
|
||||
GrepRequest {
|
||||
path: FsPath::root(),
|
||||
pattern: "needle".into(),
|
||||
glob: None,
|
||||
file_type: None,
|
||||
case_insensitive: false,
|
||||
before_context: 0,
|
||||
after_context: 0,
|
||||
multiline: false,
|
||||
output_mode: GrepOutputMode::FilesWithMatches,
|
||||
offset: 0,
|
||||
limit: 100,
|
||||
},
|
||||
&access,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(grep.paths, vec![FsPath::new("visible.txt").unwrap()]);
|
||||
assert!(
|
||||
!access
|
||||
.opened
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|path| path.starts_with(outside.path()))
|
||||
);
|
||||
}
|
||||
@@ -6,7 +6,10 @@
|
||||
|
||||
mod checked;
|
||||
pub use checked::{CheckedTarget, identity_validator};
|
||||
mod descriptor_ignore;
|
||||
mod glob;
|
||||
#[cfg(test)]
|
||||
mod ignore_tests;
|
||||
mod local;
|
||||
mod operation;
|
||||
mod search;
|
||||
|
||||
@@ -1,50 +1,20 @@
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt::Write as _;
|
||||
use std::io::{BufRead, BufReader, Read};
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::descriptor_ignore::{DescriptorIgnores, SourceBoundedReader};
|
||||
|
||||
use crate::FsAccessPolicy;
|
||||
use grep_regex::RegexMatcherBuilder;
|
||||
use grep_searcher::sinks::UTF8 as UTF8Sink;
|
||||
use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkContext, SinkMatch};
|
||||
use ignore::WalkBuilder;
|
||||
use ignore::gitignore::{Gitignore, GitignoreBuilder};
|
||||
use ignore::overrides::{Override, OverrideBuilder};
|
||||
use ignore::types::{Types, TypesBuilder};
|
||||
|
||||
use crate::{FsError, GrepOutputMode, GrepRequest, GrepResult, direct_symlink};
|
||||
|
||||
struct SourceBoundedReader<'a, R> {
|
||||
inner: R,
|
||||
remaining: &'a mut u64,
|
||||
access: &'a dyn FsAccessPolicy,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for SourceBoundedReader<'_, R> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
self.access.check_cancelled()?;
|
||||
if buffer.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
if *self.remaining == 0 {
|
||||
let mut probe = [0_u8; 1];
|
||||
return match self.inner.read(&mut probe)? {
|
||||
0 => Ok(0),
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"grep source exceeded provider byte limit",
|
||||
)),
|
||||
};
|
||||
}
|
||||
let limit = usize::try_from(*self.remaining)
|
||||
.unwrap_or(usize::MAX)
|
||||
.min(buffer.len());
|
||||
let read = self.inner.read(&mut buffer[..limit])?;
|
||||
*self.remaining = (*self.remaining).saturating_sub(read as u64);
|
||||
Ok(read)
|
||||
}
|
||||
}
|
||||
|
||||
struct ContentLine {
|
||||
path: PathBuf,
|
||||
line_number: Option<u64>,
|
||||
@@ -234,119 +204,6 @@ fn direct_file_selected(path: &Path, overrides: Option<&Override>, types: Option
|
||||
&& !types.is_some_and(|filter| filter.matched(path, false).is_ignore())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DescriptorIgnoreMatchers {
|
||||
loaded_directories: HashSet<PathBuf>,
|
||||
by_directory: BTreeMap<PathBuf, Gitignore>,
|
||||
}
|
||||
|
||||
impl DescriptorIgnoreMatchers {
|
||||
fn load_directory(
|
||||
&mut self,
|
||||
directory: &Path,
|
||||
explicit_base: &Path,
|
||||
source_bytes_remaining: &mut u64,
|
||||
access: &dyn FsAccessPolicy,
|
||||
) -> Result<(), FsError> {
|
||||
if !self.loaded_directories.insert(directory.to_path_buf()) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut builder = GitignoreBuilder::new(directory);
|
||||
let mut has_patterns = false;
|
||||
for name in [".gitignore", ".ignore"] {
|
||||
let path = directory.join(name);
|
||||
let resolved = access
|
||||
.resolve_access_path(&path)
|
||||
.map_err(|error| FsError::io(&path, error))?;
|
||||
if !access.is_readable_paths(&path, &resolved) {
|
||||
continue;
|
||||
}
|
||||
let file = match access.open_read_file(&path, &resolved) {
|
||||
Ok(file) => file,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(FsError::io(&path, error)),
|
||||
};
|
||||
if !file
|
||||
.metadata()
|
||||
.map_err(|error| FsError::io(&path, error))?
|
||||
.is_file()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let bounded = SourceBoundedReader {
|
||||
inner: file,
|
||||
remaining: source_bytes_remaining,
|
||||
access,
|
||||
};
|
||||
let mut reader = BufReader::new(bounded);
|
||||
let mut line = String::new();
|
||||
let mut line_number = 0_u64;
|
||||
loop {
|
||||
line.clear();
|
||||
let read = reader
|
||||
.read_line(&mut line)
|
||||
.map_err(|error| FsError::io(&path, error))?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
line_number = line_number.saturating_add(1);
|
||||
if line.len() > 1024 * 1024 {
|
||||
return Err(FsError::InvalidArgument(format!(
|
||||
"ignore pattern line {line_number} exceeds provider limit"
|
||||
)));
|
||||
}
|
||||
let line = line.trim_end_matches(['\r', '\n']);
|
||||
// Match the ignore crate's partial-error behavior: one invalid
|
||||
// pattern does not discard the remaining valid lines.
|
||||
if builder.add_line(Some(path.clone()), line).is_ok() {
|
||||
has_patterns = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !has_patterns {
|
||||
return Ok(());
|
||||
}
|
||||
let matcher = builder.build().map_err(|_| {
|
||||
FsError::InvalidArgument("ignore patterns could not be compiled".to_string())
|
||||
})?;
|
||||
// An explicitly selected search root is traversed even when a strict
|
||||
// ancestor ignores that directory, matching path-backed Grep behavior.
|
||||
if directory != explicit_base
|
||||
&& explicit_base.starts_with(directory)
|
||||
&& matcher
|
||||
.matched_path_or_any_parents(explicit_base, true)
|
||||
.is_ignore()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
self.by_directory.insert(directory.to_path_buf(), matcher);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_ignored(&self, root: &Path, path: &Path) -> bool {
|
||||
let mut directories = path
|
||||
.parent()
|
||||
.into_iter()
|
||||
.flat_map(Path::ancestors)
|
||||
.take_while(|directory| directory.starts_with(root))
|
||||
.collect::<Vec<_>>();
|
||||
directories.reverse();
|
||||
let mut ignored = false;
|
||||
for directory in directories {
|
||||
let Some(matcher) = self.by_directory.get(directory) else {
|
||||
continue;
|
||||
};
|
||||
let matched = matcher.matched_path_or_any_parents(path, false);
|
||||
if matched.is_ignore() {
|
||||
ignored = true;
|
||||
} else if matched.is_whitelist() {
|
||||
ignored = false;
|
||||
}
|
||||
}
|
||||
ignored
|
||||
}
|
||||
}
|
||||
|
||||
struct GrepParams {
|
||||
pattern: String,
|
||||
path: Option<PathBuf>,
|
||||
@@ -560,37 +417,16 @@ pub fn run_grep(
|
||||
|
||||
let mut visited = 0_usize;
|
||||
let mut source_bytes_remaining = crate::MAX_GREP_SOURCE_BYTES;
|
||||
let mut descriptor_ignores = DescriptorIgnoreMatchers::default();
|
||||
if traversal.is_some() {
|
||||
let mut directory = root.to_path_buf();
|
||||
descriptor_ignores.load_directory(
|
||||
&directory,
|
||||
&base,
|
||||
&mut source_bytes_remaining,
|
||||
access,
|
||||
)?;
|
||||
for component in base
|
||||
.strip_prefix(root)
|
||||
.map_err(|_| {
|
||||
FsError::InvalidArgument("grep base is outside its provider root".to_string())
|
||||
})?
|
||||
.components()
|
||||
{
|
||||
directory.push(component);
|
||||
descriptor_ignores.load_directory(
|
||||
&directory,
|
||||
&base,
|
||||
&mut source_bytes_remaining,
|
||||
access,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
let walk = if traversal.is_some() {
|
||||
let mut descriptor_ignores = traversal
|
||||
.as_ref()
|
||||
.map(|_| DescriptorIgnores::grep(root, logical_walk_root, access))
|
||||
.transpose()?;
|
||||
let mut walk = if traversal.is_some() {
|
||||
crate::walk::Walk::descriptor(logical_walk_root, access)
|
||||
} else {
|
||||
crate::walk::Walk::Plain(walker.build())
|
||||
};
|
||||
for entry in walk {
|
||||
while let Some(entry) = walk.next() {
|
||||
let entry = entry?;
|
||||
access
|
||||
.check_cancelled()
|
||||
@@ -607,29 +443,36 @@ pub fn run_grep(
|
||||
continue;
|
||||
}
|
||||
let file_type = entry.kind;
|
||||
if traversal.is_some() && file_type.as_ref().is_some_and(|kind| kind.is_dir()) {
|
||||
descriptor_ignores.load_directory(&path, &base, &mut source_bytes_remaining, access)?;
|
||||
continue;
|
||||
}
|
||||
if !file_type.map(|kind| kind.is_file()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
if traversal.is_some() {
|
||||
if let Some(ignores) = descriptor_ignores.as_mut() {
|
||||
let relative = path.strip_prefix(&base).map_err(|_| {
|
||||
FsError::InvalidArgument(
|
||||
"descriptor traversal returned a path outside its search base".to_string(),
|
||||
)
|
||||
})?;
|
||||
if relative.components().any(|component| {
|
||||
component
|
||||
.as_os_str()
|
||||
.to_str()
|
||||
.is_some_and(|name| name.starts_with('.'))
|
||||
}) || descriptor_ignores.is_ignored(root, &path)
|
||||
|| !direct_file_selected(&path, overrides.as_ref(), types.as_ref())
|
||||
let is_dir = file_type.is_some_and(|kind| kind.is_dir());
|
||||
if path != logical_walk_root
|
||||
&& (relative.components().any(|component| {
|
||||
component
|
||||
.as_os_str()
|
||||
.to_str()
|
||||
.is_some_and(|name| name.starts_with('.'))
|
||||
}) || ignores.is_ignored(&path, is_dir))
|
||||
{
|
||||
if is_dir {
|
||||
walk.skip_current_dir();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_dir {
|
||||
ignores.load_directory(&path, access)?;
|
||||
continue;
|
||||
}
|
||||
if !direct_file_selected(&path, overrides.as_ref(), types.as_ref()) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if !file_type.map(|kind| kind.is_file()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let readable = access
|
||||
.resolve_access_path(&path)
|
||||
|
||||
@@ -13,6 +13,13 @@ pub(crate) enum Walk<'a> {
|
||||
Descriptor(DescriptorWalk<'a>),
|
||||
}
|
||||
impl<'a> Walk<'a> {
|
||||
/// A rejected directory has been yielded but not enumerated yet. Keep
|
||||
/// ignore pruning inside the same bounded descriptor walk as scope pruning.
|
||||
pub fn skip_current_dir(&mut self) {
|
||||
if let Self::Descriptor(walk) = self {
|
||||
walk.pending = None;
|
||||
}
|
||||
}
|
||||
pub fn descriptor(root: &Path, access: &'a dyn FsAccessPolicy) -> Self {
|
||||
Self::Descriptor(DescriptorWalk {
|
||||
access,
|
||||
|
||||
Reference in New Issue
Block a user