fix: add bounded binary reads for ViewImage
This commit is contained in:
@@ -15,7 +15,8 @@ use thiserror::Error;
|
||||
|
||||
pub use glob::run_glob;
|
||||
pub use local::{
|
||||
resolve_access_path, run_edit, run_list, run_read, run_read_bounded, run_stat, run_write,
|
||||
resolve_access_path, run_edit, run_list, run_read, run_read_bounded, run_read_bytes,
|
||||
run_read_bytes_bounded, run_stat, run_write,
|
||||
};
|
||||
pub use operation::*;
|
||||
pub use search::run_grep;
|
||||
@@ -569,6 +570,103 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_byte_reads_preserve_binary_ranges_and_fence_versions() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let root = directory.path().canonicalize().unwrap();
|
||||
let original = b"\x89PNG\r\n\x1a\n\0line\n\xfftail";
|
||||
std::fs::write(root.join("image.bin"), original).unwrap();
|
||||
let access = RootAccess(root.clone());
|
||||
let limits = BoundedReadLimits::new(1024, 8).unwrap();
|
||||
|
||||
let first = run_read_bytes_bounded(
|
||||
&root,
|
||||
ReadBytesRequest {
|
||||
path: FsPath::new("image.bin").unwrap(),
|
||||
offset: 0,
|
||||
max_bytes: 8,
|
||||
expected_hash: None,
|
||||
},
|
||||
&access,
|
||||
limits,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(first.bytes, original[..8]);
|
||||
assert_eq!(first.total_bytes, original.len() as u64);
|
||||
assert!(!first.eof);
|
||||
|
||||
let second = run_read_bytes_bounded(
|
||||
&root,
|
||||
ReadBytesRequest {
|
||||
path: FsPath::new("image.bin").unwrap(),
|
||||
offset: 8,
|
||||
max_bytes: 8,
|
||||
expected_hash: Some(first.content_hash),
|
||||
},
|
||||
&access,
|
||||
limits,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(second.bytes, original[8..16]);
|
||||
assert_eq!(second.content_hash, first.content_hash);
|
||||
|
||||
std::fs::write(root.join("image.bin"), b"different bytes long!").unwrap();
|
||||
let error = run_read_bytes_bounded(
|
||||
&root,
|
||||
ReadBytesRequest {
|
||||
path: FsPath::new("image.bin").unwrap(),
|
||||
offset: 16,
|
||||
max_bytes: 8,
|
||||
expected_hash: Some(first.content_hash),
|
||||
},
|
||||
&access,
|
||||
limits,
|
||||
)
|
||||
.expect_err("a later chunk must not cross file versions");
|
||||
assert!(matches!(error, FsError::Conflict(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_byte_reads_reject_request_and_source_limit_overflow() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let root = directory.path().canonicalize().unwrap();
|
||||
std::fs::write(root.join("image.bin"), [0_u8; 17]).unwrap();
|
||||
let access = RootAccess(root.clone());
|
||||
let limits = BoundedReadLimits::new(16, 8).unwrap();
|
||||
|
||||
let response_error = run_read_bytes_bounded(
|
||||
&root,
|
||||
ReadBytesRequest {
|
||||
path: FsPath::new("image.bin").unwrap(),
|
||||
offset: 0,
|
||||
max_bytes: 9,
|
||||
expected_hash: None,
|
||||
},
|
||||
&access,
|
||||
limits,
|
||||
)
|
||||
.expect_err("requests above the response ceiling must be rejected");
|
||||
assert!(
|
||||
matches!(response_error, FsError::InvalidArgument(message) if message.contains("8 bytes"))
|
||||
);
|
||||
|
||||
let source_error = run_read_bytes_bounded(
|
||||
&root,
|
||||
ReadBytesRequest {
|
||||
path: FsPath::new("image.bin").unwrap(),
|
||||
offset: 0,
|
||||
max_bytes: 8,
|
||||
expected_hash: None,
|
||||
},
|
||||
&access,
|
||||
limits,
|
||||
)
|
||||
.expect_err("sources above the provider ceiling must be rejected");
|
||||
assert!(
|
||||
matches!(source_error, FsError::InvalidArgument(message) if message.contains("provider limit 16"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_stops_a_read_already_in_progress() {
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
@@ -7,8 +7,9 @@ use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{
|
||||
AtomicWriteMode, BoundedReadLimits, ContentHash, EditRequest, EditResult, EntryKind,
|
||||
FsAccessPolicy, FsError, FsPath, ListEntry, ListRequest, ListResult, ReadRequest, ReadResult,
|
||||
StatRequest, StatResult, WriteRequest, WriteResult, direct_symlink,
|
||||
FsAccessPolicy, FsError, FsPath, ListEntry, ListRequest, ListResult, ReadBytesRequest,
|
||||
ReadBytesResult, ReadRequest, ReadResult, StatRequest, StatResult, WriteRequest, WriteResult,
|
||||
direct_symlink,
|
||||
};
|
||||
|
||||
const READ_BUFFER_BYTES: usize = 16 * 1024;
|
||||
@@ -195,6 +196,137 @@ fn run_read_with_limits(
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run_read_bytes(
|
||||
root: &Path,
|
||||
request: ReadBytesRequest,
|
||||
access: &dyn FsAccessPolicy,
|
||||
) -> Result<ReadBytesResult, FsError> {
|
||||
run_read_bytes_with_limits(root, request, access, None)
|
||||
}
|
||||
|
||||
pub fn run_read_bytes_bounded(
|
||||
root: &Path,
|
||||
request: ReadBytesRequest,
|
||||
access: &dyn FsAccessPolicy,
|
||||
limits: BoundedReadLimits,
|
||||
) -> Result<ReadBytesResult, FsError> {
|
||||
run_read_bytes_with_limits(root, request, access, Some(limits))
|
||||
}
|
||||
|
||||
fn run_read_bytes_with_limits(
|
||||
root: &Path,
|
||||
request: ReadBytesRequest,
|
||||
access: &dyn FsAccessPolicy,
|
||||
limits: Option<BoundedReadLimits>,
|
||||
) -> Result<ReadBytesResult, FsError> {
|
||||
let logical = request.path;
|
||||
if request.max_bytes == 0 {
|
||||
return Err(FsError::InvalidArgument(
|
||||
"byte read max_bytes must be greater than zero".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(limits) = limits
|
||||
&& request.max_bytes > limits.max_response_bytes
|
||||
{
|
||||
return Err(FsError::InvalidArgument(format!(
|
||||
"byte read response limit is {} bytes",
|
||||
limits.max_response_bytes
|
||||
)));
|
||||
}
|
||||
|
||||
access
|
||||
.check_cancelled()
|
||||
.map_err(|error| map_io(&logical, error))?;
|
||||
let path = resolve(root, &logical)?;
|
||||
let target = require_access(&path, &logical, access, false, false)?;
|
||||
let file = access
|
||||
.open_read_file(&path, &target)
|
||||
.map_err(|error| map_io(&logical, error))?;
|
||||
let metadata = file.metadata().map_err(|error| map_io(&logical, error))?;
|
||||
if metadata.is_dir() {
|
||||
return Err(FsError::IsDirectory(PathBuf::from(logical.as_str())));
|
||||
}
|
||||
if !metadata.is_file() {
|
||||
return Err(FsError::InvalidArgument(
|
||||
"byte read source must be a regular file".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(limits) = limits
|
||||
&& metadata.len() > limits.max_source_bytes
|
||||
{
|
||||
return Err(FsError::InvalidArgument(format!(
|
||||
"byte read source exceeds provider limit {}",
|
||||
limits.max_source_bytes
|
||||
)));
|
||||
}
|
||||
|
||||
let mut bytes = Vec::with_capacity(request.max_bytes.min(READ_BUFFER_BYTES));
|
||||
let mut source_bytes = 0_u64;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut reader = BufReader::with_capacity(READ_BUFFER_BYTES, file);
|
||||
let mut buffer = [0_u8; READ_BUFFER_BYTES];
|
||||
loop {
|
||||
access
|
||||
.check_cancelled()
|
||||
.map_err(|error| map_io(&logical, error))?;
|
||||
let read_limit = limits
|
||||
.map(|limits| {
|
||||
limits
|
||||
.max_source_bytes
|
||||
.saturating_sub(source_bytes)
|
||||
.saturating_add(1)
|
||||
.min(READ_BUFFER_BYTES as u64) as usize
|
||||
})
|
||||
.unwrap_or(READ_BUFFER_BYTES);
|
||||
let read = reader
|
||||
.read(&mut buffer[..read_limit])
|
||||
.map_err(|error| map_io(&logical, error))?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
let chunk_start = source_bytes;
|
||||
source_bytes = source_bytes.saturating_add(read as u64);
|
||||
if let Some(limits) = limits
|
||||
&& source_bytes > limits.max_source_bytes
|
||||
{
|
||||
return Err(FsError::InvalidArgument(format!(
|
||||
"byte read source exceeds provider limit {}",
|
||||
limits.max_source_bytes
|
||||
)));
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
|
||||
if bytes.len() < request.max_bytes && source_bytes > request.offset {
|
||||
let start = request.offset.saturating_sub(chunk_start).min(read as u64) as usize;
|
||||
let retained = (request.max_bytes - bytes.len()).min(read - start);
|
||||
bytes.extend_from_slice(&buffer[start..start + retained]);
|
||||
}
|
||||
}
|
||||
|
||||
let content_hash: ContentHash = hasher.finalize().into();
|
||||
if request
|
||||
.expected_hash
|
||||
.is_some_and(|expected| expected != content_hash)
|
||||
{
|
||||
return Err(FsError::Conflict(logical.as_str().to_string()));
|
||||
}
|
||||
if request.offset > source_bytes {
|
||||
return Err(FsError::InvalidArgument(format!(
|
||||
"byte offset {} exceeds file size {source_bytes}",
|
||||
request.offset
|
||||
)));
|
||||
}
|
||||
let eof = request.offset.saturating_add(bytes.len() as u64) == source_bytes;
|
||||
Ok(ReadBytesResult {
|
||||
path: logical,
|
||||
bytes,
|
||||
offset: request.offset,
|
||||
total_bytes: source_bytes,
|
||||
content_hash,
|
||||
eof,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run_write(
|
||||
root: &Path,
|
||||
request: WriteRequest,
|
||||
|
||||
@@ -207,6 +207,36 @@ pub struct ReadResult {
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
/// Binary-safe byte range read with an optional whole-file version fence.
|
||||
///
|
||||
/// Unlike [`ReadRequest`], offsets and limits are bytes rather than lines. The
|
||||
/// provider still hashes the complete bounded source so callers can reject
|
||||
/// chunks from different file versions.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReadBytesRequest {
|
||||
pub path: FsPath,
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
pub offset: u64,
|
||||
#[schemars(range(min = 1, max = 9_007_199_254_740_991_usize))]
|
||||
pub max_bytes: usize,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub expected_hash: Option<ContentHash>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReadBytesResult {
|
||||
pub path: FsPath,
|
||||
pub bytes: Vec<u8>,
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
pub offset: u64,
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
pub total_bytes: u64,
|
||||
pub content_hash: ContentHash,
|
||||
pub eof: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WriteRequest {
|
||||
|
||||
Reference in New Issue
Block a user