Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: add session-owned uploaded file attachments

Browse Source
This commit is contained in:
Hare
2026-09-03 04:58:11 +09:00
parent d87441448e
commit 09a33e7283
17 changed files with 1416 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/session-store/src/fs_store.rs
+150 -2
View File
@@ -19,8 +19,12 @@ use crate::event_trace::TraceEntry;
use crate::paste_artifact::{read_from_dir, write_to_dir};
use crate::segment_log::LogEntry;
use crate::store::{Store, StoreError};
use crate::{PasteArtifactLimits, SegmentId, SessionId};
use protocol::PasteArtifactRef;
use crate::uploaded_file::{
bind_uploaded_file, delete_uploaded_file, read_uploaded_file, read_uploaded_file_by_id,
write_uploaded_file,
};
use crate::{PasteArtifactLimits, SegmentId, SessionId, UploadedFileLimits};
use protocol::{PasteArtifactRef, UploadedFileRef};
use std::fs;
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
@@ -389,6 +393,72 @@ impl Store for FsStore {
read_from_dir(&self.paste_artifact_dir(session_id), artifact_id)
}
fn write_uploaded_file(
&self,
session_id: SessionId,
file_name: &str,
media_type: &str,
content: &[u8],
limits: UploadedFileLimits,
) -> Result<UploadedFileRef, StoreError> {
let _guard = self
.append_lock
.lock()
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
write_uploaded_file(
&self.paste_artifact_dir(session_id),
file_name,
media_type,
content,
limits,
)
}
fn read_uploaded_file(
&self,
session_id: SessionId,
reference: &UploadedFileRef,
) -> Result<Vec<u8>, StoreError> {
read_uploaded_file(&self.paste_artifact_dir(session_id), reference)
}
fn read_uploaded_file_by_id(
&self,
session_id: SessionId,
artifact_id: &str,
) -> Result<(UploadedFileRef, Vec<u8>), StoreError> {
read_uploaded_file_by_id(&self.paste_artifact_dir(session_id), artifact_id)
}
fn bind_uploaded_file(
&self,
session_id: SessionId,
reference: &UploadedFileRef,
source_entry_id: &str,
) -> Result<UploadedFileRef, StoreError> {
let _guard = self
.append_lock
.lock()
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
bind_uploaded_file(
&self.paste_artifact_dir(session_id),
reference,
source_entry_id,
)
}
fn delete_uploaded_file(
&self,
session_id: SessionId,
artifact_id: &str,
) -> Result<bool, StoreError> {
let _guard = self
.append_lock
.lock()
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
delete_uploaded_file(&self.paste_artifact_dir(session_id), artifact_id)
}
fn append_trace(
&self,
session_id: SessionId,
@@ -548,6 +618,84 @@ mod tests {
);
}
#[test]
fn uploaded_files_are_session_scoped_integrity_checked_and_removable() {
let tmp = tempfile::TempDir::new().unwrap();
let store = FsStore::new(tmp.path()).unwrap();
let owner = new_session_id();
let other = new_session_id();
let limits = UploadedFileLimits {
max_file_bytes: 16,
max_session_bytes: 16,
};
let reference = store
.write_uploaded_file(owner, "notes.txt", "text/plain", b"hello", limits)
.unwrap();
assert_eq!(reference.file_name, "notes.txt");
assert_eq!(reference.media_type, "text/plain");
assert_eq!(reference.byte_len, 5);
assert_eq!(reference.source_entry_id, None);
assert_eq!(
store.read_uploaded_file(owner, &reference).unwrap(),
b"hello"
);
assert!(store.read_uploaded_file(other, &reference).is_err());
let mut forged = reference.clone();
forged.file_name = "other.txt".to_string();
assert!(matches!(
store.read_uploaded_file(owner, &forged),
Err(StoreError::ArtifactIntegrityMismatch)
));
assert!(
store
.delete_uploaded_file(owner, &reference.artifact_id)
.unwrap()
);
assert!(
!store
.delete_uploaded_file(owner, &reference.artifact_id)
.unwrap()
);
assert!(store.read_uploaded_file(owner, &reference).is_err());
}
#[test]
fn uploaded_file_validation_and_shared_quota_fail_closed() {
let tmp = tempfile::TempDir::new().unwrap();
let store = FsStore::new(tmp.path()).unwrap();
let session_id = new_session_id();
let limits = UploadedFileLimits {
max_file_bytes: 8,
max_session_bytes: 8,
};
assert!(matches!(
store.write_uploaded_file(session_id, "../secret", "text/plain", b"x", limits),
Err(StoreError::InvalidUploadedFileName)
));
assert!(matches!(
store.write_uploaded_file(session_id, "notes.txt", "not a type", b"x", limits),
Err(StoreError::InvalidUploadedFileMediaType)
));
store
.write_paste_artifact(
session_id,
"entry-1",
"1234",
PasteArtifactLimits {
max_artifact_bytes: 8,
max_session_bytes: 8,
max_session_artifacts: 4,
},
)
.unwrap();
assert!(matches!(
store.write_uploaded_file(session_id, "notes.txt", "text/plain", b"56789", limits),
Err(StoreError::ArtifactQuotaExceeded)
));
}
#[test]
fn paste_artifact_limits_and_corruption_fail_closed() {
let tmp = tempfile::TempDir::new().unwrap();
crates/session-store/src/lib.rs
+5
View File
@@ -41,6 +41,7 @@ pub mod segment;
pub mod segment_log;
pub mod store;
pub mod system_item;
pub mod uploaded_file;
pub mod worker_metadata;
pub mod worker_session_store;
@@ -66,6 +67,10 @@ pub use store::{Store, StoreError};
pub use system_item::{
PromptRenderProvenance, SystemItem, SystemReminder, SystemReminderSource, render_worker_event,
};
pub use uploaded_file::{
DEFAULT_MAX_FILES_PER_SUBMISSION, DEFAULT_MAX_SESSION_ARTIFACT_BYTES,
DEFAULT_MAX_UPLOADED_FILE_BYTES, UploadedFileLimits,
};
pub use worker_metadata::{
CombinedStore, FsWorkerStore, WorkerActiveSegmentRef, WorkerAggregateStore, WorkerMetadata,
WorkerMetadataStore, WorkerPeer, WorkerReclaimedChild, WorkerSpawnedChild,
crates/session-store/src/paste_artifact.rs
+37 -18
View File
@@ -38,6 +38,34 @@ pub(crate) struct StoredPasteArtifact {
pub content: String,
}
pub(crate) fn stored_paste_usage(artifact_dir: &Path) -> Result<(u64, u64), StoreError> {
if !artifact_dir.exists() {
return Ok((0, 0));
}
let mut aggregate = 0_u64;
let mut artifact_count = 0_u64;
for entry in fs::read_dir(artifact_dir)? {
let path = entry?.path();
let Some(name) = path.file_name().and_then(|value| value.to_str()) else {
continue;
};
if !name.ends_with(".json") || name.ends_with(".file.json") {
continue;
}
let stored: StoredPasteArtifact = serde_json::from_slice(&fs::read(&path)?)?;
verify(&stored, &stored.reference.artifact_id)?;
artifact_count = artifact_count.checked_add(1).ok_or_else(|| {
StoreError::PasteArtifactLimit("session artifact count overflow".to_string())
})?;
aggregate = aggregate
.checked_add(stored.reference.byte_len)
.ok_or_else(|| {
StoreError::PasteArtifactLimit("session aggregate size overflow".to_string())
})?;
}
Ok((aggregate, artifact_count))
}
pub(crate) fn write_to_dir(
artifact_dir: &Path,
source_entry_id: &str,
@@ -58,24 +86,15 @@ pub(crate) fn write_to_dir(
.write(true)
.open(artifact_dir.join(".aggregate.lock"))?;
FileExt::lock_exclusive(&aggregate_lock)?;
let mut aggregate = 0_u64;
let mut artifact_count = 0_u64;
for entry in fs::read_dir(artifact_dir)? {
let path = entry?.path();
if path.extension().and_then(|value| value.to_str()) != Some("json") {
continue;
}
let stored: StoredPasteArtifact = serde_json::from_slice(&fs::read(&path)?)?;
verify(&stored, &stored.reference.artifact_id)?;
artifact_count = artifact_count.checked_add(1).ok_or_else(|| {
StoreError::PasteArtifactLimit("session artifact count overflow".to_string())
})?;
aggregate = aggregate
.checked_add(stored.reference.byte_len)
.ok_or_else(|| {
StoreError::PasteArtifactLimit("session aggregate size overflow".to_string())
})?;
}
let (paste_bytes, artifact_count) = stored_paste_usage(artifact_dir)?;
let (uploaded_bytes, uploaded_count) =
crate::uploaded_file::stored_uploaded_file_usage(artifact_dir)?;
let aggregate = paste_bytes.checked_add(uploaded_bytes).ok_or_else(|| {
StoreError::PasteArtifactLimit("session aggregate size overflow".to_string())
})?;
let artifact_count = artifact_count.checked_add(uploaded_count).ok_or_else(|| {
StoreError::PasteArtifactLimit("session artifact count overflow".to_string())
})?;
let projected = aggregate.checked_add(byte_len).ok_or_else(|| {
StoreError::PasteArtifactLimit("session aggregate size overflow".to_string())
})?;
crates/session-store/src/store.rs
+73 -2
View File
@@ -13,8 +13,8 @@
use crate::event_trace::TraceEntry;
use crate::segment_log::LogEntry;
use crate::{PasteArtifactLimits, SegmentId, SessionId};
use protocol::PasteArtifactRef;
use crate::{PasteArtifactLimits, SegmentId, SessionId, UploadedFileLimits};
use protocol::{PasteArtifactRef, UploadedFileRef};
/// Errors from the persistence store.
#[derive(Debug, thiserror::Error)]
@@ -42,6 +42,30 @@ pub enum StoreError {
#[error("paste artifact size limit exceeded: {0}")]
PasteArtifactLimit(String),
#[error("uploaded file is too large")]
ArtifactTooLarge,
#[error("session artifact aggregate quota exceeded")]
ArtifactQuotaExceeded,
#[error("uploaded file reference integrity check failed")]
ArtifactIntegrityMismatch,
#[error("uploaded file name is invalid")]
InvalidUploadedFileName,
#[error("uploaded file media type is invalid")]
InvalidUploadedFileMediaType,
#[error("uploaded file is already committed to session history")]
ArtifactAlreadyCommitted,
#[error("artifact id is invalid")]
InvalidArtifactId,
#[error("artifact timestamp is invalid")]
InvalidTimestamp,
}
/// Sync persistence backend for segment logs.
@@ -150,6 +174,53 @@ pub trait Store: Send + Sync {
Err(StoreError::PasteArtifactUnsupported)
}
/// Persist a client-local file before a submission references it.
fn write_uploaded_file(
&self,
_session_id: SessionId,
_file_name: &str,
_media_type: &str,
_content: &[u8],
_limits: UploadedFileLimits,
) -> Result<UploadedFileRef, StoreError> {
Err(StoreError::PasteArtifactUnsupported)
}
/// Read and integrity-check an uploaded file owned by `session_id`.
fn read_uploaded_file(
&self,
_session_id: SessionId,
_reference: &UploadedFileRef,
) -> Result<Vec<u8>, StoreError> {
Err(StoreError::PasteArtifactUnsupported)
}
fn read_uploaded_file_by_id(
&self,
_session_id: SessionId,
_artifact_id: &str,
) -> Result<(UploadedFileRef, Vec<u8>), StoreError> {
Err(StoreError::PasteArtifactUnsupported)
}
fn bind_uploaded_file(
&self,
_session_id: SessionId,
_reference: &UploadedFileRef,
_source_entry_id: &str,
) -> Result<UploadedFileRef, StoreError> {
Err(StoreError::PasteArtifactUnsupported)
}
/// Delete an uncommitted uploaded file owned by `session_id`.
fn delete_uploaded_file(
&self,
_session_id: SessionId,
_artifact_id: &str,
) -> Result<bool, StoreError> {
Err(StoreError::PasteArtifactUnsupported)
}
/// Append a trace entry to the debug event trace file.
fn append_trace(
&self,
crates/session-store/src/uploaded_file.rs
+299
View File
@@ -0,0 +1,299 @@
use std::{
fs,
path::Path,
time::{SystemTime, UNIX_EPOCH},
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use fs4::fs_std::FileExt;
use protocol::{UploadedFileAvailability, UploadedFileRef};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use uuid::Uuid;
use crate::StoreError;
type Result<T> = std::result::Result<T, StoreError>;
pub const DEFAULT_MAX_UPLOADED_FILE_BYTES: u64 = 10 * 1024 * 1024;
pub const DEFAULT_MAX_SESSION_ARTIFACT_BYTES: u64 = 32 * 1024 * 1024;
pub const DEFAULT_MAX_FILES_PER_SUBMISSION: usize = 8;
const MAX_FILE_NAME_CHARS: usize = 255;
const MAX_MEDIA_TYPE_BYTES: usize = 127;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UploadedFileLimits {
pub max_file_bytes: u64,
pub max_session_bytes: u64,
}
impl Default for UploadedFileLimits {
fn default() -> Self {
Self {
max_file_bytes: DEFAULT_MAX_UPLOADED_FILE_BYTES,
max_session_bytes: DEFAULT_MAX_SESSION_ARTIFACT_BYTES,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
struct StoredUploadedFile {
file_name: String,
media_type: String,
created_at_ms: u64,
byte_len: u64,
sha256: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
source_entry_id: Option<String>,
content_base64: String,
}
pub(crate) fn validate_file_name(file_name: &str) -> Result<()> {
if file_name.is_empty()
|| file_name.chars().count() > MAX_FILE_NAME_CHARS
|| file_name == "."
|| file_name == ".."
|| file_name
.chars()
.any(|ch| ch.is_control() || matches!(ch, '/' | '\\'))
{
return Err(StoreError::InvalidUploadedFileName);
}
Ok(())
}
pub(crate) fn validate_media_type(media_type: &str) -> Result<()> {
let valid = !media_type.is_empty()
&& media_type.len() <= MAX_MEDIA_TYPE_BYTES
&& media_type.is_ascii()
&& !media_type
.bytes()
.any(|byte| byte.is_ascii_control() || byte == b' ')
&& media_type.split_once('/').is_some_and(|(kind, subtype)| {
!kind.is_empty()
&& !subtype.is_empty()
&& kind.bytes().chain(subtype.bytes()).all(|byte| {
byte.is_ascii_alphanumeric()
|| matches!(
byte,
b'!' | b'#' | b'$' | b'&' | b'^' | b'_' | b'.' | b'+' | b'-'
)
})
});
let allowed = media_type.starts_with("text/")
|| matches!(
media_type,
"application/json"
| "application/pdf"
| "image/png"
| "image/jpeg"
| "image/gif"
| "image/webp"
);
if !valid || !allowed {
return Err(StoreError::InvalidUploadedFileMediaType);
}
Ok(())
}
fn record_path(dir: &Path, artifact_id: &str) -> Result<std::path::PathBuf> {
let id = Uuid::parse_str(artifact_id).map_err(|_| StoreError::InvalidArtifactId)?;
Ok(dir.join(format!("{id}.file.json")))
}
fn now_ms() -> Result<u64> {
let value = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| StoreError::InvalidTimestamp)?
.as_millis();
u64::try_from(value).map_err(|_| StoreError::InvalidTimestamp)
}
fn digest(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
pub(crate) fn stored_uploaded_file_usage(dir: &Path) -> Result<(u64, u64)> {
if !dir.exists() {
return Ok((0, 0));
}
let mut bytes = 0_u64;
let mut count = 0_u64;
for entry in fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if !entry.file_type()?.is_file()
|| !path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.ends_with(".file.json"))
{
continue;
}
let stored: StoredUploadedFile = serde_json::from_slice(&fs::read(path)?)?;
bytes = bytes
.checked_add(stored.byte_len)
.ok_or(StoreError::ArtifactQuotaExceeded)?;
count = count
.checked_add(1)
.ok_or(StoreError::ArtifactQuotaExceeded)?;
}
Ok((bytes, count))
}
pub(crate) fn write_uploaded_file(
dir: &Path,
file_name: &str,
media_type: &str,
content: &[u8],
limits: UploadedFileLimits,
) -> Result<UploadedFileRef> {
validate_file_name(file_name)?;
validate_media_type(media_type)?;
let byte_len = u64::try_from(content.len()).map_err(|_| StoreError::ArtifactTooLarge)?;
if byte_len > limits.max_file_bytes {
return Err(StoreError::ArtifactTooLarge);
}
fs::create_dir_all(dir)?;
let aggregate_lock = fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(dir.join(".aggregate.lock"))?;
FileExt::lock_exclusive(&aggregate_lock)?;
let (paste_bytes, _) = crate::paste_artifact::stored_paste_usage(dir)?;
let (file_bytes, _) = stored_uploaded_file_usage(dir)?;
if paste_bytes
.checked_add(file_bytes)
.and_then(|total| total.checked_add(byte_len))
.is_none_or(|total| total > limits.max_session_bytes)
{
return Err(StoreError::ArtifactQuotaExceeded);
}
let artifact_id = Uuid::now_v7().to_string();
let created_at_ms = now_ms()?;
let sha256 = digest(content);
let stored = StoredUploadedFile {
file_name: file_name.to_owned(),
media_type: media_type.to_owned(),
created_at_ms,
byte_len,
sha256: sha256.clone(),
source_entry_id: None,
content_base64: BASE64.encode(content),
};
let path = record_path(dir, &artifact_id)?;
let temp = dir.join(format!(".{artifact_id}.file.tmp"));
fs::write(&temp, serde_json::to_vec(&stored)?)?;
fs::rename(&temp, &path)?;
Ok(UploadedFileRef {
artifact_id,
file_name: file_name.to_owned(),
media_type: media_type.to_owned(),
created_at_ms,
availability: UploadedFileAvailability::Available,
byte_len,
sha256,
source_entry_id: None,
})
}
pub(crate) fn read_uploaded_file_by_id(
dir: &Path,
artifact_id: &str,
) -> Result<(UploadedFileRef, Vec<u8>)> {
let stored: StoredUploadedFile =
serde_json::from_slice(&fs::read(record_path(dir, artifact_id)?)?)?;
let content = BASE64
.decode(&stored.content_base64)
.map_err(|_| StoreError::ArtifactIntegrityMismatch)?;
if u64::try_from(content.len()).ok() != Some(stored.byte_len)
|| digest(&content) != stored.sha256
{
return Err(StoreError::ArtifactIntegrityMismatch);
}
let reference = UploadedFileRef {
artifact_id: artifact_id.to_owned(),
file_name: stored.file_name,
media_type: stored.media_type,
created_at_ms: stored.created_at_ms,
availability: UploadedFileAvailability::Available,
byte_len: stored.byte_len,
sha256: stored.sha256,
source_entry_id: stored.source_entry_id,
};
Ok((reference, content))
}
pub(crate) fn read_uploaded_file(dir: &Path, reference: &UploadedFileRef) -> Result<Vec<u8>> {
let (stored_reference, content) = read_uploaded_file_by_id(dir, &reference.artifact_id)?;
if stored_reference.file_name != reference.file_name
|| stored_reference.media_type != reference.media_type
|| stored_reference.created_at_ms != reference.created_at_ms
|| stored_reference.byte_len != reference.byte_len
|| stored_reference.sha256 != reference.sha256
|| stored_reference.source_entry_id != reference.source_entry_id
{
return Err(StoreError::ArtifactIntegrityMismatch);
}
Ok(content)
}
pub(crate) fn bind_uploaded_file(
dir: &Path,
reference: &UploadedFileRef,
source_entry_id: &str,
) -> Result<UploadedFileRef> {
if source_entry_id.is_empty() || reference.source_entry_id.is_some() {
return Err(StoreError::ArtifactIntegrityMismatch);
}
let aggregate_lock = fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(dir.join(".aggregate.lock"))?;
FileExt::lock_exclusive(&aggregate_lock)?;
read_uploaded_file(dir, reference)?;
let path = record_path(dir, &reference.artifact_id)?;
let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
if stored.source_entry_id.is_some() {
return Err(StoreError::ArtifactAlreadyCommitted);
}
stored.source_entry_id = Some(source_entry_id.to_owned());
let temp = dir.join(format!(".{}.file.bind.tmp", reference.artifact_id));
fs::write(&temp, serde_json::to_vec(&stored)?)?;
fs::rename(&temp, path)?;
let mut bound = reference.clone();
bound.source_entry_id = Some(source_entry_id.to_owned());
Ok(bound)
}
pub(crate) fn delete_uploaded_file(dir: &Path, artifact_id: &str) -> Result<bool> {
fs::create_dir_all(dir)?;
let aggregate_lock = fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(dir.join(".aggregate.lock"))?;
FileExt::lock_exclusive(&aggregate_lock)?;
let path = record_path(dir, artifact_id)?;
let stored = match fs::read(&path) {
Ok(bytes) => serde_json::from_slice::<StoredUploadedFile>(&bytes)?,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(error) => return Err(error.into()),
};
if stored.source_entry_id.is_some() {
return Err(StoreError::ArtifactAlreadyCommitted);
}
match fs::remove_file(path) {
Ok(()) => Ok(true),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(error) => Err(error.into()),
}
}