Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: harden submit queue durability

Browse Source
This commit is contained in:
Hare
2026-09-06 01:35:26 +09:00
parent bb4c1dfe4f
commit b038f022d3
14 changed files with 1370 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/session-store/src/fs_store.rs
+116 -1
View File
@@ -22,7 +22,8 @@ use crate::store::{Store, StoreError};
use crate::uploaded_file::{
bind_uploaded_file, clear_uploaded_file_binding, copy_committed_uploaded_files,
delete_uncommitted_uploaded_files, delete_uploaded_file, list_uploaded_file_refs,
read_uploaded_file, read_uploaded_file_by_id, write_uploaded_file,
pin_uploaded_file, read_uploaded_file, read_uploaded_file_by_id, release_uploaded_file_pin,
write_uploaded_file,
};
use crate::{
PasteArtifactLimits, SegmentId, SessionId, UploadedFileLimits, UploadedFileUploadContext,
@@ -518,6 +519,32 @@ impl Store for FsStore {
}
}
fn pin_uploaded_file(
&self,
session_id: SessionId,
reference: &UploadedFileRef,
owner_id: &str,
) -> Result<(), StoreError> {
let _guard = self
.append_lock
.lock()
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
pin_uploaded_file(&self.paste_artifact_dir(session_id), reference, owner_id)
}
fn release_uploaded_file_pin(
&self,
session_id: SessionId,
artifact_id: &str,
owner_id: &str,
) -> Result<(), StoreError> {
let _guard = self
.append_lock
.lock()
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
release_uploaded_file_pin(&self.paste_artifact_dir(session_id), artifact_id, owner_id)
}
fn delete_uploaded_file(
&self,
session_id: SessionId,
@@ -865,6 +892,94 @@ mod tests {
assert!(store.read_uploaded_file(owner, &reference).is_err());
}
#[test]
fn pending_upload_pin_survives_cleanup_until_release_or_history_binding() {
let tmp = tempfile::TempDir::new().unwrap();
let store = FsStore::new(tmp.path()).unwrap();
let session_id = new_session_id();
let limits = UploadedFileLimits {
max_file_bytes: 64,
max_session_bytes: 128,
};
let pending = store
.write_uploaded_file(session_id, "pending.txt", "text/plain", b"pending", limits)
.unwrap();
store
.pin_uploaded_file(session_id, &pending, "submission-1")
.unwrap();
assert!(matches!(
store.pin_uploaded_file(session_id, &pending, "submission-other"),
Err(StoreError::ArtifactAlreadyCommitted)
));
drop(store);
let store = FsStore::new(tmp.path()).unwrap();
assert_eq!(
store.delete_uncommitted_uploaded_files(session_id).unwrap(),
0
);
assert_eq!(
store
.read_uploaded_file_by_id(session_id, &pending.artifact_id)
.unwrap()
.1,
b"pending"
);
let fork_session_id = new_session_id();
assert_eq!(
store
.copy_committed_uploaded_files(session_id, fork_session_id)
.unwrap(),
1
);
assert_eq!(
store
.read_uploaded_file_by_id(fork_session_id, &pending.artifact_id)
.unwrap()
.1,
b"pending"
);
let committed = store
.bind_uploaded_file(session_id, &pending, "entry-1")
.unwrap();
store
.create_segment(
session_id,
new_segment_id(),
&[LogEntry::InputSegmentsCheckpoint {
ts: 1,
user_segments: vec![vec![protocol::Segment::UploadedFile {
file: committed.clone(),
}]],
}],
)
.unwrap();
assert_eq!(
store.delete_uncommitted_uploaded_files(session_id).unwrap(),
0
);
let releasable = store
.write_uploaded_file(session_id, "cancelled.txt", "text/plain", b"cancel", limits)
.unwrap();
store
.pin_uploaded_file(session_id, &releasable, "submission-2")
.unwrap();
store
.release_uploaded_file_pin(session_id, &releasable.artifact_id, "submission-2")
.unwrap();
assert_eq!(
store.delete_uncommitted_uploaded_files(session_id).unwrap(),
1
);
assert!(
store
.read_uploaded_file_by_id(session_id, &releasable.artifact_id)
.is_err()
);
}
#[test]
fn uploaded_file_validation_and_shared_quota_fail_closed() {
let tmp = tempfile::TempDir::new().unwrap();
crates/session-store/src/store.rs
+20
View File
@@ -226,6 +226,26 @@ pub trait Store: Send + Sync {
Err(StoreError::PasteArtifactUnsupported)
}
/// Retain an uploaded file while a durable pending operation owns it.
fn pin_uploaded_file(
&self,
_session_id: SessionId,
_reference: &UploadedFileRef,
_owner_id: &str,
) -> Result<(), StoreError> {
Err(StoreError::PasteArtifactUnsupported)
}
/// Release a pending-operation pin without changing committed ownership.
fn release_uploaded_file_pin(
&self,
_session_id: SessionId,
_artifact_id: &str,
_owner_id: &str,
) -> Result<(), StoreError> {
Err(StoreError::PasteArtifactUnsupported)
}
/// Delete an uncommitted uploaded file owned by `session_id`.
fn delete_uploaded_file(
&self,
crates/session-store/src/uploaded_file.rs
+79 -3
View File
@@ -24,6 +24,12 @@ pub const DEFAULT_MAX_FILES_PER_SUBMISSION: usize = 8;
pub const DEFAULT_MAX_SESSION_UPLOADED_FILES: u64 = 256;
const MAX_FILE_NAME_CHARS: usize = 255;
const MAX_MEDIA_TYPE_BYTES: usize = 127;
fn validate_pending_owner_id(owner_id: &str) -> Result<()> {
if owner_id.is_empty() || owner_id.len() > 256 {
return Err(StoreError::ArtifactIntegrityMismatch);
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UploadedFileLimits {
@@ -59,6 +65,8 @@ struct StoredUploadedFile {
#[serde(default, skip_serializing_if = "Option::is_none")]
source_entry_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pending_owner_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
upload_context: Option<UploadedFileUploadContext>,
content_base64: String,
}
@@ -291,6 +299,7 @@ pub(crate) fn write_uploaded_file(
byte_len,
sha256: sha256.clone(),
source_entry_id: None,
pending_owner_id: None,
upload_context: context.cloned(),
content_base64: BASE64.encode(content),
};
@@ -376,6 +385,72 @@ pub(crate) fn clear_uploaded_file_binding(
Ok(())
}
pub(crate) fn pin_uploaded_file(
dir: &Path,
reference: &UploadedFileRef,
owner_id: &str,
) -> Result<()> {
validate_pending_owner_id(owner_id)?;
if reference.source_entry_id.is_some() {
return Err(StoreError::ArtifactAlreadyCommitted);
}
let aggregate_lock = fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(dir.join(".aggregate.lock"))?;
FileExt::lock_exclusive(&aggregate_lock)?;
let path = record_path(dir, &reference.artifact_id)?;
let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
if stored.file_name != reference.file_name
|| stored.media_type != reference.media_type
|| stored.created_at_ms != reference.created_at_ms
|| stored.byte_len != reference.byte_len
|| stored.sha256 != reference.sha256
{
return Err(StoreError::ArtifactIntegrityMismatch);
}
if stored.source_entry_id.is_some() {
return Err(StoreError::ArtifactAlreadyCommitted);
}
if let Some(existing_owner) = stored.pending_owner_id.as_deref() {
return if existing_owner == owner_id {
Ok(())
} else {
Err(StoreError::ArtifactAlreadyCommitted)
};
}
stored.pending_owner_id = Some(owner_id.to_owned());
let temp = dir.join(format!(".{}.file.pin.tmp", reference.artifact_id));
fs::write(&temp, serde_json::to_vec(&stored)?)?;
fs::rename(temp, path)?;
Ok(())
}
pub(crate) fn release_uploaded_file_pin(
dir: &Path,
artifact_id: &str,
owner_id: &str,
) -> Result<()> {
validate_pending_owner_id(owner_id)?;
let aggregate_lock = fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(dir.join(".aggregate.lock"))?;
FileExt::lock_exclusive(&aggregate_lock)?;
let path = record_path(dir, artifact_id)?;
let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
if stored.source_entry_id.is_some() || stored.pending_owner_id.as_deref() != Some(owner_id) {
return Err(StoreError::ArtifactIntegrityMismatch);
}
stored.pending_owner_id = None;
let temp = dir.join(format!(".{artifact_id}.file.unpin.tmp"));
fs::write(&temp, serde_json::to_vec(&stored)?)?;
fs::rename(temp, path)?;
Ok(())
}
pub(crate) fn bind_uploaded_file(
dir: &Path,
reference: &UploadedFileRef,
@@ -405,6 +480,7 @@ pub(crate) fn bind_uploaded_file(
return Err(StoreError::ArtifactAlreadyCommitted);
}
stored.source_entry_id = Some(source_entry_id.to_owned());
stored.pending_owner_id = None;
let temp = dir.join(format!(".{}.file.bind.tmp", reference.artifact_id));
fs::write(&temp, serde_json::to_vec(&stored)?)?;
fs::rename(&temp, path)?;
@@ -455,7 +531,7 @@ pub(crate) fn copy_committed_uploaded_files(source_dir: &Path, target_dir: &Path
}
let bytes = fs::read(&path)?;
let stored: StoredUploadedFile = serde_json::from_slice(&bytes)?;
if stored.source_entry_id.is_none() {
if stored.source_entry_id.is_none() && stored.pending_owner_id.is_none() {
continue;
}
let target = target_dir.join(name);
@@ -499,7 +575,7 @@ pub(crate) fn delete_uncommitted_uploaded_files(dir: &Path) -> Result<u64> {
continue;
}
let stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
if stored.source_entry_id.is_none() {
if stored.source_entry_id.is_none() && stored.pending_owner_id.is_none() {
fs::remove_file(path)?;
removed = removed
.checked_add(1)
@@ -523,7 +599,7 @@ pub(crate) fn delete_uploaded_file(dir: &Path, artifact_id: &str) -> Result<bool
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(error) => return Err(error.into()),
};
if stored.source_entry_id.is_some() {
if stored.source_entry_id.is_some() || stored.pending_owner_id.is_some() {
return Err(StoreError::ArtifactAlreadyCommitted);
}
match fs::remove_file(path) {