fix: merge hare/develop and reconcile lifecycle contracts

This commit is contained in:
2026-10-31 10:05:54 +09:00
32 changed files with 2934 additions and 580 deletions
+1 -10
View File
@@ -210,12 +210,7 @@ impl MergeRequestThreadEvent {
} }
fn bound_bodies(&mut self) { fn bound_bodies(&mut self) {
match self { match self {
Self::Review(value) => { Self::Review(_) => {}
truncate_body(&mut value.body);
for finding in &mut value.findings {
truncate_body(&mut finding.body);
}
}
Self::ReviewRevoked(value) => truncate_body(&mut value.reason), Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
Self::ReviewCancelled(value) => truncate_body(&mut value.reason), Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
Self::Comment(value) => truncate_body(&mut value.body), Self::Comment(value) => truncate_body(&mut value.body),
@@ -939,10 +934,6 @@ impl MergeRequestStore {
&self, &self,
i: SubmitMergeRequestReview, i: SubmitMergeRequestReview,
) -> Result<ReviewEvent, MergeRequestError> { ) -> Result<ReviewEvent, MergeRequestError> {
bounded_body("review body", &i.body)?;
for finding in &i.findings {
bounded_body("review finding", &finding.body)?;
}
let mut c = self.lock()?; let mut c = self.lock()?;
let t = c.transaction()?; let t = c.transaction()?;
let g: Option<(String, String, String, String, String, String)> = t let g: Option<(String, String, String, String, String, String)> = t
+43
View File
@@ -376,6 +376,49 @@ fn approve_for(
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent { fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
approve_for(s, "MR", "R", subject, token) approve_for(s, "MR", "R", subject, token)
} }
#[test]
fn review_submission_preserves_long_body_and_findings_on_reload() {
for (body, finding_body) in [
("b".repeat(16 * 1024 + 1), "finding".into()),
("review".into(), "指摘".repeat(4096)),
] {
let (_dir, store) = fixture();
open(&store);
request(&store, "source", "token");
let findings = vec![ReviewFinding {
severity: FindingSeverity::Major,
code: None,
path: None,
line: None,
body: finding_body,
}];
let submitted = store
.submit_review(SubmitMergeRequestReview {
merge_request_id: "MR".into(),
ticket_id: "T".into(),
current_subject_ref: "source".into(),
capability_token: "token".into(),
decision: ReviewDecision::RequestChanges,
body: body.clone(),
findings: findings.clone(),
now: at(4),
})
.unwrap();
assert_eq!(submitted.body, body);
assert_eq!(submitted.findings, findings);
let reloaded = store.get_by_id("W", "MR").unwrap();
let saved = reloaded
.thread
.iter()
.find_map(|event| match event {
MergeRequestThreadEvent::Review(review) => Some(review),
_ => None,
})
.unwrap();
assert_eq!(saved, &submitted);
}
}
#[test] #[test]
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() { fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
let (_d, store) = fixture(); let (_d, store) = fixture();
+8 -1
View File
@@ -7348,6 +7348,8 @@ pub struct WorkingDirectoryCleanupTarget {
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
pub struct WorkingDirectoryRemovalRequest { pub struct WorkingDirectoryRemovalRequest {
/// Stable for an exact resend; a new explicit request uses a new identity.
pub operation_id: String,
pub reason: String, pub reason: String,
} }
@@ -7886,6 +7888,9 @@ pub struct CurrentWorkerWorkdirAttachmentResponse {
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub environment: Option<workdir::WorkdirSessionEnvironment>, pub environment: Option<workdir::WorkdirSessionEnvironment>,
pub attached: bool, pub attached: bool,
/// The Workdir was already attached to this Worker; alias is its existing name.
#[serde(default)]
pub already_attached: bool,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
@@ -13691,9 +13696,11 @@ mod tests {
capabilities: workdir::WorkdirSessionCapabilities::EMPTY, capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
environment: None, environment: None,
attached: false, attached: false,
already_attached: false,
}; };
let value = serde_json::to_value(&response).unwrap(); let value = serde_json::to_value(&response).unwrap();
assert_eq!(value.as_object().unwrap().len(), 5); assert_eq!(value.as_object().unwrap().len(), 6);
assert_eq!(value["already_attached"], false);
assert!(value.get("connection_id").is_none()); assert!(value.get("connection_id").is_none());
assert_eq!( assert_eq!(
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(), serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
+23
View File
@@ -202,6 +202,29 @@ impl SubjektivStore {
}) })
} }
/// End a body after the Host has confirmed its Runtime removal. This is not a
/// completed Journal handoff: source coverage, positions and saved receipts stay
/// unchanged. Revocation and closing generations are atomic, so no retained Job
/// can write after deletion even while Backend Job cleanup is being retried.
pub fn release_removed_journal_body(&self, host: &JournalHostContext) -> JournalResult<()> {
self.database.try_transaction(|tx| {
super::store::check_session_host(self, tx, host)?;
let attr = host.attribution.as_ref().expect("checked Session authority");
let binding: Option<String> = tx.query_row(
"SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 ORDER BY rowid DESC LIMIT 1",
params![host.subject_id, attr.runtime_id, attr.worker_id], |r| r.get(0),
).optional()?;
let binding = binding.unwrap_or_else(|| crate::issued_id("journal-removed-body"));
tx.execute("INSERT OR IGNORE INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)",params![binding,host.subject_id,attr.runtime_id,attr.worker_id])?;
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
// Bind legacy attributed Sessions to an inactive lifetime, never to the
// successor. Retained unfinished history is not an active writer.
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT session_id,?4 FROM subject_session_attributions WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id,binding])?;
tx.execute("UPDATE journal_generations SET active=0 WHERE journal_id IN (SELECT j.id FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
Ok(())
})
}
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original /// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs. /// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
/// This transaction rejects retained writers, pending source bytes and active generations. /// This transaction rejects retained writers, pending source bytes and active generations.
+26
View File
@@ -653,6 +653,32 @@ impl SubjektivStore {
Ok(()) Ok(())
}) })
} }
/// Read the exact retained Job/attempt snapshot without granting writer
/// authority or creating a generation. Used for outcome recovery after removal.
pub fn retained_journal_generation(
&self,
host: &JournalHostContext,
id: &str,
binding: &JobAttemptBinding,
) -> JournalResult<Option<JournalGeneration>> {
self.database.try_with_connection(|tx| {
check_host(self, tx, host, false)?;
check_session_host(self, tx, host)?;
journal(tx, host, id, true)?;
let raw: Option<String> = tx.query_row(
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
params![id, binding.job_id, binding.attempt_id], |r| r.get(0),
).optional()?;
raw.map(|raw| {
let generation: JournalGeneration = serde_json::from_str(&raw)?;
if generation.journal_id != id || &generation.job != binding {
return Err(conflict("retained generation identity mismatch"));
}
Ok(generation)
}).transpose()
})
}
pub fn read_journal_generation( pub fn read_journal_generation(
&self, &self,
host: &JournalHostContext, host: &JournalHostContext,
+98 -8
View File
@@ -103,7 +103,18 @@ impl Drop for WorkdirOperationGuard {
} }
} }
fn workdir_operation_busy() -> RuntimeError { fn workdir_operation_busy(id: &str, action: &str, blocking: &WorkdirOperation) -> RuntimeError {
let (busy_reason, active_use_count) = match blocking {
WorkdirOperation::Use(count) => ("active_use", *count),
WorkdirOperation::Cleanup => ("cleanup_in_progress", 0),
};
tracing::warn!(target: "yoi::workdir_cleanup",
working_directory_id = id,
action,
busy_source = "runtime_operation",
busy_reason,
active_use_count,
"Workdir operation blocked by busy resource");
RuntimeError::WorkingDirectory( RuntimeError::WorkingDirectory(
crate::working_directory::WorkingDirectoryDiagnostic::rejected( crate::working_directory::WorkingDirectoryDiagnostic::rejected(
"working_directory_cleanup_resource_busy", "working_directory_cleanup_resource_busy",
@@ -1150,8 +1161,12 @@ impl Runtime {
let backend = { let backend = {
let mut state = self.lock()?; let mut state = self.lock()?;
state.ensure_running()?; state.ensure_running()?;
if state.workdir_operations.contains_key(working_directory_id) { if let Some(blocking) = state.workdir_operations.get(working_directory_id) {
return Err(workdir_operation_busy()); return Err(workdir_operation_busy(
working_directory_id,
"cleanup_working_directory",
blocking,
));
} }
if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) { if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) {
return Err(RuntimeError::InvalidRequest(format!( return Err(RuntimeError::InvalidRequest(format!(
@@ -6344,11 +6359,8 @@ impl RuntimeState {
} }
fn ensure_workdir_not_removing(&self, id: &str) -> Result<(), RuntimeError> { fn ensure_workdir_not_removing(&self, id: &str) -> Result<(), RuntimeError> {
if matches!( if let Some(blocking @ WorkdirOperation::Cleanup) = self.workdir_operations.get(id) {
self.workdir_operations.get(id), return Err(workdir_operation_busy(id, "admit_use", blocking));
Some(WorkdirOperation::Cleanup)
) {
return Err(workdir_operation_busy());
} }
Ok(()) Ok(())
} }
@@ -9409,6 +9421,84 @@ mod tests {
.block_on(subscription.recv()) .block_on(subscription.recv())
} }
#[test]
fn busy_admission_logs_runtime_use_and_cleanup_blockers() {
use std::io::{Read, Seek, SeekFrom};
let mut output = tempfile::tempfile().unwrap();
let writer = output.try_clone().unwrap();
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_writer(move || writer.try_clone().unwrap())
.finish();
tracing::subscriber::with_default(subscriber, || {
tracing::callsite::rebuild_interest_cache();
let (runtime, backend) = runtime_and_backend();
let first = runtime.reserve_workdir_use(["workdir-used"]).unwrap();
let second = runtime.reserve_workdir_use(["workdir-used"]).unwrap();
assert_workdir_busy(
runtime
.cleanup_working_directory("workdir-used")
.unwrap_err(),
);
drop((first, second));
assert!(runtime.inner.lock().unwrap().workdir_operations.is_empty());
let (entered, release) = WorkdirEffectGate::install(&backend.cleanup_gate);
let cleanup_runtime = runtime.clone();
let cleanup = std::thread::spawn(move || {
cleanup_runtime.cleanup_working_directory("workdir-removing")
});
entered
.recv_timeout(std::time::Duration::from_secs(5))
.unwrap();
assert_workdir_busy(
runtime
.cleanup_working_directory("workdir-removing")
.unwrap_err(),
);
assert_workdir_busy(
runtime
.reserve_workdir_use(["workdir-removing"])
.unwrap_err(),
);
release.send(()).unwrap();
assert_workdir_busy(cleanup.join().unwrap().unwrap_err());
assert!(runtime.inner.lock().unwrap().workdir_operations.is_empty());
});
output.seek(SeekFrom::Start(0)).unwrap();
let mut log = String::new();
output.read_to_string(&mut log).unwrap();
let events: Vec<_> = log
.lines()
.filter(|line| line.contains("busy_source="))
.collect();
assert_eq!(events.len(), 3, "{log}");
for (event, (id, action, reason, count)) in events.iter().zip([
("workdir-used", "cleanup_working_directory", "active_use", 2),
(
"workdir-removing",
"cleanup_working_directory",
"cleanup_in_progress",
0,
),
("workdir-removing", "admit_use", "cleanup_in_progress", 0),
]) {
for field in [
"WARN".to_string(),
"yoi::workdir_cleanup".to_string(),
format!("working_directory_id=\"{id}\""),
format!("action=\"{action}\""),
"busy_source=\"runtime_operation\"".to_string(),
format!("busy_reason=\"{reason}\""),
format!("active_use_count={count}"),
] {
assert!(event.contains(&field), "missing {field}: {event}");
}
}
}
#[test] #[test]
fn workdir_removal_reserves_only_its_target_and_releases_on_failure() { fn workdir_removal_reserves_only_its_target_and_releases_on_failure() {
let (runtime, backend) = runtime_and_backend(); let (runtime, backend) = runtime_and_backend();
@@ -28,7 +28,19 @@ pub(super) struct Lease {
id: String, id: String,
} }
fn busy() -> WorkingDirectoryDiagnostic { fn busy(id: &str, action: &str, blocking: &Entry) -> WorkingDirectoryDiagnostic {
let (busy_reason, active_use_count) = match blocking {
Entry::Use(count) => ("active_use", *count),
Entry::Create => ("creation_in_progress", 0),
Entry::Cleanup => ("cleanup_in_progress", 0),
};
tracing::warn!(target: "yoi::workdir_cleanup",
working_directory_id = id,
action,
busy_source = "materializer_occupancy",
busy_reason,
active_use_count,
"Workdir operation blocked by busy resource");
WorkingDirectoryDiagnostic::new( WorkingDirectoryDiagnostic::new(
"working_directory_cleanup_resource_busy", "working_directory_cleanup_resource_busy",
"A filesystem resource is busy or changed during removal. Release the resource and retry.", "A filesystem resource is busy or changed during removal. Release the resource and retry.",
@@ -42,7 +54,9 @@ impl Occupancy {
.lock() .lock()
.unwrap_or_else(|error| error.into_inner()); .unwrap_or_else(|error| error.into_inner());
match entries.get_mut(id) { match entries.get_mut(id) {
Some(Entry::Cleanup | Entry::Create) => return Err(busy()), Some(blocking @ (Entry::Cleanup | Entry::Create)) => {
return Err(busy(id, "acquire_use", blocking));
}
Some(Entry::Use(count)) => *count += 1, Some(Entry::Use(count)) => *count += 1,
None => { None => {
entries.insert(id.to_string(), Entry::Use(1)); entries.insert(id.to_string(), Entry::Use(1));
@@ -101,8 +115,8 @@ impl Occupancy {
.entries .entries
.lock() .lock()
.unwrap_or_else(|error| error.into_inner()); .unwrap_or_else(|error| error.into_inner());
if entries.contains_key(id) { if let Some(blocking) = entries.get(id) {
return Err(busy()); return Err(busy(id, "acquire_creation", blocking));
} }
entries.insert(id.to_string(), Entry::Create); entries.insert(id.to_string(), Entry::Create);
Ok(Lease { Ok(Lease {
@@ -116,8 +130,8 @@ impl Occupancy {
.entries .entries
.lock() .lock()
.unwrap_or_else(|error| error.into_inner()); .unwrap_or_else(|error| error.into_inner());
if entries.contains_key(id) { if let Some(blocking) = entries.get(id) {
return Err(busy()); return Err(busy(id, "acquire_cleanup", blocking));
} }
entries.insert(id.to_string(), Entry::Cleanup); entries.insert(id.to_string(), Entry::Cleanup);
Ok(Lease { Ok(Lease {
@@ -131,6 +145,71 @@ impl Occupancy {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn busy_admission_logs_blocker_and_use_count_without_changing_error() {
use std::io::{Read, Seek, SeekFrom};
let mut output = tempfile::tempfile().unwrap();
let writer = output.try_clone().unwrap();
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_writer(move || writer.try_clone().unwrap())
.finish();
tracing::subscriber::with_default(subscriber, || {
tracing::callsite::rebuild_interest_cache();
let occupancy = Occupancy::default();
for (id, leases) in [
(
"workdir-used",
vec![
occupancy.acquire_use("workdir-used").unwrap(),
occupancy.acquire_use("workdir-used").unwrap(),
],
),
(
"workdir-creating",
vec![occupancy.acquire_creation("workdir-creating").unwrap()],
),
(
"workdir-removing",
vec![occupancy.acquire_cleanup("workdir-removing").unwrap()],
),
] {
let error = occupancy.acquire_cleanup(id).unwrap_err();
assert_eq!(error.code, "working_directory_cleanup_resource_busy");
assert_eq!(
error.message,
"A filesystem resource is busy or changed during removal. Release the resource and retry."
);
drop(leases);
drop(occupancy.acquire_cleanup(id).unwrap());
}
});
output.seek(SeekFrom::Start(0)).unwrap();
let mut log = String::new();
output.read_to_string(&mut log).unwrap();
let events: Vec<_> = log.lines().collect();
assert_eq!(events.len(), 3, "{log}");
for (event, (id, reason, count)) in events.iter().zip([
("workdir-used", "active_use", 2),
("workdir-creating", "creation_in_progress", 0),
("workdir-removing", "cleanup_in_progress", 0),
]) {
for field in [
"WARN".to_string(),
"yoi::workdir_cleanup".to_string(),
format!("working_directory_id=\"{id}\""),
"action=\"acquire_cleanup\"".to_string(),
"busy_source=\"materializer_occupancy\"".to_string(),
format!("busy_reason=\"{reason}\""),
format!("active_use_count={count}"),
] {
assert!(event.contains(&field), "missing {field}: {event}");
}
}
}
#[test] #[test]
fn use_and_cleanup_leases_release_during_unwind() { fn use_and_cleanup_leases_release_during_unwind() {
let occupancy = Occupancy::default(); let occupancy = Occupancy::default();
@@ -776,25 +776,39 @@ impl WorkspaceHttpWorkdirBackend {
ATTACH_TOOL, ATTACH_TOOL,
"working_directory_id", "working_directory_id",
)?; )?;
let alias_key = workdir::WorkdirAttachmentAlias::new(alias)
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
if self.session_router.session(&alias_key).is_some() {
return Err(ToolError::InvalidArgument(format!(
"attachment alias `{alias}` already exists"
)));
}
let response = self.attach_response(alias, working_directory_id)?; let response = self.attach_response(alias, working_directory_id)?;
if let Err(error) = self.session_router.attach( if !response.attachment.attached
alias_key.clone(), || response.attachment.working_directory_id != working_directory_id
{
return Err(ToolError::ExecutionFailed(
"Workdir attachment response does not match the target".into(),
));
}
// Backend ownership decides the name. A requested alias cannot rename
// an existing attachment or create a second route to the same Workdir.
let attached_alias = &response.attachment.alias;
let alias_key = workdir::WorkdirAttachmentAlias::new(attached_alias.clone())
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
if let Some(session) = self.session_router.session(&alias_key) {
if session.workdir().id().as_str() != working_directory_id {
return Err(ToolError::ExecutionFailed("Local Workdir attachment differs from Backend; refresh attachments before retrying".into()));
}
} else if let Err(error) = self.session_router.attach(
alias_key,
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment( WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
self.client.clone(), self.client.clone(),
alias, attached_alias,
working_directory_id, working_directory_id,
response.attachment.capabilities, response.attachment.capabilities,
response.environment.clone(), response.environment.clone(),
), ),
) { ) {
let rollback = self.detach(alias); if response.attachment.already_attached {
return Err(ToolError::ExecutionFailed(format!(
"attach Workdir session route: {error}; existing Backend attachment preserved"
)));
}
let rollback = self.detach(attached_alias);
return Err(ToolError::ExecutionFailed(match rollback { return Err(ToolError::ExecutionFailed(match rollback {
Ok(_) => { Ok(_) => {
format!("attach Workdir session route: {error}; Backend attachment rolled back") format!("attach Workdir session route: {error}; Backend attachment rolled back")
@@ -805,7 +819,11 @@ impl WorkspaceHttpWorkdirBackend {
})); }));
} }
workdir_output( workdir_output(
format!("Attached Workdir {working_directory_id} as {alias}"), if response.attachment.already_attached {
format!("Workdir {working_directory_id} is already attached as {attached_alias}")
} else {
format!("Attached Workdir {working_directory_id} as {attached_alias}")
},
&response, &response,
) )
} }
@@ -817,15 +835,31 @@ impl WorkspaceHttpWorkdirBackend {
) -> Result<WorkdirSessionAttachmentResponse, ToolError> { ) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
let workspace_id = encode_path_segment(self.workspace_id()?); let workspace_id = encode_path_segment(self.workspace_id()?);
self.authorize_workdir("workdir.attach", working_directory_id)?; self.authorize_workdir("workdir.attach", working_directory_id)?;
self.execute_json::<WorkdirSessionAttachmentResponse>(WorkspaceRequest::json( let response = self
WorkspaceRequestMethod::Post, .client
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"), .execute(WorkspaceRequest::json(
serde_json::to_string(&WorkdirAttachRequest { WorkspaceRequestMethod::Post,
alias: alias.to_string(), format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
working_directory_id: working_directory_id.to_string(), serde_json::to_string(&WorkdirAttachRequest {
}) alias: alias.to_string(),
.map_err(decode_error)?, working_directory_id: working_directory_id.to_string(),
)) })
.map_err(decode_error)?,
))
.map_err(|error| {
ToolError::ExecutionFailed(format!("Workspace Workdir request failed: {error}"))
})?;
if response.status == 409
&& let Ok(error) =
serde_json::from_str::<server_api::RepositoryApiError>(&response.body)
&& error.error == "workdir_occupied"
{
return Err(ToolError::StructuredConflict {
code: "workdir_occupied".into(),
message: "Workdir is occupied by another Worker".into(),
});
}
decode_response(response)
} }
fn detach(&self, alias: &str) -> Result<ToolOutput, ToolError> { fn detach(&self, alias: &str) -> Result<ToolOutput, ToolError> {
@@ -860,7 +894,11 @@ impl WorkspaceHttpWorkdirBackend {
) )
} }
fn delete(&self, input: WorkdirDeleteInput) -> Result<ToolOutput, ToolError> { fn delete(
&self,
input: WorkdirDeleteInput,
operation_id: String,
) -> Result<ToolOutput, ToolError> {
let workdir_id = validate_identity( let workdir_id = validate_identity(
&input.working_directory_id, &input.working_directory_id,
DELETE_TOOL, DELETE_TOOL,
@@ -874,6 +912,7 @@ impl WorkspaceHttpWorkdirBackend {
WorkspaceRequestMethod::Delete, WorkspaceRequestMethod::Delete,
format!("/api/w/{workspace_id}/working-directories/{workdir_path}"), format!("/api/w/{workspace_id}/working-directories/{workdir_path}"),
serde_json::to_string(&WorkdirRemovalRequest { serde_json::to_string(&WorkdirRemovalRequest {
operation_id,
reason: reason.to_string(), reason: reason.to_string(),
}) })
.map_err(decode_error)?, .map_err(decode_error)?,
@@ -884,7 +923,7 @@ impl WorkspaceHttpWorkdirBackend {
response.disposition, response.disposition,
removal_guidance(response.failure_category.as_deref()), removal_guidance(response.failure_category.as_deref()),
if response.retryable { if response.retryable {
" After resolving the cause, repeat this WorkdirDelete with the same id and reason; current safety conditions will be rechecked." " After resolving the cause, a new explicit WorkdirDelete request will recheck current safety conditions. This failed request will not be automatically re-executed."
} else { } else {
"" ""
} }
@@ -1047,9 +1086,10 @@ impl Tool for WorkspaceHttpWorkdirTool {
self.backend.detach_managed(&input.alias, None).await self.backend.detach_managed(&input.alias, None).await
} }
WorkdirOperation::Delete => self WorkdirOperation::Delete => self.backend.delete(
.backend parse_input::<WorkdirDeleteInput>(input_json)?,
.delete(parse_input::<WorkdirDeleteInput>(input_json)?), ctx.call_id.to_string(),
),
} }
} }
} }
@@ -1111,8 +1151,14 @@ fn removal_guidance(category: Option<&str>) -> &'static str {
Some("runtime_unavailable" | "provider_unavailable") => { Some("runtime_unavailable" | "provider_unavailable") => {
"Restore the Runtime connection; the registry remains until deletion is confirmed." "Restore the Runtime connection; the registry remains until deletion is confirmed."
} }
Some("provider_observation_unknown" | "provider_cleanup_outcome_unknown") => { Some("provider_cleanup_outcome_unknown") => {
"Deletion is not confirmed. Re-observe with normal removal after restoring provider access; the registry is retained." "Deletion outcome is unknown. Restore provider access and reconcile the existing request; do not submit a fresh deletion intent. Recovery only observes the target and never repeats deletion."
}
Some("provider_observation_unknown") => {
"The target could not be observed before deletion. Restore provider access before a new request."
}
Some("request_interrupted") => {
"The request was interrupted before execution. A new explicit request is required."
} }
Some("unsupported_target") => { Some("unsupported_target") => {
"Ask the owner for the provider's supported cleanup operation." "Ask the owner for the provider's supported cleanup operation."
@@ -1312,6 +1358,8 @@ struct WorkdirAttachmentResponse {
working_directory_id: String, working_directory_id: String,
capabilities: WorkdirSessionCapabilities, capabilities: WorkdirSessionCapabilities,
attached: bool, attached: bool,
#[serde(default)]
already_attached: bool,
} }
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
@@ -1467,8 +1515,44 @@ mod tests {
}) })
} }
#[tokio::test]
async fn delete_call_identity_distinguishes_new_requests_from_exact_resends() {
let client = Arc::new(RecordingWorkspaceClient::new(
(0..3)
.map(|_| {
response(json!({
"working_directory_id": "wd-request",
"disposition": "attention_required",
"retryable": true,
"failure_category": "resource_busy"
}))
})
.collect(),
));
let tool = WorkspaceHttpWorkdirTool {
backend: WorkspaceHttpWorkdirBackend::new(client.clone()),
operation: WorkdirOperation::Delete,
};
let input = r#"{"working_directory_id":"wd-request","reason":"remove unused Workdir"}"#;
for id in ["delete-1", "delete-1", "delete-2"] {
tool.execute(input, ToolExecutionContext::new(id, "batch", 0))
.await
.unwrap();
}
let bodies: Vec<serde_json::Value> = client
.requests()
.iter()
.map(|r| serde_json::from_str(r.body.as_deref().unwrap()).unwrap())
.collect();
assert_eq!(bodies[0], bodies[1]);
assert_eq!(bodies[0]["operation_id"], "delete-1");
assert_eq!(bodies[2]["operation_id"], "delete-2");
assert_eq!(bodies[0]["reason"], bodies[2]["reason"]);
assert!(delete_schema()["properties"].get("operation_id").is_none());
}
#[test] #[test]
fn ordinary_workdir_delete_reports_safe_cause_and_exact_retry_guidance() { fn ordinary_workdir_delete_reports_safe_cause_and_new_request_guidance() {
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({ let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
"working_directory_id": "wd-retry", "working_directory_id": "wd-retry",
"disposition": "attention_required", "disposition": "attention_required",
@@ -1477,10 +1561,13 @@ mod tests {
}))])); }))]));
let backend = WorkspaceHttpWorkdirBackend::new(client.clone()); let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
let output = backend let output = backend
.delete(WorkdirDeleteInput { .delete(
working_directory_id: "wd-retry".into(), WorkdirDeleteInput {
reason: "remove unused Workdir".into(), working_directory_id: "wd-retry".into(),
}) reason: "remove unused Workdir".into(),
},
"delete-call-1".to_string(),
)
.unwrap(); .unwrap();
assert!(output.summary.contains("Confirm its owner")); assert!(output.summary.contains("Confirm its owner"));
assert!( assert!(
@@ -1488,7 +1575,11 @@ mod tests {
.summary .summary
.contains("Do not recursively delete mounted data") .contains("Do not recursively delete mounted data")
); );
assert!(output.summary.contains("same id and reason")); assert!(
output
.summary
.contains("new explicit WorkdirDelete request")
);
assert!(output.summary.len() <= 600); assert!(output.summary.len() <= 600);
let content: serde_json::Value = let content: serde_json::Value =
serde_json::from_str(output.content.as_deref().unwrap()).unwrap(); serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
@@ -1726,10 +1817,13 @@ mod tests {
.unwrap(); .unwrap();
backend.detach("checkout").unwrap(); backend.detach("checkout").unwrap();
backend backend
.delete(WorkdirDeleteInput { .delete(
working_directory_id: "wd-created".to_string(), WorkdirDeleteInput {
reason: "remove stale Workdir".to_string(), working_directory_id: "wd-created".to_string(),
}) reason: "remove stale Workdir".to_string(),
},
"delete-call-1".to_string(),
)
.unwrap(); .unwrap();
let requests = client.requests(); let requests = client.requests();
@@ -1767,7 +1861,102 @@ mod tests {
assert_eq!(requests[4].method, WorkspaceRequestMethod::Delete); assert_eq!(requests[4].method, WorkspaceRequestMethod::Delete);
let body: serde_json::Value = let body: serde_json::Value =
serde_json::from_str(requests[4].body.as_deref().unwrap()).unwrap(); serde_json::from_str(requests[4].body.as_deref().unwrap()).unwrap();
assert_eq!(body, json!({"reason": "remove stale Workdir"})); assert_eq!(
body,
json!({"reason": "remove stale Workdir", "operation_id": "delete-call-1"})
);
}
#[test]
fn attach_reports_other_worker_occupancy_without_exposing_backend_details() {
let client = Arc::new(RecordingWorkspaceClient::new(vec![WorkspaceResponse {
status: 409,
body: serde_json::to_string(&server_api::RepositoryApiError::new(
409,
"workdir_occupied",
"private backend detail",
Vec::new(),
))
.unwrap(),
}]));
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
let error = backend
.attach(WorkdirAttachInput {
alias: "work".into(),
working_directory_id: "occupied".into(),
})
.unwrap_err();
assert!(
matches!(error, ToolError::StructuredConflict { code, message } if code == "workdir_occupied" && message == "Workdir is occupied by another Worker")
);
assert_eq!(client.requests().len(), 1);
}
#[test]
fn attach_reuses_backend_alias_without_duplicate_routes_or_detach() {
let client = Arc::new(RecordingWorkspaceClient::new((0..3).map(|_| response(json!({
"workspace_id": "workspace/test", "alias": "existing", "working_directory_id": "wd-existing",
"capabilities": {"bits": 63}, "attached": true, "already_attached": true,
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL)
}))).collect()));
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
let occupied_alias = workdir::WorkdirAttachmentAlias::new("other").unwrap();
backend
.session_router
.attach(
occupied_alias.clone(),
WorkspaceAttachedWorkdirSession::handle_for_workdir(
client.clone(),
"other",
"wd-other",
),
)
.unwrap();
for alias in ["new-name", "existing", "other"] {
let output = backend
.attach(WorkdirAttachInput {
alias: alias.into(),
working_directory_id: "wd-existing".into(),
})
.unwrap();
assert_eq!(
output.summary,
"Workdir wd-existing is already attached as existing"
);
let body: serde_json::Value =
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
assert_eq!(body["alias"], "existing");
assert_eq!(body["already_attached"], true);
}
assert!(
backend
.session_router
.session(&workdir::WorkdirAttachmentAlias::new("new-name").unwrap())
.is_none()
);
assert_eq!(
backend
.session_router
.session(&occupied_alias)
.unwrap()
.workdir()
.id()
.as_str(),
"wd-other"
);
assert!(
backend
.session_router
.session(&workdir::WorkdirAttachmentAlias::new("existing").unwrap())
.is_some()
);
assert_eq!(client.requests().len(), 3);
assert!(
client
.requests()
.iter()
.all(|request| request.method == WorkspaceRequestMethod::Post)
);
} }
#[tokio::test] #[tokio::test]
@@ -2377,10 +2566,13 @@ mod tests {
); );
assert!( assert!(
backend backend
.delete(WorkdirDeleteInput { .delete(
working_directory_id: "wd".into(), WorkdirDeleteInput {
reason: "remove unused".into() working_directory_id: "wd".into(),
}) reason: "remove unused".into(),
},
"delete-intent".into(),
)
.is_err() .is_err()
); );
assert!(backend.detach_managed("checkout", None).await.is_err()); assert!(backend.detach_managed("checkout", None).await.is_err());
@@ -914,7 +914,7 @@ impl WipOperationHandler for ItemHandler {
let mut result = raw; let mut result = raw;
result["workdir_path"] = json!(item_path(Domain::Workdir, &self.id)); result["workdir_path"] = json!(item_path(Domain::Workdir, &self.id));
result["attachments_path"] = json!(ATTACHMENTS); result["attachments_path"] = json!(ATTACHMENTS);
result["attachment"] = input result["attachment"] = result
.get("alias") .get("alias")
.and_then(Json::as_str) .and_then(Json::as_str)
.and_then(|alias| self.provider.attached_reference(alias, &self.id)) .and_then(|alias| self.provider.attached_reference(alias, &self.id))
@@ -1421,10 +1421,19 @@ pub(crate) mod tests {
json!({"workspace_id":"test-workspace", "runtime_id":"runtime", "item":raw, "diagnostics":[]}) json!({"workspace_id":"test-workspace", "runtime_id":"runtime", "item":raw, "diagnostics":[]})
} }
(WorkspaceRequestMethod::Post, "workers/self/workdir-attachments") => { (WorkspaceRequestMethod::Post, "workers/self/workdir-attachments") => {
state.generation += 1; let existing = state
let raw = json!({"alias":body["alias"], "working_directory_id":body["working_directory_id"], "connection_id":format!("connection-{}", state.generation), "capabilities":{"bits":63}}); .attachments
state.attachments.push(raw); .iter()
json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "environment":workdir::WorkdirSessionEnvironment::root(workdir::WorkdirSessionCapabilities::ALL), "attached":true}) .find(|raw| raw["working_directory_id"] == body["working_directory_id"])
.cloned();
let already_attached = existing.is_some();
let raw = existing.unwrap_or_else(|| {
state.generation += 1;
let raw = json!({"alias":body["alias"], "working_directory_id":body["working_directory_id"], "connection_id":format!("connection-{}", state.generation), "capabilities":{"bits":63}});
state.attachments.push(raw.clone());
raw
});
json!({"workspace_id":"test-workspace", "alias":raw["alias"], "working_directory_id":raw["working_directory_id"], "capabilities":{"bits":63}, "environment":workdir::WorkdirSessionEnvironment::root(workdir::WorkdirSessionCapabilities::ALL), "attached":true, "already_attached":already_attached})
} }
(WorkspaceRequestMethod::Delete, p) (WorkspaceRequestMethod::Delete, p)
if p.starts_with("workers/self/workdir-attachments/") => if p.starts_with("workers/self/workdir-attachments/") =>
@@ -413,7 +413,16 @@ async fn model_repository_create_attach_selected_entries_read_and_detach_preserv
.invoke(&workdir, "attach", json!({"alias":"checkout"})) .invoke(&workdir, "attach", json!({"alias":"checkout"}))
.await; .await;
assert_eq!(attached["attached"], true); assert_eq!(attached["attached"], true);
assert_eq!(attached["already_attached"], false);
assert_eq!(attached["attachment"]["$case"], "available"); assert_eq!(attached["attachment"]["$case"], "available");
let (reused, reused_entries) = f
.invoke(&workdir, "attach", json!({"alias":"another-name"}))
.await;
assert_eq!(reused["already_attached"], true);
assert_eq!(reused["alias"], "checkout");
assert_eq!(reused["attachment"], attached["attachment"]);
assert_eq!(reused_entries, entries);
assert_eq!(f.client.catalog.state.lock().unwrap().attachments.len(), 1);
assert_eq!( assert_eq!(
f.writes("workdir-session/operations"), f.writes("workdir-session/operations"),
0, 0,
@@ -432,7 +441,7 @@ async fn model_repository_create_attach_selected_entries_read_and_detach_preserv
.unwrap() .unwrap()
.contains("isolated content") .contains("isolated content")
); );
assert_eq!(f.writes("workdir-attachments"), 1); assert_eq!(f.writes("workdir-attachments"), 2);
f.invoke(&attachment, "detach", json!({})).await; f.invoke(&attachment, "detach", json!({})).await;
assert!( assert!(
f.execute("Inspect", json!({"path":attachment,"refresh":true})) f.execute("Inspect", json!({"path":attachment,"refresh":true}))
@@ -132,6 +132,7 @@ pub(super) fn attached() -> TypeExpr {
field("capabilities", true, TypeExpr::Json), field("capabilities", true, TypeExpr::Json),
field("environment", true, TypeExpr::Json), field("environment", true, TypeExpr::Json),
field("attached", true, TypeExpr::Boolean), field("attached", true, TypeExpr::Boolean),
field("already_attached", true, TypeExpr::Boolean),
field("workdir_path", true, TypeExpr::Entry), field("workdir_path", true, TypeExpr::Entry),
field("attachments_path", true, TypeExpr::Entry), field("attachments_path", true, TypeExpr::Entry),
field("attachment", true, TypeExpr::Union { cases: vec![ field("attachment", true, TypeExpr::Union { cases: vec![
+6
View File
@@ -144,6 +144,12 @@ pub enum Error {
TicketAssignmentConflict(String), TicketAssignmentConflict(String),
#[error("Workdir attachment conflict: {0}")] #[error("Workdir attachment conflict: {0}")]
WorkdirAttachmentConflict(String), WorkdirAttachmentConflict(String),
#[error("Workdir {workdir_id} is already attached to Worker {runtime_id}:{worker_id}")]
WorkdirOccupied {
workdir_id: String,
runtime_id: String,
worker_id: String,
},
#[error("Workspace permission denied: {0}")] #[error("Workspace permission denied: {0}")]
WorkspacePermissionDenied(String), WorkspacePermissionDenied(String),
#[error("Workspace config update conflict: {0}")] #[error("Workspace config update conflict: {0}")]
+24 -4
View File
@@ -325,7 +325,7 @@ impl SqliteWorkspaceStore {
let now = Utc::now().to_rfc3339(); let now = Utc::now().to_rfc3339();
self.with_conn_mut(|conn| { self.with_conn_mut(|conn| {
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?; let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal,true)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? { let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
Some(v)=>v, Some(v)=>v,
None=>{ None=>{
@@ -385,7 +385,7 @@ impl SqliteWorkspaceStore {
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));} if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
let reconciliation=crate::store::subject_deletion_removal_matches(&tx,workspace_id,&plan.worker,&plan.operation_id,&plan.input_fingerprint)?; let reconciliation=crate::store::subject_deletion_removal_matches(&tx,workspace_id,&plan.worker,&plan.operation_id,&plan.input_fingerprint)?;
if !reconciliation { if !reconciliation {
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?; let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID,plan.policy_id=="backend-subject-removal-archive-v1")?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{ if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?; mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
return Err(stale_error(&plan,"policy content changed")); return Err(stale_error(&plan,"policy content changed"));
@@ -1068,6 +1068,7 @@ fn load_removal_policy(
w: &str, w: &str,
worker: &RuntimeWorkerRef, worker: &RuntimeWorkerRef,
erased_journal: bool, erased_journal: bool,
preserve_subject_history: bool,
) -> crate::Result<Option<WorkerRetentionPolicy>> { ) -> crate::Result<Option<WorkerRetentionPolicy>> {
let Some(mut policy) = load_policy(c, w)? else { let Some(mut policy) = load_policy(c, w)? else {
return Ok(None); return Ok(None);
@@ -1135,6 +1136,25 @@ fn load_removal_policy(
policy.archive_retention = update.archive_retention; policy.archive_retention = update.archive_retention;
policy.diagnostics_disposition = update.diagnostics_disposition; policy.diagnostics_disposition = update.diagnostics_disposition;
policy.diagnostics_retention_seconds = None; policy.diagnostics_retention_seconds = None;
} else if preserve_subject_history && policy.session_disposition == SessionDisposition::Purge && c.query_row(
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND substr(singleton_key,1,10)='subjektiv:')",
params![w, worker.runtime_id, worker.worker_id],
|r| r.get::<_, bool>(0),
)? {
// Removing a Subject body releases occupancy, not its original history.
// A configured Purge must not silently erase an unfinished Journal. Explicit
// Session erasure uses the separate, proof-bound branch above.
let update = WorkerRetentionPolicyUpdate {
policy_id: "backend-subject-removal-archive-v1".into(),
session_disposition: SessionDisposition::Archive,
metadata_disposition: policy.metadata_disposition,
archive_retention: policy.archive_retention,
diagnostics_disposition: policy.diagnostics_disposition,
diagnostics_retention_seconds: policy.diagnostics_retention_seconds,
};
policy.policy_id = update.policy_id.clone();
policy.policy_digest = worker_retention_policy_digest(&update);
policy.session_disposition = update.session_disposition;
} }
Ok(Some(policy)) Ok(Some(policy))
} }
@@ -1167,7 +1187,7 @@ fn require_current_subject_deletion_plan(
return Ok(()); return Ok(());
} }
if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() { if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() {
let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false)? let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false, true)?
.ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?; .ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?;
if plan.session_disposition != current.session_disposition if plan.session_disposition != current.session_disposition
|| plan.metadata_disposition != current.metadata_disposition || plan.metadata_disposition != current.metadata_disposition
@@ -2934,7 +2954,7 @@ mod tests {
.is_err() .is_err()
); );
let fresh = store let fresh = store
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false)) .with_conn(|c| load_removal_policy(c, "w", &req().worker, false, true))
.unwrap() .unwrap()
.unwrap(); .unwrap();
assert_eq!(fresh.session_disposition, SessionDisposition::Purge); assert_eq!(fresh.session_disposition, SessionDisposition::Purge);
File diff suppressed because it is too large Load Diff
@@ -101,6 +101,20 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) -
authorize_source(api, &attribution, "journal.job.input.read")?; authorize_source(api, &attribution, "journal.job.input.read")?;
authorize_source(api, &attribution, "journal.job.input.supply")?; authorize_source(api, &attribution, "journal.job.input.supply")?;
let host = host(api, &store, grant)?; let host = host(api, &store, grant)?;
let attr = store
.session_attribution(&grant.session_id)
.map_err(|e| Error::Store(e.to_string()))?
.ok_or_else(|| {
Error::WorkspacePermissionDenied("Journal Session attribution missing".into())
})?;
if store
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?
{
return Err(Error::WorkspacePermissionDenied(
"released Journal body cannot dispatch another attempt".into(),
));
}
let input = saved_input(&reservation.job.request, grant)?; let input = saved_input(&reservation.job.request, grant)?;
let g = store let g = store
.prepare_journal_input( .prepare_journal_input(
@@ -640,6 +654,18 @@ fn dispatch(
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
authorize_source(api, &attribution, "journal.recording.request")?; authorize_source(api, &attribution, "journal.recording.request")?;
let store = store(api)?; let store = store(api)?;
if store
.journal_worker_was_released(
&attribution.subject_id,
&attribution.runtime_id,
&attribution.worker_id,
)
.map_err(domain)?
{
// Deletion can retain an unfinished prefix. It must not mint attempts or
// transfer the removed body's write capability to its successor.
return Ok(());
}
let host = let host =
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?; JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
if let Some(g) = store if let Some(g) = store
@@ -1823,8 +1849,8 @@ fn finish_erased_release(
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation) .release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
} }
/// Normal deletion cannot substitute for Subject release, even for a stopped or disabled body. /// Ordinary removal owns writer revocation and occupancy release. Before execution
/// Explicit erasure remains a separate monotonic source/diagnostics destruction path. /// only preserve/erase source requirements are checked; an active writer is not a blocker.
pub(super) fn prepare_worker_removal( pub(super) fn prepare_worker_removal(
service: &WorkerRemovalService, service: &WorkerRemovalService,
runtime: &RuntimeRegistry, runtime: &RuntimeRegistry,
@@ -1852,7 +1878,6 @@ pub(super) fn prepare_worker_removal(
&service.journal_registration, &service.journal_registration,
) )
.map_err(|e| Error::Store(e.to_string()))?; .map_err(|e| Error::Store(e.to_string()))?;
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
let source = runtime_api::SessionPublicSource::Retained { let source = runtime_api::SessionPublicSource::Retained {
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id) worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?, .ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
@@ -1869,25 +1894,9 @@ pub(super) fn prepare_worker_removal(
finish_erased_release(&api, &attr)?; finish_erased_release(&api, &attr)?;
return Ok(Some(proof)); return Ok(Some(proof));
} }
let Some(binding_id) = host.writer_binding_id() else { // Do not require a live Controller, complete Journal, or prior release here.
return Err(Error::RepositoryConflict( // Runtime removal is the physical fence; finish_worker_removal revokes the
"Subject writer evidence is missing; ordinary deletion cannot bypass release".into(), // exact old body only after its successful, matching retention receipt.
));
};
let writer = JournalWriterBinding {
subject_id: attr.subject_id.clone(),
runtime_id: attr.runtime_id.clone(),
worker_id: attr.worker_id.clone(),
binding_id: binding_id.into(),
};
if store.journal_writer_active(&writer).map_err(domain)? {
return Err(Error::RepositoryConflict("Subject remains occupied: restore to actual Idle, finish Journal recording, then explicitly release before ordinary deletion".into()));
}
if store.journal_retention_required(&host).map_err(domain)? {
return Err(Error::RepositoryConflict(
"Released Subject has unresolved Journal material; retain occupancy and source".into(),
));
}
Ok(None) Ok(None)
} }
@@ -1990,20 +1999,6 @@ pub(super) fn validate_removal_retention(
// its retained permanent fence, including recovery after the aggregate is gone. // its retained permanent fence, including recovery after the aggregate is gone.
return Ok(()); return Ok(());
} }
let binding_id = host.writer_binding_id().ok_or_else(|| {
Error::RepositoryConflict("Subject writer evidence missing during removal recovery".into())
})?;
let binding = JournalWriterBinding {
subject_id: attr.subject_id.clone(),
runtime_id: attr.runtime_id.clone(),
worker_id: attr.worker_id.clone(),
binding_id: binding_id.into(),
};
if store.journal_writer_active(&binding).map_err(domain)? {
return Err(Error::RepositoryConflict(
"Prepared removal cannot bypass explicit Subject release".into(),
));
}
let source = runtime_api::SessionPublicSource::Retained { let source = runtime_api::SessionPublicSource::Retained {
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id) worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?, .ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
@@ -2011,37 +2006,160 @@ pub(super) fn validate_removal_retention(
let runtime = service.runtime.upgrade().ok_or_else(|| { let runtime = service.runtime.upgrade().ok_or_else(|| {
Error::Store("Runtime unavailable for source retention confirmation".into()) Error::Store("Runtime unavailable for source retention confirmation".into())
})?; })?;
if store.journal_retention_required(&host).map_err(domain)? { let retention_required = store.journal_retention_required(&host).map_err(domain)?;
if prepared.runtime_request.session_disposition if prepared.runtime_request.session_disposition
!= worker_runtime::retention::SessionDisposition::Archive == worker_runtime::retention::SessionDisposition::Purge
|| prepared.runtime_request.archive_id.is_none() && prepared.runtime_request.archive_id.is_none()
{ && !retention_required
return Err(Error::RepositoryConflict( && store
"unreflected Journal requires a retained original Session archive".into(), .journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
)); .map_err(domain)?
{
// Compatibility for an already issued, history-complete ordinary Purge
// plan. New Subject plans always Archive. Keep the old fingerprint and
// disposition; never turn this into an erasure or allow an active writer.
// A missing Runtime Worker is NOT success: execute_worker_retention still
// has to return the exact persisted receipt before anything can commit.
// A pending receipt may outlive the physical aggregate while Runtime's
// in-memory catalog still contains the Worker. Pin release is best effort
// in this historical branch; exact Runtime retention replay still enforces
// pins if the source exists, and requires a receipt if it is already gone.
if let Err(error) = retain_source(&runtime, &service.workspace_id, &attr, source, true) {
tracing::debug!(%error, "Historical Purge source unavailable; reconcile exact Runtime receipt");
} }
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, false) return Ok(());
{ }
// Recovery may already have transferred the physical source before its Backend if prepared.runtime_request.session_disposition
// catalog commit; authenticate the exact reserved archive, never another Session. != worker_runtime::retention::SessionDisposition::Archive
let archive = runtime_api::SessionPublicSource::Archived { || prepared.runtime_request.archive_id.is_none()
archive_id: prepared {
.runtime_request return Err(Error::RepositoryConflict(
.archive_id "Subject removal retains its original Session; explicit erasure is required for Purge"
.clone() .into(),
.expect("checked"), ));
}; }
retain_source(&runtime, &service.workspace_id, &attr, archive, false) let release = !retention_required;
.map_err(|_| original)?; if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, release) {
} // A lost reply may follow the physical transfer. Reconcile only the exact
} else { // reserved archive and Session, whether or not an admitted remainder exists.
// This stopped/removal-fenced Session has no admitted remainder; purge need not let archive = runtime_api::SessionPublicSource::Archived {
// force a generation. A failed release conservatively blocks ordinary deletion. archive_id: prepared
retain_source(&runtime, &service.workspace_id, &attr, source, true)?; .runtime_request
.archive_id
.clone()
.expect("checked"),
};
retain_source(&runtime, &service.workspace_id, &attr, archive, release)
.map_err(|_| original)?;
} }
Ok(()) Ok(())
} }
/// A successful Runtime receipt, not catalog liveness, authorizes ending the old
/// writer. The ordinary removal commit releases occupancy atomically with catalog
/// deletion. Until then a crash leaves the singleton occupied and the same receipt
/// can safely repeat this step; no Controller restore or Journal generation is needed.
pub(super) fn finish_worker_removal<T>(
service: &WorkerRemovalService,
target: &RuntimeWorkerRef,
prepared: &crate::retention::PreparedWorkerRemoval,
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
commit: impl FnOnce() -> T,
) -> Result<T> {
if let Some(api) = service.journal_dispatch_api.upgrade()
&& subjektiv_deletion::deleting_worker(&api, target)?
{
// Full Subject deletion already fenced the domain. Its canonical retention
// commit, not ordinary Journal writer validation, owns completion.
return Ok(commit());
}
let Some(attr) = removal_attribution(service, target)? else {
return Ok(commit());
};
let request = &prepared.runtime_request;
if prepared.plan.worker != *target
|| request.workspace_id != service.workspace_id
|| request.source_runtime_id != target.runtime_id
|| request.worker_id.to_string() != target.worker_id
|| !result.source_removed
|| result.worker_id != request.worker_id
|| result.operation_id != request.operation_id
|| result.input_fingerprint != request.input_fingerprint
|| result.session_disposition != request.session_disposition
|| result.diagnostics_disposition != request.diagnostics_disposition
|| result.diagnostics_retained
!= (request.diagnostics_disposition
== worker_runtime::retention::DiagnosticsDisposition::Retain)
{
return Err(Error::RepositoryConflict(
"Subject removal receipt does not match its plan".into(),
));
}
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
let archive = result.archive.as_ref().ok_or_else(|| {
Error::RepositoryConflict("Subject removal archive receipt is missing".into())
})?;
if Some(&archive.archive_id) != request.archive_id.as_ref()
|| archive.workspace_id != request.workspace_id
|| archive.source_runtime_id != request.source_runtime_id
|| archive.source_worker_id != request.worker_id
|| archive.source_session_id != attr.session_id
|| archive.operation_id != request.operation_id
|| archive.input_fingerprint != request.input_fingerprint
|| archive.policy_id != request.policy_id
{
return Err(Error::RepositoryConflict(
"Subject removal archive identity mismatch".into(),
));
}
}
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
Error::Store("Workspace API unavailable for Subject removal completion".into())
})?;
let gate = lifecycle_lock(&api, &attr.subject_id)?;
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
let store = store(&api)?;
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
if store.journal_session_erased(&host).map_err(domain)? {
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
// Erasure may win after Runtime archive transfer. Do not publish that
// archive or skip writer revocation: the same removal retry must take
// erased_archive_removal_proof and its physical-erasure commit path.
return Err(Error::RepositoryConflict(
"Session erased during removal; reconcile the same removal operation".into(),
));
}
return Ok(commit());
}
store.release_removed_journal_body(&host).map_err(domain)?;
// Result delivery takes the same Subject gate. Revocation is durable; release
// the gate while reconciling jobs, then recheck erasure before publishing.
drop(_guard);
for session in store
.subject_session_attributions(&attr.subject_id)
.map_err(|e| Error::Store(e.to_string()))?
.into_iter()
.filter(|a| a.runtime_id == target.runtime_id && a.worker_id == target.worker_id)
{
reconcile_session_jobs(&api, &session, false)?;
}
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
if store.journal_session_erased(&host).map_err(domain)?
&& request.session_disposition == worker_runtime::retention::SessionDisposition::Archive
{
return Err(Error::RepositoryConflict(
"Session erased during removal; reconcile the same removal operation".into(),
));
}
// Keep erasure excluded through the catalog commit: otherwise a tombstone
// could win between revocation and publication of the retained archive.
Ok(commit())
}
#[derive(Deserialize)] #[derive(Deserialize)]
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
pub(super) struct Empty {} pub(super) struct Empty {}
@@ -2196,7 +2314,28 @@ pub(super) fn reconcile_unknown(
.as_ref() .as_ref()
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?; .ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
let store = store(api)?; let store = store(api)?;
let host = host(api, &store, grant)?; let attr = store
.session_attribution(&grant.session_id)
.map_err(|e| Error::Store(e.to_string()))?
.filter(|a| a.subject_id == grant.subject_id)
.ok_or_else(|| Error::WorkspacePermissionDenied("Journal attribution mismatch".into()))?;
let released = store
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?;
let host = if released {
// Recovery reads immutable evidence, never renews a legacy writer grant.
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
if !grant.binding_id.is_empty()
&& host.writer_binding_id() != Some(grant.binding_id.as_str())
{
return Err(Error::WorkspacePermissionDenied(
"Journal recovery binding mismatch".into(),
));
}
host
} else {
host(api, &store, grant)?
};
let erased = store.journal_session_erased(&host).map_err(domain)?; let erased = store.journal_session_erased(&host).map_err(domain)?;
if !erased { if !erased {
job.load_executable_request()?; job.load_executable_request()?;
@@ -2204,70 +2343,93 @@ pub(super) fn reconcile_unknown(
job_id: job.request.job_id.clone(), job_id: job.request.job_id.clone(),
attempt_id: attempt.attempt_id.clone(), attempt_id: attempt.attempt_id.clone(),
}; };
let g = if legacy_descriptor(&job.request) { let generation = if released {
// Only reconcile already-saved legacy receipts; never redispatch descriptors.
store store
.prepare_journal_generation( .retained_journal_generation(&host, &grant.journal_id, &binding)
&host,
&grant.journal_id,
binding,
grant.selected_ranges.clone(),
)
.map_err(domain)? .map_err(domain)?
} else if legacy_descriptor(&job.request) {
Some(
store
.prepare_journal_generation(
&host,
&grant.journal_id,
binding,
grant.selected_ranges.clone(),
)
.map_err(domain)?,
)
} else { } else {
let input = saved_input(&job.request, grant)?; // Preserve active-body recovery: its saved intent can still establish
let generation = store // the original generation fence. A released body must only read it.
.prepare_journal_input(&host, &input, binding) Some(
.map_err(domain)?; store
input.validate_generation(&generation).map_err(domain)?; .prepare_journal_input(&host, &saved_input(&job.request, grant)?, binding)
generation .map_err(domain)?,
)
}; };
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone()) if let Some(g) = generation {
.map_err(domain)?; if g.selected_ranges != grant.selected_ranges {
match journal::service::finish(&store, &ctx) { return Err(Error::WorkspacePermissionDenied(
Ok(completion) => { "Journal recovery selection mismatch".into(),
let submission = BackendJobResultSubmission { ));
job_id: job.request.job_id.clone(), }
attempt_id: attempt.attempt_id.clone(), if !legacy_descriptor(&job.request) {
input_digest: attempt.input_digest.clone(), saved_input(&job.request, grant)?
result: serde_json::to_value(completion) .validate_generation(&g)
.map_err(|e| Error::Store(e.to_string()))?, .map_err(domain)?;
}; }
let current = api.store.reconcile_backend_journal_attempt( let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
api.workspace_id(), .map_err(domain)?;
attempt, match journal::service::finish(&store, &ctx) {
Some(&submission), Ok(completion) => {
&Utc::now().to_rfc3339(), let submission = BackendJobResultSubmission {
)?; job_id: job.request.job_id.clone(),
let saved = api attempt_id: attempt.attempt_id.clone(),
.store input_digest: attempt.input_digest.clone(),
.get_backend_job(api.workspace_id(), &job.request.job_id)? result: serde_json::to_value(completion)
.ok_or_else(|| Error::Store("reconciled Job missing".into()))?; .map_err(|e| Error::Store(e.to_string()))?,
notification_delivery::deliver_job_result( };
api, let current = api.store.reconcile_backend_journal_attempt(
&BackendJobResultAcceptance { api.workspace_id(),
job: saved, attempt,
attempt: current.clone(), Some(&submission),
replayed: false, &Utc::now().to_rfc3339(),
}, )?;
); let saved = api
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) { .store
drain_after_completion(api, &job.request); .get_backend_job(api.workspace_id(), &job.request.job_id)?
if job.input_disposition == crate::backend_job::JobInputDisposition::Available { .ok_or_else(|| Error::Store("reconciled Job missing".into()))?;
api.store.reclaim_backend_job_input( notification_delivery::deliver_job_result(
api.workspace_id(), api,
&job.request.job_id, &BackendJobResultAcceptance {
&job.input_digest, job: saved,
crate::backend_job::JobInputRetirementReason::Completed, attempt: current.clone(),
)?; replayed: false,
},
);
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed)
{
if !released {
drain_after_completion(api, &job.request);
}
if job.input_disposition
== crate::backend_job::JobInputDisposition::Available
{
api.store.reclaim_backend_job_input(
api.workspace_id(),
&job.request.job_id,
&job.input_digest,
crate::backend_job::JobInputRetirementReason::Completed,
)?;
}
} else {
api.schedule_backend_job_worker_cleanup(&current);
} }
} else { return Ok(());
api.schedule_backend_job_worker_cleanup(&current);
} }
return Ok(()); Err(JournalError::Conflict(_) | JournalError::PermissionDenied(_)) => {}
Err(error) => return Err(domain(error)),
} }
Err(JournalError::Conflict(_) | JournalError::PermissionDenied(_)) => {}
Err(error) => return Err(domain(error)),
} }
} }
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) { if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
@@ -2286,11 +2448,17 @@ pub(super) fn reconcile_unknown(
false false
}; };
if !confirmed_purge { if !confirmed_purge {
if released {
// Source deletion already fenced this generation. An unbound Unknown
// still lacks execution-cleanup proof: preserve Unknown, but it no
// longer owns the Subject or blocks removal of the original body.
return Ok(());
}
return Err(Error::RepositoryConflict( return Err(Error::RepositoryConflict(
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(), "Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
)); ));
} }
if !erased { if !erased && !released {
close(api, &job.request, attempt)?; close(api, &job.request, attempt)?;
} }
api.store.reconcile_backend_journal_attempt( api.store.reconcile_backend_journal_attempt(
@@ -2299,7 +2467,7 @@ pub(super) fn reconcile_unknown(
None, None,
&Utc::now().to_rfc3339(), &Utc::now().to_rfc3339(),
)?; )?;
if !erased { if !erased && !released {
// Resume the same saved body input only after canonical execution/domain fences. // Resume the same saved body input only after canonical execution/domain fences.
// dispatch selects its retained intent before it can capture any newer source. // dispatch selects its retained intent before it can capture any newer source.
let attr = store let attr = store
@@ -2316,6 +2484,10 @@ fn reconcile_session_jobs(
attr: &crate::subjektiv::SubjectSessionAttribution, attr: &crate::subjektiv::SubjectSessionAttribution,
erased: bool, erased: bool,
) -> Result<()> { ) -> Result<()> {
let released = !erased
&& store(api)?
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?;
let mut after: Option<String> = None; let mut after: Option<String> = None;
loop { loop {
let jobs = api.store.list_backend_journal_jobs( let jobs = api.store.list_backend_journal_jobs(
@@ -2330,6 +2502,29 @@ fn reconcile_session_jobs(
.store .store
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?; .ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
if released {
// Writer revocation has already fenced all Journal mutations. End
// pending/Unknown attempts without claiming their input was reflected;
// saved input, operations and original source remain retained.
if !attempt.state.terminal() {
attempt = api.store.finish_backend_job_attempt(
api.workspace_id(),
&job.request.job_id,
&id,
BackendJobAttemptState::Failed,
"journal_body_released",
"Source Worker was removed; unfinished Journal history retained",
&Utc::now().to_rfc3339(),
)?;
}
api.schedule_backend_job_worker_cleanup(&attempt);
if attempt.state == BackendJobAttemptState::Unknown {
// A saved checkpoint may settle success; otherwise retain Unknown
// until canonical Job-worker cleanup has supplied its Purge fence.
reconcile_unknown(api, job, &attempt)?;
}
continue;
}
if !erased if !erased
&& legacy_descriptor(&job.request) && legacy_descriptor(&job.request)
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available && job.input_disposition == crate::backend_job::JobInputDisposition::Available
File diff suppressed because it is too large Load Diff
@@ -314,6 +314,7 @@ impl Fixture {
&workdir_id, &workdir_id,
"account:close-test", "account:close-test",
"remove normally after session close", "remove normally after session close",
&uuid::Uuid::now_v7().to_string(),
) )
}) })
.await .await
@@ -652,6 +652,7 @@ async fn source_workdir_removal_keeps_published_drive_bytes_and_receipt() {
&removal_id, &removal_id,
"account:drive-test", "account:drive-test",
"source no longer needed", "source no longer needed",
"drive-source-removal",
) )
}) })
.await .await
@@ -819,6 +819,7 @@ mod workspace_config_integration {
}, },
"owner".into(), "owner".into(),
WorkingDirectoryRemovalRequest { WorkingDirectoryRemovalRequest {
operation_id: "unsupported-logical-cleanup".into(),
reason: "unsupported logical cleanup".into() reason: "unsupported logical cleanup".into()
} }
) )
+61 -50
View File
@@ -3253,8 +3253,8 @@ impl SqliteWorkspaceStore {
}, },
) )
.optional()?; .optional()?;
// Subject occupancy survives removal until an explicit release // An unconfirmed legacy removal cannot release Subject occupancy.
// proof. It must not resurrect the removed creation reservation: // Completed removal or explicit release does; never resurrect a reservation:
// Runtime callbacks require a live reservation or catalog member. // Runtime callbacks require a live reservation or catalog member.
if stored.as_ref().is_some_and(|row| row.6 == "removed") { if stored.as_ref().is_some_and(|row| row.6 == "removed") {
return Err(Error::RepositoryConflict(format!( return Err(Error::RepositoryConflict(format!(
@@ -11443,49 +11443,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
record.workdir_id record.workdir_id
))); )));
} }
let active_for_alias = tx // Ownership is keyed by Workdir, not by the requested local name.
.query_row(
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
FROM worker_workdir_links
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3
AND alias = ?4 AND unlinked_at IS NULL"#,
params![
record.workspace_id,
record.worker.runtime_id,
record.worker.worker_id,
record.alias,
],
read_worker_workdir_link_record,
)
.optional()?;
if let Some(mut active) = active_for_alias {
if active.workdir_id == record.workdir_id {
let capabilities = active.capabilities.intersection(record.capabilities);
if capabilities != active.capabilities {
tx.execute(
r#"UPDATE worker_workdir_links
SET capabilities = ?1
WHERE workspace_id = ?2 AND runtime_id = ?3 AND worker_id = ?4
AND workdir_id = ?5 AND alias = ?6 AND unlinked_at IS NULL"#,
params![
encode_workdir_link_capabilities(capabilities)?,
record.workspace_id,
record.worker.runtime_id,
record.worker.worker_id,
record.workdir_id,
record.alias,
],
)?;
active.capabilities = capabilities;
}
tx.commit()?;
return Ok(active);
}
return Err(Error::WorkdirAttachmentConflict(format!(
"Worker {}:{} attachment alias `{}` is already bound to Workdir {}",
record.worker.runtime_id, record.worker.worker_id, record.alias, active.workdir_id
)));
}
let active_for_workdir = tx let active_for_workdir = tx
.query_row( .query_row(
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
@@ -11495,10 +11453,45 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
read_worker_workdir_link_record, read_worker_workdir_link_record,
) )
.optional()?; .optional()?;
if let Some(active) = active_for_workdir { if let Some(mut active) = active_for_workdir {
if active.worker != record.worker {
return Err(Error::WorkdirOccupied {
workdir_id: record.workdir_id.clone(),
runtime_id: active.worker.runtime_id,
worker_id: active.worker.worker_id,
});
}
let capabilities = active.capabilities.intersection(record.capabilities);
if capabilities != active.capabilities {
tx.execute(
r#"UPDATE worker_workdir_links SET capabilities = ?1
WHERE workspace_id = ?2 AND runtime_id = ?3 AND worker_id = ?4
AND workdir_id = ?5 AND alias = ?6 AND unlinked_at IS NULL"#,
params![
encode_workdir_link_capabilities(capabilities)?,
active.workspace_id, active.worker.runtime_id, active.worker.worker_id,
active.workdir_id, active.alias,
],
)?;
active.capabilities = capabilities;
}
tx.commit()?;
return Ok(active);
}
let active_for_alias = tx
.query_row(
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
FROM worker_workdir_links
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3
AND alias = ?4 AND unlinked_at IS NULL"#,
params![record.workspace_id, record.worker.runtime_id, record.worker.worker_id, record.alias],
read_worker_workdir_link_record,
)
.optional()?;
if let Some(active) = active_for_alias {
return Err(Error::WorkdirAttachmentConflict(format!( return Err(Error::WorkdirAttachmentConflict(format!(
"Workdir {} is already attached to Worker {}:{}", "Worker {}:{} attachment alias `{}` is already bound to Workdir {}",
record.workdir_id, active.worker.runtime_id, active.worker.worker_id record.worker.runtime_id, record.worker.worker_id, record.alias, active.workdir_id
))); )));
} }
// Direct attachments (including ordinary recovery/reconnection) // Direct attachments (including ordinary recovery/reconnection)
@@ -16746,7 +16739,7 @@ fn current_worker_singleton_owner(
WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \ WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \ AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \ AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \ AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.state='succeeded') \
AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \ AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \
THEN 'created' ELSE reservation.state END \ THEN 'created' ELSE reservation.state END \
FROM worker_singleton_owners owner \ FROM worker_singleton_owners owner \
@@ -26065,6 +26058,24 @@ INSERT INTO worker_registry (
2 2
); );
for alias in ["another-name", "docs"] {
let requested = WorkerWorkdirLinkRecord {
alias: alias.into(),
..link.clone()
};
assert_eq!(
store.attach_worker_workdir(&requested).unwrap(),
downgraded_link
);
assert_eq!(
store
.list_worker_workdir_links("local-dev", &worker.worker)
.unwrap()
.len(),
2
);
}
let second_worker = WorkerRegistryRecord { let second_worker = WorkerRegistryRecord {
worker: RuntimeWorkerRef::new("embedded", "2"), worker: RuntimeWorkerRef::new("embedded", "2"),
display_name: "Browser 2".to_string(), display_name: "Browser 2".to_string(),
@@ -26080,7 +26091,7 @@ INSERT INTO worker_registry (
}; };
assert!(matches!( assert!(matches!(
store.attach_worker_workdir(&workdir_conflict), store.attach_worker_workdir(&workdir_conflict),
Err(Error::WorkdirAttachmentConflict(_)) Err(Error::WorkdirOccupied { .. })
)); ));
assert!(matches!( assert!(matches!(
store.detach_worker_workdir("local-dev", &worker.worker, Some("wrong-workdir"), "6",), store.detach_worker_workdir("local-dev", &worker.worker, Some("wrong-workdir"), "6",),
@@ -1289,8 +1289,9 @@ mod tests {
reservation(&s, &body, "body", Some("subjektiv:subject")); reservation(&s, &body, "body", Some("subjektiv:subject"));
receipt(&s, &body, "succeeded", "purge", "purge"); receipt(&s, &body, "succeeded", "purge", "purge");
let body_receipt = format!("receipt-{}-succeeded", body.worker_id); let body_receipt = format!("receipt-{}-succeeded", body.worker_id);
// A different Subject's already-removed body remains occupied until its // Another Subject's completed removal already releases its occupancy.
// own deletion/release. Deleting a delegating Subject must not release it. // Deleting the delegating Subject must preserve that independent outcome
// and must not erase the other Subject's Job.
let other_body = worker(33); let other_body = worker(33);
reservation(&s, &other_body, "other-body", Some("subjektiv:another")); reservation(&s, &other_body, "other-body", Some("subjektiv:another"));
receipt(&s, &other_body, "succeeded", "purge", "purge"); receipt(&s, &other_body, "succeeded", "purge", "purge");
@@ -1307,6 +1308,11 @@ mod tests {
}).unwrap(); }).unwrap();
let archive_before = s.worker_session_archive("w", &jw, "session").unwrap(); let archive_before = s.worker_session_archive("w", &jw, "session").unwrap();
assert!(archive_before.is_some()); assert!(archive_before.is_some());
assert!(
s.current_worker_singleton_owner("w", "subjektiv:another")
.unwrap()
.is_none()
);
s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&other_body)) s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&other_body))
.unwrap(); .unwrap();
assert_eq!(saved_prior(&s, "subject", &jw), Some(job_receipt.clone())); assert_eq!(saved_prior(&s, "subject", &jw), Some(job_receipt.clone()));
@@ -1352,7 +1358,7 @@ mod tests {
assert!( assert!(
s.current_worker_singleton_owner("w", "subjektiv:another") s.current_worker_singleton_owner("w", "subjektiv:another")
.unwrap() .unwrap()
.is_some() .is_none()
); );
s.with_conn(|c| { s.with_conn(|c| {
let old:(String,String,String)=c.query_row("SELECT state,session_disposition,metadata_disposition FROM worker_removal_operations WHERE operation_id=?1",[&job_receipt],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?; let old:(String,String,String)=c.query_row("SELECT state,session_disposition,metadata_disposition FROM worker_removal_operations WHERE operation_id=?1",[&job_receipt],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?;
+189 -77
View File
@@ -1,24 +1,15 @@
//! Persists durable deletion intent and attempts for Repository-backed Workdirs. //! Records one Repository-backed Workdir removal request and its result.
//! //!
//! Deletion intent fixes Workspace, Workdir, owning Runtime, Repository/materialization, actor, //! Caller-generated request identities distinguish an exact resend from a new explicit request;
//! and reason. Endpoints, host paths, and provider handles do not grant caller deletion authority. //! reason text is audit input, never a replay key. Failed and completed records are terminal,
//! including legacy failed records marked retryable. Retryability only advises a new explicit
//! request after resolving its cause; it is not continuing authority to delete.
//!
//! Only one unresolved request per Workdir is allowed. Attempts are claimed transactionally and
//! fenced by process/start marker and attempt identity. Recovery observes uncertain external
//! effects but never reissues deletion. Confirmed absence and the matching materialization permit
//! registry removal and the saved result to commit together. Unconfirmed effects remain pending.
//! External Workdirs follow grant revocation, not this operation. //! External Workdirs follow grant revocation, not this operation.
//!
//! Only one pending operation per Workdir is allowed; attempts are claimed transactionally.
//! Process IDs and startup markers are retained. Recovery reclaims the exact owner/attempt by
//! compare-and-set only after confirming the previous owner is gone; live or unobservable owners
//! are not displaced. In-process locks alone are insufficient.
//!
//! Server execution revalidates stored materialization and current attachments, reservations,
//! assignments, retention, and creation operations, then observes occupancy and cleanliness via
//! Runtime. Dirty resources remain; unknown observations or provider failures retain the registry
//! and record attention/retryability. Only confirmed deletion or exact working_directory_not_found
//! is deletion evidence; registry removal and completed/removed commit in the same transaction.
//!
//! Normal deletion, Workspace deletion, cleanup, and startup recovery use the same reserved Server
//! operation execution. Failed DB commits after provider deletion are reobserved through that
//! operation. Recovery relies on provider identity and results, not an exactly-once guarantee
//! across all interruption points.
use chrono::Utc; use chrono::Utc;
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
@@ -118,10 +109,22 @@ pub fn workdir_materialization_fingerprint(record: &WorkdirRegistryRecord) -> St
hex_sha256(&bytes) hex_sha256(&bytes)
} }
pub fn workdir_removal_request_id(
workspace_id: &str,
source_actor: &str,
request_id: &str,
) -> Result<String> {
validate_bounded("operation id", request_id, 256)?;
let bytes = serde_json::to_vec(&(workspace_id, source_actor, request_id))
.map_err(|error| Error::Store(error.to_string()))?;
Ok(format!("wdr_{}", &hex_sha256(&bytes)[..32]))
}
pub fn workdir_removal_intent( pub fn workdir_removal_intent(
record: &WorkdirRegistryRecord, record: &WorkdirRegistryRecord,
source_actor: &str, source_actor: &str,
reason: &str, reason: &str,
request_id: &str,
) -> Result<WorkdirRemovalIntent> { ) -> Result<WorkdirRemovalIntent> {
validate_bounded("source actor", source_actor, MAX_ACTOR_BYTES)?; validate_bounded("source actor", source_actor, MAX_ACTOR_BYTES)?;
validate_bounded("reason", reason, MAX_REASON_BYTES)?; validate_bounded("reason", reason, MAX_REASON_BYTES)?;
@@ -146,7 +149,7 @@ pub fn workdir_removal_intent(
)); ));
}; };
Ok(WorkdirRemovalIntent { Ok(WorkdirRemovalIntent {
operation_id: format!("wdr_{}", &request_fingerprint[..32]), operation_id: workdir_removal_request_id(&record.workspace_id, source_actor, request_id)?,
request_fingerprint, request_fingerprint,
workspace_id: record.workspace_id.clone(), workspace_id: record.workspace_id.clone(),
working_directory_id: record.workdir_id.clone(), working_directory_id: record.workdir_id.clone(),
@@ -285,17 +288,10 @@ impl SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let operation = let operation =
require_operation(&tx, workspace_id, operation_id, request_fingerprint)?; require_operation(&tx, workspace_id, operation_id, request_fingerprint)?;
if operation.state == WorkdirRemovalOperationState::Completed if operation.state != WorkdirRemovalOperationState::Pending {
&& operation.disposition == Some(WorkdirRemovalDisposition::Removed)
{
tx.commit()?; tx.commit()?;
return Ok(operation); return Ok(operation);
} }
if operation.state == WorkdirRemovalOperationState::Failed && !operation.retryable {
return Err(Error::InvalidInput(format!(
"Workdir removal operation `{operation_id}` is not retryable"
)));
}
if let Some((expected_owner, expected_attempt_id)) = recovery_expected { if let Some((expected_owner, expected_attempt_id)) = recovery_expected {
if operation.state != WorkdirRemovalOperationState::Pending if operation.state != WorkdirRemovalOperationState::Pending
|| operation.attempt_owner != expected_owner || operation.attempt_owner != expected_owner
@@ -473,7 +469,7 @@ impl SqliteWorkspaceStore {
&operation.operation_id, &operation.operation_id,
&operation.request_fingerprint, &operation.request_fingerprint,
)?; )?;
if current.state == WorkdirRemovalOperationState::Completed { if current.state != WorkdirRemovalOperationState::Pending {
tx.commit()?; tx.commit()?;
return Ok(current); return Ok(current);
} }
@@ -502,6 +498,39 @@ impl SqliteWorkspaceStore {
}) })
} }
/// Keep an uncertain external outcome for observation, not automatic re-execution.
pub fn record_workdir_removal_unknown(
&self,
operation: &WorkdirRemovalOperation,
) -> Result<WorkdirRemovalOperation> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let current = require_operation(
&tx,
&operation.workspace_id,
&operation.operation_id,
&operation.request_fingerprint,
)?;
if current.state != WorkdirRemovalOperationState::Pending {
tx.commit()?;
return Ok(current);
}
require_matching_attempt(operation, &current)?;
tx.execute(
"UPDATE workdir_removal_operations SET retryable=0, disposition='attention_required', failure_category='provider_cleanup_outcome_unknown', attempt_owner_pid=NULL, attempt_owner_start_marker=NULL, updated_at=?1 WHERE workspace_id=?2 AND operation_id=?3",
params![Utc::now().to_rfc3339(), operation.workspace_id, operation.operation_id],
)?;
let updated = require_operation(
&tx,
&operation.workspace_id,
&operation.operation_id,
&operation.request_fingerprint,
)?;
tx.commit()?;
Ok(updated)
})
}
pub fn commit_workdir_removal_removed( pub fn commit_workdir_removal_removed(
&self, &self,
operation: &WorkdirRemovalOperation, operation: &WorkdirRemovalOperation,
@@ -531,7 +560,7 @@ impl SqliteWorkspaceStore {
&operation.operation_id, &operation.operation_id,
&operation.request_fingerprint, &operation.request_fingerprint,
)?; )?;
if current.state == WorkdirRemovalOperationState::Completed { if current.state != WorkdirRemovalOperationState::Pending {
tx.commit()?; tx.commit()?;
return Ok(current); return Ok(current);
} }
@@ -541,24 +570,29 @@ impl SqliteWorkspaceStore {
&tx, &tx,
&operation.workspace_id, &operation.workspace_id,
&operation.working_directory_id, &operation.working_directory_id,
)?
.ok_or_else(|| Error::RegistryInconsistency(
"Workdir registry row disappeared before durable removal commit".to_string(),
))?;
require_operation_materialization(operation, &registry)?;
require_no_removal_blockers(
&tx,
&operation.workspace_id,
&operation.working_directory_id,
)?; )?;
let deleted = tx.execute( if let Some(registry) = registry {
"DELETE FROM workdir_registry WHERE workspace_id=?1 AND workdir_id=?2", require_operation_materialization(operation, &registry)?;
params![operation.workspace_id, operation.working_directory_id], require_no_removal_blockers(
)?; &tx,
if deleted != 1 { &operation.workspace_id,
return Err(Error::RegistryInconsistency( &operation.working_directory_id,
"Workdir registry deletion did not remove exactly one row".to_string(), )?;
)); tx.execute(
"DELETE FROM workdir_registry WHERE workspace_id=?1 AND workdir_id=?2",
params![operation.workspace_id, operation.working_directory_id],
)?;
} else {
let removed: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM workdir_removal_operations WHERE workspace_id=?1 AND workdir_id=?2 AND materialization_fingerprint=?3 AND state='completed' AND disposition='removed')",
params![operation.workspace_id, operation.working_directory_id, operation.materialization_fingerprint],
|row| row.get(0),
)?;
if !removed {
return Err(Error::RegistryInconsistency(
"Workdir registry disappeared without confirmed removal".to_string(),
));
}
} }
} }
let now = Utc::now().to_rfc3339(); let now = Utc::now().to_rfc3339();
@@ -585,20 +619,18 @@ impl SqliteWorkspaceStore {
}) })
} }
pub fn find_workdir_removal_operation_by_intent( pub fn completed_workdir_removal(
&self, &self,
workspace_id: &str, workspace_id: &str,
working_directory_id: &str, working_directory_id: &str,
source_actor: &str,
reason: &str,
) -> Result<Option<WorkdirRemovalOperation>> { ) -> Result<Option<WorkdirRemovalOperation>> {
self.with_conn(|conn| { self.with_conn(|conn| {
conn.query_row( conn.query_row(
&format!( &format!(
"{} WHERE workspace_id=?1 AND workdir_id=?2 AND source_actor=?3 AND reason=?4 ORDER BY created_at DESC, operation_id DESC LIMIT 1", "{} WHERE workspace_id=?1 AND workdir_id=?2 AND state='completed' AND disposition='removed' ORDER BY completed_at DESC, operation_id DESC LIMIT 1",
operation_select_sql() operation_select_sql()
), ),
params![workspace_id, working_directory_id, source_actor, reason], params![workspace_id, working_directory_id],
read_operation, read_operation,
) )
.optional() .optional()
@@ -614,7 +646,7 @@ impl SqliteWorkspaceStore {
self.with_conn(|conn| { self.with_conn(|conn| {
let mut statement = conn.prepare( let mut statement = conn.prepare(
&format!( &format!(
"{} WHERE workspace_id=?1 AND (state='pending' OR (state='failed' AND retryable=1)) ORDER BY updated_at ASC, operation_id ASC LIMIT ?2", "{} WHERE workspace_id=?1 AND state='pending' ORDER BY updated_at ASC, operation_id ASC LIMIT ?2",
operation_select_sql() operation_select_sql()
), ),
)?; )?;
@@ -1007,8 +1039,13 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn exact_replay_reuses_operation_and_conflicting_intent_fails() { async fn exact_replay_reuses_operation_and_conflicting_intent_fails() {
let (store, workdir) = seeded_store().await; let (store, workdir) = seeded_store().await;
let intent = let intent = workdir_removal_intent(
workdir_removal_intent(&workdir, "worker:W-1", "remove stale Workdir").unwrap(); &workdir,
"worker:W-1",
"remove stale Workdir",
"test-removal-request",
)
.unwrap();
let first = store.reserve_workdir_removal_operation(&intent).unwrap(); let first = store.reserve_workdir_removal_operation(&intent).unwrap();
let replay = store.reserve_workdir_removal_operation(&intent).unwrap(); let replay = store.reserve_workdir_removal_operation(&intent).unwrap();
assert_eq!(replay, first); assert_eq!(replay, first);
@@ -1023,10 +1060,15 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn failed_attempt_is_retryable_and_completed_retry_replays() { async fn failed_request_is_terminal_and_new_request_has_independent_result() {
let (store, workdir) = seeded_store().await; let (store, workdir) = seeded_store().await;
let intent = let intent = workdir_removal_intent(
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap(); &workdir,
"workspace-api",
"remove clean Workdir",
"test-removal-request",
)
.unwrap();
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap(); let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
let first = store let first = store
.begin_workdir_removal_attempt( .begin_workdir_removal_attempt(
@@ -1049,7 +1091,32 @@ mod tests {
attempt_owner(), attempt_owner(),
) )
.unwrap(); .unwrap();
assert_ne!(retry.attempt_id, first.attempt_id); assert_eq!(retry, failed);
assert!(
store
.recoverable_workdir_removal_operations("workspace-a", 100)
.unwrap()
.is_empty()
);
let next_intent = workdir_removal_intent(
&workdir,
"workspace-api",
"remove clean Workdir",
"new-request",
)
.unwrap();
let next = store
.reserve_workdir_removal_operation(&next_intent)
.unwrap();
let retry = store
.begin_workdir_removal_attempt(
&next.workspace_id,
&next.operation_id,
&next.request_fingerprint,
attempt_owner(),
)
.unwrap();
assert_ne!(retry.operation_id, failed.operation_id);
let completed = store.commit_workdir_removal_removed(&retry).unwrap(); let completed = store.commit_workdir_removal_removed(&retry).unwrap();
assert_eq!( assert_eq!(
completed.disposition, completed.disposition,
@@ -1075,7 +1142,13 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn recovery_and_completion_are_fenced_by_attempt_identity() { async fn recovery_and_completion_are_fenced_by_attempt_identity() {
let (store, workdir) = seeded_store().await; let (store, workdir) = seeded_store().await;
let intent = workdir_removal_intent(&workdir, "workspace-api", "remove Workdir").unwrap(); let intent = workdir_removal_intent(
&workdir,
"workspace-api",
"remove Workdir",
"test-removal-request",
)
.unwrap();
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap(); let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
let first = store let first = store
.begin_workdir_removal_attempt( .begin_workdir_removal_attempt(
@@ -1130,8 +1203,13 @@ mod tests {
use std::sync::{Arc, Barrier}; use std::sync::{Arc, Barrier};
let (store, workdir) = seeded_store().await; let (store, workdir) = seeded_store().await;
let intent = let intent = workdir_removal_intent(
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap(); &workdir,
"workspace-api",
"remove clean Workdir",
"test-removal-request",
)
.unwrap();
let operation = store.reserve_workdir_removal_operation(&intent).unwrap(); let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
let barrier = Arc::new(Barrier::new(3)); let barrier = Arc::new(Barrier::new(3));
let provider_calls = Arc::new(AtomicUsize::new(0)); let provider_calls = Arc::new(AtomicUsize::new(0));
@@ -1218,8 +1296,13 @@ mod tests {
"2", "2",
) )
.unwrap(); .unwrap();
let intent = let intent = workdir_removal_intent(
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap(); &workdir,
"workspace-api",
"remove clean Workdir",
"test-removal-request",
)
.unwrap();
store.reserve_workdir_removal_operation(&intent).unwrap(); store.reserve_workdir_removal_operation(&intent).unwrap();
let error = store let error = store
@@ -1236,8 +1319,13 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn pending_removal_fences_new_attachment_and_retry_rereads_live_reservation() { async fn pending_removal_fences_new_attachment_and_retry_rereads_live_reservation() {
let (store, workdir) = seeded_store().await; let (store, workdir) = seeded_store().await;
let intent = let intent = workdir_removal_intent(
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap(); &workdir,
"workspace-api",
"remove clean Workdir",
"test-removal-request",
)
.unwrap();
let pending = store.reserve_workdir_removal_operation(&intent).unwrap(); let pending = store.reserve_workdir_removal_operation(&intent).unwrap();
let error = store let error = store
@@ -1251,11 +1339,22 @@ mod tests {
store store
.reserve_worker_workdir_attachment("workspace-a", "workdir-a", "reservation-b", "3") .reserve_worker_workdir_attachment("workspace-a", "workdir-a", "reservation-b", "3")
.unwrap(); .unwrap();
let next_intent = workdir_removal_intent(
&workdir,
"workspace-api",
"remove clean Workdir",
"new-request",
)
.unwrap();
let next = store
.reserve_workdir_removal_operation(&next_intent)
.unwrap();
assert_ne!(next.operation_id, failed.operation_id);
let retry = store let retry = store
.begin_workdir_removal_attempt( .begin_workdir_removal_attempt(
&failed.workspace_id, &next.workspace_id,
&failed.operation_id, &next.operation_id,
&failed.request_fingerprint, &next.request_fingerprint,
attempt_owner(), attempt_owner(),
) )
.unwrap(); .unwrap();
@@ -1324,7 +1423,13 @@ mod tests {
Ok(()) Ok(())
}) })
.unwrap(); .unwrap();
let intent = workdir_removal_intent(&workdir, "workspace-api", "remove Workdir").unwrap(); let intent = workdir_removal_intent(
&workdir,
"workspace-api",
"remove Workdir",
"test-removal-request",
)
.unwrap();
let operation = store.reserve_workdir_removal_operation(&intent).unwrap(); let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
let categories = |guards: Vec<WorkdirRemovalGuard>| { let categories = |guards: Vec<WorkdirRemovalGuard>| {
guards guards
@@ -1447,8 +1552,13 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn retained_completion_keeps_registry_and_is_auditable() { async fn retained_completion_keeps_registry_and_is_auditable() {
let (store, workdir) = seeded_store().await; let (store, workdir) = seeded_store().await;
let intent = let intent = workdir_removal_intent(
workdir_removal_intent(&workdir, "workspace-api", "inspect dirty Workdir").unwrap(); &workdir,
"workspace-api",
"inspect dirty Workdir",
"test-removal-request",
)
.unwrap();
let operation = store.reserve_workdir_removal_operation(&intent).unwrap(); let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
let retained = store let retained = store
.complete_workdir_removal_retained( .complete_workdir_removal_retained(
@@ -1477,10 +1587,12 @@ mod tests {
attempt_owner(), attempt_owner(),
) )
.unwrap(); .unwrap();
assert_eq!(retry.state, WorkdirRemovalOperationState::Pending); assert_eq!(retry, retained);
assert!(retry.attempt_id.is_some()); assert!(
assert_eq!(retry.disposition, None); store
assert_eq!(retry.failure_category, None); .recoverable_workdir_removal_operations("workspace-a", 100)
assert!(retry.retryable); .unwrap()
.is_empty()
);
} }
} }
+6 -1
View File
@@ -20698,11 +20698,16 @@
"WorkingDirectoryRemovalRequest": { "WorkingDirectoryRemovalRequest": {
"additionalProperties": false, "additionalProperties": false,
"properties": { "properties": {
"operation_id": {
"description": "Stable for an exact resend; a new explicit request uses a new identity.",
"type": "string"
},
"reason": { "reason": {
"type": "string" "type": "string"
} }
}, },
"required": [ "required": [
"operation_id",
"reason" "reason"
], ],
"type": "object" "type": "object"
@@ -22715,7 +22720,7 @@
"info": { "info": {
"title": "Yoi Server API", "title": "Yoi Server API",
"version": "0.1.0", "version": "0.1.0",
"x-yoi-source-digest": "sha256:8cc3c45d082e52b1ea9a014721008aeb4c0699b987729d362c0c14a6f485df10" "x-yoi-source-digest": "sha256:2ac605f7ea70d0c725f9993a5c59c4a8c4266cf69184902db7b3ea7fd5932224"
}, },
"openapi": "3.1.0", "openapi": "3.1.0",
"paths": { "paths": {
@@ -1,6 +1,6 @@
// @generated by yoi-companion-openapi-typescript@1. Do not edit. // @generated by yoi-companion-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json // Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:8cc3c45d082e52b1ea9a014721008aeb4c0699b987729d362c0c14a6f485df10) // Input contract: 0.1.0 (sha256:2ac605f7ea70d0c725f9993a5c59c4a8c4266cf69184902db7b3ea7fd5932224)
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/companion-api.ts // Output: web/workspace/src/lib/generated/companion-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-repository-openapi-typescript@1. Do not edit. // @generated by yoi-repository-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json // Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:8cc3c45d082e52b1ea9a014721008aeb4c0699b987729d362c0c14a6f485df10) // Input contract: 0.1.0 (sha256:2ac605f7ea70d0c725f9993a5c59c4a8c4266cf69184902db7b3ea7fd5932224)
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
// Output: web/workspace/src/lib/generated/repository-api.ts // Output: web/workspace/src/lib/generated/repository-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-runtime-openapi-typescript@1. Do not edit. // @generated by yoi-runtime-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json // Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:8cc3c45d082e52b1ea9a014721008aeb4c0699b987729d362c0c14a6f485df10) // Input contract: 0.1.0 (sha256:2ac605f7ea70d0c725f9993a5c59c4a8c4266cf69184902db7b3ea7fd5932224)
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/runtime-api.ts // Output: web/workspace/src/lib/generated/runtime-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json // Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:8cc3c45d082e52b1ea9a014721008aeb4c0699b987729d362c0c14a6f485df10) // Input contract: 0.1.0 (sha256:2ac605f7ea70d0c725f9993a5c59c4a8c4266cf69184902db7b3ea7fd5932224)
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/ticket-api.ts // Output: web/workspace/src/lib/generated/ticket-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-workdir-openapi-typescript@1. Do not edit. // @generated by yoi-workdir-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json // Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:1053b09bb77ff0fc09fec2d3f96af04b76974c5ea52870f805381302e1b29fe4) // Input contract: 0.1.0 (sha256:2ac605f7ea70d0c725f9993a5c59c4a8c4266cf69184902db7b3ea7fd5932224)
// Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/workdir-api.ts // Output: web/workspace/src/lib/generated/workdir-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check // Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check
@@ -33,7 +33,7 @@ export type WorkingDirectoryOccupancy = { display_name: string; linked_at: strin
export type WorkingDirectoryRemovalDisposition = "removed" | "retained" | "attention_required"; export type WorkingDirectoryRemovalDisposition = "removed" | "retained" | "attention_required";
export type WorkingDirectoryRemovalRequest = { reason: string; }; export type WorkingDirectoryRemovalRequest = { operation_id: string; reason: string; };
export type WorkingDirectoryRemovalResponse = { disposition: WorkingDirectoryRemovalDisposition; failure_category?: string | null; retryable: boolean; working_directory_id: string; }; export type WorkingDirectoryRemovalResponse = { disposition: WorkingDirectoryRemovalDisposition; failure_category?: string | null; retryable: boolean; working_directory_id: string; };
@@ -1,6 +1,6 @@
// @generated by yoi-worker-openapi-typescript@1. Do not edit. // @generated by yoi-worker-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json // Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:8cc3c45d082e52b1ea9a014721008aeb4c0699b987729d362c0c14a6f485df10) // Input contract: 0.1.0 (sha256:2ac605f7ea70d0c725f9993a5c59c4a8c4266cf69184902db7b3ea7fd5932224)
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/worker-launch-api.ts // Output: web/workspace/src/lib/generated/worker-launch-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
@@ -30,7 +30,9 @@ const causes: Readonly<Record<string, string>> = {
provider_observation_unknown: provider_observation_unknown:
"The provider could not confirm the remaining Workdir. Restore provider access so the next removal can recheck its current contents.", "The provider could not confirm the remaining Workdir. Restore provider access so the next removal can recheck its current contents.",
provider_cleanup_outcome_unknown: provider_cleanup_outcome_unknown:
"The provider has not confirmed deletion. Restore provider access and retry the same removal request to recheck the remaining target. The Workdir stays listed until deletion is confirmed.", "The provider has not confirmed deletion. Restore provider access and reconcile the same request. Recovery only observes the target; it never repeats deletion.",
request_interrupted:
"The request was interrupted before execution. Submit a new request if removal is still wanted.",
authority_invalid: authority_invalid:
"The removal authority could not be verified. Ask the Workspace administrator to verify the Workdir target and owning Runtime before requesting removal again.", "The removal authority could not be verified. Ask the Workspace administrator to verify the Workdir target and owning Runtime before requesting removal again.",
provider_identity_changed: provider_identity_changed:
@@ -125,11 +127,39 @@ async function removalBody(response: Response): Promise<string> {
} }
} }
// 要求IDは結果不明時だけ保持する。確定した失敗の次の明示要求は別IDとなる。
export class WorkdirRemovalRequests {
private readonly unresolved = new Map<string, string>();
async remove(
fetchFn: typeof fetch,
workspaceId: string,
runtimeId: string,
workdirId: string,
): Promise<WorkingDirectoryRemovalResponse> {
const key = JSON.stringify([workspaceId, runtimeId, workdirId]);
const operationId = this.unresolved.get(key) ?? crypto.randomUUID();
this.unresolved.set(key, operationId);
const result = await removeWorkdir(
fetchFn,
workspaceId,
runtimeId,
workdirId,
operationId,
);
if (result.failure_category !== "provider_cleanup_outcome_unknown") {
this.unresolved.delete(key);
}
return result;
}
}
export async function removeWorkdir( export async function removeWorkdir(
fetchFn: typeof fetch, fetchFn: typeof fetch,
workspaceId: string, workspaceId: string,
runtimeId: string, runtimeId: string,
workdirId: string, workdirId: string,
operationId: string,
): Promise<WorkingDirectoryRemovalResponse> { ): Promise<WorkingDirectoryRemovalResponse> {
const response = await fetchFn( const response = await fetchFn(
workspaceApiPath( workspaceApiPath(
@@ -141,8 +171,10 @@ export async function removeWorkdir(
{ {
method: "DELETE", method: "DELETE",
headers: { "content-type": "application/json" }, headers: { "content-type": "application/json" },
// An ordinary repeated request uses the existing durable Backend removal authority. body: JSON.stringify({
body: JSON.stringify({ reason: "User requested Workdir deletion" }), operation_id: operationId,
reason: "User requested Workdir deletion",
}),
}, },
); );
if (!response.ok) { if (!response.ok) {
@@ -2,7 +2,7 @@
import { loadJson, workspaceApiPath } from '#lib/workspace/api/http.ts'; import { loadJson, workspaceApiPath } from '#lib/workspace/api/http.ts';
import { parseRuntimeCleanupPlan } from '#lib/workspace/api/runtime-workers.ts'; import { parseRuntimeCleanupPlan } from '#lib/workspace/api/runtime-workers.ts';
import { parseWorkingDirectoryListResponse } from '#lib/workspace/api/workdirs.ts'; import { parseWorkingDirectoryListResponse } from '#lib/workspace/api/workdirs.ts';
import { canRemoveWorkdir, removalCause, removalGuard, removeWorkdir, WorkdirRemovalError } from '#lib/workspace/settings/workdir-removal.ts'; import { canRemoveWorkdir, removalCause, removalGuard, WorkdirRemovalRequests, WorkdirRemovalError } from '#lib/workspace/settings/workdir-removal.ts';
import { formatCurrentWorkdirCheckout } from '#lib/workspace/settings/workdir-checkout.ts'; import { formatCurrentWorkdirCheckout } from '#lib/workspace/settings/workdir-checkout.ts';
import type { CleanupWorkdirCandidate, RuntimeCleanupPlanResponse, WorkingDirectorySummary } from '#lib/workspace/sidebar/types.ts'; import type { CleanupWorkdirCandidate, RuntimeCleanupPlanResponse, WorkingDirectorySummary } from '#lib/workspace/sidebar/types.ts';
import type { PageData as RouteData } from './$types'; import type { PageData as RouteData } from './$types';
@@ -10,6 +10,7 @@
type PageProps = { data: ResolvedPageData<RouteData> }; type PageProps = { data: ResolvedPageData<RouteData> };
let { data }: PageProps = $props(); let { data }: PageProps = $props();
const removalRequests = new WorkdirRemovalRequests();
let cleanupBusyTarget = $state<string | null>(null); let cleanupBusyTarget = $state<string | null>(null);
let refreshing = $state(false); let refreshing = $state(false);
let refreshError = $state<string | null>(null); let refreshError = $state<string | null>(null);
@@ -72,7 +73,7 @@
cleanupBusyTarget = workdir.working_directory_id; cleanupBusyTarget = workdir.working_directory_id;
feedback = null; feedback = null;
try { try {
const result = await removeWorkdir(fetch, workspaceId, runtimeId, workdir.working_directory_id); const result = await removalRequests.remove(fetch, workspaceId, runtimeId, workdir.working_directory_id);
if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return; if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return;
feedback = { feedback = {
id: workdir.working_directory_id, id: workdir.working_directory_id,
+44 -6
View File
@@ -6,6 +6,7 @@ import {
canRemoveWorkdir, canRemoveWorkdir,
removalCause, removalCause,
removeWorkdir, removeWorkdir,
WorkdirRemovalRequests,
} from "../src/lib/workspace/settings/workdir-removal.ts"; } from "../src/lib/workspace/settings/workdir-removal.ts";
import type { import type {
CleanupWorkdirCandidate, CleanupWorkdirCandidate,
@@ -57,6 +58,7 @@ Deno.test("removal wire boundary rejects malformed results and mismatched identi
"workspace", "workspace",
"runtime", "runtime",
"dir", "dir",
"request-1",
); );
} catch { } catch {
rejected = true; rejected = true;
@@ -72,6 +74,7 @@ Deno.test("removal response size is bounded before decoding provider content", a
"workspace", "workspace",
"runtime", "runtime",
"dir", "dir",
"request-1",
); );
} catch { } catch {
rejected = true; rejected = true;
@@ -179,7 +182,7 @@ Deno.test("pending unknown cleanup permits ordinary removal while current dirty
); );
}); });
Deno.test("ordinary retries send the same scoped DELETE body and expose only allowlisted HTTP causes", async () => { Deno.test("new requests after confirmed failure get distinct IDs and expose only allowlisted HTTP causes", async () => {
const requests: { path: string; body: string; method: string }[] = []; const requests: { path: string; body: string; method: string }[] = [];
const fetchFn = (async (path: RequestInfo | URL, init?: RequestInit) => { const fetchFn = (async (path: RequestInfo | URL, init?: RequestInit) => {
requests.push({ requests.push({
@@ -194,8 +197,9 @@ Deno.test("ordinary retries send the same scoped DELETE body and expose only all
failure_category: "mount_present", failure_category: "mount_present",
}); });
}) as typeof fetch; }) as typeof fetch;
const removalRequests = new WorkdirRemovalRequests();
for (let i = 0; i < 2; i++) { for (let i = 0; i < 2; i++) {
const result = await removeWorkdir( const result = await removalRequests.remove(
fetchFn, fetchFn,
"workspace/x", "workspace/x",
"runtime/x", "runtime/x",
@@ -204,8 +208,9 @@ Deno.test("ordinary retries send the same scoped DELETE body and expose only all
assert(result.retryable && result.failure_category === "mount_present"); assert(result.retryable && result.failure_category === "mount_present");
} }
assert( assert(
JSON.stringify(requests[0]) === JSON.stringify(requests[1]), JSON.parse(requests[0].body).operation_id !==
"retry changed ordinary deletion request", JSON.parse(requests[1].body).operation_id,
"new explicit request reused a failed request identity",
); );
assert( assert(
requests[0].path === requests[0].path ===
@@ -213,8 +218,9 @@ Deno.test("ordinary retries send the same scoped DELETE body and expose only all
); );
assert( assert(
requests[0].method === "DELETE" && requests[0].method === "DELETE" &&
requests[0].body === JSON.parse(requests[0].body).reason ===
JSON.stringify({ reason: "User requested Workdir deletion" }), "User requested Workdir deletion" &&
typeof JSON.parse(requests[0].body).operation_id === "string",
); );
for ( for (
const [status, category] of [ const [status, category] of [
@@ -236,6 +242,7 @@ Deno.test("ordinary retries send the same scoped DELETE body and expose only all
"workspace", "workspace",
"runtime", "runtime",
"dir", "dir",
"request-1",
); );
} catch (error) { } catch (error) {
message = error instanceof Error ? error.message : String(error); message = error instanceof Error ? error.message : String(error);
@@ -246,3 +253,34 @@ Deno.test("ordinary retries send the same scoped DELETE body and expose only all
); );
} }
}); });
Deno.test("unknown removal outcomes keep the same identity and body until a terminal result", async () => {
for (const transportLoss of [false, true]) {
const calls: string[] = [];
const requester = new WorkdirRemovalRequests();
const fetchFn = (async (_path: RequestInfo | URL, init?: RequestInit) => {
calls.push(String(init?.body));
if (calls.length === 1 && transportLoss) {
throw new Error("connection lost");
}
return Response.json({
working_directory_id: "dir",
disposition: "attention_required",
retryable: calls.length !== 1,
failure_category: calls.length === 1
? "provider_cleanup_outcome_unknown"
: "resource_busy",
});
}) as typeof fetch;
await requester.remove(fetchFn, "workspace", "runtime", "dir").catch(() =>
undefined
);
await requester.remove(fetchFn, "workspace", "runtime", "dir");
await requester.remove(fetchFn, "workspace", "runtime", "dir");
assert(calls[0] === calls[1], "unknown outcome changed request identity");
assert(
calls[1] !== calls[2],
"confirmed failure kept old request identity",
);
}
});