fix: keep runtime responsive during workdir cleanup
This commit is contained in:
+173
-1
@@ -1770,6 +1770,9 @@ impl WorkdirSession for LocalWorkdirSession {
|
|||||||
|
|
||||||
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
||||||
self.ensure_capability(WorkdirSessionCapability::Command)?;
|
self.ensure_capability(WorkdirSessionCapability::Command)?;
|
||||||
|
// Command admission, resource capture and registration must be atomic
|
||||||
|
// with terminal close, which drains commands and releases resources.
|
||||||
|
let _admission = self.inner.close_lock.lock().await;
|
||||||
self.ensure_open()?;
|
self.ensure_open()?;
|
||||||
if let Some(spill_dir) = request.spill_dir.as_deref()
|
if let Some(spill_dir) = request.spill_dir.as_deref()
|
||||||
&& !self
|
&& !self
|
||||||
@@ -1794,6 +1797,15 @@ impl WorkdirSession for LocalWorkdirSession {
|
|||||||
let telemetry = self.inner.command_telemetry.clone();
|
let telemetry = self.inner.command_telemetry.clone();
|
||||||
let command_environment = self.inner.command_environment.clone();
|
let command_environment = self.inner.command_environment.clone();
|
||||||
let (cancel, cancel_rx) = watch::channel(false);
|
let (cancel, cancel_rx) = watch::channel(false);
|
||||||
|
// A session drop only schedules command cancellation. Keep its Runtime
|
||||||
|
// resources (including cleanup exclusion) alive until the task actually
|
||||||
|
// finishes or its future is dropped, not just until the handle disappears.
|
||||||
|
let resources = self
|
||||||
|
.inner
|
||||||
|
.resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.clone();
|
||||||
let task = tokio::spawn(async move {
|
let task = tokio::spawn(async move {
|
||||||
let output = run_command(
|
let output = run_command(
|
||||||
cwd,
|
cwd,
|
||||||
@@ -1802,6 +1814,7 @@ impl WorkdirSession for LocalWorkdirSession {
|
|||||||
telemetry,
|
telemetry,
|
||||||
command_environment,
|
command_environment,
|
||||||
cancel_rx,
|
cancel_rx,
|
||||||
|
resources,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let _ = completion_tx.send(true);
|
let _ = completion_tx.send(true);
|
||||||
@@ -2131,6 +2144,7 @@ async fn run_command(
|
|||||||
telemetry: CommandTelemetry,
|
telemetry: CommandTelemetry,
|
||||||
command_environment: BTreeMap<String, String>,
|
command_environment: BTreeMap<String, String>,
|
||||||
mut cancel: watch::Receiver<bool>,
|
mut cancel: watch::Receiver<bool>,
|
||||||
|
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||||
) -> Result<CommandOutput, WorkdirError> {
|
) -> Result<CommandOutput, WorkdirError> {
|
||||||
let stdout = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
let stdout = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
||||||
let stderr = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
let stderr = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
||||||
@@ -2250,7 +2264,7 @@ async fn run_command(
|
|||||||
let stdout_path = stdout_path.to_path_buf();
|
let stdout_path = stdout_path.to_path_buf();
|
||||||
let stderr_path = stderr_path.to_path_buf();
|
let stderr_path = stderr_path.to_path_buf();
|
||||||
Some(
|
Some(
|
||||||
tokio::task::spawn_blocking(move || {
|
spawn_resource_owned_blocking(resources.clone(), move || {
|
||||||
persist_command_output(&stdout_path, &stderr_path, &spill_dir)
|
persist_command_output(&stdout_path, &stderr_path, &spill_dir)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
@@ -2272,6 +2286,18 @@ async fn run_command(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Blocking spills cannot be aborted with their async waiter. Transfer resource
|
||||||
|
// ownership into the actual filesystem task so cleanup waits for its completion.
|
||||||
|
fn spawn_resource_owned_blocking<T: Send + 'static>(
|
||||||
|
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||||
|
effect: impl FnOnce() -> T + Send + 'static,
|
||||||
|
) -> JoinHandle<T> {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let _resources = resources;
|
||||||
|
effect()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
struct CommandOutputDecoder {
|
struct CommandOutputDecoder {
|
||||||
read_offset: u64,
|
read_offset: u64,
|
||||||
@@ -4065,6 +4091,152 @@ mod tests {
|
|||||||
assert_eq!(terminal, Some((handle.0, CommandStatus::TimedOut, None)));
|
assert_eq!(terminal, Some((handle.0, CommandStatus::TimedOut, None)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn command_admission_serializes_with_close_before_capturing_resources() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let session = Arc::new(make_fs(&dir));
|
||||||
|
let held = session.inner.close_lock.lock().await;
|
||||||
|
let (closing, close_entered) = tokio::sync::oneshot::channel();
|
||||||
|
let close_session = session.clone();
|
||||||
|
let close = tokio::spawn(async move {
|
||||||
|
closing.send(()).unwrap();
|
||||||
|
WorkdirSession::close(close_session.as_ref()).await
|
||||||
|
});
|
||||||
|
close_entered.await.unwrap();
|
||||||
|
let (starting, start_entered) = tokio::sync::oneshot::channel();
|
||||||
|
let start_session = session.clone();
|
||||||
|
let start = tokio::spawn(async move {
|
||||||
|
starting.send(()).unwrap();
|
||||||
|
WorkdirSession::start_command(
|
||||||
|
start_session.as_ref(),
|
||||||
|
CommandRequest {
|
||||||
|
command: "true".into(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
output_limit: 1024,
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
spill_dir: None,
|
||||||
|
tool_call_id: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
start_entered.await.unwrap();
|
||||||
|
assert!(
|
||||||
|
!start.is_finished(),
|
||||||
|
"command started outside terminal-close admission"
|
||||||
|
);
|
||||||
|
drop(held);
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), close)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), start)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap(),
|
||||||
|
Err(WorkdirError::Unavailable(message)) if message.contains("is closed")
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
session.command_snapshot().is_empty(),
|
||||||
|
"closed admission launched a command"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn blocking_output_spill_retains_resources_after_waiter_is_cancelled() {
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct Resource(Option<tokio::sync::oneshot::Sender<()>>);
|
||||||
|
impl Drop for Resource {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.0.take().unwrap().send(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (released, mut release) = tokio::sync::oneshot::channel();
|
||||||
|
let resources: Vec<Arc<dyn WorkdirSessionResource>> =
|
||||||
|
vec![Arc::new(Resource(Some(released)))];
|
||||||
|
let (entered, started) = tokio::sync::oneshot::channel();
|
||||||
|
let (finish, wait) = std::sync::mpsc::channel();
|
||||||
|
// Exercise the resource-transfer policy at the same blocking boundary
|
||||||
|
// used by output persistence, without filesystem timing assumptions.
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
spawn_resource_owned_blocking(resources, move || {
|
||||||
|
entered.send(()).unwrap();
|
||||||
|
wait.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
});
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), started)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
waiter.abort();
|
||||||
|
assert!(waiter.await.unwrap_err().is_cancelled());
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
release.try_recv(),
|
||||||
|
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
|
||||||
|
),
|
||||||
|
"blocking spill lost its resources when the waiter was cancelled"
|
||||||
|
);
|
||||||
|
finish.send(()).unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), release)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn session_drop_retains_resources_until_command_task_cancellation_completes() {
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct Resource(Option<tokio::sync::oneshot::Sender<()>>);
|
||||||
|
impl Drop for Resource {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.0.take().unwrap().send(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let (released, mut release) = tokio::sync::oneshot::channel();
|
||||||
|
let session = LocalWorkdirSession::materialized_bound_with_environment(
|
||||||
|
Workdir::new("command-resource-session"),
|
||||||
|
dir.path().to_path_buf(),
|
||||||
|
dir.path().to_path_buf(),
|
||||||
|
SharedScope::new(Scope::writable(dir.path()).unwrap()),
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
BTreeMap::new(),
|
||||||
|
vec![Arc::new(Resource(Some(released)))],
|
||||||
|
);
|
||||||
|
WorkdirSession::start_command(
|
||||||
|
&session,
|
||||||
|
CommandRequest {
|
||||||
|
command: "true".into(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
output_limit: 1024,
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
spill_dir: None,
|
||||||
|
tool_call_id: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
// The sole executor has not polled the command task, so dropping the
|
||||||
|
// session schedules its abort but cannot yet complete cancellation.
|
||||||
|
drop(session);
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
release.try_recv(),
|
||||||
|
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
|
||||||
|
),
|
||||||
|
"resources released before the aborted command future was dropped"
|
||||||
|
);
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), release)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn closing_session_releases_runtime_resources() {
|
async fn closing_session_releases_runtime_resources() {
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
|||||||
@@ -951,10 +951,12 @@ async fn observe_repository_ref(
|
|||||||
async fn list_working_directories(
|
async fn list_working_directories(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
) -> RestResult<RuntimeHttpWorkingDirectoriesResponse> {
|
) -> RestResult<RuntimeHttpWorkingDirectoriesResponse> {
|
||||||
let working_directories = state
|
// Inventory providers may inspect the filesystem even for read-only requests.
|
||||||
.runtime
|
let working_directories =
|
||||||
.list_working_directories()
|
tokio::task::spawn_blocking(move || state.runtime.list_working_directories())
|
||||||
.map_err(RuntimeHttpRestError::runtime)?;
|
.await
|
||||||
|
.map_err(RuntimeHttpRestError::working_directory_join)?
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)?;
|
||||||
Ok(Json(RuntimeHttpWorkingDirectoriesResponse {
|
Ok(Json(RuntimeHttpWorkingDirectoriesResponse {
|
||||||
working_directories,
|
working_directories,
|
||||||
}))
|
}))
|
||||||
@@ -989,10 +991,11 @@ async fn get_working_directory(
|
|||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
Path(working_directory_id): Path<String>,
|
Path(working_directory_id): Path<String>,
|
||||||
) -> RestResult<RuntimeHttpWorkingDirectoryResponse> {
|
) -> RestResult<RuntimeHttpWorkingDirectoryResponse> {
|
||||||
let working_directory = state
|
let working_directory =
|
||||||
.runtime
|
tokio::task::spawn_blocking(move || state.runtime.working_directory(&working_directory_id))
|
||||||
.working_directory(&working_directory_id)
|
.await
|
||||||
.map_err(RuntimeHttpRestError::runtime)?;
|
.map_err(RuntimeHttpRestError::working_directory_join)?
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)?;
|
||||||
Ok(Json(RuntimeHttpWorkingDirectoryResponse {
|
Ok(Json(RuntimeHttpWorkingDirectoryResponse {
|
||||||
working_directory,
|
working_directory,
|
||||||
}))
|
}))
|
||||||
@@ -1130,10 +1133,16 @@ async fn cleanup_working_directory(
|
|||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
Path(working_directory_id): Path<String>,
|
Path(working_directory_id): Path<String>,
|
||||||
) -> RestResult<RuntimeHttpWorkingDirectoryResponse> {
|
) -> RestResult<RuntimeHttpWorkingDirectoryResponse> {
|
||||||
let working_directory = state
|
// Keep Runtime's occupancy check and deletion together, but do not run
|
||||||
.runtime
|
// synchronous filesystem/Git work on the HTTP executor.
|
||||||
.cleanup_working_directory(&working_directory_id)
|
let working_directory = tokio::task::spawn_blocking(move || {
|
||||||
.map_err(RuntimeHttpRestError::runtime)?;
|
state
|
||||||
|
.runtime
|
||||||
|
.cleanup_working_directory(&working_directory_id)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(RuntimeHttpRestError::working_directory_join)?
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)?;
|
||||||
Ok(Json(RuntimeHttpWorkingDirectoryResponse {
|
Ok(Json(RuntimeHttpWorkingDirectoryResponse {
|
||||||
working_directory,
|
working_directory,
|
||||||
}))
|
}))
|
||||||
@@ -2443,6 +2452,16 @@ impl RuntimeHttpRestError {
|
|||||||
Self::new(status, code, error.to_string())
|
Self::new(status, code, error.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn working_directory_join(_error: tokio::task::JoinError) -> Self {
|
||||||
|
// A panic payload can contain paths or credentials. A missing result
|
||||||
|
// also does not prove that cleanup made no changes; do not claim rollback.
|
||||||
|
Self::new(
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
"working_directory_task_failed",
|
||||||
|
"Working directory operation failed to return a result",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn json_rejection(error: JsonRejection) -> Self {
|
fn json_rejection(error: JsonRejection) -> Self {
|
||||||
Self::new(
|
Self::new(
|
||||||
StatusCode::BAD_REQUEST,
|
StatusCode::BAD_REQUEST,
|
||||||
@@ -4341,34 +4360,379 @@ mod tests {
|
|||||||
assert!(!error.to_string().contains('/'));
|
assert!(!error.to_string().contains('/'));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum CleanupHttpOutcome {
|
||||||
|
Removed,
|
||||||
|
Rejected(&'static str),
|
||||||
|
Panicked,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CleanupHttpGate {
|
||||||
|
entered: tokio::sync::oneshot::Sender<()>,
|
||||||
|
release: std::sync::mpsc::Receiver<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CleanupHttpBackend {
|
||||||
|
gate: Mutex<Option<CleanupHttpGate>>,
|
||||||
|
outcome: CleanupHttpOutcome,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn removed_working_directory() -> WorkingDirectoryStatus {
|
||||||
|
WorkingDirectoryStatus {
|
||||||
|
summary: crate::catalog::WorkingDirectorySummary {
|
||||||
|
working_directory_id: "cleanup-test".to_string(),
|
||||||
|
display_name: None,
|
||||||
|
repository_id: "repository-test".to_string(),
|
||||||
|
creation_selector: None,
|
||||||
|
creation_ref: None,
|
||||||
|
creation_tree: None,
|
||||||
|
current_selector: None,
|
||||||
|
current_ref: None,
|
||||||
|
current_tree: None,
|
||||||
|
observed_at_epoch_seconds: None,
|
||||||
|
materializer_kind: crate::catalog::MaterializerKind::RuntimeGitClone,
|
||||||
|
cleanup_target: None,
|
||||||
|
status: crate::catalog::WorkingDirectoryStatusKind::NotFound,
|
||||||
|
cleanliness: None,
|
||||||
|
occupied_by: None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerExecutionBackend for CleanupHttpBackend {
|
||||||
|
fn backend_id(&self) -> &str {
|
||||||
|
"http-cleanup-test"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
|
AcceptingBackend.spawn_worker(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dispatch_input(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
input: WorkerInput,
|
||||||
|
) -> WorkerExecutionResult {
|
||||||
|
AcceptingBackend.dispatch_input(worker_ref, input)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_working_directory(
|
||||||
|
&self,
|
||||||
|
working_directory_id: &str,
|
||||||
|
) -> Result<WorkingDirectoryStatus, crate::working_directory::WorkingDirectoryDiagnostic>
|
||||||
|
{
|
||||||
|
assert_eq!(working_directory_id, "cleanup-test");
|
||||||
|
let gate = self.gate.lock().unwrap().take();
|
||||||
|
if let Some(gate) = gate {
|
||||||
|
gate.entered.send(()).expect("cleanup entry receiver");
|
||||||
|
// A watchdog also bounds a regression that blocks the sole
|
||||||
|
// Tokio thread, where an async timeout could not make progress.
|
||||||
|
gate.release
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(10))
|
||||||
|
.expect("cleanup must be released after observations complete");
|
||||||
|
}
|
||||||
|
match self.outcome {
|
||||||
|
CleanupHttpOutcome::Removed => Ok(removed_working_directory()),
|
||||||
|
CleanupHttpOutcome::Rejected(code) => Err(
|
||||||
|
crate::working_directory::WorkingDirectoryDiagnostic::rejected(
|
||||||
|
code,
|
||||||
|
"Resolve the current blocker and retry removal.",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
CleanupHttpOutcome::Panicked => panic!("private provider path /secret/cleanup"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_http_router(
|
||||||
|
outcome: CleanupHttpOutcome,
|
||||||
|
gate: Option<CleanupHttpGate>,
|
||||||
|
) -> (Runtime, Router) {
|
||||||
|
let runtime = Runtime::with_execution_backend(
|
||||||
|
RuntimeOptions::default(),
|
||||||
|
Arc::new(CleanupHttpBackend {
|
||||||
|
gate: Mutex::new(gate),
|
||||||
|
outcome,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let identity = RuntimeIdentityMaterial::generate("runtime-cleanup-test").unwrap();
|
||||||
|
let workspace_identity =
|
||||||
|
RuntimeIdentityMaterial::generate("workspace-cleanup-key").unwrap();
|
||||||
|
let workspace_public_key =
|
||||||
|
crate::auth::decode_public_key(&workspace_identity.public_key).unwrap();
|
||||||
|
let workspace_fingerprint = format!(
|
||||||
|
"sha256:{}",
|
||||||
|
crate::workspace_issuer::hex_lower(&sha2::Sha256::digest(workspace_public_key))
|
||||||
|
);
|
||||||
|
let app = runtime_http_router_with_workspace_auth(
|
||||||
|
runtime.clone(),
|
||||||
|
Some("cleanup-token".to_string()),
|
||||||
|
WorkspaceRuntimeHttpAuth {
|
||||||
|
verifier: WorkspaceCapabilityVerifier::new(
|
||||||
|
vec![WorkspaceIssuerTrustRecord {
|
||||||
|
workspace_id: "local".to_string(),
|
||||||
|
backend_url: "https://backend.test".to_string(),
|
||||||
|
key_id: "workspace-cleanup-key".to_string(),
|
||||||
|
algorithm: "ed25519".to_string(),
|
||||||
|
public_key: workspace_identity.public_key,
|
||||||
|
public_key_fingerprint: workspace_fingerprint,
|
||||||
|
identity_revision: 1,
|
||||||
|
trust_generation: 1,
|
||||||
|
state: WorkspaceIssuerTrustState::Active,
|
||||||
|
registered_at_unix: 1,
|
||||||
|
updated_at_unix: 1,
|
||||||
|
}],
|
||||||
|
Arc::new(InMemoryWorkspaceClaimReplayProtection::default()),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
signer: RuntimeVerificationSigner::from_identity(&identity).unwrap(),
|
||||||
|
verifications: Arc::new(InMemoryWorkspaceRuntimeVerificationAuthority::default()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
(runtime, app)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn slow_working_directory_cleanup_does_not_block_http_ping() {
|
||||||
|
let (entered, started) = tokio::sync::oneshot::channel();
|
||||||
|
let (release, wait_for_release) = std::sync::mpsc::channel();
|
||||||
|
let (runtime, app) = cleanup_http_router(
|
||||||
|
CleanupHttpOutcome::Removed,
|
||||||
|
Some(CleanupHttpGate {
|
||||||
|
entered,
|
||||||
|
release: wait_for_release,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
runtime
|
||||||
|
.store_config_bundle(test_bundle(ProfileSelector::Builtin(
|
||||||
|
"builtin:coder".to_string(),
|
||||||
|
)))
|
||||||
|
.unwrap();
|
||||||
|
let scope = RuntimeWorkspaceScope::new("local", "local-token");
|
||||||
|
let mut request = task_request("unrelated Worker remains observable during cleanup");
|
||||||
|
request.workspace_api = Some(WorkspaceApiRef {
|
||||||
|
workspace_id: "local".to_string(),
|
||||||
|
base_url: "https://backend.test".to_string(),
|
||||||
|
});
|
||||||
|
let worker = runtime.create_worker_scoped(&scope, request).unwrap();
|
||||||
|
assert!(worker.workdir_attachments.is_empty());
|
||||||
|
let worker_path = format!("/v1/workers/{}", worker.worker_id);
|
||||||
|
let path = "/v1/working-directories/cleanup-test";
|
||||||
|
let unauthorized = empty_request(app.clone(), Method::DELETE, path).await;
|
||||||
|
assert_eq!(unauthorized.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
let error: RuntimeHttpErrorResponse = read_json(unauthorized).await;
|
||||||
|
assert_eq!(error.error.code, "unauthorized");
|
||||||
|
|
||||||
|
// A synchronous RuntimeState lock can stall this sole executor thread,
|
||||||
|
// including Tokio timeouts. Release from an independent thread on either
|
||||||
|
// completion, assertion unwind, or watchdog expiry, and report expiry as
|
||||||
|
// failure even if observations resume after the forced release.
|
||||||
|
let (observed, wait_for_observations) = std::sync::mpsc::channel();
|
||||||
|
let watchdog = std::thread::spawn(move || {
|
||||||
|
let timed_out = matches!(
|
||||||
|
wait_for_observations.recv_timeout(std::time::Duration::from_secs(5)),
|
||||||
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
|
||||||
|
);
|
||||||
|
let _ = release.send(());
|
||||||
|
timed_out
|
||||||
|
});
|
||||||
|
let cleanup = tokio::spawn(authed_empty_request(
|
||||||
|
app.clone(),
|
||||||
|
Method::DELETE,
|
||||||
|
path,
|
||||||
|
"cleanup-token",
|
||||||
|
));
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), started)
|
||||||
|
.await
|
||||||
|
.expect("cleanup provider must be entered")
|
||||||
|
.expect("cleanup entry signal");
|
||||||
|
|
||||||
|
// Unlike ping, these HTTP routes synchronously acquire RuntimeState.
|
||||||
|
// Exercise existing Worker paths so authorization/not-found shortcuts
|
||||||
|
// cannot make a global occupancy-lock regression pass unnoticed.
|
||||||
|
for observation_path in [
|
||||||
|
"/v1/workers".to_string(),
|
||||||
|
worker_path.clone(),
|
||||||
|
format!("{worker_path}/session?workspace_id=local"),
|
||||||
|
] {
|
||||||
|
let response = tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(2),
|
||||||
|
authed_empty_request(app.clone(), Method::GET, &observation_path, "cleanup-token"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| panic!("{observation_path} must finish while cleanup is gated"));
|
||||||
|
let status = response.status();
|
||||||
|
let body: serde_json::Value = read_json(response).await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{observation_path}: {body}");
|
||||||
|
if observation_path == "/v1/workers" {
|
||||||
|
assert_eq!(body["workers"].as_array().unwrap().len(), 1, "{body}");
|
||||||
|
assert_eq!(
|
||||||
|
body["workers"][0]["worker_id"],
|
||||||
|
worker.worker_id.to_string()
|
||||||
|
);
|
||||||
|
} else if observation_path == worker_path {
|
||||||
|
assert_eq!(body["worker"]["worker_id"], worker.worker_id.to_string());
|
||||||
|
} else {
|
||||||
|
// This minimal fake has no session store. Its normal, typed
|
||||||
|
// availability result still requires the real scoped HTTP read.
|
||||||
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
serde_json::to_value(runtime_api::WorkerSessionAvailability::Unavailable {
|
||||||
|
reason: runtime_api::WorkerSessionUnavailableReason::StorageUnavailable,
|
||||||
|
message: "retained session storage is unavailable".to_string(),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exercise the same scoped snapshot acquisition used by Runtime WS
|
||||||
|
// subscriptions, without requiring a socket or the ws-server feature.
|
||||||
|
let subscription = runtime
|
||||||
|
.subscribe_event_selector_scoped(
|
||||||
|
&scope,
|
||||||
|
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||||
|
)
|
||||||
|
.expect("Runtime subscription snapshot must complete while cleanup is gated");
|
||||||
|
let protocol::subscription::SubscriptionSnapshot::Workers { workers } =
|
||||||
|
subscription.snapshot()
|
||||||
|
else {
|
||||||
|
panic!("expected Runtime Worker subscription snapshot");
|
||||||
|
};
|
||||||
|
assert_eq!(workers.len(), 1);
|
||||||
|
assert_eq!(workers[0].worker_id.as_str(), worker.worker_id.to_string());
|
||||||
|
|
||||||
|
let ping = tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(2),
|
||||||
|
app.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(Method::GET)
|
||||||
|
.uri("/v1/ping")
|
||||||
|
.header(header::AUTHORIZATION, "Bearer cleanup-token")
|
||||||
|
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, "local")
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("ping must finish while cleanup is gated")
|
||||||
|
.unwrap();
|
||||||
|
let ping_status = ping.status();
|
||||||
|
let ping_body: serde_json::Value = read_json(ping).await;
|
||||||
|
assert_eq!(ping_status, StatusCode::OK, "{ping_body}");
|
||||||
|
assert_eq!(ping_body["runtime_id"], "runtime-cleanup-test");
|
||||||
|
assert_eq!(ping_body["protocol_version"], RUNTIME_HTTP_PROTOCOL_VERSION);
|
||||||
|
assert!(
|
||||||
|
!cleanup.is_finished(),
|
||||||
|
"cleanup must still be gated after ping",
|
||||||
|
);
|
||||||
|
// The watchdog owns the release sender; dropping `observed` on an
|
||||||
|
// assertion failure also releases the provider instead of leaking a task.
|
||||||
|
let observations_completed = observed.send(()).is_ok();
|
||||||
|
let watchdog_expired = watchdog.join().unwrap();
|
||||||
|
assert!(
|
||||||
|
observations_completed && !watchdog_expired,
|
||||||
|
"Worker observations, subscription snapshot, and ping must finish before cleanup is released",
|
||||||
|
);
|
||||||
|
let response = tokio::time::timeout(std::time::Duration::from_secs(2), cleanup)
|
||||||
|
.await
|
||||||
|
.expect("released cleanup must finish")
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body: serde_json::Value = read_json(response).await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{body}");
|
||||||
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
serde_json::to_value(RuntimeHttpWorkingDirectoryResponse {
|
||||||
|
working_directory: removed_working_directory(),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn working_directory_cleanup_panic_returns_bounded_http_error() {
|
||||||
|
let (_, app) = cleanup_http_router(CleanupHttpOutcome::Panicked, None);
|
||||||
|
let response = authed_empty_request(
|
||||||
|
app,
|
||||||
|
Method::DELETE,
|
||||||
|
"/v1/working-directories/cleanup-test",
|
||||||
|
"cleanup-token",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let status = response.status();
|
||||||
|
let body: serde_json::Value = read_json(response).await;
|
||||||
|
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
|
||||||
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
serde_json::json!({
|
||||||
|
"error": {
|
||||||
|
"code": "working_directory_task_failed",
|
||||||
|
"message": "Working directory operation failed to return a result",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cleanup_cause_codes_survive_http_error_serialization() {
|
async fn cleanup_cause_codes_survive_http_error_serialization() {
|
||||||
let cases = [
|
let cases = [
|
||||||
("mount_present", StatusCode::CONFLICT),
|
("working_directory_not_found", StatusCode::NOT_FOUND),
|
||||||
("mount_check_unavailable", StatusCode::SERVICE_UNAVAILABLE),
|
(
|
||||||
("permission_denied", StatusCode::FORBIDDEN),
|
"working_directory_cleanup_mount_present",
|
||||||
("resource_busy", StatusCode::CONFLICT),
|
StatusCode::CONFLICT,
|
||||||
("storage_unavailable", StatusCode::SERVICE_UNAVAILABLE),
|
),
|
||||||
("ownership_unknown", StatusCode::CONFLICT),
|
(
|
||||||
("changes_present", StatusCode::CONFLICT),
|
"working_directory_cleanup_mount_check_unavailable",
|
||||||
("changes_unknown", StatusCode::CONFLICT),
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_cleanup_permission_denied",
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_cleanup_resource_busy",
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_cleanup_storage_unavailable",
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_cleanup_ownership_unknown",
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_cleanup_changes_present",
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_cleanup_changes_unknown",
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
),
|
||||||
];
|
];
|
||||||
for (cause, status) in cases {
|
for (code, expected_status) in cases {
|
||||||
let code = format!("working_directory_cleanup_{cause}");
|
let (_, app) = cleanup_http_router(CleanupHttpOutcome::Rejected(code), None);
|
||||||
|
let response = authed_empty_request(
|
||||||
|
app,
|
||||||
|
Method::DELETE,
|
||||||
|
"/v1/working-directories/cleanup-test",
|
||||||
|
"cleanup-token",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let status = response.status();
|
||||||
|
let json: serde_json::Value = read_json(response).await;
|
||||||
|
assert_eq!(status, expected_status, "{json}");
|
||||||
|
assert_eq!(json["error"]["code"], code, "{json}");
|
||||||
let diagnostic = crate::working_directory::WorkingDirectoryDiagnostic::rejected(
|
let diagnostic = crate::working_directory::WorkingDirectoryDiagnostic::rejected(
|
||||||
&code,
|
code,
|
||||||
"Resolve the current blocker and retry removal.",
|
"Resolve the current blocker and retry removal.",
|
||||||
);
|
);
|
||||||
let response =
|
assert_eq!(
|
||||||
RuntimeHttpRestError::runtime(RuntimeError::WorkingDirectory(diagnostic))
|
json["error"]["message"],
|
||||||
.into_response();
|
RuntimeError::WorkingDirectory(diagnostic).to_string(),
|
||||||
assert_eq!(response.status(), status);
|
"{json}",
|
||||||
let body = axum::body::to_bytes(response.into_body(), 2048)
|
);
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
|
||||||
assert_eq!(json["error"]["code"], code, "{json}");
|
|
||||||
assert!(json["error"]["message"].as_str().unwrap().contains("retry"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -59,6 +59,54 @@ use tokio::sync::broadcast;
|
|||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
mod workdir_session;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum WorkdirOperation {
|
||||||
|
Use(usize),
|
||||||
|
Cleanup,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Transient admission authority, not persisted cleanup or recovery state.
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct WorkdirOperationGuard {
|
||||||
|
state: Weak<Mutex<RuntimeState>>,
|
||||||
|
ids: Vec<String>,
|
||||||
|
cleanup: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for WorkdirOperationGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let Some(state) = self.state.upgrade() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let mut state = state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
for id in &self.ids {
|
||||||
|
match state.workdir_operations.get_mut(id) {
|
||||||
|
Some(WorkdirOperation::Use(count)) if !self.cleanup && *count > 1 => *count -= 1,
|
||||||
|
Some(WorkdirOperation::Use(_)) if !self.cleanup => {
|
||||||
|
state.workdir_operations.remove(id);
|
||||||
|
}
|
||||||
|
Some(WorkdirOperation::Cleanup) if self.cleanup => {
|
||||||
|
state.workdir_operations.remove(id);
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn workdir_operation_busy() -> RuntimeError {
|
||||||
|
RuntimeError::WorkingDirectory(
|
||||||
|
crate::working_directory::WorkingDirectoryDiagnostic::rejected(
|
||||||
|
"working_directory_cleanup_resource_busy",
|
||||||
|
"Workdir has an in-progress operation; retry after it finishes",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Workspace-scoped Runtime authorization context supplied by a trusted backend.
|
/// Workspace-scoped Runtime authorization context supplied by a trusted backend.
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct RuntimeWorkspaceScope {
|
pub struct RuntimeWorkspaceScope {
|
||||||
@@ -296,6 +344,37 @@ impl Drop for WorkerOperationLease {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Runtime {
|
impl Runtime {
|
||||||
|
fn reserve_workdir_use<'a>(
|
||||||
|
&self,
|
||||||
|
ids: impl IntoIterator<Item = &'a str>,
|
||||||
|
) -> Result<WorkdirOperationGuard, RuntimeError> {
|
||||||
|
let mut ids: Vec<String> = ids.into_iter().map(str::to_string).collect();
|
||||||
|
ids.sort();
|
||||||
|
ids.dedup();
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
for id in &ids {
|
||||||
|
state.ensure_workdir_not_removing(id)?;
|
||||||
|
}
|
||||||
|
for id in &ids {
|
||||||
|
match state
|
||||||
|
.workdir_operations
|
||||||
|
.entry(id.clone())
|
||||||
|
.or_insert(WorkdirOperation::Use(0))
|
||||||
|
{
|
||||||
|
WorkdirOperation::Use(count) => *count += 1,
|
||||||
|
WorkdirOperation::Cleanup => {
|
||||||
|
unreachable!("cleanup admission checked under state lock")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(WorkdirOperationGuard {
|
||||||
|
state: Arc::downgrade(&self.inner),
|
||||||
|
ids,
|
||||||
|
cleanup: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Create a memory-backed Runtime with generated identity.
|
/// Create a memory-backed Runtime with generated identity.
|
||||||
pub fn new_memory() -> Self {
|
pub fn new_memory() -> Self {
|
||||||
Self::with_options(RuntimeOptions::default())
|
Self::with_options(RuntimeOptions::default())
|
||||||
@@ -515,8 +594,15 @@ impl Runtime {
|
|||||||
/// Create a Runtime-owned working directory through the attached execution backend.
|
/// Create a Runtime-owned working directory through the attached execution backend.
|
||||||
pub fn create_working_directory(
|
pub fn create_working_directory(
|
||||||
&self,
|
&self,
|
||||||
request: WorkingDirectoryRequest,
|
mut request: WorkingDirectoryRequest,
|
||||||
) -> Result<CatalogWorkingDirectoryStatus, RuntimeError> {
|
) -> Result<CatalogWorkingDirectoryStatus, RuntimeError> {
|
||||||
|
let id = request
|
||||||
|
.backend_workdir_id
|
||||||
|
.get_or_insert_with(|| {
|
||||||
|
crate::working_directory::next_working_directory_id(&request.repository.id)
|
||||||
|
})
|
||||||
|
.clone();
|
||||||
|
let _operation = self.reserve_workdir_use([id.as_str()])?;
|
||||||
let backend = {
|
let backend = {
|
||||||
let state = self.lock()?;
|
let state = self.lock()?;
|
||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
@@ -581,6 +667,7 @@ impl Runtime {
|
|||||||
&self,
|
&self,
|
||||||
request: WorkingDirectoryRepositoryAccessRequest,
|
request: WorkingDirectoryRepositoryAccessRequest,
|
||||||
) -> Result<(), RuntimeError> {
|
) -> Result<(), RuntimeError> {
|
||||||
|
let _operation = self.reserve_workdir_use([request.working_directory_id.as_str()])?;
|
||||||
let backend = {
|
let backend = {
|
||||||
let state = self.lock()?;
|
let state = self.lock()?;
|
||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
@@ -775,6 +862,7 @@ impl Runtime {
|
|||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
owner_worker_ref: Option<&WorkerRef>,
|
owner_worker_ref: Option<&WorkerRef>,
|
||||||
) -> Result<workdir::WorkdirSessionHandle, RuntimeError> {
|
) -> Result<workdir::WorkdirSessionHandle, RuntimeError> {
|
||||||
|
let operation = self.reserve_workdir_use([working_directory_id])?;
|
||||||
let backend = {
|
let backend = {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
@@ -809,9 +897,10 @@ impl Runtime {
|
|||||||
backend
|
backend
|
||||||
.working_directory(working_directory_id)
|
.working_directory(working_directory_id)
|
||||||
.map_err(RuntimeError::WorkingDirectory)?;
|
.map_err(RuntimeError::WorkingDirectory)?;
|
||||||
backend
|
let session = backend
|
||||||
.open_workdir_session(working_directory_id)
|
.open_workdir_session(working_directory_id)
|
||||||
.map_err(RuntimeError::WorkingDirectory)
|
.map_err(RuntimeError::WorkingDirectory)?;
|
||||||
|
Ok(workdir_session::retain_operation(session, operation))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Cleanup a Runtime-owned working directory.
|
/// Cleanup a Runtime-owned working directory.
|
||||||
@@ -819,30 +908,40 @@ impl Runtime {
|
|||||||
&self,
|
&self,
|
||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
) -> Result<CatalogWorkingDirectoryStatus, RuntimeError> {
|
) -> Result<CatalogWorkingDirectoryStatus, RuntimeError> {
|
||||||
// Hold occupancy authority through the side effect, preventing an
|
// Admit cleanup atomically with attachment/use admission, but reserve
|
||||||
// attachment from entering between the check and physical removal.
|
// only this Workdir while the provider hashes/unlinks. No filesystem
|
||||||
// This synchronous provider call must not reenter Runtime. The Git
|
// work or wait for a Workdir operation may hold the global state mutex.
|
||||||
// materializer does filesystem/Git work only. The global guard also
|
let backend = {
|
||||||
// delays unrelated state operations during hashing/unlink; releasing it
|
let mut state = self.lock()?;
|
||||||
// would require equivalent per-workdir exclusion on ALL attachment paths,
|
state.ensure_running()?;
|
||||||
// not just moving the occupied check before the provider call.
|
if state.workdir_operations.contains_key(working_directory_id) {
|
||||||
let state = self.lock()?;
|
return Err(workdir_operation_busy());
|
||||||
state.ensure_running()?;
|
|
||||||
if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) {
|
|
||||||
return Err(RuntimeError::InvalidRequest(format!(
|
|
||||||
"working directory {working_directory_id} is assigned to worker {worker_id}"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let backend = state.execution_backend.clone().ok_or_else(|| {
|
|
||||||
RuntimeError::ExecutionBackendUnavailable {
|
|
||||||
message: "working directory cleanup requires an execution backend".to_string(),
|
|
||||||
}
|
}
|
||||||
})?;
|
if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) {
|
||||||
let result = backend
|
return Err(RuntimeError::InvalidRequest(format!(
|
||||||
|
"working directory {working_directory_id} is assigned to worker {worker_id}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let backend = state.execution_backend.clone().ok_or_else(|| {
|
||||||
|
RuntimeError::ExecutionBackendUnavailable {
|
||||||
|
message: "working directory cleanup requires an execution backend".to_string(),
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
state
|
||||||
|
.workdir_operations
|
||||||
|
.insert(working_directory_id.to_string(), WorkdirOperation::Cleanup);
|
||||||
|
backend
|
||||||
|
};
|
||||||
|
// Construct/drop the RAII owner outside the state guard: failure, panic
|
||||||
|
// and request cancellation must not leave stale exclusion or deadlock.
|
||||||
|
let _operation = WorkdirOperationGuard {
|
||||||
|
state: Arc::downgrade(&self.inner),
|
||||||
|
ids: vec![working_directory_id.to_string()],
|
||||||
|
cleanup: true,
|
||||||
|
};
|
||||||
|
backend
|
||||||
.cleanup_working_directory(working_directory_id)
|
.cleanup_working_directory(working_directory_id)
|
||||||
.map_err(RuntimeError::from);
|
.map_err(RuntimeError::from)
|
||||||
drop(state);
|
|
||||||
result
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn annotate_working_directory_statuses(
|
fn annotate_working_directory_statuses(
|
||||||
@@ -1041,7 +1140,7 @@ impl Runtime {
|
|||||||
|
|
||||||
fn create_worker_with_workspace_inner(
|
fn create_worker_with_workspace_inner(
|
||||||
&self,
|
&self,
|
||||||
request: CreateWorkerRequest,
|
mut request: CreateWorkerRequest,
|
||||||
scope: Option<&RuntimeWorkspaceScope>,
|
scope: Option<&RuntimeWorkspaceScope>,
|
||||||
) -> Result<WorkerDetail, RuntimeError> {
|
) -> Result<WorkerDetail, RuntimeError> {
|
||||||
let operation_lock = self.worker_operation_lock(request.worker_id)?;
|
let operation_lock = self.worker_operation_lock(request.worker_id)?;
|
||||||
@@ -1051,6 +1150,17 @@ impl Runtime {
|
|||||||
if let Some(existing) = self.existing_worker_for_create(&request, scope)? {
|
if let Some(existing) = self.existing_worker_for_create(&request, scope)? {
|
||||||
return Ok(existing);
|
return Ok(existing);
|
||||||
}
|
}
|
||||||
|
// Assign generated materializations before publishing the durable
|
||||||
|
// request, so cleanup cannot discover an unclaimed creation mid-spawn.
|
||||||
|
for attachment in &mut request.workdir_attachment_requests {
|
||||||
|
let directory = &mut attachment.working_directory;
|
||||||
|
if directory.backend_workdir_id.is_none() {
|
||||||
|
directory.backend_workdir_id = Some(
|
||||||
|
crate::working_directory::next_working_directory_id(&directory.repository.id),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _workdir_operation = self.reserve_workdir_use(requested_workdir_ids(&request))?;
|
||||||
self.refresh_workspace_config(&request)?;
|
self.refresh_workspace_config(&request)?;
|
||||||
let (backend, worker_ref, spawn_request) = {
|
let (backend, worker_ref, spawn_request) = {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
@@ -1565,6 +1675,9 @@ impl Runtime {
|
|||||||
attachments.sort_by(|left, right| left.alias.cmp(&right.alias));
|
attachments.sort_by(|left, right| left.alias.cmp(&right.alias));
|
||||||
|
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
|
for attachment in &attachments {
|
||||||
|
state.ensure_workdir_not_removing(&attachment.working_directory_id)?;
|
||||||
|
}
|
||||||
let previous = state.worker(worker_ref)?.clone();
|
let previous = state.worker(worker_ref)?.clone();
|
||||||
{
|
{
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
@@ -1875,6 +1988,13 @@ impl Runtime {
|
|||||||
if let Some(receipt) = &owner.receipt {
|
if let Some(receipt) = &owner.receipt {
|
||||||
return Ok((Some(receipt.clone()), None));
|
return Ok((Some(receipt.clone()), None));
|
||||||
}
|
}
|
||||||
|
if let Some(preparation) = &owner.request.preparation {
|
||||||
|
for id in
|
||||||
|
preparation_workdir_ids(preparation, state.runtime_identity.as_deref())
|
||||||
|
{
|
||||||
|
state.ensure_workdir_not_removing(id)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
if candidate.restore_guard.active_request_id.as_deref() != Some(&request.request_id)
|
if candidate.restore_guard.active_request_id.as_deref() != Some(&request.request_id)
|
||||||
|| candidate
|
|| candidate
|
||||||
.pending_restore
|
.pending_restore
|
||||||
@@ -2074,6 +2194,19 @@ impl Runtime {
|
|||||||
state.ensure_workspace_owner(scope, true)?;
|
state.ensure_workspace_owner(scope, true)?;
|
||||||
state.persist_runtime_snapshot()?;
|
state.persist_runtime_snapshot()?;
|
||||||
}
|
}
|
||||||
|
// Recovery uses the persisted operation, not new preparation.
|
||||||
|
// Discard it before checking Workdir admission as well as before
|
||||||
|
// publishing the owner; unrelated cleanup must not reject recovery.
|
||||||
|
if recovering_pending {
|
||||||
|
request.preparation = None;
|
||||||
|
}
|
||||||
|
if let Some(preparation) = &request.preparation {
|
||||||
|
for id in
|
||||||
|
preparation_workdir_ids(preparation, state.runtime_identity.as_deref())
|
||||||
|
{
|
||||||
|
state.ensure_workdir_not_removing(id)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
let pending = candidate
|
let pending = candidate
|
||||||
.pending_restore
|
.pending_restore
|
||||||
.unwrap_or_else(|| PendingWorkerRestore {
|
.unwrap_or_else(|| PendingWorkerRestore {
|
||||||
@@ -2083,9 +2216,6 @@ impl Runtime {
|
|||||||
});
|
});
|
||||||
let operation_id = pending.operation_id;
|
let operation_id = pending.operation_id;
|
||||||
candidate.pending_restore = Some(pending);
|
candidate.pending_restore = Some(pending);
|
||||||
if recovering_pending {
|
|
||||||
request.preparation = None;
|
|
||||||
}
|
|
||||||
candidate.restore_guard.rotate();
|
candidate.restore_guard.rotate();
|
||||||
candidate.restore_guard.active_request_id = Some(request.request_id.clone());
|
candidate.restore_guard.active_request_id = Some(request.request_id.clone());
|
||||||
candidate.restore_guard.owners.insert(
|
candidate.restore_guard.owners.insert(
|
||||||
@@ -2517,6 +2647,28 @@ impl Runtime {
|
|||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let _workdir_operation = self.reserve_workdir_use(
|
||||||
|
requested_workdir_ids(&request.request)
|
||||||
|
.into_iter()
|
||||||
|
.chain(
|
||||||
|
request
|
||||||
|
.previous_workdir_attachments
|
||||||
|
.iter()
|
||||||
|
.map(|attachment| {
|
||||||
|
attachment
|
||||||
|
.working_directory
|
||||||
|
.summary
|
||||||
|
.working_directory_id
|
||||||
|
.as_str()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.chain(
|
||||||
|
request
|
||||||
|
.logical_workdir_attachments
|
||||||
|
.iter()
|
||||||
|
.map(|attachment| attachment.working_directory_id.as_str()),
|
||||||
|
),
|
||||||
|
)?;
|
||||||
if !reconcile && let Err(result) = backend.preflight_restore(&request) {
|
if !reconcile && let Err(result) = backend.preflight_restore(&request) {
|
||||||
// A preflight may attest an already-connected Controller snapshot.
|
// A preflight may attest an already-connected Controller snapshot.
|
||||||
// Repeated restore is read-only; do not replace its operation owner.
|
// Repeated restore is read-only; do not replace its operation owner.
|
||||||
@@ -4853,6 +5005,7 @@ struct RuntimeState {
|
|||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
next_diagnostic_id: u64,
|
next_diagnostic_id: u64,
|
||||||
workers: BTreeMap<WorkerId, WorkerRecord>,
|
workers: BTreeMap<WorkerId, WorkerRecord>,
|
||||||
|
workdir_operations: BTreeMap<String, WorkdirOperation>,
|
||||||
workspace_owners: BTreeMap<String, String>,
|
workspace_owners: BTreeMap<String, String>,
|
||||||
config_bundles: BTreeMap<String, ConfigBundle>,
|
config_bundles: BTreeMap<String, ConfigBundle>,
|
||||||
workspace_config_latest: BTreeMap<String, ConfigBundleRef>,
|
workspace_config_latest: BTreeMap<String, ConfigBundleRef>,
|
||||||
@@ -4882,6 +5035,7 @@ impl RuntimeState {
|
|||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
next_diagnostic_id: 1,
|
next_diagnostic_id: 1,
|
||||||
workers: BTreeMap::new(),
|
workers: BTreeMap::new(),
|
||||||
|
workdir_operations: BTreeMap::new(),
|
||||||
workspace_owners: BTreeMap::new(),
|
workspace_owners: BTreeMap::new(),
|
||||||
config_bundles: BTreeMap::new(),
|
config_bundles: BTreeMap::new(),
|
||||||
workspace_config_latest: BTreeMap::new(),
|
workspace_config_latest: BTreeMap::new(),
|
||||||
@@ -4912,6 +5066,7 @@ impl RuntimeState {
|
|||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
next_diagnostic_id: 1,
|
next_diagnostic_id: 1,
|
||||||
workers: BTreeMap::new(),
|
workers: BTreeMap::new(),
|
||||||
|
workdir_operations: BTreeMap::new(),
|
||||||
workspace_owners: BTreeMap::new(),
|
workspace_owners: BTreeMap::new(),
|
||||||
config_bundles: BTreeMap::new(),
|
config_bundles: BTreeMap::new(),
|
||||||
workspace_config_latest: BTreeMap::new(),
|
workspace_config_latest: BTreeMap::new(),
|
||||||
@@ -5056,6 +5211,7 @@ impl RuntimeState {
|
|||||||
workspace_backend_resource_clients: BTreeMap::new(),
|
workspace_backend_resource_clients: BTreeMap::new(),
|
||||||
next_diagnostic_id,
|
next_diagnostic_id,
|
||||||
workers,
|
workers,
|
||||||
|
workdir_operations: BTreeMap::new(),
|
||||||
config_bundles: BTreeMap::new(),
|
config_bundles: BTreeMap::new(),
|
||||||
workspace_config_latest: BTreeMap::new(),
|
workspace_config_latest: BTreeMap::new(),
|
||||||
workspace_config_fetch_gates: BTreeMap::new(),
|
workspace_config_fetch_gates: BTreeMap::new(),
|
||||||
@@ -5470,6 +5626,16 @@ impl RuntimeState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn ensure_workdir_not_removing(&self, id: &str) -> Result<(), RuntimeError> {
|
||||||
|
if matches!(
|
||||||
|
self.workdir_operations.get(id),
|
||||||
|
Some(WorkdirOperation::Cleanup)
|
||||||
|
) {
|
||||||
|
return Err(workdir_operation_busy());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn worker_id_for_workdir(&self, working_directory_id: &str) -> Option<WorkerId> {
|
fn worker_id_for_workdir(&self, working_directory_id: &str) -> Option<WorkerId> {
|
||||||
self.workers.values().find_map(|worker| {
|
self.workers.values().find_map(|worker| {
|
||||||
let has_status = worker.workdir_attachments.iter().any(|attachment| {
|
let has_status = worker.workdir_attachments.iter().any(|attachment| {
|
||||||
@@ -5478,7 +5644,23 @@ impl RuntimeState {
|
|||||||
let has_request = worker.request.as_ref().is_some_and(|request| {
|
let has_request = worker.request.as_ref().is_some_and(|request| {
|
||||||
requested_workdir_ids(request).contains(&working_directory_id)
|
requested_workdir_ids(request).contains(&working_directory_id)
|
||||||
});
|
});
|
||||||
(has_status || has_request).then_some(worker.worker_id)
|
let has_logical = worker
|
||||||
|
.logical_workdir_attachments
|
||||||
|
.iter()
|
||||||
|
.any(|attachment| attachment.working_directory_id == working_directory_id);
|
||||||
|
let has_preparation = worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||||
|
.filter(|owner| owner.receipt.is_none())
|
||||||
|
.and_then(|owner| owner.request.preparation.as_ref())
|
||||||
|
.is_some_and(|preparation| {
|
||||||
|
preparation_workdir_ids(preparation, self.runtime_identity.as_deref())
|
||||||
|
.contains(&working_directory_id)
|
||||||
|
});
|
||||||
|
(has_status || has_request || has_logical || has_preparation)
|
||||||
|
.then_some(worker.worker_id)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -6179,6 +6361,33 @@ fn durable_create_worker_request(request: &CreateWorkerRequest) -> CreateWorkerR
|
|||||||
durable
|
durable
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn preparation_workdir_ids<'a>(
|
||||||
|
preparation: &'a runtime_api::WorkerRestorePreparation,
|
||||||
|
runtime_id: Option<&str>,
|
||||||
|
) -> Vec<&'a str> {
|
||||||
|
let local = |owner: &str| runtime_id.is_none_or(|runtime_id| runtime_id == owner);
|
||||||
|
preparation
|
||||||
|
.workdir_attachments
|
||||||
|
.iter()
|
||||||
|
.flatten()
|
||||||
|
.map(|attachment| attachment.working_directory_id.as_str())
|
||||||
|
.chain(
|
||||||
|
preparation
|
||||||
|
.repository_access_workdirs
|
||||||
|
.iter()
|
||||||
|
.filter(|reference| local(&reference.runtime_id))
|
||||||
|
.map(|reference| reference.working_directory_id.as_str()),
|
||||||
|
)
|
||||||
|
.chain(
|
||||||
|
preparation
|
||||||
|
.repository_access
|
||||||
|
.iter()
|
||||||
|
.filter(|access| local(&access.materialization.runtime_id))
|
||||||
|
.map(|access| access.working_directory_id.as_str()),
|
||||||
|
)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn requested_workdir_ids(request: &CreateWorkerRequest) -> Vec<&str> {
|
fn requested_workdir_ids(request: &CreateWorkerRequest) -> Vec<&str> {
|
||||||
request
|
request
|
||||||
.workdir_attachments
|
.workdir_attachments
|
||||||
@@ -7641,6 +7850,35 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct WorkdirEffectGate {
|
||||||
|
entered: std::sync::mpsc::Sender<String>,
|
||||||
|
release: std::sync::mpsc::Receiver<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkdirEffectGate {
|
||||||
|
fn wait(slot: &Mutex<Option<Self>>, id: &str) {
|
||||||
|
let gate = slot.lock().unwrap().take();
|
||||||
|
if let Some(gate) = gate {
|
||||||
|
gate.entered.send(id.to_string()).unwrap();
|
||||||
|
gate.release
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(5))
|
||||||
|
.expect("Workdir effect did not receive release before timeout");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install(
|
||||||
|
slot: &Mutex<Option<Self>>,
|
||||||
|
) -> (
|
||||||
|
std::sync::mpsc::Receiver<String>,
|
||||||
|
std::sync::mpsc::Sender<()>,
|
||||||
|
) {
|
||||||
|
let (entered, receiver) = std::sync::mpsc::channel();
|
||||||
|
let (sender, release) = std::sync::mpsc::channel();
|
||||||
|
*slot.lock().unwrap() = Some(Self { entered, release });
|
||||||
|
(receiver, sender)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct TestExecutionBackend {
|
struct TestExecutionBackend {
|
||||||
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
||||||
@@ -7673,6 +7911,10 @@ mod tests {
|
|||||||
repository_access_available: AtomicBool,
|
repository_access_available: AtomicBool,
|
||||||
working_directory_requests: Mutex<Vec<WorkingDirectoryRequest>>,
|
working_directory_requests: Mutex<Vec<WorkingDirectoryRequest>>,
|
||||||
cleanup_occupancy_probe: Mutex<Option<Arc<Mutex<RuntimeState>>>>,
|
cleanup_occupancy_probe: Mutex<Option<Arc<Mutex<RuntimeState>>>>,
|
||||||
|
cleanup_gate: Mutex<Option<WorkdirEffectGate>>,
|
||||||
|
workdir_use_gate: Mutex<Option<WorkdirEffectGate>>,
|
||||||
|
cleanup_calls: std::sync::atomic::AtomicUsize,
|
||||||
|
cleanup_panics: AtomicBool,
|
||||||
preserve_submission_acknowledgement_id: AtomicBool,
|
preserve_submission_acknowledgement_id: AtomicBool,
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
snapshots: Mutex<BTreeMap<WorkerId, protocol::Event>>,
|
snapshots: Mutex<BTreeMap<WorkerId, protocol::Event>>,
|
||||||
@@ -7717,10 +7959,21 @@ mod tests {
|
|||||||
&self,
|
&self,
|
||||||
_id: &str,
|
_id: &str,
|
||||||
) -> Result<CatalogWorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
) -> Result<CatalogWorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||||
|
self.cleanup_calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
WorkdirEffectGate::wait(&self.cleanup_gate, _id);
|
||||||
|
if self.cleanup_panics.load(Ordering::SeqCst) {
|
||||||
|
panic!("cleanup provider failed after admission");
|
||||||
|
}
|
||||||
if let Some(state) = self.cleanup_occupancy_probe.lock().unwrap().as_ref() {
|
if let Some(state) = self.cleanup_occupancy_probe.lock().unwrap().as_ref() {
|
||||||
|
let state = state
|
||||||
|
.try_lock()
|
||||||
|
.expect("physical cleanup must not hold global Runtime state");
|
||||||
assert!(
|
assert!(
|
||||||
matches!(state.try_lock(), Err(std::sync::TryLockError::WouldBlock)),
|
matches!(
|
||||||
"occupancy must remain exclusive during physical cleanup"
|
state.workdir_operations.get(_id),
|
||||||
|
Some(WorkdirOperation::Cleanup)
|
||||||
|
),
|
||||||
|
"the target Workdir must remain reserved during physical cleanup"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Err(WorkingDirectoryDiagnostic::rejected(
|
Err(WorkingDirectoryDiagnostic::rejected(
|
||||||
@@ -7749,6 +8002,10 @@ mod tests {
|
|||||||
&self,
|
&self,
|
||||||
request: &WorkingDirectoryRequest,
|
request: &WorkingDirectoryRequest,
|
||||||
) -> Result<CatalogWorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
) -> Result<CatalogWorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||||
|
WorkdirEffectGate::wait(
|
||||||
|
&self.workdir_use_gate,
|
||||||
|
request.backend_workdir_id.as_deref().unwrap(),
|
||||||
|
);
|
||||||
self.working_directory_requests
|
self.working_directory_requests
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -7763,6 +8020,7 @@ mod tests {
|
|||||||
&self,
|
&self,
|
||||||
request: &WorkingDirectoryRepositoryAccessRequest,
|
request: &WorkingDirectoryRepositoryAccessRequest,
|
||||||
) -> Result<(), WorkingDirectoryDiagnostic> {
|
) -> Result<(), WorkingDirectoryDiagnostic> {
|
||||||
|
WorkdirEffectGate::wait(&self.workdir_use_gate, &request.working_directory_id);
|
||||||
self.repository_accesses
|
self.repository_accesses
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -8068,7 +8326,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn workdir_removal_keeps_occupancy_exclusive_during_provider_side_effect() {
|
fn workdir_removal_reserves_only_its_target_and_releases_on_failure() {
|
||||||
let (runtime, backend) = runtime_and_backend();
|
let (runtime, backend) = runtime_and_backend();
|
||||||
*backend.cleanup_occupancy_probe.lock().unwrap() = Some(runtime.inner.clone());
|
*backend.cleanup_occupancy_probe.lock().unwrap() = Some(runtime.inner.clone());
|
||||||
let error = runtime
|
let error = runtime
|
||||||
@@ -8078,11 +8336,384 @@ mod tests {
|
|||||||
matches!(error, RuntimeError::WorkingDirectory(ref diagnostic)
|
matches!(error, RuntimeError::WorkingDirectory(ref diagnostic)
|
||||||
if diagnostic.code == "working_directory_cleanup_resource_busy")
|
if diagnostic.code == "working_directory_cleanup_resource_busy")
|
||||||
);
|
);
|
||||||
// The lock is released on failure, permitting normal attachment/retry.
|
// Failure releases the per-target reservation for normal use/retry.
|
||||||
assert!(runtime.inner.try_lock().is_ok());
|
assert!(runtime.inner.lock().unwrap().workdir_operations.is_empty());
|
||||||
*backend.cleanup_occupancy_probe.lock().unwrap() = None;
|
*backend.cleanup_occupancy_probe.lock().unwrap() = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn workdir_effect_request(id: Option<&str>) -> WorkingDirectoryRequest {
|
||||||
|
WorkingDirectoryRequest {
|
||||||
|
repository: WorkingDirectoryRepository {
|
||||||
|
id: "repository-1".into(),
|
||||||
|
provider: "git".into(),
|
||||||
|
source: server_api::RepositorySource {
|
||||||
|
kind: server_api::RepositorySourceKind::LocalPath,
|
||||||
|
uri: "/unused-test-repository".into(),
|
||||||
|
},
|
||||||
|
source_revision: 1,
|
||||||
|
source_fingerprint: "sha256:test-source".into(),
|
||||||
|
selector: None,
|
||||||
|
},
|
||||||
|
display_name: None,
|
||||||
|
materializer: MaterializerKind::RuntimeGitClone,
|
||||||
|
backend_workdir_id: id.map(str::to_string),
|
||||||
|
materialization: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn workdir_access_request(id: &str) -> WorkingDirectoryRepositoryAccessRequest {
|
||||||
|
WorkingDirectoryRepositoryAccessRequest {
|
||||||
|
working_directory_id: id.into(),
|
||||||
|
materialization: RepositoryMaterializationContext {
|
||||||
|
workspace_id: "workspace-a".into(),
|
||||||
|
runtime_id: "runtime-1".into(),
|
||||||
|
operation_id: "operation-1".into(),
|
||||||
|
config_revision: 1,
|
||||||
|
config_projection_digest: "sha256:test-projection".into(),
|
||||||
|
ssh: None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_workdir_busy(error: RuntimeError) {
|
||||||
|
assert!(
|
||||||
|
matches!(error, RuntimeError::WorkingDirectory(ref diagnostic)
|
||||||
|
if diagnostic.code == "working_directory_cleanup_resource_busy"),
|
||||||
|
"{error:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn active_workdir_cleanup_rejects_same_id_admissions_without_blocking_other_workers() {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let owner = scope("workspace-a", "server-a");
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker_scoped(&owner, scoped_task_request("unrelated", "workspace-a"))
|
||||||
|
.unwrap();
|
||||||
|
let id = "workdir-cleanup-active";
|
||||||
|
let (entered, release) = WorkdirEffectGate::install(&backend.cleanup_gate);
|
||||||
|
let cleanup_runtime = runtime.clone();
|
||||||
|
let cleanup = std::thread::spawn(move || cleanup_runtime.cleanup_working_directory(id));
|
||||||
|
assert_eq!(
|
||||||
|
entered
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(5))
|
||||||
|
.unwrap(),
|
||||||
|
id
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(runtime.list_workers_scoped(&owner).unwrap().len(), 1);
|
||||||
|
assert_workdir_busy(runtime.cleanup_working_directory(id).unwrap_err());
|
||||||
|
assert_workdir_busy(
|
||||||
|
runtime
|
||||||
|
.create_working_directory(workdir_effect_request(Some(id)))
|
||||||
|
.unwrap_err(),
|
||||||
|
);
|
||||||
|
assert_workdir_busy(
|
||||||
|
runtime
|
||||||
|
.authorize_working_directory_repository_access(workdir_access_request(id))
|
||||||
|
.unwrap_err(),
|
||||||
|
);
|
||||||
|
assert_workdir_busy(
|
||||||
|
runtime
|
||||||
|
.open_workdir_session_scoped(&owner, id, None)
|
||||||
|
.unwrap_err(),
|
||||||
|
);
|
||||||
|
let mut create = scoped_task_request("conflicting attachment", "workspace-a");
|
||||||
|
create.workdir_attachments = vec![WorkingDirectoryAttachmentClaim {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
|
working_directory_id: id.into(),
|
||||||
|
relative_cwd: None,
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
}];
|
||||||
|
assert_workdir_busy(runtime.create_worker_scoped(&owner, create).unwrap_err());
|
||||||
|
let attachment = LogicalWorkdirAttachment {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
|
working_directory_id: id.into(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
};
|
||||||
|
assert_workdir_busy(
|
||||||
|
runtime
|
||||||
|
.replace_worker_workdir_attachments_scoped(
|
||||||
|
&owner,
|
||||||
|
&worker.worker_ref,
|
||||||
|
vec![attachment.clone()],
|
||||||
|
)
|
||||||
|
.unwrap_err(),
|
||||||
|
);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(&owner, &worker.worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
let restore = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
assert_workdir_busy(
|
||||||
|
runtime
|
||||||
|
.coordinate_worker_restore_operation(
|
||||||
|
&worker.worker_ref,
|
||||||
|
runtime_api::WorkerRestoreCoordinationRequest {
|
||||||
|
expected_observation_token: restore.expected_observation_token,
|
||||||
|
request_id: restore.request_id,
|
||||||
|
preparation: Some(runtime_api::WorkerRestorePreparation {
|
||||||
|
workdir_attachments: Some(vec![
|
||||||
|
runtime_api::LogicalWorkdirAttachment {
|
||||||
|
alias: attachment.alias,
|
||||||
|
working_directory_id: attachment.working_directory_id,
|
||||||
|
capabilities: attachment.capabilities,
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Some(&owner),
|
||||||
|
)
|
||||||
|
.unwrap_err(),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.worker(&worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.pending_restore
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(runtime.list_workers_scoped(&owner).unwrap().len(), 1);
|
||||||
|
// Different IDs and Worker creation remain available during cleanup.
|
||||||
|
runtime
|
||||||
|
.create_worker_scoped(
|
||||||
|
&owner,
|
||||||
|
scoped_task_request("another unrelated", "workspace-a"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let other = runtime
|
||||||
|
.create_working_directory(workdir_effect_request(Some("workdir-other")))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(other, RuntimeError::WorkingDirectory(ref diagnostic) if diagnostic.code == "working_directory_unsupported")
|
||||||
|
);
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 1);
|
||||||
|
release.send(()).unwrap();
|
||||||
|
assert_workdir_busy(cleanup.join().unwrap().unwrap_err());
|
||||||
|
assert!(runtime.lock().unwrap().workdir_operations.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn in_flight_creation_and_access_exclude_cleanup_including_generated_ids() {
|
||||||
|
for operation in ["explicit-create", "generated-create", "access"] {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let (entered, release) = WorkdirEffectGate::install(&backend.workdir_use_gate);
|
||||||
|
let effect_runtime = runtime.clone();
|
||||||
|
let effect = std::thread::spawn(move || {
|
||||||
|
if operation == "access" {
|
||||||
|
effect_runtime.authorize_working_directory_repository_access(
|
||||||
|
workdir_access_request("workdir-in-use"),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
effect_runtime
|
||||||
|
.create_working_directory(workdir_effect_request(
|
||||||
|
(operation == "explicit-create").then_some("workdir-in-use"),
|
||||||
|
))
|
||||||
|
.map(|_| ())
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let id = entered
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(5))
|
||||||
|
.unwrap();
|
||||||
|
assert!(!id.is_empty());
|
||||||
|
assert_workdir_busy(runtime.cleanup_working_directory(&id).unwrap_err());
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 0);
|
||||||
|
runtime
|
||||||
|
.create_worker(task_request("unrelated during effect"))
|
||||||
|
.unwrap();
|
||||||
|
release.send(()).unwrap();
|
||||||
|
assert!(effect.join().unwrap().is_err());
|
||||||
|
// Failed provider effects release their reservation for ordinary retry.
|
||||||
|
assert_workdir_busy(runtime.cleanup_working_directory(&id).unwrap_err());
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn logical_attachments_and_unsettled_restore_preparation_exclude_cleanup() {
|
||||||
|
for pending in [false, true] {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let owner = scope("workspace-a", "server-a");
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker_scoped(&owner, scoped_task_request("logical owner", "workspace-a"))
|
||||||
|
.unwrap();
|
||||||
|
let id = "workdir-logical-owner";
|
||||||
|
let attachment = LogicalWorkdirAttachment {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
|
working_directory_id: id.into(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
};
|
||||||
|
if pending {
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(&owner, &worker.worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
let restore = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
let preparation = runtime_api::WorkerRestorePreparation {
|
||||||
|
workdir_attachments: Some(vec![runtime_api::LogicalWorkdirAttachment {
|
||||||
|
alias: attachment.alias,
|
||||||
|
working_directory_id: attachment.working_directory_id,
|
||||||
|
capabilities: attachment.capabilities,
|
||||||
|
}]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let coordinated = runtime
|
||||||
|
.coordinate_worker_restore_operation(
|
||||||
|
&worker.worker_ref,
|
||||||
|
runtime_api::WorkerRestoreCoordinationRequest {
|
||||||
|
expected_observation_token: restore.expected_observation_token,
|
||||||
|
request_id: restore.request_id,
|
||||||
|
preparation: Some(preparation.clone()),
|
||||||
|
},
|
||||||
|
Some(&owner),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(coordinated, (None, Some(preparation)));
|
||||||
|
} else {
|
||||||
|
runtime
|
||||||
|
.replace_worker_workdir_attachments_scoped(
|
||||||
|
&owner,
|
||||||
|
&worker.worker_ref,
|
||||||
|
vec![attachment],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
matches!(runtime.cleanup_working_directory(id).unwrap_err(), RuntimeError::InvalidRequest(message) if message.contains("assigned to worker"))
|
||||||
|
);
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unowned_restore_recovery_ignores_new_preparation_for_a_workdir_being_cleaned() {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let owner = scope("workspace-a", "server-a");
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker_scoped(
|
||||||
|
&owner,
|
||||||
|
scoped_task_request("recover persisted operation", "workspace-a"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(&owner, &worker.worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
// Persisted lifecycle recovery may have no request owner (e.g. after an
|
||||||
|
// interrupted older admission). New preparation is not its authority.
|
||||||
|
runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.worker_mut(&worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.pending_restore = Some(PendingWorkerRestore {
|
||||||
|
operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
mode: WorkerRestoreMode::Explicit,
|
||||||
|
last_settled_status: WorkerStatus::Stopped,
|
||||||
|
});
|
||||||
|
let id = "workdir-irrelevant-to-recovery";
|
||||||
|
let (entered, release) = WorkdirEffectGate::install(&backend.cleanup_gate);
|
||||||
|
let cleanup_runtime = runtime.clone();
|
||||||
|
let cleanup = std::thread::spawn(move || cleanup_runtime.cleanup_working_directory(id));
|
||||||
|
entered
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(5))
|
||||||
|
.unwrap();
|
||||||
|
let restore = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
let (result, preparation) = runtime
|
||||||
|
.coordinate_worker_restore_operation(
|
||||||
|
&worker.worker_ref,
|
||||||
|
runtime_api::WorkerRestoreCoordinationRequest {
|
||||||
|
expected_observation_token: restore.expected_observation_token,
|
||||||
|
request_id: restore.request_id,
|
||||||
|
preparation: Some(runtime_api::WorkerRestorePreparation {
|
||||||
|
workdir_attachments: Some(vec![runtime_api::LogicalWorkdirAttachment {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("ignored").unwrap(),
|
||||||
|
working_directory_id: id.into(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
}]),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Some(&owner),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.unwrap().state, WorkerRestoreState::Accepted);
|
||||||
|
assert!(preparation.is_none());
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.worker(&worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.logical_workdir_attachments
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
release.send(()).unwrap();
|
||||||
|
assert_workdir_busy(cleanup.join().unwrap().unwrap_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn live_workdir_session_clones_exclude_cleanup_until_close_or_final_drop() {
|
||||||
|
for close in [false, true] {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let id = "workdir-live-session";
|
||||||
|
let provider: workdir::WorkdirSessionHandle = Arc::new(
|
||||||
|
workdir::LocalWorkdirSession::materialized_bound_with_environment(
|
||||||
|
workdir::Workdir::new(id),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
manifest::SharedScope::new(manifest::Scope::writable(root.path()).unwrap()),
|
||||||
|
workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
BTreeMap::new(),
|
||||||
|
Vec::new(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let session = workdir_session::retain_operation(
|
||||||
|
provider,
|
||||||
|
runtime.reserve_workdir_use([id]).unwrap(),
|
||||||
|
);
|
||||||
|
let clone = session.clone();
|
||||||
|
drop(session);
|
||||||
|
assert_workdir_busy(runtime.cleanup_working_directory(id).unwrap_err());
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 0);
|
||||||
|
if close {
|
||||||
|
clone.close().await.unwrap();
|
||||||
|
clone.close().await.unwrap();
|
||||||
|
// A closed wrapper may remain alive without reserving storage.
|
||||||
|
assert_workdir_busy(runtime.cleanup_working_directory(id).unwrap_err());
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 1);
|
||||||
|
} else {
|
||||||
|
drop(clone);
|
||||||
|
assert_workdir_busy(runtime.cleanup_working_directory(id).unwrap_err());
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn workdir_cleanup_panic_releases_reservation_without_poisoning_runtime() {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
backend.cleanup_panics.store(true, Ordering::SeqCst);
|
||||||
|
assert!(
|
||||||
|
std::panic::catch_unwind(std::panic::AssertUnwindSafe(
|
||||||
|
|| runtime.cleanup_working_directory("workdir-panic")
|
||||||
|
))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(runtime.lock().unwrap().workdir_operations.is_empty());
|
||||||
|
backend.cleanup_panics.store(false, Ordering::SeqCst);
|
||||||
|
runtime
|
||||||
|
.create_worker(task_request("after cleanup panic"))
|
||||||
|
.unwrap();
|
||||||
|
assert_workdir_busy(
|
||||||
|
runtime
|
||||||
|
.cleanup_working_directory("workdir-panic")
|
||||||
|
.unwrap_err(),
|
||||||
|
);
|
||||||
|
assert_eq!(backend.cleanup_calls.load(Ordering::SeqCst), 2);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn unknown_typed_intent_is_rejected_before_backend_dispatch() {
|
fn unknown_typed_intent_is_rejected_before_backend_dispatch() {
|
||||||
let (runtime, backend) = runtime_and_backend();
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
//! Retain a Runtime active-use reservation for a live provider session.
|
||||||
|
//!
|
||||||
|
//! Cloned handles share the reservation. A successful provider close releases
|
||||||
|
//! it even while closed handles remain; a failed or cancelled close retains it
|
||||||
|
//! until a later successful close or the final wrapper is dropped. Dropping the
|
||||||
|
//! wrapper does not close the provider or wait for outstanding commands, so
|
||||||
|
//! callers must keep a session handle alive while using provider-owned work.
|
||||||
|
|
||||||
|
use super::WorkdirOperationGuard;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
use workdir::{
|
||||||
|
CheckoutObservation, CheckoutRequest, CheckoutResult, CheckoutSearchRequest,
|
||||||
|
CheckoutSearchResult, CommandEvent, CommandHandle, CommandOutput, CommandOutputRequest,
|
||||||
|
CommandRequest, CommandSnapshot, CommandStatus, EditRequest, EditResult, GlobRequest,
|
||||||
|
GlobResult, GrepRequest, GrepResult, ListRequest, ListResult, ReadBytesRequest,
|
||||||
|
ReadBytesResult, ReadRequest, ReadResult, StatRequest, StatResult, Workdir, WorkdirError,
|
||||||
|
WorkdirPath, WorkdirScopeAuthorizationRequest, WorkdirScopeOverlapRequest, WorkdirSession,
|
||||||
|
WorkdirSessionCapabilities, WorkdirSessionHandle, WriteRequest, WriteResult,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(super) fn retain_operation(
|
||||||
|
session: WorkdirSessionHandle,
|
||||||
|
operation: WorkdirOperationGuard,
|
||||||
|
) -> WorkdirSessionHandle {
|
||||||
|
Arc::new(RetainedWorkdirSession {
|
||||||
|
inner: session,
|
||||||
|
operation: Mutex::new(Some(operation)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RetainedWorkdirSession {
|
||||||
|
inner: WorkdirSessionHandle,
|
||||||
|
operation: Mutex<Option<WorkdirOperationGuard>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for RetainedWorkdirSession {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
// Do not expose provider internals or reservation state containing host paths.
|
||||||
|
f.debug_struct("RetainedWorkdirSession")
|
||||||
|
.field("workdir", self.inner.workdir())
|
||||||
|
.field("capabilities", &self.inner.capabilities())
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl WorkdirSession for RetainedWorkdirSession {
|
||||||
|
fn workdir(&self) -> &Workdir {
|
||||||
|
self.inner.workdir()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||||
|
self.inner.capabilities()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn authorize_scope_path(
|
||||||
|
&self,
|
||||||
|
request: WorkdirScopeAuthorizationRequest,
|
||||||
|
) -> Result<(), WorkdirError> {
|
||||||
|
self.inner.authorize_scope_path(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn scope_rules_overlap(
|
||||||
|
&self,
|
||||||
|
request: WorkdirScopeOverlapRequest,
|
||||||
|
) -> Result<bool, WorkdirError> {
|
||||||
|
self.inner.scope_rules_overlap(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn checkout_search(
|
||||||
|
&self,
|
||||||
|
request: CheckoutSearchRequest,
|
||||||
|
) -> Result<CheckoutSearchResult, WorkdirError> {
|
||||||
|
self.inner.checkout_search(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn checkout_observe(
|
||||||
|
&self,
|
||||||
|
path: WorkdirPath,
|
||||||
|
) -> Result<CheckoutObservation, WorkdirError> {
|
||||||
|
self.inner.checkout_observe(path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn checkout_execute(
|
||||||
|
&self,
|
||||||
|
request: CheckoutRequest,
|
||||||
|
) -> Result<CheckoutResult, WorkdirError> {
|
||||||
|
self.inner.checkout_execute(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||||
|
self.inner.stat(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read(&self, request: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
||||||
|
self.inner.read(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_bytes(&self, request: ReadBytesRequest) -> Result<ReadBytesResult, WorkdirError> {
|
||||||
|
self.inner.read_bytes(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write(&self, request: WriteRequest) -> Result<WriteResult, WorkdirError> {
|
||||||
|
self.inner.write(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn edit(&self, request: EditRequest) -> Result<EditResult, WorkdirError> {
|
||||||
|
self.inner.edit(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list(&self, request: ListRequest) -> Result<ListResult, WorkdirError> {
|
||||||
|
self.inner.list(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn glob(&self, request: GlobRequest) -> Result<GlobResult, WorkdirError> {
|
||||||
|
self.inner.glob(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn grep(&self, request: GrepRequest) -> Result<GrepResult, WorkdirError> {
|
||||||
|
self.inner.grep(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
||||||
|
self.inner.start_command(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_status(&self, handle: CommandHandle) -> Result<CommandStatus, WorkdirError> {
|
||||||
|
self.inner.command_status(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_output(
|
||||||
|
&self,
|
||||||
|
request: CommandOutputRequest,
|
||||||
|
) -> Result<CommandOutput, WorkdirError> {
|
||||||
|
self.inner.command_output(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
|
||||||
|
self.inner.cancel_command(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
|
||||||
|
self.inner.subscribe_command_events()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
|
||||||
|
self.inner.command_snapshot()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn close(&self) -> Result<(), WorkdirError> {
|
||||||
|
// Provider close must succeed before releasing the active-use reservation.
|
||||||
|
// No wrapper-local terminal checks may bypass the provider's own checks.
|
||||||
|
self.inner.close().await?;
|
||||||
|
let operation = self
|
||||||
|
.operation
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
|
.take();
|
||||||
|
// Release outside the mutex: guard Drop may acquire Runtime state locks.
|
||||||
|
drop(operation);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1025,7 +1025,9 @@ fn runtime_local_workdir_router(
|
|||||||
command_output_scope.clone(),
|
command_output_scope.clone(),
|
||||||
capabilities,
|
capabilities,
|
||||||
binding.command_environment(),
|
binding.command_environment(),
|
||||||
binding.session_resources(),
|
binding.session_resources().map_err(|error| {
|
||||||
|
format!("open Workdir attachment `{alias}` resources: {error}")
|
||||||
|
})?,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.map_err(|error| format!("bind Workdir attachment `{alias}`: {error}"))?;
|
.map_err(|error| format!("bind Workdir attachment `{alias}`: {error}"))?;
|
||||||
@@ -3453,7 +3455,7 @@ where
|
|||||||
scope,
|
scope,
|
||||||
WorkdirSessionCapabilities::ALL,
|
WorkdirSessionCapabilities::ALL,
|
||||||
binding.command_environment(),
|
binding.command_environment(),
|
||||||
binding.session_resources(),
|
binding.session_resources()?,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -9501,6 +9503,42 @@ mod tests {
|
|||||||
assert_eq!(status.summary.cleanliness.as_deref(), Some("clean"));
|
assert_eq!(status.summary.cleanliness.as_deref(), Some("clean"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn local_sessions_exclude_cleanup_until_close_releases_resources() {
|
||||||
|
let runtime_base = tempfile::tempdir().unwrap();
|
||||||
|
let repo = create_clean_repo();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_base.path());
|
||||||
|
let binding = materializer
|
||||||
|
.create(&working_directory_request(repo.path()))
|
||||||
|
.unwrap();
|
||||||
|
let id = &binding.working_directory.id;
|
||||||
|
let backend = WorkerRuntimeExecutionBackend::new(FailingFactory)
|
||||||
|
.unwrap()
|
||||||
|
.with_working_directory_materializer(materializer.clone());
|
||||||
|
let session = backend.open_workdir_session(id).unwrap();
|
||||||
|
let session_clone = session.clone();
|
||||||
|
drop(session);
|
||||||
|
assert_eq!(
|
||||||
|
materializer.cleanup_working_directory(id).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
|
||||||
|
let alias = WorkdirAttachmentAlias::new("checkout").unwrap();
|
||||||
|
let attachments = BTreeMap::from([(alias, binding.clone())]);
|
||||||
|
let scope = manifest::SharedScope::new(manifest::Scope::writable(binding.root()).unwrap());
|
||||||
|
let router = runtime_local_workdir_router(&attachments, &BTreeMap::new(), scope).unwrap();
|
||||||
|
session_clone.close().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
materializer.cleanup_working_directory(id).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
// Closed handles, router and passive bindings can remain alive.
|
||||||
|
materializer.cleanup_working_directory(id).unwrap();
|
||||||
|
assert!(!binding.working_directory_root().exists());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn spawn_failure_with_existing_working_directory_preserves_workdir() {
|
fn spawn_failure_with_existing_working_directory_preserves_workdir() {
|
||||||
let runtime_base = tempfile::tempdir().unwrap();
|
let runtime_base = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|||||||
use workdir::WorkdirSessionResource;
|
use workdir::WorkdirSessionResource;
|
||||||
|
|
||||||
mod cleanup;
|
mod cleanup;
|
||||||
|
mod occupancy;
|
||||||
|
|
||||||
#[cfg(all(test, target_os = "linux"))]
|
#[cfg(all(test, target_os = "linux"))]
|
||||||
pub(crate) use cleanup::tests::fail_once as fail_workdir_cleanup_once;
|
pub(crate) use cleanup::tests::fail_once as fail_workdir_cleanup_once;
|
||||||
@@ -101,6 +102,9 @@ pub struct WorkingDirectoryBinding {
|
|||||||
working_directory_root: PathBuf,
|
working_directory_root: PathBuf,
|
||||||
command_environment: BTreeMap<String, String>,
|
command_environment: BTreeMap<String, String>,
|
||||||
session_resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
session_resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||||
|
occupancy: occupancy::Occupancy,
|
||||||
|
occupancy_id: String,
|
||||||
|
generation: occupancy::Generation,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkingDirectoryBinding {
|
impl WorkingDirectoryBinding {
|
||||||
@@ -120,8 +124,23 @@ impl WorkingDirectoryBinding {
|
|||||||
self.command_environment.clone()
|
self.command_environment.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn session_resources(&self) -> Vec<Arc<dyn WorkdirSessionResource>> {
|
/// Activate resources for a local session. Cloning a binding alone is passive;
|
||||||
self.session_resources.clone()
|
/// the returned lease excludes cleanup until the last resource clone drops.
|
||||||
|
pub fn session_resources(
|
||||||
|
&self,
|
||||||
|
) -> Result<Vec<Arc<dyn WorkdirSessionResource>>, WorkingDirectoryDiagnostic> {
|
||||||
|
let lease = self
|
||||||
|
.occupancy
|
||||||
|
.acquire_session_use(&self.occupancy_id, &self.generation)?;
|
||||||
|
if !binding_paths_are_available(self) {
|
||||||
|
return Err(WorkingDirectoryDiagnostic::new(
|
||||||
|
"working_directory_not_found",
|
||||||
|
"Workdir binding is no longer available; acquire a fresh binding",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut resources = self.session_resources.clone();
|
||||||
|
resources.push(Arc::new(lease));
|
||||||
|
Ok(resources)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn status(&self) -> WorkingDirectoryStatus {
|
pub fn status(&self) -> WorkingDirectoryStatus {
|
||||||
@@ -583,6 +602,7 @@ fn repository_access_now_epoch_seconds() -> u64 {
|
|||||||
pub struct RuntimeGitMaterializer {
|
pub struct RuntimeGitMaterializer {
|
||||||
runtime_root: PathBuf,
|
runtime_root: PathBuf,
|
||||||
repository_access: RepositoryAccessExpiryScheduler,
|
repository_access: RepositoryAccessExpiryScheduler,
|
||||||
|
occupancy: occupancy::Occupancy,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RuntimeGitMaterializer {
|
impl RuntimeGitMaterializer {
|
||||||
@@ -590,6 +610,7 @@ impl RuntimeGitMaterializer {
|
|||||||
Self {
|
Self {
|
||||||
runtime_root: runtime_root.into(),
|
runtime_root: runtime_root.into(),
|
||||||
repository_access: RepositoryAccessExpiryScheduler::new(),
|
repository_access: RepositoryAccessExpiryScheduler::new(),
|
||||||
|
occupancy: occupancy::Occupancy::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -702,6 +723,9 @@ impl RuntimeGitMaterializer {
|
|||||||
working_directory_root,
|
working_directory_root,
|
||||||
command_environment: BTreeMap::new(),
|
command_environment: BTreeMap::new(),
|
||||||
session_resources: Vec::new(),
|
session_resources: Vec::new(),
|
||||||
|
occupancy: self.occupancy.clone(),
|
||||||
|
occupancy_id: working_directory_id.to_string(),
|
||||||
|
generation: self.occupancy.generation(&working_directory_id),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -904,6 +928,7 @@ impl RuntimeGitMaterializer {
|
|||||||
request: &WorkingDirectoryRequest,
|
request: &WorkingDirectoryRequest,
|
||||||
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
||||||
validate_working_directory_id(&working_directory_id)?;
|
validate_working_directory_id(&working_directory_id)?;
|
||||||
|
let _creation_lease = self.occupancy.acquire_creation(&working_directory_id)?;
|
||||||
Self::validate_plain_http_source(request)?;
|
Self::validate_plain_http_source(request)?;
|
||||||
let request =
|
let request =
|
||||||
self.request_with_authorized_repository_access(&working_directory_id, request)?;
|
self.request_with_authorized_repository_access(&working_directory_id, request)?;
|
||||||
@@ -911,20 +936,30 @@ impl RuntimeGitMaterializer {
|
|||||||
let selector = request.repository.selector.as_deref().unwrap_or("HEAD");
|
let selector = request.repository.selector.as_deref().unwrap_or("HEAD");
|
||||||
let working_directory_root = self.working_directory_root(&working_directory_id);
|
let working_directory_root = self.working_directory_root(&working_directory_id);
|
||||||
let checkout_root = working_directory_root.join(CHECKOUT_DIR);
|
let checkout_root = working_directory_root.join(CHECKOUT_DIR);
|
||||||
if checkout_root.exists() {
|
if working_directory_root.symlink_metadata().is_ok() {
|
||||||
return Err(WorkingDirectoryDiagnostic::new(
|
return Err(WorkingDirectoryDiagnostic::new(
|
||||||
"working_directory_exists",
|
"working_directory_exists",
|
||||||
"working directory target already exists; cleanup or choose a new working_directory",
|
"working directory target already exists; cleanup or choose a new working_directory",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
fs::create_dir_all(&working_directory_root).map_err(|_| {
|
fs::create_dir_all(&self.runtime_root).map_err(|_| {
|
||||||
|
WorkingDirectoryDiagnostic::new(
|
||||||
|
"working_directory_create_failed",
|
||||||
|
"failed to create working directory storage",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
// Prepare fallible access before owning a root that must not be adopted
|
||||||
|
// by a retry after failure.
|
||||||
|
let access = RepositoryCommandAccess::prepare(&self.runtime_root, &request)?;
|
||||||
|
// Atomically own this new root before any rollback can remove it. Do
|
||||||
|
// not adopt an existing partial-cleanup directory or another creator's root.
|
||||||
|
fs::create_dir(&working_directory_root).map_err(|_| {
|
||||||
WorkingDirectoryDiagnostic::new(
|
WorkingDirectoryDiagnostic::new(
|
||||||
"working_directory_create_failed",
|
"working_directory_create_failed",
|
||||||
"failed to create working directory; backend-private path details were omitted",
|
"failed to create working directory; backend-private path details were omitted",
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let access = RepositoryCommandAccess::prepare(&self.runtime_root, &request)?;
|
|
||||||
let mut clone = repository_git_command(&request, access.as_ref());
|
let mut clone = repository_git_command(&request, access.as_ref());
|
||||||
clone
|
clone
|
||||||
.args([
|
.args([
|
||||||
@@ -1022,6 +1057,9 @@ impl RuntimeGitMaterializer {
|
|||||||
working_directory_root: working_directory_root.clone(),
|
working_directory_root: working_directory_root.clone(),
|
||||||
command_environment: BTreeMap::new(),
|
command_environment: BTreeMap::new(),
|
||||||
session_resources: Vec::new(),
|
session_resources: Vec::new(),
|
||||||
|
occupancy: self.occupancy.clone(),
|
||||||
|
occupancy_id: working_directory_id.to_string(),
|
||||||
|
generation: self.occupancy.generation(&working_directory_id),
|
||||||
};
|
};
|
||||||
if let Err(error) = self.write_record(&binding) {
|
if let Err(error) = self.write_record(&binding) {
|
||||||
let _ = fs::remove_dir_all(&working_directory_root);
|
let _ = fs::remove_dir_all(&working_directory_root);
|
||||||
@@ -1046,7 +1084,10 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
|||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
request: &WorkingDirectoryRequest,
|
request: &WorkingDirectoryRequest,
|
||||||
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
||||||
let working_directory_id = Self::working_directory_id(worker_ref, &request.repository.id);
|
let working_directory_id = request
|
||||||
|
.backend_workdir_id
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| Self::working_directory_id(worker_ref, &request.repository.id));
|
||||||
self.materialize_with_working_directory_id(working_directory_id, request)
|
self.materialize_with_working_directory_id(working_directory_id, request)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1066,6 +1107,7 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
|||||||
request: &WorkingDirectoryRepositoryAccessRequest,
|
request: &WorkingDirectoryRepositoryAccessRequest,
|
||||||
) -> Result<(), WorkingDirectoryDiagnostic> {
|
) -> Result<(), WorkingDirectoryDiagnostic> {
|
||||||
validate_working_directory_id(&request.working_directory_id)?;
|
validate_working_directory_id(&request.working_directory_id)?;
|
||||||
|
let _use_lease = self.occupancy.acquire_use(&request.working_directory_id)?;
|
||||||
let ssh = request.materialization.ssh.as_ref().ok_or_else(|| {
|
let ssh = request.materialization.ssh.as_ref().ok_or_else(|| {
|
||||||
WorkingDirectoryDiagnostic::new(
|
WorkingDirectoryDiagnostic::new(
|
||||||
"working_directory_remote_repository_access_required",
|
"working_directory_remote_repository_access_required",
|
||||||
@@ -1187,6 +1229,8 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
|||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
relative_cwd: Option<&str>,
|
relative_cwd: Option<&str>,
|
||||||
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
||||||
|
validate_working_directory_id(working_directory_id)?;
|
||||||
|
let _use_lease = self.occupancy.acquire_use(working_directory_id)?;
|
||||||
let binding = self.checked_binding(working_directory_id, relative_cwd)?;
|
let binding = self.checked_binding(working_directory_id, relative_cwd)?;
|
||||||
self.bind_repository_access(working_directory_id, binding)
|
self.bind_repository_access(working_directory_id, binding)
|
||||||
}
|
}
|
||||||
@@ -1281,7 +1325,10 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
|||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||||
validate_working_directory_id(working_directory_id)?;
|
validate_working_directory_id(working_directory_id)?;
|
||||||
cleanup::remove(self, working_directory_id)
|
let _cleanup_lease = self.occupancy.acquire_cleanup(working_directory_id)?;
|
||||||
|
let status = cleanup::remove(self, working_directory_id)?;
|
||||||
|
self.occupancy.invalidate_generation(working_directory_id);
|
||||||
|
Ok(status)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cleanup(&self, binding: &WorkingDirectoryBinding) -> Result<(), WorkingDirectoryDiagnostic> {
|
fn cleanup(&self, binding: &WorkingDirectoryBinding) -> Result<(), WorkingDirectoryDiagnostic> {
|
||||||
@@ -2878,7 +2925,7 @@ fn sanitize_path_component(value: &str) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next_working_directory_id(_repository_id: &str) -> String {
|
pub(crate) fn next_working_directory_id(_repository_id: &str) -> String {
|
||||||
let now = SystemTime::now()
|
let now = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
.map(|duration| duration.as_millis() as u64)
|
.map(|duration| duration.as_millis() as u64)
|
||||||
@@ -3933,7 +3980,8 @@ mod tests {
|
|||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
{
|
{
|
||||||
// Real SSH reauthorization updated access evidence repeatedly, but
|
// Real SSH reauthorization updated access evidence repeatedly, but
|
||||||
// did not replace this clean materialization or require manual recovery.
|
// did not replace this materialization or require manual recovery.
|
||||||
|
// Status does not inspect content; cleanup still verifies it below.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
restored
|
restored
|
||||||
.working_directory_status(&id)
|
.working_directory_status(&id)
|
||||||
@@ -3941,7 +3989,7 @@ mod tests {
|
|||||||
.summary
|
.summary
|
||||||
.cleanliness
|
.cleanliness
|
||||||
.as_deref(),
|
.as_deref(),
|
||||||
Some("clean")
|
Some("unknown")
|
||||||
);
|
);
|
||||||
restored.cleanup_working_directory(&id).unwrap();
|
restored.cleanup_working_directory(&id).unwrap();
|
||||||
assert!(!restored.working_directory_root(&id).exists());
|
assert!(!restored.working_directory_root(&id).exists());
|
||||||
@@ -4443,6 +4491,307 @@ mod tests {
|
|||||||
assert!(root.exists());
|
assert!(root.exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn materialization_context(
|
||||||
|
ssh: Option<RepositorySshMaterializationAccess>,
|
||||||
|
) -> crate::catalog::RepositoryMaterializationContext {
|
||||||
|
crate::catalog::RepositoryMaterializationContext {
|
||||||
|
workspace_id: "workspace-1".to_string(),
|
||||||
|
runtime_id: "runtime-1".to_string(),
|
||||||
|
operation_id: "operation-1".to_string(),
|
||||||
|
config_revision: 7,
|
||||||
|
config_projection_digest: "sha256:projection".to_string(),
|
||||||
|
ssh,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[test]
|
||||||
|
fn live_session_resources_block_cleanup_until_last_clone_drops() {
|
||||||
|
let repo = create_clean_repo();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let binding = materializer.create(&request(repo.path())).unwrap();
|
||||||
|
let id = &binding.working_directory.id;
|
||||||
|
// Read bindings and materializer clones must share the same admission.
|
||||||
|
let restored = materializer.bind_working_directory(id, None).unwrap();
|
||||||
|
let resources = restored.session_resources().unwrap();
|
||||||
|
let resource_clone = resources.clone();
|
||||||
|
let second_resources = binding.session_resources().unwrap();
|
||||||
|
let clone = materializer.clone();
|
||||||
|
|
||||||
|
for _ in 0..2 {
|
||||||
|
assert_eq!(
|
||||||
|
clone.cleanup_working_directory(id).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(binding.root().join("README.md").exists());
|
||||||
|
drop(resources);
|
||||||
|
drop(second_resources);
|
||||||
|
assert_eq!(
|
||||||
|
clone.cleanup_working_directory(id).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
drop(resource_clone);
|
||||||
|
clone.cleanup_working_directory(id).unwrap();
|
||||||
|
// Both bindings are still alive and passive after resource release.
|
||||||
|
assert!(!binding.working_directory_root().exists());
|
||||||
|
assert!(!restored.root().exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn active_cleanup_rejects_creation_and_access_before_effects() {
|
||||||
|
let repo = tempfile::tempdir().unwrap();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let id = "workdir-cleanup-active";
|
||||||
|
let cleanup = materializer.occupancy.acquire_cleanup(id).unwrap();
|
||||||
|
let mut create_request = request(repo.path());
|
||||||
|
create_request.backend_workdir_id = Some(id.to_string());
|
||||||
|
let access_request = WorkingDirectoryRepositoryAccessRequest {
|
||||||
|
working_directory_id: id.to_string(),
|
||||||
|
materialization: materialization_context(Some(repository_ssh_access())),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
materializer.create(&create_request).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
materializer
|
||||||
|
.materialize(&worker_ref(1), &create_request)
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
materializer
|
||||||
|
.authorize_repository_access(&access_request)
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
materializer
|
||||||
|
.bind_working_directory(id, None)
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert!(!materializer.working_directory_root(id).exists());
|
||||||
|
assert_eq!(materializer.repository_access.pending_count(), 0);
|
||||||
|
drop(cleanup);
|
||||||
|
// The cleanup reservation was transient, not a persisted lifecycle.
|
||||||
|
assert_ne!(
|
||||||
|
materializer.create(&create_request).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[test]
|
||||||
|
fn active_cleanup_rejects_existing_binding_resources_but_not_other_ids() {
|
||||||
|
let repo = create_clean_repo();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let binding = materializer.create(&request(repo.path())).unwrap();
|
||||||
|
let id = &binding.working_directory.id;
|
||||||
|
let cleanup = materializer.occupancy.acquire_cleanup(id).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
binding.session_resources().unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
materializer.cleanup_working_directory(id).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
let other = materializer.create(&request(repo.path())).unwrap();
|
||||||
|
let other_resources = other.session_resources().unwrap();
|
||||||
|
drop(other_resources);
|
||||||
|
materializer
|
||||||
|
.cleanup_working_directory(&other.working_directory.id)
|
||||||
|
.unwrap();
|
||||||
|
drop(cleanup);
|
||||||
|
let resources = binding.session_resources().unwrap();
|
||||||
|
drop(resources);
|
||||||
|
materializer.cleanup_working_directory(id).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[test]
|
||||||
|
fn cleanup_error_releases_lease_for_session_and_safe_retry() {
|
||||||
|
let repo = create_clean_repo();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let binding = materializer.create(&request(repo.path())).unwrap();
|
||||||
|
let changed = binding.root().join("untracked.txt");
|
||||||
|
fs::write(&changed, "preserve me\n").unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
materializer
|
||||||
|
.cleanup_working_directory(&binding.working_directory.id)
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"working_directory_cleanup_changes_present"
|
||||||
|
);
|
||||||
|
let resources = binding.session_resources().unwrap();
|
||||||
|
fs::remove_file(changed).unwrap();
|
||||||
|
drop(resources);
|
||||||
|
materializer
|
||||||
|
.cleanup_working_directory(&binding.working_directory.id)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[test]
|
||||||
|
fn removed_binding_cannot_activate_against_recreated_same_id() {
|
||||||
|
let repo = create_clean_repo();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let mut create = request(repo.path());
|
||||||
|
create.backend_workdir_id = Some("workdir-reused-id".into());
|
||||||
|
let old = materializer.create(&create).unwrap();
|
||||||
|
materializer
|
||||||
|
.cleanup_working_directory(&old.working_directory.id)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
old.session_resources().unwrap_err().code,
|
||||||
|
"working_directory_not_found"
|
||||||
|
);
|
||||||
|
let fresh = materializer.create(&create).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
old.session_resources().unwrap_err().code,
|
||||||
|
"working_directory_not_found"
|
||||||
|
);
|
||||||
|
let resources = fresh.session_resources().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
materializer
|
||||||
|
.cleanup_working_directory(&fresh.working_directory.id)
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
drop(resources);
|
||||||
|
materializer
|
||||||
|
.cleanup_working_directory(&fresh.working_directory.id)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn same_id_creation_is_exclusive_and_never_adopts_existing_partial_root() {
|
||||||
|
let repo = tempfile::tempdir().unwrap();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let id = "workdir-creation-owner";
|
||||||
|
let mut create = request(repo.path());
|
||||||
|
create.backend_workdir_id = Some(id.into());
|
||||||
|
let creation = materializer.occupancy.acquire_creation(id).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
materializer.clone().create(&create).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
materializer
|
||||||
|
.bind_working_directory(id, None)
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
materializer.cleanup_working_directory(id).unwrap_err().code,
|
||||||
|
"working_directory_cleanup_resource_busy"
|
||||||
|
);
|
||||||
|
assert!(!materializer.working_directory_root(id).exists());
|
||||||
|
drop(creation);
|
||||||
|
let partial = materializer.working_directory_root(id);
|
||||||
|
fs::create_dir(&partial).unwrap();
|
||||||
|
fs::write(
|
||||||
|
partial.join("owner-content"),
|
||||||
|
"preserve partial removal state",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
materializer.create(&create).unwrap_err().code,
|
||||||
|
"working_directory_exists"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_to_string(partial.join("owner-content")).unwrap(),
|
||||||
|
"preserve partial removal state"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_access_preparation_does_not_strand_a_creation_root() {
|
||||||
|
let repo = tempfile::tempdir().unwrap();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let mut create = request(repo.path());
|
||||||
|
create.backend_workdir_id = Some("workdir-access-setup-failure".into());
|
||||||
|
let mut ssh = repository_ssh_access();
|
||||||
|
ssh.known_hosts_entry =
|
||||||
|
crate::catalog::SensitiveString::new("example.test ssh-ed25519 placeholder");
|
||||||
|
create.repository.source = server_api::RepositorySource {
|
||||||
|
kind: server_api::RepositorySourceKind::Ssh,
|
||||||
|
uri: ssh.repository_uri.clone(),
|
||||||
|
};
|
||||||
|
create.materialization = Some(materialization_context(Some(ssh)));
|
||||||
|
// A local storage failure stops access setup before any SSH transport or
|
||||||
|
// agent can start. The materialization must remain retryable, not adopted.
|
||||||
|
fs::write(
|
||||||
|
runtime_root.path().join(REPOSITORY_ACCESS_DIR),
|
||||||
|
"storage obstruction",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
materializer.create(&create).unwrap_err().code,
|
||||||
|
"working_directory_repository_access_setup_failed"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!materializer
|
||||||
|
.working_directory_root(create.backend_workdir_id.as_deref().unwrap())
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_creation_and_access_release_use_leases() {
|
||||||
|
let repo = tempfile::tempdir().unwrap();
|
||||||
|
let runtime_root = tempfile::tempdir().unwrap();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||||
|
let id = "workdir-effect-errors";
|
||||||
|
let mut create_request = request(repo.path());
|
||||||
|
create_request.backend_workdir_id = Some(id.to_string());
|
||||||
|
// Validation fails after admission but before effects.
|
||||||
|
create_request.repository.provider = "unsupported".to_string();
|
||||||
|
assert_eq!(
|
||||||
|
materializer.create(&create_request).unwrap_err().code,
|
||||||
|
"working_directory_repository_provider_unsupported"
|
||||||
|
);
|
||||||
|
drop(materializer.occupancy.acquire_cleanup(id).unwrap());
|
||||||
|
// A clone failure has already created the target and must also release
|
||||||
|
// admission after rolling its filesystem effects back.
|
||||||
|
create_request.repository.provider = "git".to_string();
|
||||||
|
assert_eq!(
|
||||||
|
materializer.create(&create_request).unwrap_err().code,
|
||||||
|
"working_directory_repository_clone_failed"
|
||||||
|
);
|
||||||
|
assert!(!materializer.working_directory_root(id).exists());
|
||||||
|
drop(materializer.occupancy.acquire_cleanup(id).unwrap());
|
||||||
|
let access_request = WorkingDirectoryRepositoryAccessRequest {
|
||||||
|
working_directory_id: id.to_string(),
|
||||||
|
materialization: materialization_context(None),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
materializer
|
||||||
|
.authorize_repository_access(&access_request)
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"working_directory_remote_repository_access_required"
|
||||||
|
);
|
||||||
|
drop(materializer.occupancy.acquire_cleanup(id).unwrap());
|
||||||
|
assert!(materializer.bind_working_directory(id, None).is_err());
|
||||||
|
drop(materializer.occupancy.acquire_cleanup(id).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cleanup_working_directory_removes_clone_and_record() {
|
fn cleanup_working_directory_removes_clone_and_record() {
|
||||||
let repo = create_clean_repo();
|
let repo = create_clean_repo();
|
||||||
|
|||||||
@@ -76,19 +76,11 @@ pub(super) fn remaining_status(
|
|||||||
}
|
}
|
||||||
let mut summary = saved.working_directory.status_summary();
|
let mut summary = saved.working_directory.status_summary();
|
||||||
summary.status = WorkingDirectoryStatusKind::CleanupPending;
|
summary.status = WorkingDirectoryStatusKind::CleanupPending;
|
||||||
summary.cleanliness = Some(
|
// Observation is not removal admission: do not inventory or hash the
|
||||||
match check_mounts(
|
// surviving checkout just to list it. The witness can be stale, so neither
|
||||||
&materializer.working_directory_root(id),
|
// its old cleanliness nor a successful identity check proves current content
|
||||||
&fs::read("/proc/self/mountinfo").ok()?,
|
// is safe to remove. Actual retries still perform the full safety checks.
|
||||||
)
|
summary.cleanliness = Some("unknown".to_string());
|
||||||
.and_then(|()| retry_inventory(materializer, id, &tree, &saved))
|
|
||||||
{
|
|
||||||
Ok(_) => "clean",
|
|
||||||
Err(error) if error.code == "working_directory_cleanup_changes_present" => "dirty",
|
|
||||||
Err(_) => "unknown",
|
|
||||||
}
|
|
||||||
.to_string(),
|
|
||||||
);
|
|
||||||
Some(WorkingDirectoryStatus { summary })
|
Some(WorkingDirectoryStatus { summary })
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -974,7 +966,7 @@ pub(super) mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cleanup_pending_rechecks_current_cleanliness_and_retries_without_state_edit() {
|
fn cleanup_pending_observation_is_unknown_and_retry_checks_safety() {
|
||||||
let (_runtime, materializer, binding) = fixture();
|
let (_runtime, materializer, binding) = fixture();
|
||||||
let id = &binding.working_directory.id;
|
let id = &binding.working_directory.id;
|
||||||
fail_once(&materializer, id);
|
fail_once(&materializer, id);
|
||||||
@@ -983,12 +975,54 @@ pub(super) mod tests {
|
|||||||
status.summary.status,
|
status.summary.status,
|
||||||
WorkingDirectoryStatusKind::CleanupPending
|
WorkingDirectoryStatusKind::CleanupPending
|
||||||
);
|
);
|
||||||
assert_eq!(status.summary.cleanliness.as_deref(), Some("clean"));
|
assert_eq!(status.summary.cleanliness.as_deref(), Some("unknown"));
|
||||||
let removed = materializer.cleanup_working_directory(id).unwrap();
|
let removed = materializer.cleanup_working_directory(id).unwrap();
|
||||||
assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound);
|
assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound);
|
||||||
assert!(!binding.working_directory_root.exists());
|
assert!(!binding.working_directory_root.exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cleanup_pending_observation_does_not_open_checkout_content() {
|
||||||
|
use std::os::fd::{AsRawFd, FromRawFd};
|
||||||
|
use std::os::unix::ffi::OsStrExt;
|
||||||
|
|
||||||
|
let (_runtime, materializer, binding) = fixture();
|
||||||
|
let id = &binding.working_directory.id;
|
||||||
|
fail_once(&materializer, id);
|
||||||
|
|
||||||
|
// Observe actual file opens rather than a timing threshold or a large
|
||||||
|
// fixture. Inventory would open this surviving tracked file to hash it.
|
||||||
|
// SAFETY: inotify_init1 has no pointer arguments.
|
||||||
|
let fd = unsafe { libc::inotify_init1(libc::IN_NONBLOCK | libc::IN_CLOEXEC) };
|
||||||
|
assert!(fd >= 0, "{}", io::Error::last_os_error());
|
||||||
|
// SAFETY: fd is a freshly created descriptor, owned only by this File.
|
||||||
|
let mut events = unsafe { fs::File::from_raw_fd(fd) };
|
||||||
|
let path =
|
||||||
|
std::ffi::CString::new(binding.root.join("README.md").as_os_str().as_bytes()).unwrap();
|
||||||
|
// SAFETY: the descriptor is live and path is a valid terminated string.
|
||||||
|
let watch =
|
||||||
|
unsafe { libc::inotify_add_watch(events.as_raw_fd(), path.as_ptr(), libc::IN_OPEN) };
|
||||||
|
assert!(watch >= 0, "{}", io::Error::last_os_error());
|
||||||
|
|
||||||
|
let detail = materializer.working_directory_status(id).unwrap();
|
||||||
|
let listed = materializer.list_working_directories().unwrap();
|
||||||
|
let mut buffer = [0u8; 4096];
|
||||||
|
assert_eq!(
|
||||||
|
events
|
||||||
|
.read(&mut buffer)
|
||||||
|
.expect_err("status observation opened surviving checkout content")
|
||||||
|
.kind(),
|
||||||
|
io::ErrorKind::WouldBlock
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
detail.summary.status,
|
||||||
|
WorkingDirectoryStatusKind::CleanupPending
|
||||||
|
);
|
||||||
|
assert_eq!(detail.summary.cleanliness.as_deref(), Some("unknown"));
|
||||||
|
assert_eq!(listed.len(), 1);
|
||||||
|
assert_eq!(listed[0].summary, detail.summary);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn partial_deletion_missing_checkout_and_restart_finish_with_durable_content_authority() {
|
fn partial_deletion_missing_checkout_and_restart_finish_with_durable_content_authority() {
|
||||||
for removed in [
|
for removed in [
|
||||||
@@ -1021,7 +1055,7 @@ pub(super) mod tests {
|
|||||||
retained.summary.cleanup_target.as_ref(),
|
retained.summary.cleanup_target.as_ref(),
|
||||||
Some(&binding.working_directory.cleanup_target)
|
Some(&binding.working_directory.cleanup_target)
|
||||||
);
|
);
|
||||||
assert_eq!(retained.summary.cleanliness.as_deref(), Some("clean"));
|
assert_eq!(retained.summary.cleanliness.as_deref(), Some("unknown"));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
restarted.list_working_directories().unwrap()[0].summary,
|
restarted.list_working_directories().unwrap()[0].summary,
|
||||||
retained.summary
|
retained.summary
|
||||||
@@ -1074,7 +1108,7 @@ pub(super) mod tests {
|
|||||||
.summary
|
.summary
|
||||||
.cleanliness
|
.cleanliness
|
||||||
.as_deref(),
|
.as_deref(),
|
||||||
Some("dirty")
|
Some("unknown")
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
materializer.cleanup_working_directory(id).unwrap_err().code,
|
materializer.cleanup_working_directory(id).unwrap_err().code,
|
||||||
@@ -1117,7 +1151,7 @@ pub(super) mod tests {
|
|||||||
.summary
|
.summary
|
||||||
.cleanliness
|
.cleanliness
|
||||||
.as_deref(),
|
.as_deref(),
|
||||||
Some("clean"),
|
Some("unknown"),
|
||||||
"{resolution}"
|
"{resolution}"
|
||||||
);
|
);
|
||||||
// Renewal itself can fail at unlink. Its new witness must survive a
|
// Renewal itself can fail at unlink. Its new witness must survive a
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
//! Transient, clone-shared admission for local filesystem effects. Bindings are
|
||||||
|
//! passive; only effects and activated session resources own leases. The mutex
|
||||||
|
//! protects admission bookkeeping only, never filesystem work.
|
||||||
|
use super::WorkingDirectoryDiagnostic;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub(super) struct Occupancy {
|
||||||
|
entries: Arc<Mutex<HashMap<String, Entry>>>,
|
||||||
|
generations: Arc<Mutex<HashMap<String, Arc<()>>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Passive bindings identify a materialization generation, not merely its ID.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub(super) struct Generation(Arc<()>);
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum Entry {
|
||||||
|
Use(usize),
|
||||||
|
Create,
|
||||||
|
Cleanup,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(super) struct Lease {
|
||||||
|
occupancy: Occupancy,
|
||||||
|
id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn busy() -> WorkingDirectoryDiagnostic {
|
||||||
|
WorkingDirectoryDiagnostic::new(
|
||||||
|
"working_directory_cleanup_resource_busy",
|
||||||
|
"A filesystem resource is busy or changed during removal. Release the resource and retry.",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Occupancy {
|
||||||
|
pub(super) fn acquire_use(&self, id: &str) -> Result<Lease, WorkingDirectoryDiagnostic> {
|
||||||
|
let mut entries = self
|
||||||
|
.entries
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
|
match entries.get_mut(id) {
|
||||||
|
Some(Entry::Cleanup | Entry::Create) => return Err(busy()),
|
||||||
|
Some(Entry::Use(count)) => *count += 1,
|
||||||
|
None => {
|
||||||
|
entries.insert(id.to_string(), Entry::Use(1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Lease {
|
||||||
|
occupancy: self.clone(),
|
||||||
|
id: id.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn generation(&self, id: &str) -> Generation {
|
||||||
|
let mut generations = self
|
||||||
|
.generations
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
|
Generation(
|
||||||
|
generations
|
||||||
|
.entry(id.to_string())
|
||||||
|
.or_insert_with(|| Arc::new(()))
|
||||||
|
.clone(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Called only after successful removal, while exclusive cleanup is held.
|
||||||
|
pub(super) fn invalidate_generation(&self, id: &str) {
|
||||||
|
self.generations
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.remove(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn acquire_session_use(
|
||||||
|
&self,
|
||||||
|
id: &str,
|
||||||
|
generation: &Generation,
|
||||||
|
) -> Result<Lease, WorkingDirectoryDiagnostic> {
|
||||||
|
let lease = self.acquire_use(id)?;
|
||||||
|
let valid = self
|
||||||
|
.generations
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.get(id)
|
||||||
|
.is_some_and(|current| Arc::ptr_eq(current, &generation.0));
|
||||||
|
if !valid {
|
||||||
|
return Err(WorkingDirectoryDiagnostic::new(
|
||||||
|
"working_directory_not_found",
|
||||||
|
"Workdir binding is stale; acquire a fresh binding before opening a session",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(lease)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn acquire_creation(&self, id: &str) -> Result<Lease, WorkingDirectoryDiagnostic> {
|
||||||
|
let mut entries = self
|
||||||
|
.entries
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
|
if entries.contains_key(id) {
|
||||||
|
return Err(busy());
|
||||||
|
}
|
||||||
|
entries.insert(id.to_string(), Entry::Create);
|
||||||
|
Ok(Lease {
|
||||||
|
occupancy: self.clone(),
|
||||||
|
id: id.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn acquire_cleanup(&self, id: &str) -> Result<Lease, WorkingDirectoryDiagnostic> {
|
||||||
|
let mut entries = self
|
||||||
|
.entries
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
|
if entries.contains_key(id) {
|
||||||
|
return Err(busy());
|
||||||
|
}
|
||||||
|
entries.insert(id.to_string(), Entry::Cleanup);
|
||||||
|
Ok(Lease {
|
||||||
|
occupancy: self.clone(),
|
||||||
|
id: id.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn use_and_cleanup_leases_release_during_unwind() {
|
||||||
|
let occupancy = Occupancy::default();
|
||||||
|
for operation in ["use", "create", "cleanup"] {
|
||||||
|
let result = std::panic::catch_unwind(|| {
|
||||||
|
let _lease = match operation {
|
||||||
|
"cleanup" => occupancy.acquire_cleanup("workdir-panic").unwrap(),
|
||||||
|
"create" => occupancy.acquire_creation("workdir-panic").unwrap(),
|
||||||
|
_ => occupancy.acquire_use("workdir-panic").unwrap(),
|
||||||
|
};
|
||||||
|
panic!("effect failed after admission");
|
||||||
|
});
|
||||||
|
assert!(result.is_err());
|
||||||
|
drop(occupancy.acquire_use("workdir-panic").unwrap());
|
||||||
|
drop(occupancy.acquire_cleanup("workdir-panic").unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn simultaneous_use_and_cleanup_admit_only_one_direction() {
|
||||||
|
let occupancy = Occupancy::default();
|
||||||
|
let barrier = Arc::new(std::sync::Barrier::new(2));
|
||||||
|
std::thread::scope(|scope| {
|
||||||
|
let use_occupancy = occupancy.clone();
|
||||||
|
let use_barrier = barrier.clone();
|
||||||
|
let use_thread = scope.spawn(move || {
|
||||||
|
use_barrier.wait();
|
||||||
|
let result = use_occupancy.acquire_use("workdir-race");
|
||||||
|
// Keep the winner's lease until both admission attempts finish.
|
||||||
|
use_barrier.wait();
|
||||||
|
result
|
||||||
|
});
|
||||||
|
barrier.wait();
|
||||||
|
let cleanup = occupancy.acquire_cleanup("workdir-race");
|
||||||
|
barrier.wait();
|
||||||
|
let use_result = use_thread.join().unwrap();
|
||||||
|
assert_ne!(use_result.is_ok(), cleanup.is_ok());
|
||||||
|
let error = if let Err(error) = use_result {
|
||||||
|
error
|
||||||
|
} else {
|
||||||
|
cleanup.unwrap_err()
|
||||||
|
};
|
||||||
|
assert_eq!(error.code, "working_directory_cleanup_resource_busy");
|
||||||
|
});
|
||||||
|
drop(occupancy.acquire_cleanup("workdir-race").unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Lease {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let mut entries = self
|
||||||
|
.occupancy
|
||||||
|
.entries
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
|
match entries.get_mut(&self.id) {
|
||||||
|
Some(Entry::Use(count)) if *count > 1 => *count -= 1,
|
||||||
|
_ => {
|
||||||
|
entries.remove(&self.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -44,6 +44,11 @@ removal commits registry deletion only after Runtime confirms physical deletion
|
|||||||
or authoritative not-found. An unknown result leaves the registry in place so
|
or authoritative not-found. An unknown result leaves the registry in place so
|
||||||
retry can reconcile the provider's current state.
|
retry can reconcile the provider's current state.
|
||||||
|
|
||||||
|
A retained partial-removal Workdir is listed as `cleanup_pending` with cleanliness
|
||||||
|
`unknown`. List/detail observation checks its saved identity but does not read or
|
||||||
|
hash surviving checkout content. The saved removal witness is not a current
|
||||||
|
cleanliness assertion; only an actual removal retry runs the full content checks.
|
||||||
|
|
||||||
## Runtime safety boundary
|
## Runtime safety boundary
|
||||||
|
|
||||||
The Runtime keeps removal content/identity evidence outside the tree being
|
The Runtime keeps removal content/identity evidence outside the tree being
|
||||||
@@ -55,9 +60,22 @@ ignored/untracked content. Do not edit this witness manually.
|
|||||||
Mount-safe removal currently requires Linux `openat2` and `/proc` inspection.
|
Mount-safe removal currently requires Linux `openat2` and `/proc` inspection.
|
||||||
Unavailable inspection fails closed with `mount_check_unavailable`; there is no
|
Unavailable inspection fails closed with `mount_check_unavailable`; there is no
|
||||||
fallback to recursive deletion across an unchecked filesystem boundary.
|
fallback to recursive deletion across an unchecked filesystem boundary.
|
||||||
Removal retains the Runtime occupancy mutex through its filesystem effect.
|
Runtime HTTP list, detail and removal requests run synchronous provider work on
|
||||||
Unrelated Runtime state operations can therefore wait during hashing/unlink;
|
blocking workers rather than the HTTP executor. Removal reserves only the target
|
||||||
releasing it would require equivalent exclusion on all attachment paths.
|
Workdir under the Runtime state mutex, then releases that mutex before content
|
||||||
|
inspection and unlink. Unrelated Worker list/detail, session observation and
|
||||||
|
subscription snapshots remain available while removal runs. Conflicting use,
|
||||||
|
attachment, creation or removal of the same Workdir is rejected promptly rather
|
||||||
|
than waiting under the global mutex.
|
||||||
|
|
||||||
|
Provider leases protect in-flight creation/access and live session resources,
|
||||||
|
including their command tasks. Cloned sessions retain protection until safe close
|
||||||
|
or final resource release; passive bindings alone do not prevent removal. Removed
|
||||||
|
bindings cannot activate against a recreated Workdir with the same ID. Creation
|
||||||
|
owns a fresh root exclusively so failed creation cannot roll back another
|
||||||
|
creator's checkout or adopt an existing partial-removal directory. Reservations
|
||||||
|
are transient and released on provider success, error or unwind; they do not
|
||||||
|
replace the durable cleanup witness or attachment authority.
|
||||||
|
|
||||||
## Diagnostics
|
## Diagnostics
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user