Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: audit validated external workdir operation outcomes

Browse Source
This commit is contained in:
Hare
2026-10-21 20:39:46 +09:00
parent 7a64c38493
commit ca6d4b4b72
2 changed files with 537 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/server.rs
+164 -71
View File
@@ -1046,10 +1046,17 @@ impl ExternalProviderWorkdirSession {
}) })
} }
async fn operate( async fn operate<T, F, Fut>(
&self, &self,
operation: WorkdirSessionOperation, operation: WorkdirSessionOperation,
) -> std::result::Result<WorkdirSessionOperationResult, workdir::WorkdirError> { validate: F,
) -> std::result::Result<T, workdir::WorkdirError>
where
T: Send,
F: FnOnce(std::result::Result<WorkdirSessionOperationResult, workdir::WorkdirError>) -> Fut
+ Send,
Fut: std::future::Future<Output = std::result::Result<T, workdir::WorkdirError>> + Send,
{
let operation_kind = external_workdir_operation_kind(&operation); let operation_kind = external_workdir_operation_kind(&operation);
let operation_timeout = external_workdir_operation_timeout(&operation); let operation_timeout = external_workdir_operation_timeout(&operation);
if self.closed.load(Ordering::Acquire) { if self.closed.load(Ordering::Acquire) {
@@ -1169,6 +1176,12 @@ impl ExternalProviderWorkdirSession {
) )
} }
}; };
// Provider dispatch is finished; validation may issue bounded command cleanup
// and must not retain a data-operation slot while awaiting another operation.
drop(_admission);
// A received response is not evidence of operation success until its requested
// type and semantics (including mutation uncertainty) have been checked.
let result = validate(result).await;
let outcome = match &result { let outcome = match &result {
Ok(_) => Outcome::Succeeded, Ok(_) => Outcome::Succeeded,
Err(error) Err(error)
@@ -1275,9 +1288,15 @@ impl ExternalProviderWorkdirSession {
// Do not acquire the output lock until cancellation has unblocked any reader. // Do not acquire the output lock until cancellation has unblocked any reader.
// The reader records its last known cursor before cleanup acknowledges bytes. // The reader records its last known cursor before cleanup acknowledges bytes.
let _ = self let _ = self
.operate(WorkdirSessionOperation::CommandCancel( .operate(
command.handle.clone(), WorkdirSessionOperation::CommandCancel(command.handle.clone()),
)) |result| async move {
match result? {
WorkdirSessionOperationResult::CommandCancel => Ok(()),
_ => Err(Self::mismatch("command_cancel")),
}
},
)
.await; .await;
let mut cursor = let mut cursor =
tokio::time::timeout(std::time::Duration::from_secs(30), command.cursor.lock()) tokio::time::timeout(std::time::Duration::from_secs(30), command.cursor.lock())
@@ -1292,18 +1311,22 @@ impl ExternalProviderWorkdirSession {
} }
let drain = async { let drain = async {
loop { loop {
match self let output = self
.operate(WorkdirSessionOperation::CommandOutput( .operate(
workdir::CommandOutputRequest { WorkdirSessionOperation::CommandOutput(workdir::CommandOutputRequest {
handle: command.handle.clone(), handle: command.handle.clone(),
cursor: *cursor, cursor: *cursor,
limit: workdir::MAX_COMMAND_READ_BYTES, limit: workdir::MAX_COMMAND_READ_BYTES,
wait: true, wait: true,
}),
|result| async move {
match result? {
WorkdirSessionOperationResult::CommandOutput(output) => Ok(output),
_ => Err(Self::mismatch("command_output")),
}
}, },
)) )
.await? .await?;
{
WorkdirSessionOperationResult::CommandOutput(output) => {
*cursor = output.next_cursor; *cursor = output.next_cursor;
if output.terminal.is_some() { if output.terminal.is_some() {
*command.terminal_at.lock().unwrap() = Some(std::time::Instant::now()); *command.terminal_at.lock().unwrap() = Some(std::time::Instant::now());
@@ -1311,14 +1334,10 @@ impl ExternalProviderWorkdirSession {
} }
if output.chunks.is_empty() { if output.chunks.is_empty() {
return Err(workdir::WorkdirError::Unavailable( return Err(workdir::WorkdirError::Unavailable(
"External Workdir provider did not confirm command termination" "External Workdir provider did not confirm command termination".to_string(),
.to_string(),
)); ));
} }
} }
_ => return Err(Self::mismatch("command_output")),
}
}
}; };
tokio::time::timeout(std::time::Duration::from_secs(30), drain) tokio::time::timeout(std::time::Duration::from_secs(30), drain)
.await .await
@@ -1384,36 +1403,42 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
request: workdir::WorkdirScopeAuthorizationRequest, request: workdir::WorkdirScopeAuthorizationRequest,
) -> std::result::Result<(), workdir::WorkdirError> { ) -> std::result::Result<(), workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; self.ensure_capability(workdir::WorkdirSessionCapability::Read)?;
match self self.operate(
.operate(WorkdirSessionOperation::AuthorizeScope(request)) WorkdirSessionOperation::AuthorizeScope(request),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::AuthorizeScope => Ok(()), WorkdirSessionOperationResult::AuthorizeScope => Ok(()),
_ => Err(Self::mismatch("authorize_scope")), _ => Err(Self::mismatch("authorize_scope")),
} }
},
)
.await
} }
async fn scope_rules_overlap( async fn scope_rules_overlap(
&self, &self,
request: workdir::WorkdirScopeOverlapRequest, request: workdir::WorkdirScopeOverlapRequest,
) -> std::result::Result<bool, workdir::WorkdirError> { ) -> std::result::Result<bool, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; self.ensure_capability(workdir::WorkdirSessionCapability::Read)?;
match self self.operate(
.operate(WorkdirSessionOperation::ScopeRulesOverlap(request)) WorkdirSessionOperation::ScopeRulesOverlap(request),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::ScopeRulesOverlap { overlaps } => Ok(overlaps), WorkdirSessionOperationResult::ScopeRulesOverlap { overlaps } => Ok(overlaps),
_ => Err(Self::mismatch("scope_rules_overlap")), _ => Err(Self::mismatch("scope_rules_overlap")),
} }
},
)
.await
} }
async fn checkout_observe( async fn checkout_observe(
&self, &self,
path: workdir::WorkdirPath, path: workdir::WorkdirPath,
) -> std::result::Result<workdir::CheckoutObservation, workdir::WorkdirError> { ) -> std::result::Result<workdir::CheckoutObservation, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; self.ensure_capability(workdir::WorkdirSessionCapability::Read)?;
match self self.operate(
.operate(WorkdirSessionOperation::CheckoutObserve(path.clone())) WorkdirSessionOperation::CheckoutObserve(path.clone()),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::CheckoutObserve(mut value) WorkdirSessionOperationResult::CheckoutObserve(mut value)
if value.path == path if value.path == path
&& !value.validator.is_empty() && !value.validator.is_empty()
@@ -1424,6 +1449,9 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
} }
_ => Err(Self::mismatch("checkout_observe")), _ => Err(Self::mismatch("checkout_observe")),
} }
},
)
.await
} }
async fn checkout_execute( async fn checkout_execute(
&self, &self,
@@ -1433,10 +1461,10 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
self.ensure_capability(cap)?; self.ensure_capability(cap)?;
let target = request.target.clone(); let target = request.target.clone();
let mutation = cap != workdir::WorkdirSessionCapability::Read; let mutation = cap != workdir::WorkdirSessionCapability::Read;
match self self.operate(
.operate(WorkdirSessionOperation::CheckoutExecute(request)) WorkdirSessionOperation::CheckoutExecute(request),
.await |result| async move {
{ match result {
Ok(WorkdirSessionOperationResult::CheckoutExecute(mut value)) Ok(WorkdirSessionOperationResult::CheckoutExecute(mut value))
if value.observation.path == target if value.observation.path == target
&& !value.observation.validator.is_empty() && !value.observation.validator.is_empty()
@@ -1470,7 +1498,8 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
if mutation if mutation
&& matches!( && matches!(
error.classification_source(), error.classification_source(),
workdir::WorkdirError::Unavailable(_) | workdir::WorkdirError::Transport(_) workdir::WorkdirError::Unavailable(_)
| workdir::WorkdirError::Transport(_)
) => ) =>
{ {
let outcome = workdir::WorkdirError::OutcomeUnknown( let outcome = workdir::WorkdirError::OutcomeUnknown(
@@ -1490,6 +1519,9 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
} }
Err(error) => Err(error), Err(error) => Err(error),
} }
},
)
.await
} }
async fn checkout_search( async fn checkout_search(
&self, &self,
@@ -1505,10 +1537,10 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
Op::Grep(_) => Cap::Grep, Op::Grep(_) => Cap::Grep,
}; };
self.ensure_capability(cap)?; self.ensure_capability(cap)?;
match self self.operate(
.operate(WorkdirSessionOperation::CheckoutSearch(request)) WorkdirSessionOperation::CheckoutSearch(request),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::CheckoutSearch(value) WorkdirSessionOperationResult::CheckoutSearch(value)
if matches!( if matches!(
(&value, cap), (&value, cap),
@@ -1521,92 +1553,137 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
} }
_ => Err(Self::mismatch("checkout_search")), _ => Err(Self::mismatch("checkout_search")),
} }
},
)
.await
} }
async fn stat( async fn stat(
&self, &self,
request: workdir::StatRequest, request: workdir::StatRequest,
) -> std::result::Result<workdir::StatResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::StatResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; self.ensure_capability(workdir::WorkdirSessionCapability::Read)?;
match self.operate(WorkdirSessionOperation::Stat(request)).await? { self.operate(
WorkdirSessionOperation::Stat(request),
|result| async move {
match result? {
WorkdirSessionOperationResult::Stat(value) => Ok(value), WorkdirSessionOperationResult::Stat(value) => Ok(value),
_ => Err(Self::mismatch("stat")), _ => Err(Self::mismatch("stat")),
} }
},
)
.await
} }
async fn read( async fn read(
&self, &self,
request: workdir::ReadRequest, request: workdir::ReadRequest,
) -> std::result::Result<workdir::ReadResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::ReadResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; self.ensure_capability(workdir::WorkdirSessionCapability::Read)?;
match self.operate(WorkdirSessionOperation::Read(request)).await? { self.operate(
WorkdirSessionOperation::Read(request),
|result| async move {
match result? {
WorkdirSessionOperationResult::Read(value) => Ok(value), WorkdirSessionOperationResult::Read(value) => Ok(value),
_ => Err(Self::mismatch("read")), _ => Err(Self::mismatch("read")),
} }
},
)
.await
} }
async fn read_bytes( async fn read_bytes(
&self, &self,
request: workdir::ReadBytesRequest, request: workdir::ReadBytesRequest,
) -> std::result::Result<workdir::ReadBytesResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::ReadBytesResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; self.ensure_capability(workdir::WorkdirSessionCapability::Read)?;
match self self.operate(
.operate(WorkdirSessionOperation::ReadBytes(request)) WorkdirSessionOperation::ReadBytes(request),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::ReadBytes(value) => Ok(value), WorkdirSessionOperationResult::ReadBytes(value) => Ok(value),
_ => Err(Self::mismatch("read_bytes")), _ => Err(Self::mismatch("read_bytes")),
} }
},
)
.await
} }
async fn write( async fn write(
&self, &self,
request: workdir::WriteRequest, request: workdir::WriteRequest,
) -> std::result::Result<workdir::WriteResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::WriteResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Write)?; self.ensure_capability(workdir::WorkdirSessionCapability::Write)?;
match self self.operate(
.operate(WorkdirSessionOperation::Write(request)) WorkdirSessionOperation::Write(request),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::Write(value) => Ok(value), WorkdirSessionOperationResult::Write(value) => Ok(value),
_ => Err(Self::mismatch("write")), _ => Err(Self::mismatch("write")),
} }
},
)
.await
} }
async fn edit( async fn edit(
&self, &self,
request: workdir::EditRequest, request: workdir::EditRequest,
) -> std::result::Result<workdir::EditResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::EditResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Edit)?; self.ensure_capability(workdir::WorkdirSessionCapability::Edit)?;
match self.operate(WorkdirSessionOperation::Edit(request)).await? { self.operate(
WorkdirSessionOperation::Edit(request),
|result| async move {
match result? {
WorkdirSessionOperationResult::Edit(value) => Ok(value), WorkdirSessionOperationResult::Edit(value) => Ok(value),
_ => Err(Self::mismatch("edit")), _ => Err(Self::mismatch("edit")),
} }
},
)
.await
} }
async fn list( async fn list(
&self, &self,
request: workdir::ListRequest, request: workdir::ListRequest,
) -> std::result::Result<workdir::ListResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::ListResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; self.ensure_capability(workdir::WorkdirSessionCapability::Read)?;
match self.operate(WorkdirSessionOperation::List(request)).await? { self.operate(
WorkdirSessionOperation::List(request),
|result| async move {
match result? {
WorkdirSessionOperationResult::List(value) => Ok(value), WorkdirSessionOperationResult::List(value) => Ok(value),
_ => Err(Self::mismatch("list")), _ => Err(Self::mismatch("list")),
} }
},
)
.await
} }
async fn glob( async fn glob(
&self, &self,
request: workdir::GlobRequest, request: workdir::GlobRequest,
) -> std::result::Result<workdir::GlobResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::GlobResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Glob)?; self.ensure_capability(workdir::WorkdirSessionCapability::Glob)?;
match self.operate(WorkdirSessionOperation::Glob(request)).await? { self.operate(
WorkdirSessionOperation::Glob(request),
|result| async move {
match result? {
WorkdirSessionOperationResult::Glob(value) => Ok(value), WorkdirSessionOperationResult::Glob(value) => Ok(value),
_ => Err(Self::mismatch("glob")), _ => Err(Self::mismatch("glob")),
} }
},
)
.await
} }
async fn grep( async fn grep(
&self, &self,
request: workdir::GrepRequest, request: workdir::GrepRequest,
) -> std::result::Result<workdir::GrepResult, workdir::WorkdirError> { ) -> std::result::Result<workdir::GrepResult, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Grep)?; self.ensure_capability(workdir::WorkdirSessionCapability::Grep)?;
match self.operate(WorkdirSessionOperation::Grep(request)).await? { self.operate(
WorkdirSessionOperation::Grep(request),
|result| async move {
match result? {
WorkdirSessionOperationResult::Grep(value) => Ok(value), WorkdirSessionOperationResult::Grep(value) => Ok(value),
_ => Err(Self::mismatch("grep")), _ => Err(Self::mismatch("grep")),
} }
},
)
.await
} }
async fn start_command( async fn start_command(
&self, &self,
@@ -1631,10 +1708,10 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
"External Workdir active command limit reached".to_string(), "External Workdir active command limit reached".to_string(),
)); ));
} }
match self self.operate(
.operate(WorkdirSessionOperation::CommandStart(request)) WorkdirSessionOperation::CommandStart(request),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::CommandStart(provider_handle) => { WorkdirSessionOperationResult::CommandStart(provider_handle) => {
let duplicate_handles = { let duplicate_handles = {
let commands = self.active_commands.lock().await; let commands = self.active_commands.lock().await;
@@ -1655,8 +1732,10 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
quarantined.get_or_insert(command); quarantined.get_or_insert(command);
} }
} }
let quarantine_handle = let quarantine_handle = workdir::CommandHandle(format!(
workdir::CommandHandle(format!("quarantined-command-{}", Uuid::now_v7())); "quarantined-command-{}",
Uuid::now_v7()
));
let command = quarantined.expect("duplicate command route disappeared"); let command = quarantined.expect("duplicate command route disappeared");
*command.terminal_at.lock().unwrap() = None; *command.terminal_at.lock().unwrap() = None;
commands.insert(quarantine_handle.0.clone(), command.clone()); commands.insert(quarantine_handle.0.clone(), command.clone());
@@ -1674,7 +1753,8 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
*command.terminal_at.lock().unwrap() = None; *command.terminal_at.lock().unwrap() = None;
} }
return Err(workdir::WorkdirError::Unavailable( return Err(workdir::WorkdirError::Unavailable(
"External Workdir provider reused an active command handle".to_string(), "External Workdir provider reused an active command handle"
.to_string(),
)); ));
} }
let external_handle = let external_handle =
@@ -1687,6 +1767,9 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
} }
_ => Err(Self::mismatch("command_start")), _ => Err(Self::mismatch("command_start")),
} }
},
)
.await
} }
async fn command_status( async fn command_status(
&self, &self,
@@ -1694,13 +1777,16 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
) -> std::result::Result<workdir::CommandStatus, workdir::WorkdirError> { ) -> std::result::Result<workdir::CommandStatus, workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Command)?; self.ensure_capability(workdir::WorkdirSessionCapability::Command)?;
let provider_handle = self.provider_command_handle(&external_handle).await?; let provider_handle = self.provider_command_handle(&external_handle).await?;
match self self.operate(
.operate(WorkdirSessionOperation::CommandStatus(provider_handle)) WorkdirSessionOperation::CommandStatus(provider_handle),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::CommandStatus(status) => Ok(status), WorkdirSessionOperationResult::CommandStatus(status) => Ok(status),
_ => Err(Self::mismatch("command_status")), _ => Err(Self::mismatch("command_status")),
} }
},
)
.await
} }
async fn command_output( async fn command_output(
&self, &self,
@@ -1709,13 +1795,14 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
self.ensure_capability(workdir::WorkdirSessionCapability::Command)?; self.ensure_capability(workdir::WorkdirSessionCapability::Command)?;
let external_handle = request.handle.clone(); let external_handle = request.handle.clone();
let command = self.provider_command(&external_handle).await?; let command = self.provider_command(&external_handle).await?;
let command = &command;
let mut cursor = command.cursor.lock().await; let mut cursor = command.cursor.lock().await;
request.handle = command.handle.clone(); request.handle = command.handle.clone();
// If the response is lost, this request cursor still replays the provider page. // If the response is lost, this request cursor still replays the provider page.
match self self.operate(
.operate(WorkdirSessionOperation::CommandOutput(request)) WorkdirSessionOperation::CommandOutput(request),
.await |result| async move {
{ match result {
Ok(WorkdirSessionOperationResult::CommandOutput(output)) => { Ok(WorkdirSessionOperationResult::CommandOutput(output)) => {
*cursor = output.next_cursor; *cursor = output.next_cursor;
if output.terminal.is_some() { if output.terminal.is_some() {
@@ -1732,6 +1819,9 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
Ok(_) => Err(Self::mismatch("command_output")), Ok(_) => Err(Self::mismatch("command_output")),
Err(error) => Err(error), Err(error) => Err(error),
} }
},
)
.await
} }
async fn cancel_command( async fn cancel_command(
&self, &self,
@@ -1739,13 +1829,16 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
) -> std::result::Result<(), workdir::WorkdirError> { ) -> std::result::Result<(), workdir::WorkdirError> {
self.ensure_capability(workdir::WorkdirSessionCapability::Command)?; self.ensure_capability(workdir::WorkdirSessionCapability::Command)?;
let provider_handle = self.provider_command_handle(&external_handle).await?; let provider_handle = self.provider_command_handle(&external_handle).await?;
match self self.operate(
.operate(WorkdirSessionOperation::CommandCancel(provider_handle)) WorkdirSessionOperation::CommandCancel(provider_handle),
.await? |result| async move {
{ match result? {
WorkdirSessionOperationResult::CommandCancel => Ok(()), WorkdirSessionOperationResult::CommandCancel => Ok(()),
_ => Err(Self::mismatch("command_cancel")), _ => Err(Self::mismatch("command_cancel")),
} }
},
)
.await
} }
async fn close(&self) -> std::result::Result<(), workdir::WorkdirError> { async fn close(&self) -> std::result::Result<(), workdir::WorkdirError> {
let _close_guard = self.close_lock.lock().await; let _close_guard = self.close_lock.lock().await;
crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs
+194 -8
View File
@@ -46,7 +46,10 @@ struct ProviderProbe {
} }
impl ProviderProbe { impl ProviderProbe {
async fn receive_write(&mut self, pending: &mut PendingHttpWrite) -> (String, ProviderReply) { async fn receive_operation(
&mut self,
pending: &mut PendingHttpWrite,
) -> (String, WorkdirSessionOperation, ProviderReply) {
let command = tokio::time::timeout(BOUND, async { let command = tokio::time::timeout(BOUND, async {
tokio::select! { tokio::select! {
command = self.receiver.recv() => command.expect("provider channel closed"), command = self.receiver.recv() => command.expect("provider channel closed"),
@@ -66,6 +69,11 @@ impl ProviderProbe {
panic!("expected write dispatch, received cancellation") panic!("expected write dispatch, received cancellation")
}; };
self.dispatches += 1; self.dispatches += 1;
(operation_id.as_str().to_owned(), operation, response)
}
async fn receive_write(&mut self, pending: &mut PendingHttpWrite) -> (String, ProviderReply) {
let (operation_id, operation, response) = self.receive_operation(pending).await;
let WorkdirSessionOperation::Write(request) = operation else { let WorkdirSessionOperation::Write(request) = operation else {
panic!("expected mutation, got {operation:?}") panic!("expected mutation, got {operation:?}")
}; };
@@ -75,7 +83,7 @@ impl ProviderProbe {
); );
assert_eq!(request.content, WRITE_SECRET); assert_eq!(request.content, WRITE_SECRET);
assert_eq!(request.expected_hash, None); assert_eq!(request.expected_hash, None);
(operation_id.as_str().to_owned(), response) (operation_id, response)
} }
fn assert_dispatches(&mut self, expected: usize) { fn assert_dispatches(&mut self, expected: usize) {
@@ -200,14 +208,18 @@ impl AuditHttpFixture {
} }
fn start_write(&self) -> PendingHttpWrite { fn start_write(&self) -> PendingHttpWrite {
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations"); self.start_operation(WorkdirSessionOperation::Write(workdir::WriteRequest {
let body = serde_json::to_vec(&WorkspaceWorkdirSessionOperationRequest {
target_workdir: "attachment".into(),
operation: WorkdirSessionOperation::Write(workdir::WriteRequest {
path: workdir::WorkdirPath::new(PATH_SECRET).unwrap(), path: workdir::WorkdirPath::new(PATH_SECRET).unwrap(),
content: WRITE_SECRET.to_vec(), content: WRITE_SECRET.to_vec(),
expected_hash: None, expected_hash: None,
}), }))
}
fn start_operation(&self, operation: WorkdirSessionOperation) -> PendingHttpWrite {
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations");
let body = serde_json::to_vec(&WorkspaceWorkdirSessionOperationRequest {
target_workdir: "attachment".into(),
operation,
}) })
.unwrap(); .unwrap();
let request = self.fixture.request( let request = self.fixture.request(
@@ -259,6 +271,7 @@ impl AuditHttpFixture {
encoded_content.as_str(), encoded_content.as_str(),
PATH_SECRET, PATH_SECRET,
PROVIDER_SECRET, PROVIDER_SECRET,
PROVIDER_SECRET.trim_start_matches('/'),
pending.proof.as_str(), pending.proof.as_str(),
worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER, worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER,
self.fixture.identity.private_key.as_str(), self.fixture.identity.private_key.as_str(),
@@ -276,6 +289,10 @@ impl AuditHttpFixture {
} }
fn assert_audits(&self, expected: &[(&str, &str)]) { fn assert_audits(&self, expected: &[(&str, &str)]) {
self.assert_operation_audits(expected, "write");
}
fn assert_operation_audits(&self, expected: &[(&str, &str)], operation: &str) {
let connection = let connection =
rusqlite::Connection::open(&self.fixture.api.config.database_path).unwrap(); rusqlite::Connection::open(&self.fixture.api.config.database_path).unwrap();
let mut statement = connection let mut statement = connection
@@ -298,7 +315,10 @@ impl AuditHttpFixture {
.collect(); .collect();
assert_eq!(actual, expected, "persisted audits: {rows:?}"); assert_eq!(actual, expected, "persisted audits: {rows:?}");
for (_, _, summary) in rows { for (_, _, summary) in rows {
assert!(summary.contains("operation_kind=write"), "{summary}"); assert!(
summary.contains(&format!("operation_kind={operation}")),
"{summary}"
);
for secret in [ for secret in [
AUDIT_SECRET, AUDIT_SECRET,
PATH_SECRET, PATH_SECRET,
@@ -697,3 +717,169 @@ async fn provider_reported_unknown_preserves_received_response_and_unknown_effec
provider.assert_dispatches(1); provider.assert_dispatches(1);
h.assert_audits(&[(&dispatched_id, "requested")]); h.assert_audits(&[(&dispatched_id, "requested")]);
} }
#[derive(Clone, Copy, Debug)]
enum CheckoutReplyFault {
WrongOperation,
WrongPath,
EmptyValidator,
WrongOutput,
ProviderUnavailable,
ProviderTransport,
}
async fn assert_checkout_unknown_audit(fault: CheckoutReplyFault, fail_completion: bool) {
let (h, mut provider) = AuditHttpFixture::new().await;
if fail_completion {
h.fail_audit(false);
}
let target = workdir::WorkdirPath::new(PATH_SECRET).unwrap();
let mut pending = h.start_operation(WorkdirSessionOperation::CheckoutExecute(
workdir::CheckoutRequest {
target: target.clone(),
validator: vec![1],
operation: workdir::CheckoutOperation::Write {
content: WRITE_SECRET.to_vec(),
expected_hash: [0; 32],
},
},
));
let (operation_id, operation, reply) = provider.receive_operation(&mut pending).await;
let WorkdirSessionOperation::CheckoutExecute(request) = operation else {
panic!("expected checkout mutation, got {operation:?}");
};
assert_eq!(request.target, target);
assert_eq!(request.validator, vec![1]);
assert!(
matches!(request.operation, workdir::CheckoutOperation::Write { ref content, .. } if content == WRITE_SECRET)
);
h.apply_provider_write();
let result = match fault {
CheckoutReplyFault::WrongOperation => Ok(write_result()),
CheckoutReplyFault::ProviderUnavailable => Err(WorkdirTransportError::from_workdir_error(
&workdir::WorkdirError::Unavailable(PROVIDER_SECRET.into()),
)),
CheckoutReplyFault::ProviderTransport => Err(WorkdirTransportError::from_workdir_error(
&workdir::WorkdirError::Transport(PROVIDER_SECRET.into()),
)),
_ => {
let mut result = workdir::CheckoutResult {
observation: workdir::CheckoutObservation {
path: target,
validator: vec![2],
kind: workdir::EntryKind::File,
size: WRITE_SECRET.len() as u64,
capabilities: workdir::WorkdirSessionCapabilities::READ_WRITE,
},
output: workdir::CheckoutOutput::Write(workdir::WriteResult {
bytes_written: WRITE_SECRET.len(),
created: false,
}),
};
match fault {
CheckoutReplyFault::WrongPath => {
result.observation.path =
workdir::WorkdirPath::new(PROVIDER_SECRET.trim_start_matches('/')).unwrap()
}
CheckoutReplyFault::EmptyValidator => result.observation.validator.clear(),
CheckoutReplyFault::WrongOutput => {
result.output = workdir::CheckoutOutput::Edit(workdir::EditResult {
replacements: 1,
content_hash: [0; 32],
bytes_written: 1,
})
}
_ => unreachable!(),
}
Ok(WorkdirSessionOperationResult::CheckoutExecute(result))
}
};
reply.send(result).unwrap();
let (body, events) = h.finish(pending, StatusCode::INTERNAL_SERVER_ERROR).await;
assert_eq!(body["code"], "outcome_unknown", "{fault:?}: {body}");
assert_eq!(
body["message"],
"Workdir operation may have effects; inspect before retrying"
);
if !matches!(
fault,
CheckoutReplyFault::ProviderUnavailable | CheckoutReplyFault::ProviderTransport
) {
assert_failure_reason(&events, "operation_result_mismatch", "response_validation");
}
if fail_completion {
assert_eq!(
body["audit_failures"],
json!([{
"operation_id": operation_id,
"stage": "response_received",
"reason": "audit_store_unavailable",
"operation_outcome": {"kind":"unknown"},
}]),
"{fault:?}: {body}"
);
let audit_logs: Vec<_> = events
.iter()
.filter(|e| e["event"] == "external_workdir_audit_failed")
.collect();
assert_eq!(audit_logs.len(), 1, "{events:?}");
let event = audit_logs[0];
assert_eq!(event["operation_id"], operation_id);
assert_eq!(event["operation"], "checkout_execute");
assert_eq!(event["audit_stage"], "ResponseReceived");
assert_eq!(event["operation_outcome"], "Unknown");
h.assert_operation_audits(&[(&operation_id, "requested")], "checkout_execute");
} else {
assert!(body.get("audit_failures").is_none(), "{body}");
assert!(
events
.iter()
.all(|e| e["event"] != "external_workdir_audit_failed")
);
h.assert_operation_audits(
&[
(&operation_id, "requested"),
(&operation_id, "outcome_unknown"),
],
"checkout_execute",
);
}
assert_eq!(std::fs::read(h.effect_path()).unwrap(), WRITE_SECRET);
provider.assert_dispatches(1);
}
#[tokio::test]
async fn checkout_invalid_reply_records_unknown_in_healthy_completion_audit() {
for fault in [
CheckoutReplyFault::WrongOperation,
CheckoutReplyFault::WrongPath,
CheckoutReplyFault::EmptyValidator,
CheckoutReplyFault::WrongOutput,
] {
assert_checkout_unknown_audit(fault, false).await;
}
}
#[tokio::test]
async fn checkout_invalid_reply_records_unknown_in_failed_completion_receipt_and_log() {
for fault in [
CheckoutReplyFault::WrongOperation,
CheckoutReplyFault::WrongPath,
CheckoutReplyFault::EmptyValidator,
CheckoutReplyFault::WrongOutput,
] {
assert_checkout_unknown_audit(fault, true).await;
}
}
#[tokio::test]
async fn checkout_uncertain_reply_normalization_matches_audit_outcome() {
for fault in [
CheckoutReplyFault::ProviderUnavailable,
CheckoutReplyFault::ProviderTransport,
] {
for fail_completion in [false, true] {
assert_checkout_unknown_audit(fault, fail_completion).await;
}
}
}