chore: integrate backend Job notifications with diagnostics
This commit is contained in:
@@ -469,7 +469,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
(
|
||||
95,
|
||||
"Common notification routing and durable Worker delivery".into()
|
||||
)
|
||||
),
|
||||
(96,"Backend Job common notification routing".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
//! Backend Job adapter for the common notification ledger. Results, not Worker
|
||||
//! lifecycle, are the durable source. Fixed Worker recipients never follow leases.
|
||||
use super::*;
|
||||
use crate::backend_job::{
|
||||
BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState,
|
||||
BackendJobResultAcceptance, BackendJobState,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct JobNotificationSource {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub delivery_id: String,
|
||||
}
|
||||
|
||||
/// Domain facts, not a second notification policy. Rendering is performed by the
|
||||
/// domain adapter; only this routing module chooses deliver/suppress/defer.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum JobNotificationFacts {
|
||||
Result(String),
|
||||
NoFindings,
|
||||
StaleContent,
|
||||
Failed(NotificationFailureCode),
|
||||
}
|
||||
|
||||
pub(in crate::store) fn install(conn: &Connection) -> Result<()> {
|
||||
// Generalize the existing source parent without changing any sequence, request,
|
||||
// or receipt. The migration caller disables FK enforcement for this rebuild.
|
||||
let triggers = conn
|
||||
.prepare("SELECT name,sql FROM sqlite_schema WHERE type='trigger'")?
|
||||
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let indexes = conn.prepare("SELECT sql FROM sqlite_schema WHERE type='index' AND tbl_name='notification_sources' AND sql IS NOT NULL")?
|
||||
.query_map([], |r| r.get::<_,String>(0))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
for (name, _) in &triggers {
|
||||
conn.execute_batch(&format!("DROP TRIGGER \"{}\"", name.replace('"', "\"\"")))?;
|
||||
}
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE notification_sources_job_upgrade (
|
||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workspace_id TEXT NOT NULL, ticket_id TEXT, input_event_id TEXT NOT NULL,
|
||||
role TEXT NOT NULL, assignment_id TEXT NOT NULL, kind TEXT NOT NULL,
|
||||
actor TEXT, event_index INTEGER, facts_json TEXT NOT NULL CHECK(json_valid(facts_json)),
|
||||
CHECK(ticket_id IS NOT NULL OR role='backend_job'),
|
||||
UNIQUE(workspace_id,ticket_id,input_event_id,role,assignment_id),
|
||||
FOREIGN KEY(workspace_id,ticket_id) REFERENCES typed_tickets(workspace_id,ticket_id));
|
||||
INSERT INTO notification_sources_job_upgrade SELECT * FROM notification_sources;
|
||||
DROP TABLE notification_sources;
|
||||
ALTER TABLE notification_sources_job_upgrade RENAME TO notification_sources;",
|
||||
)?;
|
||||
for sql in indexes {
|
||||
conn.execute_batch(&sql)?;
|
||||
}
|
||||
for (_, sql) in triggers {
|
||||
conn.execute_batch(&sql)?;
|
||||
}
|
||||
conn.execute_batch("CREATE TRIGGER notification_job_source_guard BEFORE INSERT ON notification_sources WHEN NEW.role='backend_job' BEGIN
|
||||
SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)
|
||||
WHERE j.workspace_id=NEW.workspace_id AND j.job_id=json_extract(NEW.facts_json,'$.backend_job.job_id') AND a.attempt_id=json_extract(NEW.facts_json,'$.backend_job.attempt_id')
|
||||
AND j.state='completed' AND a.state='completed' AND a.attempt=j.current_attempt AND j.result_digest=json_extract(NEW.facts_json,'$.result_digest'))
|
||||
THEN RAISE(ABORT,'Job notification requires a committed current result') END;
|
||||
END;
|
||||
CREATE UNIQUE INDEX notification_job_identity ON notification_sources(workspace_id,input_event_id) WHERE role='backend_job';
|
||||
CREATE TABLE notification_job_targets (
|
||||
workspace_id TEXT NOT NULL, delivery_id TEXT NOT NULL, target_json TEXT NOT NULL CHECK(json_valid(target_json)),
|
||||
PRIMARY KEY(workspace_id,delivery_id),
|
||||
FOREIGN KEY(workspace_id,delivery_id) REFERENCES notification_requests(workspace_id,delivery_id));
|
||||
CREATE TRIGGER notification_job_targets_no_update BEFORE UPDATE ON notification_job_targets BEGIN SELECT RAISE(ABORT,'Job notification target is immutable'); END;
|
||||
CREATE TRIGGER notification_job_targets_no_delete BEFORE DELETE ON notification_job_targets BEGIN SELECT RAISE(ABORT,'Job notification target is immutable'); END;
|
||||
CREATE TABLE notification_job_legacy_receipts (
|
||||
workspace_id TEXT NOT NULL, delivery_id TEXT NOT NULL, record_json TEXT NOT NULL CHECK(json_valid(record_json)),
|
||||
runtime_accepted INTEGER NOT NULL CHECK(runtime_accepted IN (0,1)),
|
||||
PRIMARY KEY(workspace_id,delivery_id),
|
||||
FOREIGN KEY(workspace_id,delivery_id) REFERENCES notification_decisions(workspace_id,delivery_id));
|
||||
CREATE TRIGGER notification_job_legacy_no_update BEFORE UPDATE ON notification_job_legacy_receipts BEGIN SELECT RAISE(ABORT,'Legacy notification evidence is immutable'); END;
|
||||
CREATE TRIGGER notification_job_legacy_no_delete BEFORE DELETE ON notification_job_legacy_receipts BEGIN SELECT RAISE(ABORT,'Legacy notification evidence is immutable'); END;")?;
|
||||
// Includes no-target results and the crash gap before the former reservation.
|
||||
let identities = conn.prepare("SELECT a.workspace_id,a.job_id,a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j USING(workspace_id,job_id) WHERE a.state='completed' AND j.state='completed' AND a.attempt=j.current_attempt")?
|
||||
.query_map([], |r| Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?,r.get::<_,String>(2)?)))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
for (workspace, job, attempt) in identities {
|
||||
capture(conn, &workspace, &job, &attempt)?;
|
||||
import_legacy(conn, &workspace, &job, &attempt)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(in crate::store) fn capture(
|
||||
conn: &Connection,
|
||||
workspace: &str,
|
||||
job: &str,
|
||||
attempt: &str,
|
||||
) -> Result<()> {
|
||||
let id = retained_delivery_id(conn, workspace, job, attempt)?;
|
||||
let source = JobNotificationSource {
|
||||
job_id: job.into(),
|
||||
attempt_id: attempt.into(),
|
||||
delivery_id: id.clone(),
|
||||
};
|
||||
let job = read_backend_job(conn, workspace, job)?
|
||||
.ok_or_else(|| Error::Store("Job notification source missing".into()))?;
|
||||
let target = &job.request.notification_target;
|
||||
let facts = serde_json::json!({"principal_kind":if target.is_some(){Some("worker")}else{None}, "principal_id":null,"runtime_id":target.as_ref().map(|w|&w.runtime_id),"worker_id":target.as_ref().map(|w|&w.worker_id),"status":null,"case_state":"open","released":false,"backend_job":source,"recipient_kind":"fixed_worker","job_purpose":job.request.purpose,"result_digest":job.result_digest});
|
||||
conn.execute("INSERT OR IGNORE INTO notification_sources(workspace_id,ticket_id,input_event_id,role,assignment_id,kind,facts_json) VALUES(?1,NULL,?2,'backend_job','','backend_job_result',?3)",params![workspace,id,json(&facts)?])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(in crate::store) fn pending(
|
||||
store: &SqliteWorkspaceStore,
|
||||
workspace: &str,
|
||||
limit: usize,
|
||||
) -> Result<Vec<JobNotificationSource>> {
|
||||
store.with_conn_mut(|conn| {
|
||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let rows = tx.prepare("SELECT s.facts_json,s.sequence FROM notification_sources s LEFT JOIN notification_decisions d ON d.source_sequence=s.sequence WHERE s.workspace_id=?1 AND s.role='backend_job' AND (d.source_sequence IS NULL OR d.disposition='deliver') AND NOT EXISTS(SELECT 1 FROM notification_acknowledgements a WHERE a.workspace_id=?1 AND a.delivery_id=s.input_event_id) AND NOT EXISTS(SELECT 1 FROM notification_job_legacy_receipts l WHERE l.workspace_id=?1 AND l.delivery_id=s.input_event_id AND l.runtime_accepted=1) ORDER BY COALESCE(d.last_offered,0),s.sequence LIMIT ?2")?
|
||||
.query_map(params![workspace,limit.clamp(1,100) as i64],|r|Ok((r.get::<_,String>(0)?,r.get::<_,i64>(1)?)))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let tick:i64=tx.query_row("SELECT COALESCE(MAX(last_offered),0)+1 FROM notification_decisions WHERE workspace_id=?1",[workspace],|r|r.get(0))?;
|
||||
let mut sources=Vec::new();
|
||||
for (facts,sequence) in rows {
|
||||
let facts:serde_json::Value=decode(&facts)?;
|
||||
sources.push(serde_json::from_value(facts["backend_job"].clone()).map_err(|e|Error::Store(e.to_string()))?);
|
||||
tx.execute("UPDATE notification_decisions SET last_offered=?2 WHERE source_sequence=?1",params![sequence,tick])?;
|
||||
}
|
||||
tx.commit()?; Ok(sources)
|
||||
})
|
||||
}
|
||||
|
||||
fn policy(target: bool, facts: &JobNotificationFacts) -> (&'static str, NotificationReason) {
|
||||
if !target {
|
||||
return ("suppress", NotificationReason::NoRecipient);
|
||||
}
|
||||
match facts {
|
||||
JobNotificationFacts::Result(_) => ("deliver", NotificationReason::Result),
|
||||
JobNotificationFacts::NoFindings => ("suppress", NotificationReason::NoFindings),
|
||||
JobNotificationFacts::StaleContent => ("suppress", NotificationReason::StaleContent),
|
||||
JobNotificationFacts::Failed(_) => ("defer", NotificationReason::SourceUnavailable),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::store) fn route(
|
||||
store: &SqliteWorkspaceStore,
|
||||
workspace: &str,
|
||||
source: &JobNotificationSource,
|
||||
facts: &JobNotificationFacts,
|
||||
) -> Result<bool> {
|
||||
let invalid;
|
||||
let facts = match facts {
|
||||
JobNotificationFacts::Result(content)
|
||||
if content.len() > crate::backend_job::MAX_JOB_DELIVERY_BYTES =>
|
||||
{
|
||||
invalid = JobNotificationFacts::Failed(NotificationFailureCode::ContentTooLarge);
|
||||
&invalid
|
||||
}
|
||||
JobNotificationFacts::Result(content) if content.is_empty() || content.contains('\0') => {
|
||||
invalid = JobNotificationFacts::Failed(NotificationFailureCode::RenderingFailed);
|
||||
&invalid
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
store.with_conn_mut(|conn| {
|
||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job=read_backend_job(&tx,workspace,&source.job_id)?.ok_or_else(||Error::InvalidInput("Job notification source disappeared".into()))?;
|
||||
let attempt=read_backend_job_attempt(&tx,workspace,&source.job_id,&source.attempt_id)?.ok_or_else(||Error::InvalidInput("Job notification attempt disappeared".into()))?;
|
||||
if job.state!=BackendJobState::Completed || attempt.state!=BackendJobAttemptState::Completed || attempt.attempt!=job.current_attempt || source.delivery_id!=retained_delivery_id(&tx,workspace,&source.job_id,&source.attempt_id)? {
|
||||
return Err(Error::InvalidInput("Job notification requires its current committed result".into()));
|
||||
}
|
||||
capture(&tx,workspace,&source.job_id,&source.attempt_id)?;
|
||||
import_legacy(&tx, workspace, &source.job_id, &source.attempt_id)?;
|
||||
let existing:Option<String>=tx.query_row("SELECT disposition FROM notification_decisions WHERE workspace_id=?1 AND delivery_id=?2",params![workspace,source.delivery_id],|r|r.get(0)).optional()?;
|
||||
if let Some(disposition)=existing {
|
||||
let ack:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM notification_acknowledgements WHERE workspace_id=?1 AND delivery_id=?2) OR EXISTS(SELECT 1 FROM notification_job_legacy_receipts WHERE workspace_id=?1 AND delivery_id=?2 AND runtime_accepted=1)",params![workspace,source.delivery_id],|r|r.get(0))?;
|
||||
tx.commit()?; return Ok(disposition=="deliver" && !ack);
|
||||
}
|
||||
let (disposition,reason)=policy(job.request.notification_target.is_some(),facts);
|
||||
save_decision(&tx, workspace, &source.delivery_id, disposition, reason)?;
|
||||
if let JobNotificationFacts::Failed(failure)=facts { save_failure(&tx,workspace,&source.delivery_id,*failure)?; }
|
||||
// Pin bytes with the decision, before any network call. A prepared body is
|
||||
// never rerendered when content changes or an acknowledgement is lost.
|
||||
if disposition=="deliver" {
|
||||
if let (Some(target),JobNotificationFacts::Result(content))=(&job.request.notification_target,facts) {
|
||||
tx.execute("INSERT OR IGNORE INTO notification_requests VALUES(?1,?2,?3,?4)",params![workspace,source.delivery_id,content,json(target)?])?;
|
||||
}
|
||||
}
|
||||
tx.commit()?; Ok(disposition=="deliver")
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::store) fn request(
|
||||
store: &SqliteWorkspaceStore,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
) -> Result<Option<(String, RuntimeWorkerRef)>> {
|
||||
store.with_conn(|conn| {
|
||||
let row:Option<(String,String)>=conn.query_row("SELECT content,target_json FROM notification_requests WHERE workspace_id=?1 AND delivery_id=?2",params![workspace,id],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
|
||||
row.map(|(content,target)|Ok((content,decode(&target)?))).transpose()
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::store) fn ack(
|
||||
store: &SqliteWorkspaceStore,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
target: &RuntimeWorkerRef,
|
||||
) -> Result<()> {
|
||||
store.with_conn_mut(|conn| {
|
||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let saved:String=tx.query_row("SELECT target_json FROM notification_requests WHERE workspace_id=?1 AND delivery_id=?2",params![workspace,id],|r|r.get(0))?;
|
||||
if saved!=json(target)? {return Err(Error::InvalidInput("Job notification target changed".into()));}
|
||||
let authorized:String=tx.query_row("SELECT target_json FROM notification_job_targets WHERE workspace_id=?1 AND delivery_id=?2",params![workspace,id],|r|r.get(0))?;
|
||||
let authorized_target:TicketWorkerNotificationTarget=decode(&authorized)?;
|
||||
if authorized_target.worker!=*target {return Err(Error::InvalidInput("Job acknowledgement target differs from authorized send".into()));}
|
||||
tx.execute("INSERT OR IGNORE INTO notification_acknowledgements VALUES(?1,?2,?3)",params![workspace,id,authorized])?;
|
||||
tx.commit()?; Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn save_decision(
|
||||
conn: &Connection,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
disposition: &str,
|
||||
reason: NotificationReason,
|
||||
) -> Result<()> {
|
||||
conn.execute("INSERT INTO notification_decisions(source_sequence,workspace_id,delivery_id,disposition,reason,purpose) SELECT sequence,workspace_id,?2,?3,?4,'result' FROM notification_sources WHERE workspace_id=?1 AND input_event_id=?2 AND role='backend_job'",params![workspace,id,disposition,code(reason)?])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Old rows did not store request bytes. Only the original deterministic renderers
|
||||
// may reconstruct an interrupted request; no current Ticket contents participate.
|
||||
// Accepted legacy rows keep their actual evidence, never a fabricated new request.
|
||||
fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: &str) -> Result<()> {
|
||||
let id = retained_delivery_id(conn, workspace, job_id, attempt_id)?;
|
||||
let exists:bool=conn.query_row("SELECT EXISTS(SELECT 1 FROM notification_decisions WHERE workspace_id=?1 AND delivery_id=?2)",params![workspace,id],|r|r.get(0))?;
|
||||
if exists {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(old) = read_backend_job_delivery(conn, workspace, &id)? else {
|
||||
return Ok(());
|
||||
};
|
||||
let accepted =
|
||||
old.state == BackendJobDeliveryState::Completed && old.failure_category.is_none();
|
||||
let count:i64=conn.query_row("SELECT COUNT(*) FROM backend_job_deliveries WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace,job_id,attempt_id],|r|r.get(0))?;
|
||||
let mut content = None;
|
||||
let mut failure = legacy_failure(&old);
|
||||
let (disposition, reason) = match old.failure_category.as_deref() {
|
||||
_ if accepted => ("deliver", NotificationReason::Result),
|
||||
_ if count > 1 => {
|
||||
failure = Some(NotificationFailureCode::PreparationFailed);
|
||||
("defer", NotificationReason::SourceUnavailable)
|
||||
}
|
||||
Some("notification_suppressed_no_findings") => ("suppress", NotificationReason::NoFindings),
|
||||
Some("notification_suppressed_stale_ticket_content") => {
|
||||
("suppress", NotificationReason::StaleContent)
|
||||
}
|
||||
_ if old.state == BackendJobDeliveryState::Completed => {
|
||||
("defer", NotificationReason::SourceUnavailable)
|
||||
}
|
||||
Some(
|
||||
"invalid_checker_input"
|
||||
| "invalid_checker_result"
|
||||
| "current_ticket_content_unavailable"
|
||||
| "notification_too_large",
|
||||
) => ("defer", NotificationReason::SourceUnavailable),
|
||||
_ => {
|
||||
let job = read_backend_job(conn, workspace, job_id)?
|
||||
.ok_or_else(|| Error::Store("Legacy Job result missing".into()))?;
|
||||
let attempt = read_backend_job_attempt(conn, workspace, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::Store("Legacy Job attempt missing".into()))?;
|
||||
let acceptance = BackendJobResultAcceptance {
|
||||
job,
|
||||
attempt,
|
||||
replayed: true,
|
||||
};
|
||||
let facts = if acceptance.job.request.notification_target.as_ref() != Some(&old.target)
|
||||
{
|
||||
JobNotificationFacts::Failed(NotificationFailureCode::PreparationFailed)
|
||||
} else {
|
||||
legacy_result_facts(&acceptance)
|
||||
};
|
||||
let decision = policy(acceptance.job.request.notification_target.is_some(), &facts);
|
||||
match facts {
|
||||
JobNotificationFacts::Result(body) => content = Some(body),
|
||||
JobNotificationFacts::Failed(code) => failure = Some(code),
|
||||
// A valid empty result could not have passed the old sender's
|
||||
// findings gate even if it crashed before recording suppression.
|
||||
JobNotificationFacts::NoFindings => failure = None,
|
||||
JobNotificationFacts::StaleContent => {}
|
||||
}
|
||||
decision
|
||||
}
|
||||
};
|
||||
save_decision(conn, workspace, &id, disposition, reason)?;
|
||||
conn.execute(
|
||||
"INSERT INTO notification_job_legacy_receipts VALUES(?1,?2,?3,?4)",
|
||||
params![workspace, id, json(&old)?, accepted],
|
||||
)?;
|
||||
if !accepted && disposition != "suppress" {
|
||||
if let Some(failure) = failure {
|
||||
save_failure(conn, workspace, &id, failure)?;
|
||||
}
|
||||
}
|
||||
if disposition == "deliver" {
|
||||
if let Some(content) = content {
|
||||
conn.execute(
|
||||
"INSERT INTO notification_requests VALUES(?1,?2,?3,?4)",
|
||||
params![workspace, id, content, json(&old.target)?],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn legacy_result_facts(acceptance: &BackendJobResultAcceptance) -> JobNotificationFacts {
|
||||
let content = if acceptance.job.request.purpose == crate::ticket_item_checker::PURPOSE {
|
||||
let Some(result) = &acceptance.job.result else {
|
||||
return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult);
|
||||
};
|
||||
match crate::ticket_item_checker::validate_result(&acceptance.job.request.input, result) {
|
||||
Ok(result) if result.findings.is_empty() => return JobNotificationFacts::NoFindings,
|
||||
Ok(_) => {}
|
||||
Err(_) => return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult),
|
||||
}
|
||||
crate::ticket_item_checker::render_result(acceptance)
|
||||
} else {
|
||||
Ok(format!(
|
||||
"Backend Job `{}` completed attempt `{}` for `{}`. Durable result digest: {}.",
|
||||
acceptance.job.request.job_id,
|
||||
acceptance.attempt.attempt_id,
|
||||
acceptance.job.request.purpose,
|
||||
acceptance
|
||||
.job
|
||||
.result_digest
|
||||
.as_deref()
|
||||
.unwrap_or("unavailable")
|
||||
))
|
||||
};
|
||||
match content {
|
||||
Ok(content) if content.len() > crate::backend_job::MAX_JOB_DELIVERY_BYTES => {
|
||||
JobNotificationFacts::Failed(NotificationFailureCode::ContentTooLarge)
|
||||
}
|
||||
Ok(content) if !content.is_empty() && !content.contains('\0') => {
|
||||
JobNotificationFacts::Result(content)
|
||||
}
|
||||
_ => JobNotificationFacts::Failed(NotificationFailureCode::RenderingFailed),
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_failure(old: &BackendJobDeliveryRecord) -> Option<NotificationFailureCode> {
|
||||
use NotificationFailureCode as F;
|
||||
match old.failure_category.as_deref() {
|
||||
Some("notification_outcome_unknown") => Some(F::AcceptanceUnknown),
|
||||
Some("notification_rejected") => Some(F::RuntimeRejected),
|
||||
Some("notification_too_large") => Some(F::ContentTooLarge),
|
||||
Some("invalid_checker_input" | "invalid_checker_result") => Some(F::InvalidResult),
|
||||
Some("current_ticket_content_unavailable") => Some(F::SourceLookupFailed),
|
||||
_ if matches!(
|
||||
old.state,
|
||||
BackendJobDeliveryState::Sending
|
||||
| BackendJobDeliveryState::Unknown
|
||||
| BackendJobDeliveryState::Completed
|
||||
) =>
|
||||
{
|
||||
Some(F::AcceptanceUnknown)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// None authorizes this exact send target; a typed denial is not a DB failure.
|
||||
pub(in crate::store) fn authorize(
|
||||
store: &SqliteWorkspaceStore,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
target: &TicketWorkerNotificationTarget,
|
||||
) -> Result<Option<NotificationFailureCode>> {
|
||||
store.with_conn_mut(|conn| {
|
||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let recipient:String=tx.query_row("SELECT target_json FROM notification_requests WHERE workspace_id=?1 AND delivery_id=?2",params![workspace,id],|r|r.get(0))?;
|
||||
if decode::<RuntimeWorkerRef>(&recipient)?!=target.worker {return Ok(Some(NotificationFailureCode::PreparationFailed));}
|
||||
let present:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM worker_registry w WHERE w.workspace_id=?1 AND w.runtime_id=?2 AND w.worker_id=?3 AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id AND r.state IN ('executing','failed','succeeded')))",params![workspace,target.worker.runtime_id,target.worker.worker_id],|r|r.get(0))?;
|
||||
if !present {return Ok(Some(NotificationFailureCode::TargetUnavailable));}
|
||||
let binding:Option<(String,Option<String>)>=tx.query_row("SELECT binding_id,revoked_at FROM workspace_runtime_bindings WHERE workspace_id=?1 AND runtime_id=?2",params![workspace,target.worker.runtime_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()?;
|
||||
let Some((binding,revoked))=binding else {return Ok(Some(NotificationFailureCode::TargetUnavailable));};
|
||||
if revoked.is_some() || binding!=target.authentication_binding_id {return Ok(Some(NotificationFailureCode::BindingRevoked));}
|
||||
let saved:Option<String>=tx.query_row("SELECT target_json FROM notification_job_targets WHERE workspace_id=?1 AND delivery_id=?2",params![workspace,id],|r|r.get(0)).optional()?;
|
||||
let serialized=json(target)?;
|
||||
if saved.as_ref().is_some_and(|saved|*saved!=serialized) {return Ok(Some(NotificationFailureCode::BindingRevoked));}
|
||||
tx.execute("INSERT OR IGNORE INTO notification_job_targets VALUES(?1,?2,?3)",params![workspace,id,serialized])?;
|
||||
tx.commit()?; Ok(None)
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::store) fn verify(conn: &Connection) -> Result<()> {
|
||||
for name in [
|
||||
"notification_job_identity",
|
||||
"notification_job_source_guard",
|
||||
"notification_job_targets",
|
||||
"notification_job_targets_no_update",
|
||||
"notification_job_targets_no_delete",
|
||||
"notification_job_legacy_receipts",
|
||||
"notification_job_legacy_no_update",
|
||||
"notification_job_legacy_no_delete",
|
||||
] {
|
||||
let present: bool = conn.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?1)",
|
||||
[name],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if !present {
|
||||
return Err(Error::Store(format!(
|
||||
"Job notification constraint {name} is missing"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn retained_delivery_id(
|
||||
conn: &Connection,
|
||||
workspace: &str,
|
||||
job: &str,
|
||||
attempt: &str,
|
||||
) -> Result<String> {
|
||||
// The production legacy sender used the derived ID, but retained receipts are
|
||||
// authoritative even when imported history contains a non-derived identity.
|
||||
// If history contains multiple identities, an actual ACK wins; otherwise the
|
||||
// importer defers the ambiguity instead of manufacturing another send.
|
||||
let saved:Option<String>=conn.query_row("SELECT delivery_id FROM backend_job_deliveries WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 ORDER BY (state='completed' AND failure_category IS NULL) DESC,created_at,delivery_id LIMIT 1",params![workspace,job,attempt],|r|r.get(0)).optional()?;
|
||||
Ok(saved.unwrap_or_else(|| crate::backend_job::delivery_id(job, attempt)))
|
||||
}
|
||||
|
||||
pub(in crate::store) fn source(
|
||||
store: &SqliteWorkspaceStore,
|
||||
workspace: &str,
|
||||
job: &str,
|
||||
attempt: &str,
|
||||
) -> Result<JobNotificationSource> {
|
||||
store.with_conn(|conn| {
|
||||
let facts:String=conn.query_row("SELECT json_extract(facts_json,'$.backend_job') FROM notification_sources WHERE workspace_id=?1 AND role='backend_job' AND json_extract(facts_json,'$.backend_job.job_id')=?2 AND json_extract(facts_json,'$.backend_job.attempt_id')=?3",params![workspace,job,attempt],|r|r.get(0))?;
|
||||
decode(&facts)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,639 @@
|
||||
// Real SQLite result transactions, migration, reopen and concurrent recovery.
|
||||
fn job_notice_fixture(
|
||||
store: &SqliteWorkspaceStore,
|
||||
id: &str,
|
||||
target: bool,
|
||||
) -> (BackendJobResultAcceptance, JobNotificationSource) {
|
||||
let mut request = resource_job_request(id, id, id);
|
||||
request.purpose = "notification_test".into();
|
||||
request.grants = Default::default();
|
||||
request.profile = "builtin:backend-job".into();
|
||||
request.notification_target = target.then(|| RuntimeWorkerRef::new("r", "replacement"));
|
||||
let worker = RuntimeWorkerRef::new("r", format!("job-worker-{id}"));
|
||||
let mut registration = store
|
||||
.get_worker_registry("w", &RuntimeWorkerRef::new("r", "replacement"))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
registration.worker = worker.clone();
|
||||
store.upsert_worker_registry(®istration).unwrap();
|
||||
let reservation = store
|
||||
.reserve_backend_job("w", &request, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
store
|
||||
.claim_backend_job_attempt_dispatch(
|
||||
"w",
|
||||
id,
|
||||
&reservation.attempt.attempt_id,
|
||||
RESOURCE_JOB_NOW,
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.bind_backend_job_attempt_worker(
|
||||
"w",
|
||||
id,
|
||||
&reservation.attempt.attempt_id,
|
||||
&worker,
|
||||
Some("run"),
|
||||
RESOURCE_JOB_NOW,
|
||||
)
|
||||
.unwrap();
|
||||
let acceptance = store
|
||||
.accept_backend_job_result(
|
||||
"w",
|
||||
&worker,
|
||||
&BackendJobResultSubmission {
|
||||
job_id: id.into(),
|
||||
attempt_id: reservation.attempt.attempt_id.clone(),
|
||||
input_digest: request.input_digest().unwrap(),
|
||||
result: serde_json::json!({"valid":true}),
|
||||
},
|
||||
RESOURCE_JOB_NOW,
|
||||
)
|
||||
.unwrap();
|
||||
let source = JobNotificationSource {
|
||||
job_id: id.into(),
|
||||
attempt_id: acceptance.attempt.attempt_id.clone(),
|
||||
delivery_id: crate::backend_job::delivery_id(id, &acceptance.attempt.attempt_id),
|
||||
};
|
||||
(acceptance, source)
|
||||
}
|
||||
|
||||
fn job_notice_evidence(
|
||||
store: &SqliteWorkspaceStore,
|
||||
source: &JobNotificationSource,
|
||||
) -> (String, String, bool, Option<String>) {
|
||||
store.with_conn(|conn|Ok(conn.query_row("SELECT disposition,reason,EXISTS(SELECT 1 FROM notification_acknowledgements WHERE workspace_id='w' AND delivery_id=?1),(SELECT code FROM notification_failures WHERE workspace_id='w' AND delivery_id=?1 ORDER BY sequence DESC LIMIT 1) FROM notification_decisions WHERE workspace_id='w' AND delivery_id=?1",[&source.delivery_id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?)))?)).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_notification_routing_common_policy_distinguishes_suppression_failures_and_acceptance()
|
||||
{
|
||||
let (_directory, store) = tracking_store_fixture().await;
|
||||
for (name, target, facts, disposition, reason, failure) in [
|
||||
(
|
||||
"normal",
|
||||
true,
|
||||
JobNotificationFacts::Result("result notice".into()),
|
||||
"deliver",
|
||||
"result",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"no-target",
|
||||
false,
|
||||
JobNotificationFacts::Result("unused".into()),
|
||||
"suppress",
|
||||
"no_recipient",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"clean",
|
||||
true,
|
||||
JobNotificationFacts::NoFindings,
|
||||
"suppress",
|
||||
"no_findings",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"stale",
|
||||
true,
|
||||
JobNotificationFacts::StaleContent,
|
||||
"suppress",
|
||||
"stale_content",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"missing",
|
||||
true,
|
||||
JobNotificationFacts::Failed(NotificationFailureCode::SourceLookupFailed),
|
||||
"defer",
|
||||
"source_unavailable",
|
||||
Some("source_lookup_failed"),
|
||||
),
|
||||
(
|
||||
"invalid",
|
||||
true,
|
||||
JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult),
|
||||
"defer",
|
||||
"source_unavailable",
|
||||
Some("invalid_result"),
|
||||
),
|
||||
(
|
||||
"oversize",
|
||||
true,
|
||||
JobNotificationFacts::Result(
|
||||
"x".repeat(crate::backend_job::MAX_JOB_DELIVERY_BYTES + 1),
|
||||
),
|
||||
"defer",
|
||||
"source_unavailable",
|
||||
Some("content_too_large"),
|
||||
),
|
||||
(
|
||||
"render",
|
||||
true,
|
||||
JobNotificationFacts::Result(String::new()),
|
||||
"defer",
|
||||
"source_unavailable",
|
||||
Some("rendering_failed"),
|
||||
),
|
||||
] {
|
||||
let (acceptance, source) = job_notice_fixture(&store, name, target);
|
||||
assert_eq!(
|
||||
store.route_job_notification("w", &source, &facts).unwrap(),
|
||||
disposition == "deliver"
|
||||
);
|
||||
assert_eq!(
|
||||
job_notice_evidence(&store, &source),
|
||||
(
|
||||
disposition.into(),
|
||||
reason.into(),
|
||||
false,
|
||||
failure.map(String::from)
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
store.get_backend_job("w", name).unwrap().unwrap().result,
|
||||
acceptance.job.result
|
||||
);
|
||||
assert_eq!(
|
||||
store.get_backend_job("w", name).unwrap().unwrap().state,
|
||||
BackendJobState::Completed
|
||||
);
|
||||
if disposition != "deliver" {
|
||||
assert!(
|
||||
store
|
||||
.job_notification_request("w", &source.delivery_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
!store
|
||||
.route_job_notification(
|
||||
"w",
|
||||
&source,
|
||||
&JobNotificationFacts::Result("new rendering".into())
|
||||
)
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
store
|
||||
.list_pending_job_notifications("w", 100)
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_notification_result_commit_reopen_parallel_recovery_and_fixed_target_receipt() {
|
||||
let (directory, store) = tracking_store_fixture().await;
|
||||
let (accepted, source) = job_notice_fixture(&store, "commit-gap", true);
|
||||
// No delivery reservation or routing callback has run. The result transaction
|
||||
// alone leaves one recoverable source, including after reopening SQLite.
|
||||
assert!(
|
||||
store
|
||||
.job_notification_request("w", &source.delivery_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let submission = BackendJobResultSubmission {
|
||||
job_id: source.job_id.clone(),
|
||||
attempt_id: source.attempt_id.clone(),
|
||||
input_digest: accepted.attempt.input_digest.clone(),
|
||||
result: accepted.job.result.clone().unwrap(),
|
||||
};
|
||||
assert!(
|
||||
store
|
||||
.accept_backend_job_result(
|
||||
"w",
|
||||
accepted.attempt.worker.as_ref().unwrap(),
|
||||
&submission,
|
||||
RESOURCE_JOB_NOW
|
||||
)
|
||||
.unwrap()
|
||||
.replayed
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.accept_backend_job_result(
|
||||
"other",
|
||||
accepted.attempt.worker.as_ref().unwrap(),
|
||||
&submission,
|
||||
RESOURCE_JOB_NOW
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let mut stale = submission.clone();
|
||||
stale.attempt_id = "old-attempt".into();
|
||||
assert!(
|
||||
store
|
||||
.accept_backend_job_result(
|
||||
"w",
|
||||
accepted.attempt.worker.as_ref().unwrap(),
|
||||
&stale,
|
||||
RESOURCE_JOB_NOW
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
drop(store);
|
||||
let path = directory.path().join("workspace.db");
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.list_pending_job_notifications("w", 100)
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
|
||||
let threads = (0..2)
|
||||
.map(|i| {
|
||||
let path = path.clone();
|
||||
let source = source.clone();
|
||||
let barrier = barrier.clone();
|
||||
std::thread::spawn(move || {
|
||||
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
|
||||
barrier.wait();
|
||||
store
|
||||
.route_job_notification(
|
||||
"w",
|
||||
&source,
|
||||
&JobNotificationFacts::Result(format!("immutable rendering {i}")),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.job_notification_request("w", &source.delivery_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let results = threads
|
||||
.into_iter()
|
||||
.map(|thread| thread.join().unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(results[0], results[1]);
|
||||
assert!(
|
||||
store
|
||||
.route_job_notification("w", &source, &JobNotificationFacts::StaleContent)
|
||||
.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.job_notification_request("w", &source.delivery_id)
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
results[0]
|
||||
);
|
||||
let target = TicketWorkerNotificationTarget {
|
||||
worker: RuntimeWorkerRef::new("r", "replacement"),
|
||||
authentication_binding_id: "tracking-binding".into(),
|
||||
};
|
||||
assert_eq!(
|
||||
store
|
||||
.authorize_job_notification("w", &source.delivery_id, &target)
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
let mut replacement = target.clone();
|
||||
replacement.worker.worker_id = "another-worker".into();
|
||||
assert!(
|
||||
store
|
||||
.authorize_job_notification("w", &source.delivery_id, &replacement)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
replacement = target.clone();
|
||||
replacement.authentication_binding_id = "replacement-binding".into();
|
||||
assert!(
|
||||
store
|
||||
.authorize_job_notification("w", &source.delivery_id, &replacement)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
store
|
||||
.record_notification_delivery_failure(
|
||||
"w",
|
||||
&NotificationDeliveryKey::WorkerNotification {
|
||||
delivery_id: source.delivery_id.clone(),
|
||||
},
|
||||
NotificationFailureCode::AcceptanceUnknown,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!job_notice_evidence(&store, &source).2);
|
||||
assert_eq!(
|
||||
store
|
||||
.job_notification_request("w", &source.delivery_id)
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
results[0]
|
||||
);
|
||||
store.with_conn(|conn| {conn.execute("UPDATE workspace_runtime_bindings SET revoked_at=?1 WHERE workspace_id='w' AND runtime_id='r'",[RESOURCE_JOB_NOW])?;Ok(())}).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.authorize_job_notification("w", &source.delivery_id, &target)
|
||||
.unwrap(),
|
||||
Some(NotificationFailureCode::BindingRevoked)
|
||||
);
|
||||
// A receipt about the already authorized send remains valid after revocation.
|
||||
store
|
||||
.ack_job_notification("w", &source.delivery_id, &target.worker)
|
||||
.unwrap();
|
||||
store
|
||||
.ack_job_notification("w", &source.delivery_id, &target.worker)
|
||||
.unwrap();
|
||||
assert!(job_notice_evidence(&store, &source).2);
|
||||
assert!(
|
||||
store
|
||||
.list_pending_job_notifications("w", 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.get_backend_job("w", &source.job_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.current_attempt,
|
||||
1
|
||||
);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*) FROM notification_sources WHERE role='backend_job'",
|
||||
[],
|
||||
|r| r.get::<_, i64>(0)
|
||||
)?,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
conn.query_row("SELECT COUNT(*) FROM notification_requests", [], |r| r
|
||||
.get::<_, i64>(0))?,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
conn.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |r| r
|
||||
.get::<_, i64>(
|
||||
0
|
||||
))?,
|
||||
0
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn job_notice_legacy_seed(
|
||||
conn: &Connection,
|
||||
name: &str,
|
||||
state: Option<&str>,
|
||||
category: Option<&str>,
|
||||
target: bool,
|
||||
) -> String {
|
||||
let mut request = resource_job_request(name, name, name);
|
||||
request.purpose = "legacy_notice".into();
|
||||
request.grants = Default::default();
|
||||
request.notification_target =
|
||||
target.then(|| RuntimeWorkerRef::new("old-runtime", "old-worker"));
|
||||
let attempt = attempt_id(name, 1);
|
||||
let id = crate::backend_job::delivery_id(name, &attempt);
|
||||
conn.execute("INSERT INTO backend_jobs(workspace_id,job_id,purpose,input_digest,input_ref,request_json,intent_fingerprint,state,current_attempt,result_json,result_digest,created_at,updated_at,completed_at) VALUES('w',?1,'legacy_notice',?2,'test://input',?3,'fingerprint','completed',1,'{}','digest',?4,?4,?4)",params![name,request.input_digest().unwrap(),serde_json::to_string(&request).unwrap(),RESOURCE_JOB_NOW]).unwrap();
|
||||
conn.execute("INSERT INTO backend_job_attempts(workspace_id,job_id,attempt_id,attempt,input_digest,state,deadline_at,result_json,result_digest,created_at,updated_at,completed_at) VALUES('w',?1,?2,1,?3,'completed',?4,'{}','digest',?4,?4,?4)",params![name,attempt,request.input_digest().unwrap(),RESOURCE_JOB_NOW]).unwrap();
|
||||
if let Some(state) = state {
|
||||
conn.execute("INSERT INTO backend_job_deliveries(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,failure_category,created_at,updated_at,delivered_at) VALUES('w',?1,?2,?3,'old-runtime','old-worker',?4,?5,?6,?6,CASE WHEN ?4='completed' THEN ?6 ELSE NULL END)",params![id,name,attempt,state,category,RESOURCE_JOB_NOW]).unwrap();
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_notification_migration_retains_legacy_acks_suppressions_unknown_bytes_and_commit_gaps() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("workspace.db");
|
||||
let conn = tracking_schema91_fixture(&path);
|
||||
migrate_ticket_tracking_v91_to_v92(&conn).unwrap();
|
||||
migrate_assignment_delivery_v92_to_v93(&conn).unwrap();
|
||||
migrate_ticket_posts_v93_to_v94(&conn).unwrap();
|
||||
migrate_notification_routing_v94_to_v95(&conn).unwrap();
|
||||
let accepted = job_notice_legacy_seed(&conn, "ack", Some("completed"), None, true);
|
||||
let clean = job_notice_legacy_seed(
|
||||
&conn,
|
||||
"clean",
|
||||
Some("completed"),
|
||||
Some("notification_suppressed_no_findings"),
|
||||
true,
|
||||
);
|
||||
let stale = job_notice_legacy_seed(
|
||||
&conn,
|
||||
"stale",
|
||||
Some("completed"),
|
||||
Some("notification_suppressed_stale_ticket_content"),
|
||||
true,
|
||||
);
|
||||
let unknown = job_notice_legacy_seed(
|
||||
&conn,
|
||||
"unknown",
|
||||
Some("unknown"),
|
||||
Some("notification_outcome_unknown"),
|
||||
true,
|
||||
);
|
||||
let ambiguous = job_notice_legacy_seed(
|
||||
&conn,
|
||||
"ambiguous",
|
||||
Some("completed"),
|
||||
Some("unrecognized_legacy_reason"),
|
||||
true,
|
||||
);
|
||||
job_notice_legacy_seed(&conn, "no-target", None, None, false);
|
||||
job_notice_legacy_seed(&conn, "commit-gap", None, None, true);
|
||||
conn.execute("UPDATE backend_job_deliveries SET delivery_id='legacy-stable-request' WHERE delivery_id=?1",[&unknown]).unwrap();
|
||||
let unknown = "legacy-stable-request".to_string();
|
||||
let checker = job_notice_legacy_seed(&conn, "checker-unknown", Some("sending"), None, true);
|
||||
let mut checker_request: BackendJobRequest = serde_json::from_str(
|
||||
&conn
|
||||
.query_row(
|
||||
"SELECT request_json FROM backend_jobs WHERE job_id='checker-unknown'",
|
||||
[],
|
||||
|r| r.get::<_, String>(0),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
checker_request.purpose = crate::ticket_item_checker::PURPOSE.into();
|
||||
checker_request.input = serde_json::json!({"ticket_id":"deleted-ticket","ticket_resource_key":"T-42","content_digest":"original-digest","title":"title","body":"Writer-only scope","edit":{"kind":"create"}});
|
||||
let checker_result = serde_json::json!({"findings":[{"category":"writer_scope","quote":"Writer-only scope","reason":"This is writer scope.","suggestion":"Remove it."}]});
|
||||
conn.execute(
|
||||
"UPDATE backend_jobs SET request_json=?1,result_json=?2 WHERE job_id='checker-unknown'",
|
||||
params![
|
||||
serde_json::to_string(&checker_request).unwrap(),
|
||||
checker_result.to_string()
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"UPDATE backend_job_attempts SET result_json=?1 WHERE job_id='checker-unknown'",
|
||||
[checker_result.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
let before = stored_fixture_rows(&conn, "backend_job_deliveries");
|
||||
// Force migration failure after the source parent rebuild and verify rollback
|
||||
// restores the old parent, rows, constraints, version and connection pragmas.
|
||||
conn.execute_batch("CREATE TABLE notification_job_targets(collision TEXT)")
|
||||
.unwrap();
|
||||
assert!(migrate_job_notifications_v95_to_v96(&conn).is_err());
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 95);
|
||||
assert_eq!(stored_fixture_rows(&conn, "backend_job_deliveries"), before);
|
||||
assert_eq!(
|
||||
conn.query_row("PRAGMA foreign_keys", [], |r| r.get::<_, i64>(0))
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
conn.execute_batch("DROP TABLE notification_job_targets")
|
||||
.unwrap();
|
||||
migrate_job_notifications_v95_to_v96(&conn).unwrap();
|
||||
assert_eq!(stored_fixture_rows(&conn, "backend_job_deliveries"), before);
|
||||
for (id, disposition, reason, runtime_accepted) in [
|
||||
(&accepted, "deliver", "result", true),
|
||||
(&clean, "suppress", "no_findings", false),
|
||||
(&stale, "suppress", "stale_content", false),
|
||||
(&unknown, "deliver", "result", false),
|
||||
(&ambiguous, "defer", "source_unavailable", false),
|
||||
] {
|
||||
let got=conn.query_row("SELECT d.disposition,d.reason,l.runtime_accepted FROM notification_decisions d JOIN notification_job_legacy_receipts l USING(workspace_id,delivery_id) WHERE d.workspace_id='w' AND d.delivery_id=?1",[id],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?,r.get::<_,bool>(2)?))).unwrap();
|
||||
assert_eq!(got, (disposition.into(), reason.into(), runtime_accepted));
|
||||
}
|
||||
assert_eq!(
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*) FROM notification_acknowledgements",
|
||||
[],
|
||||
|r| r.get::<_, i64>(0)
|
||||
)
|
||||
.unwrap(),
|
||||
0,
|
||||
"never fabricate modern ACKs or request bodies for old receipts"
|
||||
);
|
||||
assert_eq!(
|
||||
conn.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |r| r
|
||||
.get::<_, i64>(
|
||||
0
|
||||
))
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
let pinned: String = conn
|
||||
.query_row(
|
||||
"SELECT content FROM notification_requests WHERE delivery_id=?1",
|
||||
[&unknown],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(pinned.contains("Durable result digest: digest."));
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open_for_test(path).unwrap();
|
||||
let pending = store.list_pending_job_notifications("w", 100).unwrap();
|
||||
assert_eq!(pending.len(), 4);
|
||||
let checker_body = store
|
||||
.job_notification_request("w", &checker)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0;
|
||||
assert!(checker_body.contains("Content digest: original-digest"));
|
||||
assert!(checker_body.contains("Quote: \"Writer-only scope\""));
|
||||
assert!(checker_body.contains("not a new user request, approval gate"));
|
||||
assert!(pending.iter().all(|s| s.delivery_id != accepted
|
||||
&& s.delivery_id != clean
|
||||
&& s.delivery_id != stale
|
||||
&& s.delivery_id != ambiguous));
|
||||
assert_eq!(
|
||||
store
|
||||
.job_notification_request("w", &unknown)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.0,
|
||||
pinned
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_notification_source_is_atomic_with_result_and_schema_guards_are_verified() {
|
||||
let (directory, store) = tracking_store_fixture().await;
|
||||
let mut request = resource_job_request("atomic-source", "atomic-source", "atomic-source");
|
||||
request.purpose = "notification_test".into();
|
||||
request.grants = Default::default();
|
||||
let worker = RuntimeWorkerRef::new("r", "replacement");
|
||||
let reservation = store
|
||||
.reserve_backend_job("w", &request, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
store
|
||||
.claim_backend_job_attempt_dispatch(
|
||||
"w",
|
||||
&request.job_id,
|
||||
&reservation.attempt.attempt_id,
|
||||
RESOURCE_JOB_NOW,
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.bind_backend_job_attempt_worker(
|
||||
"w",
|
||||
&request.job_id,
|
||||
&reservation.attempt.attempt_id,
|
||||
&worker,
|
||||
Some("run"),
|
||||
RESOURCE_JOB_NOW,
|
||||
)
|
||||
.unwrap();
|
||||
let submission = BackendJobResultSubmission {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id: reservation.attempt.attempt_id.clone(),
|
||||
input_digest: request.input_digest().unwrap(),
|
||||
result: serde_json::json!({"saved":true}),
|
||||
};
|
||||
store.with_conn(|conn| {conn.execute_batch("CREATE TRIGGER reject_job_source BEFORE INSERT ON notification_sources WHEN NEW.role='backend_job' BEGIN SELECT RAISE(ABORT,'injected source commit failure'); END;")?;Ok(())}).unwrap();
|
||||
assert!(
|
||||
store
|
||||
.accept_backend_job_result("w", &worker, &submission, RESOURCE_JOB_NOW)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.get_backend_job("w", &request.job_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.state,
|
||||
BackendJobState::Pending
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.list_pending_job_notifications("w", 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER reject_job_source")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
store
|
||||
.accept_backend_job_result("w", &worker, &submission, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.list_pending_job_notifications("w", 100)
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER notification_job_targets_no_update")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
assert!(
|
||||
matches!(SqliteWorkspaceStore::open_for_test(directory.path().join("workspace.db")),Err(error) if error.to_string().contains("notification_job_targets_no_update"))
|
||||
);
|
||||
}
|
||||
@@ -9,7 +9,8 @@
|
||||
//! | --- | --- | --- |
|
||||
//! | Assign, Answer, content and post edits, case, tags, tracking, release | Ticket transaction captures facts; `notification_routing` decides both recipients and non-notification reasons | Saved assignment/Worker requests and ACKs; mutation wakeup and periodic rounds |
|
||||
//! | Workspace startup, Orchestrator create/restore/replacement, idle | Wakeup only; never a new request or a second policy | Same saved pending decisions |
|
||||
//! | MR/handoff and Backend Job | Separate adapters to be connected by follow-up work | Not inferred from comments or display state |
|
||||
//! | Backend Job result | Result transaction captures source; checker supplies domain facts; common routing records result/suppression/failure | Fixed Worker request bytes, binding, ACK and legacy receipt; same recovery rounds |
|
||||
//! | MR/handoff | Separate adapter to be connected by follow-up work | Not inferred from comments or display state |
|
||||
//! | Explicit WorkerNotify/WorkerSendInput | Authenticated caller communication | Not automatic business routing |
|
||||
//! | Internal SubWorker result and Web subscription | Parent/child execution and UI synchronization respectively | Remain separate; no Ticket policy in Runtime |
|
||||
//!
|
||||
@@ -121,10 +122,17 @@ pub(super) fn record_notification_failure(
|
||||
pub(super) fn notification_target(
|
||||
api: &WorkspaceApi,
|
||||
input: &TicketWorkerNotification,
|
||||
) -> std::result::Result<TicketWorkerNotificationTarget, NotificationFailureCode> {
|
||||
fixed_worker_target(api, &input.worker)
|
||||
}
|
||||
|
||||
fn fixed_worker_target(
|
||||
api: &WorkspaceApi,
|
||||
recipient: &RuntimeWorkerRef,
|
||||
) -> std::result::Result<TicketWorkerNotificationTarget, NotificationFailureCode> {
|
||||
let worker = api
|
||||
.runtime
|
||||
.worker(&input.worker)
|
||||
.worker(recipient)
|
||||
.map_err(|_| NotificationFailureCode::TargetUnavailable)?;
|
||||
if worker.workspace.workspace_id.as_deref() != Some(&api.config.workspace_id) {
|
||||
return Err(NotificationFailureCode::WorkspaceMismatch);
|
||||
@@ -134,14 +142,14 @@ pub(super) fn notification_target(
|
||||
}
|
||||
let binding = api
|
||||
.config_store
|
||||
.get_workspace_runtime_binding(&api.config.workspace_id, &input.worker.runtime_id)
|
||||
.get_workspace_runtime_binding(&api.config.workspace_id, &recipient.runtime_id)
|
||||
.map_err(|_| NotificationFailureCode::SourceLookupFailed)?
|
||||
.ok_or(NotificationFailureCode::TargetUnavailable)?;
|
||||
if binding.revoked_at.is_some() {
|
||||
return Err(NotificationFailureCode::BindingRevoked);
|
||||
}
|
||||
Ok(TicketWorkerNotificationTarget {
|
||||
worker: input.worker.clone(),
|
||||
worker: recipient.clone(),
|
||||
authentication_binding_id: binding.binding_id,
|
||||
})
|
||||
}
|
||||
@@ -264,6 +272,7 @@ pub(super) fn recover_ticket_notifications(api: &WorkspaceApi) {
|
||||
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
|
||||
return;
|
||||
};
|
||||
recover_job_notifications(api);
|
||||
dispatch_ticket_worker_notifications(api);
|
||||
if let Some(orchestrator) = find_workspace_orchestrator(api) {
|
||||
dispatch_orchestrator_assignment_attention(api, &orchestrator.worker);
|
||||
@@ -339,3 +348,195 @@ mod tests {
|
||||
assert!(lifecycle.begin().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
/// Recover notification sources, not Job execution. This same bounded round runs
|
||||
/// on startup and periodically; no Worker restoration or assignment is implicit.
|
||||
pub(super) fn recover_job_notifications(api: &WorkspaceApi) {
|
||||
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
|
||||
return;
|
||||
};
|
||||
let sources = match api
|
||||
.store
|
||||
.list_pending_job_notifications(&api.config.workspace_id, 20)
|
||||
{
|
||||
Ok(sources) => sources,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error,"Job notification scan failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for source in sources {
|
||||
if api.notification_delivery_lifecycle.stopped() {
|
||||
break;
|
||||
}
|
||||
let acceptance = (|| -> Result<BackendJobResultAcceptance> {
|
||||
let job = api
|
||||
.store
|
||||
.get_backend_job(&api.config.workspace_id, &source.job_id)?
|
||||
.ok_or_else(|| Error::Store("Job notification result disappeared".into()))?;
|
||||
let attempt = api
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
&api.config.workspace_id,
|
||||
&source.job_id,
|
||||
&source.attempt_id,
|
||||
)?
|
||||
.ok_or_else(|| Error::Store("Job notification attempt disappeared".into()))?;
|
||||
Ok(BackendJobResultAcceptance {
|
||||
job,
|
||||
attempt,
|
||||
replayed: true,
|
||||
})
|
||||
})();
|
||||
match acceptance {
|
||||
Ok(acceptance) => deliver_job_result(api, &acceptance),
|
||||
Err(error) => tracing::warn!(%error,"Job notification source lookup failed"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn job_result_facts(
|
||||
api: &WorkspaceApi,
|
||||
acceptance: &BackendJobResultAcceptance,
|
||||
) -> crate::store::JobNotificationFacts {
|
||||
use crate::store::JobNotificationFacts as F;
|
||||
let content = if acceptance.job.request.purpose == ticket_item_checker::PURPOSE {
|
||||
let input = match ticket_item_checker::parse_input(&acceptance.job.request.input) {
|
||||
Ok(input) => input,
|
||||
Err(_) => return F::Failed(NotificationFailureCode::InvalidResult),
|
||||
};
|
||||
let ticket = match browser_ticket_backend(api)
|
||||
.and_then(|backend| backend.show(input.ticket_id.into()).map_err(Error::from))
|
||||
{
|
||||
Ok(ticket) => ticket,
|
||||
Err(Error::Ticket(ticket::TicketError::NotFound(_))) => {
|
||||
return F::Failed(NotificationFailureCode::SourceNotFound);
|
||||
}
|
||||
Err(_) => return F::Failed(NotificationFailureCode::SourceLookupFailed),
|
||||
};
|
||||
match ticket_item_checker::result_facts(acceptance, &ticket) {
|
||||
Ok(ticket_item_checker::TicketItemCheckFacts::Findings(content)) => content,
|
||||
Ok(ticket_item_checker::TicketItemCheckFacts::NoFindings) => {
|
||||
return F::NoFindings;
|
||||
}
|
||||
Ok(ticket_item_checker::TicketItemCheckFacts::Stale) => return F::StaleContent,
|
||||
Err(_) => return F::Failed(NotificationFailureCode::InvalidResult),
|
||||
}
|
||||
} else {
|
||||
format!(
|
||||
"Backend Job `{}` completed attempt `{}` for `{}`. Durable result digest: {}.",
|
||||
acceptance.job.request.job_id,
|
||||
acceptance.attempt.attempt_id,
|
||||
acceptance.job.request.purpose,
|
||||
acceptance
|
||||
.job
|
||||
.result_digest
|
||||
.as_deref()
|
||||
.unwrap_or("unavailable")
|
||||
)
|
||||
};
|
||||
if content.len() > MAX_JOB_DELIVERY_BYTES {
|
||||
F::Failed(NotificationFailureCode::ContentTooLarge)
|
||||
} else {
|
||||
F::Result(content)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) {
|
||||
use crate::store::JobNotificationFacts;
|
||||
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
|
||||
return;
|
||||
};
|
||||
let source = match api.store.job_notification_source(
|
||||
&api.config.workspace_id,
|
||||
&acceptance.job.request.job_id,
|
||||
&acceptance.attempt.attempt_id,
|
||||
) {
|
||||
Ok(source) => source,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error,"Job notification source lookup failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let workspace = &api.config.workspace_id;
|
||||
let key = NotificationDeliveryKey::WorkerNotification {
|
||||
delivery_id: source.delivery_id.clone(),
|
||||
};
|
||||
// The prepared body wins over all subsequent rendering and content changes.
|
||||
let request = match api
|
||||
.store
|
||||
.job_notification_request(workspace, &source.delivery_id)
|
||||
{
|
||||
Ok(request) => request,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error,"Job notification request lookup failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let facts = match &request {
|
||||
Some((content, _)) => JobNotificationFacts::Result(content.clone()),
|
||||
None => job_result_facts(api, acceptance),
|
||||
};
|
||||
match api.store.route_job_notification(workspace, &source, &facts) {
|
||||
Ok(true) => {}
|
||||
Ok(false) => return,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error,"Job notification routing failed; committed source retained");
|
||||
return;
|
||||
}
|
||||
}
|
||||
let request = match api
|
||||
.store
|
||||
.job_notification_request(workspace, &source.delivery_id)
|
||||
{
|
||||
Ok(Some(request)) => request,
|
||||
_ => {
|
||||
record_notification_failure(api, &key, NotificationFailureCode::PreparationFailed);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let (content, target) = request;
|
||||
if acceptance.job.request.notification_target.as_ref() != Some(&target) {
|
||||
record_notification_failure(api, &key, NotificationFailureCode::PreparationFailed);
|
||||
return;
|
||||
}
|
||||
let authorized = match fixed_worker_target(api, &target) {
|
||||
Ok(target) => target,
|
||||
Err(code) => {
|
||||
record_notification_failure(api, &key, code);
|
||||
return;
|
||||
}
|
||||
};
|
||||
match api
|
||||
.store
|
||||
.authorize_job_notification(workspace, &source.delivery_id, &authorized)
|
||||
{
|
||||
Ok(None) => {}
|
||||
Ok(Some(code)) => {
|
||||
record_notification_failure(api, &key, code);
|
||||
return;
|
||||
}
|
||||
Err(_) => {
|
||||
record_notification_failure(api, &key, NotificationFailureCode::SourceLookupFailed);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let result = api.runtime.send_input(
|
||||
&target,
|
||||
WorkerInputRequest {
|
||||
kind: WorkerInputKind::Notify,
|
||||
content,
|
||||
submission_request_id: Some(source.delivery_id.clone()),
|
||||
segments: None,
|
||||
},
|
||||
);
|
||||
if let Some(failure) = notification_failure(&result) {
|
||||
record_notification_failure(api, &key, failure);
|
||||
} else if api
|
||||
.store
|
||||
.ack_job_notification(workspace, &source.delivery_id, &target)
|
||||
.is_err()
|
||||
{
|
||||
record_notification_failure(api, &key, NotificationFailureCode::AcknowledgementFailed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
//! Runtime acceptance, handling and execution authority remain independent. Explicit user
|
||||
//! communication, internal child notifications and Web subscriptions are not business routes.
|
||||
use super::*;
|
||||
#[path = "job_notification_routing.rs"]
|
||||
pub(super) mod jobs;
|
||||
pub use jobs::{JobNotificationFacts, JobNotificationSource};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -39,6 +42,7 @@ impl NotificationEventKind {
|
||||
"tags" | "tags_set" | "tags_changed" => Self::Tags,
|
||||
"release" => Self::Release,
|
||||
"assignment_changed" => Self::AssignmentChanged,
|
||||
"backend_job_result" => Self::Result,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
@@ -51,6 +55,9 @@ pub enum NotificationReason {
|
||||
Attention,
|
||||
Result,
|
||||
NoRecipient,
|
||||
NoFindings,
|
||||
StaleContent,
|
||||
SourceUnavailable,
|
||||
UnsupportedRecipient,
|
||||
HistoricalOnly,
|
||||
CaseClosed,
|
||||
@@ -66,7 +73,10 @@ pub enum NotificationReason {
|
||||
pub enum NotificationFailureCode {
|
||||
ConfigurationUnavailable,
|
||||
RenderingFailed,
|
||||
ContentTooLarge,
|
||||
InvalidResult,
|
||||
SourceLookupFailed,
|
||||
SourceNotFound,
|
||||
TargetUnavailable,
|
||||
TargetStopped,
|
||||
WorkspaceMismatch,
|
||||
@@ -371,7 +381,7 @@ fn read_source(
|
||||
Ok((
|
||||
row.get(0)?,
|
||||
row.get(1)?,
|
||||
row.get(2)?,
|
||||
row.get::<_, Option<String>>(2)?.unwrap_or_default(),
|
||||
row.get(3)?,
|
||||
row.get(4)?,
|
||||
row.get(5)?,
|
||||
@@ -469,7 +479,7 @@ fn decision_for_source(
|
||||
}
|
||||
|
||||
pub(super) fn route_pending(conn: &Connection, workspace: &str) -> Result<()> {
|
||||
let mut stmt = conn.prepare("SELECT s.* FROM notification_sources s LEFT JOIN notification_decisions d ON d.source_sequence=s.sequence WHERE s.workspace_id=?1 AND d.source_sequence IS NULL ORDER BY s.sequence LIMIT 500")?;
|
||||
let mut stmt = conn.prepare("SELECT s.* FROM notification_sources s LEFT JOIN notification_decisions d ON d.source_sequence=s.sequence WHERE s.workspace_id=?1 AND s.role!='backend_job' AND d.source_sequence IS NULL ORDER BY s.sequence LIMIT 500")?;
|
||||
let mut rows = stmt.query([workspace])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let source = source_from_row(row)?;
|
||||
|
||||
@@ -843,6 +843,7 @@ async fn notification_routing_fresh_and_upgraded_schema_and_guards_match() {
|
||||
migrate_assignment_delivery_v92_to_v93(&conn).unwrap();
|
||||
migrate_ticket_posts_v93_to_v94(&conn).unwrap();
|
||||
migrate_notification_routing_v94_to_v95(&conn).unwrap();
|
||||
migrate_job_notifications_v95_to_v96(&conn).unwrap();
|
||||
let snapshot = |conn: &Connection| -> Vec<(String, String)> {
|
||||
conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::<rusqlite::Result<_>>().unwrap()
|
||||
};
|
||||
|
||||
@@ -181,7 +181,7 @@ use crate::authority::{
|
||||
};
|
||||
use crate::backend_job::{
|
||||
ABSOLUTE_MAX_CONCURRENT_JOBS, BackendJobAttemptRecord, BackendJobAttemptState,
|
||||
BackendJobDeliveryState, BackendJobRequest, BackendJobReservation, BackendJobResultAcceptance,
|
||||
BackendJobRequest, BackendJobReservation, BackendJobResultAcceptance,
|
||||
BackendJobResultSubmission, BackendJobState, BackendJobWorkerCleanupState,
|
||||
MAX_JOB_DELIVERY_BYTES, allocation_key,
|
||||
};
|
||||
@@ -3973,38 +3973,7 @@ impl WorkspaceApi {
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let deliveries = api
|
||||
.store
|
||||
.recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?;
|
||||
for delivery in deliveries {
|
||||
let Some(job) = api
|
||||
.store
|
||||
.get_backend_job(&api.config.workspace_id, &delivery.job_id)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some(attempt) = api.store.get_backend_job_attempt(
|
||||
&api.config.workspace_id,
|
||||
&delivery.job_id,
|
||||
&delivery.attempt_id,
|
||||
)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if job.state == BackendJobState::Completed
|
||||
&& attempt.state == BackendJobAttemptState::Completed
|
||||
{
|
||||
api.deliver_backend_job_result(
|
||||
&BackendJobResultAcceptance {
|
||||
job,
|
||||
attempt,
|
||||
replayed: true,
|
||||
},
|
||||
&delivery.target,
|
||||
);
|
||||
}
|
||||
}
|
||||
notification_delivery::recover_job_notifications(&api);
|
||||
api.recover_backend_jobs()?;
|
||||
if let Some(orchestrator) = find_online_workspace_orchestrator(&api) {
|
||||
dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker);
|
||||
@@ -4892,175 +4861,19 @@ impl WorkspaceApi {
|
||||
&verified_submission,
|
||||
&accepted_at,
|
||||
)?;
|
||||
if let Some(target) = acceptance.job.request.notification_target.as_ref() {
|
||||
self.deliver_backend_job_result(&acceptance, target);
|
||||
}
|
||||
notification_delivery::deliver_job_result(self, &acceptance);
|
||||
let _ = self.schedule_backend_job_worker_cleanup(&acceptance.attempt);
|
||||
self.drain_reserved_backend_jobs();
|
||||
Ok(acceptance)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn deliver_backend_job_result(
|
||||
&self,
|
||||
acceptance: &BackendJobResultAcceptance,
|
||||
target: &RuntimeWorkerRef,
|
||||
_target: &RuntimeWorkerRef,
|
||||
) {
|
||||
let delivery_id = crate::backend_job::delivery_id(
|
||||
&acceptance.job.request.job_id,
|
||||
&acceptance.attempt.attempt_id,
|
||||
);
|
||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let Ok((delivery, claimed)) = self.store.reserve_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
&acceptance.job.request.job_id,
|
||||
target,
|
||||
&now,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
if !claimed || delivery.state != BackendJobDeliveryState::Sending {
|
||||
return;
|
||||
}
|
||||
|
||||
let content = if acceptance.job.request.purpose == ticket_item_checker::PURPOSE {
|
||||
let checker_input =
|
||||
match ticket_item_checker::parse_input(&acceptance.job.request.input) {
|
||||
Ok(input) => input,
|
||||
Err(error) => {
|
||||
let _ = self.store.finish_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
BackendJobDeliveryState::Failed,
|
||||
Some("invalid_checker_input"),
|
||||
Some(&error.to_string()),
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let current_ticket = match browser_ticket_backend(self).and_then(|backend| {
|
||||
backend
|
||||
.show(checker_input.ticket_id.clone().into())
|
||||
.map_err(Error::from)
|
||||
}) {
|
||||
Ok(ticket) => ticket,
|
||||
Err(error) => {
|
||||
let _ = self.store.finish_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
BackendJobDeliveryState::Failed,
|
||||
Some("current_ticket_content_unavailable"),
|
||||
Some(&error.to_string()),
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
match ticket_item_checker::notification(acceptance, ¤t_ticket) {
|
||||
Ok(ticket_item_checker::TicketItemCheckNotification::Deliver(content)) => content,
|
||||
Ok(ticket_item_checker::TicketItemCheckNotification::NoFindings) => {
|
||||
let _ = self.store.finish_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
BackendJobDeliveryState::Completed,
|
||||
Some("notification_suppressed_no_findings"),
|
||||
None,
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
);
|
||||
return;
|
||||
}
|
||||
Ok(ticket_item_checker::TicketItemCheckNotification::Stale) => {
|
||||
let detail = format!(
|
||||
"checked Ticket content digest {} is no longer current ({})",
|
||||
checker_input.content_digest,
|
||||
ticket_item_checker::content_digest(¤t_ticket)
|
||||
);
|
||||
let _ = self.store.finish_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
BackendJobDeliveryState::Completed,
|
||||
Some("notification_suppressed_stale_ticket_content"),
|
||||
Some(&detail),
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = self.store.finish_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
BackendJobDeliveryState::Failed,
|
||||
Some("invalid_checker_result"),
|
||||
Some(&error.to_string()),
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
format!(
|
||||
"Backend Job `{}` completed attempt `{}` for `{}`. Durable result digest: {}.",
|
||||
acceptance.job.request.job_id,
|
||||
acceptance.attempt.attempt_id,
|
||||
acceptance.job.request.purpose,
|
||||
acceptance
|
||||
.job
|
||||
.result_digest
|
||||
.as_deref()
|
||||
.unwrap_or("unavailable")
|
||||
)
|
||||
};
|
||||
if content.len() > MAX_JOB_DELIVERY_BYTES {
|
||||
let _ = self.store.finish_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
BackendJobDeliveryState::Failed,
|
||||
Some("notification_too_large"),
|
||||
Some("Backend Job notification content exceeded its configured byte limit"),
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let delivery_result = self.runtime.send_input(
|
||||
target,
|
||||
WorkerInputRequest {
|
||||
kind: WorkerInputKind::Notify,
|
||||
content,
|
||||
submission_request_id: Some(delivery.delivery_id.clone()),
|
||||
segments: None,
|
||||
},
|
||||
);
|
||||
let (state, category, detail): (_, Option<String>, Option<String>) = match delivery_result {
|
||||
Ok(result) if result.state == InternalWorkerOperationState::Accepted => {
|
||||
(BackendJobDeliveryState::Completed, None, None)
|
||||
}
|
||||
Ok(result) if result.disposition == WorkerInputDisposition::Unknown => (
|
||||
BackendJobDeliveryState::Unknown,
|
||||
Some("notification_outcome_unknown".to_string()),
|
||||
result.diagnostics.first().map(|item| item.message.clone()),
|
||||
),
|
||||
Ok(result) => (
|
||||
BackendJobDeliveryState::Failed,
|
||||
Some("notification_rejected".to_string()),
|
||||
result.diagnostics.first().map(|item| item.message.clone()),
|
||||
),
|
||||
Err(error) => (
|
||||
BackendJobDeliveryState::Unknown,
|
||||
Some("notification_outcome_unknown".to_string()),
|
||||
Some(error.message()),
|
||||
),
|
||||
};
|
||||
// Delivery is intentionally separate from result authority. A delivery
|
||||
// bookkeeping failure must never roll back or re-run completed work.
|
||||
let _ = self.store.finish_backend_job_delivery(
|
||||
&self.config.workspace_id,
|
||||
&delivery_id,
|
||||
state,
|
||||
category.as_deref(),
|
||||
detail.as_deref(),
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
);
|
||||
notification_delivery::deliver_job_result(self, acceptance);
|
||||
}
|
||||
|
||||
fn register_workspace_runtime_binding(&self, binding: WorkspaceRuntimeBinding) -> Result<()> {
|
||||
@@ -46923,7 +46736,16 @@ mod tests {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn job_notification_evidence(
|
||||
api: &WorkspaceApi,
|
||||
id: &str,
|
||||
) -> (String, String, bool, Option<String>) {
|
||||
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
||||
conn.query_row("SELECT d.disposition,d.reason,EXISTS(SELECT 1 FROM notification_acknowledgements a WHERE a.workspace_id=d.workspace_id AND a.delivery_id=d.delivery_id),(SELECT code FROM notification_failures f WHERE f.workspace_id=d.workspace_id AND f.delivery_id=d.delivery_id ORDER BY sequence DESC LIMIT 1) FROM notification_decisions d WHERE d.workspace_id=?1 AND d.delivery_id=?2",rusqlite::params![TEST_WORKSPACE_ID,id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?))).unwrap()
|
||||
}
|
||||
|
||||
fn spawn_ticket_check_source(api: &WorkspaceApi, name: &str) -> RuntimeWorkerRef {
|
||||
seed_test_delivery_binding(api);
|
||||
api.spawn_workspace_worker(
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
WorkerSpawnRequest {
|
||||
@@ -47601,21 +47423,9 @@ mod tests {
|
||||
);
|
||||
let stale_delivery_id =
|
||||
crate::backend_job::delivery_id(&stale_job.request.job_id, &stale_attempt.attempt_id);
|
||||
let (stale_delivery, claimed) = api
|
||||
.store
|
||||
.reserve_backend_job_delivery(
|
||||
TEST_WORKSPACE_ID,
|
||||
&stale_delivery_id,
|
||||
&stale_job.request.job_id,
|
||||
&source,
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!claimed);
|
||||
assert_eq!(stale_delivery.state, BackendJobDeliveryState::Completed);
|
||||
assert_eq!(
|
||||
stale_delivery.failure_category.as_deref(),
|
||||
Some("notification_suppressed_stale_ticket_content")
|
||||
job_notification_evidence(&api, &stale_delivery_id),
|
||||
("suppress".into(), "stale_content".into(), false, None)
|
||||
);
|
||||
|
||||
api.accept_backend_job_result(
|
||||
@@ -47633,20 +47443,9 @@ mod tests {
|
||||
¤t_job.request.job_id,
|
||||
¤t_attempt.attempt_id,
|
||||
);
|
||||
let (clean_delivery, claimed) = api
|
||||
.store
|
||||
.reserve_backend_job_delivery(
|
||||
TEST_WORKSPACE_ID,
|
||||
&clean_delivery_id,
|
||||
¤t_job.request.job_id,
|
||||
&source,
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!claimed);
|
||||
assert_eq!(
|
||||
clean_delivery.failure_category.as_deref(),
|
||||
Some("notification_suppressed_no_findings")
|
||||
job_notification_evidence(&api, &clean_delivery_id),
|
||||
("suppress".into(), "no_findings".into(), false, None)
|
||||
);
|
||||
|
||||
let Json(_) = execute_worker_ticket_test_operation(
|
||||
@@ -47962,21 +47761,15 @@ mod tests {
|
||||
);
|
||||
let delivery_id =
|
||||
crate::backend_job::delivery_id(&request.job_id, &dispatched.attempt.attempt_id);
|
||||
let (delivery, claimed) = api
|
||||
.store
|
||||
.reserve_backend_job_delivery(
|
||||
TEST_WORKSPACE_ID,
|
||||
&delivery_id,
|
||||
&request.job_id,
|
||||
¬ification_target,
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!claimed);
|
||||
assert_eq!(delivery.state, BackendJobDeliveryState::Unknown);
|
||||
assert_eq!(
|
||||
delivery.failure_category.as_deref(),
|
||||
Some("notification_outcome_unknown")
|
||||
let evidence = job_notification_evidence(&api, &delivery_id);
|
||||
assert_eq!(evidence.0, "deliver");
|
||||
assert!(!evidence.2);
|
||||
assert_eq!(evidence.3.as_deref(), Some("acceptance_unknown"));
|
||||
assert!(
|
||||
api.store
|
||||
.job_notification_request(TEST_WORKSPACE_ID, &delivery_id)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -48444,6 +48237,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
seed_test_delivery_binding(&api);
|
||||
let target = api
|
||||
.spawn_workspace_worker(
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
@@ -48535,18 +48329,15 @@ mod tests {
|
||||
.any(|request_id| request_id.as_deref() == Some(delivery_id.as_str())),
|
||||
"recovery must replay the durable delivery id as the tracked notification id: {request_ids:?}"
|
||||
);
|
||||
let (delivery, claimed) = api
|
||||
.store
|
||||
.reserve_backend_job_delivery(
|
||||
TEST_WORKSPACE_ID,
|
||||
&delivery_id,
|
||||
&request.job_id,
|
||||
&target,
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!claimed);
|
||||
assert_eq!(delivery.state, BackendJobDeliveryState::Completed);
|
||||
assert_eq!(job_notification_evidence(&api, &delivery_id).2, true);
|
||||
assert!(
|
||||
api.store
|
||||
.list_pending_job_notifications(TEST_WORKSPACE_ID, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
notification_delivery::recover_job_notifications(&api);
|
||||
assert!(execution.take_input_request_ids().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1794,3 +1794,5 @@ async fn catalog_shutdown_stops_notification_hook_and_fences_later_dispatch() {
|
||||
);
|
||||
delivery_stop(&api, &orchestrator);
|
||||
}
|
||||
|
||||
include!("server_job_notification_tests.rs");
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
// Uses the real Controller/session fixtures above; only model output and the
|
||||
// deliberately lost Runtime response are substituted.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn job_notification_real_worker_ack_loss_reopen_and_parallel_recovery_commits_once() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution, _) = delivery_real_fixture(dir.path(), false).await;
|
||||
let target = spawn_ticket_check_source(&api, "fixed-job-result-target");
|
||||
let request = BackendJobRequest {
|
||||
job_id: "real-job-notification".into(),
|
||||
purpose: "result_notification_fixture".into(),
|
||||
input_ref: "test://job/result".into(),
|
||||
input: json!({"fixed":true}),
|
||||
instruction: "Return a result.".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: Some(target.clone()),
|
||||
limits: Default::default(),
|
||||
};
|
||||
let dispatched = api.dispatch_backend_job(&request).unwrap();
|
||||
let worker = dispatched.attempt.worker.as_ref().unwrap();
|
||||
let submission = BackendJobResultSubmission {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id: dispatched.attempt.attempt_id.clone(),
|
||||
input_digest: request.input_digest().unwrap(),
|
||||
result: json!({"result":"saved"}),
|
||||
};
|
||||
let accepted = api
|
||||
.store
|
||||
.accept_backend_job_result(TEST_WORKSPACE_ID, worker, &submission, TEST_CREATED_AT)
|
||||
.unwrap();
|
||||
let id = crate::backend_job::delivery_id(&request.job_id, &accepted.attempt.attempt_id);
|
||||
// Result commit without its wakeup. No old reserve record is necessary.
|
||||
assert!(
|
||||
api.store
|
||||
.job_notification_request(TEST_WORKSPACE_ID, &id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
execution
|
||||
.lose_responses
|
||||
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
notification_delivery::recover_job_notifications(&api);
|
||||
delivery_wait_idle(&api, &target, 1).await;
|
||||
let pinned = api
|
||||
.store
|
||||
.job_notification_request(TEST_WORKSPACE_ID, &id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let evidence = job_notification_evidence(&api, &id);
|
||||
assert!(!evidence.2);
|
||||
assert_eq!(evidence.3.as_deref(), Some("acceptance_unknown"));
|
||||
assert_eq!(delivery_receipts(&api, &target).len(), 1);
|
||||
// Reopen only Backend persistence, retaining the real Runtime receipt/session.
|
||||
let mut reopened = api.clone();
|
||||
reopened.store =
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap());
|
||||
execution
|
||||
.lose_responses
|
||||
.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
let barrier = Arc::new(std::sync::Barrier::new(4));
|
||||
std::thread::scope(|scope| {
|
||||
let joins = (0..4)
|
||||
.map(|_| {
|
||||
let api = reopened.clone();
|
||||
let barrier = barrier.clone();
|
||||
scope.spawn(move || {
|
||||
barrier.wait();
|
||||
notification_delivery::recover_job_notifications(&api);
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for join in joins {
|
||||
join.join().unwrap();
|
||||
}
|
||||
});
|
||||
delivery_wait_idle(&reopened, &target, 1).await;
|
||||
assert!(job_notification_evidence(&reopened, &id).2);
|
||||
assert_eq!(delivery_receipts(&reopened, &target).len(), 1);
|
||||
assert_eq!(delivery_commits(&reopened, &target), vec![pinned.0.clone()]);
|
||||
let attempts = execution.attempts.lock().unwrap();
|
||||
let retries = attempts
|
||||
.iter()
|
||||
.filter(|attempt| attempt.0 == id)
|
||||
.collect::<Vec<_>>();
|
||||
assert!(retries.len() >= 2);
|
||||
assert!(retries.iter().all(|attempt| attempt.1 == pinned.0));
|
||||
drop(attempts);
|
||||
assert!(
|
||||
reopened
|
||||
.store
|
||||
.accept_backend_job_result(TEST_WORKSPACE_ID, worker, &submission, TEST_CREATED_AT)
|
||||
.unwrap()
|
||||
.replayed
|
||||
);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &request.job_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.current_attempt,
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.store
|
||||
.list_pending_job_notifications(TEST_WORKSPACE_ID, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
delivery_stop(&reopened, &target);
|
||||
delivery_stop(&reopened, worker);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn job_notification_stopped_fixed_recipient_is_not_replaced_or_restored() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution, _) = delivery_real_fixture(dir.path(), false).await;
|
||||
let target = spawn_ticket_check_source(&api, "stopped-job-target");
|
||||
let request = BackendJobRequest {
|
||||
job_id: "stopped-fixed-job".into(),
|
||||
purpose: "result_notification_fixture".into(),
|
||||
input_ref: "test://job/stopped".into(),
|
||||
input: json!({"fixed":true}),
|
||||
instruction: "Return a result.".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: Some(target.clone()),
|
||||
limits: Default::default(),
|
||||
};
|
||||
let dispatched = api.dispatch_backend_job(&request).unwrap();
|
||||
let worker = dispatched.attempt.worker.as_ref().unwrap();
|
||||
let accepted = api
|
||||
.store
|
||||
.accept_backend_job_result(
|
||||
TEST_WORKSPACE_ID,
|
||||
worker,
|
||||
&BackendJobResultSubmission {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id: dispatched.attempt.attempt_id.clone(),
|
||||
input_digest: request.input_digest().unwrap(),
|
||||
result: json!({"saved":true}),
|
||||
},
|
||||
TEST_CREATED_AT,
|
||||
)
|
||||
.unwrap();
|
||||
let id = crate::backend_job::delivery_id(&request.job_id, &accepted.attempt.attempt_id);
|
||||
delivery_stop(&api, &target);
|
||||
let replacement = spawn_ticket_check_source(&api, "different-job-target");
|
||||
notification_delivery::recover_job_notifications(&api);
|
||||
let evidence = job_notification_evidence(&api, &id);
|
||||
assert!(!evidence.2);
|
||||
assert_eq!(evidence.3.as_deref(), Some("target_stopped"));
|
||||
assert!(
|
||||
!execution
|
||||
.attempts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|attempt| attempt.0 == id)
|
||||
);
|
||||
assert_eq!(
|
||||
api.store
|
||||
.job_notification_request(TEST_WORKSPACE_ID, &id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.1,
|
||||
target
|
||||
);
|
||||
assert_eq!(api.runtime.worker(&target).unwrap().state, "stopped");
|
||||
assert_eq!(
|
||||
api.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &request.job_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.result,
|
||||
Some(json!({"saved":true}))
|
||||
);
|
||||
delivery_stop(&api, &replacement);
|
||||
delivery_stop(&api, worker);
|
||||
}
|
||||
@@ -67,8 +67,9 @@ mod notification_diagnostics;
|
||||
#[path = "notification_routing.rs"]
|
||||
mod notification_routing;
|
||||
pub use notification_routing::{
|
||||
NotificationDeliveryKey, NotificationEventKind, NotificationFailureCode, NotificationPurpose,
|
||||
NotificationReason, TicketWorkerNotification, TicketWorkerNotificationTarget,
|
||||
JobNotificationFacts, JobNotificationSource, NotificationDeliveryKey, NotificationEventKind,
|
||||
NotificationFailureCode, NotificationPurpose, NotificationReason, TicketWorkerNotification,
|
||||
TicketWorkerNotificationTarget,
|
||||
};
|
||||
#[path = "assignment_tracking.rs"]
|
||||
mod assignment_tracking;
|
||||
@@ -81,7 +82,7 @@ pub use ticket::{
|
||||
};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 95;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 96;
|
||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
@@ -364,6 +365,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: "Common notification routing and durable Worker delivery",
|
||||
apply: migrate_notification_routing_v94_to_v95,
|
||||
},
|
||||
Migration {
|
||||
version: 96,
|
||||
name: "Backend Job common notification routing",
|
||||
apply: migrate_job_notifications_v95_to_v96,
|
||||
},
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -1704,6 +1710,40 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
||||
failure_detail: Option<&str>,
|
||||
now: &str,
|
||||
) -> Result<BackendJobAttemptRecord>;
|
||||
fn job_notification_source(
|
||||
&self,
|
||||
workspace: &str,
|
||||
job: &str,
|
||||
attempt: &str,
|
||||
) -> Result<JobNotificationSource>;
|
||||
fn list_pending_job_notifications(
|
||||
&self,
|
||||
workspace: &str,
|
||||
limit: usize,
|
||||
) -> Result<Vec<JobNotificationSource>>;
|
||||
fn route_job_notification(
|
||||
&self,
|
||||
workspace: &str,
|
||||
source: &JobNotificationSource,
|
||||
facts: &JobNotificationFacts,
|
||||
) -> Result<bool>;
|
||||
fn job_notification_request(
|
||||
&self,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
) -> Result<Option<(String, RuntimeWorkerRef)>>;
|
||||
fn authorize_job_notification(
|
||||
&self,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
target: &TicketWorkerNotificationTarget,
|
||||
) -> Result<Option<NotificationFailureCode>>;
|
||||
fn ack_job_notification(
|
||||
&self,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
target: &RuntimeWorkerRef,
|
||||
) -> Result<()>;
|
||||
fn reserve_backend_job_delivery(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -6973,6 +7013,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
.ok_or_else(|| Error::Store("accepted Backend Job disappeared".to_string()))?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
|
||||
.ok_or_else(|| Error::Store("accepted Backend Job attempt disappeared".to_string()))?;
|
||||
notification_routing::jobs::capture(&tx,workspace_id,&submission.job_id,&submission.attempt_id)?;
|
||||
tx.commit()?;
|
||||
Ok(BackendJobResultAcceptance { job, attempt, replayed: false })
|
||||
})
|
||||
@@ -7187,6 +7228,53 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
})
|
||||
}
|
||||
|
||||
fn job_notification_source(
|
||||
&self,
|
||||
workspace: &str,
|
||||
job: &str,
|
||||
attempt: &str,
|
||||
) -> Result<JobNotificationSource> {
|
||||
notification_routing::jobs::source(self, workspace, job, attempt)
|
||||
}
|
||||
fn list_pending_job_notifications(
|
||||
&self,
|
||||
workspace: &str,
|
||||
limit: usize,
|
||||
) -> Result<Vec<JobNotificationSource>> {
|
||||
notification_routing::jobs::pending(self, workspace, limit)
|
||||
}
|
||||
fn route_job_notification(
|
||||
&self,
|
||||
workspace: &str,
|
||||
source: &JobNotificationSource,
|
||||
facts: &JobNotificationFacts,
|
||||
) -> Result<bool> {
|
||||
notification_routing::jobs::route(self, workspace, source, facts)
|
||||
}
|
||||
fn job_notification_request(
|
||||
&self,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
) -> Result<Option<(String, RuntimeWorkerRef)>> {
|
||||
notification_routing::jobs::request(self, workspace, id)
|
||||
}
|
||||
fn authorize_job_notification(
|
||||
&self,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
target: &TicketWorkerNotificationTarget,
|
||||
) -> Result<Option<NotificationFailureCode>> {
|
||||
notification_routing::jobs::authorize(self, workspace, id, target)
|
||||
}
|
||||
fn ack_job_notification(
|
||||
&self,
|
||||
workspace: &str,
|
||||
id: &str,
|
||||
target: &RuntimeWorkerRef,
|
||||
) -> Result<()> {
|
||||
notification_routing::jobs::ack(self, workspace, id, target)
|
||||
}
|
||||
|
||||
fn reserve_backend_job_delivery(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -14137,11 +14225,48 @@ fn migrate_notification_routing_v94_to_v95(conn: &Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_job_notifications_v95_to_v96(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 95 {
|
||||
return Err(Error::Store(
|
||||
"Job notification migration requires schema 95".into(),
|
||||
));
|
||||
}
|
||||
let foreign_keys: bool = conn.query_row("PRAGMA foreign_keys", [], |r| r.get(0))?;
|
||||
let legacy_alter: bool = conn.query_row("PRAGMA legacy_alter_table", [], |r| r.get(0))?;
|
||||
conn.execute_batch("PRAGMA foreign_keys=OFF; PRAGMA legacy_alter_table=ON;")?;
|
||||
let result = (|| -> Result<()> {
|
||||
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
|
||||
notification_routing::jobs::install(&tx)?;
|
||||
let violations: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM pragma_foreign_key_check)",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if violations {
|
||||
return Err(Error::Store(
|
||||
"Job notification migration foreign-key violation".into(),
|
||||
));
|
||||
}
|
||||
tx.execute("INSERT INTO __yoi_schema_migrations(version,name) VALUES(96,'Backend Job common notification routing')",[])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})();
|
||||
let restore = conn
|
||||
.execute_batch(&format!(
|
||||
"PRAGMA legacy_alter_table={}; PRAGMA foreign_keys={};",
|
||||
i32::from(legacy_alter),
|
||||
i32::from(foreign_keys)
|
||||
))
|
||||
.map_err(Error::from);
|
||||
result.and(restore)
|
||||
}
|
||||
|
||||
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
||||
create_workspace_schema_structure(conn)?;
|
||||
assignment_delivery::install(conn)?;
|
||||
assignment_delivery::install_post_inputs(conn)?;
|
||||
notification_routing::install(conn)?;
|
||||
notification_routing::jobs::install(conn)?;
|
||||
ticket::verify_sqlite_ticket_schema(conn).map_err(|error| {
|
||||
Error::Store(format!(
|
||||
"fresh Workspace Ticket schema verification failed: {error}"
|
||||
@@ -14400,6 +14525,9 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? >= 95 {
|
||||
notification_routing::verify(conn)?;
|
||||
}
|
||||
if current_schema_version(conn)? >= 96 {
|
||||
notification_routing::jobs::verify(conn)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -15156,6 +15284,7 @@ mod tests {
|
||||
include!("assignment_delivery_tests.rs");
|
||||
include!("notification_routing_tests.rs");
|
||||
include!("notification_diagnostics_tests.rs");
|
||||
include!("job_notification_tests.rs");
|
||||
include!("store_workspace_config_tests.rs");
|
||||
|
||||
fn drive_grant_fixture(store: &SqliteWorkspaceStore) -> server_api::DriveGrantResponse {
|
||||
@@ -17369,6 +17498,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
WorkspaceSchemaMigrationStep {version:93, name:"Durable assignment deliveries and source-bound startup fences".into()},
|
||||
WorkspaceSchemaMigrationStep {version:94, name:"Canonical Ticket posts".into()},
|
||||
WorkspaceSchemaMigrationStep {version:95, name:"Common notification routing and durable Worker delivery".into()},
|
||||
WorkspaceSchemaMigrationStep {version:96, name:"Backend Job common notification routing".into()},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -17474,6 +17604,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(93, "Durable assignment deliveries and source-bound startup fences".to_string()),
|
||||
(94, "Canonical Ticket posts".to_string()),
|
||||
(95, "Common notification routing and durable Worker delivery".to_string()),
|
||||
(96, "Backend Job common notification routing".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -20093,7 +20224,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(92, "Ticket-owned assignment tracking".into()),
|
||||
(93, "Durable assignment deliveries and source-bound startup fences".into()),
|
||||
(94, "Canonical Ticket posts".into()),
|
||||
(95, "Common notification routing and durable Worker delivery".into())]);
|
||||
(95, "Common notification routing and durable Worker delivery".into()),
|
||||
(96, "Backend Job common notification routing".into())]);
|
||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||
}
|
||||
|
||||
@@ -13,9 +13,10 @@
|
||||
//!
|
||||
//! Results are checked for unknown fields, duplicates, count/string limits, and quotes that are
|
||||
//! exact substrings of the checked title/body. Invalid results end as invalid_result.
|
||||
//! Delivery rereads the Ticket and notifies the original Worker only of findings matching the
|
||||
//! saved digest. No findings and stale results end delivery with distinct suppression reasons,
|
||||
//! without redelivery on recovery. Advice is neither an approval gate nor implementation guidance;
|
||||
//! Before routing, the adapter rereads the Ticket and reports findings/content freshness.
|
||||
//! Common routing records clean/stale suppression separately from Runtime acceptance. Once
|
||||
//! a request is pinned, recovery reuses those bytes even if the Ticket later changes.
|
||||
//! Advice is neither an approval gate nor implementation guidance;
|
||||
//! no findings do not prove the entire specification correct.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
@@ -217,18 +218,18 @@ fn validate_finding_text(field: &str, value: &str, max_bytes: usize) -> Result<(
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TicketItemCheckNotification {
|
||||
Deliver(String),
|
||||
pub enum TicketItemCheckFacts {
|
||||
Findings(String),
|
||||
NoFindings,
|
||||
Stale,
|
||||
}
|
||||
|
||||
/// Decide whether a completed check should produce an advisory. Clean and stale
|
||||
/// results remain distinct so delivery recovery can record why no input was sent.
|
||||
pub fn notification(
|
||||
/// Domain facts about validated results and the current checked content.
|
||||
/// The common routing policy owns whether these facts produce an advisory.
|
||||
pub fn result_facts(
|
||||
acceptance: &BackendJobResultAcceptance,
|
||||
current_ticket: &ticket::Ticket,
|
||||
) -> Result<TicketItemCheckNotification> {
|
||||
) -> Result<TicketItemCheckFacts> {
|
||||
let input = parse_input(&acceptance.job.request.input)?;
|
||||
let result = validate_result(
|
||||
&acceptance.job.request.input,
|
||||
@@ -239,12 +240,32 @@ pub fn notification(
|
||||
if input.content_digest != content_digest(current_ticket)
|
||||
|| input.ticket_id != current_ticket.meta.id
|
||||
{
|
||||
return Ok(TicketItemCheckNotification::Stale);
|
||||
return Ok(TicketItemCheckFacts::Stale);
|
||||
}
|
||||
if result.findings.is_empty() {
|
||||
return Ok(TicketItemCheckNotification::NoFindings);
|
||||
return Ok(TicketItemCheckFacts::NoFindings);
|
||||
}
|
||||
|
||||
Ok(TicketItemCheckFacts::Findings(render_result(acceptance)?))
|
||||
}
|
||||
|
||||
/// Stable rendering from the original checked snapshot. Used to pin pre-upgrade
|
||||
/// unacknowledged requests without reinterpreting them against newer Ticket text.
|
||||
pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result<String> {
|
||||
let input = parse_input(&acceptance.job.request.input)?;
|
||||
let result = validate_result(
|
||||
&acceptance.job.request.input,
|
||||
acceptance
|
||||
.job
|
||||
.result
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::InvalidInput("completed checker has no result".into()))?,
|
||||
)?;
|
||||
if result.findings.is_empty() {
|
||||
return Err(Error::InvalidInput(
|
||||
"clean checker result has no advisory body".into(),
|
||||
));
|
||||
}
|
||||
let mut content = format!(
|
||||
"[Ticket item checker advisory]\nTicket: {}\nContent digest: {}\n\nThis is a post-save advisory based only on the inspected Ticket text. It is not a new user request, approval gate, workflow blocker, or implementation instruction.\n",
|
||||
input.ticket_resource_key, input.content_digest
|
||||
@@ -262,7 +283,7 @@ pub fn notification(
|
||||
finding.suggestion
|
||||
));
|
||||
}
|
||||
Ok(TicketItemCheckNotification::Deliver(content))
|
||||
Ok(content)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -45,6 +45,8 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"repository_ssh_host_trust_keys",
|
||||
"repository_ssh_host_trusts",
|
||||
"server_secret_objects",
|
||||
"notification_job_targets",
|
||||
"notification_job_legacy_receipts",
|
||||
"notification_assignment_targets",
|
||||
"notification_acknowledgements",
|
||||
"notification_requests",
|
||||
@@ -489,6 +491,8 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
// commit; update guards remain installed throughout.
|
||||
let mut ticket_delete_guards = Vec::new();
|
||||
for (table, trigger) in [
|
||||
("notification_job_targets", "notification_job_targets_no_delete"),
|
||||
("notification_job_legacy_receipts", "notification_job_legacy_no_delete"),
|
||||
("notification_assignment_targets", "notification_assignment_targets_no_delete"),
|
||||
("notification_sources", "notification_sources_no_delete"),
|
||||
("notification_decisions", "notification_decisions_no_delete"),
|
||||
|
||||
Reference in New Issue
Block a user