Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: merge develop into hare/develop

Browse Source
This commit is contained in:
Hare
2026-09-18 21:36:04 +09:00
parent d7917512ac fba12b89bc
commit b06e2860ca
202 changed files with 18312 additions and 3945 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/Cargo.toml
+3 -2
View File
@@ -12,7 +12,7 @@ path = "src/main.rs"
[features]
default = ["typescript"]
typescript = ["dep:ts-rs", "workspace-api/typescript"]
typescript = ["dep:ts-rs", "server-api/typescript"]
[dependencies]
async-trait.workspace = true
@@ -23,6 +23,8 @@ config-source.workspace = true
flow = { path = "../flow" }
manifest.workspace = true
protocol = { workspace = true }
runtime-api.workspace = true
server-api.workspace = true
project-record.workspace = true
reqwest = { version = "0.13", default-features = false, features = ["blocking", "json", "native-tls"] }
ring.workspace = true
@@ -40,7 +42,6 @@ tokio = { workspace = true, features = ["fs", "macros", "net", "rt-multi-thread"
tower.workspace = true
tokio-tungstenite = { workspace = true, features = ["rustls-tls-webpki-roots"] }
worker.workspace = true
workspace-api.workspace = true
workdir = { workspace = true, features = ["http-client"] }
worker-runtime.workspace = true
toml.workspace = true
crates/workspace-server/src/auth.rs
+1 -1
View File
@@ -1,8 +1,8 @@
use axum::http::{HeaderMap, header};
use chrono::{DateTime, Duration, Utc};
pub use server_api::{ActorAuthMethod, AuthPublicConfig, AuthenticatedUser, RequestActor};
use sha2::{Digest, Sha256};
use uuid::Uuid;
pub use workspace_api::{ActorAuthMethod, AuthPublicConfig, AuthenticatedUser, RequestActor};
use crate::{Error, Result, store::ControlPlaneStore};
crates/workspace-server/src/companion.rs
+2 -2
View File
@@ -1,9 +1,9 @@
use workspace_api::{
use server_api::{
CompanionLifecycleState, CompanionMessageDisposition, CompanionTransportSummary, Diagnostic,
DiagnosticSeverity,
};
pub use workspace_api::{
pub use server_api::{
CompanionCancelRequest, CompanionMessageRequest, CompanionMessageResponse,
CompanionStatusResponse, CompanionTranscriptProjection,
};
crates/workspace-server/src/hosts.rs
+600 -240
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/latest_schema.sql
+41
View File
@@ -772,6 +772,47 @@ CREATE TABLE "worker_registry" (
PRIMARY KEY (workspace_id, worker_id),
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE TABLE worker_registry_observations (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
availability TEXT NOT NULL CHECK (availability IN ('observed', 'unavailable')),
worker_json TEXT,
connection_generation INTEGER NOT NULL,
subject_revision INTEGER NOT NULL,
snapshot_revision INTEGER NOT NULL,
projection_revision INTEGER NOT NULL,
observed_at TEXT NOT NULL,
PRIMARY KEY (workspace_id, runtime_id, worker_id),
FOREIGN KEY (workspace_id, worker_id)
REFERENCES worker_registry(workspace_id, worker_id) ON DELETE CASCADE
);
CREATE TABLE worker_registry_projection_cursors (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
connection_generation INTEGER NOT NULL,
snapshot_revision INTEGER NOT NULL,
PRIMARY KEY (workspace_id, runtime_id)
);
CREATE TABLE worker_registry_projection_revisions (
workspace_id TEXT PRIMARY KEY,
revision INTEGER NOT NULL
);
CREATE TABLE worker_registry_projection_removals (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
projection_revision INTEGER NOT NULL,
PRIMARY KEY (workspace_id, runtime_id, worker_id)
);
CREATE TABLE worker_registry_projection_diagnostics (
diagnostic_id INTEGER PRIMARY KEY AUTOINCREMENT,
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
reason TEXT NOT NULL,
observed_at TEXT NOT NULL
);
CREATE TABLE worker_removal_operations (
operation_id TEXT PRIMARY KEY, plan_id TEXT NOT NULL UNIQUE, input_fingerprint TEXT NOT NULL,
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
crates/workspace-server/src/lib.rs
+3 -2
View File
@@ -30,6 +30,7 @@ pub mod skills;
pub mod store;
pub mod workdir_create_operations;
mod workdir_removal;
mod worker_projection;
pub mod worker_source;
pub mod workspace_catalog;
mod workspace_deletion;
@@ -48,11 +49,11 @@ pub use server::{
AuthConfig, ServerConfig, WorkspaceApi, WorkspaceServerApi, build_router,
build_workspace_server_router, serve, serve_workspace_catalog,
};
pub use store::{ControlPlaneStore, SqliteWorkspaceStore, WorkspaceRecord};
pub use workspace_api::{
pub use server_api::{
GitCommitSummary, GitRemoteSummary, GitRepositorySummary, RepositorySummary,
WorkspaceCreateResponse,
};
pub use store::{ControlPlaneStore, SqliteWorkspaceStore, WorkspaceRecord};
pub use workspace_catalog::{
InitialRepositoryIntent, WorkspaceCatalogService, WorkspaceCreateRequest, WorkspaceCreateResult,
};
crates/workspace-server/src/memory_staging.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use memory::extract::{CandidateKind, StagingRecord};
use memory::schema::{EvidenceOrigin, EvidenceOriginKind, SourceEvidenceRef, SourceRef};
use workspace_api::{
use server_api::{
Diagnostic, DiagnosticSeverity, MemoryCandidateKind, MemoryEvidenceOrigin,
MemoryEvidenceOriginKind, MemorySourceEvidenceRef, MemorySourceRef, MemoryStagingEntry,
MemoryStagingEvidence, MemoryStagingListResponse, MemoryStagingRecord,
crates/workspace-server/src/profile_settings.rs
+4 -4
View File
@@ -4,16 +4,16 @@ use std::path::{Component, Path, PathBuf};
use config_source::{ConfigContentType, ConfigSchemaContribution, VirtualPath};
use manifest::{ProfileSource, builtin_profile_catalog_snapshot, resolve_profile_artifact_value};
use serde::Deserialize;
use server_api::{
ProfileSettingsResponse, WorkspaceMetadataSettingsResponse, WorkspaceProfileSourceProvenance,
WorkspaceProfileSourceSummary, WorkspaceProfileSummary,
};
use sha2::{Digest, Sha256};
use worker::EffectivePromptCatalog;
use worker_runtime::config_bundle::{
ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
};
use worker_runtime::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput};
use workspace_api::{
ProfileSettingsResponse, WorkspaceMetadataSettingsResponse, WorkspaceProfileSourceProvenance,
WorkspaceProfileSourceSummary, WorkspaceProfileSummary,
};
use crate::config_source::{
WorkspaceConfigSchemaProvider, WorkspaceConfigState, evaluate_workspace_config_state,
crates/workspace-server/src/records.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
pub use workspace_api::{
pub use server_api::{
ObjectiveDetail, ObjectiveEventDetail, ObjectiveLinkedTicketSummary, ObjectiveResourceSummary,
ObjectiveSummary, QueryPage,
};
crates/workspace-server/src/repositories.rs
+5 -8
View File
@@ -4,7 +4,7 @@ use std::{
process::Command,
};
use workspace_api::{
use server_api::{
Diagnostic, DiagnosticSeverity, GitCommitSummary, GitRemoteSummary, GitRepositorySummary,
RepositoryDiagnostic, RepositoryObservedStatus, RepositorySource, RepositorySummary,
};
@@ -629,7 +629,7 @@ mod tests {
let projection = reader.list();
assert_eq!(
projection.items[0].source.kind,
workspace_api::RepositorySourceKind::Invalid
server_api::RepositorySourceKind::Invalid
);
let diagnostics = projection.items[0].diagnostics.as_ref().unwrap();
assert!(diagnostics.iter().any(|diagnostic| {
@@ -642,7 +642,7 @@ mod tests {
#[test]
fn remote_source_is_visible_but_local_provider_operations_fail_closed() {
let source = RepositorySource {
kind: workspace_api::RepositorySourceKind::Ssh,
kind: server_api::RepositorySourceKind::Ssh,
uri: "git@example.test:org/repository.git".to_string(),
};
let reader = RepositoryRegistryReader::new(vec![ConfiguredRepository {
@@ -660,10 +660,7 @@ mod tests {
let projection = reader.list();
let summary = &projection.items[0];
assert_eq!(
summary.source.kind,
workspace_api::RepositorySourceKind::Ssh
);
assert_eq!(summary.source.kind, server_api::RepositorySourceKind::Ssh);
assert_eq!(
summary.observed_status,
RepositoryObservedStatus::Unverified
@@ -766,7 +763,7 @@ mod tests {
);
let source_descriptor = RepositorySource {
kind: workspace_api::RepositorySourceKind::LocalPath,
kind: server_api::RepositorySourceKind::LocalPath,
uri: path.display().to_string(),
};
let reader = RepositoryRegistryReader::new(vec![ConfiguredRepository {
crates/workspace-server/src/repository_access.rs
+10 -10
View File
@@ -11,16 +11,16 @@ use ring::hmac;
use ring::rand::{SecureRandom, SystemRandom};
use rusqlite::{OptionalExtension, TransactionBehavior, params};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use ssh_key::private::Ed25519Keypair;
use ssh_key::{Algorithm, HashAlg, LineEnding, PrivateKey, PublicKey};
use workspace_api::{
use server_api::{
CreateRepositorySshCredentialRequest, DeleteRepositorySshCredentialRequest,
DeleteRepositorySshHostTrustRequest, GenerateRepositorySshCredentialRequest,
PutRepositorySshHostTrustRequest, RepositoryAccessMode, RepositoryAccessProjection,
RepositorySshAccessBinding, RepositorySshCredential, RepositorySshHostTrust,
RepositorySshPublicKey, RotateRepositorySshCredentialRequest,
};
use sha2::{Digest, Sha256};
use ssh_key::private::Ed25519Keypair;
use ssh_key::{Algorithm, HashAlg, LineEnding, PrivateKey, PublicKey};
use crate::config_source::{
EvaluatedConfigCandidate, WorkspaceConfigSchemaProvider, WorkspaceConfigState,
@@ -138,7 +138,7 @@ pub fn project_repository_access_state(
fn validate_repository_access_source(
repository_key: &str,
source: &workspace_api::RepositorySource,
source: &server_api::RepositorySource,
) -> Result<()> {
if crate::repository_source::is_plain_http_repository_source(source) {
return Err(Error::InvalidInput(format!(
@@ -205,7 +205,7 @@ fn project_repository_access_evaluation(
}
let mut bindings = Vec::with_capacity(config.repository_access.len());
for (repository_key, access) in config.repository_access {
workspace_api::validate_repository_key(&repository_key)
server_api::validate_repository_key(&repository_key)
.map_err(|error| Error::InvalidInput(format!("invalid Repository key: {error}")))?;
validate_identifier("credential_id", &access.ssh.credential)?;
validate_identifier("host_trust_id", &access.ssh.host_trust)?;
@@ -213,7 +213,7 @@ fn project_repository_access_evaluation(
.get_repository_by_key(workspace_id, &repository_key)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Repository `{repository_key}`")))?;
validate_repository_access_source(&repository_key, &repository.source)?;
if repository.source.kind != workspace_api::RepositorySourceKind::Ssh {
if repository.source.kind != server_api::RepositorySourceKind::Ssh {
return Err(Error::InvalidInput(format!(
"Repository `{repository_key}` is not an ssh:// Repository"
)));
@@ -1940,8 +1940,8 @@ mod tests {
DEFAULT_SCHEMA_VERSION, SnapshotEnvironment, ToolchainContract, VirtualPath,
WorkspaceConfigSchemaBundle,
};
use server_api::{RepositoryObservedStatus, RepositorySource, RepositorySourceKind};
use ssh_key::private::Ed25519Keypair;
use workspace_api::{RepositoryObservedStatus, RepositorySource, RepositorySourceKind};
fn test_private_key(seed: u8) -> (String, String) {
let key = PrivateKey::from(Ed25519Keypair::from_seed(&[seed; 32]));
@@ -1995,8 +1995,8 @@ mod tests {
assert!(error.to_string().contains("unsupported plain HTTP"));
assert!(error.to_string().contains("HTTPS or SSH"));
let mismatched = workspace_api::RepositorySource {
kind: workspace_api::RepositorySourceKind::Https,
let mismatched = server_api::RepositorySource {
kind: server_api::RepositorySourceKind::Https,
uri: "http://git.example.test/team/project.git".to_string(),
};
let error = validate_repository_access_source("remote", &mismatched).unwrap_err();
crates/workspace-server/src/repository_source.rs
+1 -1
View File
@@ -1,8 +1,8 @@
use std::path::Path;
use server_api::{RepositorySource, RepositorySourceKind};
use sha2::{Digest, Sha256};
use url::Url;
use workspace_api::{RepositorySource, RepositorySourceKind};
use crate::{Error, Result};
crates/workspace-server/src/server.rs
+1004 -419
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/skills.rs
+2 -2
View File
@@ -4,12 +4,12 @@ use config_source::{
ConfigSchemaContribution, MarkdownDocumentProjection, VirtualPath, project_markdown_document,
};
use serde::Deserialize;
use worker::skill::SkillActivationResponse;
use workspace_api::{
use server_api::{
SKILL_CATALOG_AUTHORITY, SkillActivationStatus, SkillCatalogEntry, SkillCatalogResponse,
SkillDetailResponse, SkillDiagnostic, SkillDiagnosticSeverity, SkillProjectionIdentity,
SkillProjectionStatus, SkillProvenance, SkillResourceRef, SkillSourceKind,
};
use worker::skill::SkillActivationResponse;
use crate::config_source::{
WorkspaceConfigSchemaProvider, WorkspaceConfigState, evaluate_workspace_config_state,
crates/workspace-server/src/store.rs
+954 -18
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/workdir_create_operations.rs
+3 -3
View File
@@ -549,14 +549,14 @@ mod tests {
repository_key: "main".to_string(),
kind: "git".to_string(),
provider: Some("git".to_string()),
source: workspace_api::RepositorySource {
kind: workspace_api::RepositorySourceKind::LocalPath,
source: server_api::RepositorySource {
kind: server_api::RepositorySourceKind::LocalPath,
uri: "/tmp/main".to_string(),
},
default_ref: Some("develop".to_string()),
source_revision: 1,
source_fingerprint: "sha256:test".to_string(),
observed_status: workspace_api::RepositoryObservedStatus::Unverified,
observed_status: server_api::RepositoryObservedStatus::Unverified,
observed_at: None,
created_at: "2026-08-24T00:00:00Z".to_string(),
updated_at: "2026-08-24T00:00:00Z".to_string(),
crates/workspace-server/src/workdir_removal.rs
+1 -1
View File
@@ -879,7 +879,7 @@ fn hex_sha256(bytes: &[u8]) -> String {
mod tests {
use super::*;
use crate::store::{AccountRecord, ControlPlaneStore, RepositoryRecord, WorkspaceRecord};
use workspace_api::{RepositoryObservedStatus, RepositorySource, RepositorySourceKind};
use server_api::{RepositoryObservedStatus, RepositorySource, RepositorySourceKind};
fn attempt_owner() -> WorkdirRemovalAttemptOwner {
WorkdirRemovalAttemptOwner {
crates/workspace-server/src/worker_projection.rs
+279
View File
@@ -0,0 +1,279 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use chrono::Utc;
use protocol::subscription::{
EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot,
};
use tokio::sync::broadcast;
use tokio::task::JoinHandle;
use crate::runtime_subscription::{
BrokerSubscriptionEvent, RuntimeSubscriptionBroker, RuntimeSubscriptionBrokerError,
};
use crate::store::{
ControlPlaneStore, WorkerRegistryProjectionCommit, WorkerRegistryProjectionRecord,
};
use worker_runtime::identity::RuntimeWorkerRef;
const PROJECTION_EVENT_CAPACITY: usize = 256;
#[derive(Debug, Clone)]
pub enum WorkerProjectionChange {
Upsert(WorkerRegistryProjectionRecord),
Removed(RuntimeWorkerRef),
}
#[derive(Debug, Clone)]
pub struct WorkerProjectionEvent {
pub revision: u64,
pub changes: Vec<WorkerProjectionChange>,
}
/// Owns the durable Runtime-to-Workspace Worker projection and its independent
/// low-frequency publication stream.
///
/// One Runtime subscription is kept alive per registered Runtime even when no
/// browser is connected. Browser subscriptions therefore never become the
/// authority for populating `worker_registry` observations.
pub struct WorkerProjectionService {
workspace_id: String,
store: Arc<dyn ControlPlaneStore>,
events: broadcast::Sender<WorkerProjectionEvent>,
connection_generation_epoch: u64,
tasks: Mutex<HashMap<String, JoinHandle<()>>>,
}
impl WorkerProjectionService {
pub fn new(workspace_id: impl Into<String>, store: Arc<dyn ControlPlaneStore>) -> Arc<Self> {
let (events, _) = broadcast::channel(PROJECTION_EVENT_CAPACITY);
Arc::new(Self {
workspace_id: workspace_id.into(),
store,
events,
connection_generation_epoch: Utc::now().timestamp_nanos_opt().unwrap_or_default().max(0)
as u64,
tasks: Mutex::new(HashMap::new()),
})
}
pub fn shutdown(&self) {
for (_, task) in self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.drain()
{
task.abort();
}
}
pub fn subscribe(&self) -> broadcast::Receiver<WorkerProjectionEvent> {
self.events.subscribe()
}
pub async fn attach_runtime(
self: &Arc<Self>,
broker: &RuntimeSubscriptionBroker,
runtime_id: &str,
) -> Result<(), RuntimeSubscriptionBrokerError> {
let mut subscription =
broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?;
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop {
let Ok(Some(event)) = tokio::time::timeout_at(deadline, subscription.recv()).await
else {
break;
};
let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. });
if let Err(error) = self.apply_broker_event(runtime_id, event).await {
tracing::warn!(
runtime_id,
error = %error,
"failed to initialize durable Worker projection"
);
}
if initialized {
break;
}
}
let service = Arc::downgrade(self);
let runtime_id = runtime_id.to_string();
let task_runtime_id = runtime_id.clone();
let task = tokio::spawn(async move {
while let Some(event) = subscription.recv().await {
let Some(service) = service.upgrade() else {
return;
};
if let Err(error) = service
.apply_broker_event(task_runtime_id.as_str(), event)
.await
{
tracing::warn!(
runtime_id = %task_runtime_id,
error = %error,
"failed to update durable Worker projection"
);
}
}
});
if let Some(previous) = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.insert(runtime_id, task)
{
previous.abort();
}
Ok(())
}
pub fn seed_observation(
&self,
runtime_id: &str,
worker: &protocol::subscription::SubscriptionWorker,
observed_at: &str,
) -> crate::Result<()> {
let commit = self.store.apply_worker_registry_observation(
self.workspace_id.as_str(),
runtime_id,
0,
worker,
observed_at,
)?;
self.publish_commit(commit)
}
pub fn publish_catalog_change(&self, worker: &RuntimeWorkerRef) -> crate::Result<()> {
let commit = self
.store
.publish_worker_registry_catalog_change(self.workspace_id.as_str(), worker)?;
self.publish_commit(commit)
}
pub fn publish_removed(&self, worker: RuntimeWorkerRef) -> crate::Result<()> {
let commit = self
.store
.publish_worker_registry_removal(self.workspace_id.as_str(), &worker)?;
if commit.changed_workers.is_empty() {
return Ok(());
}
let _ = self.events.send(WorkerProjectionEvent {
revision: commit.revision,
changes: vec![WorkerProjectionChange::Removed(worker)],
});
Ok(())
}
async fn apply_broker_event(
&self,
runtime_id: &str,
event: BrokerSubscriptionEvent,
) -> crate::Result<()> {
let observed_at = Utc::now().to_rfc3339();
let projection_generation = |connection_generation: u64| {
self.connection_generation_epoch
.saturating_add(connection_generation)
};
let commit = match event {
BrokerSubscriptionEvent::Snapshot {
connection_generation,
snapshot_revision,
snapshot: SubscriptionSnapshot::Workers { workers },
} => self.store.reconcile_worker_registry_snapshot(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
snapshot_revision,
workers.as_slice(),
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Snapshot { .. } => return Ok(()),
BrokerSubscriptionEvent::Event {
connection_generation,
payload: SubscriptionEventPayload::WorkerUpserted { worker },
..
} => self.store.apply_worker_registry_observation(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
&worker,
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Event {
connection_generation,
subject_revision,
payload: SubscriptionEventPayload::WorkerRemoved { worker_id, .. },
} => self.store.mark_worker_registry_observation_unavailable(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
Some(worker_id.as_str()),
Some(subject_revision),
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Event { .. } => return Ok(()),
BrokerSubscriptionEvent::Disconnected {
connection_generation,
..
}
| BrokerSubscriptionEvent::Rejected {
connection_generation,
..
}
| BrokerSubscriptionEvent::Closed {
connection_generation,
..
} => self.store.mark_worker_registry_observation_unavailable(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
None,
None,
observed_at.as_str(),
)?,
};
self.publish_commit(commit)
}
fn publish_commit(&self, commit: WorkerRegistryProjectionCommit) -> crate::Result<()> {
if commit.changed_workers.is_empty() {
return Ok(());
}
let changes = commit
.changed_workers
.iter()
.filter_map(|worker| {
match self
.store
.worker_registry_projection(self.workspace_id.as_str(), worker)
{
Ok(Some(record)) => Some(WorkerProjectionChange::Upsert(record)),
Ok(None) => None,
Err(error) => {
tracing::warn!(
runtime_id = %worker.runtime_id,
worker_id = %worker.worker_id,
error = %error,
"failed to load committed Worker projection"
);
None
}
}
})
.collect::<Vec<_>>();
if !changes.is_empty() {
let _ = self.events.send(WorkerProjectionEvent {
revision: commit.revision,
changes,
});
}
Ok(())
}
}
impl Drop for WorkerProjectionService {
fn drop(&mut self) {
self.shutdown();
}
}
crates/workspace-server/src/workspace_catalog.rs
+4 -4
View File
@@ -4,8 +4,8 @@ use chrono::{SecondsFormat, Utc};
use sha2::{Digest, Sha256};
use uuid::Uuid;
pub use workspace_api::{InitialRepositoryIntent, WorkspaceCreateRequest};
use workspace_api::{RepositoryObservedStatus, RepositorySource};
pub use server_api::{InitialRepositoryIntent, WorkspaceCreateRequest};
use server_api::{RepositoryObservedStatus, RepositorySource};
use crate::repository_source::{parse_repository_source, repository_source_fingerprint};
use crate::store::{
@@ -96,7 +96,7 @@ impl WorkspaceCatalogService {
normalize_required("display_name", request.display_name, MAX_DISPLAY_NAME_BYTES)?;
let repository_source = validate_repository_source(&request.repository.uri)?;
let repository_uri = repository_source.uri.clone();
workspace_api::validate_repository_key(&request.repository.repository_key)
server_api::validate_repository_key(&request.repository.repository_key)
.map_err(|error| Error::InvalidInput(format!("invalid Repository key: {error}")))?;
let repository_key = request.repository.repository_key.clone();
let default_ref = request
@@ -228,7 +228,7 @@ mod tests {
InMemoryWorkspaceSigningMaterialStore, WorkspaceSigningMaterialStore,
WorkspaceSigningPrivateMaterial, identity_error,
};
use workspace_api::RepositorySourceKind;
use server_api::RepositorySourceKind;
struct FailFirstMaterialWrite {
inner: Arc<InMemoryWorkspaceSigningMaterialStore>,
crates/workspace-server/src/workspace_deletion.rs
+8 -3
View File
@@ -1,7 +1,6 @@
use chrono::Utc;
use rusqlite::{OptionalExtension, params};
use sha2::{Digest, Sha256};
use workspace_api::{
use server_api::{
WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, WORKSPACE_DELETION_MAX_BLOCKERS,
WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS, WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES,
WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES, WORKSPACE_DELETION_MAX_REVISION_BYTES,
@@ -9,6 +8,7 @@ use workspace_api::{
WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, WorkspaceDeletionResourceCounts,
WorkspaceDeletionState,
};
use sha2::{Digest, Sha256};
use crate::store::{SqliteWorkspaceStore, WorkspaceRecord};
use crate::{Error, Result};
@@ -67,6 +67,11 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"worker_mutation_source_proof_jtis",
"worker_orphan_diagnostics",
"worker_registry",
"worker_registry_observations",
"worker_registry_projection_cursors",
"worker_registry_projection_diagnostics",
"worker_registry_projection_removals",
"worker_registry_projection_revisions",
"worker_removal_operations",
"worker_retention_audit_events",
"worker_session_archives",
@@ -200,7 +205,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
) -> Result<WorkspaceDeletionReservation> {
validate_operation_id(&request.operation_id)?;
if request.expected_revision.len() > WORKSPACE_DELETION_MAX_REVISION_BYTES
|| request.confirmation.len() > workspace_api::WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES
|| request.confirmation.len() > server_api::WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES
{
return Err(Error::InvalidInput(
"Workspace deletion request exceeds bounded field limits".to_string(),
crates/workspace-server/src/workspace_subscription.rs
+103 -220
View File
@@ -1,4 +1,4 @@
use std::collections::{BTreeMap, HashMap, HashSet};
use std::collections::HashMap;
use axum::extract::ws::{Message as WsMessage, WebSocket};
use futures::{SinkExt, StreamExt};
@@ -10,11 +10,10 @@ use protocol::subscription::{
use tokio::sync::mpsc;
use worker_runtime::identity::RuntimeWorkerRef;
use crate::runtime_subscription::{BrokerSubscriptionEvent, RuntimeSubscriptionBroker};
use crate::runtime_subscription::RuntimeSubscriptionBroker;
use crate::server::{
WorkspaceApi, authorize_browser_worker_method, connect_workspace_worker_protocol,
};
use crate::store::WorkspaceResourceKind;
const OUTBOUND_CAPACITY: usize = 256;
@@ -293,62 +292,31 @@ async fn run_worker_protocol(
async fn run_workspace_workers(
api: WorkspaceApi,
broker: RuntimeSubscriptionBroker,
_broker: RuntimeSubscriptionBroker,
request_id: protocol::subscription::SubscriptionRequestId,
subscription_id: SubscriptionId,
outbound: mpsc::Sender<WsMessage>,
) {
let runtime_ids = broker.runtime_ids();
let mut pending = runtime_ids.iter().cloned().collect::<HashSet<_>>();
let (events, mut event_receiver) = mpsc::channel(OUTBOUND_CAPACITY);
let mut upstreams = tokio::task::JoinSet::new();
for runtime_id in runtime_ids {
let Ok(mut subscription) =
broker.subscribe(&runtime_id, EventSubscriptionSelector::RuntimeWorkers)
else {
pending.remove(&runtime_id);
continue;
};
let sender = events.clone();
upstreams.spawn(async move {
while let Some(event) = subscription.recv().await {
if sender.send((runtime_id.clone(), event)).await.is_err() {
break;
}
}
});
}
drop(events);
let mut workers = HashMap::<String, BTreeMap<String, SubscriptionWorker>>::new();
while !pending.is_empty() {
let Some((runtime_id, event)) = event_receiver.recv().await else {
return;
};
match event {
BrokerSubscriptionEvent::Snapshot { snapshot, .. } => {
install_snapshot(&api, &mut workers, &runtime_id, snapshot);
pending.remove(&runtime_id);
}
BrokerSubscriptionEvent::Disconnected { .. }
| BrokerSubscriptionEvent::Rejected { .. }
| BrokerSubscriptionEvent::Closed { .. } => {
pending.remove(&runtime_id);
}
BrokerSubscriptionEvent::Event { .. } => {}
}
}
let mut revisions = HashMap::<RuntimeWorkerRef, u64>::new();
let mut initial_workers = Vec::new();
for (runtime_id, runtime) in &mut workers {
for worker in runtime.values_mut() {
let worker_ref = RuntimeWorkerRef::new(runtime_id, worker.worker_id.as_str());
worker.subject_revision = next_revision(&mut revisions, &worker_ref);
initial_workers.push(worker.clone());
}
}
sort_workers(&mut initial_workers);
// Subscribe before reading so commits racing with the snapshot are replayed.
let mut events = api.worker_projection.subscribe();
let Ok((snapshot_revision, records)) = api.store.worker_registry_projection_snapshot(
&api.config.workspace_id,
api.config.max_records.max(1),
) else {
let _ = send_rejected(
&outbound,
request_id,
SubscriptionRejectionCode::Internal,
"failed to read durable Worker projection".to_string(),
)
.await;
return;
};
let mut workers = records
.into_iter()
.filter_map(|record| project_registry_worker(&api, record))
.collect::<Vec<_>>();
sort_workers(&mut workers);
if send_frame(
&outbound,
SubscriptionFrame::new(SubscriptionFramePayload::Response(
@@ -356,10 +324,8 @@ async fn run_workspace_workers(
request_id,
subscription_id: subscription_id.clone(),
selector: EventSubscriptionSelector::WorkspaceWorkers,
snapshot_revision: 1,
snapshot: SubscriptionSnapshot::Workers {
workers: initial_workers,
},
snapshot_revision,
snapshot: SubscriptionSnapshot::Workers { workers },
},
)),
)
@@ -369,174 +335,103 @@ async fn run_workspace_workers(
return;
}
while let Some((runtime_id, event)) = event_receiver.recv().await {
match event {
BrokerSubscriptionEvent::Snapshot { snapshot, .. } => {
let removed = workers.remove(&runtime_id).unwrap_or_default();
for worker in removed.values() {
let worker_ref = RuntimeWorkerRef::new(&runtime_id, worker.worker_id.as_str());
let revision = next_revision(&mut revisions, &worker_ref);
if send_event(
&outbound,
&subscription_id,
revision,
SubscriptionEventPayload::WorkerRemoved {
worker_id: worker.worker_id.clone(),
runtime_id: Some(runtime_id.clone()),
},
)
.await
.is_err()
{
return;
}
}
install_snapshot(&api, &mut workers, &runtime_id, snapshot);
if let Some(current) = workers.get_mut(&runtime_id) {
for worker in current.values_mut() {
let worker_ref =
RuntimeWorkerRef::new(&runtime_id, worker.worker_id.as_str());
let revision = next_revision(&mut revisions, &worker_ref);
worker.subject_revision = revision;
if send_event(
&outbound,
&subscription_id,
revision,
SubscriptionEventPayload::WorkerUpserted {
worker: worker.clone(),
},
)
loop {
match events.recv().await {
Ok(event) if event.revision <= snapshot_revision => continue,
Ok(event) => {
for change in event.changes {
let payload = match change {
crate::worker_projection::WorkerProjectionChange::Upsert(record) => {
let Some(mut worker) = project_registry_worker(&api, record) else {
continue;
};
worker.subject_revision = event.revision;
SubscriptionEventPayload::WorkerUpserted { worker }
}
crate::worker_projection::WorkerProjectionChange::Removed(worker) => {
let Ok(worker_id) =
protocol::subscription::SubscriptionWorkerId::new(worker.worker_id)
else {
continue;
};
SubscriptionEventPayload::WorkerRemoved {
worker_id,
runtime_id: Some(worker.runtime_id),
}
}
};
if send_event(&outbound, &subscription_id, event.revision, payload)
.await
.is_err()
{
return;
}
{
return;
}
}
}
BrokerSubscriptionEvent::Event { payload, .. } => match payload {
SubscriptionEventPayload::WorkerUpserted { mut worker } => {
worker.runtime_id = Some(runtime_id.clone());
let Ok(Some(resource_key)) = api.store.resource_key(
&api.config.workspace_id,
WorkspaceResourceKind::Worker,
worker.worker_id.as_str(),
) else {
continue;
};
worker.resource_key = Some(resource_key);
if !project_repository_key(&api, &mut worker) {
continue;
}
let worker_ref = RuntimeWorkerRef::new(&runtime_id, worker.worker_id.as_str());
let revision = next_revision(&mut revisions, &worker_ref);
worker.subject_revision = revision;
workers
.entry(runtime_id)
.or_default()
.insert(worker.worker_id.to_string(), worker.clone());
if send_event(
&outbound,
&subscription_id,
revision,
SubscriptionEventPayload::WorkerUpserted { worker },
)
.await
.is_err()
{
return;
}
}
SubscriptionEventPayload::WorkerRemoved { worker_id, .. } => {
workers
.entry(runtime_id.clone())
.or_default()
.remove(worker_id.as_str());
let worker_ref = RuntimeWorkerRef::new(&runtime_id, worker_id.as_str());
let revision = next_revision(&mut revisions, &worker_ref);
if send_event(
&outbound,
&subscription_id,
revision,
SubscriptionEventPayload::WorkerRemoved {
worker_id,
runtime_id: Some(runtime_id),
},
)
.await
.is_err()
{
return;
}
}
_ => {}
},
BrokerSubscriptionEvent::Disconnected { .. } => {}
BrokerSubscriptionEvent::Rejected { code, message, .. } => {
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
let _ = send_frame(
&outbound,
SubscriptionFrame::new(SubscriptionFramePayload::Event(
SubscriptionEvent::SubscriptionClosed {
subscription_id: subscription_id.clone(),
code: rejection_termination(code),
message,
},
)),
)
.await;
return;
}
BrokerSubscriptionEvent::Closed { code, message, .. } => {
let _ = send_frame(
&outbound,
SubscriptionFrame::new(SubscriptionFramePayload::Event(
SubscriptionEvent::SubscriptionClosed {
subscription_id: subscription_id.clone(),
code,
message,
code: protocol::subscription::SubscriptionTerminationCode::Lagged,
message: "durable Worker projection subscriber lagged; reconnect for a fresh snapshot".to_string(),
},
)),
)
.await;
return;
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
}
}
}
fn install_snapshot(
fn project_registry_worker(
api: &WorkspaceApi,
workers: &mut HashMap<String, BTreeMap<String, SubscriptionWorker>>,
runtime_id: &str,
snapshot: SubscriptionSnapshot,
) {
let SubscriptionSnapshot::Workers {
workers: snapshot_workers,
} = snapshot
else {
return;
};
let mut projected = BTreeMap::new();
for mut worker in snapshot_workers {
worker.runtime_id = Some(runtime_id.to_string());
let Ok(Some(resource_key)) = api.store.resource_key(
&api.config.workspace_id,
WorkspaceResourceKind::Worker,
worker.worker_id.as_str(),
) else {
continue;
};
worker.resource_key = Some(resource_key);
if !project_repository_key(api, &mut worker) {
continue;
record: crate::store::WorkerRegistryProjectionRecord,
) -> Option<SubscriptionWorker> {
let runtime_id = record.registry.worker.runtime_id.clone();
let mut worker = if let Some(observation) = record.observation {
let mut worker = observation.worker;
worker.availability = observation.availability;
worker.subject_revision = observation.projection_revision;
worker
} else {
SubscriptionWorker {
worker_id: protocol::subscription::SubscriptionWorkerId::new(
record.registry.worker.worker_id.clone(),
)
.ok()?,
runtime_id: Some(runtime_id.clone()),
resource_key: record.resource_key.clone(),
availability: protocol::subscription::SubscriptionWorkerAvailability::Unavailable,
subject_revision: 0,
worker_state: None,
state: protocol::subscription::SubscriptionWorkerState::Stopped,
has_running_internal_workers: false,
workspace_id: Some(record.registry.workspace_id.clone()),
display_name: Some(record.registry.display_name.clone()),
profile: record.registry.profile.clone(),
repository_id: None,
repository_key: None,
working_directory_id: None,
}
projected.insert(worker.worker_id.to_string(), worker);
};
worker.runtime_id = Some(runtime_id);
worker.resource_key = record.resource_key;
worker.workspace_id = Some(record.registry.workspace_id);
worker.display_name = Some(record.registry.display_name);
worker.profile = record.registry.profile;
if !project_repository_key(api, &mut worker) {
return None;
}
workers.insert(runtime_id.to_string(), projected);
Some(worker)
}
fn project_repository_key(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -> bool {
let Some(repository_id) = worker.repository_id.take() else {
worker.repository_key = None;
return true;
};
let Ok(Some(repository)) = api
@@ -549,6 +444,14 @@ fn project_repository_key(api: &WorkspaceApi, worker: &mut SubscriptionWorker) -
true
}
fn sort_workers(workers: &mut [SubscriptionWorker]) {
workers.sort_by(|left, right| {
left.runtime_id
.cmp(&right.runtime_id)
.then_with(|| left.worker_id.cmp(&right.worker_id))
});
}
async fn send_event(
outbound: &mpsc::Sender<WsMessage>,
subscription_id: &SubscriptionId,
@@ -586,23 +489,3 @@ async fn send_frame(
.await
.map_err(|_| ())
}
fn next_revision(revisions: &mut HashMap<RuntimeWorkerRef, u64>, worker: &RuntimeWorkerRef) -> u64 {
let revision = revisions.entry(worker.clone()).or_insert(0);
*revision = revision.saturating_add(1);
*revision
}
fn sort_workers(workers: &mut [SubscriptionWorker]) {
workers.sort_by(|left, right| {
left.runtime_id
.cmp(&right.runtime_id)
.then_with(|| left.worker_id.cmp(&right.worker_id))
});
}
fn rejection_termination(code: SubscriptionRejectionCode) -> SubscriptionTerminationCode {
match code {
SubscriptionRejectionCode::Unauthorized => SubscriptionTerminationCode::Unauthorized,
SubscriptionRejectionCode::ResourceNotFound => SubscriptionTerminationCode::ResourceGone,
_ => SubscriptionTerminationCode::ServerShutdown,
}
}