feat(drive): add durable Workspace document storage and lifecycle
Use object_store 0.13.2 immutable local blobs and FeatureStorage SQLite trees, revision CAS, same-transaction receipts, bounded reads/search, staged/orphan GC and retryable deletion. Attach Server authority to serialize cached Drive operations with deletion reservation. Document fsync compensation, Linux guarantees and offline DB-plus-blob restore. Validation: root cargo check; workspace-drive 29 unit + 12 integration tests; feature-storage 13 tests; workspace-server 649 library + 11 binary tests with test-threads=4; cargo fmt --all -- --check; git diff --check HEAD. Default-parallel Server run hit the existing ticket_checker_burst timeout assertion (648 passed); the isolated filter and bounded-parallel full suite passed. Fault tests are not physical power-cut evidence.
This commit is contained in:
@@ -17,6 +17,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
futures.workspace = true
|
||||
config-source.workspace = true
|
||||
feature-storage.workspace = true
|
||||
workspace-drive.workspace = true
|
||||
subjektiv.workspace = true
|
||||
flow = { path = "../flow" }
|
||||
manifest.workspace = true
|
||||
|
||||
@@ -104,13 +104,10 @@ impl ResolvedWorkspaceBackendConfig {
|
||||
host_config: &ServerHostConfigFile,
|
||||
) -> Result<Self> {
|
||||
let workspace_root = workspace_root.as_ref();
|
||||
let data_root = ServerConfig::default_workspace_backend_data_root(&workspace.workspace_id);
|
||||
let database_path = ServerConfig::default_server_database_path();
|
||||
let (browser_public_url, browser_rp_id) =
|
||||
resolve_browser_public_url(&host_config.browser.public_url)?;
|
||||
let mut server = ServerConfig::local_dev(workspace_root.to_path_buf(), workspace);
|
||||
server.database_path = database_path.clone();
|
||||
server.embedded_runtime_store_root = data_root.join("embedded-runtime");
|
||||
let database_path = server.database_path.clone();
|
||||
server.max_records = DEFAULT_MAX_RECORDS;
|
||||
server.remote_runtime_sources = Vec::new();
|
||||
server.auth = AuthConfig::Passkey {
|
||||
|
||||
@@ -36,6 +36,7 @@ mod worker_projection;
|
||||
pub mod worker_source;
|
||||
pub mod workspace_catalog;
|
||||
mod workspace_deletion;
|
||||
mod workspace_drive_host;
|
||||
pub mod workspace_signing_identity;
|
||||
mod workspace_subscription;
|
||||
|
||||
@@ -151,6 +152,8 @@ pub enum Error {
|
||||
WorkerSourceIdentity(String),
|
||||
#[error("Workspace signing identity error ({code}): {message}")]
|
||||
WorkspaceSigningIdentity { code: String, message: String },
|
||||
#[error("Workspace Drive error: {0}")]
|
||||
Drive(#[from] workspace_drive::Error),
|
||||
#[error("store error: {0}")]
|
||||
Store(String),
|
||||
}
|
||||
|
||||
@@ -252,6 +252,9 @@ pub struct ServerConfig {
|
||||
pub workspace_created_at: String,
|
||||
pub workspace_execution_root: PathBuf,
|
||||
pub database_path: PathBuf,
|
||||
/// Resolved Yoi data directory, independent of the authority database location.
|
||||
/// Custom database deployments must explicitly configure this path as well.
|
||||
pub data_root: PathBuf,
|
||||
pub embedded_runtime_store_root: PathBuf,
|
||||
pub static_assets_dir: Option<PathBuf>,
|
||||
pub auth: AuthConfig,
|
||||
@@ -268,14 +271,22 @@ impl ServerConfig {
|
||||
workspace: WorkspaceRecord,
|
||||
) -> Self {
|
||||
let workspace_id = workspace.workspace_id.clone();
|
||||
let embedded_runtime_store_root = Self::default_embedded_runtime_store_root(&workspace_id);
|
||||
let database_path = Self::default_server_database_path();
|
||||
// Reuse the startup-resolved path; Drive never reads the environment.
|
||||
let data_root = database_path
|
||||
.parent()
|
||||
.and_then(Path::parent)
|
||||
.expect("default Server database is below the resolved data directory")
|
||||
.to_path_buf();
|
||||
let embedded_runtime_store_root =
|
||||
Self::embedded_runtime_store_root_for_data_dir(&data_root, &workspace_id);
|
||||
Self {
|
||||
workspace_id: workspace.workspace_id,
|
||||
workspace_display_name: workspace.display_name,
|
||||
workspace_created_at: workspace.created_at,
|
||||
workspace_execution_root: workspace_execution_root.into(),
|
||||
database_path,
|
||||
data_root,
|
||||
embedded_runtime_store_root,
|
||||
static_assets_dir: None,
|
||||
auth: AuthConfig::Passkey {
|
||||
@@ -345,6 +356,15 @@ impl ServerConfig {
|
||||
Self::default_workspace_backend_data_root(workspace_id).join("embedded-runtime")
|
||||
}
|
||||
|
||||
pub fn with_data_root(mut self, root: impl Into<PathBuf>) -> Self {
|
||||
self.data_root = root.into();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn drive_blob_root(&self) -> Result<PathBuf> {
|
||||
crate::workspace_drive_host::drive_blob_root(&self.data_root, &self.workspace_id)
|
||||
}
|
||||
|
||||
pub fn with_embedded_runtime_store_root(mut self, root: impl Into<PathBuf>) -> Self {
|
||||
self.embedded_runtime_store_root = root.into();
|
||||
self
|
||||
@@ -1554,6 +1574,7 @@ pub struct WorkspaceApi {
|
||||
config_schema_registry: crate::config_source::WorkspaceConfigSchemaRegistry,
|
||||
prompt_projection_cache: crate::prompt_settings::WorkspacePromptProjectionCache,
|
||||
feature_storage: crate::WorkspaceFeatureStorage,
|
||||
drive_host: Arc<crate::workspace_drive_host::WorkspaceDriveHost>,
|
||||
subjektiv_registration: crate::RegisteredFeature,
|
||||
authority: SqliteWorkspaceAuthority,
|
||||
_worker_projection_shutdown: Arc<WorkerProjectionShutdownGuard>,
|
||||
@@ -2334,6 +2355,7 @@ impl WorkspaceServerApi {
|
||||
.await?
|
||||
.ok_or_else(|| Error::InvalidInput("Workspace no longer exists".to_string()))?;
|
||||
|
||||
api.fence_drive().await?;
|
||||
let mut child_operation_ids = operation.child_operation_ids.clone();
|
||||
let mut blockers = Vec::new();
|
||||
for worker in self
|
||||
@@ -2428,11 +2450,18 @@ impl WorkspaceServerApi {
|
||||
None,
|
||||
);
|
||||
}
|
||||
api.feature_storage.delete().map_err(|error| {
|
||||
Error::Store(format!(
|
||||
"failed to delete Workspace Feature storage: {error}"
|
||||
))
|
||||
})?;
|
||||
api.purge_drive().await?;
|
||||
let feature_storage = api.feature_storage.clone();
|
||||
tokio::task::spawn_blocking(move || feature_storage.delete())
|
||||
.await
|
||||
.map_err(|error| {
|
||||
Error::Store(format!("Feature storage deletion task failed: {error}"))
|
||||
})?
|
||||
.map_err(|error| {
|
||||
Error::Store(format!(
|
||||
"failed to delete Workspace Feature storage: {error}"
|
||||
))
|
||||
})?;
|
||||
WorkspaceSigningIdentityService::new(self.store.clone(), self.signing_materials.clone())
|
||||
.delete_material(&operation.workspace_id)?;
|
||||
let completed = self.store.finalize_workspace_deletion(operation_id)?;
|
||||
@@ -3201,6 +3230,55 @@ fn embedded_runtime_request_audience(config: &ServerConfig) -> crate::Result<Str
|
||||
}
|
||||
|
||||
impl WorkspaceApi {
|
||||
/// Trusted Workspace-scoped Drive access for backend adapters. This is not
|
||||
/// model authorization; callers must authorize requests before using it.
|
||||
/// All synchronous Drive operations on the returned handle must run through
|
||||
/// `tokio::task::spawn_blocking` when called from an async context.
|
||||
/// Drive transactions serialize with server DB writes so deletion reservation
|
||||
/// fences even previously returned handles at its authority commit.
|
||||
pub async fn drive(&self) -> Result<workspace_drive::Drive> {
|
||||
let workspace = self
|
||||
.store
|
||||
.get_workspace(&self.config.workspace_id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::WorkspaceConfigConflict("Workspace no longer exists".into()))?;
|
||||
if workspace.state != "active" {
|
||||
if workspace.state == "deleting" {
|
||||
self.fence_drive().await?;
|
||||
}
|
||||
return Err(Error::WorkspaceConfigConflict(
|
||||
"Workspace is not active".into(),
|
||||
));
|
||||
}
|
||||
self.drive_host
|
||||
.open(
|
||||
&self.feature_storage,
|
||||
self.config.drive_blob_root()?,
|
||||
self.config.database_path.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn fence_drive(&self) -> Result<()> {
|
||||
self.drive_host
|
||||
.fence(
|
||||
&self.feature_storage,
|
||||
self.config.drive_blob_root()?,
|
||||
self.config.database_path.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn purge_drive(&self) -> Result<()> {
|
||||
self.drive_host
|
||||
.purge(
|
||||
&self.feature_storage,
|
||||
self.config.drive_blob_root()?,
|
||||
self.config.database_path.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Server-managed, Workspace-scoped storage for trusted Feature repositories.
|
||||
/// Feature registration and SQL access stay inside the Server process.
|
||||
pub fn feature_storage(&self) -> &crate::WorkspaceFeatureStorage {
|
||||
@@ -3472,7 +3550,10 @@ impl WorkspaceApi {
|
||||
.map_err(|error| {
|
||||
Error::Store(format!("failed to register subjektiv storage: {error}"))
|
||||
})?;
|
||||
let drive_host =
|
||||
Arc::new(crate::workspace_drive_host::WorkspaceDriveHost::new(&feature_storage).await?);
|
||||
let mut api = Self {
|
||||
drive_host,
|
||||
config_store,
|
||||
repository_secrets,
|
||||
signing_identities,
|
||||
@@ -3510,6 +3591,16 @@ impl WorkspaceApi {
|
||||
worker_control_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||
attachment_upload_grants: Arc::new(Mutex::new(HashMap::new())),
|
||||
};
|
||||
// Restore the domain fence before this API can issue any Drive handle,
|
||||
// including blocked/failed deletions not scheduled for automatic recovery.
|
||||
if api
|
||||
.store
|
||||
.get_workspace(&api.config.workspace_id)
|
||||
.await?
|
||||
.is_some_and(|workspace| workspace.state == "deleting")
|
||||
{
|
||||
api.fence_drive().await?;
|
||||
}
|
||||
// The observer snapshot retains the unconfigured authority, not itself:
|
||||
// source observation uses Repository access + Runtime authority without a cycle.
|
||||
let observer_api = api.clone();
|
||||
@@ -7265,6 +7356,19 @@ impl server_api::ServerApi for ServerApiContractService {
|
||||
}
|
||||
let operation = reservation.operation;
|
||||
if operation.state != WorkspaceDeletionState::Succeeded {
|
||||
let workspace_api = api
|
||||
.api_for_workspace(&operation.workspace_id)
|
||||
.await
|
||||
.map_err(|error| ApiError::from(error).into_repository_api_error())?
|
||||
.ok_or_else(|| {
|
||||
ApiError::from(Error::InvalidInput("Workspace no longer exists".into()))
|
||||
.into_repository_api_error()
|
||||
})?;
|
||||
// Persist before returning deletion start or scheduling resource cleanup.
|
||||
workspace_api
|
||||
.fence_drive()
|
||||
.await
|
||||
.map_err(|error| ApiError::from(error).into_repository_api_error())?;
|
||||
api.schedule_workspace_deletion(request.operation_id).await;
|
||||
}
|
||||
Ok(operation)
|
||||
@@ -32220,6 +32324,7 @@ impl IntoResponse for ApiError {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod auth_logging_tests;
|
||||
mod drive_tests;
|
||||
mod subject_spawn_tests;
|
||||
mod subjektiv_jobs_tests;
|
||||
mod ticket_evidence_tests;
|
||||
@@ -42803,6 +42908,7 @@ mod tests {
|
||||
let mut config = ServerConfig::local_dev(workspace_root.clone(), test_workspace())
|
||||
.with_embedded_runtime_store_root(store_root);
|
||||
config.database_path = workspace_root.join(".test-yoi-server.db");
|
||||
config.data_root = workspace_root.join(".test-yoi-data");
|
||||
config.backend_base_url = Some("http://127.0.0.1:8787".to_string());
|
||||
let source = server_api::RepositorySource {
|
||||
kind: server_api::RepositorySourceKind::LocalPath,
|
||||
@@ -57778,6 +57884,7 @@ mod tests {
|
||||
let mut config = ServerConfig::local_dev(workspace_root, test_workspace())
|
||||
.with_embedded_runtime_store_root(default_root.clone());
|
||||
config.database_path = ServerConfig::server_database_path_for_data_dir(&data_dir);
|
||||
config.data_root = data_dir.clone();
|
||||
let store = test_control_store(&config);
|
||||
let app = build_inner_router(
|
||||
WorkspaceApi::new_with_execution_backend(
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
use super::*;
|
||||
use workspace_drive::{Error as DriveError, Mutation};
|
||||
|
||||
#[test]
|
||||
fn custom_database_does_not_change_explicit_drive_placement() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(temp.path());
|
||||
config.database_path = temp.path().join("custom-authority/authority.sqlite");
|
||||
config.data_root = temp.path().join("explicit-yoi-data");
|
||||
let mut workspace = test_workspace();
|
||||
workspace.workspace_id = "other-internal-id".into();
|
||||
workspace.display_name = "../Not a storage key".into();
|
||||
let scoped = config
|
||||
.for_catalog_workspace(&workspace, Vec::new())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
scoped.drive_blob_root().unwrap(),
|
||||
config.data_root.join("drives/other-internal-id/blobs")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cached_drive_handle_rejects_mutation_after_deletion_reservation() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(temp.path());
|
||||
config.repositories.clear();
|
||||
let store = Arc::new(test_control_store(&config));
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
let owner = format!("account-{}", config.workspace_id);
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "retained-workspace".into(),
|
||||
owner_account_id: owner.clone(),
|
||||
display_name: "Retain".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let api = WorkspaceApi::new(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let cached = api.drive().await.unwrap();
|
||||
let for_root = cached.clone();
|
||||
let parent = tokio::task::spawn_blocking(move || for_root.root().unwrap().id)
|
||||
.await
|
||||
.unwrap();
|
||||
let preflight = store
|
||||
.workspace_deletion_preflight(&owner, &config.workspace_id)
|
||||
.unwrap();
|
||||
store
|
||||
.reserve_workspace_deletion(
|
||||
&owner,
|
||||
&config.workspace_id,
|
||||
&WorkspaceDeletionRequest {
|
||||
operation_id: "delete-cached-drive".into(),
|
||||
expected_revision: preflight.expected_revision,
|
||||
confirmation: preflight.display_name,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
// No WorkspaceApi access or host/domain fence between reservation and the
|
||||
// cached operation: the committed server authority must reject it directly.
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
cached.mutate(
|
||||
"cached-write",
|
||||
"owner",
|
||||
Mutation::CreateFolder {
|
||||
parent,
|
||||
name: "forbidden".into(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(result, Err(DriveError::Fenced)), "{result:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deleting_workspace_reopens_fenced_drive_and_purges_blobs_before_feature_storage() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(temp.path());
|
||||
config.repositories.clear();
|
||||
let store = Arc::new(test_control_store(&config));
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
let owner = format!("account-{}", config.workspace_id);
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "retained-workspace".into(),
|
||||
owner_account_id: owner.clone(),
|
||||
display_name: "Retain".into(),
|
||||
state: "active".into(),
|
||||
created_at: "1".into(),
|
||||
updated_at: "1".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let root = config.drive_blob_root().unwrap();
|
||||
// Use a separate embedded-runtime namespace for the first API: this test
|
||||
// reconstructs Drive authority in-process rather than spawning/restarting
|
||||
// product processes (runtime observers may retain their namespace).
|
||||
let first_api = WorkspaceApi::new(config.clone(), store.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
// Merely constructing a Workspace API must not create blob storage.
|
||||
assert!(!root.exists());
|
||||
let drive = first_api.drive().await.unwrap();
|
||||
let parent = tokio::task::spawn_blocking(move || {
|
||||
let parent = drive.root().unwrap().id;
|
||||
drive
|
||||
.mutate(
|
||||
"upload",
|
||||
"owner",
|
||||
Mutation::CreateFile {
|
||||
parent,
|
||||
name: "file".into(),
|
||||
content_type: "text/plain".into(),
|
||||
bytes: b"content".to_vec(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
parent
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let preflight = store
|
||||
.workspace_deletion_preflight(&owner, &config.workspace_id)
|
||||
.unwrap();
|
||||
store
|
||||
.reserve_workspace_deletion(
|
||||
&owner,
|
||||
&config.workspace_id,
|
||||
&WorkspaceDeletionRequest {
|
||||
operation_id: "delete-drive-workspace".into(),
|
||||
expected_revision: preflight.expected_revision,
|
||||
confirmation: preflight.display_name,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
// Blocked operations are not automatically resumed on restart, but their
|
||||
// Workspace remains deleting and must still reconstruct a fenced Drive.
|
||||
store
|
||||
.update_workspace_deletion_operation(
|
||||
"delete-drive-workspace",
|
||||
WorkspaceDeletionState::Blocked,
|
||||
&[],
|
||||
&[],
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
// Simulate a crash between the catalog reservation and Drive fencing.
|
||||
let feature_storage = first_api.feature_storage.clone();
|
||||
tokio::task::spawn_blocking(move || feature_storage.shutdown())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
drop(first_api);
|
||||
let recovered = WorkspaceServerApi::new(config.clone(), store.clone());
|
||||
let api = recovered
|
||||
.api_for_workspace(&config.workspace_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
api.drive().await,
|
||||
Err(Error::WorkspaceConfigConflict(_))
|
||||
));
|
||||
let reopened = api
|
||||
.drive_host
|
||||
.open(
|
||||
&api.feature_storage,
|
||||
root.clone(),
|
||||
config.database_path.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mutation = tokio::task::spawn_blocking(move || {
|
||||
reopened.mutate(
|
||||
"restart-write",
|
||||
"owner",
|
||||
Mutation::CreateFolder {
|
||||
parent,
|
||||
name: "forbidden".into(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(mutation, Err(DriveError::Fenced)), "{mutation:?}");
|
||||
let completed = recovered
|
||||
.execute_workspace_deletion("delete-drive-workspace")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, WorkspaceDeletionState::Succeeded);
|
||||
assert!(!root.exists());
|
||||
assert!(
|
||||
!config
|
||||
.database_path
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join("feature-storage")
|
||||
.join(&config.workspace_id)
|
||||
.exists()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_workspace("retained-workspace")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
//! Trusted Host placement and lifecycle for Workspace Drive (no HTTP/DTO surface).
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use tokio::sync::Mutex;
|
||||
use workspace_drive::Drive;
|
||||
|
||||
use crate::{Error, Result, WorkspaceFeatureStorage};
|
||||
|
||||
pub(crate) fn drive_blob_root(data_root: &Path, workspace_id: &str) -> Result<PathBuf> {
|
||||
if !data_root.is_absolute() {
|
||||
return Err(Error::Config(
|
||||
"Server data_root must be an absolute path".into(),
|
||||
));
|
||||
}
|
||||
if workspace_id.is_empty()
|
||||
|| workspace_id.len() > 128
|
||||
|| !workspace_id
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
|
||||
{
|
||||
return Err(Error::Config(
|
||||
"Drive requires an internal Workspace storage identifier".into(),
|
||||
));
|
||||
}
|
||||
Ok(data_root.join("drives").join(workspace_id).join("blobs"))
|
||||
}
|
||||
|
||||
// Reject redirected Host-managed ancestors before opening or deleting storage.
|
||||
// These directories are server-owned, not writable by Drive API clients.
|
||||
fn reject_symlink_ancestors(path: &Path) -> Result<()> {
|
||||
for ancestor in path.ancestors() {
|
||||
match std::fs::symlink_metadata(ancestor) {
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => {
|
||||
return Err(Error::Config(
|
||||
"Drive storage path contains a symbolic link".into(),
|
||||
));
|
||||
}
|
||||
Ok(metadata) if !metadata.is_dir() => {
|
||||
return Err(Error::Config(
|
||||
"Drive storage path contains a non-directory".into(),
|
||||
));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) struct WorkspaceDriveHost {
|
||||
registration: crate::RegisteredFeature,
|
||||
purged: AtomicBool,
|
||||
drive: Mutex<Option<Drive>>,
|
||||
}
|
||||
|
||||
impl WorkspaceDriveHost {
|
||||
pub(crate) async fn new(storage: &WorkspaceFeatureStorage) -> Result<Self> {
|
||||
let storage = storage.clone();
|
||||
let registration = tokio::task::spawn_blocking(move || Drive::register(&storage))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive registration task failed: {error}")))?
|
||||
.map_err(|error| Error::Store(format!("failed to register Drive storage: {error}")))?;
|
||||
Ok(Self {
|
||||
registration,
|
||||
purged: AtomicBool::new(false),
|
||||
drive: Mutex::new(None),
|
||||
})
|
||||
}
|
||||
pub(crate) async fn open(
|
||||
&self,
|
||||
storage: &WorkspaceFeatureStorage,
|
||||
root: PathBuf,
|
||||
authority_database: PathBuf,
|
||||
) -> Result<Drive> {
|
||||
let mut cached = self.drive.lock().await;
|
||||
if let Some(drive) = cached.as_ref() {
|
||||
return Ok(drive.clone());
|
||||
}
|
||||
let storage = storage.clone();
|
||||
let registration = self.registration.clone();
|
||||
let drive = tokio::task::spawn_blocking(move || {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain IMMEDIATE transactions lock the attached server authority
|
||||
// as well as Drive metadata. Deletion reservation and cached-handle
|
||||
// operations therefore serialize with other server DB writes.
|
||||
Drive::open_with_workspace_authority(
|
||||
&storage,
|
||||
®istration,
|
||||
&root,
|
||||
&authority_database,
|
||||
)
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive open task failed: {error}")))??;
|
||||
*cached = Some(drive.clone());
|
||||
Ok(drive)
|
||||
}
|
||||
|
||||
pub(crate) async fn fence(
|
||||
&self,
|
||||
storage: &WorkspaceFeatureStorage,
|
||||
root: PathBuf,
|
||||
authority_database: PathBuf,
|
||||
) -> Result<()> {
|
||||
if self.purged.load(Ordering::Acquire) {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root, authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || drive.fence().map_err(Error::from))
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive fence task failed: {error}")))?
|
||||
}
|
||||
|
||||
pub(crate) async fn purge(
|
||||
&self,
|
||||
storage: &WorkspaceFeatureStorage,
|
||||
root: PathBuf,
|
||||
authority_database: PathBuf,
|
||||
) -> Result<()> {
|
||||
if self.purged.load(Ordering::Acquire) {
|
||||
return Ok(());
|
||||
}
|
||||
let drive = self.open(storage, root.clone(), authority_database).await?;
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
reject_symlink_ancestors(&root)?;
|
||||
// Domain purge persists its tombstone before removing blobs/nodes/receipts.
|
||||
drive.purge()?;
|
||||
// Remove the empty blob root, never the data root or another Workspace.
|
||||
// Missing roots are successful retries; other failures retain FeatureStorage.
|
||||
match std::fs::remove_dir_all(&root) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::Store(format!("Drive purge task failed: {error}")))??;
|
||||
// A later signing-material/finalization failure can retry after the
|
||||
// feature DB has been closed and removed by FeatureStorage.delete.
|
||||
self.purged.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use workspace_drive::Mutation;
|
||||
|
||||
fn seed_authority(data_root: &Path) -> PathBuf {
|
||||
let path = data_root.join("server.db");
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE workspaces (workspace_id TEXT PRIMARY KEY, state TEXT NOT NULL);
|
||||
INSERT INTO workspaces VALUES ('workspace-a', 'active');",
|
||||
)
|
||||
.unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_path_uses_explicit_data_root_and_internal_workspace_id() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let data = temp.path().join("data");
|
||||
assert_eq!(
|
||||
drive_blob_root(&data, "workspace-a").unwrap(),
|
||||
data.join("drives/workspace-a/blobs")
|
||||
);
|
||||
for id in [
|
||||
"",
|
||||
".",
|
||||
"..",
|
||||
"../other",
|
||||
"/absolute",
|
||||
"a/b",
|
||||
"a\\b",
|
||||
"display name",
|
||||
] {
|
||||
assert!(drive_blob_root(&data, id).is_err(), "accepted {id:?}");
|
||||
}
|
||||
assert!(drive_blob_root(Path::new("relative"), "workspace-a").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drive_fence_survives_host_reconstruction_before_purge() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let feature_root = temp.path().join("feature-storage");
|
||||
let root = drive_blob_root(temp.path(), "workspace-a").unwrap();
|
||||
let authority = seed_authority(temp.path());
|
||||
let storage = crate::FeatureStorage::new(&feature_root)
|
||||
.workspace("workspace-a")
|
||||
.unwrap();
|
||||
let host = WorkspaceDriveHost::new(&storage).await.unwrap();
|
||||
let drive = host
|
||||
.open(&storage, root.clone(), authority.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let parent = tokio::task::spawn_blocking(move || drive.root().unwrap().id)
|
||||
.await
|
||||
.unwrap();
|
||||
host.fence(&storage, root.clone(), authority.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
drop(host);
|
||||
drop(storage);
|
||||
let storage = crate::FeatureStorage::new(&feature_root)
|
||||
.workspace("workspace-a")
|
||||
.unwrap();
|
||||
let host = WorkspaceDriveHost::new(&storage).await.unwrap();
|
||||
let reopened = host.open(&storage, root, authority.clone()).await.unwrap();
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
reopened.mutate(
|
||||
"after-restart",
|
||||
"owner",
|
||||
Mutation::CreateFolder {
|
||||
parent,
|
||||
name: "forbidden".into(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(result, Err(workspace_drive::Error::Fenced)),
|
||||
"{result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drive_purge_removes_only_its_blob_root_and_retries_after_feature_delete() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let feature_root = temp.path().join("feature-storage");
|
||||
let root = drive_blob_root(temp.path(), "workspace-a").unwrap();
|
||||
let authority = seed_authority(temp.path());
|
||||
let other_root = drive_blob_root(temp.path(), "workspace-b").unwrap();
|
||||
std::fs::create_dir_all(&other_root).unwrap();
|
||||
std::fs::write(other_root.join("keep"), b"other workspace").unwrap();
|
||||
let storage = crate::FeatureStorage::new(&feature_root)
|
||||
.workspace("workspace-a")
|
||||
.unwrap();
|
||||
let host = WorkspaceDriveHost::new(&storage).await.unwrap();
|
||||
let drive = host
|
||||
.open(&storage, root.clone(), authority.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let parent = drive.root().unwrap().id;
|
||||
drive
|
||||
.mutate(
|
||||
"upload",
|
||||
"owner",
|
||||
Mutation::CreateFile {
|
||||
parent,
|
||||
name: "file".into(),
|
||||
content_type: "text/plain".into(),
|
||||
bytes: b"content".to_vec(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
host.purge(&storage, root.clone(), authority.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!root.exists());
|
||||
assert_eq!(
|
||||
std::fs::read(other_root.join("keep")).unwrap(),
|
||||
b"other workspace"
|
||||
);
|
||||
let feature_exists = feature_root.join("workspace-a/features/workspace-drive.sqlite");
|
||||
assert!(
|
||||
feature_exists.exists(),
|
||||
"purge must retain the fenced DB: {}",
|
||||
feature_exists.display()
|
||||
);
|
||||
let storage_for_delete = storage.clone();
|
||||
tokio::task::spawn_blocking(move || storage_for_delete.delete())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
host.fence(&storage, root.clone(), authority.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
host.purge(&storage, root.clone(), authority.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!root.exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn redirected_drive_root_blocks_cleanup_and_preserves_external_files() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = drive_blob_root(temp.path(), "workspace-a").unwrap();
|
||||
let authority = seed_authority(temp.path());
|
||||
let outside = temp.path().join("outside");
|
||||
std::fs::create_dir_all(&outside).unwrap();
|
||||
std::fs::write(outside.join("keep"), b"external").unwrap();
|
||||
let storage = crate::FeatureStorage::new(temp.path().join("feature-storage"))
|
||||
.workspace("workspace-a")
|
||||
.unwrap();
|
||||
let host = WorkspaceDriveHost::new(&storage).await.unwrap();
|
||||
host.fence(&storage, root.clone(), authority.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
std::os::unix::fs::symlink(&outside, &root).unwrap();
|
||||
assert!(host.purge(&storage, root, authority).await.is_err());
|
||||
assert_eq!(std::fs::read(outside.join("keep")).unwrap(), b"external");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user