From 5b1b0688bc69cc4f4a93bd79e2e396fcd7b524ac Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 26 Jul 2026 11:10:00 +0900 Subject: [PATCH] objective: move records to sqlite authority --- crates/workspace-server/src/lib.rs | 5 +- crates/workspace-server/src/main.rs | 107 ++++- .../workspace-server/src/objective_import.rs | 260 ++++++++++++ crates/workspace-server/src/records.rs | 170 ++++---- crates/workspace-server/src/server.rs | 62 +++ crates/workspace-server/src/store.rs | 383 +++++++++++++++++- ...jectives-and-staging-to-sqlite-20260726.sh | 66 +++ 7 files changed, 959 insertions(+), 94 deletions(-) create mode 100644 crates/workspace-server/src/objective_import.rs create mode 100755 scripts/import-objectives-and-staging-to-sqlite-20260726.sh diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index e0988e99..460502e0 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -1,8 +1,8 @@ //! Local workspace web control plane backend bootstrap. //! //! This crate deliberately provides backend building blocks and an HTTP router; -//! it is not the product CLI facade. Existing `.yoi` Ticket and Objective files -//! remain the canonical project records and are read through bounded bridge APIs. +//! it is not the product CLI facade. Tickets and Objectives are served through +//! Backend authority surfaces rather than Worker-local filesystem access. pub mod auth; pub mod companion; @@ -10,6 +10,7 @@ pub mod config; pub mod hosts; pub mod identity; pub mod memory_staging; +pub mod objective_import; pub mod observation; pub mod profile_settings; pub mod records; diff --git a/crates/workspace-server/src/main.rs b/crates/workspace-server/src/main.rs index 2202d5f7..51a14b80 100644 --- a/crates/workspace-server/src/main.rs +++ b/crates/workspace-server/src/main.rs @@ -17,6 +17,7 @@ enum Command { Init(InitOptions), ConfigDefault, ConfigDiff(WorkspacePathOptions), + ImportObjectives(ImportObjectivesOptions), Skills(SkillsCommand), Help, } @@ -36,6 +37,12 @@ struct WorkspacePathOptions { workspace: PathBuf, } +#[derive(Debug)] +struct ImportObjectivesOptions { + workspace: PathBuf, + database: Option, +} + #[derive(Debug)] enum SkillsCommand { List(WorkspacePathOptions), @@ -72,6 +79,7 @@ async fn run() -> Result<(), Box> { Command::Init(options) => run_init(options).await, Command::ConfigDefault => run_config_default(), Command::ConfigDiff(options) => run_config_diff(options), + Command::ImportObjectives(options) => run_import_objectives(options).await, Command::Skills(command) => run_skills(command), Command::Help => Ok(()), } @@ -92,6 +100,15 @@ fn parse_command(args: &[String]) -> Result { Ok(Command::Init(parse_init_options(rest)?)) } "config" => parse_config_command(rest), + "import-objectives" => { + if rest.iter().any(|arg| arg == "--help" || arg == "-h") { + print_import_objectives_help(); + return Ok(Command::Help); + } + Ok(Command::ImportObjectives(parse_import_objectives_options( + rest, + )?)) + } "skills" => parse_skills_command(rest), "serve" => { if rest.iter().any(|arg| arg == "--help" || arg == "-h") { @@ -105,7 +122,7 @@ fn parse_command(args: &[String]) -> Result { Ok(Command::Help) } other => Err(CliError(format!( - "unknown command `{other}`; expected `init`, `config`, `skills`, or `serve`" + "unknown command `{other}`; expected `init`, `config`, `import-objectives`, `skills`, or `serve`" ))), } } @@ -169,6 +186,45 @@ fn run_config_diff(options: WorkspacePathOptions) -> Result<(), Box Result<(), Box> { + let identity = WorkspaceIdentity::load_required(&options.workspace)?; + let database_path = options + .database + .unwrap_or_else(ServerConfig::default_server_database_path); + let store = SqliteWorkspaceStore::open(&database_path)?; + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: identity.workspace_id.clone(), + owner_account_id: None, + display_name: identity.display_name.clone(), + state: "active".to_string(), + created_at: identity.created_at.clone(), + updated_at: chrono::Utc::now().to_rfc3339(), + }) + .await?; + let report = + yoi_workspace_server::objective_import::import_legacy_objectives_and_memory_staging( + &options.workspace, + &identity.workspace_id, + &store, + )?; + println!( + "imported objectives={} resources={} ticket_links={} skipped_ticket_links={} memory_staging={} invalid_records={}", + report.objectives_imported, + report.objective_resources_imported, + report.objective_ticket_links_imported, + report.objective_ticket_links_skipped, + report.memory_staging_records_imported, + report.invalid_records.len(), + ); + for invalid in report.invalid_records { + eprintln!("invalid objective record: {invalid}"); + } + Ok(()) +} + fn run_skills(command: SkillsCommand) -> Result<(), Box> { match command { SkillsCommand::List(options) => { @@ -377,6 +433,47 @@ fn parse_workspace_path_options(args: &[String]) -> Result Result { + let mut workspace = std::env::current_dir() + .map_err(|error| CliError(format!("failed to read current dir: {error}")))?; + let mut database = None; + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + match arg.as_str() { + "--workspace" => { + let value = iter + .next() + .ok_or_else(|| CliError("--workspace requires a path".to_string()))?; + workspace = PathBuf::from(value); + } + value if value.starts_with("--workspace=") => { + workspace = PathBuf::from(value_after_equals(arg, "--workspace")?); + } + "--database" => { + let value = iter + .next() + .ok_or_else(|| CliError("--database requires a path".to_string()))?; + database = Some(PathBuf::from(value)); + } + value if value.starts_with("--database=") => { + database = Some(PathBuf::from(value_after_equals(arg, "--database")?)); + } + other => { + return Err(CliError(format!( + "unknown import-objectives option `{other}`" + ))); + } + } + } + let workspace = workspace + .canonicalize() + .map_err(|error| CliError(format!("failed to canonicalize workspace: {error}")))?; + Ok(ImportObjectivesOptions { + workspace, + database, + }) +} + fn parse_init_options(args: &[String]) -> Result { let mut workspace = std::env::current_dir() .map_err(|error| CliError(format!("failed to read current dir: {error}")))?; @@ -453,7 +550,7 @@ fn parse_listen(value: &str) -> Result { fn print_help() { println!( - "yoi-workspace-server\n\nUsage:\n yoi-workspace-server init [OPTIONS]\n yoi-workspace-server config [OPTIONS]\n yoi-workspace-server skills [OPTIONS]\n yoi-workspace-server serve [OPTIONS]\n\nOptions:\n -h, --help Print help" + "yoi-workspace-server\n\nUsage:\n yoi-workspace-server init [OPTIONS]\n yoi-workspace-server config [OPTIONS]\n yoi-workspace-server import-objectives [OPTIONS]\n yoi-workspace-server skills [OPTIONS]\n yoi-workspace-server serve [OPTIONS]\n\nOptions:\n -h, --help Print help" ); } @@ -469,6 +566,12 @@ fn print_config_help() { ); } +fn print_import_objectives_help() { + println!( + "yoi-workspace-server import-objectives\n\nUsage:\n yoi-workspace-server import-objectives [OPTIONS]\n\nDescription:\n Imports legacy .yoi/objectives records and .yoi/memory/_staging JSON files into the Workspace SQLite authority.\n\nOptions:\n --workspace Workspace root (defaults to cwd)\n --database Server SQLite DB path (defaults to canonical server DB)\n -h, --help Print help" + ); +} + fn print_skills_help() { println!( "yoi-workspace-server skills\n\nUsage:\n yoi-workspace-server skills list [OPTIONS]\n yoi-workspace-server skills lint [OPTIONS]\n yoi-workspace-server skills show [OPTIONS]\n\nDescription:\n Uses the Workspace backend Skill catalog/lint/detail authority. Catalog output is lightweight and omits full SKILL.md bodies; detail output includes the body. allowed-tools and scripts are diagnostics only.\n\nOptions:\n --workspace Workspace root (defaults to cwd)\n -h, --help Print help" diff --git a/crates/workspace-server/src/objective_import.rs b/crates/workspace-server/src/objective_import.rs new file mode 100644 index 00000000..4d9a1e07 --- /dev/null +++ b/crates/workspace-server/src/objective_import.rs @@ -0,0 +1,260 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use chrono::Utc; +use project_record::validate_record_id; +use serde::Deserialize; + +use crate::store::{ + ControlPlaneStore, MemoryStagingRecord, ObjectiveRecord, ObjectiveResourceRecord, + ObjectiveTicketLinkRecord, +}; +use crate::{Error, Result}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObjectiveImportReport { + pub objectives_imported: usize, + pub objective_resources_imported: usize, + pub objective_ticket_links_imported: usize, + pub objective_ticket_links_skipped: usize, + pub memory_staging_records_imported: usize, + pub invalid_records: Vec, +} + +#[derive(Debug, Deserialize)] +struct ObjectiveFrontmatter { + title: String, + state: String, + #[serde(default)] + created_at: Option, + #[serde(default)] + updated_at: Option, + #[serde(default)] + linked_tickets: Vec, +} + +pub fn import_legacy_objectives_and_memory_staging( + workspace_root: &Path, + workspace_id: &str, + store: &S, +) -> Result { + let mut report = ObjectiveImportReport { + objectives_imported: 0, + objective_resources_imported: 0, + objective_ticket_links_imported: 0, + objective_ticket_links_skipped: 0, + memory_staging_records_imported: 0, + invalid_records: Vec::new(), + }; + + import_objectives(workspace_root, workspace_id, store, &mut report)?; + import_memory_staging(workspace_root, workspace_id, store, &mut report)?; + Ok(report) +} + +fn import_objectives( + workspace_root: &Path, + workspace_id: &str, + store: &S, + report: &mut ObjectiveImportReport, +) -> Result<()> { + let root = workspace_root.join(".yoi/objectives"); + if !root.exists() { + return Ok(()); + } + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if !path.is_dir() { + continue; + } + let objective_id = entry.file_name().to_string_lossy().to_string(); + if let Err(err) = validate_record_id(&objective_id) { + report + .invalid_records + .push(format!("{objective_id}: invalid objective id: {err}")); + continue; + } + match import_one_objective(&path, workspace_id, &objective_id, store) { + Ok(item) => { + report.objectives_imported += 1; + report.objective_resources_imported += item.resources; + report.objective_ticket_links_imported += item.links; + report.objective_ticket_links_skipped += item.skipped_links; + } + Err(err) => report + .invalid_records + .push(format!("{objective_id}: {err}")), + } + } + Ok(()) +} + +struct ImportedObjectiveCounts { + resources: usize, + links: usize, + skipped_links: usize, +} + +fn import_one_objective( + objective_dir: &Path, + workspace_id: &str, + objective_id: &str, + store: &S, +) -> Result { + let item_path = objective_dir.join("item.md"); + let raw = fs::read_to_string(&item_path)?; + let (frontmatter, body) = split_frontmatter(&raw, objective_id)?; + let meta: ObjectiveFrontmatter = serde_yaml::from_str(frontmatter)?; + let now = Utc::now().to_rfc3339(); + let created_at = meta.created_at.clone().unwrap_or_else(|| now.clone()); + let updated_at = meta.updated_at.clone().unwrap_or_else(|| now.clone()); + + store.upsert_objective(&ObjectiveRecord { + workspace_id: workspace_id.to_string(), + objective_id: objective_id.to_string(), + title: meta.title, + state: meta.state, + body_md: body.trim_start().to_string(), + created_at: created_at.clone(), + updated_at: updated_at.clone(), + })?; + + let mut links = Vec::new(); + let mut skipped_links = 0; + for ticket_id in meta.linked_tickets { + if validate_record_id(&ticket_id).is_ok() { + links.push(ObjectiveTicketLinkRecord { + workspace_id: workspace_id.to_string(), + objective_id: objective_id.to_string(), + ticket_id, + kind: "linked".to_string(), + created_at: now.clone(), + }); + } else { + skipped_links += 1; + } + } + let imported_links = links.len(); + let links_imported = if let Err(err) = + store.replace_objective_ticket_links(workspace_id, objective_id, &links) + { + // The legacy filesystem allowed links to tickets that may no longer + // exist. Preserve the Objective itself and make the skipped count clear + // rather than failing the whole import on a foreign-key mismatch. + skipped_links += imported_links; + eprintln!( + "warning: skipped {imported_links} ticket link(s) for objective {objective_id}: {err}" + ); + 0 + } else { + imported_links + }; + + let mut resources = 0; + for file in walk_files(objective_dir)? { + let relative = file + .strip_prefix(objective_dir) + .map_err(|err| Error::Store(err.to_string()))? + .to_string_lossy() + .replace('\\', "/"); + if relative == "item.md" || relative.starts_with("_staging/") { + continue; + } + let Ok(body) = fs::read_to_string(&file) else { + continue; + }; + store.upsert_objective_resource(&ObjectiveResourceRecord { + workspace_id: workspace_id.to_string(), + objective_id: objective_id.to_string(), + resource_path: relative, + body, + media_type: Some("text/markdown".to_string()), + created_at: created_at.clone(), + updated_at: updated_at.clone(), + })?; + resources += 1; + } + + Ok(ImportedObjectiveCounts { + resources, + links: links_imported, + skipped_links, + }) +} + +fn import_memory_staging( + workspace_root: &Path, + workspace_id: &str, + store: &S, + report: &mut ObjectiveImportReport, +) -> Result<()> { + let root = workspace_root.join(".yoi/memory/_staging"); + if !root.exists() { + return Ok(()); + } + let now = Utc::now().to_rfc3339(); + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("json") { + continue; + } + let raw_json = fs::read_to_string(&path)?; + let candidate_id = candidate_id_from_json(&raw_json) + .or_else(|| { + path.file_stem() + .map(|stem| stem.to_string_lossy().to_string()) + }) + .ok_or_else(|| Error::Store(format!("missing candidate id: {}", path.display())))?; + store.upsert_memory_staging_record(&MemoryStagingRecord { + workspace_id: workspace_id.to_string(), + candidate_id, + raw_json, + source_path: Some(path.to_string_lossy().to_string()), + imported_at: now.clone(), + })?; + report.memory_staging_records_imported += 1; + } + Ok(()) +} + +fn candidate_id_from_json(raw_json: &str) -> Option { + serde_json::from_str::(raw_json) + .ok() + .and_then(|value| { + value + .get("id") + .and_then(|id| id.as_str()) + .map(str::to_string) + }) +} + +fn split_frontmatter<'a>(raw: &'a str, label: &str) -> Result<(&'a str, &'a str)> { + let rest = raw + .strip_prefix("---\n") + .ok_or_else(|| Error::MissingFrontmatter(label.to_string()))?; + let Some((frontmatter, body)) = rest.split_once("\n---\n") else { + return Err(Error::MissingFrontmatter(label.to_string())); + }; + Ok((frontmatter, body)) +} + +fn walk_files(root: &Path) -> Result> { + let mut files = Vec::new(); + walk_files_inner(root, &mut files)?; + Ok(files) +} + +fn walk_files_inner(path: &Path, files: &mut Vec) -> Result<()> { + for entry in fs::read_dir(path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + walk_files_inner(&path, files)?; + } else if path.is_file() { + files.push(path); + } + } + Ok(()) +} diff --git a/crates/workspace-server/src/records.rs b/crates/workspace-server/src/records.rs index 5f3b82fa..228d0e39 100644 --- a/crates/workspace-server/src/records.rs +++ b/crates/workspace-server/src/records.rs @@ -1,4 +1,3 @@ -use std::fs; use std::path::{Path, PathBuf}; use project_record::validate_record_id; @@ -8,14 +7,17 @@ use ticket::{ TicketWorkspaceActionPriority, project_ticket_workspace_item, }; +use crate::store::{ControlPlaneStore, SqliteWorkspaceStore}; use crate::{Error, Result}; const DETAIL_BODY_LIMIT: usize = 64 * 1024; const SUMMARY_BODY_LIMIT: usize = 240; -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct LocalProjectRecordReader { + workspace_id: String, workspace_root: PathBuf, + store: SqliteWorkspaceStore, ticket_backend: SqliteTicketBackend, } @@ -26,8 +28,12 @@ impl LocalProjectRecordReader { workspace_id: impl Into, ) -> Result { let workspace_root = workspace_root.into(); + let database_path = database_path.into(); + let workspace_id = workspace_id.into(); Ok(Self { + workspace_id: workspace_id.clone(), workspace_root, + store: SqliteWorkspaceStore::open(&database_path)?, ticket_backend: SqliteTicketBackend::new(database_path, workspace_id), }) } @@ -96,61 +102,65 @@ impl LocalProjectRecordReader { pub fn list_objectives(&self, limit: usize) -> Result> { let mut items = Vec::new(); - let mut invalid_records = Vec::new(); - let root = self.workspace_root.join(".yoi/objectives"); - if !root.exists() { - return Ok(ProjectRecordList { - items, - invalid_records, - record_authority: "local_yoi_project_records".to_string(), + for record in self.store.list_objectives(&self.workspace_id, limit)? { + let linked_tickets = self + .store + .list_objective_ticket_links(&self.workspace_id, &record.objective_id)? + .into_iter() + .map(|link| link.ticket_id) + .collect::>(); + items.push(ObjectiveSummary { + id: record.objective_id, + title: record.title, + state: record.state, + updated_at: Some(record.updated_at), + summary: summarize_body(&record.body_md), + linked_tickets, + record_source: "workspace-sqlite".to_string(), }); } - - for entry in fs::read_dir(&root)? { - let entry = entry?; - let path = entry.path(); - if !path.is_dir() { - continue; - } - let id = entry.file_name().to_string_lossy().to_string(); - match read_objective_summary(&path, &id) { - Ok(item) => items.push(item), - Err(error) => invalid_records.push(InvalidProjectRecord { - label: id, - reason: error.to_string(), - }), - } - } - items.sort_by(|a, b| { - b.updated_at - .cmp(&a.updated_at) - .then_with(|| a.id.cmp(&b.id)) - }); - items.truncate(limit.min(200)); Ok(ProjectRecordList { items, - invalid_records, - record_authority: "local_yoi_project_records".to_string(), + invalid_records: Vec::new(), + record_authority: "workspace-sqlite".to_string(), }) } pub fn objective(&self, id: &str) -> Result { validate_project_id(id)?; - let path = self.workspace_root.join(".yoi/objectives").join(id); - let raw = fs::read_to_string(path.join("item.md"))?; - let (frontmatter, body) = split_frontmatter(&raw, id)?; - let meta: ObjectiveFrontmatter = serde_yaml::from_str(frontmatter)?; - let (body, body_truncated) = truncate_body(body, DETAIL_BODY_LIMIT); + let record = self + .store + .get_objective(&self.workspace_id, id)? + .ok_or_else(|| Error::Store(format!("unknown objective `{id}`")))?; + let linked_tickets = self + .store + .list_objective_ticket_links(&self.workspace_id, &record.objective_id)? + .into_iter() + .map(|link| link.ticket_id) + .collect::>(); + let resources = self + .store + .list_objective_resources(&self.workspace_id, &record.objective_id)? + .into_iter() + .map(|resource| ObjectiveResourceSummary { + path: resource.resource_path, + media_type: resource.media_type, + bytes: resource.body.len(), + updated_at: resource.updated_at, + }) + .collect(); + let (body, body_truncated) = truncate_body(&record.body_md, DETAIL_BODY_LIMIT); Ok(ObjectiveDetail { - id: id.to_string(), - title: meta.title, - state: meta.state, - created_at: meta.created_at, - updated_at: meta.updated_at, - linked_tickets: meta.linked_tickets, + id: record.objective_id, + title: record.title, + state: record.state, + created_at: Some(record.created_at), + updated_at: Some(record.updated_at), + linked_tickets, + resources, body, body_truncated, - record_source: "local_yoi_objective".to_string(), + record_source: "workspace-sqlite".to_string(), }) } } @@ -226,47 +236,18 @@ pub struct ObjectiveDetail { pub created_at: Option, pub updated_at: Option, pub linked_tickets: Vec, + pub resources: Vec, pub body: String, pub body_truncated: bool, pub record_source: String, } -#[derive(Debug, Deserialize)] -struct ObjectiveFrontmatter { - title: String, - state: String, - #[serde(default)] - created_at: Option, - #[serde(default)] - updated_at: Option, - #[serde(default)] - linked_tickets: Vec, -} - -fn read_objective_summary(path: &Path, id: &str) -> Result { - validate_project_id(id)?; - let raw = fs::read_to_string(path.join("item.md"))?; - let (frontmatter, body) = split_frontmatter(&raw, id)?; - let meta: ObjectiveFrontmatter = serde_yaml::from_str(frontmatter)?; - Ok(ObjectiveSummary { - id: id.to_string(), - title: meta.title, - state: meta.state, - updated_at: meta.updated_at, - summary: summarize_body(body), - linked_tickets: meta.linked_tickets, - record_source: "local_yoi_objective".to_string(), - }) -} - -fn split_frontmatter<'a>(raw: &'a str, label: &str) -> Result<(&'a str, &'a str)> { - let rest = raw - .strip_prefix("---\n") - .ok_or_else(|| Error::MissingFrontmatter(label.to_string()))?; - let Some((frontmatter, body)) = rest.split_once("\n---\n") else { - return Err(Error::MissingFrontmatter(label.to_string())); - }; - Ok((frontmatter, body)) +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ObjectiveResourceSummary { + pub path: String, + pub media_type: Option, + pub bytes: usize, + pub updated_at: String, } fn validate_project_id(id: &str) -> Result<()> { @@ -300,10 +281,13 @@ fn truncate_body(body: &str, limit: usize) -> (String, bool) { #[cfg(test)] mod tests { - use super::*; + use std::fs; - #[test] - fn reads_sqlite_yoi_ticket_and_objective_records_without_migration() { + use super::*; + use crate::store::WorkspaceRecord; + + #[tokio::test] + async fn reads_sqlite_yoi_ticket_and_objective_records_without_filesystem_authority() { let dir = tempfile::tempdir().unwrap(); write_ticket(dir.path(), "00000000001J2", "Read bridge", "ready"); let db_path = dir.path().join("workspace.db"); @@ -313,6 +297,24 @@ mod tests { )) .unwrap(); write_objective(dir.path(), "00000000001J3", "Control plane", "active"); + let store = SqliteWorkspaceStore::open(&db_path).unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "workspace-test".to_string(), + owner_account_id: None, + display_name: "Workspace Test".to_string(), + state: "active".to_string(), + created_at: "2026-01-01T00:00:00Z".to_string(), + updated_at: "2026-01-01T00:00:00Z".to_string(), + }) + .await + .unwrap(); + crate::objective_import::import_legacy_objectives_and_memory_staging( + dir.path(), + "workspace-test", + &store, + ) + .unwrap(); let reader = LocalProjectRecordReader::new(dir.path(), &db_path, "workspace-test").unwrap(); let tickets = reader.list_tickets(20).unwrap(); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 8f6ad640..782b201a 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -8312,6 +8312,7 @@ mod tests { async fn serves_bounded_read_apis_and_static_spa_separately() { let dir = tempfile::tempdir().unwrap(); write_objective(dir.path(), "00000000001J3", "API Objective", "active"); + write_memory_staging(dir.path(), "00000000001J4"); let static_dir = dir.path().join("static"); std::fs::create_dir_all(static_dir.join("assets")).unwrap(); std::fs::write(static_dir.join("index.html"), "
Yoi Workspace
").unwrap(); @@ -8325,6 +8326,36 @@ mod tests { "API Ticket", ticket::TicketWorkflowState::Ready, ); + let sqlite_store = SqliteWorkspaceStore::open(&config.database_path).unwrap(); + sqlite_store + .upsert_workspace(&WorkspaceRecord { + workspace_id: TEST_WORKSPACE_ID.to_string(), + owner_account_id: None, + display_name: "Test Workspace".to_string(), + state: "active".to_string(), + created_at: "2026-01-01T00:00:00Z".to_string(), + updated_at: "2026-01-01T00:00:00Z".to_string(), + }) + .await + .unwrap(); + crate::objective_import::import_legacy_objectives_and_memory_staging( + dir.path(), + TEST_WORKSPACE_ID, + &sqlite_store, + ) + .unwrap(); + assert_eq!( + sqlite_store + .count_memory_staging_records(TEST_WORKSPACE_ID) + .unwrap(), + 1 + ); + write_objective( + dir.path(), + "00000000001J5", + "Filesystem Only Objective", + "active", + ); config.static_assets_dir = Some(static_dir); let api = WorkspaceApi::new_with_execution_backend( config, @@ -8412,8 +8443,10 @@ mod tests { assert_eq!(tickets["items"][0]["state"], "ready"); let objectives = get_json(app.clone(), "/api/objectives").await; + assert_eq!(objectives["items"].as_array().unwrap().len(), 1); assert_eq!(objectives["items"][0]["id"], "00000000001J3"); assert_eq!(objectives["items"][0]["summary"], "Objective body."); + assert_eq!(objectives["record_authority"], "workspace-sqlite"); let scoped_objectives = get_json( app.clone(), &format!("/api/w/{TEST_WORKSPACE_ID}/objectives"), @@ -8432,6 +8465,11 @@ mod tests { ) .await; assert_eq!(scoped_objective["id"], "00000000001J3"); + assert_eq!(scoped_objective["record_source"], "workspace-sqlite"); + assert_eq!( + scoped_objective["resources"][0]["path"], + "memory-architecture-overview.md" + ); let repositories = get_json(app.clone(), "/api/repositories").await; assert_eq!(repositories["items"][0]["id"], TEST_REPOSITORY_ID); @@ -9327,6 +9365,25 @@ mod tests { backend.create(input).unwrap(); } + fn write_memory_staging(root: &Path, candidate_id: &str) { + let staging_dir = root.join(".yoi/memory/_staging"); + std::fs::create_dir_all(&staging_dir).unwrap(); + std::fs::write( + staging_dir.join(format!("{candidate_id}.json")), + format!( + r#"{{ + "schema_version": 2, + "id": "{candidate_id}", + "kind": "working_assumption", + "claim": "Objective staging import test", + "why_useful": "Verifies SQLite push" +}} +"#, + ), + ) + .unwrap(); + } + fn write_objective(root: &Path, id: &str, title: &str, state: &str) { let objective_dir = root.join(".yoi/objectives").join(id); std::fs::create_dir_all(&objective_dir).unwrap(); @@ -9346,5 +9403,10 @@ Objective body. ), ) .unwrap(); + std::fs::write( + objective_dir.join("memory-architecture-overview.md"), + "# Memory architecture\n\nResource body.\n", + ) + .unwrap(); } } diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index a6cc22f5..2a464eb6 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -62,6 +62,11 @@ const MIGRATIONS: &[Migration] = &[ name: "webauthn challenge state", apply: add_webauthn_challenge_state, }, + Migration { + version: 10, + name: "sqlite objective authority and memory staging import", + apply: create_objective_sqlite_authority_tables, + }, ]; struct Migration { @@ -221,6 +226,46 @@ pub struct WorkerWorkdirLinkRecord { pub unlinked_at: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ObjectiveRecord { + pub workspace_id: String, + pub objective_id: String, + pub title: String, + pub state: String, + pub body_md: String, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ObjectiveTicketLinkRecord { + pub workspace_id: String, + pub objective_id: String, + pub ticket_id: String, + pub kind: String, + pub created_at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ObjectiveResourceRecord { + pub workspace_id: String, + pub objective_id: String, + pub resource_path: String, + pub body: String, + pub media_type: Option, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct MemoryStagingRecord { + pub workspace_id: String, + pub candidate_id: String, + pub raw_json: String, + pub source_path: Option, + pub imported_at: String, +} + #[async_trait] pub trait ControlPlaneStore: Send + Sync { async fn schema_version(&self) -> Result; @@ -230,6 +275,33 @@ pub trait ControlPlaneStore: Send + Sync { fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>; fn list_repositories(&self, workspace_id: &str) -> Result>; + fn upsert_objective(&self, record: &ObjectiveRecord) -> Result<()>; + fn list_objectives(&self, workspace_id: &str, limit: usize) -> Result>; + fn get_objective( + &self, + workspace_id: &str, + objective_id: &str, + ) -> Result>; + fn replace_objective_ticket_links( + &self, + workspace_id: &str, + objective_id: &str, + links: &[ObjectiveTicketLinkRecord], + ) -> Result<()>; + fn list_objective_ticket_links( + &self, + workspace_id: &str, + objective_id: &str, + ) -> Result>; + fn upsert_objective_resource(&self, record: &ObjectiveResourceRecord) -> Result<()>; + fn list_objective_resources( + &self, + workspace_id: &str, + objective_id: &str, + ) -> Result>; + fn upsert_memory_staging_record(&self, record: &MemoryStagingRecord) -> Result<()>; + fn count_memory_staging_records(&self, workspace_id: &str) -> Result; + fn upsert_account(&self, record: &AccountRecord) -> Result<()>; fn get_account(&self, account_id: &str) -> Result>; fn get_account_by_handle(&self, kind: &str, handle: &str) -> Result>; @@ -480,6 +552,200 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn upsert_objective(&self, record: &ObjectiveRecord) -> Result<()> { + self.with_conn(|conn| { + conn.execute( + r#"INSERT INTO objectives ( + workspace_id, objective_id, title, state, body_md, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(objective_id) DO UPDATE SET + workspace_id = excluded.workspace_id, + title = excluded.title, + state = excluded.state, + body_md = excluded.body_md, + created_at = excluded.created_at, + updated_at = excluded.updated_at"#, + params![ + record.workspace_id, + record.objective_id, + record.title, + record.state, + record.body_md, + record.created_at, + record.updated_at, + ], + )?; + Ok(()) + }) + } + + fn list_objectives(&self, workspace_id: &str, limit: usize) -> Result> { + self.with_conn(|conn| { + let mut stmt = conn.prepare( + r#"SELECT workspace_id, objective_id, title, state, body_md, created_at, updated_at + FROM objectives + WHERE workspace_id = ?1 + ORDER BY updated_at DESC, objective_id ASC + LIMIT ?2"#, + )?; + let rows = + stmt.query_map(params![workspace_id, limit as i64], read_objective_record)?; + rows.collect::, _>>() + .map_err(Error::from) + }) + } + + fn get_objective( + &self, + workspace_id: &str, + objective_id: &str, + ) -> Result> { + self.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, objective_id, title, state, body_md, created_at, updated_at + FROM objectives + WHERE workspace_id = ?1 AND objective_id = ?2"#, + params![workspace_id, objective_id], + read_objective_record, + ) + .optional() + .map_err(Error::from) + }) + } + + fn replace_objective_ticket_links( + &self, + workspace_id: &str, + objective_id: &str, + links: &[ObjectiveTicketLinkRecord], + ) -> Result<()> { + self.with_conn(|conn| { + conn.execute( + "DELETE FROM objective_ticket_links WHERE workspace_id = ?1 AND objective_id = ?2", + params![workspace_id, objective_id], + )?; + for link in links { + conn.execute( + r#"INSERT INTO objective_ticket_links ( + workspace_id, objective_id, ticket_id, kind, created_at + ) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(objective_id, ticket_id, kind) DO UPDATE SET + workspace_id = excluded.workspace_id, + created_at = excluded.created_at"#, + params![ + link.workspace_id, + link.objective_id, + link.ticket_id, + link.kind, + link.created_at, + ], + )?; + } + Ok(()) + }) + } + + fn list_objective_ticket_links( + &self, + workspace_id: &str, + objective_id: &str, + ) -> Result> { + self.with_conn(|conn| { + let mut stmt = conn.prepare( + r#"SELECT workspace_id, objective_id, ticket_id, kind, created_at + FROM objective_ticket_links + WHERE workspace_id = ?1 AND objective_id = ?2 + ORDER BY ticket_id ASC, kind ASC"#, + )?; + let rows = stmt.query_map( + params![workspace_id, objective_id], + read_objective_ticket_link_record, + )?; + rows.collect::, _>>() + .map_err(Error::from) + }) + } + + fn upsert_objective_resource(&self, record: &ObjectiveResourceRecord) -> Result<()> { + self.with_conn(|conn| { + conn.execute( + r#"INSERT INTO objective_resources ( + workspace_id, objective_id, resource_path, body, media_type, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(objective_id, resource_path) DO UPDATE SET + workspace_id = excluded.workspace_id, + body = excluded.body, + media_type = excluded.media_type, + updated_at = excluded.updated_at"#, + params![ + record.workspace_id, + record.objective_id, + record.resource_path, + record.body, + record.media_type, + record.created_at, + record.updated_at, + ], + )?; + Ok(()) + }) + } + + fn list_objective_resources( + &self, + workspace_id: &str, + objective_id: &str, + ) -> Result> { + self.with_conn(|conn| { + let mut stmt = conn.prepare( + r#"SELECT workspace_id, objective_id, resource_path, body, media_type, created_at, updated_at + FROM objective_resources + WHERE workspace_id = ?1 AND objective_id = ?2 + ORDER BY resource_path ASC"#, + )?; + let rows = stmt.query_map( + params![workspace_id, objective_id], + read_objective_resource_record, + )?; + rows.collect::, _>>() + .map_err(Error::from) + }) + } + + fn upsert_memory_staging_record(&self, record: &MemoryStagingRecord) -> Result<()> { + self.with_conn(|conn| { + conn.execute( + r#"INSERT INTO memory_staging_records ( + workspace_id, candidate_id, raw_json, source_path, imported_at + ) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(workspace_id, candidate_id) DO UPDATE SET + raw_json = excluded.raw_json, + source_path = excluded.source_path, + imported_at = excluded.imported_at"#, + params![ + record.workspace_id, + record.candidate_id, + record.raw_json, + record.source_path, + record.imported_at, + ], + )?; + Ok(()) + }) + } + + fn count_memory_staging_records(&self, workspace_id: &str) -> Result { + self.with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM memory_staging_records WHERE workspace_id = ?1", + params![workspace_id], + |row| row.get::<_, i64>(0), + ) + .map(|count| count as usize) + .map_err(Error::from) + }) + } + fn upsert_account(&self, record: &AccountRecord) -> Result<()> { self.with_conn(|conn| { conn.execute( @@ -1265,6 +1531,44 @@ fn read_worker_workdir_link_record( }) } +fn read_objective_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(ObjectiveRecord { + workspace_id: row.get(0)?, + objective_id: row.get(1)?, + title: row.get(2)?, + state: row.get(3)?, + body_md: row.get(4)?, + created_at: row.get(5)?, + updated_at: row.get(6)?, + }) +} + +fn read_objective_ticket_link_record( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(ObjectiveTicketLinkRecord { + workspace_id: row.get(0)?, + objective_id: row.get(1)?, + ticket_id: row.get(2)?, + kind: row.get(3)?, + created_at: row.get(4)?, + }) +} + +fn read_objective_resource_record( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(ObjectiveResourceRecord { + workspace_id: row.get(0)?, + objective_id: row.get(1)?, + resource_path: row.get(2)?, + body: row.get(3)?, + media_type: row.get(4)?, + created_at: row.get(5)?, + updated_at: row.get(6)?, + }) +} + fn worker_registry_select_sql(where_clause: &str) -> String { format!( "SELECT workspace_id, runtime_id, runtime_worker_id, display_name, profile, \ @@ -1381,6 +1685,53 @@ fn add_webauthn_challenge_state(conn: &Connection) -> Result<()> { Ok(()) } +fn create_objective_sqlite_authority_tables(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" +PRAGMA foreign_keys = OFF; +ALTER TABLE objective_ticket_links RENAME TO objective_ticket_links_old; +CREATE TABLE objective_ticket_links ( + workspace_id TEXT NOT NULL, + objective_id TEXT NOT NULL, + ticket_id TEXT NOT NULL, + kind TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (objective_id, ticket_id, kind), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + FOREIGN KEY (objective_id) REFERENCES objectives(objective_id) ON DELETE CASCADE +); +INSERT OR IGNORE INTO objective_ticket_links (workspace_id, objective_id, ticket_id, kind, created_at) + SELECT workspace_id, objective_id, ticket_id, kind, created_at FROM objective_ticket_links_old; +DROP TABLE objective_ticket_links_old; +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS objective_resources ( + workspace_id TEXT NOT NULL, + objective_id TEXT NOT NULL, + resource_path TEXT NOT NULL, + body TEXT NOT NULL, + media_type TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (objective_id, resource_path), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + FOREIGN KEY (objective_id) REFERENCES objectives(objective_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS memory_staging_records ( + workspace_id TEXT NOT NULL, + candidate_id TEXT NOT NULL, + raw_json TEXT NOT NULL, + source_path TEXT, + imported_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, candidate_id), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE +); +"#, + )?; + Ok(()) +} + fn create_account_identity_tables(conn: &Connection) -> Result<()> { conn.execute_batch( r#" @@ -1916,12 +2267,32 @@ CREATE TABLE IF NOT EXISTS objectives ( CREATE TABLE IF NOT EXISTS objective_ticket_links ( workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE, objective_id TEXT NOT NULL REFERENCES objectives(objective_id) ON DELETE CASCADE, - ticket_id TEXT NOT NULL REFERENCES tickets(ticket_id) ON DELETE CASCADE, + ticket_id TEXT NOT NULL, kind TEXT NOT NULL, created_at TEXT NOT NULL, PRIMARY KEY (objective_id, ticket_id, kind) ); +CREATE TABLE IF NOT EXISTS objective_resources ( + workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + objective_id TEXT NOT NULL REFERENCES objectives(objective_id) ON DELETE CASCADE, + resource_path TEXT NOT NULL, + body TEXT NOT NULL, + media_type TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (objective_id, resource_path) +); + +CREATE TABLE IF NOT EXISTS memory_staging_records ( + workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + candidate_id TEXT NOT NULL, + raw_json TEXT NOT NULL, + source_path TEXT, + imported_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, candidate_id) +); + CREATE TABLE IF NOT EXISTS repositories ( workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE, repository_id TEXT PRIMARY KEY, @@ -2053,7 +2424,7 @@ mod tests { let db = dir.path().join("control-plane.sqlite"); let store = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 9); + assert_eq!(store.schema_version().await.unwrap(), 10); let record = WorkspaceRecord { workspace_id: "local-dev".to_string(), @@ -2066,7 +2437,7 @@ mod tests { store.upsert_workspace(&record).await.unwrap(); let reopened = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(reopened.schema_version().await.unwrap(), 9); + assert_eq!(reopened.schema_version().await.unwrap(), 10); assert_eq!( reopened.get_workspace("local-dev").await.unwrap(), Some(record) @@ -2283,7 +2654,7 @@ mod tests { .unwrap(); let store = SqliteWorkspaceStore::from_connection(conn).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 9); + assert_eq!(store.schema_version().await.unwrap(), 10); store .with_conn(|conn| { @@ -2374,7 +2745,7 @@ mod tests { #[tokio::test] async fn repository_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 9); + assert_eq!(store.schema_version().await.unwrap(), 10); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -2511,7 +2882,7 @@ mod tests { #[tokio::test] async fn account_and_login_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 9); + assert_eq!(store.schema_version().await.unwrap(), 10); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { account_id: "acct-user-alice".to_string(), diff --git a/scripts/import-objectives-and-staging-to-sqlite-20260726.sh b/scripts/import-objectives-and-staging-to-sqlite-20260726.sh new file mode 100755 index 00000000..976c2a2a --- /dev/null +++ b/scripts/import-objectives-and-staging-to-sqlite-20260726.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: scripts/import-objectives-and-staging-to-sqlite-20260726.sh [--workspace PATH] [--database PATH] [--apply] + +Imports legacy .yoi/objectives records and .yoi/memory/_staging JSON files into +Workspace SQLite authority by invoking yoi-workspace-server import-objectives. + +By default this is a dry-run of the command line only. Pass --apply to execute. +EOF +} + +workspace="$(pwd)" +database="${YOI_SERVER_DB:-$HOME/.local/share/yoi/server/server.db}" +apply=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --workspace) + workspace="$2" + shift 2 + ;; + --workspace=*) + workspace="${1#--workspace=}" + shift + ;; + --database) + database="$2" + shift 2 + ;; + --database=*) + database="${1#--database=}" + shift + ;; + --apply) + apply=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +cmd=(cargo run -q -p yoi-workspace-server -- import-objectives --workspace "$workspace" --database "$database") + +printf 'Workspace: %s\n' "$workspace" +printf 'Database: %s\n' "$database" +printf 'Command: ' +printf '%q ' "${cmd[@]}" +printf '\n' + +if [[ "$apply" -ne 1 ]]; then + echo 'Dry run only. Re-run with --apply to import.' + exit 0 +fi + +"${cmd[@]}"