From 5ec1839ec7f1d3d18777d4f21f83a1fb551e782e Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 26 Sep 2024 23:43:51 +0900 Subject: [PATCH 1/3] fix typo --- src/lib/server/{aritcle-git.ts => article-git.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/lib/server/{aritcle-git.ts => article-git.ts} (100%) diff --git a/src/lib/server/aritcle-git.ts b/src/lib/server/article-git.ts similarity index 100% rename from src/lib/server/aritcle-git.ts rename to src/lib/server/article-git.ts -- 2.43.0 From f11bdf15fc7f6ce6df195cd4f5882c40850a8d85 Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 26 Sep 2024 23:44:12 +0900 Subject: [PATCH 2/3] fix typo #2 --- src/lib/server/database/init.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/server/database/init.ts b/src/lib/server/database/init.ts index b96b31e..e539271 100644 --- a/src/lib/server/database/init.ts +++ b/src/lib/server/database/init.ts @@ -2,7 +2,7 @@ import type { Postgres } from '$lib/server/database'; import fs from 'fs'; import compile from '$lib/server/article'; -import article_git from '$lib/server/aritcle-git'; +import article_git from '$lib/server/article-git'; import { format } from '$lib/scripts/formatted_date'; -- 2.43.0 From dd057dd335562c285fdd182c773490b90c1b628e Mon Sep 17 00:00:00 2001 From: Hare Date: Fri, 27 Sep 2024 00:16:57 +0900 Subject: [PATCH 3/3] Refactor: Update imports for server modules --- src/hooks.server.ts | 6 +- .../{article.ts => article_compiler.ts} | 0 .../{database/init.ts => data_loader.ts} | 117 ++++-------------- .../database/{get_connection.ts => pool.ts} | 9 +- .../{database.ts => database/postgres.ts} | 12 ++ src/lib/server/database/table.ts | 77 ++++++++++++ src/lib/server/{article-git.ts => git.ts} | 0 src/routes/+page.server.ts | 2 +- src/routes/api/fetch_articles/+server.ts | 6 +- .../article/[category]/[id]/+page.server.ts | 2 +- .../[category]/[series]/[id]/+page.server.ts | 2 +- 11 files changed, 129 insertions(+), 104 deletions(-) rename src/lib/server/{article.ts => article_compiler.ts} (100%) rename src/lib/server/{database/init.ts => data_loader.ts} (53%) rename src/lib/server/database/{get_connection.ts => pool.ts} (62%) rename src/lib/server/{database.ts => database/postgres.ts} (69%) create mode 100644 src/lib/server/database/table.ts rename src/lib/server/{article-git.ts => git.ts} (100%) diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 9ed28ca..84108ac 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -1,9 +1,11 @@ import type { Handle } from '@sveltejs/kit' -import { getConnection } from '$lib/server/database/get_connection' +import { getPool } from '$lib/server/database/pool'; +import git from '$lib/server/git'; export const handle: Handle = async ({ event, resolve }) => { - const pg = getConnection(); + const pg = getPool; event.locals.db = pg; + event.locals.git = git; const result = await resolve(event); return result; diff --git a/src/lib/server/article.ts b/src/lib/server/article_compiler.ts similarity index 100% rename from src/lib/server/article.ts rename to src/lib/server/article_compiler.ts diff --git a/src/lib/server/database/init.ts b/src/lib/server/data_loader.ts similarity index 53% rename from src/lib/server/database/init.ts rename to src/lib/server/data_loader.ts index e539271..a050696 100644 --- a/src/lib/server/database/init.ts +++ b/src/lib/server/data_loader.ts @@ -1,90 +1,21 @@ -// initialize -import type { Postgres } from '$lib/server/database'; +import type { Postgres } from '$lib/server/database/postgres'; +import type { SimpleGit } from 'simple-git'; import fs from 'fs'; -import compile from '$lib/server/article'; -import article_git from '$lib/server/article-git'; - +import compile from '$lib/server/article_compiler'; +import type { TableSchema } from '$lib/server/database/table'; +import server_table from '$lib/server/database/table'; import { format } from '$lib/scripts/formatted_date'; -export default async function init(db: Postgres) { - await cloneRepo(); - await createTable(db, [ - { - name: 'article', - columns: [ - { name: 'seq', type: 'serial', constraint: 'primary key' }, - { name: 'id', type: 'text', constraint: 'not null' }, - { name: 'released_at', type: 'timestamp', constraint: 'not null' }, - { name: 'updated_at', type: 'timestamp', constraint: 'not null' }, - { name: 'author', type: 'text', constraint: 'not null' }, - { name: 'email', type: 'text', constraint: 'not null' }, - { name: 'title', type: 'text', constraint: 'not null' }, - { name: 'category', type: 'text', constraint: 'not null' }, - { name: 'tags', type: 'text[]', constraint: 'not null' }, - { name: 'image', type: 'text', constraint: '' }, - { name: 'publish', type: 'text', constraint: 'not null' }, - { name: 'content', type: 'text', constraint: 'not null' }, - ], - }, - { - name: 'article_comment', - columns: [ - { name: 'id', type: 'serial', constraint: 'primary key' }, - { name: 'article', type: 'integer', constraint: 'not null' }, - { name: 'posted_at', type: 'timestamp', constraint: 'not null' }, - { name: 'content', type: 'text', constraint: 'not null' }, - ], - }, - { - name: 'thread', - columns: [ - { name: 'seq', type: 'serial', constraint: 'primary key' }, - { name: 'id', type: 'text', constraint: 'not null' }, - { name: 'title', type: 'text', constraint: 'not null' }, - { name: 'category', type: 'text', constraint: 'not null' }, - { name: 'created_at', type: 'timestamp', constraint: 'not null' }, - { name: 'updated_at', type: 'timestamp', constraint: 'not null' }, - { name: 'tags', type: 'text[]', constraint: 'not null' }, - { name: 'content', type: 'text', constraint: 'not null' }, - ], - }, - { - name: 'thread_post', - columns: [ - { name: 'seq', type: 'serial', constraint: 'primary key' }, - { name: 'thread_id', type: 'integer', constraint: 'not null' }, - { name: 'title', type: 'text', constraint: 'not null' }, - { name: 'posted_at', type: 'timestamp', constraint: 'not null' }, - { name: 'content', type: 'text', constraint: 'not null' }, - ], - }, - { - name: 'thread_comment', - columns: [ - { name: 'id', type: 'serial', constraint: 'primary key' }, - { name: 'thread', type: 'integer', constraint: 'not null' }, - { name: 'posted_at', type: 'timestamp', constraint: 'not null' }, - { name: 'content', type: 'text', constraint: 'not null' }, - ], - }, - { - name: 'tag', - columns: [ - { name: 'seq', type: 'serial', constraint: 'primary key' }, - { name: 'name', type: 'text', constraint: 'not null' }, - { name: 'ref_count', type: 'integer', constraint: 'not null' }, - ], - } - ]); - +const load = async (db: Postgres, git: SimpleGit) => { const articleFiles = await crawlArticles(db); await db.query('update tag set ref_count = 0'); + await db.begin(); for (const { path, id } of articleFiles) { + await db.savepoint(id); console.log(`Processing ${id}...`); - await db.begin(); try { - const gitlog = await article_git.log({ + const gitlog = await git.log({ file: path.slice(11), strictDate: true, }); @@ -135,36 +66,38 @@ export default async function init(db: Postgres) { } } catch (err) { console.log(err); - await db.rollback(); + await db.rollbackTo(id); } finally { console.log(''); - await db.commit(); + await db.releaseSavepoint(id); } } + await db.commit(); +} - await db.release(); +export async function init(db: Postgres, git: SimpleGit) { + await createTable(db, server_table); + await cloneRepo(git); + await load(db, git); +} + +export async function reload(db: Postgres, git: SimpleGit) { + await cloneRepo(git); + await load(db, git); } type ArticleFileItem = { path: string, id: string, } -export type TableSchema = { - name: string, - columns: { - name: string, - type: string, - constraint: string, - }[], -} -const cloneRepo = async () => { +const cloneRepo = async (git: SimpleGit) => { if (fs.existsSync('./articles/')) { console.log('Pulling articles from git..'); - await article_git.pull(); + await git.pull(); } else { console.log('Cloning articles from git..'); - await article_git.clone('git@gitea.hareworks.net:Hare/blog-articles.git', 'articles'); + await git.clone('git@gitea.hareworks.net:Hare/blog-articles.git', 'articles'); } } diff --git a/src/lib/server/database/get_connection.ts b/src/lib/server/database/pool.ts similarity index 62% rename from src/lib/server/database/get_connection.ts rename to src/lib/server/database/pool.ts index 818b226..ed588bc 100644 --- a/src/lib/server/database/get_connection.ts +++ b/src/lib/server/database/pool.ts @@ -12,12 +12,13 @@ const connectionString = `postgres://${PG_USER}:${PG_PASS}@${PG_HOST}:${PG_PORT} let pool = new Pool({ connectionString }); -export const getConnection = () => pool; +export const getPool = pool; export const reConnect = async () => { pool = new Pool({ connectionString }); } -import init_db from '$lib/server/database/init'; -import PG from '$lib/server/database'; -await init_db(await PG(pool)); \ No newline at end of file +import { init as init_data} from '$lib/server/data_loader'; +import PG from '$lib/server/database/postgres'; +import git from '$lib/server/git'; +await init_data(await PG(pool), git); \ No newline at end of file diff --git a/src/lib/server/database.ts b/src/lib/server/database/postgres.ts similarity index 69% rename from src/lib/server/database.ts rename to src/lib/server/database/postgres.ts index cc4d196..e268789 100644 --- a/src/lib/server/database.ts +++ b/src/lib/server/database/postgres.ts @@ -24,9 +24,21 @@ export class Postgres { await this.client!.query('commit'); } + async savepoint(name: string) { + await this.client!.query(`savepoint ${name}`); + } + async rollback() { await this.client!.query('rollback'); } + + async rollbackTo(name: string) { + await this.client!.query(`rollback to ${name}`); + } + + async releaseSavepoint(name: string) { + await this.client!.query(`release savepoint ${name}`); + } } export default async ( diff --git a/src/lib/server/database/table.ts b/src/lib/server/database/table.ts new file mode 100644 index 0000000..c09d24a --- /dev/null +++ b/src/lib/server/database/table.ts @@ -0,0 +1,77 @@ +export type TableSchema = { + name: string, + columns: { + name: string, + type: string, + constraint: string, + }[], +} + +export default [ + { + name: 'article', + columns: [ + { name: 'seq', type: 'serial', constraint: 'primary key' }, + { name: 'id', type: 'text', constraint: 'not null' }, + { name: 'released_at', type: 'timestamp', constraint: 'not null' }, + { name: 'updated_at', type: 'timestamp', constraint: 'not null' }, + { name: 'author', type: 'text', constraint: 'not null' }, + { name: 'email', type: 'text', constraint: 'not null' }, + { name: 'title', type: 'text', constraint: 'not null' }, + { name: 'category', type: 'text', constraint: 'not null' }, + { name: 'tags', type: 'text[]', constraint: 'not null' }, + { name: 'image', type: 'text', constraint: '' }, + { name: 'publish', type: 'text', constraint: 'not null' }, + { name: 'content', type: 'text', constraint: 'not null' }, + ], + }, + { + name: 'article_comment', + columns: [ + { name: 'id', type: 'serial', constraint: 'primary key' }, + { name: 'article', type: 'integer', constraint: 'not null' }, + { name: 'posted_at', type: 'timestamp', constraint: 'not null' }, + { name: 'content', type: 'text', constraint: 'not null' }, + ], + }, + { + name: 'thread', + columns: [ + { name: 'seq', type: 'serial', constraint: 'primary key' }, + { name: 'id', type: 'text', constraint: 'not null' }, + { name: 'title', type: 'text', constraint: 'not null' }, + { name: 'category', type: 'text', constraint: 'not null' }, + { name: 'created_at', type: 'timestamp', constraint: 'not null' }, + { name: 'updated_at', type: 'timestamp', constraint: 'not null' }, + { name: 'tags', type: 'text[]', constraint: 'not null' }, + { name: 'content', type: 'text', constraint: 'not null' }, + ], + }, + { + name: 'thread_post', + columns: [ + { name: 'seq', type: 'serial', constraint: 'primary key' }, + { name: 'thread_id', type: 'integer', constraint: 'not null' }, + { name: 'title', type: 'text', constraint: 'not null' }, + { name: 'posted_at', type: 'timestamp', constraint: 'not null' }, + { name: 'content', type: 'text', constraint: 'not null' }, + ], + }, + { + name: 'thread_comment', + columns: [ + { name: 'id', type: 'serial', constraint: 'primary key' }, + { name: 'thread', type: 'integer', constraint: 'not null' }, + { name: 'posted_at', type: 'timestamp', constraint: 'not null' }, + { name: 'content', type: 'text', constraint: 'not null' }, + ], + }, + { + name: 'tag', + columns: [ + { name: 'seq', type: 'serial', constraint: 'primary key' }, + { name: 'name', type: 'text', constraint: 'not null' }, + { name: 'ref_count', type: 'integer', constraint: 'not null' }, + ], + } +] as TableSchema[]; \ No newline at end of file diff --git a/src/lib/server/article-git.ts b/src/lib/server/git.ts similarity index 100% rename from src/lib/server/article-git.ts rename to src/lib/server/git.ts diff --git a/src/routes/+page.server.ts b/src/routes/+page.server.ts index 306e33e..a570d26 100644 --- a/src/routes/+page.server.ts +++ b/src/routes/+page.server.ts @@ -1,6 +1,6 @@ import { error } from '@sveltejs/kit'; import type { PageServerLoad } from './$types'; -import PG from '$lib/server/database'; +import PG from '$lib/server/database/postgres'; import { format } from '$lib/scripts/formatted_date'; let data: { diff --git a/src/routes/api/fetch_articles/+server.ts b/src/routes/api/fetch_articles/+server.ts index 7b6eea2..80fa33c 100644 --- a/src/routes/api/fetch_articles/+server.ts +++ b/src/routes/api/fetch_articles/+server.ts @@ -6,9 +6,9 @@ import { TOKEN } from '$env/static/private' -import PG from '$lib/server/database'; +import PG from '$lib/server/database/postgres'; import { building } from '$app/environment'; -import init_db from '$lib/server/database/init'; +import { reload as reload_data } from '$lib/server/data_loader'; export const POST: RequestHandler = async ({ url, locals }) => { const token = url.searchParams.get('token'); @@ -17,6 +17,6 @@ export const POST: RequestHandler = async ({ url, locals }) => { return error(401, 'Unauthorized'); } - if (!building) await init_db(await PG(locals.db)); + if (!building) await reload_data(await PG(locals.db), locals.git); return new Response(String(token)); }; diff --git a/src/routes/article/[category]/[id]/+page.server.ts b/src/routes/article/[category]/[id]/+page.server.ts index 54410e4..946403d 100644 --- a/src/routes/article/[category]/[id]/+page.server.ts +++ b/src/routes/article/[category]/[id]/+page.server.ts @@ -1,6 +1,6 @@ import type { Article } from '$lib/article'; import type { PageServerLoad } from './$types'; -import PG from '$lib/server/database'; +import PG from '$lib/server/database/postgres'; import { error } from '@sveltejs/kit'; export const load: PageServerLoad = async ({ params, locals }) => { diff --git a/src/routes/article/[category]/[series]/[id]/+page.server.ts b/src/routes/article/[category]/[series]/[id]/+page.server.ts index 522fd97..2c1ac33 100644 --- a/src/routes/article/[category]/[series]/[id]/+page.server.ts +++ b/src/routes/article/[category]/[series]/[id]/+page.server.ts @@ -1,6 +1,6 @@ import type { Article } from '$lib/article'; import type { PageServerLoad } from './$types'; -import PG from '$lib/server/database'; +import PG from '$lib/server/database/postgres'; import { error } from '@sveltejs/kit'; export const load: PageServerLoad = async ({ params, locals }) => { -- 2.43.0