blog.hareworks.net/src/lib/server/data_loader.ts

import type { Postgres } from '$lib/server/database/postgres';
import { type SimpleGit, CheckRepoActions } from 'simple-git';
import fs from 'fs';
import compileArticle from '$lib/server/article_compiler';
import type { TableSchema } from '$lib/server/database/table';
import server_table from '$lib/server/database/table';
import { format } from '$lib/scripts/formatted_date';

const load = async (db: Postgres, git: SimpleGit) => {
  const articleFiles = await crawlArticles(db);
  await db.query('update tag set ref_count = 0');

  await db.begin();
  let i = 0;
  for (const { path, id } of articleFiles) {
    await db.savepoint(`load${i}`);
    console.log(`Processing ${id}...`);
    try {
      const gitlog = await git.log({
        file: path.slice(11),
        strictDate: true,
      });
      const res = await db.query('select * from article where id = $1', [id]);

      // console.log(gitlog);
      const latest = gitlog.latest!;
      const oldest = gitlog.all[gitlog.all.length - 1];

      const author = oldest.author_name;
      const email = oldest.author_email;
      const released_at = new Date(oldest.date);
      const updated_at = new Date(latest.date);
      console.log(`Author: ${author} <${email}>\nReleased at: ${format(released_at)}\nUpdated at: ${format(updated_at)}`);

      const category = path.split('/')[3];
      const compiled = await compileArticle(fs.readFileSync(path, 'utf-8'));
      const title = compiled.data.fm.title;
      const tags: string[] = compiled.data.fm.tags;
      const image = compiled.data.fm.image;
      const publish = compiled.data.fm.publish;
      const content = compiled.code
        .replace(/>{@html `<code class="language-/g, '><code class="language-')
        .replace(/<\/code>`}<\/pre>/g, '</code></pre>')

      if (res.rowCount == 0) {
        console.log(`New article: ${id}`);
        await db.query(
          'insert into article (id, released_at, updated_at, author, email, title, category, tags, image, publish, content) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)',
          [id, released_at, updated_at, author, email, title, category, tags, image, publish, content]
        );
        // } else if (res.rows[0].updated_at < updated_at) {
      } else if (true) {
        console.log(`Update article: ${id}`);
        await db.query(
          'update article set released_at = $2, updated_at = $3, author = $4, email = $5, title = $6, category = $7, tags = $8, image = $9, publish = $10, content = $11 where id = $1',
          [id, released_at, updated_at, author, email, title, category, tags, image, publish, content]
        );
      } else {
        console.log(`Article ${id} is already up-to-date`);
      }
      for (const tag of tags) {
        if ((await db.query('select * from tag where name = $1', [tag])).rowCount == 0) {
          db.query('insert into tag (name, ref_count) values ($1, 1)', [tag]);
        } else {
          db.query('update tag set ref_count = ref_count + 1 where name = $1', [tag]);
        }
      }
    } catch (err) {
      console.log(err);
      await db.rollbackTo(`load${i}`);
    } finally {
      console.log('');
      await db.releaseSavepoint(`load${i}`);
      i++;
    }
  }
  await db.commit();
}

type ArticleFileItem = {
  path: string,
  id: string,
}

const cloneRepo = async (git: SimpleGit) => {
  const isRepoRoot = await git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT);
  if (isRepoRoot) {
    console.log('Pulling articles from git..');
    await git.pull();
  } else {
    console.log('Cloning articles from git..');
    await git.clone('git@gitea.hareworks.net:Hare/blog-articles.git', './');
  }
}

const createTable = async (db: Postgres, schemas: TableSchema[]) => {
  await db.begin();
  try {
    for (const schema of schemas) {
      const res = await db.query(`select * from information_schema.tables where table_name = '${schema.name}'`)
      if (res.rowCount == 0) {
        console.log(`Creating table ${schema.name}`);
        const columnStr = schema.columns.map(c => `${c.name} ${c.type} ${c.constraint}`).join(', ');
        await db.query(`create table ${schema.name} (${columnStr})`);
      } else {
        console.log(`Table ${schema.name} already exists`);
      }
    }
  } catch (err) {
    console.error(err);
    await db.rollback();
  } finally {
    await db.commit();
  }
}

const crawlArticles = async (db: Postgres): Promise<ArticleFileItem[]> => {
  const articleFiles: ArticleFileItem[] = [];
  function scanDir(path: string) {
    const files = fs.readdirSync(path);
    for (const file of files) {
      const dir = `${path}/${file}`;
      const stat = fs.statSync(dir);
      if (stat.isDirectory()) {
        scanDir(dir);
      } else {
        articleFiles.push({ path: `${path}/${file}`, id: file.replace('.md', '') });
      }
    }
  }
  scanDir('./articles/article');
  return articleFiles;
}

export async function init(db: Postgres, git: SimpleGit) {
  await createTable(db, server_table);
  await cloneRepo(git);
  await load(db, git);
}

export async function reload(db: Postgres, git: SimpleGit) {
  await cloneRepo(git);
  await load(db, git);
}