Add SEO and agent-readable documentation

site/decodal-site/src/layouts/ManualLayout.astro

@@ -11,9 +11,12 @@ const {
   playground = false,
   frontpage = false,
   locale,
+  noindex = false,
+  standalone = false,
 } = Astro.props;
 
 const pathname = Astro.url.pathname;
+const site = Astro.site ?? new URL('https://decodal.hareworks.net');
 const language = locale ?? 'en';
 const pageDescription = description ?? (language === 'ja'
   ? 'Decodalは、設定、制約、デフォルトのための決定的なデータ言語です。'
@@ -22,6 +25,20 @@ const docsLocale = language === 'ja' ? 'ja' : 'en';
 const otherDocsLocale = docsLocale === 'ja' ? 'en' : 'ja';
 const isDocsPage = pathname.startsWith('/docs/');
 const localizedDocPath = active && active !== 'index' ? `${active}/` : '';
+const canonicalUrl = new URL(pathname, site).href;
+const englishUrl = new URL(frontpage ? '/' : `/docs/en/${localizedDocPath}`, site).href;
+const japaneseUrl = new URL(frontpage ? '/ja/' : `/docs/ja/${localizedDocPath}`, site).href;
+const markdownPath = frontpage
+  ? language === 'ja' ? '/ja.md' : '/index.md'
+  : isDocsPage
+    ? active === 'index'
+      ? `/docs/${docsLocale}.md`
+      : `/docs/${docsLocale}/${active}.md`
+    : undefined;
+const markdownUrl = markdownPath ? new URL(markdownPath, site).href : undefined;
+const sitemapUrl = new URL('/sitemap-index.xml', site).href;
+const llmsUrl = new URL('/llms.txt', site).href;
+const socialImageUrl = new URL('/og.png', site).href;
 const homeHref = language === 'ja' ? '/ja/' : '/';
 const alternateHref = frontpage
   ? language === 'ja' ? '/' : '/ja/'
@@ -32,6 +49,53 @@ const manualLabel = language === 'ja' ? 'マニュアル' : 'Manual';
 const manualHref = `/docs/${docsLocale}/introduction/`;
 const manualIndexHref = `/docs/${docsLocale}/`;
 const nav = getNav(docsLocale);
+const localizedPage = !noindex && (frontpage || isDocsPage);
+const structuredData = isDocsPage
+  ? {
+      '@context': 'https://schema.org',
+      '@type': 'TechArticle',
+      headline: title,
+      description: pageDescription,
+      inLanguage: language,
+      url: canonicalUrl,
+      isPartOf: {
+        '@type': 'WebSite',
+        name: 'Decodal',
+        url: new URL('/', site).href,
+      },
+    }
+  : playground
+    ? {
+        '@context': 'https://schema.org',
+        '@type': 'WebApplication',
+        name: 'Decodal Playground',
+        description: pageDescription,
+        applicationCategory: 'DeveloperApplication',
+        operatingSystem: 'Web',
+        url: canonicalUrl,
+      }
+    : {
+        '@context': 'https://schema.org',
+        '@graph': [
+          {
+            '@type': 'WebSite',
+            name: 'Decodal',
+            description: pageDescription,
+            inLanguage: language,
+            url: canonicalUrl,
+          },
+          {
+            '@type': 'SoftwareSourceCode',
+            name: 'Decodal',
+            description: pageDescription,
+            codeRepository: 'https://gitea.hareworks.net/Hare/Decodal',
+            license: [
+              'https://opensource.org/license/mit',
+              'https://www.apache.org/licenses/LICENSE-2.0',
+            ],
+          },
+        ],
+      };
 
 function isCurrent(path) {
   return pathname === path || (path !== '/' && pathname.startsWith(path));
@@ -55,9 +119,33 @@ function renderNav(items, prefix = []) {
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta name="description" content={pageDescription} />
-    {(frontpage || isDocsPage) && <link rel="alternate" hreflang="en" href={frontpage ? '/' : `/docs/en/${localizedDocPath}`} />}
-    {(frontpage || isDocsPage) && <link rel="alternate" hreflang="ja" href={frontpage ? '/ja/' : `/docs/ja/${localizedDocPath}`} />}
-    {(frontpage || isDocsPage) && <link rel="alternate" hreflang="x-default" href={frontpage ? '/' : `/docs/en/${localizedDocPath}`} />}
+    {noindex && <meta name="robots" content="noindex" />}
+    {!noindex && <link rel="canonical" href={canonicalUrl} />}
+    <link rel="icon" href="/favicon.svg" type="image/svg+xml" />
+    <link rel="sitemap" href={sitemapUrl} />
+    <link rel="describedby" href={llmsUrl} type="text/markdown" />
+    {markdownUrl && <link rel="alternate" type="text/markdown" href={markdownUrl} />}
+    {localizedPage && <link rel="alternate" hreflang="en" href={englishUrl} />}
+    {localizedPage && <link rel="alternate" hreflang="ja" href={japaneseUrl} />}
+    {localizedPage && <link rel="alternate" hreflang="x-default" href={englishUrl} />}
+    <meta property="og:type" content={isDocsPage ? 'article' : 'website'} />
+    <meta property="og:site_name" content="Decodal" />
+    <meta property="og:title" content={title} />
+    <meta property="og:description" content={pageDescription} />
+    <meta property="og:url" content={canonicalUrl} />
+    <meta property="og:locale" content={language === 'ja' ? 'ja_JP' : 'en_US'} />
+    {localizedPage && <meta property="og:locale:alternate" content={language === 'ja' ? 'en_US' : 'ja_JP'} />}
+    <meta property="og:image" content={socialImageUrl} />
+    <meta property="og:image:type" content="image/png" />
+    <meta property="og:image:width" content="1200" />
+    <meta property="og:image:height" content="630" />
+    <meta property="og:image:alt" content="Decodal — Deferred Constraint Data Language" />
+    <meta name="twitter:card" content="summary_large_image" />
+    <meta name="twitter:title" content={title} />
+    <meta name="twitter:description" content={pageDescription} />
+    <meta name="twitter:image" content={socialImageUrl} />
+    <meta name="twitter:image:alt" content="Decodal — Deferred Constraint Data Language" />
+    {!noindex && <script type="application/ld+json" is:inline set:html={JSON.stringify(structuredData)} />}
     <title>{title}</title>
   </head>
   <body>
@@ -73,8 +161,8 @@ function renderNav(items, prefix = []) {
         {(frontpage || isDocsPage) && <a class="language-switch" href={alternateHref} hreflang={otherDocsLocale} aria-label={alternateAriaLabel}>{alternateLabel}</a>}
       </nav>
     </header>
-    <div class:list={{ layout: true, 'wide-layout': playground || frontpage, 'playground-layout': playground }}>
-      {!playground && !frontpage && (
+    <div class:list={{ layout: true, 'wide-layout': playground || frontpage || standalone, 'playground-layout': playground }}>
+      {!playground && !frontpage && !standalone && (
         <aside class="sidebar">
           <div class="sidebar-content">
             <a class="sidebar-title" href={manualIndexHref}>{manualLabel}</a>
@@ -82,7 +170,7 @@ function renderNav(items, prefix = []) {
           </div>
         </aside>
       )}
-      <main class={playground ? 'playground' : frontpage ? 'frontpage' : ''}>
+      <main class={playground ? 'playground' : frontpage ? 'frontpage' : standalone ? 'standalone' : ''}>
         <slot />
       </main>
     </div>

site/decodal-site/src/lib/agent-content.js

@@ -0,0 +1,134 @@
+import { docTitle, orderedDocSlugs, sourceMarkdown } from './docs.js';
+
+export const SITE_URL = 'https://decodal.hareworks.net';
+
+const homePages = {
+  en: `# Decodal
+
+Decodal is the Deferred Constraint Data Language: a small deterministic language for describing, composing, validating, and materializing structured data.
+
+Configuration values, constraints, defaults, and derived values share one expression system. The host owns filesystems and external I/O; Decodal owns deterministic evaluation.
+
+\`\`\`dcdl
+let
+    Port = Int & >= 1 & <= 65535;
+    Service = {
+        host = String;
+        port = Port default 8080;
+    };
+in
+{
+    host = "127.0.0.1";
+} as Service;
+\`\`\`
+
+## Documentation
+
+- [English manual](/docs/en.md)
+- [Embedding guide](/docs/en/embedding.md)
+- [Packages and integrations](/docs/en/components.md)
+- [Japanese site](/ja.md)
+
+## Resources
+
+- [Browser playground](/playground/)
+- [Source repository](https://gitea.hareworks.net/Hare/Decodal)
+`,
+  ja: `# Decodal
+
+Decodalは、構造化データの記述、合成、検証、具体化のための小さく決定的な言語、Deferred Constraint Data Languageである。
+
+設定値、制約、デフォルト、派生値をひとつの式体系で扱う。filesystemや外部I/Oはホストが担い、Decodalは決定的な評価を担う。
+
+\`\`\`dcdl
+let
+    Port = Int & >= 1 & <= 65535;
+    Service = {
+        host = String;
+        port = Port default 8080;
+    };
+in
+{
+    host = "127.0.0.1";
+} as Service;
+\`\`\`
+
+## ドキュメント
+
+- [日本語マニュアル](/docs/ja.md)
+- [組み込みガイド](/docs/ja/embedding.md)
+- [パッケージと統合](/docs/ja/components.md)
+- [English site](/index.md)
+
+## リソース
+
+- [ブラウザPlayground](/playground/)
+- [ソースリポジトリ](https://gitea.hareworks.net/Hare/Decodal)
+`,
+};
+
+export function homeMarkdown(locale = 'en') {
+  return homePages[locale] ?? homePages.en;
+}
+
+function markdownUrl(slug, locale) {
+  return slug === 'index'
+    ? `${SITE_URL}/docs/${locale}.md`
+    : `${SITE_URL}/docs/${locale}/${slug}.md`;
+}
+
+function manualLinks(locale) {
+  return orderedDocSlugs(locale, { includeIndex: true })
+    .map((slug) => `- [${docTitle(slug, locale)}](${markdownUrl(slug, locale)})`)
+    .join('\n');
+}
+
+export function llmsText() {
+  return `# Decodal
+
+> Decodal is a deterministic data language for configuration, constraints, defaults, validation, and materialization.
+
+Use the English manual as the default reference. Japanese translations cover the same document structure. Markdown endpoints are generated directly from the manual sources rather than converted from HTML.
+
+## English documentation
+
+${manualLinks('en')}
+
+## 日本語ドキュメント
+
+${manualLinks('ja')}
+
+## Complete manuals
+
+- [Complete English manual](${SITE_URL}/llms-full.txt): All English manual pages in one text resource.
+- [日本語マニュアル全文](${SITE_URL}/ja/llms-full.txt): 日本語マニュアルの全ページをまとめたテキスト。
+
+## Resources
+
+- [Project overview](${SITE_URL}/index.md): Concise English overview.
+- [プロジェクト概要](${SITE_URL}/ja.md): 日本語の概要。
+- [Browser playground](${SITE_URL}/playground/): Interactive Decodal runtime and editor.
+- [Source repository](https://gitea.hareworks.net/Hare/Decodal): Rust, JavaScript, language service, and website source.
+`;
+}
+
+export function fullManual(locale = 'en') {
+  const heading = locale === 'ja' ? 'Decodalマニュアル全文' : 'Complete Decodal Manual';
+  const note = locale === 'ja'
+    ? 'このファイルは日本語マニュアルの原本から生成されている。各章の個別Markdownは /llms.txt を参照。'
+    : 'This file is generated from the English manual sources. See /llms.txt for links to individual Markdown pages.';
+  const pages = orderedDocSlugs(locale, { includeIndex: true })
+    .map((slug) => sourceMarkdown(slug, locale).trim())
+    .join('\n\n---\n\n');
+  return `# ${heading}\n\n${note}\n\n${pages}\n`;
+}
+
+export function textResponse(content, language = 'en', contentType = 'text/markdown') {
+  return new Response(content, {
+    headers: {
+      'Content-Type': `${contentType}; charset=utf-8`,
+      'Content-Language': language,
+      'Content-Signal': 'search=yes, ai-input=yes, ai-train=no',
+    },
+  });
+}

site/decodal-site/src/lib/docs.js

@@ -173,6 +173,16 @@ export function allDocSlugs(locale = 'en') {
   return Object.keys(docsByLocale[locale] ?? {}).filter((slug) => slug !== 'index');
 }
 
+function flattenNav(items) {
+  return items.flatMap((item) => [item.slug, ...(item.children ? flattenNav(item.children) : [])]);
+}
+
+export function orderedDocSlugs(locale = 'en', { includeIndex = false } = {}) {
+  const localizedDocs = docsByLocale[locale] ?? docsByLocale.en;
+  const slugs = flattenNav(getNav(locale)).filter((slug) => localizedDocs[slug]);
+  return includeIndex ? ['index', ...slugs] : slugs;
+}
+
 function findDocTitle(slug, items) {
   for (const item of items) {
     if (item.slug === slug) return item.title;
@@ -185,13 +195,51 @@ function findDocTitle(slug, items) {
 }
 
 export function docTitle(slug, locale = 'en') {
+  if (slug === 'index') return locale === 'ja' ? 'Decodalマニュアル' : 'Decodal Manual';
   return findDocTitle(slug, getNav(locale)) ?? slug;
 }
 
-export function renderMarkdown(slug, locale = 'en') {
+function getSource(slug, locale) {
   const localizedDocs = docsByLocale[locale] ?? docsByLocale.en;
-  const source = localizedDocs[slug] ?? localizedDocs.index;
-  const html = marked.parse(source ?? '# Not found\n');
+  return localizedDocs[slug] ?? localizedDocs.index ?? '# Not found\n';
+}
+
+function stripInlineMarkdown(value) {
+  return value
+    .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+    .replace(/[`*_~]/g, '')
+    .replace(/<[^>]+>/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+export function docDescription(slug, locale = 'en') {
+  const blocks = getSource(slug, locale)
+    .replace(/```[\s\S]*?```/g, '')
+    .split(/\n\s*\n/)
+    .map((block) => block.trim())
+    .filter((block) => block && !block.startsWith('#') && !/^[-*]\s/.test(block));
+  const description = stripInlineMarkdown(blocks[0] ?? '');
+  if (description.length <= 180) return description;
+  const shortened = description.slice(0, 177).replace(/\s+\S*$/, '').trimEnd();
+  return `${shortened || description.slice(0, 177)}…`;
+}
+
+export function sourceMarkdown(slug, locale = 'en') {
+  return getSource(slug, locale).replace(/\]\(([^)\s]+)\)/g, (all, destination) => {
+    const [href, hash = ''] = destination.split('#', 2);
+    if (!href.endsWith('.md')) return all;
+    const target = normalizeDocLink(locale, slug, href.slice(0, -3));
+    const pathname = target === 'index'
+      ? `/docs/${locale}.md`
+      : `/docs/${locale}/${target}.md`;
+    return `](${pathname}${hash ? `#${hash}` : ''})`;
+  });
+}
+
+export function renderMarkdown(slug, locale = 'en') {
+  const html = marked.parse(getSource(slug, locale));
   return html.replace(/href="([^"#][^"]*)\.md(#[^"]*)?"/g, (_all, href, hash = '') => {
     const target = normalizeDocLink(locale, slug, href);
     const pathname = target === 'index' ? `/docs/${locale}/` : `/docs/${locale}/${target}/`;

site/decodal-site/src/pages/404.astro

@@ -0,0 +1,15 @@
+---
+import ManualLayout from '../layouts/ManualLayout.astro';
+---
+<ManualLayout
+  title="Page not found — Decodal"
+  description="The requested Decodal page could not be found."
+  noindex
+  standalone
+>
+  <article class="markdown not-found">
+    <h1>Page not found</h1>
+    <p>The requested page does not exist or has moved.</p>
+    <p><a href="/">Return home</a> or <a href="/docs/en/">open the manual</a>.</p>
+  </article>
+</ManualLayout>

site/decodal-site/src/pages/docs/en/[...slug].astro

@@ -1,6 +1,6 @@
 ---
 import ManualLayout from '../../../layouts/ManualLayout.astro';
-import { allDocSlugs, docTitle, renderMarkdown } from '../../../lib/docs.js';
+import { allDocSlugs, docDescription, docTitle, renderMarkdown } from '../../../lib/docs.js';
 import '@fontsource/ibm-plex-sans-jp/latin-400.css';
 import '@fontsource/ibm-plex-sans-jp/latin-600.css';
 
@@ -14,7 +14,8 @@ export function getStaticPaths() {
 const { slug } = Astro.props;
 const html = renderMarkdown(slug, 'en');
 const title = docTitle(slug, 'en');
+const description = docDescription(slug, 'en');
 ---
-<ManualLayout title={`${title} — Decodal Manual`} active={slug} locale="en">
+<ManualLayout title={`${title} — Decodal Manual`} description={description} active={slug} locale="en">
   <article class="markdown" set:html={html} />
 </ManualLayout>

site/decodal-site/src/pages/docs/en/[...slug]/index.md.ts

@@ -0,0 +1,10 @@
+import { textResponse } from '../../../../lib/agent-content.js';
+import { allDocSlugs, sourceMarkdown } from '../../../../lib/docs.js';
+
+export function getStaticPaths() {
+  return allDocSlugs('en').map((slug) => ({ params: { slug }, props: { slug } }));
+}
+
+export function GET({ props }) {
+  return textResponse(sourceMarkdown(props.slug, 'en'), 'en');
+}

site/decodal-site/src/pages/docs/en/index.md.ts

@@ -0,0 +1,6 @@
+import { textResponse } from '../../../lib/agent-content.js';
+import { sourceMarkdown } from '../../../lib/docs.js';
+
+export function GET() {
+  return textResponse(sourceMarkdown('index', 'en'), 'en');
+}

site/decodal-site/src/pages/docs/ja/[...slug].astro

@@ -1,6 +1,6 @@
 ---
 import ManualLayout from '../../../layouts/ManualLayout.astro';
-import { allDocSlugs, docTitle, renderMarkdown } from '../../../lib/docs.js';
+import { allDocSlugs, docDescription, docTitle, renderMarkdown } from '../../../lib/docs.js';
 import '@fontsource/ibm-plex-sans-jp/400.css';
 import '@fontsource/ibm-plex-sans-jp/600.css';
 
@@ -14,7 +14,8 @@ export function getStaticPaths() {
 const { slug } = Astro.props;
 const html = renderMarkdown(slug, 'ja');
 const title = docTitle(slug, 'ja');
+const description = docDescription(slug, 'ja');
 ---
-<ManualLayout title={`${title} — Decodalマニュアル`} active={slug} locale="ja">
+<ManualLayout title={`${title} — Decodalマニュアル`} description={description} active={slug} locale="ja">
   <article class="markdown" set:html={html} />
 </ManualLayout>

site/decodal-site/src/pages/docs/ja/[...slug]/index.md.ts

@@ -0,0 +1,10 @@
+import { textResponse } from '../../../../lib/agent-content.js';
+import { allDocSlugs, sourceMarkdown } from '../../../../lib/docs.js';
+
+export function getStaticPaths() {
+  return allDocSlugs('ja').map((slug) => ({ params: { slug }, props: { slug } }));
+}
+
+export function GET({ props }) {
+  return textResponse(sourceMarkdown(props.slug, 'ja'), 'ja');
+}

site/decodal-site/src/pages/docs/ja/index.md.ts

@@ -0,0 +1,6 @@
+import { textResponse } from '../../../lib/agent-content.js';
+import { sourceMarkdown } from '../../../lib/docs.js';
+
+export function GET() {
+  return textResponse(sourceMarkdown('index', 'ja'), 'ja');
+}

site/decodal-site/src/pages/index.md.ts

@@ -0,0 +1,5 @@
+import { homeMarkdown, textResponse } from '../lib/agent-content.js';
+
+export function GET() {
+  return textResponse(homeMarkdown('en'), 'en');
+}

site/decodal-site/src/pages/ja/index.md.ts

@@ -0,0 +1,5 @@
+import { homeMarkdown, textResponse } from '../../lib/agent-content.js';
+
+export function GET() {
+  return textResponse(homeMarkdown('ja'), 'ja');
+}

site/decodal-site/src/pages/ja/llms-full.txt.ts

@@ -0,0 +1,5 @@
+import { fullManual, textResponse } from '../../lib/agent-content.js';
+
+export function GET() {
+  return textResponse(fullManual('ja'), 'ja', 'text/plain');
+}

site/decodal-site/src/pages/llms-full.txt.ts

@@ -0,0 +1,5 @@
+import { fullManual, textResponse } from '../lib/agent-content.js';
+
+export function GET() {
+  return textResponse(fullManual('en'), 'en', 'text/plain');
+}

site/decodal-site/src/pages/llms.txt.ts

@@ -0,0 +1,5 @@
+import { llmsText, textResponse } from '../lib/agent-content.js';
+
+export function GET() {
+  return textResponse(llmsText(), 'en', 'text/plain');
+}

site/decodal-site/src/pages/robots.txt.ts

@@ -0,0 +1,12 @@
+import { SITE_URL, textResponse } from '../lib/agent-content.js';
+
+const robots = `User-agent: *
+Content-Signal: search=yes, ai-input=yes, ai-train=no
+Allow: /
+
+Sitemap: ${SITE_URL}/sitemap-index.xml
+`;
+
+export function GET() {
+  return textResponse(robots, 'en', 'text/plain');
+}