build.ts
20.6 kB · typescript · 572 lines
1import { createHash } from "node:crypto";2import { existsSync, mkdirSync, readFileSync, readdirSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";3import { basename, dirname, extname, join, normalize, relative, resolve } from "node:path";4import { deflateSync } from "node:zlib";5import { collect as gitRoutes, forest, isGit, mirror, owner, print as gitPrint, render as gitRender, type Hooks } from "../git/git.ts";6import { collect as blogRoutes, isBlog, render as blogRender, type Hooks as Posts } from "./blog.ts";7import { index, stamp, type Index } from "./links.ts";89/* TYPES */1011export type Bytes = string | Uint8Array;1213export type Output = { path: string; bytes: Bytes; type?: string };1415export type Link = { route: string; name?: string; at?: string; source?: string };1617export type Route = {18 route: string;19 kind?: string;20 name?: string;21 data?: unknown;22 source?: string;23 inputs?: string[];24 urls?: Link[];25 at?: string;26 hidden?: boolean;27 sitemap?: boolean;28};2930export type Node = { name: string; href?: string; nodes?: Node[]; open?: boolean; lazy?: string; icon?: string; figure?: { dark: string; light: string }; text?: string; dates?: string[] };3132export type Input = { name: string; path: string; files: string[]; missing: boolean };3334export type Bundle = { path: string; out: string; hash: boolean; files: string[]; ext?: string };3536export type Icons = { rows: string[]; svg: string };3738export type Site = {39 root: string;40 out: string;41 config: Config;42 inputs: Record<string, Input>;43 kit: Bundle | null;44 routes: Route[];45 nav: Node[];46 stamp: string;47 index: Index | null;48 copies: Output[];49 styles: string[];50 serves: Map<string, string>;51 made: Set<string>;52 ships: Map<string, string>;53 asset: (name: string) => string;54 input: (name: string) => Input;55 bytes: (file: string) => Uint8Array;56};5758export type Decl = { path: string; out?: string; hash?: boolean; files?: string[]; ext?: string };5960export type Config = {61 title?: string;62 name?: string;63 root?: string;64 inputs?: Record<string, { path: string; ext?: string; deep?: boolean }>;65 kit?: Decl;66 assets?: Decl[];67 manifest?: Record<string, unknown>;68 robots?: { disallow?: string[] };69 llms?: { about?: string; links?: { href: string; name?: string; note?: string }[] };70 [key: string]: unknown;71};7273export type Picked = { routes: Route[]; nav?: Node[] };7475export type Spec = {76 root: string;77 out: string;78 config?: Config;79 templates?: string[];80 prepare?: () => Promise<void> | void;81 collect: (site: Site) => Promise<Picked> | Picked;82 render: (site: Site, route: Route) => Promise<Output[]> | Output[];83 globals?: (site: Site) => Promise<Output[]> | Output[];84 inline?: string[];85 icons?: Icons;86 git?: Hooks;87 blog?: Posts;88 asset?: (name: string, body: Uint8Array) => Bytes;89};9091export type Record_ = { hash: string; at: string; outputs: string[]; types?: Record<string, string> };9293export type Manifest = Record<string, Record_>;9495/* FILES */9697const SKIP = new Set(["node_modules", "dist", ".git", ".cache", "target", "data", "pkg"]);9899export function walk(dir: string, deep = true): string[] {100 if (!existsSync(dir)) return [];101 const out: string[] = [];102 for (const item of readdirSync(dir, { withFileTypes: true })) {103 if (item.name.startsWith(".")) continue;104 const path = join(dir, item.name);105 if (item.isDirectory()) {106 if (deep && !SKIP.has(item.name)) out.push(...walk(path, deep));107 } else out.push(path);108 }109 return out.sort();110}111112const cache = new Map<string, Uint8Array>();113114export function forget() {115 cache.clear();116}117118export function bytes(file: string): Uint8Array {119 const hit = cache.get(file);120 if (hit) return hit;121 const data = new Uint8Array(readFileSync(file));122 cache.set(file, data);123 return data;124}125126const digest = (parts: Bytes[]) => {127 const h = createHash("sha256");128 for (const part of parts) h.update(part);129 return h.digest("hex");130};131132const short = (text: string) => text.slice(0, 8);133134const escape = (text: string) =>135 text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);136137/* SCAN */138139function inputs(root: string, config: Config): Record<string, Input> {140 const found: Record<string, Input> = {};141 for (const [name, decl] of Object.entries(config.inputs ?? {})) {142 const path = resolve(root, decl.path);143 const all = existsSync(path) && statSync(path).isFile() ? [path] : walk(path, decl.deep ?? false);144 const files = decl.ext ? all.filter((f) => f.endsWith(decl.ext!)) : all;145 found[name] = { name, path, files, missing: !existsSync(path) };146 }147 return found;148}149150function templates(spec: Spec): string {151 const here = resolve(import.meta.dir, "..");152 const dirs = [here, ...(spec.templates ?? []).map((d) => resolve(spec.root, d))];153 const parts: Bytes[] = [];154 for (const dir of dirs) for (const file of walk(dir)) parts.push(relative(dir, file), bytes(file));155 return digest(parts);156}157158const outDir = (decl: Decl) => (decl.out ?? "ui").replace(/^\/|\/$/g, "");159160function bundle(root: string, decl: Decl): Bundle {161 const path = resolve(root, decl.path);162 const named = decl.files ?? walk(path, false).map((f) => f.slice(path.length + 1));163 const files = decl.ext ? named.filter((f) => f.endsWith(decl.ext!)) : named;164 return { path, out: outDir(decl), hash: decl.hash ?? false, files };165}166167const decls = (config: Config): Decl[] => [...(config.kit ? [config.kit] : []), ...(config.assets ?? [])];168169/* ASSETS */170171const IMPORT = /((?:\bfrom|\bimport)\s*\(?\s*)(["'])(\.\.?\/[^"']+)\2/g;172173const URLS = /(\burl\(\s*)(["']?)([^"')]+)\2(\s*\))/g;174175const LOADS = /(@import\s+)(["'])([^"']+)\2/g;176177const OUTSIDE = /^(?:[a-z][a-z0-9+.-]*:|[/#])/i;178179function place(one: Bundle, spec: Spec, assets: Map<string, string>, copies: Output[], serves: Map<string, string>) {180 const known = new Set(one.files);181 const busy = new Set<string>();182 const point = (name: string, target: string): string | null => {183 if (OUTSIDE.test(target)) return null;184 const dep = normalize(join(dirname(name), target));185 if (known.has(dep)) return visit(dep);186 const placed = serves.get(join(one.path, dep));187 if (placed) return placed;188 if (existsSync(join(one.path, dep))) throw new Error(`ssg: ${name} names ${dep}, which the bundle's files do not list`);189 return null;190 };191 const visit = (name: string): string => {192 const hit = assets.get(name);193 if (hit) return hit;194 if (busy.has(name)) throw new Error(`ssg: ${name} imports itself around a cycle`);195 busy.add(name);196 const file = join(one.path, name);197 if (!existsSync(file)) throw new Error(`ssg: asset missing: ${file}`);198 const raw = bytes(file);199 let body = spec.asset ? spec.asset(name, raw) : raw;200 if (one.hash && /\.m?js$/.test(name)) {201 const text = typeof body === "string" ? body : new TextDecoder().decode(body);202 body = text.replace(IMPORT, (whole, head: string, quote: string, target: string) => {203 const to = point(name, target);204 return to ? `${head}${quote}${to}${quote}` : whole;205 });206 }207 if (one.hash && /\.css$/.test(name)) {208 const text = typeof body === "string" ? body : new TextDecoder().decode(body);209 body = text210 .replace(URLS, (whole, head: string, quote: string, target: string, tail: string) => {211 const to = point(name, target);212 return to ? `${head}${quote}${to}${quote}${tail}` : whole;213 })214 .replace(LOADS, (whole, head: string, quote: string, target: string) => {215 const to = point(name, target);216 return to ? `${head}${quote}${to}${quote}` : whole;217 });218 }219 const data = typeof body === "string" ? new TextEncoder().encode(body) : body;220 const stem = name.replace(/(\.[^.]+)$/, "");221 const path = one.hash ? `${one.out}/${stem}-${short(digest([data]))}${extname(name)}` : `${one.out}/${name}`;222 assets.set(name, `/${path}`);223 serves.set(join(one.path, name), `/${path}`);224 copies.push({ path, bytes: data });225 busy.delete(name);226 return `/${path}`;227 };228 for (const name of one.files) visit(name);229}230231const shows = (nodes: Node[], href: string): boolean =>232 nodes.some((node) => node.href === href || shows(node.nodes ?? [], href));233234export async function scan(spec: Spec): Promise<Site> {235 if (spec.prepare) await spec.prepare();236 const root = resolve(spec.root);237 const config = spec.config ?? (JSON.parse(readFileSync(join(root, "site.json"), "utf8")) as Config);238 const found = inputs(root, config);239 const list = decls(config).map((decl) => bundle(root, decl));240 const assets = new Map<string, string>();241 const copies: Output[] = [];242 const serves = new Map<string, string>();243 for (const one of list) place(one, spec, assets, copies, serves);244 const site: Site = {245 root,246 out: resolve(spec.out),247 config,248 inputs: found,249 kit: config.kit ? list[0] : null,250 routes: [],251 nav: [],252 stamp: "",253 index: null,254 copies,255 styles: [...assets].filter(([name]) => name.endsWith(".css")).map(([, href]) => href).sort(),256 serves,257 made: new Set(copies.map((one) => one.path)),258 ships: new Map(),259 asset: (name) => {260 const hit = assets.get(name);261 if (!hit) throw new Error(`ssg: no asset named ${name}`);262 return hit;263 },264 input: (name) => {265 const hit = found[name];266 if (!hit) throw new Error(`ssg: ${name} is not declared under inputs in site.json`);267 return hit;268 },269 bytes,270 };271 const picked = await spec.collect(site);272 site.routes = picked.routes;273 site.nav = picked.nav ?? [];274 const written = blogRoutes(site);275 if (written.routes.length) site.routes = [...site.routes, ...written.routes];276 const repo = gitRoutes(site);277 if (repo.routes.length) {278 site.routes = [...site.routes, ...repo.routes];279 if (repo.node && !shows(site.nav, repo.node.href!)) site.nav = [...site.nav, repo.node];280 }281 site.index = index(site);282 site.stamp = digest([283 templates(spec),284 JSON.stringify(site.nav),285 JSON.stringify(config),286 JSON.stringify([...assets]),287 stamp(site.index),288 ]);289 return site;290}291292/* FINGERPRINT */293294export function label(site: Site, file: string): string {295 let base = "";296 let name = "";297 for (const one of Object.values(site.inputs)) {298 if (one.path.length <= base.length) continue;299 if (file !== one.path && !file.startsWith(`${one.path}/`)) continue;300 base = one.path;301 name = one.name;302 }303 if (!base) return basename(file);304 return file === base ? name : `${name}/${file.slice(base.length + 1)}`;305}306307export function fingerprint(site: Site, route: Route, spec?: Spec): string {308 if (isGit(route)) return gitPrint(site, route, spec?.git);309 const files = route.inputs ?? (route.source ? [route.source] : []);310 const named = files.map((file) => [label(site, file), file] as const).sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));311 const parts: Bytes[] = [route.route, route.kind ?? "", JSON.stringify(route.data ?? null), site.stamp];312 for (const [name, file] of named) {313 parts.push(name);314 if (!existsSync(file)) {315 parts.push("gone");316 continue;317 }318 if (statSync(file).isDirectory()) for (const inner of walk(file)) parts.push(relative(file, inner), bytes(inner));319 else parts.push(bytes(file));320 }321 return digest(parts).slice(0, 16);322}323324/* RENDER */325326export async function render(site: Site, route: Route, spec: Spec): Promise<Output[]> {327 if (isGit(route)) return spec.git?.page ? gitRender(site, route, spec) : [];328 if (isBlog(route)) return spec.blog?.page ? blogRender(site, route, spec) : [];329 return await spec.render(site, route);330}331332/* ICONS */333334const SIGNATURE = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);335336const TABLE = new Uint32Array(256).map((_, n) => {337 let c = n;338 for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;339 return c >>> 0;340});341342function crc32(data: Uint8Array): number {343 let c = 0xffffffff;344 for (const b of data) c = TABLE[(c ^ b) & 0xff] ^ (c >>> 8);345 return (c ^ 0xffffffff) >>> 0;346}347348function chunk(type: string, body: Uint8Array): Uint8Array {349 const out = new Uint8Array(12 + body.length);350 const view = new DataView(out.buffer);351 view.setUint32(0, body.length);352 out.set(new TextEncoder().encode(type), 4);353 out.set(body, 8);354 view.setUint32(8 + body.length, crc32(out.subarray(4, 8 + body.length)));355 return out;356}357358function png(size: number, dark: (x: number, y: number) => boolean): Uint8Array {359 const raw = new Uint8Array(size * (size + 1));360 for (let y = 0; y < size; y++) for (let x = 0; x < size; x++) raw[y * (size + 1) + 1 + x] = dark(x, y) ? 0 : 255;361 const head = new Uint8Array(13);362 const view = new DataView(head.buffer);363 view.setUint32(0, size);364 view.setUint32(4, size);365 head[8] = 8;366 const parts = [SIGNATURE, chunk("IHDR", head), chunk("IDAT", new Uint8Array(deflateSync(raw))), chunk("IEND", new Uint8Array(0))];367 const out = new Uint8Array(parts.reduce((n, part) => n + part.length, 0));368 let at = 0;369 for (const part of parts) {370 out.set(part, at);371 at += part.length;372 }373 return out;374}375376function icons({ rows, svg }: Icons): Output[] {377 const n = rows.length;378 const mark = (size: number) => png(size, (x, y) => rows[Math.floor((y * n) / size)][Math.floor((x * n) / size)] === "1");379 return [380 { path: "favicon.svg", bytes: svg },381 { path: "favicon.png", bytes: mark(40) },382 { path: "apple-touch-icon.png", bytes: mark(180) },383 { path: "icon-192.png", bytes: mark(192) },384 { path: "icon-512.png", bytes: mark(512) },385 ];386}387388/* GLOBALS */389390const clean = (root: string) => (root ?? "").replace(/\/$/, "");391392const AGENTS = ["GPTBot", "ClaudeBot", "Claude-Web", "CCBot", "Google-Extended", "anthropic-ai", "PerplexityBot"];393394function links(site: Site, spec: Spec): Link[] {395 const out: Link[] = [];396 for (const route of site.routes) {397 if (route.hidden && !route.sitemap) continue;398 const away = mirror(site, route, spec.git);399 const list = route.urls ?? (route.route.endsWith("/") ? [{ route: route.route, name: route.name }] : []);400 for (const one of list) {401 if (away && owner(one.route)) continue;402 out.push({ route: one.route, name: one.name ?? one.route, at: one.at || route.at });403 }404 }405 return out.sort((a, b) => a.route.localeCompare(b.route));406}407408function robots(site: Site, root: string): string {409 const deny = (site.config.robots?.disallow ?? []).map((path) => `Disallow: ${path}`);410 const lines: string[] = [];411 for (const agent of AGENTS) lines.push(`User-agent: ${agent}`, "Allow: /", "");412 lines.push("User-agent: *", "Allow: /", ...deny, "");413 lines.push(`Sitemap: ${root}/sitemap.xml`, "");414 return lines.join("\n");415}416417function llms(site: Site, root: string): string {418 const decl = site.config.llms ?? {};419 const known = new Set<string>();420 for (const route of site.routes) {421 known.add(route.route);422 for (const one of route.urls ?? []) known.add(one.route);423 }424 const rows = (decl.links ?? [])425 .filter((one) => known.has(one.href))426 .map((one) => `- [${one.name ?? one.href}](${root}${one.href})${one.note ? `: ${one.note}` : ""}`);427 const head = [`# ${site.config.title ?? site.config.name ?? ""}`, "", `> ${root}`, ""];428 if (decl.about) head.push(decl.about, "");429 return [...head, ...rows, ""].join("\n");430}431432export async function globals(site: Site, spec: Spec): Promise<Output[]> {433 const out: Output[] = [...site.copies];434 const root = clean(site.config.root as string);435 const shown = links(site, spec);436 const urls = shown.map((l) => `<url><loc>${escape(root + l.route)}</loc><lastmod>${l.at || today()}</lastmod></url>`);437 out.push({438 path: "sitemap.xml",439 bytes: `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`,440 });441 out.push({ path: "robots.txt", bytes: robots(site, root) });442 if (site.config.llms) out.push({ path: "llms.txt", bytes: llms(site, root) });443 if (site.config.manifest) out.push({ path: "manifest.webmanifest", bytes: JSON.stringify(site.config.manifest, null, 2) + "\n" });444 if (spec.icons) out.push(...icons(spec.icons));445 const wood = forest(site);446 if (wood) out.push({ path: "git/tree.json", bytes: JSON.stringify(wood), type: "application/json" });447 const pub = site.config.inputs?.public ? site.input("public") : null;448 if (pub) for (const file of walk(pub.path)) out.push({ path: file.slice(pub.path.length + 1), bytes: bytes(file) });449 if (spec.globals) out.push(...(await spec.globals(site)));450 return out;451}452453/* GUARD */454455const SCRIPT = /<script\b([^>]*)>([\s\S]*?)<\/script>/g;456457const TYPED = /\btype\s*=\s*["']?([^"'\s>]*)/;458459const RUNS = new Set(["", "module", "text/javascript", "application/javascript", "text/ecmascript", "application/ecmascript"]);460461export function guard(path: string, html: string, known: Set<string>) {462 if (path.startsWith("raw/")) return;463 for (const [, attrs, body] of html.matchAll(SCRIPT)) {464 if (/\bsrc\s*=/.test(attrs)) continue;465 if (!RUNS.has((attrs.match(TYPED)?.[1] ?? "").trim().toLowerCase())) continue;466 if (known.has(body)) continue;467 throw new Error(`ssg: ${path} carries an inline script the boot list does not know: ${body.slice(0, 60)}`);468 }469}470471/* WRITE */472473const today = () => new Date().toISOString().slice(0, 10);474475function put(out: string, item: Output): boolean {476 const path = join(out, item.path);477 const body = typeof item.bytes === "string" ? new TextEncoder().encode(item.bytes) : item.bytes;478 if (existsSync(path)) {479 const old = new Uint8Array(readFileSync(path));480 if (old.length === body.length && Buffer.compare(old, body) === 0) return false;481 }482 mkdirSync(dirname(path), { recursive: true });483 writeFileSync(path, body);484 return true;485}486487/* BUILD */488489function typed(outputs: Output[]): Record<string, string> | undefined {490 const out: Record<string, string> = {};491 for (const item of outputs) if (item.type) out[item.path] = item.type;492 return Object.keys(out).length ? out : undefined;493}494495export async function build(spec: Spec, options: { manifest?: string; force?: boolean; verify?: boolean } = {}) {496 const site = await scan(spec);497 const path = options.manifest ? resolve(spec.root, options.manifest) : "";498 const old: Manifest = path && existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : {};499 const next: Manifest = {};500 const kept = new Set<string>();501 const known = new Set(spec.inline ?? []);502 const verify = options.verify ?? true;503 let rendered = 0;504 let written = 0;505 for (const route of site.routes) {506 if (isGit(route) && !spec.git?.page) continue;507 if (isBlog(route) && !spec.blog?.page) continue;508 const hash = fingerprint(site, route, spec);509 const was = old[route.route];510 const same = !options.force && !!was && was.hash === hash;511 if (was && same && (!verify || was.outputs.every((p) => existsSync(join(site.out, p))))) {512 next[route.route] = was;513 route.at = route.at || was.at;514 for (const p of was.outputs) {515 kept.add(p);516 site.made.add(p);517 }518 continue;519 }520 const outputs = await render(site, route, spec);521 for (const item of outputs) {522 if (item.path.endsWith(".html")) guard(item.path, typeof item.bytes === "string" ? item.bytes : new TextDecoder().decode(item.bytes), known);523 if (put(site.out, item)) written++;524 }525 rendered++;526 const at = route.at || (was && same ? was.at : today());527 next[route.route] = { hash, at, outputs: outputs.map((o) => o.path), types: typed(outputs) };528 route.at = at;529 for (const item of outputs) {530 kept.add(item.path);531 site.made.add(item.path);532 }533 }534 for (const item of await globals(site, spec)) {535 if (put(site.out, item)) written++;536 kept.add(item.path);537 }538 let removed = 0;539 const drop = (p: string) => {540 const file = join(site.out, p);541 if (!existsSync(file)) return;542 unlinkSync(file);543 removed++;544 let dir = dirname(file);545 while (dir !== site.out && existsSync(dir) && readdirSync(dir).length === 0) {546 rmdirSync(dir);547 dir = dirname(dir);548 }549 };550 for (const record of Object.values(old)) for (const p of record.outputs) if (!kept.has(p)) drop(p);551 for (const out of new Set(decls(site.config).map(outDir))) {552 for (const file of walk(join(site.out, out))) {553 const p = relative(site.out, file);554 if (!kept.has(p)) drop(p);555 }556 }557 if (path) {558 mkdirSync(dirname(path), { recursive: true });559 writeFileSync(path, JSON.stringify(next, null, 2) + "\n");560 }561 return { site, manifest: next, rendered, written, removed };562}563564/* HELPERS */565566export const page = (route: string) => (route === "/" ? "index.html" : `${route.replace(/^\/|\/$/g, "")}/index.html`);567568export const jsonText = (data: unknown) => JSON.stringify(data).replace(/</g, "\\u003c");569570export const jsonScript = (data: unknown) => `<script type="application/ld+json">${jsonText(data)}</script>`;571572export { escape, digest, short, today };