push.ts

1import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";2import { tmpdir } from "node:os";3import { dirname, join } from "node:path";4import { spec } from "./build.ts";5import { need } from "../lib/env.ts";6import { client, del, getText, putBytes } from "../../aws/s3.ts";7import type { S3Client } from "bun";8import type { Manifest, Output } from "kit/ssg/build.ts";910const { build, digest, globals, today } = await import("kit/ssg/build.ts");1112/* WHERE */1314const PREFIX = "site/";15const GUARD = ["cdn/", "art/"];16const REMOTE = "data/build/manifest.json";17const ASSET = "@";18const BATCH = 32;19const DRY = process.env.DRY === "1";20const HOLD = join(process.env.DRY_DIR ?? "/tmp/carlomitchener", "manifest.json");2122/* HEADERS */2324const IMMUTABLE = "public, max-age=31536000, immutable";25const REVALIDATE = "public, max-age=0, must-revalidate";2627const HASHED = [/(^|\/)lib-[^/]+\.js$/, /(^|\/)lib-[^/]+\.css$/, /\.wasm$/, /-[0-9a-f]{8}\.[^./]+$/];2829const TYPES: Record<string, string> = {30  html: "text/html; charset=utf-8",31  js: "text/javascript; charset=utf-8",32  mjs: "text/javascript; charset=utf-8",33  css: "text/css; charset=utf-8",34  json: "application/json",35  map: "application/json",36  webmanifest: "application/manifest+json",37  xml: "application/xml",38  txt: "text/plain; charset=utf-8",39  md: "text/markdown; charset=utf-8",40  tex: "text/plain; charset=utf-8",41  wasm: "application/wasm",42  svg: "image/svg+xml",43  png: "image/png",44  jpg: "image/jpeg",45  jpeg: "image/jpeg",46  gif: "image/gif",47  webp: "image/webp",48  ico: "image/x-icon",49  pdf: "application/pdf",50  woff2: "font/woff2",51};5253const kind = (path: string) => TYPES[(path.match(/\.([^./]+)$/)?.[1] ?? "").toLowerCase()] ?? "application/octet-stream";5455const cache = (path: string) => (HASHED.some((re) => re.test(path)) ? IMMUTABLE : REVALIDATE);5657/* GUARD */5859const mine = (path: string) => !GUARD.some((one) => path.startsWith(one));6061async function sweep(s3: S3Client, prefix: string, out: string[]): Promise<string[]> {62  let token: string | undefined;63  do {64    const page = await s3.list({ prefix, delimiter: "/", maxKeys: 1000, continuationToken: token });65    for (const item of page?.contents ?? []) out.push(item.key.slice(PREFIX.length));66    for (const one of page?.commonPrefixes ?? []) {67      const next = typeof one === "string" ? one : one.prefix;68      if (!GUARD.some((one) => next === PREFIX + one)) await sweep(s3, next, out);69    }70    token = page?.isTruncated ? (page.nextContinuationToken ?? undefined) : undefined;71  } while (token);72  return out;73}7475/* MANIFEST */7677function assets(items: Output[], old: Manifest): Manifest {78  const out: Manifest = {};79  for (const item of items) {80    const body = typeof item.bytes === "string" ? new TextEncoder().encode(item.bytes) : item.bytes;81    const hash = digest([body]).slice(0, 16);82    const was = old[ASSET + item.path];83    out[ASSET + item.path] = was && was.hash === hash ? was : { hash, at: today(), outputs: [item.path] };84  }85  return out;86}8788function typing(manifest: Manifest): Map<string, string> {89  const out = new Map<string, string>();90  for (const record of Object.values(manifest)) for (const [path, type] of Object.entries(record.types ?? {})) out.set(path, type);91  return out;92}9394function spread(manifest: Manifest): Map<string, string> {95  const out = new Map<string, string>();96  for (const [key, record] of Object.entries(manifest)) for (const path of record.outputs) out.set(path, key);97  return out;98}99100/* LOCAL */101102const held = () => (existsSync(HOLD) ? readFileSync(HOLD, "utf8") : null);103104function hold(text: string) {105  mkdirSync(dirname(HOLD), { recursive: true });106  writeFileSync(HOLD, text);107}108109/* PUSH */110111export async function push(options: { dry?: boolean } = {}): Promise<{ rendered: number; uploaded: number; deleted: number }> {112  const s3 = DRY ? null : client(need("CARLOMITCHENER_BUCKET"));113  const found = s3 ? await getText(s3, REMOTE) : held();114  const old: Manifest = found ? JSON.parse(found) : {};115  const carry = join(tmpdir(), `carlomitchener-remote-${process.pid}.json`);116  writeFileSync(carry, JSON.stringify(old, null, 2) + "\n");117  const done = await build(spec, { manifest: carry, verify: false });118  rmSync(carry, { force: true });119  const next: Manifest = { ...done.manifest, ...assets(await globals(done.site, spec), old) };120  const want = spread(next);121  const had = spread(old);122  const types = typing(next);123  const upload: string[] = [];124  for (const [path, key] of want) {125    if (!mine(path)) continue;126    const was = old[key];127    if (was && was.hash === next[key]!.hash && had.has(path)) continue;128    upload.push(path);129  }130  const seen = found || !s3 ? [...had.keys()] : await sweep(s3, PREFIX, []);131  const remove = seen.filter((path) => mine(path) && !want.has(path));132  const text = JSON.stringify(next, null, 2) + "\n";133  if (!options.dry) {134    if (s3) {135      for (let i = 0; i < upload.length; i += BATCH) {136        await Promise.all(137          upload.slice(i, i + BATCH).map((path) =>138            putBytes(s3, PREFIX + path, new Uint8Array(readFileSync(join(done.site.out, path))), {139              type: types.get(path) ?? kind(path),140              cacheControl: types.has(path) ? REVALIDATE : cache(path),141            }),142          ),143        );144      }145      if (remove.length) await del(s3, remove.map((path) => PREFIX + path), BATCH);146      await putBytes(s3, REMOTE, text, { type: "application/json", cacheControl: REVALIDATE });147    } else hold(text);148  }149  return { rendered: done.rendered, uploaded: upload.length, deleted: remove.length };150}151152/* MAIN */153154if (import.meta.main) {155  const dry = process.argv.includes("--dry");156  const done = await push({ dry });157  console.log(`push${dry ? " --dry" : ""}${DRY ? " --local" : ""}: ${done.rendered} rendered, ${done.uploaded} uploaded, ${done.deleted} deleted`);158}