more robust error handling, config hotreload

Jacky Zhao

2023-08-05 c402f0c3857a75cc101c3459866c94e646fd2957

more robust error handling, config hotreload

 content/features/upcoming features.md

@@ -4,8 +4,6 @@

## high priority

- images in same folder are broken on shortest path mode
- watch mode for config/source code
- block links: https://help.obsidian.md/Linking+notes+and+files/Internal+links#Link+to+a+block+in+a+note
- note/header/block transcludes: https://help.obsidian.md/Linking+notes+and+files/Embedding+files

@@ -22,7 +20,5 @@
- https://help.obsidian.md/Advanced+topics/Using+Obsidian+URI
- audio/video embed styling
- Canvas
- mermaid styling: https://mermaid.js.org/config/theming.html#theme-variables-reference-table
  - https://github.com/jackyzha0/quartz/issues/331
- parse all images in page: use this for page lists if applicable?
- CV mode? with print stylesheet

 package-lock.json

@@ -1,12 +1,12 @@
{
  "name": "@jackyzha0/quartz",
  "version": "4.0.6",
  "version": "4.0.7",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@jackyzha0/quartz",
      "version": "4.0.6",
      "version": "4.0.7",
      "license": "MIT",
      "dependencies": {
        "@clack/prompts": "^0.6.3",

 package.json

@@ -2,7 +2,7 @@
  "name": "@jackyzha0/quartz",
  "description": "🌱 publish your digital garden and notes as a website",
  "private": true,
  "version": "4.0.6",
  "version": "4.0.7",
  "type": "module",
  "author": "jackyzha0 <j.zhao2k19@gmail.com>",
  "license": "MIT",

 quartz/bootstrap-cli.mjs

@@ -9,9 +9,13 @@
import fs from "fs"
import { intro, isCancel, outro, select, text } from "@clack/prompts"
import { rimraf } from "rimraf"
import chokidar from "chokidar"
import prettyBytes from "pretty-bytes"
import { execSync, spawnSync } from "child_process"
import { transform as cssTransform } from "lightningcss"
import http from "http"
import serveHandler from "serve-handler"
import { WebSocketServer } from "ws"

const ORIGIN_NAME = "origin"
const UPSTREAM_NAME = "upstream"
@@ -287,86 +291,132 @@
    console.log(chalk.green("Done!"))
  })
  .command("build", "Build Quartz into a bundle of static HTML files", BuildArgv, async (argv) => {
    const result = await esbuild
      .build({
        entryPoints: [fp],
        outfile: path.join("quartz", cacheFile),
        bundle: true,
        keepNames: true,
        minify: true,
        platform: "node",
        format: "esm",
        jsx: "automatic",
        jsxImportSource: "preact",
        packages: "external",
        metafile: true,
        sourcemap: true,
        plugins: [
          sassPlugin({
            type: "css-text",
            cssImports: true,
            async transform(css) {
              const { code } = cssTransform({
                filename: "style.css",
                code: Buffer.from(css),
                minify: true,
              })
              return code.toString()
            },
          }),
          {
            name: "inline-script-loader",
            setup(build) {
              build.onLoad({ filter: /\.inline\.(ts|js)$/ }, async (args) => {
                let text = await promises.readFile(args.path, "utf8")

                // remove default exports that we manually inserted
                text = text.replace("export default", "")
                text = text.replace("export", "")

                const sourcefile = path.relative(path.resolve("."), args.path)
                const resolveDir = path.dirname(sourcefile)
                const transpiled = await esbuild.build({
                  stdin: {
                    contents: text,
                    loader: "ts",
                    resolveDir,
                    sourcefile,
                  },
                  write: false,
                  bundle: true,
                  platform: "browser",
                  format: "esm",
                })
                const rawMod = transpiled.outputFiles[0].text
                return {
                  contents: rawMod,
                  loader: "text",
                }
              })
            },
    console.log(chalk.bgGreen.black(`\n Quartz v${version} \n`))
    const ctx = await esbuild.context({
      entryPoints: [fp],
      outfile: path.join("quartz", cacheFile),
      bundle: true,
      keepNames: true,
      minify: true,
      platform: "node",
      format: "esm",
      jsx: "automatic",
      jsxImportSource: "preact",
      packages: "external",
      metafile: true,
      sourcemap: true,
      plugins: [
        sassPlugin({
          type: "css-text",
          cssImports: true,
          async transform(css) {
            const { code } = cssTransform({
              filename: "style.css",
              code: Buffer.from(css),
              minify: true,
            })
            return code.toString()
          },
        ],
      })
      .catch((err) => {
        }),
        {
          name: "inline-script-loader",
          setup(build) {
            build.onLoad({ filter: /\.inline\.(ts|js)$/ }, async (args) => {
              let text = await promises.readFile(args.path, "utf8")

              // remove default exports that we manually inserted
              text = text.replace("export default", "")
              text = text.replace("export", "")

              const sourcefile = path.relative(path.resolve("."), args.path)
              const resolveDir = path.dirname(sourcefile)
              const transpiled = await esbuild.build({
                stdin: {
                  contents: text,
                  loader: "ts",
                  resolveDir,
                  sourcefile,
                },
                write: false,
                bundle: true,
                platform: "browser",
                format: "esm",
              })
              const rawMod = transpiled.outputFiles[0].text
              return {
                contents: rawMod,
                loader: "text",
              }
            })
          },
        },
      ],
    })

    let clientRefresh = () => {}
    let closeHandler = null
    const build = async () => {
      const result = await ctx.rebuild().catch((err) => {
        console.error(`${chalk.red("Couldn't parse Quartz configuration:")} ${fp}`)
        console.log(`Reason: ${chalk.grey(err)}`)
        process.exit(1)
      })

    if (argv.bundleInfo) {
      const outputFileName = "quartz/.quartz-cache/transpiled-build.mjs"
      const meta = result.metafile.outputs[outputFileName]
      console.log(
        `Successfully transpiled ${Object.keys(meta.inputs).length} files (${prettyBytes(
          meta.bytes,
        )})`,
      )
      console.log(await esbuild.analyzeMetafile(result.metafile, { color: true }))
      if (argv.bundleInfo) {
        const outputFileName = "quartz/.quartz-cache/transpiled-build.mjs"
        const meta = result.metafile.outputs[outputFileName]
        console.log(
          `Successfully transpiled ${Object.keys(meta.inputs).length} files (${prettyBytes(
            meta.bytes,
          )})`,
        )
        console.log(await esbuild.analyzeMetafile(result.metafile, { color: true }))
      }

      // bypass module cache
      const { default: buildQuartz } = await import(cacheFile + `?update=${new Date()}`)
      if (closeHandler) {
        await closeHandler()
      }

      closeHandler = await buildQuartz(argv, clientRefresh)
      clientRefresh()
    }

    const { default: buildQuartz } = await import(cacheFile)
    buildQuartz(argv, version)
    await build()
    if (argv.serve) {
      const wss = new WebSocketServer({ port: 3001 })
      const connections = []
      wss.on("connection", (ws) => connections.push(ws))
      clientRefresh = () => connections.forEach((conn) => conn.send("rebuild"))
      const server = http.createServer(async (req, res) => {
        await serveHandler(req, res, {
          public: argv.output,
          directoryListing: false,
        })
        const status = res.statusCode
        const statusString =
          status >= 200 && status < 300
            ? chalk.green(`[${status}]`)
            : status >= 300 && status < 400
            ? chalk.yellow(`[${status}]`)
            : chalk.red(`[${status}]`)
        console.log(statusString + chalk.grey(` ${req.url}`))
      })
      server.listen(argv.port)
      console.log(chalk.cyan(`Started a Quartz server listening at http://localhost:${argv.port}`))
      console.log("hint: exit with ctrl+c")
      chokidar
        .watch(["**/*.ts", "**/*.tsx", "**/*.scss", "package.json"], {
          ignoreInitial: true,
        })
        .on("all", async () => {
          console.log(chalk.yellow("Detected a source code change, doing a hard rebuild..."))
          await build()
        })
    } else {
      ctx.dispose()
    }
  })
  .showHelpOnFail(false)
  .help()

 quartz/build.ts

@@ -4,8 +4,6 @@
import { rimraf } from "rimraf"
import { isGitIgnored } from "globby"
import chalk from "chalk"
import http from "http"
import serveHandler from "serve-handler"
import { parseMarkdown } from "./processors/parse"
import { filterContent } from "./processors/filter"
import { emitContent } from "./processors/emit"
@@ -13,18 +11,17 @@
import { FilePath, joinSegments, slugifyFilePath } from "./path"
import chokidar from "chokidar"
import { ProcessedContent } from "./plugins/vfile"
import WebSocket, { WebSocketServer } from "ws"
import { Argv, BuildCtx } from "./ctx"
import { glob, toPosixPath } from "./glob"
import { trace } from "./trace"

async function buildQuartz(argv: Argv, version: string) {
async function buildQuartz(argv: Argv, clientRefresh: () => void) {
  const ctx: BuildCtx = {
    argv,
    cfg,
    allSlugs: [],
  }

  console.log(chalk.bgGreen.black(`\n Quartz v${version} \n`))
  const perf = new PerfTimer()
  const output = argv.output

@@ -57,15 +54,17 @@
  console.log(chalk.green(`Done processing ${fps.length} files in ${perf.timeSince()}`))

  if (argv.serve) {
    await startServing(ctx, parsedFiles)
    return startServing(ctx, parsedFiles, clientRefresh)
  }
}

async function startServing(ctx: BuildCtx, initialContent: ProcessedContent[]) {
// setup watcher for rebuilds
async function startServing(
  ctx: BuildCtx,
  initialContent: ProcessedContent[],
  clientRefresh: () => void,
) {
  const { argv } = ctx
  const wss = new WebSocketServer({ port: 3001 })
  const connections: WebSocket[] = []
  wss.on("connection", (ws) => connections.push(ws))

  const ignored = await isGitIgnored()
  const contentMap = new Map<FilePath, ProcessedContent>()
@@ -78,6 +77,12 @@
  let toRebuild: Set<FilePath> = new Set()
  let toRemove: Set<FilePath> = new Set()
  async function rebuild(fp: string, action: "add" | "change" | "delete") {
    if (path.extname(fp) !== ".md") {
      // dont bother rebuilding for non-content files, just refresh
      clientRefresh()
      return
    }

    fp = toPosixPath(fp)
    if (!ignored(fp)) {
      const filePath = joinSegments(argv.directory, fp) as FilePath
@@ -120,7 +125,8 @@
        } catch {
          console.log(chalk.yellow(`Rebuild failed. Waiting on a change to fix the error...`))
        }
        connections.forEach((conn) => conn.send("rebuild"))

        clientRefresh()
        toRebuild.clear()
        toRemove.clear()
      }, 250)
@@ -137,31 +143,12 @@
    .on("add", (fp) => rebuild(fp, "add"))
    .on("change", (fp) => rebuild(fp, "change"))
    .on("unlink", (fp) => rebuild(fp, "delete"))

  const server = http.createServer(async (req, res) => {
    await serveHandler(req, res, {
      public: argv.output,
      directoryListing: false,
    })
    const status = res.statusCode
    const statusString =
      status >= 200 && status < 300
        ? chalk.green(`[${status}]`)
        : status >= 300 && status < 400
        ? chalk.yellow(`[${status}]`)
        : chalk.red(`[${status}]`)
    console.log(statusString + chalk.grey(` ${req.url}`))
  })
  server.listen(argv.port)
  console.log(chalk.cyan(`Started a Quartz server listening at http://localhost:${argv.port}`))
  console.log("hint: exit with ctrl+c")
}

export default async (argv: Argv, version: string) => {
export default async (argv: Argv, clientRefresh: () => void) => {
  try {
    await buildQuartz(argv, version)
  } catch {
    console.log(chalk.red("\nExiting Quartz due to a fatal error"))
    process.exit(1)
    return await buildQuartz(argv, clientRefresh)
  } catch (err) {
    trace("\nExiting Quartz due to a fatal error", err as Error)
  }
}

 quartz/path.ts

@@ -1,5 +1,4 @@
import { slug } from "github-slugger"
import { trace } from "./trace"

// Quartz Paths
// Things in boxes are not actual types but rather sources which these types can be acquired from
@@ -43,18 +42,6 @@
//                                             └────────────┤ MD File ├─────┴─────────────────┘
//                                                          └─────────┘

const STRICT_TYPE_CHECKS = false
const HARD_EXIT_ON_FAIL = false

function conditionCheck<T>(name: string, label: "pre" | "post", s: T, chk: (x: any) => x is T) {
  if (STRICT_TYPE_CHECKS && !chk(s)) {
    trace(`${name} failed ${label}-condition check: ${s} does not pass ${chk.name}`, new Error())
    if (HARD_EXIT_ON_FAIL) {
      process.exit(1)
    }
  }
}

/// Utility type to simulate nominal types in TypeScript
type SlugLike<T> = string & { __brand: T }

@@ -102,36 +89,29 @@

export function getClientSlug(window: Window): ClientSlug {
  const res = window.location.href as ClientSlug
  conditionCheck(getClientSlug.name, "post", res, isClientSlug)
  return res
}

export function getCanonicalSlug(window: Window): CanonicalSlug {
  const res = window.document.body.dataset.slug! as CanonicalSlug
  conditionCheck(getCanonicalSlug.name, "post", res, isCanonicalSlug)
  return res
}

export function canonicalizeClient(slug: ClientSlug): CanonicalSlug {
  conditionCheck(canonicalizeClient.name, "pre", slug, isClientSlug)
  const { pathname } = new URL(slug)
  let fp = pathname.slice(1)
  fp = fp.replace(new RegExp(_getFileExtension(fp) + "$"), "")
  const res = _canonicalize(fp) as CanonicalSlug
  conditionCheck(canonicalizeClient.name, "post", res, isCanonicalSlug)
  return res
}

export function canonicalizeServer(slug: ServerSlug): CanonicalSlug {
  conditionCheck(canonicalizeServer.name, "pre", slug, isServerSlug)
  let fp = slug as string
  const res = _canonicalize(fp) as CanonicalSlug
  conditionCheck(canonicalizeServer.name, "post", res, isCanonicalSlug)
  return res
}

export function slugifyFilePath(fp: FilePath): ServerSlug {
  conditionCheck(slugifyFilePath.name, "pre", fp, isFilePath)
  fp = _stripSlashes(fp) as FilePath
  const withoutFileExt = fp.replace(new RegExp(_getFileExtension(fp) + "$"), "")
  let slug = withoutFileExt
@@ -145,7 +125,6 @@
    slug = slug.replace(/_index$/, "index")
  }

  conditionCheck(slugifyFilePath.name, "post", slug, isServerSlug)
  return slug as ServerSlug
}

@@ -165,13 +144,11 @@

  let joined = joinSegments(_stripSlashes(prefix), _stripSlashes(fp))
  const res = (_addRelativeToStart(joined) + anchor) as RelativeURL
  conditionCheck(transformInternalLink.name, "post", res, isRelativeURL)
  return res
}

// resolve /a/b/c to ../../
export function pathToRoot(slug: CanonicalSlug): RelativeURL {
  conditionCheck(pathToRoot.name, "pre", slug, isCanonicalSlug)
  let rootPath = slug
    .split("/")
    .filter((x) => x !== "")
@@ -179,15 +156,11 @@
    .join("/")

  const res = _addRelativeToStart(rootPath) as RelativeURL
  conditionCheck(pathToRoot.name, "post", res, isRelativeURL)
  return res
}

export function resolveRelative(current: CanonicalSlug, target: CanonicalSlug): RelativeURL {
  conditionCheck(resolveRelative.name, "pre", current, isCanonicalSlug)
  conditionCheck(resolveRelative.name, "pre", target, isCanonicalSlug)
  const res = joinSegments(pathToRoot(current), target) as RelativeURL
  conditionCheck(resolveRelative.name, "post", res, isRelativeURL)
  return res
}


 quartz/plugins/transformers/ofm.ts

@@ -184,7 +184,7 @@

              // embed cases
              if (value.startsWith("!")) {
                const ext: string | undefined = path.extname(fp).toLowerCase()
                const ext: string = path.extname(fp).toLowerCase()
                const url = slugifyFilePath(fp as FilePath) + ext
                if ([".png", ".jpg", ".jpeg", ".gif", ".bmp", ".svg"].includes(ext)) {
                  const dims = alias ?? ""
@@ -218,8 +218,8 @@
                    type: "html",
                    value: `<iframe src="${url}"></iframe>`,
                  }
                } else {
                  // TODO: this is the node embed case
                } else if (ext === "") {
                  // TODO: note embed
                }
                // otherwise, fall through to regular link
              }

 quartz/processors/emit.ts

@@ -37,7 +37,6 @@
      }
    } catch (err) {
      trace(`Failed to emit from plugin \`${emitter.name}\``, err as Error)
      throw err
    }
  }


 quartz/processors/parse.ts

@@ -103,7 +103,6 @@
        }
      } catch (err) {
        trace(`\nFailed to process \`${fp}\``, err as Error)
        throw err
      }
    }


 quartz/trace.ts

@@ -1,4 +1,5 @@
import chalk from "chalk"
import process from "process"

const rootFile = /.*at file:/
export function trace(msg: string, err: Error) {
@@ -28,4 +29,5 @@
      }
    }
  }
  process.exit(1)
}