update spinners

Jacky Zhao

2023-06-04 1406ee0f05dc688e1b7635d3ce14e63d1c42a973

update spinners

 package-lock.json

@@ -13,7 +13,6 @@
        "@napi-rs/simple-git": "^0.1.8",
        "chalk": "^4.1.2",
        "cli-spinner": "^0.2.10",
        "env-paths": "^3.0.0",
        "esbuild-sass-plugin": "^2.9.0",
        "github-slugger": "^2.0.0",
        "globby": "^13.1.4",
@@ -34,29 +33,29 @@
        "remark-frontmatter": "^4.0.1",
        "remark-gfm": "^3.0.1",
        "remark-math": "^5.1.1",
        "remark-obsidian-callout": "^1.1.3",
        "remark-parse": "^10.0.1",
        "remark-rehype": "^10.1.0",
        "remark-smartypants": "^2.0.0",
        "require-from-string": "^2.0.2",
        "rimraf": "^5.0.1",
        "serve-handler": "^6.1.5",
        "shiki": "^0.14.2",
        "to-vfile": "^7.2.4",
        "unified": "^10.1.2",
        "unist-util-visit": "^4.1.2",
        "vfile": "^5.3.7",
        "workerpool": "^6.4.0",
        "yargs": "^17.7.2"
      },
      "bin": {
        "quartz": "quartz/bootstrap.mjs"
        "quartz": "quartz/bootstrap-cli.mjs"
      },
      "devDependencies": {
        "@types/cli-spinner": "^0.2.1",
        "@types/hast": "^2.3.4",
        "@types/node": "^20.1.2",
        "@types/pretty-time": "^1.1.2",
        "@types/require-from-string": "^1.2.1",
        "@types/serve-handler": "^6.1.1",
        "@types/workerpool": "^6.4.0",
        "@types/yargs": "^17.0.24",
        "esbuild": "^0.17.18",
        "typescript": "^5.0.4"
@@ -938,12 +937,6 @@
      "integrity": "sha512-4i+Y+O5H80Rh01lY/3Z0hB/UWc4R64ReE83joEpVsIG3iQWpYx66k6pQh1amJNZquKtJQyu/RcfkTtvL0KwssA==",
      "dev": true
    },
    "node_modules/@types/require-from-string": {
      "version": "1.2.1",
      "resolved": "https://registry.npmjs.org/@types/require-from-string/-/require-from-string-1.2.1.tgz",
      "integrity": "sha512-mIDK7lTHc0uW67SxPIqkwCrxmdKBV5aAET560hyZnT8c6Ekp9Aah3GPqe8Pl1Yzn/i2NMYmYv+HiMLwjGDCIAQ==",
      "dev": true
    },
    "node_modules/@types/serve-handler": {
      "version": "6.1.1",
      "resolved": "https://registry.npmjs.org/@types/serve-handler/-/serve-handler-6.1.1.tgz",
@@ -958,6 +951,15 @@
      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz",
      "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ=="
    },
    "node_modules/@types/workerpool": {
      "version": "6.4.0",
      "resolved": "https://registry.npmjs.org/@types/workerpool/-/workerpool-6.4.0.tgz",
      "integrity": "sha512-SIF2/169pDsLKeM8GQGHkOFifGalDbZgiBSaLUnnlVSRsAOenkAvQ6h4uhV2W+PZZczS+8LQxACwNkSykdT91A==",
      "dev": true,
      "dependencies": {
        "@types/node": "*"
      }
    },
    "node_modules/@types/yargs": {
      "version": "17.0.24",
      "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz",
@@ -998,7 +1000,8 @@
    "node_modules/ansi-sequence-parser": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.0.tgz",
      "integrity": "sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ=="
      "integrity": "sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==",
      "peer": true
    },
    "node_modules/ansi-styles": {
      "version": "4.3.0",
@@ -1347,17 +1350,6 @@
        "url": "https://github.com/fb55/entities?sponsor=1"
      }
    },
    "node_modules/env-paths": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz",
      "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==",
      "engines": {
        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
      },
      "funding": {
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
    "node_modules/esbuild": {
      "version": "0.17.19",
      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.19.tgz",
@@ -2114,7 +2106,8 @@
    "node_modules/jsonc-parser": {
      "version": "3.2.0",
      "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz",
      "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w=="
      "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==",
      "peer": true
    },
    "node_modules/katex": {
      "version": "0.16.7",
@@ -3412,6 +3405,14 @@
        "url": "https://opencollective.com/unified"
      }
    },
    "node_modules/remark-obsidian-callout": {
      "version": "1.1.3",
      "resolved": "https://registry.npmjs.org/remark-obsidian-callout/-/remark-obsidian-callout-1.1.3.tgz",
      "integrity": "sha512-q2jZgPMHP5Wyq0WgeGdwboPoepqN7Ib77iWagEKThZjtl1wUd7/f2tdErdPr/LeZu9K7I6r52XylwhLs5GJi9g==",
      "dependencies": {
        "unist-util-visit": "^4.1.2"
      }
    },
    "node_modules/remark-parse": {
      "version": "10.0.2",
      "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.2.tgz",
@@ -3476,14 +3477,6 @@
        "node": ">=0.10.0"
      }
    },
    "node_modules/require-from-string": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
      "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
      "engines": {
        "node": ">=0.10.0"
      }
    },
    "node_modules/resolve": {
      "version": "1.22.2",
      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz",
@@ -3717,6 +3710,7 @@
      "version": "0.14.2",
      "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.2.tgz",
      "integrity": "sha512-ltSZlSLOuSY0M0Y75KA+ieRaZ0Trf5Wl3gutE7jzLuIcWxLp5i/uEnLoQWNvgKXQ5OMpGkJnVMRLAuzjc0LJ2A==",
      "peer": true,
      "dependencies": {
        "ansi-sequence-parser": "^1.1.0",
        "jsonc-parser": "^3.2.0",
@@ -4158,12 +4152,14 @@
    "node_modules/vscode-oniguruma": {
      "version": "1.7.0",
      "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz",
      "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA=="
      "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==",
      "peer": true
    },
    "node_modules/vscode-textmate": {
      "version": "8.0.0",
      "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz",
      "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg=="
      "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==",
      "peer": true
    },
    "node_modules/web-namespaces": {
      "version": "2.0.1",
@@ -4188,6 +4184,11 @@
        "node": ">= 8"
      }
    },
    "node_modules/workerpool": {
      "version": "6.4.0",
      "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.4.0.tgz",
      "integrity": "sha512-i3KR1mQMNwY2wx20ozq2EjISGtQWDIfV56We+yGJ5yDs8jTwQiLLaqHlkBHITlCuJnYlVRmXegxFxZg7gqI++A=="
    },
    "node_modules/wrap-ansi": {
      "version": "6.2.0",
      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",

 package.json

@@ -2,6 +2,7 @@
  "name": "@jackyzha0/quartz",
  "description": "🌱 publish your digital garden and notes as a website",
  "version": "4.0.3",
  "type": "module",
  "author": "jackyzha0 <j.zhao2k19@gmail.com>",
  "license": "MIT",
  "homepage": "https://quartz.jzhao.xyz",
@@ -21,14 +22,13 @@
    "quartz"
  ],
  "bin": {
    "quartz": "./quartz/bootstrap.mjs"
    "quartz": "./quartz/bootstrap-cli.mjs"
  },
  "dependencies": {
    "@inquirer/prompts": "^1.0.3",
    "@napi-rs/simple-git": "^0.1.8",
    "chalk": "^4.1.2",
    "cli-spinner": "^0.2.10",
    "env-paths": "^3.0.0",
    "esbuild-sass-plugin": "^2.9.0",
    "github-slugger": "^2.0.0",
    "globby": "^13.1.4",
@@ -49,17 +49,17 @@
    "remark-frontmatter": "^4.0.1",
    "remark-gfm": "^3.0.1",
    "remark-math": "^5.1.1",
    "remark-obsidian-callout": "^1.1.3",
    "remark-parse": "^10.0.1",
    "remark-rehype": "^10.1.0",
    "remark-smartypants": "^2.0.0",
    "require-from-string": "^2.0.2",
    "rimraf": "^5.0.1",
    "serve-handler": "^6.1.5",
    "shiki": "^0.14.2",
    "to-vfile": "^7.2.4",
    "unified": "^10.1.2",
    "unist-util-visit": "^4.1.2",
    "vfile": "^5.3.7",
    "workerpool": "^6.4.0",
    "yargs": "^17.7.2"
  },
  "devDependencies": {
@@ -67,8 +67,8 @@
    "@types/hast": "^2.3.4",
    "@types/node": "^20.1.2",
    "@types/pretty-time": "^1.1.2",
    "@types/require-from-string": "^1.2.1",
    "@types/serve-handler": "^6.1.1",
    "@types/workerpool": "^6.4.0",
    "@types/yargs": "^17.0.24",
    "esbuild": "^0.17.18",
    "typescript": "^5.0.4"

 quartz/build.ts

@@ -24,10 +24,10 @@
  const perf = new PerfTimer()
  const output = argv.output

  if (argv.verbose) {
    const pluginCount = Object.values(cfg.plugins).flat().length
    const pluginNames = (key: 'transformers' | 'filters' | 'emitters') => cfg.plugins[key].map(plugin => plugin.name)
    console.log(`Loaded ${pluginCount} plugins`)
  if (argv.verbose) {
    console.log(`  Transformers: ${pluginNames('transformers').join(", ")}`)
    console.log(`  Filters: ${pluginNames('filters').join(", ")}`)
    console.log(`  Emitters: ${pluginNames('emitters').join(", ")}`)
@@ -37,10 +37,8 @@
  if (argv.clean) {
    perf.addEvent('clean')
    await rimraf(output)
    if (argv.verbose) {
      console.log(`Cleaned output directory \`${output}\` in ${perf.timeSince('clean')}`)
    }
  }

  // glob
  perf.addEvent('glob')
@@ -49,12 +47,8 @@
    ignore: cfg.configuration.ignorePatterns,
    gitignore: true,
  })

  if (argv.verbose) {
    console.log(`Found ${fps.length} input files in ${perf.timeSince('glob')}`)

  }

  const filePaths = fps.map(fp => `${argv.directory}${path.sep}${fp}`)
  const parsedFiles = await parseMarkdown(cfg.plugins.transformers, argv.directory, filePaths, argv.verbose)
  const filteredContent = filterContent(cfg.plugins.filters, parsedFiles, argv.verbose)

 quartz/log.ts

New file
@@ -0,0 +1,26 @@
import { Spinner } from 'cli-spinner'

export class QuartzLogger {
  verbose: boolean
  spinner: Spinner | undefined
  constructor(verbose: boolean) {
    this.verbose = verbose
  }

  start(text: string) {
    if (this.verbose) {
      console.log(text)
    } else {
      this.spinner = new Spinner(`%s ${text}`)
      this.spinner.setSpinnerString(18)
      this.spinner.start()
    }
  }

  success(text: string) {
    if (!this.verbose) {
      this.spinner!.stop(true)
    }
    console.log(text)
  }
}

 quartz/processors/emit.ts

@@ -7,6 +7,7 @@
import { ProcessedContent } from "../plugins/vfile"
import { QUARTZ, slugify } from "../path"
import { globbyStream } from "globby"
import chalk from "chalk"

export async function emitContent(contentFolder: string, output: string, cfg: QuartzConfig, content: ProcessedContent[], verbose: boolean) {
  const perf = new PerfTimer()
@@ -23,6 +24,7 @@

  let emittedFiles = 0
  for (const emitter of cfg.plugins.emitters) {
    try {
    const emitted = await emitter.emit(cfg.configuration, content, staticResources, emit)
    emittedFiles += emitted.length

@@ -31,6 +33,10 @@
        console.log(`[emit:${emitter.name}] ${file}`)
      }
    }
    } catch (err) {
      console.log(chalk.red(`Failed to emit from plugin \`${emitter.name}\`: `) + err)
      process.exit(1)
    }
  }

  const staticPath = path.join(QUARTZ, "static")

 quartz/processors/parse.ts

@@ -12,9 +12,11 @@
import os from 'os'
import workerpool, { Promise as WorkerPromise } from 'workerpool'
import { QuartzTransformerPlugin } from '../plugins/types'
import { QuartzLogger } from '../log'
import chalk from 'chalk'

export type QuartzProcessor = Processor<MDRoot, HTMLRoot, void>
export function createProcessor(transformers: QuartzTransformerPlugin[]): any {
export function createProcessor(transformers: QuartzTransformerPlugin[]): QuartzProcessor {
  // base Markdown -> MD AST
  let processor = unified().use(remarkParse)

@@ -41,15 +43,11 @@
  }
}

async function transpileWorkerScript(verbose: boolean) {
async function transpileWorkerScript() {
  // transpile worker script
  const cacheFile = "./.quartz-cache/transpiled-worker.mjs"
  const fp = "./quartz/worker.ts"
  if (verbose) {
    console.log("Transpiling worker script")
  }

  await esbuild.build({
  return esbuild.build({
    entryPoints: [fp],
    outfile: path.join("quartz", cacheFile),
    bundle: true,
@@ -75,16 +73,11 @@
  })
}

export async function parseMarkdown(transformers: QuartzTransformerPlugin[], baseDir: string, fps: string[], verbose: boolean): Promise<ProcessedContent[]> {
  const perf = new PerfTimer()

  const CHUNK_SIZE = 128
  let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
export function createFileParser(baseDir: string, fps: string[], verbose: boolean) {
  return async (processor: QuartzProcessor) => {
  const res: ProcessedContent[] = []
  if (concurrency === 1) {
    // single-thread
    const processor = createProcessor(transformers)
    for (const fp of fps) {
      try {
      const file = await read(fp)

      // base data properties that plugins may use
@@ -92,14 +85,38 @@
      file.data.filePath = fp

      const ast = processor.parse(file)
      res.push([await processor.run(ast, file), file])
        const newAst = await processor.run(ast, file)
        res.push([newAst, file])

      if (verbose) {
        console.log(`[process] ${fp} -> ${file.data.slug}`)
      }
      } catch (err) {
        console.log(chalk.red(`Failed to process \`${fp}\`: `) + err)
        process.exit(1)
    }
    }

    return res
  }
}

export async function parseMarkdown(transformers: QuartzTransformerPlugin[], baseDir: string, fps: string[], verbose: boolean): Promise<ProcessedContent[]> {
  const perf = new PerfTimer()
  const log = new QuartzLogger(verbose)

  const CHUNK_SIZE = 128
  let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
  let res: ProcessedContent[] = []

  log.start(`Parsing input files using ${concurrency} threads`)
  if (concurrency === 1) {
    // single-thread
    const processor = createProcessor(transformers)
    const parse = createFileParser(baseDir, fps, verbose)
    res = await parse(processor)
  } else {
    await transpileWorkerScript(verbose)
    await transpileWorkerScript()
    const pool = workerpool.pool(
      './quartz/bootstrap-worker.mjs',
      {
@@ -113,11 +130,12 @@
    for (const chunk of chunks(fps, CHUNK_SIZE)) {
      childPromises.push(pool.exec('parseFiles', [baseDir, chunk, verbose]))
    }

    const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)
    res.push(...results.flat())
    res = results.flat()
    await pool.terminate()
  }

  console.log(`Parsed and transformed ${res.length} Markdown files with ${concurrency} cores in ${perf.timeSince()}`)
  log.success(`Parsed ${res.length} Markdown files in ${perf.timeSince()}`)
  return res
}

 quartz/worker.ts

@@ -1,30 +1,11 @@
import { read } from "to-vfile"
import config from "../quartz.config"
import { createProcessor } from "./processors/parse"
import { slugify } from "./path"
import path from "path"
import { ProcessedContent } from "./plugins/vfile"
import { createFileParser, createProcessor } from "./processors/parse"

const transformers = config.plugins.transformers
const processor = createProcessor(transformers)

// only called from worker thread
export async function parseFiles(baseDir: string, fps: string[], verbose: boolean) {
  const res: ProcessedContent[] = []
  for (const fp of fps) {
    const file = await read(fp)

    // base data properties that plugins may use
    file.data.slug = slugify(path.relative(baseDir, file.path))
    file.data.filePath = fp

    const ast = processor.parse(file)
    res.push([await processor.run(ast, file), file])

    if (verbose) {
      console.log(`[process] ${fp} -> ${file.data.slug}`)
    }
  }

  return res
  const parse = createFileParser(baseDir, fps, verbose)
  return parse(processor)
}