From 49bd6bc3ffe1d3507e00bae62c12d9b045363090 Mon Sep 17 00:00:00 2001
From: Jacky Zhao <j.zhao2k19@gmail.com>
Date: Wed, 09 Aug 2023 05:52:49 +0000
Subject: [PATCH] better concurrency debugging, --concurrency flag for npx quartz build
---
quartz/build.ts | 21 +---------
.gitignore | 1
quartz/trace.ts | 30 +++++++++++----
quartz/sourcemap.ts | 19 +++++++++
quartz/ctx.ts | 1
quartz/worker.ts | 3 +
content/build.md | 1
quartz/bootstrap-cli.mjs | 4 ++
quartz/processors/parse.ts | 10 ++++-
9 files changed, 62 insertions(+), 28 deletions(-)
diff --git a/.gitignore b/.gitignore
index 9be70c6..5177d66 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@
tsconfig.tsbuildinfo
.obsidian
.quartz-cache
+private/
diff --git a/content/build.md b/content/build.md
index 541c9d2..bce9d54 100644
--- a/content/build.md
+++ b/content/build.md
@@ -24,3 +24,4 @@
> - `-o` or `--output`: the output folder. This is normally just `public`
> - `--serve`: run a local hot-reloading server to preview your Quartz
> - `--port`: what port to run the local preview server on
+> - `--concurrency`: how many threads to use to parse notes
diff --git a/quartz/bootstrap-cli.mjs b/quartz/bootstrap-cli.mjs
index 71d6957..adcdb9f 100755
--- a/quartz/bootstrap-cli.mjs
+++ b/quartz/bootstrap-cli.mjs
@@ -84,6 +84,10 @@
default: false,
describe: "show detailed bundle information",
},
+ concurrency: {
+ number: true,
+ describe: "how many threads to use to parse notes"
+ }
}
function escapePath(fp) {
diff --git a/quartz/build.ts b/quartz/build.ts
index e6dc96a..c25efbf 100644
--- a/quartz/build.ts
+++ b/quartz/build.ts
@@ -1,19 +1,5 @@
import sourceMapSupport from "source-map-support"
-sourceMapSupport.install({
- retrieveSourceMap(source) {
- // source map hack to get around query param
- // import cache busting
- if (source.includes(".quartz-cache")) {
- let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
- return {
- map: fs.readFileSync(realSource, "utf8"),
- }
- } else {
- return null
- }
- },
-})
-
+sourceMapSupport.install(options)
import path from "path"
import { PerfTimer } from "./perf"
import { rimraf } from "rimraf"
@@ -23,14 +9,13 @@
import { filterContent } from "./processors/filter"
import { emitContent } from "./processors/emit"
import cfg from "../quartz.config"
-import { FilePath, ServerSlug, joinSegments, slugifyFilePath } from "./path"
+import { FilePath, joinSegments, slugifyFilePath } from "./path"
import chokidar from "chokidar"
import { ProcessedContent } from "./plugins/vfile"
import { Argv, BuildCtx } from "./ctx"
import { glob, toPosixPath } from "./glob"
import { trace } from "./trace"
-import { fileURLToPath } from "url"
-import fs from "fs"
+import { options } from "./sourcemap"
async function buildQuartz(argv: Argv, clientRefresh: () => void) {
const ctx: BuildCtx = {
diff --git a/quartz/ctx.ts b/quartz/ctx.ts
index 8a7b803..dad5cef 100644
--- a/quartz/ctx.ts
+++ b/quartz/ctx.ts
@@ -7,6 +7,7 @@
output: string
serve: boolean
port: number
+ concurrency?: number
}
export interface BuildCtx {
diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts
index 52dc519..1340020 100644
--- a/quartz/processors/parse.ts
+++ b/quartz/processors/parse.ts
@@ -56,6 +56,8 @@
platform: "node",
format: "esm",
packages: "external",
+ sourcemap: true,
+ sourcesContent: false,
plugins: [
{
name: "css-and-scripts-as-text",
@@ -116,7 +118,7 @@
const log = new QuartzLogger(argv.verbose)
const CHUNK_SIZE = 128
- let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
+ let concurrency = ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism())
let res: ProcessedContent[] = []
log.start(`Parsing input files using ${concurrency} threads`)
@@ -142,7 +144,11 @@
childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs]))
}
- const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)
+ const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch((err) => {
+ const errString = err.toString().slice("Error:".length)
+ console.error(errString)
+ process.exit(1)
+ })
res = results.flat()
await pool.terminate()
}
diff --git a/quartz/sourcemap.ts b/quartz/sourcemap.ts
new file mode 100644
index 0000000..8b05c8b
--- /dev/null
+++ b/quartz/sourcemap.ts
@@ -0,0 +1,19 @@
+import fs from "fs"
+import sourceMapSupport from "source-map-support"
+import { fileURLToPath } from "url"
+
+export const options: sourceMapSupport.Options = {
+ // source map hack to get around query param
+ // import cache busting
+ retrieveSourceMap(source) {
+ if (source.includes(".quartz-cache")) {
+ let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
+ return {
+ map: fs.readFileSync(realSource, "utf8"),
+ }
+ } else {
+ return null
+ }
+ },
+}
+
diff --git a/quartz/trace.ts b/quartz/trace.ts
index 337ffe0..ca3a5a0 100644
--- a/quartz/trace.ts
+++ b/quartz/trace.ts
@@ -1,17 +1,22 @@
import chalk from "chalk"
import process from "process"
+import { isMainThread } from "workerpool"
const rootFile = /.*at file:/
export function trace(msg: string, err: Error) {
const stack = err.stack
- console.log()
- console.log(
+
+ const lines: string[] = []
+
+ lines.push("")
+ lines.push(
"\n" +
- chalk.bgRed.black.bold(" ERROR ") +
- "\n" +
- chalk.red(` ${msg}`) +
- (err.message.length > 0 ? `: ${err.message}` : ""),
+ chalk.bgRed.black.bold(" ERROR ") +
+ "\n" +
+ chalk.red(` ${msg}`) +
+ (err.message.length > 0 ? `: ${err.message}` : ""),
)
+
if (!stack) {
return
}
@@ -23,11 +28,20 @@
}
if (!line.includes("node_modules")) {
- console.log(` ${line}`)
+ lines.push(` ${line}`)
if (rootFile.test(line)) {
reachedEndOfLegibleTrace = true
}
}
}
- process.exit(1)
+
+ const traceMsg = lines.join("\n")
+ if (!isMainThread) {
+ // gather lines and throw
+ throw new Error(traceMsg)
+ } else {
+ // print and exit
+ console.error(traceMsg)
+ process.exit(1)
+ }
}
diff --git a/quartz/worker.ts b/quartz/worker.ts
index d97c483..db11cbb 100644
--- a/quartz/worker.ts
+++ b/quartz/worker.ts
@@ -1,7 +1,10 @@
+import sourceMapSupport from "source-map-support"
+sourceMapSupport.install(options)
import cfg from "../quartz.config"
import { Argv, BuildCtx } from "./ctx"
import { FilePath, ServerSlug } from "./path"
import { createFileParser, createProcessor } from "./processors/parse"
+import { options } from "./sourcemap"
// only called from worker thread
export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {
--
Gitblit v1.10.0