From 3ac6b42e16dca5a44ed3fec2c0314f1dbbc2322b Mon Sep 17 00:00:00 2001
From: Jacky Zhao <j.zhao2k19@gmail.com>
Date: Sun, 16 Jul 2023 06:02:12 +0000
Subject: [PATCH] finish path refactoring, add sourcemap + better trace support

---
 quartz/processors/parse.ts |   38 +++++++++++++++++++++++++-------------
 1 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts
index f937701..a782475 100644
--- a/quartz/processors/parse.ts
+++ b/quartz/processors/parse.ts
@@ -7,13 +7,14 @@
 import { ProcessedContent } from '../plugins/vfile'
 import { PerfTimer } from '../perf'
 import { read } from 'to-vfile'
-import { slugify } from '../path'
+import { FilePath, ServerSlug, slugifyFilePath } from '../path'
 import path from 'path'
 import os from 'os'
 import workerpool, { Promise as WorkerPromise } from 'workerpool'
 import { QuartzTransformerPluginInstance } from '../plugins/types'
 import { QuartzLogger } from '../log'
 import chalk from 'chalk'
+import { trace } from '../trace'
 
 export type QuartzProcessor = Processor<MDRoot, HTMLRoot, void>
 export function createProcessor(transformers: QuartzTransformerPluginInstance[]): QuartzProcessor {
@@ -21,8 +22,8 @@
   let processor = unified().use(remarkParse)
 
   // MD AST -> MD AST transforms
-  for (const plugin of transformers) {
-    processor = processor.use(plugin.markdownPlugins())
+  for (const plugin of transformers.filter(p => p.markdownPlugins)) {
+    processor = processor.use(plugin.markdownPlugins!())
   }
 
   // MD AST -> HTML AST
@@ -30,8 +31,8 @@
 
 
   // HTML AST -> HTML AST transforms
-  for (const plugin of transformers) {
-    processor = processor.use(plugin.htmlPlugins())
+  for (const plugin of transformers.filter(p => p.htmlPlugins)) {
+    processor = processor.use(plugin.htmlPlugins!())
   }
 
   return processor
@@ -73,15 +74,24 @@
   })
 }
 
-export function createFileParser(baseDir: string, fps: string[], verbose: boolean) {
+export function createFileParser(transformers: QuartzTransformerPluginInstance[], baseDir: string, fps: FilePath[], allSlugs: ServerSlug[], verbose: boolean) {
   return async (processor: QuartzProcessor) => {
     const res: ProcessedContent[] = []
     for (const fp of fps) {
       try {
         const file = await read(fp)
 
+        // strip leading and trailing whitespace
+        file.value = file.value.toString().trim()
+
+        // Text -> Text transforms
+        for (const plugin of transformers.filter(p => p.textTransform)) {
+          file.value = plugin.textTransform!(file.value)
+        }
+
         // base data properties that plugins may use
-        file.data.slug = slugify(path.relative(baseDir, file.path))
+        file.data.slug = slugifyFilePath(path.relative(baseDir, file.path) as FilePath)
+        file.data.allSlugs = allSlugs
         file.data.filePath = fp
 
         const ast = processor.parse(file)
@@ -92,7 +102,7 @@
           console.log(`[process] ${fp} -> ${file.data.slug}`)
         }
       } catch (err) {
-        console.log(chalk.red(`\nFailed to process \`${fp}\`: `) + err)
+        trace(`\nFailed to process \`${fp}\``, err as Error)
         process.exit(1)
       }
     }
@@ -101,19 +111,21 @@
   }
 }
 
-export async function parseMarkdown(transformers: QuartzTransformerPluginInstance[], baseDir: string, fps: string[], verbose: boolean): Promise<ProcessedContent[]> {
+export async function parseMarkdown(transformers: QuartzTransformerPluginInstance[], baseDir: string, fps: FilePath[], verbose: boolean): Promise<ProcessedContent[]> {
   const perf = new PerfTimer()
   const log = new QuartzLogger(verbose)
 
   const CHUNK_SIZE = 128
   let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
-  let res: ProcessedContent[] = []
 
+  // get all slugs ahead of time as each thread needs a copy
+  const allSlugs = fps.map(fp => slugifyFilePath(path.relative(baseDir, path.resolve(fp)) as FilePath))
+
+  let res: ProcessedContent[] = []
   log.start(`Parsing input files using ${concurrency} threads`)
   if (concurrency === 1) {
-    // single-thread
     const processor = createProcessor(transformers)
-    const parse = createFileParser(baseDir, fps, verbose)
+    const parse = createFileParser(transformers, baseDir, fps, allSlugs, verbose)
     res = await parse(processor)
   } else {
     await transpileWorkerScript()
@@ -128,7 +140,7 @@
 
     const childPromises: WorkerPromise<ProcessedContent[]>[] = []
     for (const chunk of chunks(fps, CHUNK_SIZE)) {
-      childPromises.push(pool.exec('parseFiles', [baseDir, chunk, verbose]))
+      childPromises.push(pool.exec('parseFiles', [baseDir, chunk, allSlugs, verbose]))
     }
 
     const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)

--
Gitblit v1.10.0