feat: Update FlexSearch and Add Support for All Languages (#2108)

Amir Pourmand

2025-09-17 03ccac2872822442b4e11457e1c3452167a3f639

feat: Update FlexSearch and Add Support for All Languages (#2108)

* chore(deps): update flexsearch to version 0.8.205 and adjust search encoder.

* refactor(search): enhance search encoder and update search results type

- Improved the encoder function to filter out empty tokens.
- Updated the search results type from a specific FlexSearch type to a more generic 'any' type for flexibility.
- Removed redundant rtl property from the index configuration.

* refactor(search): remove rtl property from search index configuration

* refactor(search): improve encoder function formatting

- Updated the encoder function to use consistent arrow function syntax for better readability.

* refactor(search): update search results type to DefaultDocumentSearchResults

- Imported DefaultDocumentSearchResults from FlexSearch for improved type safety.
- Changed the type of searchResults from 'any' to DefaultDocumentSearchResults<Item> for better clarity and maintainability.

 package-lock.json

@@ -20,7 +20,7 @@
        "cli-spinner": "^0.2.10",
        "d3": "^7.9.0",
        "esbuild-sass-plugin": "^3.3.1",
        "flexsearch": "0.7.43",
        "flexsearch": "^0.8.205",
        "github-slugger": "^2.0.0",
        "globby": "^14.1.0",
        "gray-matter": "^4.0.3",
@@ -3189,9 +3189,36 @@
      }
    },
    "node_modules/flexsearch": {
      "version": "0.7.43",
      "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.7.43.tgz",
      "integrity": "sha512-c5o/+Um8aqCSOXGcZoqZOm+NqtVwNsvVpWv6lfmSclU954O3wvQKxxK8zj74fPaSJbXpSLTs4PRhh+wnoCXnKg=="
      "version": "0.8.205",
      "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.205.tgz",
      "integrity": "sha512-REFjMqy86DKkCTJ4gIE42c9MVm9t1vUWfEub/8taixYuhvyu4jd4XmFALk5VuKW4GH4VLav8A4BJboTsslHF1w==",
      "funding": [
        {
          "type": "github",
          "url": "https://github.com/ts-thomas"
        },
        {
          "type": "opencollective",
          "url": "https://opencollective.com/flexsearch"
        },
        {
          "type": "patreon",
          "url": "https://patreon.com/user?u=96245532"
        },
        {
          "type": "liberapay",
          "url": "https://liberapay.com/ts-thomas"
        },
        {
          "type": "paypal",
          "url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW"
        },
        {
          "type": "bountysource",
          "url": "https://salt.bountysource.com/teams/ts-thomas"
        }
      ],
      "license": "Apache-2.0"
    },
    "node_modules/format": {
      "version": "0.2.2",

 package.json

@@ -46,7 +46,7 @@
    "cli-spinner": "^0.2.10",
    "d3": "^7.9.0",
    "esbuild-sass-plugin": "^3.3.1",
    "flexsearch": "0.7.43",
    "flexsearch": "^0.8.205",
    "github-slugger": "^2.0.0",
    "globby": "^14.1.0",
    "gray-matter": "^4.0.3",

 quartz/components/scripts/search.inline.ts

@@ -1,4 +1,4 @@
import FlexSearch from "flexsearch"
import FlexSearch, { DefaultDocumentSearchResults } from "flexsearch"
import { ContentDetails } from "../../plugins/emitters/contentIndex"
import { registerEscapeHandler, removeAllChildren } from "./util"
import { FullSlug, normalizeRelativeURLs, resolveRelative } from "../../util/path"
@@ -9,15 +9,21 @@
  title: string
  content: string
  tags: string[]
  [key: string]: any
}

// Can be expanded with things like "term" in the future
type SearchType = "basic" | "tags"
let searchType: SearchType = "basic"
let currentSearchTerm: string = ""
const encoder = (str: string) => str.toLowerCase().split(/([^a-z]|[^\x00-\x7F])/)
const encoder = (str: string) => {
  return str
    .toLowerCase()
    .split(/\s+/)
    .filter((token) => token.length > 0)
}

let index = new FlexSearch.Document<Item>({
  charset: "latin:extra",
  encode: encoder,
  document: {
    id: "id",
@@ -397,7 +403,7 @@
    searchLayout.classList.toggle("display-results", currentSearchTerm !== "")
    searchType = currentSearchTerm.startsWith("#") ? "tags" : "basic"

    let searchResults: FlexSearch.SimpleDocumentSearchResultSetUnit[]
    let searchResults: DefaultDocumentSearchResults<Item>
    if (searchType === "tags") {
      currentSearchTerm = currentSearchTerm.substring(1).trim()
      const separatorIndex = currentSearchTerm.indexOf(" ")
@@ -410,7 +416,7 @@
          // return at least 10000 documents, so it is enough to filter them by tag (implemented in flexsearch)
          limit: Math.max(numSearchResults, 10000),
          index: ["title", "content"],
          tag: tag,
          tag: { tags: tag },
        })
        for (let searchResult of searchResults) {
          searchResult.result = searchResult.result.slice(0, numSearchResults)