From decdab093f0f82daa8d86e17f7bd03e231a69142 Mon Sep 17 00:00:00 2001 From: whyour Date: Tue, 18 Aug 2026 23:56:11 +0800 Subject: [PATCH] perf: speed up large file tree listings --- back/config/util.ts | 129 +++++++++++++++++++++-------- test/back/fileTreeListing.test.cjs | 57 +++++++++++++ 2 files changed, 150 insertions(+), 36 deletions(-) create mode 100644 test/back/fileTreeListing.test.cjs diff --git a/back/config/util.ts b/back/config/util.ts index f22f7e01..649f97a1 100644 --- a/back/config/util.ts +++ b/back/config/util.ts @@ -214,48 +214,105 @@ export function dirSort(a: IFile, b: IFile): number { } } +const FILE_SYSTEM_READ_CONCURRENCY = 32; + +type FileSystemTaskRunner = (task: () => Promise) => Promise; + +function createFileSystemTaskRunner(concurrency: number): FileSystemTaskRunner { + let activeCount = 0; + const queue: Array<() => void> = []; + + const runNext = () => { + while (activeCount < concurrency && queue.length > 0) { + activeCount += 1; + queue.shift()?.(); + } + }; + + return (task: () => Promise) => + new Promise((resolve, reject) => { + queue.push(() => { + task() + .then(resolve, reject) + .finally(() => { + activeCount -= 1; + runNext(); + }); + }); + runNext(); + }); +} + +async function readDirsWithRunner( + dir: string, + baseDir: string, + blacklist: string[], + sort: (a: IFile, b: IFile) => number, + runFileSystemTask: FileSystemTaskRunner, +): Promise { + const relativePath = path.relative(baseDir, dir); + const entries = await runFileSystemTask(() => + fs.readdir(dir, { withFileTypes: true }), + ); + + const items = await Promise.all( + entries.map(async (entry): Promise => { + if (blacklist.includes(entry.name) || entry.isSymbolicLink()) { + return undefined; + } + + const subPath = path.join(dir, entry.name); + const stats = await runFileSystemTask(() => fs.lstat(subPath)); + if (stats.isSymbolicLink()) { + return undefined; + } + const key = path.join(relativePath, entry.name); + + if (stats.isDirectory()) { + const children = await readDirsWithRunner( + subPath, + baseDir, + blacklist, + sort, + runFileSystemTask, + ); + return { + title: entry.name, + key, + type: 'directory', + parent: relativePath, + createTime: stats.birthtime.getTime(), + children, + }; + } + + return { + title: entry.name, + type: 'file', + key, + parent: relativePath, + size: stats.size, + createTime: stats.birthtime.getTime(), + }; + }), + ); + + return items.filter((item): item is IFile => Boolean(item)).sort(sort); +} + export async function readDirs( dir: string, baseDir: string = '', blacklist: string[] = [], sort: (a: IFile, b: IFile) => number = dirSort, ): Promise { - const relativePath = path.relative(baseDir, dir); - const files = await fs.readdir(dir); - const result: IFile[] = []; - - for (const file of files) { - const subPath = path.join(dir, file); - const stats = await fs.lstat(subPath); - const key = path.join(relativePath, file); - - if (blacklist.includes(file) || stats.isSymbolicLink()) { - continue; - } - - if (stats.isDirectory()) { - const children = await readDirs(subPath, baseDir, blacklist, sort); - result.push({ - title: file, - key, - type: 'directory', - parent: relativePath, - createTime: stats.birthtime.getTime(), - children: children.sort(sort), - }); - } else { - result.push({ - title: file, - type: 'file', - key, - parent: relativePath, - size: stats.size, - createTime: stats.birthtime.getTime(), - }); - } - } - - return result.sort(sort); + return readDirsWithRunner( + dir, + baseDir, + blacklist, + sort, + createFileSystemTaskRunner(FILE_SYSTEM_READ_CONCURRENCY), + ); } export async function readDir( diff --git a/test/back/fileTreeListing.test.cjs b/test/back/fileTreeListing.test.cjs new file mode 100644 index 00000000..711d3b96 --- /dev/null +++ b/test/back/fileTreeListing.test.cjs @@ -0,0 +1,57 @@ +require('ts-node/register/transpile-only'); + +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { afterEach, test } = require('node:test'); +const { readDirs } = require('../../back/config/util'); + +const temporaryDirectories = []; + +async function temporaryRoot() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-file-tree-')); + temporaryDirectories.push(root); + return root; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +test('readDirs preserves the recursive tree contract and ignores blocked entries', async () => { + const root = await temporaryRoot(); + await fs.mkdir(path.join(root, 'folder')); + await fs.mkdir(path.join(root, 'node_modules')); + await fs.writeFile(path.join(root, 'z.js'), 'z'); + await fs.writeFile(path.join(root, 'folder', 'a.js'), 'alpha'); + await fs.writeFile(path.join(root, 'node_modules', 'hidden.js'), 'hidden'); + await fs.symlink(path.join(root, 'folder'), path.join(root, 'linked-folder')); + + const result = await readDirs(root, root, ['node_modules'], (a, b) => { + if (a.type === b.type) return a.title.localeCompare(b.title); + return a.type === 'directory' ? -1 : 1; + }); + + assert.deepEqual( + result.map(({ title, type, key, parent }) => ({ + title, + type, + key, + parent, + })), + [ + { title: 'folder', type: 'directory', key: 'folder', parent: '' }, + { title: 'z.js', type: 'file', key: 'z.js', parent: '' }, + ], + ); + assert.equal(result[0].children.length, 1); + assert.equal(result[0].children[0].title, 'a.js'); + assert.equal(result[0].children[0].key, path.join('folder', 'a.js')); + assert.equal(result[0].children[0].parent, 'folder'); + assert.equal(result[0].children[0].size, 5); +});