perf: speed up large file tree listings

back/config/util.ts

@@ -214,48 +214,105 @@ export function dirSort(a: IFile, b: IFile): number {
   }
 }
 
+const FILE_SYSTEM_READ_CONCURRENCY = 32;
+
+type FileSystemTaskRunner = <T>(task: () => Promise<T>) => Promise<T>;
+
+function createFileSystemTaskRunner(concurrency: number): FileSystemTaskRunner {
+  let activeCount = 0;
+  const queue: Array<() => void> = [];
+
+  const runNext = () => {
+    while (activeCount < concurrency && queue.length > 0) {
+      activeCount += 1;
+      queue.shift()?.();
+    }
+  };
+
+  return <T>(task: () => Promise<T>) =>
+    new Promise<T>((resolve, reject) => {
+      queue.push(() => {
+        task()
+          .then(resolve, reject)
+          .finally(() => {
+            activeCount -= 1;
+            runNext();
+          });
+      });
+      runNext();
+    });
+}
+
+async function readDirsWithRunner(
+  dir: string,
+  baseDir: string,
+  blacklist: string[],
+  sort: (a: IFile, b: IFile) => number,
+  runFileSystemTask: FileSystemTaskRunner,
+): Promise<IFile[]> {
+  const relativePath = path.relative(baseDir, dir);
+  const entries = await runFileSystemTask(() =>
+    fs.readdir(dir, { withFileTypes: true }),
+  );
+
+  const items = await Promise.all(
+    entries.map(async (entry): Promise<IFile | undefined> => {
+      if (blacklist.includes(entry.name) || entry.isSymbolicLink()) {
+        return undefined;
+      }
+
+      const subPath = path.join(dir, entry.name);
+      const stats = await runFileSystemTask(() => fs.lstat(subPath));
+      if (stats.isSymbolicLink()) {
+        return undefined;
+      }
+      const key = path.join(relativePath, entry.name);
+
+      if (stats.isDirectory()) {
+        const children = await readDirsWithRunner(
+          subPath,
+          baseDir,
+          blacklist,
+          sort,
+          runFileSystemTask,
+        );
+        return {
+          title: entry.name,
+          key,
+          type: 'directory',
+          parent: relativePath,
+          createTime: stats.birthtime.getTime(),
+          children,
+        };
+      }
+
+      return {
+        title: entry.name,
+        type: 'file',
+        key,
+        parent: relativePath,
+        size: stats.size,
+        createTime: stats.birthtime.getTime(),
+      };
+    }),
+  );
+
+  return items.filter((item): item is IFile => Boolean(item)).sort(sort);
+}
+
 export async function readDirs(
   dir: string,
   baseDir: string = '',
   blacklist: string[] = [],
   sort: (a: IFile, b: IFile) => number = dirSort,
 ): Promise<IFile[]> {
-  const relativePath = path.relative(baseDir, dir);
+  return readDirsWithRunner(
-  const files = await fs.readdir(dir);
+    dir,
-  const result: IFile[] = [];
+    baseDir,
-
+    blacklist,
-  for (const file of files) {
+    sort,
-    const subPath = path.join(dir, file);
+    createFileSystemTaskRunner(FILE_SYSTEM_READ_CONCURRENCY),
-    const stats = await fs.lstat(subPath);
+  );
-    const key = path.join(relativePath, file);
-
-    if (blacklist.includes(file) || stats.isSymbolicLink()) {
-      continue;
-    }
-
-    if (stats.isDirectory()) {
-      const children = await readDirs(subPath, baseDir, blacklist, sort);
-      result.push({
-        title: file,
-        key,
-        type: 'directory',
-        parent: relativePath,
-        createTime: stats.birthtime.getTime(),
-        children: children.sort(sort),
-      });
-    } else {
-      result.push({
-        title: file,
-        type: 'file',
-        key,
-        parent: relativePath,
-        size: stats.size,
-        createTime: stats.birthtime.getTime(),
-      });
-    }
-  }
-
-  return result.sort(sort);
 }
 
 export async function readDir(

test/back/fileTreeListing.test.cjs

@@ -0,0 +1,57 @@
+require('ts-node/register/transpile-only');
+
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { afterEach, test } = require('node:test');
+const { readDirs } = require('../../back/config/util');
+
+const temporaryDirectories = [];
+
+async function temporaryRoot() {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-file-tree-'));
+  temporaryDirectories.push(root);
+  return root;
+}
+
+afterEach(async () => {
+  await Promise.all(
+    temporaryDirectories
+      .splice(0)
+      .map((directory) => fs.rm(directory, { recursive: true, force: true })),
+  );
+});
+
+test('readDirs preserves the recursive tree contract and ignores blocked entries', async () => {
+  const root = await temporaryRoot();
+  await fs.mkdir(path.join(root, 'folder'));
+  await fs.mkdir(path.join(root, 'node_modules'));
+  await fs.writeFile(path.join(root, 'z.js'), 'z');
+  await fs.writeFile(path.join(root, 'folder', 'a.js'), 'alpha');
+  await fs.writeFile(path.join(root, 'node_modules', 'hidden.js'), 'hidden');
+  await fs.symlink(path.join(root, 'folder'), path.join(root, 'linked-folder'));
+
+  const result = await readDirs(root, root, ['node_modules'], (a, b) => {
+    if (a.type === b.type) return a.title.localeCompare(b.title);
+    return a.type === 'directory' ? -1 : 1;
+  });
+
+  assert.deepEqual(
+    result.map(({ title, type, key, parent }) => ({
+      title,
+      type,
+      key,
+      parent,
+    })),
+    [
+      { title: 'folder', type: 'directory', key: 'folder', parent: '' },
+      { title: 'z.js', type: 'file', key: 'z.js', parent: '' },
+    ],
+  );
+  assert.equal(result[0].children.length, 1);
+  assert.equal(result[0].children[0].title, 'a.js');
+  assert.equal(result[0].children[0].key, path.join('folder', 'a.js'));
+  assert.equal(result[0].children[0].parent, 'folder');
+  assert.equal(result[0].children[0].size, 5);
+});