Batch committed include reads to prevent web event-loop stalls

README.md

@@ -125,6 +125,12 @@ approve historical code: missing bindings and stale payloads need current-source
 verification and a new submission. Maintenance and actively merging batches are
 not settled by this reconciliation.
 
+Header-includer discovery reads committed source blobs in one `git cat-file
+--batch` operation and reuses that snapshot until HEAD or source roots change.
+Submitted overrides apply only to the current query. Reverse include traversal
+handles cycles without losing transitive dependents; uncommitted working-tree
+text is never used as the baseline.
+
 ## Building td
 
 `node tools/build-td/build.mjs` (Node 22.21) writes `dist/td-windows-x64.exe`, `dist/td-linux-x64` and

src/server/git.js

@@ -13,7 +13,7 @@ export class Repo {
 
   git(args, opts = {}) {
     try {
-      return execFileSync('git', args, { cwd: this.dir, encoding: opts.encoding ?? 'utf8', maxBuffer: 64 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], input: opts.input });
+      return execFileSync('git', args, { cwd: this.dir, encoding: opts.encoding === undefined ? 'utf8' : opts.encoding, maxBuffer: 64 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], input: opts.input });
     } catch (e) {
       // git's own explanation, not just "Command failed".
       const why = `${e.stderr ?? ''}${e.stdout ?? ''}`.trim();
@@ -44,6 +44,31 @@ export class Repo {
     }
   }
 
+  /** Reads a committed source snapshot with one Git process, not one process per file. */
+  showMany(commit, files) {
+    const result = new Map();
+    if (!files.length) return result;
+    const data = this.git(['cat-file', '--batch'], { input: files.map((f) => `${commit}:${f}\n`).join(''), encoding: null });
+    let offset = 0;
+    for (const file of files) {
+      const end = data.indexOf(10, offset);
+      if (end < 0) throw new Error('incomplete git cat-file response');
+      const header = data.toString('utf8', offset, end);
+      offset = end + 1;
+      if (header.endsWith(' missing')) {
+        result.set(file, null);
+        continue;
+      }
+      const match = /^[0-9a-f]+ blob (\d+)$/.exec(header);
+      if (!match) throw new Error(`expected source blob for ${file}`);
+      const size = Number(match[1]);
+      if (offset + size >= data.length || data[offset + size] !== 10) throw new Error('incomplete git source blob');
+      result.set(file, data.toString('utf8', offset, offset + size));
+      offset += size + 1;
+    }
+    return result;
+  }
+
   /** Size in bytes of a file at a commit, or null if it does not exist there. */
   size(commit, file) {
     try {

src/server/includers.js

@@ -5,6 +5,18 @@
 import { listSources } from '../shared/project.js';
 
 const INCLUDE_RE = /^\s*#\s*include\s+"([^"]+)"/gm;
+const snapshots = new WeakMap();
+
+function sourceSnapshot(repo, project) {
+  const head = repo.head();
+  const roots = JSON.stringify([project.sourceDirs, project.includeDirs]);
+  const cached = snapshots.get(repo);
+  if (cached?.head === head && cached.roots === roots) return cached;
+  const paths = listSources(repo.dir, project).filter((file) => !/[\r\n]/.test(file));
+  const snapshot = { head, roots, sources: repo.showMany(head, paths) };
+  snapshots.set(repo, snapshot);
+  return snapshot;
+}
 
 /**
  * @param {import('./git.js').Repo} repo
@@ -15,9 +27,10 @@ const INCLUDE_RE = /^\s*#\s*include\s+"([^"]+)"/gm;
  */
 export function sourceIncluders(repo, project, headers, files = {}) {
   if (!headers.length) return [];
-  const read = (f) => files[f] ?? repo.show('HEAD', f);
+  const { sources } = sourceSnapshot(repo, project);
+  const read = (f) => Object.hasOwn(files, f) ? files[f] : sources.get(f);
   const roots = [...project.sourceDirs, ...project.includeDirs].map((d) => d.replace(/\/+$/, ''));
-  const known = new Set([...listSources(repo.dir, project), ...Object.keys(files)]);
+  const known = new Set([...sources.keys(), ...Object.keys(files)]);
   const resolve = (from, spec) => {
     const dir = from.slice(0, from.lastIndexOf('/') + 1);
     for (const base of [dir.replace(/\/$/, ''), ...roots]) {
@@ -34,18 +47,24 @@ export function sourceIncluders(repo, project, headers, files = {}) {
     }
     return edges.get(f);
   };
-  const want = new Set(headers);
-  const reaches = new Map();
-  const hits = (f, seen = new Set()) => {
-    if (reaches.has(f)) return reaches.get(f);
-    if (seen.has(f)) return false;
-    seen.add(f);
-    const r = includesOf(f).some((h) => want.has(h) || hits(h, seen));
-    reaches.set(f, r);
-    return r;
-  };
+  const reverse = new Map();
+  for (const file of known) {
+    for (const dependency of includesOf(file)) {
+      if (!reverse.has(dependency)) reverse.set(dependency, []);
+      reverse.get(dependency).push(file);
+    }
+  }
+  const affected = new Set(headers);
+  const pending = [...headers];
+  while (pending.length) {
+    for (const parent of reverse.get(pending.pop()) ?? []) {
+      if (affected.has(parent)) continue;
+      affected.add(parent);
+      pending.push(parent);
+    }
+  }
   const units = listSources(repo.dir, project, { unitsOnly: true });
-  return units.filter((u) => hits(u));
+  return units.filter((u) => affected.has(u));
 }
 
 function normalize(p) {

test/includers.test.js

@@ -0,0 +1,55 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+import { Repo } from '../src/server/git.js';
+import { sourceIncluders } from '../src/server/includers.js';
+
+function fixture() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'td-includers-'));
+  execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: dir });
+  const repo = new Repo(dir);
+  const project = { sourceDirs: ['src'], includeDirs: [], compile: { units: { 'src/a.cpp': 0, 'src/b.cpp': 0 } } };
+  repo.commitFiles({
+    'src/target.h': 'int value();\n',
+    'src/left.h': '#include "right.h"\n#include "target.h"\n',
+    'src/right.h': '#include "left.h"\n',
+    'src/a.cpp': '#include "left.h"\n',
+    'src/b.cpp': '#include "right.h"\n',
+    'src/한글 file.h': 'const char* name = "이름";\n',
+  }, 'fixture', 'test <test@example.invalid>');
+  return { dir, repo, project };
+}
+
+test('batch blob reads preserve UTF-8 bytes, spaces, empty files and absent paths', () => {
+  const { dir, repo } = fixture();
+  try {
+    repo.commitFiles({ 'src/empty.h': '' }, 'empty header', 'test <test@example.invalid>');
+    const files = ['src/한글 file.h', 'src/empty.h', 'src/missing.h', 'src/target.h'];
+    const snapshot = repo.showMany(repo.head(), files);
+    assert.equal(snapshot.get(files[0]), 'const char* name = "이름";\n');
+    assert.equal(snapshot.get(files[1]), '');
+    assert.equal(snapshot.get(files[2]), null);
+    assert.equal(snapshot.get(files[3]), 'int value();\n');
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test('include cycles, submitted overrides and head changes preserve transitive dependencies', () => {
+  const { dir, repo, project } = fixture();
+  try {
+    assert.deepEqual(sourceIncluders(repo, project, ['src/target.h']), ['src/a.cpp', 'src/b.cpp']);
+    fs.writeFileSync(path.join(dir, 'src/left.h'), 'int dirtyWorkingTree;\n');
+    assert.deepEqual(sourceIncluders(repo, project, ['src/target.h']), ['src/a.cpp', 'src/b.cpp'], 'reads committed sources, not uncommitted data');
+    assert.deepEqual(sourceIncluders(repo, project, ['src/target.h'], { 'src/left.h': '#include "right.h"\n' }), [], 'overrides apply to this query only');
+    assert.deepEqual(sourceIncluders(repo, project, ['src/target.h']), ['src/a.cpp', 'src/b.cpp']);
+    repo.commitFiles({ 'src/left.h': '#include "right.h"\n' }, 'remove target include', 'test <test@example.invalid>');
+    assert.deepEqual(sourceIncluders(repo, project, ['src/target.h']), [], 'a new commit invalidates cached source content');
+    assert.deepEqual(sourceIncluders(repo, project, ['src/new.h'], { 'src/a.cpp': '#include "new.h"\n', 'src/new.h': 'int next();\n' }), ['src/a.cpp']);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});