Batches, auto-merge, standalone td, one-line install, self-update, server Ghidra

- Batches: claim every open function of a unit or address range (td claim --unit/--range),
  submit the file once; one compile, per-function routing, merged as a whole. A failing
  function returns the batch; peer verdicts carry over to functions whose compiled code is
  unchanged (code fingerprints). Submissions may not change functions they do not hold.
- Byte-identical functions are approved and merged without human review
  (matchPolicy.autoMergeIdentical); peer-confirmed ones need one click per batch.
- Remote check: td check compiles on the server when there is no local toolchain.
- /api/original: assembly and pseudocode of the original from the server's Ghidra export
  (objdump before an export exists); td disasm/decompile fall back to it.
- td as a standalone executable (Node single executable application; tools/build-td),
  one-line installers (install.ps1 / install.sh) with checksum verification, td connect for
  any MCP client, td mcp --workspace, self-update (td update, hourly check).
- Server-side Ghidra project: built once per version, refreshed after merges, downloadable
  for contributors (td ghidra download); JDK 21 and the pinned Ghidra in deploy/install.sh.
- Agent instructions: work continuously, batches first, peer reviews in between.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RNwtXPYqFu3oKaaXwYwNWW
This commit is contained in:
Clank Together 2026-10-18 02:13:09 +02:00
commit 125f5c4e7d
27 changed files with 1751 additions and 241 deletions

1
.gitignore vendored
View file

@ -2,3 +2,4 @@ node_modules/
*.log *.log
# Operational notes about the deployment, not part of the tool # Operational notes about the deployment, not part of the tool
PORTS-TO-FORWARD.md PORTS-TO-FORWARD.md
dist/

View file

@ -5,6 +5,9 @@ set -eu
HOST="${1:?usage: install.sh <public-hostname>}" HOST="${1:?usage: install.sh <public-hostname>}"
NODE_VERSION=v22.21.0 NODE_VERSION=v22.21.0
GHIDRA_VERSION=11.2.1
GHIDRA_URL=https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_11.2.1_build/ghidra_11.2.1_PUBLIC_20241105.zip
GHIDRA_SHA256=ce4db5117da0fbaf8f33863fec4f40902f754f06b68945a59fb1c0f9b1bc461c
SRC=$(cd "$(dirname "$0")/.." && pwd) SRC=$(cd "$(dirname "$0")/.." && pwd)
APP=/opt/togetherdecomp APP=/opt/togetherdecomp
DATA=/var/lib/togetherdecomp DATA=/var/lib/togetherdecomp
@ -12,7 +15,8 @@ DATA=/var/lib/togetherdecomp
echo "== packages" echo "== packages"
export DEBIAN_FRONTEND=noninteractive export DEBIAN_FRONTEND=noninteractive
apt-get update -q apt-get update -q
apt-get install -y -q git curl ca-certificates nftables caddy docker.io xz-utils >/dev/null # binutils: objdump disassembles original functions for /api/original before a Ghidra export exists.
apt-get install -y -q git curl ca-certificates nftables caddy docker.io xz-utils binutils >/dev/null
echo "== node $NODE_VERSION" echo "== node $NODE_VERSION"
if [ "$(/opt/node/bin/node --version 2>/dev/null || true)" != "$NODE_VERSION" ]; then if [ "$(/opt/node/bin/node --version 2>/dev/null || true)" != "$NODE_VERSION" ]; then
@ -25,6 +29,19 @@ if [ "$(/opt/node/bin/node --version 2>/dev/null || true)" != "$NODE_VERSION" ];
rm -rf "$tmp" rm -rf "$tmp"
fi fi
echo "== ghidra $GHIDRA_VERSION (server-side project and pseudocode)"
# The JDK, not just the runtime: Ghidra compiles its .java scripts.
apt-get install -y -q openjdk-21-jdk-headless unzip >/dev/null
GHIDRA_HOME="/opt/ghidra/ghidra_${GHIDRA_VERSION}_PUBLIC"
if [ ! -x "$GHIDRA_HOME/support/analyzeHeadless" ]; then
tmp=$(mktemp -d)
curl -fsSL "$GHIDRA_URL" -o "$tmp/ghidra.zip"
echo "$GHIDRA_SHA256 $tmp/ghidra.zip" | sha256sum -c - >/dev/null
mkdir -p /opt/ghidra
unzip -q -o "$tmp/ghidra.zip" -d /opt/ghidra
rm -rf "$tmp"
fi
echo "== docker sandbox settings" echo "== docker sandbox settings"
mkdir -p /etc/docker mkdir -p /etc/docker
cat > /etc/docker/daemon.json <<'EOF' cat > /etc/docker/daemon.json <<'EOF'
@ -53,13 +70,14 @@ if [ ! -f "$DATA/server.json" ]; then
fi fi
# Production settings, keeping anything else already configured. # Production settings, keeping anything else already configured.
UIDGID="$(id -u togetherdecomp):$(id -g togetherdecomp)" UIDGID="$(id -u togetherdecomp):$(id -g togetherdecomp)"
runuser -u togetherdecomp -- $NODE -e ' runuser -u togetherdecomp -- env GHIDRA_HOME="$GHIDRA_HOME" $NODE -e '
const fs = require("fs"); const [file, host, user] = process.argv.slice(1); const fs = require("fs"); const [file, host, user] = process.argv.slice(1);
const c = JSON.parse(fs.readFileSync(file, "utf8")); const c = JSON.parse(fs.readFileSync(file, "utf8"));
c.listen = { host: "127.0.0.1", port: 8080 }; c.listen = { host: "127.0.0.1", port: 8080 };
c.publicUrl = "https://" + host; c.publicUrl = "https://" + host;
c.trustProxy = true; c.trustProxy = true;
c.allowUnsandboxedRunners = false; c.allowUnsandboxedRunners = false;
c.ghidraHome = process.env.GHIDRA_HOME || c.ghidraHome || null;
for (const r of Object.values(c.runners || {})) if (r.type === "docker") r.user = user; for (const r of Object.values(c.runners || {})) if (r.type === "docker") r.user = user;
fs.writeFileSync(file, JSON.stringify(c, null, 2) + "\n"); fs.writeFileSync(file, JSON.stringify(c, null, 2) + "\n");
' "$DATA/server.json" "$HOST" "$UIDGID" ' "$DATA/server.json" "$HOST" "$UIDGID"

View file

@ -0,0 +1,113 @@
// Exports the disassembly and the decompiler output of every function as JSON lines, for the
// coordinator's /api/original (contributors then need no local Ghidra):
// {"addr":"0x140099b90","asm":"...","c":"..."}
// Args: <output.jsonl> [decompile timeout seconds, default 30]
// Run on a project built with `td ghidra build` (symbol names applied), with -readOnly.
// @category ClankTogether
import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileResults;
import ghidra.app.script.GhidraScript;
import ghidra.program.model.address.Address;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.FunctionIterator;
import ghidra.program.model.listing.Instruction;
import ghidra.program.model.listing.InstructionIterator;
import ghidra.program.model.symbol.Reference;
import ghidra.program.model.symbol.Symbol;
public class ExportOriginals extends GhidraScript {
@Override
protected void run() throws Exception {
String[] args = getScriptArgs();
int timeout = args.length > 1 ? Integer.parseInt(args[1]) : 30;
DecompInterface ifc = new DecompInterface();
ifc.openProgram(currentProgram);
int n = 0;
int failed = 0;
try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(args[0]), StandardCharsets.UTF_8))) {
FunctionIterator it = currentProgram.getFunctionManager().getFunctions(true);
while (it.hasNext() && !monitor.isCancelled()) {
Function f = it.next();
if (f.isExternal() || f.isThunk()) {
continue;
}
String c;
DecompileResults r = ifc.decompileFunction(f, timeout, monitor);
if (r.decompileCompleted() && r.getDecompiledFunction() != null) {
c = r.getDecompiledFunction().getC();
} else {
c = null;
failed++;
}
out.write("{\"addr\":\"0x" + Long.toHexString(f.getEntryPoint().getOffset()) + "\",\"asm\":" + json(disassemble(f)) + ",\"c\":" + (c == null ? "null" : json(c)) + "}\n");
if (++n % 1000 == 0) {
println("exported " + n + " functions");
}
}
} finally {
ifc.dispose();
}
println("exported " + n + " functions, " + failed + " without decompiler output");
}
private String disassemble(Function f) {
StringBuilder sb = new StringBuilder();
Address entry = f.getEntryPoint();
sb.append("; ").append(f.getName(true)).append(" @ ").append(entry).append(" ").append(f.getSignature().getPrototypeString()).append('\n');
InstructionIterator it = currentProgram.getListing().getInstructions(f.getBody(), true);
while (it.hasNext()) {
Instruction insn = it.next();
Address a = insn.getAddress();
if (!a.equals(entry)) {
for (Symbol s : currentProgram.getSymbolTable().getSymbols(a)) {
sb.append(s.getName(true)).append(":\n");
}
}
StringBuilder bytes = new StringBuilder();
try {
for (byte b : insn.getBytes()) {
bytes.append(String.format("%02x ", b & 0xff));
}
} catch (Exception e) {
bytes.append("?? ");
}
StringBuilder refs = new StringBuilder();
for (Reference ref : insn.getReferencesFrom()) {
Symbol s = currentProgram.getSymbolTable().getPrimarySymbol(ref.getToAddress());
if (s != null) {
refs.append(" ; -> ").append(s.getName(true));
}
}
sb.append(String.format(" +0x%04x %08x %-24s %s%s%n", a.subtract(entry), a.getOffset(), bytes.toString().trim(), insn.toString(), refs));
}
return sb.toString();
}
private static String json(String s) {
StringBuilder sb = new StringBuilder("\"");
for (int i = 0; i < s.length(); i++) {
char ch = s.charAt(i);
switch (ch) {
case '"': sb.append("\\\""); break;
case '\\': sb.append("\\\\"); break;
case '\n': sb.append("\\n"); break;
case '\r': sb.append("\\r"); break;
case '\t': sb.append("\\t"); break;
default:
if (ch < 0x20) {
sb.append(String.format("\\u%04x", (int) ch));
} else {
sb.append(ch);
}
}
}
return sb.append('"').toString();
}
}

View file

@ -2,11 +2,13 @@
// `td` - contributor CLI for Clank Together. // `td` - contributor CLI for Clank Together.
import path from 'node:path'; import path from 'node:path';
import { spawnSync } from 'node:child_process';
import { parseArgs } from 'node:util'; import { parseArgs } from 'node:util';
import { Workspace } from './workspace.js'; import { Workspace } from './workspace.js';
import * as ops from './ops.js'; import * as ops from './ops.js';
import * as ghidra from './ghidra.js'; import * as ghidra from './ghidra.js';
import { startMcp } from './mcp.js'; import { startMcp } from './mcp.js';
import { maybeUpdate } from './update.js';
import { hex } from '../shared/annotations.js'; import { hex } from '../shared/annotations.js';
const HELP = `td - Clank Together contributor client const HELP = `td - Clank Together contributor client
@ -18,17 +20,23 @@ Setup
[--toolchain <toolchain root>] compiler layout of the project (e.g. re/toolchain) [--toolchain <toolchain root>] compiler layout of the project (e.g. re/toolchain)
[--vc <VC dir>] [--sdk <SDK dir>] [--image <docker image>] [--vc <VC dir>] [--sdk <SDK dir>] [--image <docker image>]
td sync pull main and refresh the symbol database snapshot td sync pull main and refresh the symbol database snapshot
td ghidra install | build | sync pinned Ghidra: install, (re)build the project, re-apply symbols td ghidra download the server's Ghidra project (names applied, refreshed after merges)
td ghidra install | build | sync pinned Ghidra: install it, analyze locally, re-apply symbols
td update update td now (it also checks by itself once an hour)
td connect [--workspace <dir>] connect your AI (registers with Claude Code, prints MCP settings)
td mcp [--workspace <dir>] run the MCP server for your agent (stdio) td mcp [--workspace <dir>] run the MCP server for your agent (stdio)
Work Work
td tasks [--state S] [--limit N] list claimable tasks td tasks [--state S] [--limit N] list claimable tasks
td claim [taskId] claim a task, prints its context td claim [taskId] claim a task, prints its context
td claim --unit <file> [--limit N] claim every open function of a source file (a batch)
td claim --range <start>-<end> [--limit N] claim every open function of an address range
td task <taskId> show a task's context td task <taskId> show a task's context
td check [files...] compile + compare locally (default: changed files) td check [files...] compile + compare locally (default: changed files)
td submit <taskId> [files...] [--notes T] submit for server verification td submit [taskId...] [--notes T] submit everything you hold (or these tasks) as one batch
td batch <batchId> state of a submitted batch, per function
td result <attemptId> show a verification result td result <attemptId> show a verification result
td heartbeat <taskId> | td release <taskId> td heartbeat [taskId] | td release <taskId> extend your leases (all without an id)
td config [ghidraHome|server <value>] show or change workspace settings td config [ghidraHome|server <value>] show or change workspace settings
td symbol <id|name|0xaddr> look up a symbol td symbol <id|name|0xaddr> look up a symbol
td symbol new <kind> <0xaddr> [--name N] get a stable ID for a new global/vtable/string/function td symbol new <kind> <0xaddr> [--name N] get a stable ID for a new global/vtable/string/function
@ -59,6 +67,29 @@ function printCheck(r) {
for (const c of r.referenceConflicts) console.log(`conflict: ${c.message ?? c}`); for (const c of r.referenceConflicts) console.log(`conflict: ${c.message ?? c}`);
} }
/** How an MCP client starts us: the td binary itself, or node + this script when run from source. */
async function launcher() {
const sea = await import('node:sea').catch(() => null);
return sea?.isSea?.() ? { command: process.execPath, args: [] } : { command: process.execPath, args: [path.resolve(process.argv[1])] };
}
/**
* Registers the MCP server with Claude Code when it is installed, and prints the settings
* for every other MCP client.
*/
async function connect(dir) {
const ws = Workspace.find(dir);
const l = await launcher();
const args = [...l.args, 'mcp', '--workspace', ws.dir];
const claude = spawnSync(process.platform === 'win32' ? 'claude.cmd' : 'claude', ['mcp', 'add', '--scope', 'user', 'clank-together', '--', l.command, ...args], { encoding: 'utf8', shell: process.platform === 'win32' });
if (claude.status === 0) console.log('Claude Code: connected (MCP server "clank-together"). Start claude and ask it to read AGENTS.md and claim a task.');
else console.log('Claude Code: not found, or already connected (claude mcp list shows it).');
console.log(`\nOther MCP clients (Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI, ...): add this to their MCP server settings:
${JSON.stringify({ mcpServers: { 'clank-together': { command: l.command, args } } }, null, 2)}
Agents without MCP can run the td commands directly: td tasks, td claim, td check, td submit, td peer ...`);
}
async function main() { async function main() {
const { values, positionals } = parseArgs({ const { values, positionals } = parseArgs({
allowPositionals: true, allowPositionals: true,
@ -74,8 +105,12 @@ async function main() {
state: { type: 'string' }, state: { type: 'string' },
limit: { type: 'string', default: '20' }, limit: { type: 'string', default: '20' },
notes: { type: 'string' }, notes: { type: 'string' },
unit: { type: 'string' },
range: { type: 'string' },
files: { type: 'string' },
reason: { type: 'string' }, reason: { type: 'string' },
json: { type: 'boolean', default: false }, json: { type: 'boolean', default: false },
force: { type: 'boolean', default: false },
workspace: { type: 'string' }, workspace: { type: 'string' },
help: { type: 'boolean', short: 'h' }, help: { type: 'boolean', short: 'h' },
}, },
@ -83,22 +118,24 @@ async function main() {
const [cmd, ...args] = positionals; const [cmd, ...args] = positionals;
if (!cmd || values.help) return print(HELP); if (!cmd || values.help) return print(HELP);
// Self-update (standalone binary only): at most once an hour, never fatal.
const serverHint = cmd === 'setup' ? args[0] : (() => { try { return Workspace.find(values.workspace ?? process.cwd()).config.server; } catch { return null; } })();
const updated = await maybeUpdate({ server: serverHint, force: cmd === 'update' });
if (updated) process.stderr.write(`${updated}\n`);
if (cmd === 'update') return print(updated ?? 'td is up to date');
if (cmd === 'setup') { if (cmd === 'setup') {
const [server, dir] = args; const [server, dir] = args;
if (!server || !dir) throw new Error('usage: td setup <server-url> <dir> --token <API token from the website>'); if (!server || !dir) throw new Error('usage: td setup <server-url> <dir> --token <API token from the website>');
const ws = await ops.setup({ server, dir, token: values.token, exe: values.exe, toolchain: values.toolchain, runner: values.runner, vc: values.vc, sdk: values.sdk, image: values.image }); const ws = await ops.setup({ server, dir, token: values.token, exe: values.exe, toolchain: values.toolchain, runner: values.runner, vc: values.vc, sdk: values.sdk, image: values.image });
const remote = ws.config.runners?.[ws.project.toolchain]?.type === 'remote';
console.log(`\nworkspace ready at ${ws.dir} console.log(`\nworkspace ready at ${ws.dir}
next: compiling: ${remote ? "on the server (no local compiler needed); add one later with td setup ... --toolchain <dir>" : 'locally'}
cd ${path.relative(process.cwd(), ws.dir) || '.'} original code: ${ws.config.ghidraHome ? 'local Ghidra' : "from the server; a local Ghidra is optional (td ghidra install && td ghidra build)"}
td ghidra install && td ghidra build (one-time, takes a while) connect your AI with: td connect --workspace "${ws.dir}"`);
td check src/... (self-test once there is code)
connect your agent (any MCP client works):
Claude Code: claude mcp add clank-together -- node "${path.resolve(process.argv[1])}" mcp --workspace "${ws.dir}"
other clients (Cursor, Windsurf, VS Code, Cline, Claude Desktop, ...), in their MCP server settings:
${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [path.resolve(process.argv[1]), 'mcp', '--workspace', ws.dir] } } }, null, 2).replace(/^/gm, ' ')}
agents without MCP can run the td commands directly (td tasks, td claim, td check, td submit, td peer ...).`);
return; return;
} }
if (cmd === 'connect') return connect(values.workspace ?? process.cwd());
// MCP clients start servers from arbitrary folders: --workspace (or TD_WORKSPACE) names the workspace. // MCP clients start servers from arbitrary folders: --workspace (or TD_WORKSPACE) names the workspace.
if (cmd === 'mcp') return startMcp(values.workspace ?? process.env.TD_WORKSPACE ?? process.cwd()); if (cmd === 'mcp') return startMcp(values.workspace ?? process.env.TD_WORKSPACE ?? process.cwd());
@ -112,12 +149,20 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat
console.log(`#${String(t.id).padEnd(6)} ${t.symbol_id} ${t.addr} size ${String(t.size ?? '?').padEnd(6)} d${t.difficulty} ${t.state.padEnd(13)} best ${(t.best_score * 100).toFixed(0)}% ${t.name ?? ''} ${t.unit ?? ''}`); console.log(`#${String(t.id).padEnd(6)} ${t.symbol_id} ${t.addr} size ${String(t.size ?? '?').padEnd(6)} d${t.difficulty} ${t.state.padEnd(13)} best ${(t.best_score * 100).toFixed(0)}% ${t.name ?? ''} ${t.unit ?? ''}`);
} }
return; return;
case 'claim': case 'claim': {
// td claim [taskId] | td claim --unit <file> | td claim --range 0x140001000-0x140002000 [--limit N]
if (values.unit || values.range) {
const [start, end] = values.range ? values.range.split('-') : [];
return print(await ops.claimBatch(ws, { unit: values.unit ? ops.relPath(ws, values.unit) : undefined, start, end, limit: Number(values.limit) || 50 }));
}
return print(await client.post('/api/tasks/claim', { taskId: args[0] ? Number(args[0]) : undefined })); return print(await client.post('/api/tasks/claim', { taskId: args[0] ? Number(args[0]) : undefined }));
}
case 'task': case 'task':
return print(await client.get(`/api/tasks/${args[0]}`)); return print(await client.get(`/api/tasks/${args[0]}`));
case 'heartbeat': case 'heartbeat':
return print(await client.post(`/api/tasks/${args[0]}/heartbeat`)); return print(args[0] ? await client.post(`/api/tasks/${args[0]}/heartbeat`) : await client.post('/api/tasks/heartbeat-all'));
case 'batch':
return print(await client.get(`/api/batches/${args[0]}`));
case 'release': case 'release':
return print(await client.post(`/api/tasks/${args[0]}/release`)); return print(await client.post(`/api/tasks/${args[0]}/release`));
case 'check': { case 'check': {
@ -125,8 +170,11 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat
return values.json ? print(r) : printCheck(r); return values.json ? print(r) : printCheck(r);
} }
case 'submit': { case 'submit': {
const [taskId, ...files] = args; // td submit everything you hold, as one batch
return print(await ops.submit(ws, Number(taskId), { files: files.map((f) => ops.relPath(ws, f)), notes: values.notes, agentMeta: { client: 'td-cli' } })); // td submit <taskId>... [--files a.cpp,b.h]
const ids = args.filter((x) => /^\d+$/.test(x)).map(Number);
const files = values.files ? values.files.split(',').map((f) => ops.relPath(ws, f)) : [];
return print(await ops.submit(ws, ids, { files, notes: values.notes, agentMeta: { client: 'td-cli' } }));
} }
case 'result': { case 'result': {
const r = await client.get(`/api/attempts/${args[0]}`); const r = await client.get(`/api/attempts/${args[0]}`);
@ -166,13 +214,15 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat
if (args[0] === 'install') return print(`Ghidra installed at ${await ghidra.installGhidra(ws)}`); if (args[0] === 'install') return print(`Ghidra installed at ${await ghidra.installGhidra(ws)}`);
if (args[0] === 'build') return void (await ghidra.buildGhidraProject(ws), print('Ghidra project built')); if (args[0] === 'build') return void (await ghidra.buildGhidraProject(ws), print('Ghidra project built'));
if (args[0] === 'sync') return void (await ghidra.syncGhidraProject(ws), print('symbols applied')); if (args[0] === 'sync') return void (await ghidra.syncGhidraProject(ws), print('symbols applied'));
throw new Error('usage: td ghidra install|build|sync'); if (args[0] === 'download') {
const r = await ghidra.downloadGhidraProject(ws, { force: values.force });
return print(r.updated ? `Ghidra project of ${r.info.version} downloaded${ws.config.ghidraHome ? '' : ' (open it with Ghidra 11.2.1, or run td ghidra install)'}` : 'the Ghidra project is up to date');
}
throw new Error('usage: td ghidra install|download|build|sync');
case 'decompile': case 'decompile':
case 'disasm': { case 'disasm': {
const s = ws.findSymbol(args[0]); // Local Ghidra if configured, otherwise the server's export.
const a = s?.addr ?? (/^0x/i.test(args[0] ?? '') ? Number.parseInt(args[0], 16) : null); return print(await ops.original(ws, args[0], cmd === 'decompile' ? 'c' : 'asm'));
if (a == null) throw new Error(`unknown symbol ${args[0]}`);
return print(cmd === 'decompile' ? await ghidra.decompile(ws, a) : await ghidra.disassemble(ws, a));
} }
default: default:
print(HELP); print(HELP);

View file

@ -13,7 +13,22 @@ import { hex } from '../shared/annotations.js';
import { loadToolchainLock } from '../shared/project.js'; import { loadToolchainLock } from '../shared/project.js';
import { Api } from './api.js'; import { Api } from './api.js';
const SCRIPTS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts'); const SCRIPT_NAMES = ['ApplySymbols.java', 'DecompileAt.java', 'DisassembleAt.java', 'ExportFunctions.java', 'ExportOriginals.java'];
/**
* The Ghidra scripts: next to the sources when run from a checkout or npm install; extracted
* into the workspace when td runs as a single executable (they are embedded as assets).
*/
async function scriptsDir(ws) {
const sea = await import('node:sea').catch(() => null);
if (sea?.isSea?.()) {
const dir = path.join(ws.tdDir, 'ghidra_scripts');
fs.mkdirSync(dir, { recursive: true });
for (const name of SCRIPT_NAMES) fs.writeFileSync(path.join(dir, name), sea.getAsset(name, 'utf8'));
return dir;
}
return path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts');
}
const PROJECT_NAME = 'td'; const PROJECT_NAME = 'td';
const PROGRAM = 'target.exe'; const PROGRAM = 'target.exe';
@ -33,7 +48,7 @@ async function runHeadless(ws, args, timeoutMs) {
const exe = headless(ws); const exe = headless(ws);
const projDir = path.join(ws.tdDir, 'ghidra'); const projDir = path.join(ws.tdDir, 'ghidra');
fs.mkdirSync(projDir, { recursive: true }); fs.mkdirSync(projDir, { recursive: true });
const full = [projDir, PROJECT_NAME, ...args, '-scriptPath', SCRIPTS_DIR]; const full = [projDir, PROJECT_NAME, ...args, '-scriptPath', await scriptsDir(ws)];
// .bat files need a shell on Windows. // .bat files need a shell on Windows.
const r = os.platform() === 'win32' ? await run('cmd.exe', ['/d', '/s', '/c', exe, ...full], { timeoutMs }) : await run(exe, full, { timeoutMs }); const r = os.platform() === 'win32' ? await run('cmd.exe', ['/d', '/s', '/c', exe, ...full], { timeoutMs }) : await run(exe, full, { timeoutMs });
if (r.code !== 0) throw new Error(`Ghidra failed (exit ${r.code}):\n${r.output.slice(-4000)}`); if (r.code !== 0) throw new Error(`Ghidra failed (exit ${r.code}):\n${r.output.slice(-4000)}`);
@ -104,3 +119,28 @@ export function decompile(ws, addr) {
export function disassemble(ws, addr) { export function disassemble(ws, addr) {
return query(ws, 'DisassembleAt.java', ['0x' + hex(addr)]); return query(ws, 'DisassembleAt.java', ['0x' + hex(addr)]);
} }
/**
* Downloads the server's Ghidra project (built and refreshed on the server, with the shared symbol
* names) into the workspace, replacing the local one. Nothing happens when it is up to date.
*/
export async function downloadGhidraProject(ws, { force = false } = {}) {
const client = new Api(ws.config.server, ws.config.token);
const info = await client.get('/api/ghidra-project');
if (!force && ws.config.ghidraPackage === info.sha256 && fs.existsSync(path.join(ws.tdDir, 'ghidra', `${PROJECT_NAME}.gpr`))) {
return { updated: false, info };
}
const tgz = path.join(ws.tdDir, 'ghidra-project.tar.gz');
console.error(`downloading the Ghidra project of ${info.version} (${(info.size / 1048576).toFixed(0)} MB)...`);
await client.download(info.download, tgz, info.sha256);
const projDir = path.join(ws.tdDir, 'ghidra');
fs.rmSync(projDir, { recursive: true, force: true });
fs.mkdirSync(projDir, { recursive: true });
const tar = os.platform() === 'win32' ? path.join(process.env.SystemRoot ?? 'C:\Windows', 'System32', 'tar.exe') : 'tar';
const r = await run(tar, ['-xzf', tgz, '-C', projDir], { timeoutMs: 1800_000 });
if (r.code !== 0) throw new Error(`could not unpack the Ghidra project: ${r.output}`);
fs.rmSync(tgz, { force: true });
ws.config.ghidraPackage = info.sha256;
ws.save();
return { updated: true, info };
}

View file

@ -4,7 +4,6 @@
import readline from 'node:readline'; import readline from 'node:readline';
import { Workspace } from './workspace.js'; import { Workspace } from './workspace.js';
import * as ops from './ops.js'; import * as ops from './ops.js';
import * as ghidra from './ghidra.js';
import { hex } from '../shared/annotations.js'; import { hex } from '../shared/annotations.js';
const PROTOCOL_VERSIONS = ['2025-06-18', '2025-03-26', '2024-11-05']; const PROTOCOL_VERSIONS = ['2025-06-18', '2025-03-26', '2024-11-05'];
@ -33,9 +32,9 @@ const TOOLS = [
}, },
{ {
name: 'td_heartbeat', name: 'td_heartbeat',
description: 'Extend the lease on a task you are still working on.', description: 'Extend your leases while you are still working (all of them, or one taskId). Leases expire after two hours otherwise.',
inputSchema: { type: 'object', properties: { taskId: int('task id') }, required: ['taskId'] }, inputSchema: { type: 'object', properties: { taskId: int('task id; omit for all') } },
run: async (ws, a) => ops.api(ws).post(`/api/tasks/${a.taskId}/heartbeat`), run: async (ws, a) => (a.taskId ? ops.api(ws).post(`/api/tasks/${a.taskId}/heartbeat`) : ops.api(ws).post('/api/tasks/heartbeat-all')),
}, },
{ {
name: 'td_release_task', name: 'td_release_task',
@ -47,13 +46,13 @@ const TOOLS = [
name: 'td_decompile', name: 'td_decompile',
description: 'Ghidra decompiler output for a function in the original executable (by symbol ID, name or hex address). A starting point only; the result must be rewritten into matching C++.', description: 'Ghidra decompiler output for a function in the original executable (by symbol ID, name or hex address). A starting point only; the result must be rewritten into matching C++.',
inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] }, inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] },
run: async (ws, a) => ghidra.decompile(ws, resolveAddr(ws, a.symbol)), run: async (ws, a) => ops.original(ws, a.symbol, 'c'),
}, },
{ {
name: 'td_disassemble', name: 'td_disassemble',
description: 'Disassembly of a function in the original executable, the ground truth to match.', description: 'Disassembly of a function in the original executable, the ground truth to match.',
inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] }, inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] },
run: async (ws, a) => ghidra.disassemble(ws, resolveAddr(ws, a.symbol)), run: async (ws, a) => ops.original(ws, a.symbol, 'asm'),
}, },
{ {
name: 'td_lookup_symbol', name: 'td_lookup_symbol',
@ -82,9 +81,24 @@ const TOOLS = [
}, },
{ {
name: 'td_submit', name: 'td_submit',
description: 'Submit your work on a leased task for server verification. Defaults to all changed files the task allows. Ends the lease.', description: 'Submit your work for server verification, as one batch: every task you hold (default) or the given taskIds, with all changed files they allow. Byte-identical functions are merged automatically; others go to peer review. Ends the leases. Do not wait for the result: claim the next work right away and check results later.',
inputSchema: { type: 'object', properties: { taskId: int('task id'), files: { type: 'array', items: { type: 'string' } }, notes: str('short notes for reviewers and later agents') }, required: ['taskId'] }, inputSchema: {
run: async (ws, a) => ops.submit(ws, a.taskId, { files: a.files, notes: a.notes, agentMeta: { client: 'td-mcp' } }), type: 'object',
properties: { taskIds: { type: 'array', items: { type: 'integer' }, description: 'tasks to submit; default: all you hold' }, taskId: int('a single task (same as taskIds: [id])'), files: { type: 'array', items: { type: 'string' } }, notes: str('short notes for reviewers and later agents') },
},
run: async (ws, a) => ops.submit(ws, a.taskIds ?? (a.taskId ? [a.taskId] : []), { files: a.files, notes: a.notes, agentMeta: { client: 'td-mcp' } }),
},
{
name: 'td_claim_batch',
description: 'Claim many functions at once: every open function of a source file (unit), or of an address range of the current version, up to limit (default 50). Work through them in the file, td_check, then td_submit them together. Much cheaper than one function at a time.',
inputSchema: { type: 'object', properties: { unit: str('repo-relative source file, e.g. Engine/Source/FrameXML/LoadXML.cpp'), start: str('range start, hex'), end: str('range end, hex (exclusive)'), limit: int('max functions, default 50') } },
run: async (ws, a) => ops.claimBatch(ws, a),
},
{
name: 'td_get_batch',
description: 'State of a submitted batch, per function (approved, peer_review, in_review, needs_cleanup, merged...).',
inputSchema: { type: 'object', properties: { batchId: int('batch id') }, required: ['batchId'] },
run: async (ws, a) => ops.api(ws).get(`/api/batches/${a.batchId}`),
}, },
{ {
name: 'td_get_result', name: 'td_get_result',
@ -132,12 +146,6 @@ const TOOLS = [
}, },
]; ];
function resolveAddr(ws, q) {
const s = ws.findSymbol(q);
if (s?.addr != null) return s.addr;
if (/^0x[0-9a-f]+$/i.test(q)) return Number.parseInt(q, 16);
throw new Error(`unknown symbol ${q}`);
}
export function startMcp(dir) { export function startMcp(dir) {
const ws = Workspace.find(dir); const ws = Workspace.find(dir);
@ -163,7 +171,7 @@ export function startMcp(dir) {
protocolVersion: PROTOCOL_VERSIONS.includes(requested) ? requested : PROTOCOL_VERSIONS[0], protocolVersion: PROTOCOL_VERSIONS.includes(requested) ? requested : PROTOCOL_VERSIONS[0],
capabilities: { tools: {} }, capabilities: { tools: {} },
serverInfo: { name: 'clank-together', version: '0.1.0' }, serverInfo: { name: 'clank-together', version: '0.1.0' },
instructions: 'Tools for a collaborative matching decompilation. Read AGENTS.md in the workspace root first. Workflow: td_claim_task -> td_disassemble/td_decompile -> write C++ in the task file with the given annotation -> td_check until exact -> td_submit -> td_get_result. Also review the work of others: td_list_peer_reviews -> td_claim_peer_review -> read the original and the submission (no compiling or diffing) -> td_submit_peer_review.', instructions: INSTRUCTIONS,
}); });
} }
case 'ping': case 'ping':
@ -185,3 +193,17 @@ export function startMcp(dir) {
} }
}); });
} }
// What every connected agent is told when it starts. Work continues until the user stops it.
const INSTRUCTIONS = [
'Clank Together: a collaborative matching decompilation. Read AGENTS.md in the workspace first.',
'Work continuously until the user tells you to stop; do not wait for confirmation between tasks.',
'Loop: td_claim_batch (a whole unit: every open function of one source file) or td_claim_task ->',
'read the originals (td_disassemble, td_decompile) -> write the C++ with the given annotations ->',
'td_check until as many functions as possible are byte-identical -> td_submit (everything you hold) ->',
'immediately claim the next work. Do not wait for verification; look at results later with td_get_batch / td_get_result',
'and fix what came back (needs_cleanup).',
'Between batches, take a peer review when one is waiting: td_list_peer_reviews -> td_claim_peer_review ->',
'read the original and the submission only (no compiling or diffing) -> td_submit_peer_review for every function.',
'Call td_heartbeat every hour or so while working on a big batch.',
].join(' ');

View file

@ -6,12 +6,13 @@ import path from 'node:path';
import { execFileSync } from 'node:child_process'; import { execFileSync } from 'node:child_process';
import { Api } from './api.js'; import { Api } from './api.js';
import { Workspace, sha256File } from './workspace.js'; import { Workspace, sha256File } from './workspace.js';
import { checkUnits } from '../shared/check.js'; import { checkUnits, summarizeCheck } from '../shared/check.js';
import { reconcile } from '../shared/reconcile.js'; import { reconcile } from '../shared/reconcile.js';
import { lintSubmission, matchesAny } from '../shared/lint.js'; import { lintSubmission, matchesAny } from '../shared/lint.js';
import { isUnit, loadToolchainLock } from '../shared/project.js'; import { isUnit, loadToolchainLock } from '../shared/project.js';
import { hex } from '../shared/annotations.js'; import { hex } from '../shared/annotations.js';
import { run } from '../shared/build.js'; import { run } from '../shared/build.js';
import { rememberServer } from './update.js';
export function api(ws) { export function api(ws) {
return new Api(ws.config.server, ws.config.token); return new Api(ws.config.server, ws.config.token);
@ -43,6 +44,7 @@ export async function setup({ server, dir, token, exe, runner, vc, sdk, image, t
} }
const ws = new Workspace(abs); const ws = new Workspace(abs);
ws.config.server = server.replace(/\/+$/, ''); ws.config.server = server.replace(/\/+$/, '');
rememberServer(ws.config.server);
ws.config.token = useToken; ws.config.token = useToken;
ws.config.user = me.name; ws.config.user = me.name;
fs.mkdirSync(ws.tdDir, { recursive: true }); fs.mkdirSync(ws.tdDir, { recursive: true });
@ -60,11 +62,11 @@ export async function setup({ server, dir, token, exe, runner, vc, sdk, image, t
log(`target verified: sha256 ${info.version.sha256}`); log(`target verified: sha256 ${info.version.sha256}`);
const lock = loadToolchainLock(abs)[info.toolchain] ?? {}; const lock = loadToolchainLock(abs)[info.toolchain] ?? {};
ws.config.runners = { [info.toolchain]: runnerConfig({ runner, vc, sdk, image: image ?? lock.image, toolchain, project: ws.project }) }; ws.config.runners = { [info.toolchain]: runnerConfig({ runner, vc, sdk, image: image ?? (runner === 'docker' ? lock.image : undefined), toolchain, project: ws.project }) };
ws.save(); ws.save();
await refreshSymbols(ws); await refreshSymbols(ws);
ws.config.envDigest = await computeEnvDigest(ws, info.toolchain); ws.config.envDigest = await computeEnvDigest(ws, info.toolchain);
if (lock.clSha256 && ws.config.envDigest !== lock.clSha256) { if (lock.clSha256 && ws.config.envDigest && ws.config.envDigest !== lock.clSha256) {
log(`WARNING: your cl.exe (sha256 ${ws.config.envDigest}) is not the pinned one (${lock.clSha256}); results may differ from the server`); log(`WARNING: your cl.exe (sha256 ${ws.config.envDigest}) is not the pinned one (${lock.clSha256}); results may differ from the server`);
} }
ws.save(); ws.save();
@ -90,7 +92,10 @@ function runnerConfig({ runner, vc, sdk, image, toolchain, project }) {
if (type === 'wine') return { type, clPath, wine: 'wine', toolchainRoot: root, jobs, env: { WINEDEBUG: '-all' } }; if (type === 'wine') return { type, clPath, wine: 'wine', toolchainRoot: root, jobs, env: { WINEDEBUG: '-all' } };
return { type, clPath, toolchainRoot: root, jobs, env: { PATH: `${path.dirname(clPath)};${process.env.PATH ?? ''}` } }; return { type, clPath, toolchainRoot: root, jobs, env: { PATH: `${path.dirname(clPath)};${process.env.PATH ?? ''}` } };
} }
// Nothing local given: compile on the server (remote check).
if (!runner && !vc && !image) return { type: 'remote' };
const type = runner ?? (os.platform() === 'win32' && vc ? 'native' : 'docker'); const type = runner ?? (os.platform() === 'win32' && vc ? 'native' : 'docker');
if (type === 'remote') return { type };
if (type === 'docker') { if (type === 'docker') {
if (!image) throw new Error('no docker image configured; pass --image or set it in toolchain.lock.json'); if (!image) throw new Error('no docker image configured; pass --image or set it in toolchain.lock.json');
return { type, image }; return { type, image };
@ -108,6 +113,7 @@ function runnerConfig({ runner, vc, sdk, image, toolchain, project }) {
async function computeEnvDigest(ws, toolchain) { async function computeEnvDigest(ws, toolchain) {
const r = ws.config.runners?.[toolchain]; const r = ws.config.runners?.[toolchain];
if (!r) return null; if (!r) return null;
if (r.type === 'remote') return null; // the server's compiler is the pinned one
if (r.type !== 'docker') return fs.existsSync(r.clPath) ? sha256File(r.clPath) : null; if (r.type !== 'docker') return fs.existsSync(r.clPath) ? sha256File(r.clPath) : null;
const out = await run('docker', ['run', '--rm', '--network', 'none', r.image, 'sha256sum', '/opt/msvc/bin/cl.exe'], { timeoutMs: 120_000 }); const out = await run('docker', ['run', '--rm', '--network', 'none', r.image, 'sha256sum', '/opt/msvc/bin/cl.exe'], { timeoutMs: 120_000 });
return /^[0-9a-f]{64}/.exec(out.output)?.[0] ?? null; return /^[0-9a-f]{64}/.exec(out.output)?.[0] ?? null;
@ -130,7 +136,13 @@ export async function sync(ws) {
pulled = false; pulled = false;
} }
const snap = await refreshSymbols(ws); const snap = await refreshSymbols(ws);
return { head: ws.head(), fastForwarded: pulled, symbols: snap.symbols.length, note: pulled ? null : 'could not fast-forward (local commits or conflicting changes); merge origin/main yourself' }; // A newer Ghidra project on the server is only announced (it is a large download).
let ghidra = null;
try {
const g = await api(ws).get('/api/ghidra-project');
if (g.sha256 !== ws.config.ghidraPackage) ghidra = `a newer Ghidra project is available (${(g.size / 1048576).toFixed(0)} MB): td ghidra download`;
} catch {}
return { head: ws.head(), fastForwarded: pulled, symbols: snap.symbols.length, ghidra, note: pulled ? null : 'could not fast-forward (local commits or conflicting changes); merge origin/main yourself' };
} }
/** Compiles and compares locally with the same pipeline the server uses. */ /** Compiles and compares locally with the same pipeline the server uses. */
@ -139,6 +151,8 @@ export async function check(ws, files) {
const targets = files?.length ? files : ws.changedFiles(); const targets = files?.length ? files : ws.changedFiles();
const units = targets.filter(isUnit); const units = targets.filter(isUnit);
if (!units.length) throw new Error('nothing to check: pass .cpp files or modify some'); if (!units.length) throw new Error('nothing to check: pass .cpp files or modify some');
const runner = ws.config.runners?.[project.toolchain];
if (!runner || runner.type === 'remote') return remoteCheck(ws, targets);
const result = await checkUnits({ const result = await checkUnits({
repoDir: ws.dir, repoDir: ws.dir,
jobDir: path.join(ws.tdDir, 'build'), jobDir: path.join(ws.tdDir, 'build'),
@ -150,50 +164,72 @@ export async function check(ws, files) {
runners: ws.config.runners, runners: ws.config.runners,
}); });
const rec = reconcile(result, ws.symbols); const rec = reconcile(result, ws.symbols);
return summarize(result, rec, new Set(targets)); return summarizeCheck(result, rec, new Set(targets));
} }
/** `files`: the files the user asked about; annotation problems elsewhere in the repo are not theirs. */ /**
export function summarize(result, rec, files = null) { * Compiles and compares on the server (its sandboxed compiler), for contributors without a
return { * local toolchain. Sends the changed files and the commit they are based on; nothing is submitted.
annotationErrors: files ? result.annotationErrors.filter((e) => files.has(e.file)) : result.annotationErrors, */
units: result.units.map((u) => ({ async function remoteCheck(ws, targets) {
file: u.file, const files = {};
compiled: u.compiled, for (const f of new Set([...targets, ...ws.changedFiles()])) {
log: u.compiled ? undefined : u.log, const abs = path.join(ws.dir, f);
lintErrors: u.lintErrors, if (fs.existsSync(abs) && /\.(c|cc|cpp|cxx|h|hpp|inl)$/i.test(f)) files[f.replace(/\\/g, '/')] = fs.readFileSync(abs, 'utf8');
errors: u.errors, }
unannotated: u.unannotated, const r = await api(ws).post('/api/check', { baseCommit: ws.head(), files });
functions: u.functions.map((f) => ({ if (r.gate === 'rejected') {
id: f.id, const e = new Error('the static gate rejects this code:\n' + r.lintErrors.map((x) => ` ${x.file ?? ''}${x.line ? ':' + x.line : ''} ${x.message}`).join('\n'));
name: f.demangled, e.details = r.lintErrors;
addr: f.addr != null ? '0x' + hex(f.addr) : null, throw e;
status: f.status, }
score: f.score, if (r.gate === 'conflict') throw new Error(`your changes conflict with newer work on main (${r.conflicts.join(', ')}); run td sync and merge`);
size: f.size, return { ...r, compiledOn: 'server' };
diffRegions: f.diffRegions?.slice(0, 12), }
relocations: f.relocations,
})), /**
})), * The original function as text. A local Ghidra (td ghidra install/build) answers locally;
referenceConflicts: rec.conflicts, * otherwise the server's Ghidra export does.
unknownReferences: rec.unknown.map((x) => ({ symbol: x.decorated, addr: '0x' + hex(x.addr) })), */
}; export async function original(ws, query, kind) {
if (ws.config.ghidraHome) {
const { decompile, disassemble } = await import('./ghidra.js');
const s = ws.findSymbol(query);
const addr = s?.addr ?? (/^0x[0-9a-f]+$/i.test(query) ? Number.parseInt(query, 16) : null);
if (addr == null) throw new Error(`unknown symbol ${query}`);
return kind === 'asm' ? disassemble(ws, addr) : decompile(ws, addr);
}
const r = await api(ws).get(`/api/original/${encodeURIComponent(query)}`);
if (kind === 'asm') return r.asm;
return r.pseudocode ?? `// no pseudocode for ${r.symbol ?? query} yet (${r.source}); the assembly is available with td disasm`;
} }
/** Submits files for a leased task. Defaults to all changed files the task allows. */ /** Submits files for a leased task. Defaults to all changed files the task allows. */
export async function submit(ws, taskId, { files, notes, agentMeta } = {}) { /**
const ctx = await api(ws).get(`/api/tasks/${taskId}`); * Submits work. `taskIds` empty = every task you hold, as one batch (a whole unit you claimed
const candidates = files?.length ? files : ws.changedFiles().filter((f) => matchesAny(f, ctx.allowedPaths)); * with claim-batch, or several single tasks). Defaults to all changed files those tasks allow.
if (!candidates.length) throw new Error(`no changed files match the task's allowed paths (${ctx.allowedPaths.join(', ')})`); */
export async function submit(ws, taskIds = [], { files, notes, agentMeta } = {}) {
const me = await api(ws).get('/api/me');
const ids = taskIds.length ? taskIds.map(Number) : me.leases;
if (!ids.length) throw new Error('you hold no tasks; claim one (td claim) or a whole unit (td claim --unit <file>)');
const allowed = taskIds.length ? [...new Set((await Promise.all(ids.map((id) => api(ws).get(`/api/tasks/${id}`)))).flatMap((c) => c.allowedPaths))] : me.leasedAllowedPaths;
const candidates = files?.length ? files : ws.changedFiles().filter((f) => matchesAny(f, allowed));
if (!candidates.length) throw new Error(`no changed files match the allowed paths of your tasks (${allowed.join(', ')})`);
const contents = {}; const contents = {};
for (const f of candidates) contents[f.replace(/\\/g, '/')] = fs.readFileSync(path.join(ws.dir, f), 'utf8'); for (const f of candidates) contents[f.replace(/\\/g, '/')] = fs.readFileSync(path.join(ws.dir, f), 'utf8');
const errors = lintSubmission(contents, { allowedPaths: ctx.allowedPaths, limits: ws.project.lintLimits, allowedPragmas: ws.project.allowedPragmas }); const errors = lintSubmission(contents, { allowedPaths: allowed, limits: ws.project.lintLimits, allowedPragmas: ws.project.allowedPragmas });
if (errors.length) { if (errors.length) {
const e = new Error('submission fails the static gate:\n' + errors.map((x) => ` ${x.file ?? ''}${x.line ? ':' + x.line : ''} ${x.message}`).join('\n')); const e = new Error('submission fails the static gate:\n' + errors.map((x) => ` ${x.file ?? ''}${x.line ? ':' + x.line : ''} ${x.message}`).join('\n'));
e.details = errors; e.details = errors;
throw e; throw e;
} }
return api(ws).post(`/api/tasks/${taskId}/submit`, { baseCommit: ws.head(), files: contents, notes, agentMeta, envDigest: ws.config.envDigest }); return api(ws).post('/api/batches', { taskIds: ids, baseCommit: ws.head(), files: contents, notes, agentMeta, envDigest: ws.config.envDigest });
}
/** Claims every open function of a unit (source file) or an address range, as one batch. */
export function claimBatch(ws, { unit, start, end, limit } = {}) {
return api(ws).post('/api/tasks/claim-batch', { unit, start, end, limit });
} }
export function relPath(ws, f) { export function relPath(ws, f) {

93
src/client/update.js Normal file
View file

@ -0,0 +1,93 @@
// Self-update of the td binary (only when td runs as the standalone executable; a checkout or
// npm install updates through git/npm). At most once an hour td compares its own SHA-256 with
// the server's /download/SHA256SUMS; a newer build is downloaded, verified and swapped in for
// the next start. TD_NO_UPDATE=1 turns it off.
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import crypto from 'node:crypto';
const CHECK_EVERY_MS = 3600_000;
const STATE_DIR = path.join(os.homedir(), '.clank');
const STATE_FILE = path.join(STATE_DIR, 'update.json');
const ASSET = process.platform === 'win32' ? 'td-windows-x64.exe' : 'td-linux-x64';
function readState() {
try {
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
} catch {
return {};
}
}
function writeState(s) {
try {
fs.mkdirSync(STATE_DIR, { recursive: true });
fs.writeFileSync(STATE_FILE, JSON.stringify(s));
} catch {}
}
/** Remembers the server for updates of a td that is used outside a workspace. */
export function rememberServer(server) {
writeState({ ...readState(), server });
}
async function ownHash(exe, state) {
const st = fs.statSync(exe);
const key = `${st.size}:${st.mtimeMs}`;
if (state.selfKey === key && state.selfHash) return state.selfHash;
const hash = crypto.createHash('sha256').update(fs.readFileSync(exe)).digest('hex');
state.selfKey = key;
state.selfHash = hash;
return hash;
}
/**
* Checks for and installs a newer td. Returns a message when an update was installed (it takes
* effect on the next start), otherwise null. Never throws: an update must not break a command.
*/
export async function maybeUpdate({ server, force = false, log = (m) => process.stderr.write(m + '\n') } = {}) {
try {
if (process.env.TD_NO_UPDATE === '1') return null;
const sea = await import('node:sea').catch(() => null);
if (!sea?.isSea?.()) return null;
const exe = process.execPath;
// Leftover from a Windows update (a running .exe can be renamed, not replaced).
fs.rmSync(exe + '.old', { force: true });
const state = readState();
const base = (server ?? state.server)?.replace(/\/+$/, '');
if (!base) return null;
if (!force && state.checkedAt && Date.now() - state.checkedAt < CHECK_EVERY_MS) return null;
state.checkedAt = Date.now();
const res = await fetch(`${base}/download/SHA256SUMS`, { signal: AbortSignal.timeout(10_000) });
if (!res.ok) return writeState(state), null;
const want = new RegExp(`^([0-9a-f]{64})\\s+${ASSET.replace('.', '\\.')}$`, 'm').exec(await res.text())?.[1];
if (!want || want === (await ownHash(exe, state))) return writeState(state), null;
log(`td: downloading an update from ${base} ...`);
const bin = await fetch(`${base}/download/${ASSET}`, { signal: AbortSignal.timeout(600_000) });
if (!bin.ok) return writeState(state), null;
const data = Buffer.from(await bin.arrayBuffer());
if (crypto.createHash('sha256').update(data).digest('hex') !== want) {
log('td: the update failed its checksum check and was not installed');
return writeState(state), null;
}
const next = exe + '.new';
fs.writeFileSync(next, data, { mode: 0o755 });
if (process.platform === 'win32') {
fs.renameSync(exe, exe + '.old');
fs.renameSync(next, exe);
} else {
fs.renameSync(next, exe);
}
delete state.selfKey;
delete state.selfHash;
writeState(state);
return 'td was updated; the new version runs from the next command on';
} catch {
return null;
}
}

View file

@ -11,10 +11,14 @@ import { Tasks } from './tasks.js';
import { Verifier } from './verify.js'; import { Verifier } from './verify.js';
import { Merger } from './merge.js'; import { Merger } from './merge.js';
import { PeerReviews } from './peer.js'; import { PeerReviews } from './peer.js';
import { Originals } from './original.js';
import { downloadsDir, installPs1, installSh } from './install.js';
import { ServerGhidra } from './ghidra.js';
import { Router, HttpError, readJson, sendJson, RateLimiter } from './http.js'; import { Router, HttpError, readJson, sendJson, RateLimiter } from './http.js';
import { PEImage } from '../shared/pe.js'; import { PEImage } from '../shared/pe.js';
import { loadProject, loadToolchainLock } from '../shared/project.js'; import { loadProject, loadToolchainLock } from '../shared/project.js';
import { DEFAULT_LIMITS } from '../shared/lint.js'; import { DEFAULT_LIMITS, lintSubmission } from '../shared/lint.js';
import { summarizeCheck } from '../shared/check.js';
import { hex } from '../shared/annotations.js'; import { hex } from '../shared/annotations.js';
const WEB_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'web'); const WEB_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'web');
@ -30,6 +34,8 @@ export const DEFAULT_CONFIG = {
runners: { msvc2010: { type: 'docker', image: 'togetherdecomp/msvc2010:1', memory: '1g', cpus: 1 } }, runners: { msvc2010: { type: 'docker', image: 'togetherdecomp/msvc2010:1', memory: '1g', cpus: 1 } },
allowUnsandboxedRunners: false, allowUnsandboxedRunners: false,
pushRemote: null, pushRemote: null,
ghidraHome: null,
ghidraRefreshMinutes: 30,
distributeTarget: false, distributeTarget: false,
gitCloneUrl: null, gitCloneUrl: null,
gitWebUrl: null, gitWebUrl: null,
@ -66,16 +72,20 @@ export function openServer(dataDir, { log = console } = {}) {
} }
const tasks = new Tasks({ store, config, repo, project }); const tasks = new Tasks({ store, config, repo, project });
const verifier = new Verifier({ store, config, repo, project, tasks, getPe, log });
const peers = new PeerReviews({ store, project }); const peers = new PeerReviews({ store, project });
const verifier = new Verifier({ store, config, repo, project, tasks, getPe, peers, log });
const merger = new Merger({ store, config, repo, project, tasks, verifier, getPe, peers, log }); const merger = new Merger({ store, config, repo, project, tasks, verifier, getPe, peers, log });
const accounts = new Accounts(store); const accounts = new Accounts(store);
return { config, store, repo, project, tasks, verifier, merger, getPe, accounts, peers }; const originals = new Originals({ store });
const ghidra = new ServerGhidra({ config, store, originals, log });
merger.onMerged = (commit) => ghidra.scheduleRefresh(commit);
return { config, store, repo, project, tasks, verifier, merger, getPe, accounts, peers, originals, ghidra };
} }
export function createApp(ctx) { export function createApp(ctx) {
const { config, store, repo, project, tasks, merger, accounts, peers } = ctx; const { config, store, repo, project, tasks, merger, accounts, peers } = ctx;
const router = new Router(); const router = new Router();
downloadsRoot = downloadsDir(config);
const submitLimiter = new RateLimiter(config.submitsPerHour, 3600_000); const submitLimiter = new RateLimiter(config.submitsPerHour, 3600_000);
const symbolLimiter = new RateLimiter(120, 3600_000); const symbolLimiter = new RateLimiter(120, 3600_000);
const authFailLimiter = new RateLimiter(60, 600_000); const authFailLimiter = new RateLimiter(60, 600_000);
@ -203,7 +213,8 @@ export function createApp(ctx) {
router.on('GET', '/api/me', ({ req }) => { router.on('GET', '/api/me', ({ req }) => {
const user = auth(req); const user = auth(req);
return { name: user.name, role: user.role, via: user.via, leases: tasks.leasesOf(user.id) }; const leases = tasks.leasesOf(user.id);
return { name: user.name, role: user.role, via: user.via, leases, leasedAllowedPaths: leases.length ? tasks.allowedPathsForTasks(leases) : [] };
}); });
router.on('POST', '/api/me/password', async ({ req }) => { router.on('POST', '/api/me/password', async ({ req }) => {
@ -265,22 +276,85 @@ export function createApp(ctx) {
submitLimiter.check(user.id); submitLimiter.check(user.id);
const taskId = intParam(params.id); const taskId = intParam(params.id);
const body = await readJson(req); const body = await readJson(req);
const t = tasks.requireLease(user, taskId); const r = submitBatch(user, [taskId], body);
return { attemptId: r.attempts[0].attemptId, batchId: r.batchId, state: 'queued' };
});
/** Validates a submission body and queues it as one batch for the given tasks. */
const submitBatch = (user, taskIds, body) => {
if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash'); if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash');
if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) throw new HttpError(400, 'files must be an object of path -> content'); if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) throw new HttpError(400, 'files must be an object of path -> content');
const notes = body.notes ? String(body.notes).slice(0, DEFAULT_LIMITS.maxNotesBytes) : null; const r = tasks.submitBatch(user, {
const meta = body.agentMeta ? JSON.stringify(body.agentMeta).slice(0, 1000) : null; taskIds,
const id = store.tx(() => { baseCommit: body.baseCommit,
const r = store.run( files: body.files,
`INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', ?)`, notes: body.notes ? String(body.notes).slice(0, DEFAULT_LIMITS.maxNotesBytes) : null,
t.id, user.id, body.baseCommit, JSON.stringify(body.files), notes, meta, body.envDigest ? String(body.envDigest).slice(0, 200) : null, now(), agentMeta: body.agentMeta ? JSON.stringify(body.agentMeta).slice(0, 1000) : null,
); envDigest: body.envDigest ? String(body.envDigest).slice(0, 200) : null,
store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), t.id);
store.event('submit', user.id, { task: t.id, attempt: Number(r.lastInsertRowid) });
return Number(r.lastInsertRowid);
}); });
ctx.verifier.pump(); ctx.verifier.pump();
return { attemptId: id, state: 'queued' }; return r;
};
// Bulk work: claim every open function of a unit (source file) or an address range, submit the
// file once. Verified with one compile; merged as a whole.
router.on('POST', '/api/tasks/claim-batch', async ({ req }) => {
const user = auth(req);
const body = await readJson(req, 4096);
const hexOrNull = (x) => (x == null ? null : typeof x === 'string' ? Number.parseInt(x, 16) : Number(x));
const ids = tasks.claimBatch(user, { unit: body.unit ? String(body.unit) : null, start: hexOrNull(body.start), end: hexOrNull(body.end), limit: body.limit ? Number(body.limit) : 50 });
const v = store.currentVersion();
const ph = ids.map(() => '?').join(',');
const rows = store.all(
`SELECT t.id AS task, s.id AS symbol, s.name, s.unit, a.addr, a.size, t.state, t.best_score FROM tasks t JOIN symbols s ON s.id = t.symbol_id
LEFT JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? WHERE t.id IN (${ph}) ORDER BY a.addr`,
v.id, ...ids,
);
return {
tasks: rows.map((r) => ({ ...r, addr: r.addr != null ? '0x' + hex(r.addr) : null, annotation: `// FUNCTION: ${r.symbol}` })),
allowedPaths: tasks.allowedPathsForTasks(ids),
baseCommit: repo.head(),
note: 'Write or fix every function, check, then submit them together (td submit --all). Context of one task: GET /api/tasks/<id>.',
};
});
router.on('POST', '/api/batches', async ({ req }) => {
const user = auth(req);
submitLimiter.check(user.id);
const body = await readJson(req, 8 * 1024 * 1024);
const r = submitBatch(user, (body.taskIds ?? []).map((x) => intParam(x)), body);
return { ...r, state: 'queued' };
});
router.on('GET', '/api/batches/:id', ({ req, params }) => {
auth(req);
const b = store.get('SELECT b.*, u.name AS author FROM batches b JOIN users u ON u.id = b.user_id WHERE b.id = ?', intParam(params.id));
if (!b) throw new HttpError(404, 'no such batch');
const result = b.result_json ? JSON.parse(b.result_json) : {};
const attempts = store.all(
`SELECT a.id, a.task_id, t.symbol_id, s.name, a.state, a.score, a.result_json FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id WHERE a.batch_id = ? ORDER BY a.id`,
b.id,
);
return {
id: b.id,
author: b.author,
state: b.state,
baseCommit: b.base_commit,
mergedCommit: b.merged_commit,
returned: result.returned ?? null,
mergeFailure: result.mergeFailure ?? null,
changedHeaders: result.changedHeaders ?? [],
changedOutside: result.outside ?? [],
functions: attempts.map((a) => {
const fb = a.result_json ? JSON.parse(a.result_json).feedback : null;
return { attemptId: a.id, taskId: a.task_id, symbol: a.symbol_id, name: a.name, state: a.state, score: a.score, summary: fb?.summary ?? null, status: fb?.target?.status ?? null };
}),
};
});
router.on('POST', '/api/tasks/heartbeat-all', ({ req }) => {
const user = auth(req);
return tasks.heartbeatAll(user);
}); });
router.on('GET', '/api/attempts/:id', ({ req, params }) => { router.on('GET', '/api/attempts/:id', ({ req, params }) => {
@ -288,6 +362,8 @@ export function createApp(ctx) {
const a = store.get('SELECT a.*, u.name AS author FROM attempts a JOIN users u ON u.id = a.user_id WHERE a.id = ?', intParam(params.id)); const a = store.get('SELECT a.*, u.name AS author FROM attempts a JOIN users u ON u.id = a.user_id WHERE a.id = ?', intParam(params.id));
if (!a) throw new HttpError(404, 'no such attempt'); if (!a) throw new HttpError(404, 'no such attempt');
const result = a.result_json ? JSON.parse(a.result_json) : null; const result = a.result_json ? JSON.parse(a.result_json) : null;
const batch = a.batch_id ? store.get('SELECT result_json FROM batches WHERE id = ?', a.batch_id) : null;
const batchResult = batch?.result_json ? JSON.parse(batch.result_json) : {};
const isReviewer = hasRole(user, 'reviewer'); const isReviewer = hasRole(user, 'reviewer');
return { return {
id: a.id, id: a.id,
@ -300,10 +376,11 @@ export function createApp(ctx) {
finishedAt: a.finished_at, finishedAt: a.finished_at,
notes: a.notes, notes: a.notes,
feedback: result?.feedback ?? null, feedback: result?.feedback ?? null,
mergeFailure: result?.mergeFailure ?? null, batchId: a.batch_id,
changedHeaders: result?.changedHeaders ?? [], mergeFailure: batchResult.mergeFailure ?? result?.mergeFailure ?? null,
files: JSON.parse(a.files_json), changedHeaders: batchResult.changedHeaders ?? result?.changedHeaders ?? [],
mergedFiles: isReviewer ? result?.mergedFiles ?? null : undefined, files: store.attemptFiles(a),
mergedFiles: isReviewer ? batchResult.mergedFiles ?? result?.mergedFiles ?? null : undefined,
reviews: store.all('SELECT r.action, r.comment, r.created_at, u.name AS reviewer FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ?', a.id), reviews: store.all('SELECT r.action, r.comment, r.created_at, u.name AS reviewer FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ?', a.id),
peerReviews: peers.verdicts(a.id), peerReviews: peers.verdicts(a.id),
peerRequired: peers.required, peerRequired: peers.required,
@ -334,6 +411,67 @@ export function createApp(ctx) {
return { id: r.symbol.id, kind: r.symbol.kind, created: r.created, annotation: `// ${{ function: 'FUNCTION', global: 'GLOBAL', vtable: 'VTABLE', string: 'STRING' }[kind]}: ${r.symbol.id}` }; return { id: r.symbol.id, kind: r.symbol.kind, created: r.created, annotation: `// ${{ function: 'FUNCTION', global: 'GLOBAL', vtable: 'VTABLE', string: 'STRING' }[kind]}: ${r.symbol.id}` };
}); });
// --- the original function (assembly, pseudocode) for contributors without a local Ghidra.
// Derived from the exe, so approved accounts only, like the exe itself.
router.on('GET', '/api/original/:symbol', async ({ req, params }) => {
auth(req);
return ctx.originals.get(params.symbol);
});
// --- the server's Ghidra project of the current version (td ghidra download). Derived from the
// exe, so approved accounts only.
router.on('GET', '/api/ghidra-project', ({ req }) => {
auth(req);
const info = ctx.ghidra.info();
if (!info) throw new HttpError(404, 'no Ghidra project has been built on this server yet');
return { ...info, download: `${config.publicUrl}/api/ghidra-project/download` };
});
router.on('GET', '/api/ghidra-project/download', ({ req, res }) => {
auth(req);
const file = ctx.ghidra.packagePath();
if (!file || !fs.existsSync(file)) throw new HttpError(404, 'no Ghidra project has been built on this server yet');
res.writeHead(200, { 'content-type': 'application/gzip', 'content-length': fs.statSync(file).size, 'content-disposition': 'attachment; filename="ghidra-project.tar.gz"' });
fs.createReadStream(file).pipe(res);
return undefined;
});
// --- remote check: compile and compare in the server's sandbox without submitting
// (contributors without a local MSVC toolchain). Same static gate as a submission.
const checkLimiter = new RateLimiter(config.checksPerHour ?? 60, 3600_000);
let checksRunning = 0;
const checkWaiters = [];
const withCheckSlot = async (fn) => {
const max = config.remoteCheckConcurrency ?? 2;
if (checksRunning >= max) await new Promise((r) => checkWaiters.push(r));
checksRunning++;
try {
return await fn();
} finally {
checksRunning--;
checkWaiters.shift()?.();
}
};
router.on('POST', '/api/check', async ({ req }) => {
const user = auth(req);
checkLimiter.check(user.id);
const body = await readJson(req);
if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash');
if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) throw new HttpError(400, 'files must be an object of path -> content');
const allowed = [...project.sourceDirs.map((d) => `${d}/**`)];
const lintErrors = lintSubmission(body.files, { allowedPaths: allowed, limits: project.lintLimits, allowedPragmas: project.allowedPragmas });
if (lintErrors.length) return { gate: 'rejected', lintErrors };
if (!repo.isAncestor(body.baseCommit)) throw new HttpError(409, 'your base commit is not part of main; run td sync');
const merged = repo.mergeOntoHead(body.baseCommit, body.files);
if (merged.conflicts.length) return { gate: 'conflict', conflicts: merged.conflicts };
return withCheckSlot(async () => {
const { check, rec } = await ctx.verifier.check(merged.files, `check-${user.id}-${Date.now()}`);
return { gate: 'ok', head: repo.head(), ...summarizeCheck(check, rec, new Set(Object.keys(merged.files))) };
});
});
// --- peer review (agents, API token or session) // --- peer review (agents, API token or session)
router.on('GET', '/api/peer-reviews', ({ req, query }) => { router.on('GET', '/api/peer-reviews', ({ req, query }) => {
@ -369,7 +507,7 @@ export function createApp(ctx) {
router.on('GET', '/api/review/queue', ({ req }) => { router.on('GET', '/api/review/queue', ({ req }) => {
const user = auth(req, { role: 'reviewer', session: true }); const user = auth(req, { role: 'reviewer', session: true });
return store.all( return store.all(
`SELECT a.id, a.task_id, a.score, a.created_at, a.finished_at, u.name AS author, t.symbol_id, s.name, `SELECT a.id, a.batch_id, a.task_id, a.score, a.created_at, a.finished_at, u.name AS author, t.symbol_id, s.name,
(SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.action = 'approve') AS approvals, (SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.action = 'approve') AS approvals,
(SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.reviewer_id = ?) AS reviewed_by_me (SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.reviewer_id = ?) AS reviewed_by_me
FROM attempts a JOIN users u ON u.id = a.user_id JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id FROM attempts a JOIN users u ON u.id = a.user_id JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
@ -386,6 +524,15 @@ export function createApp(ctx) {
return r; return r;
}); });
// One click for a whole batch: every function of it that waits for a human.
router.on('POST', '/api/batches/:id/review', async ({ req, params }) => {
const user = auth(req, { role: 'reviewer', session: true });
const body = await readJson(req, 16 * 1024);
const r = merger.reviewBatch(user, intParam(params.id), String(body.action ?? ''), body.comment ? String(body.comment) : null);
setImmediate(() => merger.pump());
return r;
});
// --- maintainers and admins (browser session only) // --- maintainers and admins (browser session only)
router.on('POST', '/api/admin/tasks/:id', async ({ req, params }) => { router.on('POST', '/api/admin/tasks/:id', async ({ req, params }) => {
@ -425,6 +572,13 @@ export function createApp(ctx) {
if (url.pathname.startsWith('/repo.git/')) { if (url.pathname.startsWith('/repo.git/')) {
return serveGit(req, res, { projectRoot: config.dataDir, subPath: '/repo' + url.pathname.slice('/repo.git'.length), query: url.searchParams }); return serveGit(req, res, { projectRoot: config.dataDir, subPath: '/repo' + url.pathname.slice('/repo.git'.length), query: url.searchParams });
} }
if (req.method === 'GET' && (url.pathname === '/install.ps1' || url.pathname === '/install.sh')) {
const script = url.pathname.endsWith('.ps1') ? installPs1(config, project) : installSh(config, project);
if (!script) throw new HttpError(503, 'the td binaries are not published on this server yet');
res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-cache', 'x-content-type-options': 'nosniff' });
return res.end(script);
}
if (req.method === 'GET' && url.pathname.startsWith('/download/')) return serveDownload(url.pathname.slice(10), res);
if (req.method === 'GET' && !url.pathname.startsWith('/api/')) return serveStatic(url.pathname, res); if (req.method === 'GET' && !url.pathname.startsWith('/api/')) return serveStatic(url.pathname, res);
const m = router.match(req.method, url.pathname); const m = router.match(req.method, url.pathname);
if (!m) throw new HttpError(404, 'not found'); if (!m) throw new HttpError(404, 'not found');
@ -442,6 +596,19 @@ export function createApp(ctx) {
return http.createServer(handler); return http.createServer(handler);
} }
// Public downloads of the td binaries (tools/build-td output copied to <dataDir>/downloads).
const DOWNLOADS = new Set(['td-windows-x64.exe', 'td-linux-x64', 'SHA256SUMS']);
let downloadsRoot = null;
function serveDownload(name, res) {
const file = DOWNLOADS.has(name) && downloadsRoot ? path.join(downloadsRoot, name) : null;
if (!file || !fs.existsSync(file)) {
res.writeHead(404, { 'content-type': 'text/plain' });
return res.end('not found');
}
res.writeHead(200, { 'content-type': name === 'SHA256SUMS' ? 'text/plain' : 'application/octet-stream', 'content-length': fs.statSync(file).size, 'content-disposition': `attachment; filename="${name}"`, 'x-content-type-options': 'nosniff' });
fs.createReadStream(file).pipe(res);
}
const STATIC = { '/': 'index.html', '/review': 'index.html', '/app.js': 'app.js', '/style.css': 'style.css', '/favicon.svg': 'favicon.svg' }; const STATIC = { '/': 'index.html', '/review': 'index.html', '/app.js': 'app.js', '/style.css': 'style.css', '/favicon.svg': 'favicon.svg' };
const TYPES = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css; charset=utf-8', '.svg': 'image/svg+xml' }; const TYPES = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css; charset=utf-8', '.svg': 'image/svg+xml' };

32
src/server/batches.js Normal file
View file

@ -0,0 +1,32 @@
// A batch merges as a whole. When one of its attempts fails (verification, peer review, human
// review or the merge re-check), the whole batch goes back to its author: every other attempt
// that was still on its way is marked returned, and its task becomes claimable again (reserved
// for the author for a while). Peer verdicts stay attached to their attempts, so a resubmission
// whose compiled code is unchanged for a function carries them over (Verifier.carryOver).
import { now } from './db.js';
export function returnBatch(store, batchId, reason) {
if (!batchId) return;
const b = store.get('SELECT * FROM batches WHERE id = ?', batchId);
if (!b || ['merged', 'returned', 'superseded'].includes(b.state)) return;
const pending = store.all(`SELECT * FROM attempts WHERE batch_id = ? AND state IN ('queued', 'verifying', 'in_review', 'peer_review', 'approved')`, batchId);
for (const a of pending) {
store.run(`UPDATE attempts SET state = 'returned', result_json = json_set(COALESCE(result_json, '{}'), '$.returned', ?) WHERE id = ?`, reason, a.id);
store.run('DELETE FROM peer_leases WHERE attempt_id = ?', a.id);
store.run(
`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ? AND state IN ('verifying', 'in_review', 'peer_review', 'approved', 'merging')`,
a.user_id, now() + 6 * 3600_000, now(), a.task_id,
);
}
store.run(`UPDATE batches SET state = 'returned', result_json = json_set(COALESCE(result_json, '{}'), '$.returned', ?) WHERE id = ?`, reason, batchId);
store.event('batch_returned', b.user_id, { batch: batchId, reason });
}
/** Attempts of a batch with their task's symbol. */
export function batchAttempts(store, batchId) {
return store.all(
`SELECT a.*, t.symbol_id FROM attempts a JOIN tasks t ON t.id = a.task_id WHERE a.batch_id = ? ORDER BY a.id`,
batchId,
);
}

View file

@ -37,6 +37,8 @@ Usage: td-server <command> [options] (all commands accept --data <dir>, defaul
symbols load <symbols.json> import symbols with fixed IDs and per-version addresses symbols load <symbols.json> import symbols with fixed IDs and per-version addresses
symbols units <csv> assign functions to translation units (csv: start,end,file) symbols units <csv> assign functions to translation units (csv: start,end,file)
symbols library <id...> mark symbols as library code (no tasks) symbols library <id...> mark symbols as library code (no tasks)
originals import <jsonl> [--version V] assembly + pseudocode of the original (ExportOriginals.java)
ghidra build | refresh the server's Ghidra project (download, pseudocode); refresh also runs after merges
rebuild compile all of main and refresh match status rebuild compile all of main and refresh match status
tasks generate create tasks for functions that have none tasks generate create tasks for functions that have none
`; `;
@ -239,6 +241,25 @@ async function main() {
} }
} }
if (cmd === 'ghidra' && (sub === 'build' || sub === 'refresh')) {
// The server's Ghidra project (needs ghidraHome in server.json and a JDK 21).
const ctx = openServer(dataDir);
if (!ctx.ghidra.enabled) throw new Error('set ghidraHome in server.json to a Ghidra 11.2.1 installation');
ctx.store.setMeta('ghidra.commit', ctx.repo.head());
const info = sub === 'build' ? await ctx.ghidra.build() : await ctx.ghidra.refresh();
console.log(JSON.stringify(info, null, 2));
return;
}
if (cmd === 'originals' && sub === 'import') {
// ghidra_scripts/ExportOriginals.java output, for the current version unless --version is given.
const ctx = openServer(dataDir);
const v = values.version ?? ctx.store.currentVersion()?.id;
const n = await ctx.originals.import(v, rest[0]);
console.log(`imported ${n} functions for ${v}`);
return;
}
if (cmd === 'tasks' && sub === 'generate') { if (cmd === 'tasks' && sub === 'generate') {
const ctx = openServer(dataDir); const ctx = openServer(dataDir);
console.log(`created ${ctx.tasks.generate()} tasks`); console.log(`created ${ctx.tasks.generate()} tasks`);

View file

@ -142,6 +142,25 @@ CREATE TABLE IF NOT EXISTS func_status (
confirmed INTEGER NOT NULL DEFAULT 0 confirmed INTEGER NOT NULL DEFAULT 0
); );
-- A batch is one submission: the files once, and one attempt per task (function) it covers.
-- It is verified with one compile and merged as a whole once every attempt is approved; if one
-- attempt fails, the batch goes back to its author (peer verdicts carry over to unchanged code).
CREATE TABLE IF NOT EXISTS batches (
id INTEGER PRIMARY KEY,
user_id INTEGER NOT NULL REFERENCES users(id),
base_commit TEXT NOT NULL,
files_json TEXT NOT NULL,
notes TEXT,
agent_meta TEXT,
env_digest TEXT,
state TEXT NOT NULL, -- queued, verifying, open, merging, merged, returned, superseded
result_json TEXT,
merged_commit TEXT,
created_at INTEGER NOT NULL,
finished_at INTEGER
);
CREATE INDEX IF NOT EXISTS batches_state ON batches(state);
-- Peer review of attempts that are not byte-identical: other agents read the original's -- Peer review of attempts that are not byte-identical: other agents read the original's
-- assembly and pseudocode and the submitted code, and judge whether the logic is the same. -- assembly and pseudocode and the submitted code, and judge whether the logic is the same.
CREATE TABLE IF NOT EXISTS peer_reviews ( CREATE TABLE IF NOT EXISTS peer_reviews (
@ -171,6 +190,15 @@ CREATE TABLE IF NOT EXISTS unit_deps (
); );
CREATE INDEX IF NOT EXISTS unit_deps_header ON unit_deps(header); CREATE INDEX IF NOT EXISTS unit_deps_header ON unit_deps(header);
-- The original functions as text (Ghidra export): assembly with symbol names, and pseudocode.
CREATE TABLE IF NOT EXISTS original_code (
version_id TEXT NOT NULL,
addr INTEGER NOT NULL,
asm TEXT,
c TEXT,
PRIMARY KEY (version_id, addr)
);
CREATE TABLE IF NOT EXISTS events ( CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
at INTEGER NOT NULL, at INTEGER NOT NULL,
@ -189,6 +217,19 @@ export class Store {
this.migrateUsers(); this.migrateUsers();
this.db.exec(SCHEMA); this.db.exec(SCHEMA);
this.addColumn('func_status', 'confirmed', 'INTEGER NOT NULL DEFAULT 0'); this.addColumn('func_status', 'confirmed', 'INTEGER NOT NULL DEFAULT 0');
// Fingerprint of main's compiled code per function (submissions may only change their own tasks).
this.addColumn('func_status', 'code_hash', 'TEXT');
this.addColumn('attempts', 'batch_id', 'INTEGER REFERENCES batches(id)');
this.db.exec('CREATE INDEX IF NOT EXISTS attempts_batch ON attempts(batch_id)');
}
/** Submitted files of an attempt (kept once per batch). */
attemptFiles(attempt) {
if (attempt.batch_id) {
const b = this.get('SELECT files_json FROM batches WHERE id = ?', attempt.batch_id);
if (b) return JSON.parse(b.files_json);
}
return JSON.parse(attempt.files_json || '{}');
} }
/** Adds a column to a table created by an older schema. */ /** Adds a column to a table created by an older schema. */

137
src/server/ghidra.js Normal file
View file

@ -0,0 +1,137 @@
// The server's own Ghidra project of the current target version, kept up to date with the symbol
// database. Contributors download it (td ghidra download) instead of analyzing the exe
// themselves, and the coordinator exports assembly and pseudocode from it for /api/original.
//
// build import + auto-analysis + symbol names (once per version, ~15 min)
// refresh re-apply the names, re-export the originals, pack the project (a few minutes);
// runs on its own after merges, at most every `ghidraRefreshMinutes` (default 30)
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { run } from '../shared/build.js';
import { hex } from '../shared/annotations.js';
const SCRIPTS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts');
const PROJECT = 'td';
const PROGRAM = 'target.exe';
export class ServerGhidra {
constructor({ config, store, originals, log = console }) {
Object.assign(this, { config, store, originals, log });
this.dir = path.join(config.dataDir, 'ghidra');
this.projectDir = path.join(this.dir, 'project');
this.busy = null;
this.timer = null;
}
get enabled() {
return !!this.config.ghidraHome && fs.existsSync(path.join(this.config.ghidraHome, 'support', 'analyzeHeadless'));
}
/** Metadata of the packed project, or null. */
info() {
try {
return JSON.parse(fs.readFileSync(path.join(this.dir, 'package.json'), 'utf8'));
} catch {
return null;
}
}
packagePath() {
const i = this.info();
return i ? path.join(this.dir, i.file) : null;
}
async headless(args, timeoutMs) {
fs.mkdirSync(this.projectDir, { recursive: true });
const exe = path.join(this.config.ghidraHome, 'support', 'analyzeHeadless');
// Ghidra 11.2 is made for JDK 21; the system default may be newer.
const jdk21 = '/usr/lib/jvm/java-21-openjdk-amd64';
const javaHome = this.config.ghidraJavaHome ?? (fs.existsSync(jdk21) ? jdk21 : process.env.JAVA_HOME);
const env = { ...process.env, MAXMEM: this.config.ghidraMaxMem ?? '6G', ...(javaHome ? { JAVA_HOME: javaHome, PATH: `${javaHome}/bin:${process.env.PATH}` } : {}) };
const r = await run(exe, [this.projectDir, PROJECT, ...args, '-scriptPath', SCRIPTS_DIR], { timeoutMs, env });
if (r.code !== 0) throw new Error(`Ghidra failed (exit ${r.code}): ${r.output.slice(-2000)}`);
return r.output;
}
writeSymbols(versionId) {
const file = path.join(this.dir, 'symbols.tsv');
const lines = [];
for (const s of this.store.symbolMap(versionId).values()) {
if (s.addr == null) continue;
lines.push([hex(s.addr), s.kind, s.id, s.name ?? '', s.decorated ?? '', s.library ? 1 : 0].join('\t'));
}
fs.writeFileSync(file, lines.join('\n') + '\n');
return file;
}
/** Creates the project from the current version's exe (replacing an old one). */
async build() {
const v = this.store.currentVersion();
fs.rmSync(this.projectDir, { recursive: true, force: true });
fs.mkdirSync(this.dir, { recursive: true });
const exe = path.join(this.dir, PROGRAM);
fs.copyFileSync(v.file, exe);
this.log.info(`ghidra: analyzing ${v.id} (this takes a while)`);
await this.headless(['-import', exe, '-overwrite', '-postScript', 'ApplySymbols.java', this.writeSymbols(v.id)], 4 * 3600_000);
fs.rmSync(exe, { force: true });
this.store.setMeta('ghidra.version', v.id);
await this.refresh({ applySymbols: false });
}
/** Re-applies the symbol names, re-exports the originals and packs the project. */
async refresh({ applySymbols = true } = {}) {
const v = this.store.currentVersion();
if (this.store.getMeta('ghidra.version') !== v.id) return this.build();
const t0 = Date.now();
if (applySymbols) await this.headless(['-process', PROGRAM, '-noanalysis', '-postScript', 'ApplySymbols.java', this.writeSymbols(v.id)], 3600_000);
const jsonl = path.join(this.dir, 'originals.jsonl');
await this.headless(['-process', PROGRAM, '-noanalysis', '-readOnly', '-postScript', 'ExportOriginals.java', jsonl, '30'], 6 * 3600_000);
const n = await this.originals.import(v.id, jsonl);
fs.rmSync(jsonl, { force: true });
// Pack the project for download (tar.gz: Windows' tar.exe and Linux tar both unpack it).
const head = this.store.getMeta('ghidra.commit') ?? null;
const name = `ghidra-${v.id}-${Date.now()}.tar.gz`;
const tmp = path.join(this.dir, name + '.part');
const r = await run('tar', ['-czf', tmp, '-C', this.projectDir, `${PROJECT}.gpr`, `${PROJECT}.rep`], { timeoutMs: 1800_000 });
if (r.code !== 0) throw new Error(`packing the Ghidra project failed: ${r.output}`);
const old = this.info();
fs.renameSync(tmp, path.join(this.dir, name));
const sha256 = await sha256File(path.join(this.dir, name));
const info = { version: v.id, file: name, sha256, size: fs.statSync(path.join(this.dir, name)).size, builtAt: Date.now(), commit: head, functions: n, ghidra: this.config.ghidraVersion ?? '11.2.1' };
fs.writeFileSync(path.join(this.dir, 'package.json'), JSON.stringify(info, null, 2));
if (old?.file && old.file !== name) fs.rmSync(path.join(this.dir, old.file), { force: true });
this.store.setMeta('ghidra.refreshedAt', Date.now());
this.log.info(`ghidra: refreshed ${v.id} in ${((Date.now() - t0) / 1000).toFixed(0)}s (${n} functions exported, package ${(info.size / 1048576).toFixed(0)} MB)`);
return info;
}
/** Called after merges: refresh soon, but at most every ghidraRefreshMinutes. */
scheduleRefresh(commit) {
if (!this.enabled) return;
if (commit) this.store.setMeta('ghidra.commit', commit);
if (this.timer || this.busy) return;
const every = (this.config.ghidraRefreshMinutes ?? 30) * 60_000;
const last = Number(this.store.getMeta('ghidra.refreshedAt') ?? 0);
const wait = Math.max(0, last + every - Date.now());
this.timer = setTimeout(() => {
this.timer = null;
this.busy = this.refresh()
.catch((e) => this.log.error(`ghidra refresh failed: ${e.message}`))
.finally(() => {
this.busy = null;
});
}, wait);
this.timer.unref?.();
}
}
function sha256File(file) {
return new Promise((resolve, reject) => {
const h = crypto.createHash('sha256');
fs.createReadStream(file).on('data', (d) => h.update(d)).on('end', () => resolve(h.digest('hex'))).on('error', reject);
});
}

111
src/server/install.js Normal file
View file

@ -0,0 +1,111 @@
// One-line installers. The scripts are generated per request with this server's URL, the project
// name and the SHA-256 of the td binaries in <dataDir>/downloads (built by tools/build-td).
//
// Windows: $env:TD_TOKEN='td_...'; irm https://<host>/install.ps1 | iex
// Linux: curl -fsSL https://<host>/install.sh | TD_TOKEN=td_... sh
import fs from 'node:fs';
import path from 'node:path';
export function downloadsDir(config) {
return path.join(config.dataDir, 'downloads');
}
function checksums(config) {
const file = path.join(downloadsDir(config), 'SHA256SUMS');
const out = {};
if (!fs.existsSync(file)) return out;
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
const m = /^([0-9a-f]{64})\s+(\S+)$/.exec(line.trim());
if (m) out[m[2]] = m[1];
}
return out;
}
const slug = (name) => String(name ?? 'project').replace(/[^A-Za-z0-9_.-]/g, '-');
export function installPs1(config, project) {
const sha = checksums(config)['td-windows-x64.exe'];
if (!sha) return null;
return `# Clank Together installer for Windows. Usage (PowerShell, one line):
# $env:TD_TOKEN='td_...'; irm ${config.publicUrl}/install.ps1 | iex
# Installs td (a single program, no Node.js needed), sets up the ${project.name} workspace and
# connects your AI. Optional: $env:TD_DIR (workspace folder), $env:TD_TOOLCHAIN (local compiler root).
$ErrorActionPreference = 'Stop'
$ProgressPreference = 'SilentlyContinue'
$server = '${config.publicUrl}'
$token = $env:TD_TOKEN
if (-not $token) { $token = Read-Host 'API token (log in on $server and create one under Account)' }
$dir = if ($env:TD_DIR) { $env:TD_DIR } else { Join-Path $HOME 'clank\\${slug(project.name)}' }
$bin = Join-Path $env:LOCALAPPDATA 'ClankTogether\\bin'
New-Item -ItemType Directory -Force $bin | Out-Null
if (-not (Get-Command git -ErrorAction SilentlyContinue)) {
Write-Host 'Installing git (winget)...'
winget install --id Git.Git -e --silent --accept-package-agreements --accept-source-agreements | Out-Null
$env:Path += ';' + (Join-Path $env:ProgramFiles 'Git\\cmd')
if (-not (Get-Command git -ErrorAction SilentlyContinue)) { throw 'git is required: install it from https://git-scm.com and run this line again' }
}
Write-Host 'Downloading td...'
$exe = Join-Path $bin 'td.exe'
Invoke-WebRequest "$server/download/td-windows-x64.exe" -OutFile "$exe.part" -UseBasicParsing
if ((Get-FileHash "$exe.part" -Algorithm SHA256).Hash.ToLower() -ne '${sha}') { Remove-Item "$exe.part"; throw 'td download failed its checksum check' }
Move-Item -Force "$exe.part" $exe
$userPath = [Environment]::GetEnvironmentVariable('Path', 'User')
if (-not $userPath) { $userPath = '' }
if (($userPath -split ';') -notcontains $bin) { [Environment]::SetEnvironmentVariable('Path', ($userPath.TrimEnd(';') + ';' + $bin), 'User') }
if (($env:Path -split ';') -notcontains $bin) { $env:Path += ';' + $bin }
$setup = @('setup', $server, $dir, '--token', $token)
if ($env:TD_TOOLCHAIN) { $setup += @('--toolchain', $env:TD_TOOLCHAIN) }
& $exe @setup
if ($LASTEXITCODE -ne 0) { throw 'td setup failed (see above)' }
& $exe connect --workspace $dir
Write-Host ''
Write-Host "Done. Workspace: $dir (td is on your PATH in new terminals)"
`;
}
export function installSh(config, project) {
const sha = checksums(config)['td-linux-x64'];
if (!sha) return null;
return `#!/bin/sh
# Clank Together installer for Linux. Usage (one line):
# curl -fsSL ${config.publicUrl}/install.sh | TD_TOKEN=td_... sh
# Installs td (a single program, no Node.js needed), sets up the ${project.name} workspace and
# connects your AI. Optional: TD_DIR (workspace folder), TD_TOOLCHAIN (local compiler root).
set -eu
SERVER='${config.publicUrl}'
case "$(uname -s)-$(uname -m)" in
Linux-x86_64) ;;
*) echo "td is available for Linux x86_64 and Windows x64; this is $(uname -s) $(uname -m)" >&2; exit 1 ;;
esac
TOKEN="\${TD_TOKEN:-}"
if [ -z "$TOKEN" ]; then printf 'API token (log in on %s and create one under Account): ' "$SERVER"; read -r TOKEN < /dev/tty; fi
DIR="\${TD_DIR:-$HOME/clank/${slug(project.name)}}"
BIN="$HOME/.local/bin"
mkdir -p "$BIN"
command -v git >/dev/null 2>&1 || { echo 'git is required: install it (e.g. sudo apt install git) and run this line again' >&2; exit 1; }
echo 'Downloading td...'
curl -fsSL "$SERVER/download/td-linux-x64" -o "$BIN/td.part"
echo "${sha} $BIN/td.part" | sha256sum -c - >/dev/null || { rm -f "$BIN/td.part"; echo 'td download failed its checksum check' >&2; exit 1; }
mv "$BIN/td.part" "$BIN/td"
chmod +x "$BIN/td"
case ":$PATH:" in
*":$BIN:"*) ;;
*) echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$HOME/.profile"; PATH="$BIN:$PATH" ;;
esac
if [ -n "\${TD_TOOLCHAIN:-}" ]; then
"$BIN/td" setup "$SERVER" "$DIR" --token "$TOKEN" --toolchain "$TD_TOOLCHAIN"
else
"$BIN/td" setup "$SERVER" "$DIR" --token "$TOKEN"
fi
"$BIN/td" connect --workspace "$DIR"
echo
echo "Done. Workspace: $DIR (td is in ~/.local/bin)"
`;
}

View file

@ -4,6 +4,7 @@ import { HttpError } from './http.js';
import { now } from './db.js'; import { now } from './db.js';
import { hasRole } from './accounts.js'; import { hasRole } from './accounts.js';
import { STATUS } from '../shared/compare.js'; import { STATUS } from '../shared/compare.js';
import { returnBatch, batchAttempts } from './batches.js';
export class Merger { export class Merger {
constructor({ store, config, repo, project, tasks, verifier, getPe, peers, log = console }) { constructor({ store, config, repo, project, tasks, verifier, getPe, peers, log = console }) {
@ -12,6 +13,7 @@ export class Merger {
} }
start() { start() {
this.store.run(`UPDATE batches SET state = 'open' WHERE state = 'merging'`);
this.store.run(`UPDATE tasks SET state = 'approved' WHERE state = 'merging'`); this.store.run(`UPDATE tasks SET state = 'approved' WHERE state = 'merging'`);
this.timer = setInterval(() => this.pump(), 3000); this.timer = setInterval(() => this.pump(), 3000);
} }
@ -21,15 +23,32 @@ export class Merger {
} }
requiredApprovals(attempt) { requiredApprovals(attempt) {
const result = attempt.result_json ? JSON.parse(attempt.result_json) : {}; const batch = attempt.batch_id ? this.store.get('SELECT result_json FROM batches WHERE id = ?', attempt.batch_id) : null;
const result = batch?.result_json ? JSON.parse(batch.result_json) : {};
const p = this.project.matchPolicy; const p = this.project.matchPolicy;
return result.changedHeaders?.length ? p.approvalsWithHeaderChanges : p.approvals; return result.changedHeaders?.length ? p.approvalsWithHeaderChanges : p.approvals;
} }
/** A human decision on one attempt (peer-confirmed, not byte-identical). */
review(reviewer, attemptId, action, comment) { review(reviewer, attemptId, action, comment) {
if (!hasRole(reviewer, 'reviewer')) throw new HttpError(403, 'only reviewers can review');
if (!['approve', 'request_changes', 'reject'].includes(action)) throw new HttpError(400, 'invalid action');
return this.store.tx(() => this.reviewOne(reviewer, attemptId, action, comment));
}
/** The same decision for every attempt of a batch that waits for human review. */
reviewBatch(reviewer, batchId, action, comment) {
if (!hasRole(reviewer, 'reviewer')) throw new HttpError(403, 'only reviewers can review'); if (!hasRole(reviewer, 'reviewer')) throw new HttpError(403, 'only reviewers can review');
if (!['approve', 'request_changes', 'reject'].includes(action)) throw new HttpError(400, 'invalid action'); if (!['approve', 'request_changes', 'reject'].includes(action)) throw new HttpError(400, 'invalid action');
return this.store.tx(() => { return this.store.tx(() => {
const waiting = this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'in_review'`, batchId);
if (!waiting.length) throw new HttpError(409, 'nothing in this batch waits for human review');
const results = waiting.map((w) => ({ attemptId: w.id, ...this.reviewOne(reviewer, w.id, action, comment) }));
return { batchId, results };
});
}
reviewOne(reviewer, attemptId, action, comment) {
const attempt = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId); const attempt = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId);
if (!attempt) throw new HttpError(404, 'no such attempt'); if (!attempt) throw new HttpError(404, 'no such attempt');
if (attempt.state !== 'in_review') throw new HttpError(409, `attempt is ${attempt.state}`); if (attempt.state !== 'in_review') throw new HttpError(409, `attempt is ${attempt.state}`);
@ -52,60 +71,96 @@ export class Merger {
const newState = action === 'reject' ? 'rejected' : 'needs_cleanup'; const newState = action === 'reject' ? 'rejected' : 'needs_cleanup';
this.store.run('UPDATE attempts SET state = ? WHERE id = ?', newState, attemptId); this.store.run('UPDATE attempts SET state = ? WHERE id = ?', newState, attemptId);
this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), attempt.task_id); this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), attempt.task_id);
returnBatch(this.store, attempt.batch_id, `a reviewer asked for changes to ${attempt.task_id}`);
return { state: newState }; return { state: newState };
});
} }
/** Merges the oldest open batch whose attempts are all approved. */
async pump() { async pump() {
if (this.busy) return; if (this.busy) return;
const task = this.store.get(`SELECT * FROM tasks WHERE state = 'approved' ORDER BY updated_at LIMIT 1`); const batch = this.store.get(
if (!task) return; `SELECT b.* FROM batches b WHERE b.state = 'open'
AND EXISTS (SELECT 1 FROM attempts a WHERE a.batch_id = b.id)
AND NOT EXISTS (SELECT 1 FROM attempts a WHERE a.batch_id = b.id AND a.state != 'approved')
ORDER BY b.id LIMIT 1`,
);
if (!batch) return;
this.busy = true; this.busy = true;
try { try {
await this.merge(task); await this.merge(batch);
} catch (e) { } catch (e) {
this.log.error(`merge of task ${task.id} failed: ${e.stack}`); this.log.error(`merge of batch ${batch.id} failed: ${e.stack}`);
try { try {
this.repo.reset(); this.repo.reset();
} catch {} } catch {}
this.store.run(`UPDATE tasks SET state = 'in_review', updated_at = ? WHERE id = ?`, now(), task.id); this.store.run(`UPDATE batches SET state = 'open' WHERE id = ?`, batch.id);
this.store.event('merge_error', null, { task: task.id, error: e.message }); this.store.event('merge_error', null, { batch: batch.id, error: e.message });
} finally { } finally {
this.busy = false; this.busy = false;
} }
} }
async merge(task) { async merge(batch) {
this.store.run(`UPDATE tasks SET state = 'merging', updated_at = ? WHERE id = ?`, now(), task.id); this.store.run(`UPDATE batches SET state = 'merging' WHERE id = ?`, batch.id);
const attempt = this.store.get(`SELECT * FROM attempts WHERE task_id = ? AND state = 'approved' ORDER BY id DESC LIMIT 1`, task.id); const attempts = batchAttempts(this.store, batch.id);
const author = this.store.get('SELECT name FROM users WHERE id = ?', attempt.user_id).name; for (const a of attempts) this.store.run(`UPDATE tasks SET state = 'merging', updated_at = ? WHERE id = ?`, now(), a.task_id);
const files = JSON.parse(attempt.files_json); const author = this.store.get('SELECT name FROM users WHERE id = ?', batch.user_id).name;
const files = JSON.parse(batch.files_json);
const merged = this.repo.mergeOntoHead(attempt.base_commit, files);
const bounce = (summary, details) => { const bounce = (summary, details) => {
this.store.run(`UPDATE attempts SET state = 'needs_cleanup', result_json = json_set(COALESCE(result_json, '{}'), '$.mergeFailure', json(?)) WHERE id = ?`, JSON.stringify({ summary, ...details }), attempt.id); this.store.run(`UPDATE batches SET result_json = json_set(COALESCE(result_json, '{}'), '$.mergeFailure', json(?)) WHERE id = ?`, JSON.stringify({ summary, ...details }), batch.id);
this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), task.id); returnBatch(this.store, batch.id, summary);
this.log.info(`merge of task ${task.id} bounced: ${summary}`); this.log.info(`merge of batch ${batch.id} bounced: ${summary}`);
}; };
const merged = this.repo.mergeOntoHead(batch.base_commit, files);
if (merged.conflicts.length) return bounce('conflicts with newer work on main', { conflicts: merged.conflicts }); if (merged.conflicts.length) return bounce('conflicts with newer work on main', { conflicts: merged.conflicts });
// Re-verify on top of the current main: other merges may have landed since review. // Re-verify on top of the current main: other merges may have landed since verification.
const { check, rec } = await this.verifier.check(merged.files, `merge-${attempt.id}`); const { check, rec, changedHeaders } = await this.verifier.check(merged.files, `merge-${batch.id}`);
const evaln = this.verifier.evaluate({ check, rec, targetId: task.symbol_id, files: merged.files, changedHeaders: Object.keys(merged.files).filter((f) => !/\.(c|cpp|cc|cxx)$/i.test(f)) }); const targets = new Set(attempts.map((a) => a.symbol_id));
// Not byte-identical is fine if other agents confirmed the logic; anything worse bounces. const outside = this.verifier.changedOutside(check, merged.files, targets);
const peerConfirmed = evaln.state === 'peer_review' && this.peers?.confirmed(attempt.id); const confirmedIds = new Set();
if (evaln.state !== 'in_review' && !peerConfirmed) return bounce(`re-verification on current main failed: ${evaln.feedback.summary}`, { feedback: evaln.feedback }); const results = [];
for (const a of attempts) {
const ev = this.verifier.evaluate({ check, rec, targetId: a.symbol_id, files: merged.files, changedHeaders, outside });
// Byte-identical, or confirmed by peers and approved by a human; anything worse bounces.
const identical = ev.state === 'approved' || ev.state === 'in_review';
const peerConfirmed = ev.state === 'peer_review' && this.peers?.confirmed(a.id);
if (!identical && !peerConfirmed) return bounce(`re-verification on current main failed for ${a.symbol_id}: ${ev.feedback.summary}`, { feedback: ev.feedback });
if (peerConfirmed) confirmedIds.add(a.symbol_id);
results.push({ id: a.symbol_id, name: ev.feedback.target?.demangled ?? null, status: ev.feedback.target?.status, peerConfirmed: !!peerConfirmed });
}
const reviewers = this.store.all(`SELECT u.name FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ? AND r.action = 'approve'`, attempt.id).map((r) => r.name); const reviewers = this.store.all(
const target = evaln.feedback.target; `SELECT DISTINCT u.name FROM reviews r JOIN users u ON u.id = r.reviewer_id JOIN attempts a ON a.id = r.attempt_id WHERE a.batch_id = ? AND r.action = 'approve'`,
const message = `Decompile ${task.symbol_id}${target?.demangled ? ` (${target.demangled})` : ''}\n\nAttempt #${attempt.id} by ${author}, status ${target?.status}.\nReviewed-by: ${reviewers.join(', ')}\n`; batch.id,
).map((r) => r.name);
const peers = this.store.all(
`SELECT DISTINCT u.name FROM peer_reviews p JOIN users u ON u.id = p.reviewer_id JOIN attempts a ON a.id = p.attempt_id WHERE a.batch_id = ? AND p.verdict = 'same'`,
batch.id,
).map((r) => r.name);
const identicalCount = results.filter((r) => !r.peerConfirmed).length;
const title =
results.length === 1
? `Decompile ${results[0].id}${results[0].name ? ` (${results[0].name})` : ''}`
: `Decompile ${results.length} functions in ${[...new Set(Object.keys(files))].slice(0, 2).join(', ')}${Object.keys(files).length > 2 ? ', ...' : ''}`;
const lines = results.map((r) => `- ${r.id}${r.name ? ` ${r.name}` : ''}: ${r.peerConfirmed ? 'logic confirmed by peer review' : 'byte-identical'}`);
const message =
`${title}\n\nBatch #${batch.id} by ${author}: ${identicalCount} byte-identical, ${results.length - identicalCount} confirmed by peer review.\n\n${lines.join('\n')}\n` +
(peers.length ? `\nPeer-reviewed-by: ${peers.join(', ')}` : '') +
(reviewers.length ? `\nReviewed-by: ${reviewers.join(', ')}` : '') +
(!reviewers.length && !confirmedIds.size ? '\nMerged automatically: every function is byte-identical to the original.' : '') +
'\n';
const commit = this.repo.commitFiles(merged.files, message, `${author} <${author.replace(/[^A-Za-z0-9_.-]/g, '_')}@users.clank-together.invalid>`); const commit = this.repo.commitFiles(merged.files, message, `${author} <${author.replace(/[^A-Za-z0-9_.-]/g, '_')}@users.clank-together.invalid>`);
this.store.tx(() => { this.store.tx(() => {
this.applyCheckToDatabase(check, rec, commit, attempt.user_id, { confirmedId: peerConfirmed ? task.symbol_id : null }); this.applyCheckToDatabase(check, rec, commit, batch.user_id, { confirmedIds });
this.store.run(`UPDATE attempts SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, attempt.id); for (const a of attempts) {
this.store.run(`UPDATE tasks SET state = 'done', lease_user = NULL, updated_at = ? WHERE id = ?`, now(), task.id); this.store.run(`UPDATE attempts SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, a.id);
this.store.event('merged', attempt.user_id, { task: task.id, attempt: attempt.id, commit }); this.store.run(`UPDATE tasks SET state = 'done', lease_user = NULL, updated_at = ? WHERE id = ?`, now(), a.task_id);
}
this.store.run(`UPDATE batches SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, batch.id);
this.store.event('merged', batch.user_id, { batch: batch.id, functions: attempts.length, commit });
}); });
this.tasks.generate(); this.tasks.generate();
if (this.config.pushRemote) { if (this.config.pushRemote) {
@ -115,14 +170,15 @@ export class Merger {
this.log.error(`push to ${this.config.pushRemote} failed: ${e.message}`); this.log.error(`push to ${this.config.pushRemote} failed: ${e.message}`);
} }
} }
this.log.info(`merged task ${task.id} as ${commit.slice(0, 10)}`); this.onMerged?.(commit);
this.log.info(`merged batch ${batch.id} (${attempts.length} functions) as ${commit.slice(0, 10)}`);
} }
/** /**
* Records what a check on main learned: per-function status, decorated * Records what a check on main learned: per-function status, decorated
* names and new symbols for previously unknown references. * names and new symbols for previously unknown references.
*/ */
applyCheckToDatabase(check, rec, commit, userId, { confirmedId = null } = {}) { applyCheckToDatabase(check, rec, commit, userId, { confirmedIds = new Set() } = {}) {
const v = this.store.currentVersion(); const v = this.store.currentVersion();
const pe = this.getPe(); const pe = this.getPe();
for (const u of check.units) { for (const u of check.units) {
@ -135,10 +191,10 @@ export class Merger {
} }
for (const f of u.functions) { for (const f of u.functions) {
this.store.run( this.store.run(
`INSERT INTO func_status (symbol_id, file, status, score, commit_hash, updated_at, confirmed) VALUES (?, ?, ?, ?, ?, ?, ?) `INSERT INTO func_status (symbol_id, file, status, score, commit_hash, updated_at, confirmed, code_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(symbol_id) DO UPDATE SET file = excluded.file, status = excluded.status, score = excluded.score, commit_hash = excluded.commit_hash, updated_at = excluded.updated_at, ON CONFLICT(symbol_id) DO UPDATE SET file = excluded.file, status = excluded.status, score = excluded.score, commit_hash = excluded.commit_hash, updated_at = excluded.updated_at, code_hash = COALESCE(excluded.code_hash, func_status.code_hash),
confirmed = CASE WHEN excluded.confirmed = 1 THEN 1 WHEN excluded.status IN ('exact', 'match_unresolved') THEN 0 ELSE func_status.confirmed END`, confirmed = CASE WHEN excluded.confirmed = 1 THEN 1 WHEN excluded.status IN ('exact', 'match_unresolved') THEN 0 ELSE func_status.confirmed END`,
f.id, u.file, f.status, f.score, commit, now(), f.id === confirmedId ? 1 : 0, f.id, u.file, f.status, f.score, commit, now(), confirmedIds.has(f.id) ? 1 : 0, f.codeHash ?? null,
); );
if (f.demangled) this.store.run('UPDATE symbols SET name = ? WHERE id = ? AND name IS NULL', f.demangled, f.id); if (f.demangled) this.store.run('UPDATE symbols SET name = ? WHERE id = ? AND name IS NULL', f.demangled, f.id);
this.store.run('UPDATE symbols SET unit = ? WHERE id = ? AND unit IS NULL', u.file, f.id); this.store.run('UPDATE symbols SET unit = ? WHERE id = ? AND unit IS NULL', u.file, f.id);

95
src/server/original.js Normal file
View file

@ -0,0 +1,95 @@
// The original function, for contributors without a local Ghidra: assembly and pseudocode from a
// Ghidra export (ghidra_scripts/ExportOriginals.java, imported with `td-server originals import`),
// and live disassembly by objdump for functions the export does not cover.
import fs from 'node:fs';
import readline from 'node:readline';
import { run } from '../shared/build.js';
import { hex } from '../shared/annotations.js';
import { HttpError } from './http.js';
export class Originals {
constructor({ store }) {
this.store = store;
}
/** Imports an ExportOriginals JSON-lines file for a version; returns the number of functions. */
async import(versionId, file) {
const rl = readline.createInterface({ input: fs.createReadStream(file, 'utf8'), crlfDelay: Infinity });
let n = 0;
const insert = this.store.db.prepare(
`INSERT INTO original_code (version_id, addr, asm, c) VALUES (?, ?, ?, ?)
ON CONFLICT(version_id, addr) DO UPDATE SET asm = excluded.asm, c = excluded.c`,
);
this.store.db.exec('BEGIN');
try {
for await (const line of rl) {
if (!line.trim()) continue;
const r = JSON.parse(line);
insert.run(versionId, Number.parseInt(r.addr, 16), r.asm ?? null, r.c ?? null);
if (++n % 5000 === 0) {
this.store.db.exec('COMMIT');
this.store.db.exec('BEGIN');
}
}
this.store.db.exec('COMMIT');
} catch (e) {
this.store.db.exec('ROLLBACK');
throw e;
}
return n;
}
/** {symbol, name, version, addr, size, asm, pseudocode, source} for a symbol ID or a hex address. */
async get(query) {
const v = this.store.currentVersion();
if (!v) throw new HttpError(404, 'no current version');
let sym;
if (/^0x[0-9a-f]+$/i.test(query)) {
sym = this.store.get(
`SELECT s.id, s.name, a.addr, a.size FROM symbol_addresses a JOIN symbols s ON s.id = a.symbol_id WHERE a.version_id = ? AND a.addr = ?`,
v.id,
Number.parseInt(query, 16),
) ?? { id: null, name: null, addr: Number.parseInt(query, 16), size: null };
} else {
sym = this.store.get(
`SELECT s.id, s.name, a.addr, a.size FROM symbols s LEFT JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? WHERE s.id = ? OR s.name = ? OR s.decorated = ? LIMIT 1`,
v.id,
query,
query,
query,
);
}
if (!sym) throw new HttpError(404, `unknown symbol ${query}`);
if (sym.addr == null) throw new HttpError(404, `${sym.id} has no address in ${v.id}`);
const row = this.store.get('SELECT asm, c FROM original_code WHERE version_id = ? AND addr = ?', v.id, sym.addr);
const asm = row?.asm ?? (await this.objdump(v.file, sym.addr, sym.size ?? 256));
return {
symbol: sym.id,
name: sym.name,
version: v.id,
addr: '0x' + hex(sym.addr),
size: sym.size,
asm,
pseudocode: row?.c ?? null,
source: row ? 'ghidra export' : 'objdump (no pseudocode yet)',
note: 'Pseudocode is Ghidra output, a reading aid; it is not matching code.',
};
}
async objdump(file, addr, size) {
const r = await run('objdump', ['-d', '-M', 'intel', '--no-addresses', `--start-address=0x${hex(addr)}`, `--stop-address=0x${hex(addr + Math.min(size, 65536))}`, file], { timeoutMs: 30_000 });
if (r.code !== 0) {
// Older binutils lack --no-addresses.
const r2 = await run('objdump', ['-d', '-M', 'intel', `--start-address=0x${hex(addr)}`, `--stop-address=0x${hex(addr + Math.min(size, 65536))}`, file], { timeoutMs: 30_000 });
return cleanObjdump(r2.output);
}
return cleanObjdump(r.output);
}
}
function cleanObjdump(out) {
const lines = out.split('\n');
const start = lines.findIndex((l) => /^\s*[0-9a-f]+ <|^Disassembly of section/.test(l));
return lines.slice(start >= 0 ? start : 0).filter((l) => l.trim() && !/^Disassembly of section/.test(l)).join('\n');
}

View file

@ -10,6 +10,7 @@ import { HttpError } from './http.js';
import { now } from './db.js'; import { now } from './db.js';
import { untrusted } from './tasks.js'; import { untrusted } from './tasks.js';
import { hex } from '../shared/annotations.js'; import { hex } from '../shared/annotations.js';
import { returnBatch } from './batches.js';
export const PEER_INSTRUCTIONS = [ export const PEER_INSTRUCTIONS = [
'Decide whether the submitted C++ does the same thing as the original function. It is not byte-identical; your judgment is what counts.', 'Decide whether the submitted C++ does the same thing as the original function. It is not byte-identical; your judgment is what counts.',
@ -36,7 +37,7 @@ export class PeerReviews {
/** Attempts waiting for peer review that `user` may take. */ /** Attempts waiting for peer review that `user` may take. */
available(user, limit = 20) { available(user, limit = 20) {
return this.store.all( return this.store.all(
`SELECT a.id AS attempt_id, a.task_id, t.symbol_id, s.name, a.finished_at, `SELECT a.id AS attempt_id, a.batch_id, a.task_id, t.symbol_id, s.name, a.finished_at,
(SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS confirmations, (SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS confirmations,
(SELECT COUNT(*) FROM peer_leases l WHERE l.attempt_id = a.id AND l.expires_at > ?) AS in_progress (SELECT COUNT(*) FROM peer_leases l WHERE l.attempt_id = a.id AND l.expires_at > ?) AS in_progress
FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
@ -51,28 +52,56 @@ export class PeerReviews {
).filter((r) => r.confirmations + r.in_progress < this.required); ).filter((r) => r.confirmations + r.in_progress < this.required);
} }
/** Leases one review (a given attempt, or the oldest available) and returns its context. */ /**
* Leases a review (a given attempt, or the oldest available) together with every other function
* of the same batch that waits for this reviewer, and returns the context of all of them.
*/
claim(user, attemptId = null) { claim(user, attemptId = null) {
return this.store.tx(() => { return this.store.tx(() => {
const options = this.available(user, 200); const options = this.available(user, 200);
const pick = attemptId ? options.find((o) => o.attempt_id === attemptId) : options[0]; const pick = attemptId ? options.find((o) => o.attempt_id === attemptId) : options[0];
if (!pick) throw new HttpError(attemptId ? 409 : 404, attemptId ? 'this attempt is not open for your review (yours, already reviewed, or enough reviewers)' : 'no attempts are waiting for peer review'); if (!pick) throw new HttpError(attemptId ? 409 : 404, attemptId ? 'this attempt is not open for your review (yours, already reviewed, or enough reviewers)' : 'no attempts are waiting for peer review');
const group = pick.batch_id ? options.filter((o) => o.batch_id === pick.batch_id) : [pick];
for (const o of group) {
this.store.run( this.store.run(
`INSERT INTO peer_leases (attempt_id, reviewer_id, expires_at) VALUES (?, ?, ?) `INSERT INTO peer_leases (attempt_id, reviewer_id, expires_at) VALUES (?, ?, ?)
ON CONFLICT(attempt_id, reviewer_id) DO UPDATE SET expires_at = excluded.expires_at`, ON CONFLICT(attempt_id, reviewer_id) DO UPDATE SET expires_at = excluded.expires_at`,
pick.attempt_id, user.id, now() + this.leaseMs, o.attempt_id, user.id, now() + this.leaseMs,
); );
this.store.event('peer_claim', user.id, { attempt: pick.attempt_id }); }
return this.context(pick.attempt_id); this.store.event('peer_claim', user.id, { attempt: pick.attempt_id, functions: group.length });
return this.context(pick.attempt_id, group.map((o) => o.attempt_id));
}); });
} }
/** One function to review: where the original is and how to read it. */
functionInfo(attemptId, v) {
const row = this.store.get(
`SELECT a.id, s.id AS symbol, s.name, ad.addr, ad.size FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
LEFT JOIN symbol_addresses ad ON ad.symbol_id = s.id AND ad.version_id = ? WHERE a.id = ?`,
v.id,
attemptId,
);
return {
attemptId: row.id,
symbol: row.symbol,
name: row.name,
addr: row.addr != null ? '0x' + hex(row.addr) : null,
size: row.size,
annotation: `// FUNCTION: ${row.symbol}`,
read: [`td_disassemble ${row.symbol}`, `td_decompile ${row.symbol}`],
};
}
release(user, attemptId) { release(user, attemptId) {
this.store.run('DELETE FROM peer_leases WHERE attempt_id = ? AND reviewer_id = ?', attemptId, user.id); this.store.run('DELETE FROM peer_leases WHERE attempt_id = ? AND reviewer_id = ?', attemptId, user.id);
} }
/** What a reviewer sees: the function to compare against and the submitted code, nothing machine-judged. */ /**
context(attemptId) { * What a reviewer sees: the functions to compare against and the submitted code, nothing
* machine-judged. `attemptIds`: the functions of the batch this review covers.
*/
context(attemptId, attemptIds = null) {
const a = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId); const a = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId);
if (!a) throw new HttpError(404, 'no such attempt'); if (!a) throw new HttpError(404, 'no such attempt');
const t = this.store.get('SELECT * FROM tasks WHERE id = ?', a.task_id); const t = this.store.get('SELECT * FROM tasks WHERE id = ?', a.task_id);
@ -82,12 +111,16 @@ export class PeerReviews {
v.id, v.id,
t.symbol_id, t.symbol_id,
); );
const result = a.result_json ? JSON.parse(a.result_json) : {}; const batch = a.batch_id ? this.store.get('SELECT * FROM batches WHERE id = ?', a.batch_id) : null;
const batchResult = batch?.result_json ? JSON.parse(batch.result_json) : {};
const ids = attemptIds ?? (a.batch_id ? this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'peer_review' ORDER BY id`, a.batch_id).map((r) => r.id) : [a.id]);
return { return {
attemptId: a.id, attemptId: a.id,
required: this.required, required: this.required,
confirmations: this.store.get(`SELECT COUNT(*) AS n FROM peer_reviews WHERE attempt_id = ? AND verdict = 'same'`, a.id).n, confirmations: this.store.get(`SELECT COUNT(*) AS n FROM peer_reviews WHERE attempt_id = ? AND verdict = 'same'`, a.id).n,
instructions: PEER_INSTRUCTIONS, instructions: PEER_INSTRUCTIONS,
// Every function this review covers (one batch); give one verdict per attemptId.
functions: (ids.length ? ids : [a.id]).map((id) => this.functionInfo(id, v)),
original: { original: {
symbol: s.id, symbol: s.id,
name: s.name, name: s.name,
@ -98,7 +131,7 @@ export class PeerReviews {
}, },
submission: { submission: {
annotation: `// FUNCTION: ${s.id}`, annotation: `// FUNCTION: ${s.id}`,
files: result.mergedFiles ?? JSON.parse(a.files_json), files: batchResult.mergedFiles ?? this.store.attemptFiles(a),
notes: untrusted(a.notes), notes: untrusted(a.notes),
}, },
leaseExpires: this.store.get('SELECT expires_at FROM peer_leases WHERE attempt_id = ? ORDER BY expires_at DESC LIMIT 1', a.id)?.expires_at ?? null, leaseExpires: this.store.get('SELECT expires_at FROM peer_leases WHERE attempt_id = ? ORDER BY expires_at DESC LIMIT 1', a.id)?.expires_at ?? null,
@ -135,6 +168,8 @@ export class PeerReviews {
`UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, `UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`,
a.user_id, now() + 6 * 3600_000, now(), a.task_id, a.user_id, now() + 6 * 3600_000, now(), a.task_id,
); );
// The batch merges as a whole, so it goes back with this function; confirmed ones keep their verdicts.
returnBatch(this.store, a.batch_id, `a peer review found a difference in ${a.task_id}`);
return { state: 'needs_cleanup', confirmations: taken }; return { state: 'needs_cleanup', confirmations: taken };
} }
const confirmations = taken + 1; const confirmations = taken + 1;

View file

@ -75,10 +75,46 @@ export class Tasks {
return this.store.all(`SELECT id FROM tasks WHERE lease_user = ? AND state = 'leased'`, userId).map((r) => r.id); return this.store.all(`SELECT id FROM tasks WHERE lease_user = ? AND state = 'leased'`, userId).map((r) => r.id);
} }
/** How many tasks (functions) one account may hold at once, single claims and batches together. */
get maxLeased() {
return this.config.maxLeasedTasks ?? 200;
}
/**
* Claims several tasks at once: every claimable function of a unit (source file) or of an
* address range in the current version, up to `limit`. They are submitted together as a batch.
*/
claimBatch(user, { unit = null, start = null, end = null, limit = 50 } = {}) {
if (!unit && (start == null || end == null)) throw new HttpError(400, 'give a unit (source file) or an address range (start, end)');
return this.store.tx(() => {
const held = this.leasesOf(user.id).length;
const room = Math.min(Math.max(1, limit | 0), 200, this.maxLeased - held);
if (room <= 0) throw new HttpError(409, `you already hold ${held} tasks; submit or release some first`);
const v = this.store.currentVersion();
const ph = CLAIMABLE.map(() => '?').join(',');
const where = unit ? 's.unit = ?' : 'a.addr >= ? AND a.addr < ?';
const args = unit ? [unit] : [start, end];
const rows = this.store.all(
`SELECT t.id FROM tasks t JOIN symbols s ON s.id = t.symbol_id
JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ?
WHERE t.state IN (${ph}) AND ${where}
AND (t.preferred_user IS NULL OR t.preferred_user = ? OR t.preferred_until < ?)
ORDER BY a.addr LIMIT ?`,
v.id, ...CLAIMABLE, ...args, user.id, now(), room,
);
if (!rows.length) throw new HttpError(404, unit ? `no claimable tasks in ${unit}` : 'no claimable tasks in that range');
const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000;
const head = this.repo.head();
for (const r of rows) this.store.run(`UPDATE tasks SET state = 'leased', lease_user = ?, lease_expires = ?, lease_base_commit = ?, updated_at = ? WHERE id = ?`, user.id, expires, head, now(), r.id);
this.store.event('claim_batch', user.id, { tasks: rows.length, unit, start, end });
return rows.map((r) => r.id);
});
}
claim(user, taskId) { claim(user, taskId) {
return this.store.tx(() => { return this.store.tx(() => {
const max = this.config.maxLeasesPerUser ?? 3; const max = this.maxLeased;
if (this.leasesOf(user.id).length >= max) throw new HttpError(409, `you already hold ${max} leases; submit or release one first`); if (this.leasesOf(user.id).length >= max) throw new HttpError(409, `you already hold ${max} tasks; submit or release some first`);
let t; let t;
if (taskId) { if (taskId) {
t = this.get(taskId); t = this.get(taskId);
@ -114,12 +150,59 @@ export class Tasks {
return expires; return expires;
} }
/** Extends every lease the account holds (an agent working through a batch). */
heartbeatAll(user) {
const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000;
const r = this.store.run(`UPDATE tasks SET lease_expires = ?, updated_at = ? WHERE lease_user = ? AND state = 'leased'`, expires, now(), user.id);
return { leases: Number(r.changes), leaseExpires: expires };
}
release(user, taskId) { release(user, taskId) {
const t = this.requireLease(user, taskId); const t = this.requireLease(user, taskId);
this.reopen(t.id); this.reopen(t.id);
this.store.event('release', user.id, { task: t.id }); this.store.event('release', user.id, { task: t.id });
} }
/**
* One submission for one or more leased tasks: the files once, an attempt per task. Verified
* with a single compile and merged as a whole (see Verifier.verifyBatch, Merger).
*/
submitBatch(user, { taskIds, baseCommit, files, notes = null, agentMeta = null, envDigest = null }) {
const ids = [...new Set((taskIds ?? []).map(Number))];
if (!ids.length) throw new HttpError(400, 'taskIds must list the tasks this submission covers');
if (ids.length > 200) throw new HttpError(400, 'at most 200 tasks per submission');
return this.store.tx(() => {
for (const id of ids) this.requireLease(user, id);
const b = this.store.run(
`INSERT INTO batches (user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', ?)`,
user.id, baseCommit, JSON.stringify(files), notes, agentMeta, envDigest, now(),
);
const batchId = Number(b.lastInsertRowid);
const attempts = [];
for (const id of ids) {
const r = this.store.run(
`INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, batch_id, created_at) VALUES (?, ?, ?, '{}', ?, ?, ?, 'queued', ?, ?)`,
id, user.id, baseCommit, notes, agentMeta, envDigest, batchId, now(),
);
attempts.push({ taskId: id, attemptId: Number(r.lastInsertRowid) });
this.store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), id);
}
this.store.event('submit', user.id, { batch: batchId, tasks: ids.length });
return { batchId, attempts };
});
}
/** Allowed paths of a set of tasks: their unit files plus the project headers. */
allowedPathsForTasks(taskIds) {
const out = new Set();
for (const id of taskIds) {
const t = this.get(id);
const sym = this.store.get('SELECT * FROM symbols WHERE id = ?', t.symbol_id);
for (const p of this.allowedPaths(sym)) out.add(p);
}
return [...out];
}
/** Returns a task to the pool, keeping its attempt history. */ /** Returns a task to the pool, keeping its attempt history. */
reopen(taskId) { reopen(taskId) {
const t = this.get(taskId); const t = this.get(taskId);
@ -164,8 +247,8 @@ export class Tasks {
); );
let bestFiles = null; let bestFiles = null;
if (t.best_attempt) { if (t.best_attempt) {
const best = this.store.get('SELECT files_json FROM attempts WHERE id = ?', t.best_attempt); const best = this.store.get('SELECT * FROM attempts WHERE id = ?', t.best_attempt);
bestFiles = best ? JSON.parse(best.files_json) : null; bestFiles = best ? this.store.attemptFiles(best) : null;
} }
const neighbours = sym.unit const neighbours = sym.unit
? this.store.all( ? this.store.all(

View file

@ -9,24 +9,35 @@ import { reconcile } from '../shared/reconcile.js';
import { STATUS } from '../shared/compare.js'; import { STATUS } from '../shared/compare.js';
import { listSources, isUnit } from '../shared/project.js'; import { listSources, isUnit } from '../shared/project.js';
import { now } from './db.js'; import { now } from './db.js';
import { returnBatch, batchAttempts } from './batches.js';
const RANK = { [STATUS.EXACT]: 3, [STATUS.UNRESOLVED]: 2, [STATUS.MISMATCH]: 1, [STATUS.MISSING]: 0, [STATUS.NO_ADDRESS]: 0 }; const RANK = { [STATUS.EXACT]: 3, [STATUS.UNRESOLVED]: 2, [STATUS.MISMATCH]: 1, [STATUS.MISSING]: 0, [STATUS.NO_ADDRESS]: 0 };
export class Verifier { export class Verifier {
constructor({ store, config, repo, project, tasks, getPe, log = console }) { constructor({ store, config, repo, project, tasks, getPe, peers, log = console }) {
this.store = store; this.store = store;
this.config = config; this.config = config;
this.repo = repo; this.repo = repo;
this.project = project; this.project = project;
this.tasks = tasks; this.tasks = tasks;
this.getPe = getPe; this.getPe = getPe;
this.peers = peers;
this.log = log; this.log = log;
this.running = 0; this.running = 0;
this.jobsDir = path.join(config.dataDir, 'jobs'); this.jobsDir = path.join(config.dataDir, 'jobs');
} }
start() { start() {
this.store.run(`UPDATE batches SET state = 'queued' WHERE state = 'verifying'`);
this.store.run(`UPDATE attempts SET state = 'queued' WHERE state = 'verifying'`); this.store.run(`UPDATE attempts SET state = 'queued' WHERE state = 'verifying'`);
// Attempts from before batches existed become batches of one.
for (const a of this.store.all(`SELECT * FROM attempts WHERE state = 'queued' AND batch_id IS NULL`)) {
const b = this.store.run(
`INSERT INTO batches (user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', ?)`,
a.user_id, a.base_commit, a.files_json, a.notes, a.agent_meta, a.env_digest, a.created_at,
);
this.store.run('UPDATE attempts SET batch_id = ? WHERE id = ?', Number(b.lastInsertRowid), a.id);
}
this.timer = setInterval(() => this.pump(), 2000); this.timer = setInterval(() => this.pump(), 2000);
this.pump(); this.pump();
} }
@ -38,17 +49,23 @@ export class Verifier {
pump() { pump() {
const max = this.config.verifyConcurrency ?? 2; const max = this.config.verifyConcurrency ?? 2;
while (this.running < max) { while (this.running < max) {
const a = this.store.tx(() => { const b = this.store.tx(() => {
const row = this.store.get(`SELECT * FROM attempts WHERE state = 'queued' ORDER BY id LIMIT 1`); const row = this.store.get(`SELECT * FROM batches WHERE state = 'queued' ORDER BY id LIMIT 1`);
if (row) this.store.run(`UPDATE attempts SET state = 'verifying' WHERE id = ?`, row.id); if (row) {
this.store.run(`UPDATE batches SET state = 'verifying' WHERE id = ?`, row.id);
this.store.run(`UPDATE attempts SET state = 'verifying' WHERE batch_id = ? AND state = 'queued'`, row.id);
}
return row; return row;
}); });
if (!a) return; if (!b) return;
this.running++; this.running++;
this.verify(a) this.verifyBatch(b)
.catch((e) => { .catch((e) => {
this.log.error(`attempt ${a.id}: verifier crashed: ${e.stack}`); this.log.error(`batch ${b.id}: verifier crashed: ${e.stack}`);
this.finish(a, 'needs_cleanup', 0, { summary: 'internal verifier error; a maintainer has been notified', internalError: true }); for (const a of batchAttempts(this.store, b.id)) {
if (a.state === 'verifying') this.finish(a, 'needs_cleanup', 0, { summary: 'internal verifier error; a maintainer has been notified', internalError: true });
}
returnBatch(this.store, b.id, 'internal verifier error');
}) })
.finally(() => { .finally(() => {
this.running--; this.running--;
@ -118,39 +135,91 @@ export class Verifier {
return out; return out;
} }
async verify(attempt) { /**
const task = this.tasks.get(attempt.task_id); * Functions of the submitted files whose compiled code differs from main although they are not
const sym = this.store.get('SELECT * FROM symbols WHERE id = ?', task.symbol_id); * part of the submission: a submission may only change the functions it holds the lease for.
const submitted = JSON.parse(attempt.files_json); * Unknown on main (no fingerprint yet, e.g. before the first rebuild) is not counted.
*/
changedOutside(check, files, targets) {
const fileSet = new Set(Object.keys(files));
const seen = new Map();
for (const u of check.units) for (const f of u.functions) if (fileSet.has(f.file) && !targets.has(f.id) && f.codeHash) seen.set(f.id, f);
const out = [];
for (const [id, f] of seen) {
const main = this.store.get('SELECT code_hash FROM func_status WHERE symbol_id = ?', id);
if (!main) out.push({ id, file: f.file, why: 'new or newly annotated' });
else if (main.code_hash && main.code_hash !== f.codeHash) out.push({ id, file: f.file, why: 'compiled code changed' });
}
return out;
}
/**
* A function that three peers already confirmed keeps that confirmation when its compiled code is
* unchanged in a resubmission (the batch came back because of another function).
*/
carryOver(attempt, target) {
if (!target?.codeHash) return false;
const prev = this.store.all(`SELECT id, result_json FROM attempts WHERE task_id = ? AND id != ? ORDER BY id DESC LIMIT 5`, attempt.task_id, attempt.id);
for (const p of prev) {
const hash = p.result_json ? JSON.parse(p.result_json).feedback?.target?.codeHash : null;
if (hash !== target.codeHash || !this.peers?.confirmed(p.id)) continue;
this.store.run(
`INSERT OR IGNORE INTO peer_reviews (attempt_id, reviewer_id, verdict, reason, agent_meta, created_at)
SELECT ?, reviewer_id, verdict, reason, agent_meta, created_at FROM peer_reviews WHERE attempt_id = ?`,
attempt.id, p.id,
);
return true;
}
return false;
}
async verifyBatch(batch) {
const attempts = batchAttempts(this.store, batch.id);
const submitted = JSON.parse(batch.files_json);
const finishAll = (state, feedback) => {
for (const a of attempts) this.finish(a, state, 0, feedback);
this.store.run(`UPDATE batches SET state = 'returned', finished_at = ?, result_json = ? WHERE id = ?`, now(), JSON.stringify({ feedback }), batch.id);
};
// Stage 0: static gate. // Stage 0: static gate.
const lintErrors = lintSubmission(submitted, { const lintErrors = lintSubmission(submitted, {
allowedPaths: this.tasks.allowedPaths(sym), allowedPaths: this.tasks.allowedPathsForTasks(attempts.map((a) => a.task_id)),
limits: this.project.lintLimits, limits: this.project.lintLimits,
allowedPragmas: this.project.allowedPragmas, allowedPragmas: this.project.allowedPragmas,
}); });
if (lintErrors.length) return this.finish(attempt, 'rejected', 0, { summary: 'rejected by the static gate', lintErrors }); if (lintErrors.length) return finishAll('rejected', { summary: 'rejected by the static gate', lintErrors });
if (!this.repo.isAncestor(batch.base_commit)) return finishAll('rejected', { summary: `base commit ${batch.base_commit} is not part of main; sync and resubmit` });
const merged = this.repo.mergeOntoHead(batch.base_commit, submitted);
if (merged.conflicts.length) return finishAll('needs_cleanup', { summary: 'your change conflicts with newer work on main; sync, merge and resubmit', conflicts: merged.conflicts });
if (!this.repo.isAncestor(attempt.base_commit)) { // Stages 1-3: one compile for the whole batch, then every function on its own.
return this.finish(attempt, 'rejected', 0, { summary: `base commit ${attempt.base_commit} is not part of main; sync and resubmit` }); const { check, rec, changedHeaders } = await this.check(merged.files, `batch-${batch.id}`);
const targets = new Set(attempts.map((a) => a.symbol_id));
const outside = this.changedOutside(check, merged.files, targets);
let failed = 0;
for (const a of attempts) {
const ev = this.evaluate({ check, rec, targetId: a.symbol_id, files: merged.files, changedHeaders, outside });
ev.feedback.headCommit = this.repo.head();
if (attempts.length > 1) ev.feedback.batch = { id: batch.id, functions: attempts.length };
if (batch.env_digest && this.config.envDigest && batch.env_digest !== this.config.envDigest) {
ev.feedback.warnings.push('your local toolchain differs from the pinned one; local results may not match the server');
} }
const merged = this.repo.mergeOntoHead(attempt.base_commit, submitted); if (ev.state === 'peer_review' && this.carryOver(a, ev.feedback.target)) {
if (merged.conflicts.length) { ev.state = 'in_review';
return this.finish(attempt, 'needs_cleanup', 0, { summary: 'your change conflicts with newer work on main; sync, merge and resubmit', conflicts: merged.conflicts }); ev.feedback.summary += '; the compiled code is unchanged since three peers confirmed it, so their verdicts carry over';
} }
if (ev.state === 'needs_cleanup' || ev.state === 'rejected') failed++;
// Stages 1-3: compile, compare, regressions. this.finish(a, ev.state, ev.score, ev.feedback);
const { check, rec, changedHeaders } = await this.check(merged.files, `attempt-${attempt.id}`);
const evaluation = this.evaluate({ check, rec, targetId: task.symbol_id, files: merged.files, changedHeaders });
evaluation.feedback.headCommit = this.repo.head();
if (attempt.env_digest && this.config.envDigest && attempt.env_digest !== this.config.envDigest) {
evaluation.feedback.warnings.push('your local toolchain differs from the pinned one; local results may not match the server');
} }
return this.finish(attempt, evaluation.state, evaluation.score, evaluation.feedback, { mergedFiles: merged.files, check: summarizeCheck(check), rec, changedHeaders }); this.store.run(
`UPDATE batches SET state = 'open', finished_at = ?, result_json = ? WHERE id = ?`,
now(), JSON.stringify({ mergedFiles: merged.files, check: summarizeCheck(check), changedHeaders, outside }), batch.id,
);
if (failed) returnBatch(this.store, batch.id, `${failed} of ${attempts.length} functions need work; see each function's feedback`);
} }
/** Routes a check result: rejected, needs_cleanup or in_review. */ /** Routes a check result: rejected, needs_cleanup or in_review. */
evaluate({ check, rec, targetId, files, changedHeaders }) { evaluate({ check, rec, targetId, files, changedHeaders, outside = [] }) {
const fileSet = new Set(Object.keys(files)); const fileSet = new Set(Object.keys(files));
const policy = this.project.matchPolicy; const policy = this.project.matchPolicy;
const feedback = { summary: '', warnings: [] }; const feedback = { summary: '', warnings: [] };
@ -200,12 +269,18 @@ export class Verifier {
} else if (regressions.length) { } else if (regressions.length) {
feedback.summary = `your change breaks ${regressions.length} function(s) that matched before`; feedback.summary = `your change breaks ${regressions.length} function(s) that matched before`;
state = 'needs_cleanup'; state = 'needs_cleanup';
} else if (outside.length) {
feedback.summary = `the submission changes functions it does not hold: ${outside.map((o) => o.id).join(', ')}; claim them as well (td claim --unit) or undo those changes`;
feedback.changedOutside = outside;
state = 'needs_cleanup';
} else if (wrongRefs.length) { } else if (wrongRefs.length) {
feedback.summary = `references point at the wrong symbols: ${wrongRefs.map((c) => c.message).join('; ')}`; feedback.summary = `references point at the wrong symbols: ${wrongRefs.map((c) => c.message).join('; ')}`;
state = 'needs_cleanup'; state = 'needs_cleanup';
} else if (target.status === STATUS.EXACT || (target.status === STATUS.UNRESOLVED && policy.reviewAt === 'match_unresolved')) { } else if (target.status === STATUS.EXACT || (target.status === STATUS.UNRESOLVED && policy.reviewAt === 'match_unresolved')) {
feedback.summary = target.status === STATUS.EXACT ? 'exact match, sent to human review' : 'bytes match (some referenced data is not yet named), sent to human review'; // Byte-identical needs no human: approved directly, merged with its batch.
state = 'in_review'; const auto = policy.autoMergeIdentical !== false;
feedback.summary = (target.status === STATUS.EXACT ? 'byte-identical' : 'byte-identical (some referenced data is not yet named)') + (auto ? ', approved automatically' : ', sent to human review');
state = auto ? 'approved' : 'in_review';
} else if (target.status === STATUS.MISMATCH || target.status === STATUS.UNRESOLVED) { } else if (target.status === STATUS.MISMATCH || target.status === STATUS.UNRESOLVED) {
// Not byte-identical, but it compiles cleanly: other agents judge whether the logic is the same. // Not byte-identical, but it compiles cleanly: other agents judge whether the logic is the same.
feedback.summary = `not byte-identical (score ${(score * 100).toFixed(1)}%), sent to peer review: ${policy.peerReviews} other agents compare it with the original`; feedback.summary = `not byte-identical (score ${(score * 100).toFixed(1)}%), sent to peer review: ${policy.peerReviews} other agents compare it with the original`;
@ -221,9 +296,10 @@ export class Verifier {
const task = this.tasks.get(attempt.task_id); const task = this.tasks.get(attempt.task_id);
this.store.tx(() => { this.store.tx(() => {
this.store.run('UPDATE attempts SET state = ?, score = ?, result_json = ?, finished_at = ? WHERE id = ?', state, score, JSON.stringify({ feedback, ...extra }), now(), attempt.id); this.store.run('UPDATE attempts SET state = ?, score = ?, result_json = ?, finished_at = ? WHERE id = ?', state, score, JSON.stringify({ feedback, ...extra }), now(), attempt.id);
let taskState = state === 'in_review' || state === 'peer_review' ? state : 'needs_cleanup'; const onTheWay = ['in_review', 'peer_review', 'approved'].includes(state);
let taskState = onTheWay ? state : 'needs_cleanup';
const better = state !== 'rejected' && score > task.best_score; const better = state !== 'rejected' && score > task.best_score;
const cycles = task.cleanup_cycles + (state === 'in_review' || state === 'peer_review' ? 0 : 1); const cycles = task.cleanup_cycles + (onTheWay ? 0 : 1);
if (taskState === 'needs_cleanup' && cycles >= this.project.matchPolicy.maxCleanupCycles) taskState = 'hard'; if (taskState === 'needs_cleanup' && cycles >= this.project.matchPolicy.maxCleanupCycles) taskState = 'hard';
if (state === 'rejected' && !task.best_attempt) taskState = 'open'; if (state === 'rejected' && !task.best_attempt) taskState = 'open';
this.store.run( this.store.run(

View file

@ -180,6 +180,9 @@ $('token-form').addEventListener('submit', async (e) => {
const t = await post('/api/me/tokens', formData(e.target)); const t = await post('/api/me/tokens', formData(e.target));
e.target.reset(); e.target.reset();
$('new-token-value').textContent = t.token; $('new-token-value').textContent = t.token;
const origin = location.origin;
$('install-win').textContent = `$env:TD_TOKEN='${t.token}'; irm ${origin}/install.ps1 | iex`;
$('install-linux').textContent = `curl -fsSL ${origin}/install.sh | TD_TOKEN=${t.token} sh`;
$('new-token').hidden = false; $('new-token').hidden = false;
await loadTokens(); await loadTokens();
} catch (err) { } catch (err) {
@ -187,6 +190,15 @@ $('token-form').addEventListener('submit', async (e) => {
} }
}); });
$('copy-install').addEventListener('click', async () => {
try {
await navigator.clipboard.writeText($('install-win').textContent);
$('copy-install').textContent = 'Copied';
} catch {
$('copy-install').textContent = 'Select the line and copy it';
}
});
$('password-form').addEventListener('submit', async (e) => { $('password-form').addEventListener('submit', async (e) => {
e.preventDefault(); e.preventDefault();
try { try {
@ -264,13 +276,41 @@ async function loadQueue() {
const q = await api('/api/review/queue'); const q = await api('/api/review/queue');
const body = $('queue-body'); const body = $('queue-body');
body.replaceChildren(); body.replaceChildren();
if (!q.length) return body.append(el('p', 'Nothing waiting for review.', 'muted')); if (!q.length) return body.append(el('p', 'Nothing waiting for review. Byte-identical work merges on its own; this queue only holds peer-confirmed functions.', 'muted'));
body.append( // One block per batch: the functions three peers confirmed, approved together with one click.
table( const batches = new Map();
['Attempt', 'Symbol', 'Name', 'Author', 'Score', 'Approvals', ''], for (const a of q) {
q.map((a) => [`#${a.id}`, a.symbol_id, a.name || '', a.author, `${(a.score * 100).toFixed(1)}%`, a.approvals, button(a.reviewed_by_me ? 'Reviewed' : 'Open', () => openAttempt(a.id))]), const key = a.batch_id ?? `a${a.id}`;
if (!batches.has(key)) batches.set(key, []);
batches.get(key).push(a);
}
for (const [key, items] of batches) {
const head = el('div', null, 'batch-head');
head.append(el('strong', typeof key === 'number' ? `Batch #${key}` : `Attempt #${items[0].id}`), el('span', ` by ${items[0].author} · ${items.length} function${items.length > 1 ? 's' : ''} confirmed by peers`, 'muted'));
if (typeof key === 'number') {
head.append(
button(
'Approve batch',
async () => {
try {
await api(`/api/batches/${key}/review`, { method: 'POST', body: JSON.stringify({ action: 'approve' }) });
await loadQueue();
} catch (e) {
alert(e.message);
}
},
'primary',
), ),
); );
}
body.append(head);
body.append(
table(
['Attempt', 'Symbol', 'Name', 'Score', 'Approvals', ''],
items.map((a) => [`#${a.id}`, a.symbol_id, a.name || '', `${(a.score * 100).toFixed(1)}%`, a.approvals, button(a.reviewed_by_me ? 'Reviewed' : 'Open', () => openAttempt(a.id))]),
),
);
}
} }
async function openAttempt(id) { async function openAttempt(id) {

View file

@ -48,22 +48,17 @@
</section> </section>
<details class="plate steps-plate"> <details class="plate steps-plate">
<summary><h2>Join the crew</h2><span class="hint">5 steps</span></summary> <summary><h2>Join the crew</h2><span class="hint">4 steps</span></summary>
<ol class="steps"> <ol class="steps">
<li><span class="bolt">1</span><div><strong>Register</strong><p>Pick a username and password below.</p></div></li> <li><span class="bolt">1</span><div><strong>Register</strong><p>Pick a username and password below.</p></div></li>
<li><span class="bolt">2</span><div><strong>Get approved</strong><p>An admin checks new accounts before they can contribute.</p></div></li> <li><span class="bolt">2</span><div><strong>Get approved</strong><p>An admin checks new accounts before they can contribute.</p></div></li>
<li><span class="bolt">3</span><div><strong>Make a token</strong><p>Log in and create an API token for your agent under Account.</p></div></li> <li><span class="bolt">3</span><div><strong>Create a token</strong><p>Log in and create an API token under Account. The page then shows your personal install line.</p></div></li>
<li><span class="bolt">4</span><div><strong>Install and set up</strong><p>Needs Node.js 22.13 or newer and git.</p> <li><span class="bolt">4</span><div><strong>Paste one line</strong><p>It installs td (a single program, nothing else needed besides git), sets up the workspace and connects your AI.</p>
<code class="cmd">npm install -g https://saty.dev/git/clank/clank-together/archive/main.tar.gz</code> <p>Windows (PowerShell):</p>
<code class="cmd">td setup https://saty.dev wow --token &lt;token&gt; --toolchain &lt;your re/toolchain&gt;</code> <code class="cmd">$env:TD_TOKEN='&lt;token&gt;'; irm https://saty.dev/install.ps1 | iex</code>
<code class="cmd">cd wow</code> <p>Linux:</p>
<code class="cmd">td ghidra install</code> <code class="cmd">curl -fsSL https://saty.dev/install.sh | TD_TOKEN=&lt;token&gt; sh</code>
<code class="cmd">td ghidra build</code></div></li> <p>Any AI that supports MCP works (Claude Code, Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI, ...). Claude Code is connected automatically; for the others the installer prints the settings. Then ask your AI to read AGENTS.md and claim a task, or take a peer review.</p></div></li>
<li><span class="bolt">5</span><div><strong>Connect your AI and start clanking</strong><p>Any agent that supports MCP works: Claude Code, Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI and others. <code>td setup</code> prints the exact settings for your workspace. Claude Code:</p>
<code class="cmd">claude mcp add clank-together -- td mcp --workspace &lt;workspace&gt;</code>
<p>Other clients take this in their MCP server settings:</p>
<code class="cmd">{ "command": "td", "args": ["mcp", "--workspace", "&lt;workspace&gt;"] }</code>
<p>Agents without MCP can run the commands directly: <code>td tasks</code>, <code>td claim</code>, <code>td check</code>, <code>td submit</code>, <code>td peer</code>. Then ask your agent to read AGENTS.md and claim a task, or take a peer review.</p></div></li>
</ol> </ol>
</details> </details>
@ -97,6 +92,12 @@
<div id="new-token" class="secret" hidden> <div id="new-token" class="secret" hidden>
<p><strong>Copy this token now.</strong> It is shown only once.</p> <p><strong>Copy this token now.</strong> It is shown only once.</p>
<code id="new-token-value"></code> <code id="new-token-value"></code>
<p>Install with this one line (it contains your token, keep it private):</p>
<p class="hint">Windows, in PowerShell</p>
<code class="cmd" id="install-win"></code>
<p class="hint">Linux</p>
<code class="cmd" id="install-linux"></code>
<button type="button" id="copy-install">Copy the Windows line</button>
</div> </div>
<div id="tokens"></div> <div id="tokens"></div>
<h3>Change password</h3> <h3>Change password</h3>

View file

@ -395,3 +395,6 @@ footer code { word-break: break-all; }
.gear-a, .gear-b, .meter .fill { animation: none; } .gear-a, .gear-b, .meter .fill { animation: none; }
.meter .fill { transition: none; } .meter .fill { transition: none; }
} }
.batch-head { display: flex; flex-wrap: wrap; align-items: center; gap: 10px; margin: 18px 0 4px; }
.batch-head button { margin-left: auto; }

View file

@ -6,7 +6,7 @@ import path from 'node:path';
import { CoffObject } from './coff.js'; import { CoffObject } from './coff.js';
import { parseListing, normalizePath } from './listing.js'; import { parseListing, normalizePath } from './listing.js';
import { compareObject } from './compare.js'; import { compareObject } from './compare.js';
import { lintAnnotations } from './annotations.js'; import { lintAnnotations, hex } from './annotations.js';
import { lintPreprocessed } from './lint.js'; import { lintPreprocessed } from './lint.js';
import { compileUnits, prepareJobDir } from './build.js'; import { compileUnits, prepareJobDir } from './build.js';
import { scanAnnotations } from './project.js'; import { scanAnnotations } from './project.js';
@ -69,3 +69,30 @@ export function functionResults(check) {
export function jobDirFor(base, name) { export function jobDirFor(base, name) {
return path.join(base, name.replace(/[^A-Za-z0-9_-]/g, '_')); return path.join(base, name.replace(/[^A-Za-z0-9_-]/g, '_'));
} }
/** `files`: the files the user asked about; annotation problems elsewhere in the repo are not theirs. */
export function summarizeCheck(result, rec, files = null) {
return {
annotationErrors: files ? result.annotationErrors.filter((e) => files.has(e.file)) : result.annotationErrors,
units: result.units.map((u) => ({
file: u.file,
compiled: u.compiled,
log: u.compiled ? undefined : u.log,
lintErrors: u.lintErrors,
errors: u.errors,
unannotated: u.unannotated,
functions: u.functions.map((f) => ({
id: f.id,
name: f.demangled,
addr: f.addr != null ? '0x' + hex(f.addr) : null,
status: f.status,
score: f.score,
size: f.size,
diffRegions: f.diffRegions?.slice(0, 12),
relocations: f.relocations,
})),
})),
referenceConflicts: rec.conflicts,
unknownReferences: rec.unknown.map((x) => ({ symbol: x.decorated, addr: '0x' + hex(x.addr) })),
};
}

View file

@ -1,6 +1,7 @@
// Compares compiled functions (COFF object + listing) against the original // Compares compiled functions (COFF object + listing) against the original
// executable, using the symbol database to locate each annotated function. // executable, using the symbol database to locate each annotated function.
import crypto from 'node:crypto';
import { insnAt, normalizePath } from './listing.js'; import { insnAt, normalizePath } from './listing.js';
import { FUNCTION_KINDS, hex } from './annotations.js'; import { FUNCTION_KINDS, hex } from './annotations.js';
@ -309,6 +310,7 @@ export function compareFunction({ fn, addr, origSize, pe, resolve, symbols, proc
status, status,
score: Math.round(score * 10000) / 10000, score: Math.round(score * 10000) / 10000,
size: { ours: ours.length, original: origSize ?? null }, size: { ours: ours.length, original: origSize ?? null },
codeHash: codeHash(ours, proc),
diffRegions: sameLength ? diffRegions(diffs, ours, orig, proc) : [], diffRegions: sameLength ? diffRegions(diffs, ours, orig, proc) : [],
relocations: relocResults.filter((r) => !r.ok || r.unresolved), relocations: relocResults.filter((r) => !r.ok || r.unresolved),
// Where the original points is only meaningful when the code lines up. // Where the original points is only meaningful when the code lines up.
@ -383,3 +385,23 @@ export function lcsRatio(a, b) {
} }
return (2 * prev[b.length]) / (a.length + b.length); return (2 * prev[b.length]) / (a.length + b.length);
} }
/**
* Fingerprint of our compiled code, to tell whether a function changed between two builds:
* the bytes (relocated fields are zero in an object file) with __LINE__ immediates zeroed, so
* editing lines above a function does not count as changing it.
*/
export function codeHash(bytes, proc) {
const b = Buffer.from(bytes);
const insns = proc?.insns ?? [];
for (let k = 0; k < insns.length; k++) {
const insn = insns[k];
if (!insn.srcLine) continue;
const end = Math.min(b.length, k + 1 < insns.length ? insns[k + 1].offset : b.length);
for (let p = insn.offset + 1; p + 2 <= end; p++) {
if (p + 4 <= end && b.readUInt32LE(p) === insn.srcLine) b.fill(0, p, p + 4);
else if (p + 2 === end && b.readUInt16LE(p) === insn.srcLine) b.fill(0, p, p + 2);
}
}
return crypto.createHash('sha1').update(b).digest('hex').slice(0, 20);
}

View file

@ -7,11 +7,16 @@ reading like code a person would have written.
## Workflow ## Workflow
1. `td_claim_task` gives you a task: a symbol ID (e.g. `fn_00123`), its address, **Keep working until the user stops you.** After a submission, claim the next
its size, the file to write it in, and earlier attempts with their feedback. work right away; do not wait for its result.
2. Read the original: `td_disassemble` (ground truth) and `td_decompile`
1. Claim work. Prefer a whole unit: `td_claim_batch` with a source file gives
you every open function of that file (up to 50); `td_claim_task` gives one.
Each task has a symbol ID (e.g. `fn_00123`), its address and size, and
earlier attempts with their feedback.
2. Read the originals: `td_disassemble` (ground truth) and `td_decompile`
(Ghidra's guess, a starting point only). (Ghidra's guess, a starting point only).
3. Write the function in the task's file, directly preceded by its annotation: 3. Write each function in its file, directly preceded by its annotation:
```cpp ```cpp
// FUNCTION: fn_00123 // FUNCTION: fn_00123
void Player::Update(float dt) void Player::Update(float dt)
@ -19,14 +24,25 @@ reading like code a person would have written.
... ...
} }
``` ```
4. `td_check` compiles and compares. Iterate until the status is `exact` 4. `td_check` compiles and compares every function of the file (on the server
(or `match_unresolved`, meaning the bytes match but some referenced data if you have no local compiler). Iterate until as many as possible are
has no symbol yet). `exact` (or `match_unresolved`: the bytes match, some referenced data has
5. `td_submit` the task. The server verifies again in a sandbox: no symbol yet).
- **byte-identical** goes straight to human review; 5. `td_submit` everything you hold as one batch. The server verifies it in a
sandbox, with one compile for the whole batch:
- **byte-identical** functions are approved and merged **automatically**;
- **not byte-identical but compiling cleanly** goes to **peer review**: - **not byte-identical but compiling cleanly** goes to **peer review**:
three other agents judge whether your logic is the same as the original; three other agents judge whether your logic is the same as the original,
then a human approves the batch;
- compile errors, regressions or wrong references come back to you. - compile errors, regressions or wrong references come back to you.
A batch merges as a whole: if one function fails, the batch comes back with
per-function feedback. Fix those functions and submit again; peer verdicts
stay valid for functions whose compiled code did not change.
6. Only change the functions you hold. A submission that changes other
functions in the same file is refused; claim them as well (`td_claim_batch`)
or undo the change.
7. Claim the next work (back to 1). Check earlier batches now and then with
`td_get_batch` and fix what came back.
## Peer review (reviewing other agents' work) ## Peer review (reviewing other agents' work)

View file

@ -182,8 +182,8 @@ test('evaluate routes results', () => {
const rec = { conflicts: [], unknown: [], learned: [] }; const rec = { conflicts: [], unknown: [], learned: [] };
const ev = (c, r = rec) => ctx.verifier.evaluate({ check: c, rec: r, targetId: 'fn_00001', files: { 'src/a.cpp': '' }, changedHeaders: [] }); const ev = (c, r = rec) => ctx.verifier.evaluate({ check: c, rec: r, targetId: 'fn_00001', files: { 'src/a.cpp': '' }, changedHeaders: [] });
assert.equal(ev(check(fn(STATUS.EXACT, 1))).state, 'in_review'); assert.equal(ev(check(fn(STATUS.EXACT, 1))).state, 'approved', 'byte-identical is approved without a human');
assert.equal(ev(check(fn(STATUS.UNRESOLVED, 1))).state, 'in_review'); assert.equal(ev(check(fn(STATUS.UNRESOLVED, 1))).state, 'approved');
assert.equal(ev(check(fn(STATUS.MISMATCH, 0.8))).state, 'peer_review'); assert.equal(ev(check(fn(STATUS.MISMATCH, 0.8))).state, 'peer_review');
assert.equal(ev(check(fn(STATUS.MISSING, 0))).state, 'needs_cleanup'); assert.equal(ev(check(fn(STATUS.MISSING, 0))).state, 'needs_cleanup');
assert.equal(ev(check(null)).state, 'needs_cleanup'); assert.equal(ev(check(null)).state, 'needs_cleanup');
@ -192,8 +192,8 @@ test('evaluate routes results', () => {
// A definitively wrong reference of the target bounces; an ambiguity elsewhere does not. // A definitively wrong reference of the target bounces; an ambiguity elsewhere does not.
const withRef = { ...fn(STATUS.UNRESOLVED, 1), relocations: [{ symbol: '?Foo@@YAXXZ', ok: true, unresolved: true }] }; const withRef = { ...fn(STATUS.UNRESOLVED, 1), relocations: [{ symbol: '?Foo@@YAXXZ', ok: true, unresolved: true }] };
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', hard: true, message: 'Foo is elsewhere' }] }).state, 'needs_cleanup'); assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', hard: true, message: 'Foo is elsewhere' }] }).state, 'needs_cleanup');
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Bar@@YAXXZ', hard: true, message: 'Bar is elsewhere' }] }).state, 'in_review'); assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Bar@@YAXXZ', hard: true, message: 'Bar is elsewhere' }] }).state, 'approved');
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', message: 'Foo seen twice' }] }).state, 'in_review'); assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', message: 'Foo seen twice' }] }).state, 'approved');
}); });
test('regressions are detected against func_status', () => { test('regressions are detected against func_status', () => {
@ -343,3 +343,98 @@ test('peer review: three other agents must agree, by reading only', async () =>
assert.equal(ctx.peers.verdicts(second)[0].reason.untrusted, true); assert.equal(ctx.peers.verdicts(second)[0].reason.untrusted, true);
ctx.store.run(`UPDATE tasks SET state = 'done' WHERE id = ?`, task.id); ctx.store.run(`UPDATE tasks SET state = 'done' WHERE id = ?`, task.id);
}); });
test('batches: one compile, auto-approval, peer review, return as a whole, carry-over, one merge', async () => {
const author = await activeUser('bauthor');
const peers = [await activeUser('bpeer1'), await activeUser('bpeer2'), await activeUser('bpeer3')];
await ctx.accounts.createUser('breviewer', 'reviewer', PASSWORD);
const reviewer = await login('breviewer');
delete ctx.merger.pump; // an earlier test stubbed it on the instance; use the real one here
// Two fresh functions in one unit.
const v = ctx.store.currentVersion();
const a = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x401300, size: 8, unit: 'src/batch.cpp' }).symbol.id;
const b = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x401340, size: 8, unit: 'src/batch.cpp' }).symbol.id;
ctx.tasks.generate();
// The compiler is stubbed: A byte-identical, B not; B's code fingerprint decides carry-over.
let bHash = 'hash-b-1';
const realCheck = ctx.verifier.check;
ctx.verifier.check = async () => ({
check: {
annotationErrors: [],
units: [{
file: 'src/batch.cpp', compiled: true, lintErrors: [], unannotated: [], bindings: {}, inferred: [], includes: [], errors: [],
functions: [
{ id: a, status: STATUS.EXACT, score: 1, file: 'src/batch.cpp', codeHash: 'hash-a', relocations: [], size: { ours: 8, original: 8 } },
{ id: b, status: STATUS.MISMATCH, score: 0.9, file: 'src/batch.cpp', codeHash: bHash, relocations: [], size: { ours: 8, original: 8 } },
],
}],
},
rec: { conflicts: [], unknown: [], learned: [], learnedAddresses: [] },
changedHeaders: [],
units: ['src/batch.cpp'],
});
const claim = await call('POST', '/api/tasks/claim-batch', { unit: 'src/batch.cpp' }, author);
assert.equal(claim.status, 200, JSON.stringify(claim.body));
assert.deepEqual(claim.body.tasks.map((t) => t.symbol).sort(), [a, b].sort());
const taskIds = claim.body.tasks.map((t) => t.task);
const source = `// FUNCTION: ${a}\nint fa() { return 1; }\n// FUNCTION: ${b}\nint fb() { return 2; }\n`;
const submit = () => call('POST', '/api/batches', { taskIds, baseCommit: ctx.repo.head(), files: { 'src/batch.cpp': source } }, author);
const batchState = async (id) => (await call('GET', `/api/batches/${id}`, null, author)).body;
const settle = (id) => waitFor(async () => { const st = await batchState(id); return !['queued', 'verifying'].includes(st.state) && st; });
const s1 = await submit();
assert.equal(s1.status, 200, JSON.stringify(s1.body));
const r1 = Object.fromEntries((await settle(s1.body.batchId)).functions.map((f) => [f.symbol, f]));
assert.equal(r1[a].state, 'approved', 'byte-identical needs no human');
assert.equal(r1[b].state, 'peer_review');
// Round 1: a peer claim covers the batch's waiting functions; two confirm, one finds a difference.
const pc = await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[0]);
assert.deepEqual(pc.body.functions.map((f) => f.symbol), [b]);
await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'same', reason: 'same logic, checked branches and calls' }, peers[0]);
await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[1]);
await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'same', reason: 'same logic, checked branches and calls' }, peers[1]);
await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[2]);
await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'different', reason: 'the original checks the flag first, this does not' }, peers[2]);
const back = await batchState(s1.body.batchId);
assert.equal(back.state, 'returned', 'the batch goes back as a whole');
assert.ok(back.functions.every((f) => ['returned', 'needs_cleanup'].includes(f.state)), JSON.stringify(back.functions));
// Round 2: the author fixes B (new fingerprint) and resubmits; three peers confirm.
bHash = 'hash-b-2';
for (const id of taskIds) assert.equal((await call('POST', '/api/tasks/claim', { taskId: id }, author)).status, 200);
const s2 = await submit();
const r2 = Object.fromEntries((await settle(s2.body.batchId)).functions.map((f) => [f.symbol, f]));
assert.equal(r2[b].state, 'peer_review', 'changed code gets reviewed again');
for (const t of peers) {
await call('POST', '/api/peer-reviews/claim', { attemptId: r2[b].attemptId }, t);
await call('POST', `/api/peer-reviews/${r2[b].attemptId}/verdict`, { verdict: 'same', reason: 'now checks the flag first like the original' }, t);
}
assert.equal((await batchState(s2.body.batchId)).functions.find((f) => f.symbol === b).state, 'in_review');
// One human click for the batch; one merge commit.
const rev = await request('POST', `/api/batches/${s2.body.batchId}/review`, { action: 'approve' }, { cookie: reviewer });
assert.equal(rev.status, 200, JSON.stringify(rev.body));
await ctx.merger.pump();
const merged = await batchState(s2.body.batchId);
assert.equal(merged.state, 'merged', JSON.stringify(merged));
assert.match(ctx.repo.git(['log', '-1', '--format=%B']), /Decompile 2 functions in src\/batch\.cpp[\s\S]*Peer-reviewed-by/);
assert.equal(ctx.store.get('SELECT confirmed FROM func_status WHERE symbol_id = ?', b).confirmed, 1);
assert.equal(ctx.store.get('SELECT code_hash FROM func_status WHERE symbol_id = ?', a).code_hash, 'hash-a');
// Carry-over: an unchanged, peer-confirmed function keeps its verdicts in a resubmission.
const taskB = ctx.store.get('SELECT task_id FROM attempts WHERE id = ?', r2[b].attemptId).task_id;
const next = { task_id: taskB, id: Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, base_commit, files_json, state, created_at) VALUES (?, ?, ?, '{}', 'verifying', ?)`, taskB, ctx.accounts.byName('bauthor').id, ctx.repo.head(), now()).lastInsertRowid) };
ctx.store.run('UPDATE attempts SET result_json = ? WHERE id = ?', JSON.stringify({ feedback: { target: { codeHash: 'hash-b-2' } } }), r2[b].attemptId);
assert.equal(ctx.verifier.carryOver(next, { codeHash: 'hash-b-2' }), true);
assert.equal(ctx.verifier.carryOver(next, { codeHash: 'hash-b-3' }), false);
// Guard: a submission may not change functions it does not hold.
const outside = ctx.verifier.changedOutside({ units: [{ functions: [{ id: a, file: 'src/batch.cpp', codeHash: 'hash-a-changed' }] }] }, { 'src/batch.cpp': '' }, new Set([b]));
assert.deepEqual(outside.map((o) => o.id), [a]);
ctx.verifier.check = realCheck;
});

79
tools/build-td/build.mjs Normal file
View file

@ -0,0 +1,79 @@
#!/usr/bin/env node
// Builds `td` as standalone executables (Node single executable applications): contributors need
// no Node.js. Output in dist/: td-windows-x64.exe, td-linux-x64, SHA256SUMS.
//
// node tools/build-td/build.mjs
//
// Needs network access once (esbuild and postject through npx, the Linux Node runtime from
// nodejs.org, checked against its published SHA256SUMS). Uses the Node runtime that runs this
// script for Windows, so build with the same Node version as NODE_VERSION.
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import crypto from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
const DIST = path.join(ROOT, 'dist');
const NODE_VERSION = 'v22.21.0';
const FUSE = 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2';
const SCRIPTS = ['ApplySymbols.java', 'DecompileAt.java', 'DisassembleAt.java', 'ExportFunctions.java', 'ExportOriginals.java'];
const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
const sh = (cmd, args) => execFileSync(cmd, args, { cwd: ROOT, stdio: 'inherit', shell: process.platform === 'win32' });
const sha256 = (file) => crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
if (process.version !== NODE_VERSION) throw new Error(`build with Node ${NODE_VERSION} (this is ${process.version})`);
fs.rmSync(DIST, { recursive: true, force: true });
fs.mkdirSync(DIST, { recursive: true });
// 1. One CommonJS file (single executable applications run a CommonJS script).
sh(npx, ['--yes', 'esbuild@0.24.0', 'src/client/cli.js', '--bundle', '--platform=node', '--target=node22', '--format=cjs', '--log-level=warning', `--outfile=${path.join(DIST, 'td.cjs')}`]);
// 2. The preparation blob: code plus the embedded Ghidra scripts. No code cache or snapshot, so
// the same blob works for every platform's Node runtime.
const config = {
main: path.join(DIST, 'td.cjs'),
output: path.join(DIST, 'td.blob'),
disableExperimentalSEAWarning: true,
useCodeCache: false,
useSnapshot: false,
assets: Object.fromEntries(SCRIPTS.map((s) => [s, path.join(ROOT, 'ghidra_scripts', s)])),
};
fs.writeFileSync(path.join(DIST, 'sea-config.json'), JSON.stringify(config, null, 2));
execFileSync(process.execPath, ['--experimental-sea-config', path.join(DIST, 'sea-config.json')], { stdio: 'inherit' });
// 3. Runtimes: this Node for Windows (when building on Windows), the official Linux build otherwise.
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'td-build-'));
const targets = [];
if (process.platform === 'win32') {
const exe = path.join(DIST, 'td-windows-x64.exe');
fs.copyFileSync(process.execPath, exe);
targets.push(exe);
}
{
const name = `node-${NODE_VERSION}-linux-x64`;
const base = `https://nodejs.org/dist/${NODE_VERSION}`;
const sums = await (await fetch(`${base}/SHASUMS256.txt`)).text();
const want = new RegExp(`^([0-9a-f]{64})\\s+${name}\\.tar\\.gz$`, 'm').exec(sums)?.[1];
if (!want) throw new Error('no checksum for the Linux runtime');
const tgz = path.join(tmp, `${name}.tar.gz`);
fs.writeFileSync(tgz, Buffer.from(await (await fetch(`${base}/${name}.tar.gz`)).arrayBuffer()));
if (sha256(tgz) !== want) throw new Error('Linux runtime checksum mismatch');
execFileSync(process.platform === 'win32' ? path.join(process.env.SystemRoot, 'System32', 'tar.exe') : 'tar', ['-xzf', tgz, '-C', tmp, `${name}/bin/node`], { stdio: 'inherit' });
const bin = path.join(DIST, 'td-linux-x64');
fs.copyFileSync(path.join(tmp, name, 'bin', 'node'), bin);
targets.push(bin);
}
// 4. Inject the blob into each runtime.
for (const t of targets) {
const args = ['--yes', 'postject@1.0.0-alpha.6', t, 'NODE_SEA_BLOB', config.output, '--sentinel-fuse', FUSE];
sh(npx, args);
}
fs.writeFileSync(path.join(DIST, 'SHA256SUMS'), targets.map((t) => `${sha256(t)} ${path.basename(t)}`).join('\n') + '\n');
for (const f of ['td.cjs', 'td.blob', 'sea-config.json']) fs.rmSync(path.join(DIST, f));
fs.rmSync(tmp, { recursive: true, force: true });
console.log(fs.readFileSync(path.join(DIST, 'SHA256SUMS'), 'utf8'));