From 125f5c4e7d587ff634405cafac82d98a76eb1880 Mon Sep 17 00:00:00 2001 From: Clank Together Date: Sun, 18 Oct 2026 02:13:09 +0200 Subject: [PATCH] Batches, auto-merge, standalone td, one-line install, self-update, server Ghidra - Batches: claim every open function of a unit or address range (td claim --unit/--range), submit the file once; one compile, per-function routing, merged as a whole. A failing function returns the batch; peer verdicts carry over to functions whose compiled code is unchanged (code fingerprints). Submissions may not change functions they do not hold. - Byte-identical functions are approved and merged without human review (matchPolicy.autoMergeIdentical); peer-confirmed ones need one click per batch. - Remote check: td check compiles on the server when there is no local toolchain. - /api/original: assembly and pseudocode of the original from the server's Ghidra export (objdump before an export exists); td disasm/decompile fall back to it. - td as a standalone executable (Node single executable application; tools/build-td), one-line installers (install.ps1 / install.sh) with checksum verification, td connect for any MCP client, td mcp --workspace, self-update (td update, hourly check). - Server-side Ghidra project: built once per version, refreshed after merges, downloadable for contributors (td ghidra download); JDK 21 and the pinned Ghidra in deploy/install.sh. - Agent instructions: work continuously, batches first, peer reviews in between. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RNwtXPYqFu3oKaaXwYwNWW --- .gitignore | 1 + deploy/install.sh | 22 ++- ghidra_scripts/ExportOriginals.java | 113 +++++++++++++++ src/client/cli.js | 92 +++++++++--- src/client/ghidra.js | 44 +++++- src/client/mcp.js | 54 ++++--- src/client/ops.js | 108 +++++++++----- src/client/update.js | 93 +++++++++++++ src/server/app.js | 209 +++++++++++++++++++++++++--- src/server/batches.js | 32 +++++ src/server/cli.js | 21 +++ src/server/db.js | 41 ++++++ src/server/ghidra.js | 137 ++++++++++++++++++ src/server/install.js | 111 +++++++++++++++ src/server/merge.js | 170 ++++++++++++++-------- src/server/original.js | 95 +++++++++++++ src/server/peer.js | 61 ++++++-- src/server/tasks.js | 91 +++++++++++- src/server/verify.js | 144 ++++++++++++++----- src/server/web/app.js | 54 ++++++- src/server/web/index.html | 27 ++-- src/server/web/style.css | 3 + src/shared/check.js | 29 +++- src/shared/compare.js | 22 +++ template/AGENTS.md | 36 +++-- test/server.test.js | 103 +++++++++++++- tools/build-td/build.mjs | 79 +++++++++++ 27 files changed, 1751 insertions(+), 241 deletions(-) create mode 100644 ghidra_scripts/ExportOriginals.java create mode 100644 src/client/update.js create mode 100644 src/server/batches.js create mode 100644 src/server/ghidra.js create mode 100644 src/server/install.js create mode 100644 src/server/original.js create mode 100644 tools/build-td/build.mjs diff --git a/.gitignore b/.gitignore index 2cdd7e7..37f3cac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules/ *.log # Operational notes about the deployment, not part of the tool PORTS-TO-FORWARD.md +dist/ diff --git a/deploy/install.sh b/deploy/install.sh index 068e2b2..5ab730e 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -5,6 +5,9 @@ set -eu HOST="${1:?usage: install.sh }" NODE_VERSION=v22.21.0 +GHIDRA_VERSION=11.2.1 +GHIDRA_URL=https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_11.2.1_build/ghidra_11.2.1_PUBLIC_20241105.zip +GHIDRA_SHA256=ce4db5117da0fbaf8f33863fec4f40902f754f06b68945a59fb1c0f9b1bc461c SRC=$(cd "$(dirname "$0")/.." && pwd) APP=/opt/togetherdecomp DATA=/var/lib/togetherdecomp @@ -12,7 +15,8 @@ DATA=/var/lib/togetherdecomp echo "== packages" export DEBIAN_FRONTEND=noninteractive apt-get update -q -apt-get install -y -q git curl ca-certificates nftables caddy docker.io xz-utils >/dev/null +# binutils: objdump disassembles original functions for /api/original before a Ghidra export exists. +apt-get install -y -q git curl ca-certificates nftables caddy docker.io xz-utils binutils >/dev/null echo "== node $NODE_VERSION" if [ "$(/opt/node/bin/node --version 2>/dev/null || true)" != "$NODE_VERSION" ]; then @@ -25,6 +29,19 @@ if [ "$(/opt/node/bin/node --version 2>/dev/null || true)" != "$NODE_VERSION" ]; rm -rf "$tmp" fi +echo "== ghidra $GHIDRA_VERSION (server-side project and pseudocode)" +# The JDK, not just the runtime: Ghidra compiles its .java scripts. +apt-get install -y -q openjdk-21-jdk-headless unzip >/dev/null +GHIDRA_HOME="/opt/ghidra/ghidra_${GHIDRA_VERSION}_PUBLIC" +if [ ! -x "$GHIDRA_HOME/support/analyzeHeadless" ]; then + tmp=$(mktemp -d) + curl -fsSL "$GHIDRA_URL" -o "$tmp/ghidra.zip" + echo "$GHIDRA_SHA256 $tmp/ghidra.zip" | sha256sum -c - >/dev/null + mkdir -p /opt/ghidra + unzip -q -o "$tmp/ghidra.zip" -d /opt/ghidra + rm -rf "$tmp" +fi + echo "== docker sandbox settings" mkdir -p /etc/docker cat > /etc/docker/daemon.json <<'EOF' @@ -53,13 +70,14 @@ if [ ! -f "$DATA/server.json" ]; then fi # Production settings, keeping anything else already configured. UIDGID="$(id -u togetherdecomp):$(id -g togetherdecomp)" -runuser -u togetherdecomp -- $NODE -e ' +runuser -u togetherdecomp -- env GHIDRA_HOME="$GHIDRA_HOME" $NODE -e ' const fs = require("fs"); const [file, host, user] = process.argv.slice(1); const c = JSON.parse(fs.readFileSync(file, "utf8")); c.listen = { host: "127.0.0.1", port: 8080 }; c.publicUrl = "https://" + host; c.trustProxy = true; c.allowUnsandboxedRunners = false; +c.ghidraHome = process.env.GHIDRA_HOME || c.ghidraHome || null; for (const r of Object.values(c.runners || {})) if (r.type === "docker") r.user = user; fs.writeFileSync(file, JSON.stringify(c, null, 2) + "\n"); ' "$DATA/server.json" "$HOST" "$UIDGID" diff --git a/ghidra_scripts/ExportOriginals.java b/ghidra_scripts/ExportOriginals.java new file mode 100644 index 0000000..1a6eb14 --- /dev/null +++ b/ghidra_scripts/ExportOriginals.java @@ -0,0 +1,113 @@ +// Exports the disassembly and the decompiler output of every function as JSON lines, for the +// coordinator's /api/original (contributors then need no local Ghidra): +// {"addr":"0x140099b90","asm":"...","c":"..."} +// Args: [decompile timeout seconds, default 30] +// Run on a project built with `td ghidra build` (symbol names applied), with -readOnly. +// @category ClankTogether + +import java.io.BufferedWriter; +import java.io.OutputStreamWriter; +import java.io.FileOutputStream; +import java.nio.charset.StandardCharsets; + +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.DecompileResults; +import ghidra.app.script.GhidraScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionIterator; +import ghidra.program.model.listing.Instruction; +import ghidra.program.model.listing.InstructionIterator; +import ghidra.program.model.symbol.Reference; +import ghidra.program.model.symbol.Symbol; + +public class ExportOriginals extends GhidraScript { + + @Override + protected void run() throws Exception { + String[] args = getScriptArgs(); + int timeout = args.length > 1 ? Integer.parseInt(args[1]) : 30; + DecompInterface ifc = new DecompInterface(); + ifc.openProgram(currentProgram); + int n = 0; + int failed = 0; + try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(args[0]), StandardCharsets.UTF_8))) { + FunctionIterator it = currentProgram.getFunctionManager().getFunctions(true); + while (it.hasNext() && !monitor.isCancelled()) { + Function f = it.next(); + if (f.isExternal() || f.isThunk()) { + continue; + } + String c; + DecompileResults r = ifc.decompileFunction(f, timeout, monitor); + if (r.decompileCompleted() && r.getDecompiledFunction() != null) { + c = r.getDecompiledFunction().getC(); + } else { + c = null; + failed++; + } + out.write("{\"addr\":\"0x" + Long.toHexString(f.getEntryPoint().getOffset()) + "\",\"asm\":" + json(disassemble(f)) + ",\"c\":" + (c == null ? "null" : json(c)) + "}\n"); + if (++n % 1000 == 0) { + println("exported " + n + " functions"); + } + } + } finally { + ifc.dispose(); + } + println("exported " + n + " functions, " + failed + " without decompiler output"); + } + + private String disassemble(Function f) { + StringBuilder sb = new StringBuilder(); + Address entry = f.getEntryPoint(); + sb.append("; ").append(f.getName(true)).append(" @ ").append(entry).append(" ").append(f.getSignature().getPrototypeString()).append('\n'); + InstructionIterator it = currentProgram.getListing().getInstructions(f.getBody(), true); + while (it.hasNext()) { + Instruction insn = it.next(); + Address a = insn.getAddress(); + if (!a.equals(entry)) { + for (Symbol s : currentProgram.getSymbolTable().getSymbols(a)) { + sb.append(s.getName(true)).append(":\n"); + } + } + StringBuilder bytes = new StringBuilder(); + try { + for (byte b : insn.getBytes()) { + bytes.append(String.format("%02x ", b & 0xff)); + } + } catch (Exception e) { + bytes.append("?? "); + } + StringBuilder refs = new StringBuilder(); + for (Reference ref : insn.getReferencesFrom()) { + Symbol s = currentProgram.getSymbolTable().getPrimarySymbol(ref.getToAddress()); + if (s != null) { + refs.append(" ; -> ").append(s.getName(true)); + } + } + sb.append(String.format(" +0x%04x %08x %-24s %s%s%n", a.subtract(entry), a.getOffset(), bytes.toString().trim(), insn.toString(), refs)); + } + return sb.toString(); + } + + private static String json(String s) { + StringBuilder sb = new StringBuilder("\""); + for (int i = 0; i < s.length(); i++) { + char ch = s.charAt(i); + switch (ch) { + case '"': sb.append("\\\""); break; + case '\\': sb.append("\\\\"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: + if (ch < 0x20) { + sb.append(String.format("\\u%04x", (int) ch)); + } else { + sb.append(ch); + } + } + } + return sb.append('"').toString(); + } +} diff --git a/src/client/cli.js b/src/client/cli.js index 33ccc3f..a288cc8 100644 --- a/src/client/cli.js +++ b/src/client/cli.js @@ -2,11 +2,13 @@ // `td` - contributor CLI for Clank Together. import path from 'node:path'; +import { spawnSync } from 'node:child_process'; import { parseArgs } from 'node:util'; import { Workspace } from './workspace.js'; import * as ops from './ops.js'; import * as ghidra from './ghidra.js'; import { startMcp } from './mcp.js'; +import { maybeUpdate } from './update.js'; import { hex } from '../shared/annotations.js'; const HELP = `td - Clank Together contributor client @@ -18,17 +20,23 @@ Setup [--toolchain ] compiler layout of the project (e.g. re/toolchain) [--vc ] [--sdk ] [--image ] td sync pull main and refresh the symbol database snapshot - td ghidra install | build | sync pinned Ghidra: install, (re)build the project, re-apply symbols + td ghidra download the server's Ghidra project (names applied, refreshed after merges) + td ghidra install | build | sync pinned Ghidra: install it, analyze locally, re-apply symbols + td update update td now (it also checks by itself once an hour) + td connect [--workspace ] connect your AI (registers with Claude Code, prints MCP settings) td mcp [--workspace ] run the MCP server for your agent (stdio) Work td tasks [--state S] [--limit N] list claimable tasks td claim [taskId] claim a task, prints its context + td claim --unit [--limit N] claim every open function of a source file (a batch) + td claim --range - [--limit N] claim every open function of an address range td task show a task's context td check [files...] compile + compare locally (default: changed files) - td submit [files...] [--notes T] submit for server verification + td submit [taskId...] [--notes T] submit everything you hold (or these tasks) as one batch + td batch state of a submitted batch, per function td result show a verification result - td heartbeat | td release + td heartbeat [taskId] | td release extend your leases (all without an id) td config [ghidraHome|server ] show or change workspace settings td symbol look up a symbol td symbol new <0xaddr> [--name N] get a stable ID for a new global/vtable/string/function @@ -59,6 +67,29 @@ function printCheck(r) { for (const c of r.referenceConflicts) console.log(`conflict: ${c.message ?? c}`); } +/** How an MCP client starts us: the td binary itself, or node + this script when run from source. */ +async function launcher() { + const sea = await import('node:sea').catch(() => null); + return sea?.isSea?.() ? { command: process.execPath, args: [] } : { command: process.execPath, args: [path.resolve(process.argv[1])] }; +} + +/** + * Registers the MCP server with Claude Code when it is installed, and prints the settings + * for every other MCP client. + */ +async function connect(dir) { + const ws = Workspace.find(dir); + const l = await launcher(); + const args = [...l.args, 'mcp', '--workspace', ws.dir]; + const claude = spawnSync(process.platform === 'win32' ? 'claude.cmd' : 'claude', ['mcp', 'add', '--scope', 'user', 'clank-together', '--', l.command, ...args], { encoding: 'utf8', shell: process.platform === 'win32' }); + if (claude.status === 0) console.log('Claude Code: connected (MCP server "clank-together"). Start claude and ask it to read AGENTS.md and claim a task.'); + else console.log('Claude Code: not found, or already connected (claude mcp list shows it).'); + console.log(`\nOther MCP clients (Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI, ...): add this to their MCP server settings: +${JSON.stringify({ mcpServers: { 'clank-together': { command: l.command, args } } }, null, 2)} + +Agents without MCP can run the td commands directly: td tasks, td claim, td check, td submit, td peer ...`); +} + async function main() { const { values, positionals } = parseArgs({ allowPositionals: true, @@ -74,8 +105,12 @@ async function main() { state: { type: 'string' }, limit: { type: 'string', default: '20' }, notes: { type: 'string' }, + unit: { type: 'string' }, + range: { type: 'string' }, + files: { type: 'string' }, reason: { type: 'string' }, json: { type: 'boolean', default: false }, + force: { type: 'boolean', default: false }, workspace: { type: 'string' }, help: { type: 'boolean', short: 'h' }, }, @@ -83,22 +118,24 @@ async function main() { const [cmd, ...args] = positionals; if (!cmd || values.help) return print(HELP); + // Self-update (standalone binary only): at most once an hour, never fatal. + const serverHint = cmd === 'setup' ? args[0] : (() => { try { return Workspace.find(values.workspace ?? process.cwd()).config.server; } catch { return null; } })(); + const updated = await maybeUpdate({ server: serverHint, force: cmd === 'update' }); + if (updated) process.stderr.write(`${updated}\n`); + if (cmd === 'update') return print(updated ?? 'td is up to date'); + if (cmd === 'setup') { const [server, dir] = args; if (!server || !dir) throw new Error('usage: td setup --token '); const ws = await ops.setup({ server, dir, token: values.token, exe: values.exe, toolchain: values.toolchain, runner: values.runner, vc: values.vc, sdk: values.sdk, image: values.image }); + const remote = ws.config.runners?.[ws.project.toolchain]?.type === 'remote'; console.log(`\nworkspace ready at ${ws.dir} -next: - cd ${path.relative(process.cwd(), ws.dir) || '.'} - td ghidra install && td ghidra build (one-time, takes a while) - td check src/... (self-test once there is code) -connect your agent (any MCP client works): - Claude Code: claude mcp add clank-together -- node "${path.resolve(process.argv[1])}" mcp --workspace "${ws.dir}" - other clients (Cursor, Windsurf, VS Code, Cline, Claude Desktop, ...), in their MCP server settings: -${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [path.resolve(process.argv[1]), 'mcp', '--workspace', ws.dir] } } }, null, 2).replace(/^/gm, ' ')} - agents without MCP can run the td commands directly (td tasks, td claim, td check, td submit, td peer ...).`); + compiling: ${remote ? "on the server (no local compiler needed); add one later with td setup ... --toolchain " : 'locally'} + original code: ${ws.config.ghidraHome ? 'local Ghidra' : "from the server; a local Ghidra is optional (td ghidra install && td ghidra build)"} +connect your AI with: td connect --workspace "${ws.dir}"`); return; } + if (cmd === 'connect') return connect(values.workspace ?? process.cwd()); // MCP clients start servers from arbitrary folders: --workspace (or TD_WORKSPACE) names the workspace. if (cmd === 'mcp') return startMcp(values.workspace ?? process.env.TD_WORKSPACE ?? process.cwd()); @@ -112,12 +149,20 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat console.log(`#${String(t.id).padEnd(6)} ${t.symbol_id} ${t.addr} size ${String(t.size ?? '?').padEnd(6)} d${t.difficulty} ${t.state.padEnd(13)} best ${(t.best_score * 100).toFixed(0)}% ${t.name ?? ''} ${t.unit ?? ''}`); } return; - case 'claim': + case 'claim': { + // td claim [taskId] | td claim --unit | td claim --range 0x140001000-0x140002000 [--limit N] + if (values.unit || values.range) { + const [start, end] = values.range ? values.range.split('-') : []; + return print(await ops.claimBatch(ws, { unit: values.unit ? ops.relPath(ws, values.unit) : undefined, start, end, limit: Number(values.limit) || 50 })); + } return print(await client.post('/api/tasks/claim', { taskId: args[0] ? Number(args[0]) : undefined })); + } case 'task': return print(await client.get(`/api/tasks/${args[0]}`)); case 'heartbeat': - return print(await client.post(`/api/tasks/${args[0]}/heartbeat`)); + return print(args[0] ? await client.post(`/api/tasks/${args[0]}/heartbeat`) : await client.post('/api/tasks/heartbeat-all')); + case 'batch': + return print(await client.get(`/api/batches/${args[0]}`)); case 'release': return print(await client.post(`/api/tasks/${args[0]}/release`)); case 'check': { @@ -125,8 +170,11 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat return values.json ? print(r) : printCheck(r); } case 'submit': { - const [taskId, ...files] = args; - return print(await ops.submit(ws, Number(taskId), { files: files.map((f) => ops.relPath(ws, f)), notes: values.notes, agentMeta: { client: 'td-cli' } })); + // td submit everything you hold, as one batch + // td submit ... [--files a.cpp,b.h] + const ids = args.filter((x) => /^\d+$/.test(x)).map(Number); + const files = values.files ? values.files.split(',').map((f) => ops.relPath(ws, f)) : []; + return print(await ops.submit(ws, ids, { files, notes: values.notes, agentMeta: { client: 'td-cli' } })); } case 'result': { const r = await client.get(`/api/attempts/${args[0]}`); @@ -166,13 +214,15 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat if (args[0] === 'install') return print(`Ghidra installed at ${await ghidra.installGhidra(ws)}`); if (args[0] === 'build') return void (await ghidra.buildGhidraProject(ws), print('Ghidra project built')); if (args[0] === 'sync') return void (await ghidra.syncGhidraProject(ws), print('symbols applied')); - throw new Error('usage: td ghidra install|build|sync'); + if (args[0] === 'download') { + const r = await ghidra.downloadGhidraProject(ws, { force: values.force }); + return print(r.updated ? `Ghidra project of ${r.info.version} downloaded${ws.config.ghidraHome ? '' : ' (open it with Ghidra 11.2.1, or run td ghidra install)'}` : 'the Ghidra project is up to date'); + } + throw new Error('usage: td ghidra install|download|build|sync'); case 'decompile': case 'disasm': { - const s = ws.findSymbol(args[0]); - const a = s?.addr ?? (/^0x/i.test(args[0] ?? '') ? Number.parseInt(args[0], 16) : null); - if (a == null) throw new Error(`unknown symbol ${args[0]}`); - return print(cmd === 'decompile' ? await ghidra.decompile(ws, a) : await ghidra.disassemble(ws, a)); + // Local Ghidra if configured, otherwise the server's export. + return print(await ops.original(ws, args[0], cmd === 'decompile' ? 'c' : 'asm')); } default: print(HELP); diff --git a/src/client/ghidra.js b/src/client/ghidra.js index eb38f20..975b2e8 100644 --- a/src/client/ghidra.js +++ b/src/client/ghidra.js @@ -13,7 +13,22 @@ import { hex } from '../shared/annotations.js'; import { loadToolchainLock } from '../shared/project.js'; import { Api } from './api.js'; -const SCRIPTS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts'); +const SCRIPT_NAMES = ['ApplySymbols.java', 'DecompileAt.java', 'DisassembleAt.java', 'ExportFunctions.java', 'ExportOriginals.java']; + +/** + * The Ghidra scripts: next to the sources when run from a checkout or npm install; extracted + * into the workspace when td runs as a single executable (they are embedded as assets). + */ +async function scriptsDir(ws) { + const sea = await import('node:sea').catch(() => null); + if (sea?.isSea?.()) { + const dir = path.join(ws.tdDir, 'ghidra_scripts'); + fs.mkdirSync(dir, { recursive: true }); + for (const name of SCRIPT_NAMES) fs.writeFileSync(path.join(dir, name), sea.getAsset(name, 'utf8')); + return dir; + } + return path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts'); +} const PROJECT_NAME = 'td'; const PROGRAM = 'target.exe'; @@ -33,7 +48,7 @@ async function runHeadless(ws, args, timeoutMs) { const exe = headless(ws); const projDir = path.join(ws.tdDir, 'ghidra'); fs.mkdirSync(projDir, { recursive: true }); - const full = [projDir, PROJECT_NAME, ...args, '-scriptPath', SCRIPTS_DIR]; + const full = [projDir, PROJECT_NAME, ...args, '-scriptPath', await scriptsDir(ws)]; // .bat files need a shell on Windows. const r = os.platform() === 'win32' ? await run('cmd.exe', ['/d', '/s', '/c', exe, ...full], { timeoutMs }) : await run(exe, full, { timeoutMs }); if (r.code !== 0) throw new Error(`Ghidra failed (exit ${r.code}):\n${r.output.slice(-4000)}`); @@ -104,3 +119,28 @@ export function decompile(ws, addr) { export function disassemble(ws, addr) { return query(ws, 'DisassembleAt.java', ['0x' + hex(addr)]); } + +/** + * Downloads the server's Ghidra project (built and refreshed on the server, with the shared symbol + * names) into the workspace, replacing the local one. Nothing happens when it is up to date. + */ +export async function downloadGhidraProject(ws, { force = false } = {}) { + const client = new Api(ws.config.server, ws.config.token); + const info = await client.get('/api/ghidra-project'); + if (!force && ws.config.ghidraPackage === info.sha256 && fs.existsSync(path.join(ws.tdDir, 'ghidra', `${PROJECT_NAME}.gpr`))) { + return { updated: false, info }; + } + const tgz = path.join(ws.tdDir, 'ghidra-project.tar.gz'); + console.error(`downloading the Ghidra project of ${info.version} (${(info.size / 1048576).toFixed(0)} MB)...`); + await client.download(info.download, tgz, info.sha256); + const projDir = path.join(ws.tdDir, 'ghidra'); + fs.rmSync(projDir, { recursive: true, force: true }); + fs.mkdirSync(projDir, { recursive: true }); + const tar = os.platform() === 'win32' ? path.join(process.env.SystemRoot ?? 'C:\Windows', 'System32', 'tar.exe') : 'tar'; + const r = await run(tar, ['-xzf', tgz, '-C', projDir], { timeoutMs: 1800_000 }); + if (r.code !== 0) throw new Error(`could not unpack the Ghidra project: ${r.output}`); + fs.rmSync(tgz, { force: true }); + ws.config.ghidraPackage = info.sha256; + ws.save(); + return { updated: true, info }; +} diff --git a/src/client/mcp.js b/src/client/mcp.js index a007c7a..55e4fe8 100644 --- a/src/client/mcp.js +++ b/src/client/mcp.js @@ -4,7 +4,6 @@ import readline from 'node:readline'; import { Workspace } from './workspace.js'; import * as ops from './ops.js'; -import * as ghidra from './ghidra.js'; import { hex } from '../shared/annotations.js'; const PROTOCOL_VERSIONS = ['2025-06-18', '2025-03-26', '2024-11-05']; @@ -33,9 +32,9 @@ const TOOLS = [ }, { name: 'td_heartbeat', - description: 'Extend the lease on a task you are still working on.', - inputSchema: { type: 'object', properties: { taskId: int('task id') }, required: ['taskId'] }, - run: async (ws, a) => ops.api(ws).post(`/api/tasks/${a.taskId}/heartbeat`), + description: 'Extend your leases while you are still working (all of them, or one taskId). Leases expire after two hours otherwise.', + inputSchema: { type: 'object', properties: { taskId: int('task id; omit for all') } }, + run: async (ws, a) => (a.taskId ? ops.api(ws).post(`/api/tasks/${a.taskId}/heartbeat`) : ops.api(ws).post('/api/tasks/heartbeat-all')), }, { name: 'td_release_task', @@ -47,13 +46,13 @@ const TOOLS = [ name: 'td_decompile', description: 'Ghidra decompiler output for a function in the original executable (by symbol ID, name or hex address). A starting point only; the result must be rewritten into matching C++.', inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] }, - run: async (ws, a) => ghidra.decompile(ws, resolveAddr(ws, a.symbol)), + run: async (ws, a) => ops.original(ws, a.symbol, 'c'), }, { name: 'td_disassemble', description: 'Disassembly of a function in the original executable, the ground truth to match.', inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] }, - run: async (ws, a) => ghidra.disassemble(ws, resolveAddr(ws, a.symbol)), + run: async (ws, a) => ops.original(ws, a.symbol, 'asm'), }, { name: 'td_lookup_symbol', @@ -82,9 +81,24 @@ const TOOLS = [ }, { name: 'td_submit', - description: 'Submit your work on a leased task for server verification. Defaults to all changed files the task allows. Ends the lease.', - inputSchema: { type: 'object', properties: { taskId: int('task id'), files: { type: 'array', items: { type: 'string' } }, notes: str('short notes for reviewers and later agents') }, required: ['taskId'] }, - run: async (ws, a) => ops.submit(ws, a.taskId, { files: a.files, notes: a.notes, agentMeta: { client: 'td-mcp' } }), + description: 'Submit your work for server verification, as one batch: every task you hold (default) or the given taskIds, with all changed files they allow. Byte-identical functions are merged automatically; others go to peer review. Ends the leases. Do not wait for the result: claim the next work right away and check results later.', + inputSchema: { + type: 'object', + properties: { taskIds: { type: 'array', items: { type: 'integer' }, description: 'tasks to submit; default: all you hold' }, taskId: int('a single task (same as taskIds: [id])'), files: { type: 'array', items: { type: 'string' } }, notes: str('short notes for reviewers and later agents') }, + }, + run: async (ws, a) => ops.submit(ws, a.taskIds ?? (a.taskId ? [a.taskId] : []), { files: a.files, notes: a.notes, agentMeta: { client: 'td-mcp' } }), + }, + { + name: 'td_claim_batch', + description: 'Claim many functions at once: every open function of a source file (unit), or of an address range of the current version, up to limit (default 50). Work through them in the file, td_check, then td_submit them together. Much cheaper than one function at a time.', + inputSchema: { type: 'object', properties: { unit: str('repo-relative source file, e.g. Engine/Source/FrameXML/LoadXML.cpp'), start: str('range start, hex'), end: str('range end, hex (exclusive)'), limit: int('max functions, default 50') } }, + run: async (ws, a) => ops.claimBatch(ws, a), + }, + { + name: 'td_get_batch', + description: 'State of a submitted batch, per function (approved, peer_review, in_review, needs_cleanup, merged...).', + inputSchema: { type: 'object', properties: { batchId: int('batch id') }, required: ['batchId'] }, + run: async (ws, a) => ops.api(ws).get(`/api/batches/${a.batchId}`), }, { name: 'td_get_result', @@ -132,12 +146,6 @@ const TOOLS = [ }, ]; -function resolveAddr(ws, q) { - const s = ws.findSymbol(q); - if (s?.addr != null) return s.addr; - if (/^0x[0-9a-f]+$/i.test(q)) return Number.parseInt(q, 16); - throw new Error(`unknown symbol ${q}`); -} export function startMcp(dir) { const ws = Workspace.find(dir); @@ -163,7 +171,7 @@ export function startMcp(dir) { protocolVersion: PROTOCOL_VERSIONS.includes(requested) ? requested : PROTOCOL_VERSIONS[0], capabilities: { tools: {} }, serverInfo: { name: 'clank-together', version: '0.1.0' }, - instructions: 'Tools for a collaborative matching decompilation. Read AGENTS.md in the workspace root first. Workflow: td_claim_task -> td_disassemble/td_decompile -> write C++ in the task file with the given annotation -> td_check until exact -> td_submit -> td_get_result. Also review the work of others: td_list_peer_reviews -> td_claim_peer_review -> read the original and the submission (no compiling or diffing) -> td_submit_peer_review.', + instructions: INSTRUCTIONS, }); } case 'ping': @@ -185,3 +193,17 @@ export function startMcp(dir) { } }); } + +// What every connected agent is told when it starts. Work continues until the user stops it. +const INSTRUCTIONS = [ + 'Clank Together: a collaborative matching decompilation. Read AGENTS.md in the workspace first.', + 'Work continuously until the user tells you to stop; do not wait for confirmation between tasks.', + 'Loop: td_claim_batch (a whole unit: every open function of one source file) or td_claim_task ->', + 'read the originals (td_disassemble, td_decompile) -> write the C++ with the given annotations ->', + 'td_check until as many functions as possible are byte-identical -> td_submit (everything you hold) ->', + 'immediately claim the next work. Do not wait for verification; look at results later with td_get_batch / td_get_result', + 'and fix what came back (needs_cleanup).', + 'Between batches, take a peer review when one is waiting: td_list_peer_reviews -> td_claim_peer_review ->', + 'read the original and the submission only (no compiling or diffing) -> td_submit_peer_review for every function.', + 'Call td_heartbeat every hour or so while working on a big batch.', +].join(' '); diff --git a/src/client/ops.js b/src/client/ops.js index 01523dc..28a4495 100644 --- a/src/client/ops.js +++ b/src/client/ops.js @@ -6,12 +6,13 @@ import path from 'node:path'; import { execFileSync } from 'node:child_process'; import { Api } from './api.js'; import { Workspace, sha256File } from './workspace.js'; -import { checkUnits } from '../shared/check.js'; +import { checkUnits, summarizeCheck } from '../shared/check.js'; import { reconcile } from '../shared/reconcile.js'; import { lintSubmission, matchesAny } from '../shared/lint.js'; import { isUnit, loadToolchainLock } from '../shared/project.js'; import { hex } from '../shared/annotations.js'; import { run } from '../shared/build.js'; +import { rememberServer } from './update.js'; export function api(ws) { return new Api(ws.config.server, ws.config.token); @@ -43,6 +44,7 @@ export async function setup({ server, dir, token, exe, runner, vc, sdk, image, t } const ws = new Workspace(abs); ws.config.server = server.replace(/\/+$/, ''); + rememberServer(ws.config.server); ws.config.token = useToken; ws.config.user = me.name; fs.mkdirSync(ws.tdDir, { recursive: true }); @@ -60,11 +62,11 @@ export async function setup({ server, dir, token, exe, runner, vc, sdk, image, t log(`target verified: sha256 ${info.version.sha256}`); const lock = loadToolchainLock(abs)[info.toolchain] ?? {}; - ws.config.runners = { [info.toolchain]: runnerConfig({ runner, vc, sdk, image: image ?? lock.image, toolchain, project: ws.project }) }; + ws.config.runners = { [info.toolchain]: runnerConfig({ runner, vc, sdk, image: image ?? (runner === 'docker' ? lock.image : undefined), toolchain, project: ws.project }) }; ws.save(); await refreshSymbols(ws); ws.config.envDigest = await computeEnvDigest(ws, info.toolchain); - if (lock.clSha256 && ws.config.envDigest !== lock.clSha256) { + if (lock.clSha256 && ws.config.envDigest && ws.config.envDigest !== lock.clSha256) { log(`WARNING: your cl.exe (sha256 ${ws.config.envDigest}) is not the pinned one (${lock.clSha256}); results may differ from the server`); } ws.save(); @@ -90,7 +92,10 @@ function runnerConfig({ runner, vc, sdk, image, toolchain, project }) { if (type === 'wine') return { type, clPath, wine: 'wine', toolchainRoot: root, jobs, env: { WINEDEBUG: '-all' } }; return { type, clPath, toolchainRoot: root, jobs, env: { PATH: `${path.dirname(clPath)};${process.env.PATH ?? ''}` } }; } + // Nothing local given: compile on the server (remote check). + if (!runner && !vc && !image) return { type: 'remote' }; const type = runner ?? (os.platform() === 'win32' && vc ? 'native' : 'docker'); + if (type === 'remote') return { type }; if (type === 'docker') { if (!image) throw new Error('no docker image configured; pass --image or set it in toolchain.lock.json'); return { type, image }; @@ -108,6 +113,7 @@ function runnerConfig({ runner, vc, sdk, image, toolchain, project }) { async function computeEnvDigest(ws, toolchain) { const r = ws.config.runners?.[toolchain]; if (!r) return null; + if (r.type === 'remote') return null; // the server's compiler is the pinned one if (r.type !== 'docker') return fs.existsSync(r.clPath) ? sha256File(r.clPath) : null; const out = await run('docker', ['run', '--rm', '--network', 'none', r.image, 'sha256sum', '/opt/msvc/bin/cl.exe'], { timeoutMs: 120_000 }); return /^[0-9a-f]{64}/.exec(out.output)?.[0] ?? null; @@ -130,7 +136,13 @@ export async function sync(ws) { pulled = false; } const snap = await refreshSymbols(ws); - return { head: ws.head(), fastForwarded: pulled, symbols: snap.symbols.length, note: pulled ? null : 'could not fast-forward (local commits or conflicting changes); merge origin/main yourself' }; + // A newer Ghidra project on the server is only announced (it is a large download). + let ghidra = null; + try { + const g = await api(ws).get('/api/ghidra-project'); + if (g.sha256 !== ws.config.ghidraPackage) ghidra = `a newer Ghidra project is available (${(g.size / 1048576).toFixed(0)} MB): td ghidra download`; + } catch {} + return { head: ws.head(), fastForwarded: pulled, symbols: snap.symbols.length, ghidra, note: pulled ? null : 'could not fast-forward (local commits or conflicting changes); merge origin/main yourself' }; } /** Compiles and compares locally with the same pipeline the server uses. */ @@ -139,6 +151,8 @@ export async function check(ws, files) { const targets = files?.length ? files : ws.changedFiles(); const units = targets.filter(isUnit); if (!units.length) throw new Error('nothing to check: pass .cpp files or modify some'); + const runner = ws.config.runners?.[project.toolchain]; + if (!runner || runner.type === 'remote') return remoteCheck(ws, targets); const result = await checkUnits({ repoDir: ws.dir, jobDir: path.join(ws.tdDir, 'build'), @@ -150,50 +164,72 @@ export async function check(ws, files) { runners: ws.config.runners, }); const rec = reconcile(result, ws.symbols); - return summarize(result, rec, new Set(targets)); + return summarizeCheck(result, rec, new Set(targets)); } -/** `files`: the files the user asked about; annotation problems elsewhere in the repo are not theirs. */ -export function summarize(result, rec, files = null) { - return { - annotationErrors: files ? result.annotationErrors.filter((e) => files.has(e.file)) : result.annotationErrors, - units: result.units.map((u) => ({ - file: u.file, - compiled: u.compiled, - log: u.compiled ? undefined : u.log, - lintErrors: u.lintErrors, - errors: u.errors, - unannotated: u.unannotated, - functions: u.functions.map((f) => ({ - id: f.id, - name: f.demangled, - addr: f.addr != null ? '0x' + hex(f.addr) : null, - status: f.status, - score: f.score, - size: f.size, - diffRegions: f.diffRegions?.slice(0, 12), - relocations: f.relocations, - })), - })), - referenceConflicts: rec.conflicts, - unknownReferences: rec.unknown.map((x) => ({ symbol: x.decorated, addr: '0x' + hex(x.addr) })), - }; +/** + * Compiles and compares on the server (its sandboxed compiler), for contributors without a + * local toolchain. Sends the changed files and the commit they are based on; nothing is submitted. + */ +async function remoteCheck(ws, targets) { + const files = {}; + for (const f of new Set([...targets, ...ws.changedFiles()])) { + const abs = path.join(ws.dir, f); + if (fs.existsSync(abs) && /\.(c|cc|cpp|cxx|h|hpp|inl)$/i.test(f)) files[f.replace(/\\/g, '/')] = fs.readFileSync(abs, 'utf8'); + } + const r = await api(ws).post('/api/check', { baseCommit: ws.head(), files }); + if (r.gate === 'rejected') { + const e = new Error('the static gate rejects this code:\n' + r.lintErrors.map((x) => ` ${x.file ?? ''}${x.line ? ':' + x.line : ''} ${x.message}`).join('\n')); + e.details = r.lintErrors; + throw e; + } + if (r.gate === 'conflict') throw new Error(`your changes conflict with newer work on main (${r.conflicts.join(', ')}); run td sync and merge`); + return { ...r, compiledOn: 'server' }; +} + +/** + * The original function as text. A local Ghidra (td ghidra install/build) answers locally; + * otherwise the server's Ghidra export does. + */ +export async function original(ws, query, kind) { + if (ws.config.ghidraHome) { + const { decompile, disassemble } = await import('./ghidra.js'); + const s = ws.findSymbol(query); + const addr = s?.addr ?? (/^0x[0-9a-f]+$/i.test(query) ? Number.parseInt(query, 16) : null); + if (addr == null) throw new Error(`unknown symbol ${query}`); + return kind === 'asm' ? disassemble(ws, addr) : decompile(ws, addr); + } + const r = await api(ws).get(`/api/original/${encodeURIComponent(query)}`); + if (kind === 'asm') return r.asm; + return r.pseudocode ?? `// no pseudocode for ${r.symbol ?? query} yet (${r.source}); the assembly is available with td disasm`; } /** Submits files for a leased task. Defaults to all changed files the task allows. */ -export async function submit(ws, taskId, { files, notes, agentMeta } = {}) { - const ctx = await api(ws).get(`/api/tasks/${taskId}`); - const candidates = files?.length ? files : ws.changedFiles().filter((f) => matchesAny(f, ctx.allowedPaths)); - if (!candidates.length) throw new Error(`no changed files match the task's allowed paths (${ctx.allowedPaths.join(', ')})`); +/** + * Submits work. `taskIds` empty = every task you hold, as one batch (a whole unit you claimed + * with claim-batch, or several single tasks). Defaults to all changed files those tasks allow. + */ +export async function submit(ws, taskIds = [], { files, notes, agentMeta } = {}) { + const me = await api(ws).get('/api/me'); + const ids = taskIds.length ? taskIds.map(Number) : me.leases; + if (!ids.length) throw new Error('you hold no tasks; claim one (td claim) or a whole unit (td claim --unit )'); + const allowed = taskIds.length ? [...new Set((await Promise.all(ids.map((id) => api(ws).get(`/api/tasks/${id}`)))).flatMap((c) => c.allowedPaths))] : me.leasedAllowedPaths; + const candidates = files?.length ? files : ws.changedFiles().filter((f) => matchesAny(f, allowed)); + if (!candidates.length) throw new Error(`no changed files match the allowed paths of your tasks (${allowed.join(', ')})`); const contents = {}; for (const f of candidates) contents[f.replace(/\\/g, '/')] = fs.readFileSync(path.join(ws.dir, f), 'utf8'); - const errors = lintSubmission(contents, { allowedPaths: ctx.allowedPaths, limits: ws.project.lintLimits, allowedPragmas: ws.project.allowedPragmas }); + const errors = lintSubmission(contents, { allowedPaths: allowed, limits: ws.project.lintLimits, allowedPragmas: ws.project.allowedPragmas }); if (errors.length) { const e = new Error('submission fails the static gate:\n' + errors.map((x) => ` ${x.file ?? ''}${x.line ? ':' + x.line : ''} ${x.message}`).join('\n')); e.details = errors; throw e; } - return api(ws).post(`/api/tasks/${taskId}/submit`, { baseCommit: ws.head(), files: contents, notes, agentMeta, envDigest: ws.config.envDigest }); + return api(ws).post('/api/batches', { taskIds: ids, baseCommit: ws.head(), files: contents, notes, agentMeta, envDigest: ws.config.envDigest }); +} + +/** Claims every open function of a unit (source file) or an address range, as one batch. */ +export function claimBatch(ws, { unit, start, end, limit } = {}) { + return api(ws).post('/api/tasks/claim-batch', { unit, start, end, limit }); } export function relPath(ws, f) { diff --git a/src/client/update.js b/src/client/update.js new file mode 100644 index 0000000..870e907 --- /dev/null +++ b/src/client/update.js @@ -0,0 +1,93 @@ +// Self-update of the td binary (only when td runs as the standalone executable; a checkout or +// npm install updates through git/npm). At most once an hour td compares its own SHA-256 with +// the server's /download/SHA256SUMS; a newer build is downloaded, verified and swapped in for +// the next start. TD_NO_UPDATE=1 turns it off. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; + +const CHECK_EVERY_MS = 3600_000; +const STATE_DIR = path.join(os.homedir(), '.clank'); +const STATE_FILE = path.join(STATE_DIR, 'update.json'); +const ASSET = process.platform === 'win32' ? 'td-windows-x64.exe' : 'td-linux-x64'; + +function readState() { + try { + return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')); + } catch { + return {}; + } +} + +function writeState(s) { + try { + fs.mkdirSync(STATE_DIR, { recursive: true }); + fs.writeFileSync(STATE_FILE, JSON.stringify(s)); + } catch {} +} + +/** Remembers the server for updates of a td that is used outside a workspace. */ +export function rememberServer(server) { + writeState({ ...readState(), server }); +} + +async function ownHash(exe, state) { + const st = fs.statSync(exe); + const key = `${st.size}:${st.mtimeMs}`; + if (state.selfKey === key && state.selfHash) return state.selfHash; + const hash = crypto.createHash('sha256').update(fs.readFileSync(exe)).digest('hex'); + state.selfKey = key; + state.selfHash = hash; + return hash; +} + +/** + * Checks for and installs a newer td. Returns a message when an update was installed (it takes + * effect on the next start), otherwise null. Never throws: an update must not break a command. + */ +export async function maybeUpdate({ server, force = false, log = (m) => process.stderr.write(m + '\n') } = {}) { + try { + if (process.env.TD_NO_UPDATE === '1') return null; + const sea = await import('node:sea').catch(() => null); + if (!sea?.isSea?.()) return null; + const exe = process.execPath; + // Leftover from a Windows update (a running .exe can be renamed, not replaced). + fs.rmSync(exe + '.old', { force: true }); + + const state = readState(); + const base = (server ?? state.server)?.replace(/\/+$/, ''); + if (!base) return null; + if (!force && state.checkedAt && Date.now() - state.checkedAt < CHECK_EVERY_MS) return null; + state.checkedAt = Date.now(); + + const res = await fetch(`${base}/download/SHA256SUMS`, { signal: AbortSignal.timeout(10_000) }); + if (!res.ok) return writeState(state), null; + const want = new RegExp(`^([0-9a-f]{64})\\s+${ASSET.replace('.', '\\.')}$`, 'm').exec(await res.text())?.[1]; + if (!want || want === (await ownHash(exe, state))) return writeState(state), null; + + log(`td: downloading an update from ${base} ...`); + const bin = await fetch(`${base}/download/${ASSET}`, { signal: AbortSignal.timeout(600_000) }); + if (!bin.ok) return writeState(state), null; + const data = Buffer.from(await bin.arrayBuffer()); + if (crypto.createHash('sha256').update(data).digest('hex') !== want) { + log('td: the update failed its checksum check and was not installed'); + return writeState(state), null; + } + const next = exe + '.new'; + fs.writeFileSync(next, data, { mode: 0o755 }); + if (process.platform === 'win32') { + fs.renameSync(exe, exe + '.old'); + fs.renameSync(next, exe); + } else { + fs.renameSync(next, exe); + } + delete state.selfKey; + delete state.selfHash; + writeState(state); + return 'td was updated; the new version runs from the next command on'; + } catch { + return null; + } +} diff --git a/src/server/app.js b/src/server/app.js index f65bdab..72d9fd6 100644 --- a/src/server/app.js +++ b/src/server/app.js @@ -11,10 +11,14 @@ import { Tasks } from './tasks.js'; import { Verifier } from './verify.js'; import { Merger } from './merge.js'; import { PeerReviews } from './peer.js'; +import { Originals } from './original.js'; +import { downloadsDir, installPs1, installSh } from './install.js'; +import { ServerGhidra } from './ghidra.js'; import { Router, HttpError, readJson, sendJson, RateLimiter } from './http.js'; import { PEImage } from '../shared/pe.js'; import { loadProject, loadToolchainLock } from '../shared/project.js'; -import { DEFAULT_LIMITS } from '../shared/lint.js'; +import { DEFAULT_LIMITS, lintSubmission } from '../shared/lint.js'; +import { summarizeCheck } from '../shared/check.js'; import { hex } from '../shared/annotations.js'; const WEB_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'web'); @@ -30,6 +34,8 @@ export const DEFAULT_CONFIG = { runners: { msvc2010: { type: 'docker', image: 'togetherdecomp/msvc2010:1', memory: '1g', cpus: 1 } }, allowUnsandboxedRunners: false, pushRemote: null, + ghidraHome: null, + ghidraRefreshMinutes: 30, distributeTarget: false, gitCloneUrl: null, gitWebUrl: null, @@ -66,16 +72,20 @@ export function openServer(dataDir, { log = console } = {}) { } const tasks = new Tasks({ store, config, repo, project }); - const verifier = new Verifier({ store, config, repo, project, tasks, getPe, log }); const peers = new PeerReviews({ store, project }); + const verifier = new Verifier({ store, config, repo, project, tasks, getPe, peers, log }); const merger = new Merger({ store, config, repo, project, tasks, verifier, getPe, peers, log }); const accounts = new Accounts(store); - return { config, store, repo, project, tasks, verifier, merger, getPe, accounts, peers }; + const originals = new Originals({ store }); + const ghidra = new ServerGhidra({ config, store, originals, log }); + merger.onMerged = (commit) => ghidra.scheduleRefresh(commit); + return { config, store, repo, project, tasks, verifier, merger, getPe, accounts, peers, originals, ghidra }; } export function createApp(ctx) { const { config, store, repo, project, tasks, merger, accounts, peers } = ctx; const router = new Router(); + downloadsRoot = downloadsDir(config); const submitLimiter = new RateLimiter(config.submitsPerHour, 3600_000); const symbolLimiter = new RateLimiter(120, 3600_000); const authFailLimiter = new RateLimiter(60, 600_000); @@ -203,7 +213,8 @@ export function createApp(ctx) { router.on('GET', '/api/me', ({ req }) => { const user = auth(req); - return { name: user.name, role: user.role, via: user.via, leases: tasks.leasesOf(user.id) }; + const leases = tasks.leasesOf(user.id); + return { name: user.name, role: user.role, via: user.via, leases, leasedAllowedPaths: leases.length ? tasks.allowedPathsForTasks(leases) : [] }; }); router.on('POST', '/api/me/password', async ({ req }) => { @@ -265,22 +276,85 @@ export function createApp(ctx) { submitLimiter.check(user.id); const taskId = intParam(params.id); const body = await readJson(req); - const t = tasks.requireLease(user, taskId); + const r = submitBatch(user, [taskId], body); + return { attemptId: r.attempts[0].attemptId, batchId: r.batchId, state: 'queued' }; + }); + + /** Validates a submission body and queues it as one batch for the given tasks. */ + const submitBatch = (user, taskIds, body) => { if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash'); if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) throw new HttpError(400, 'files must be an object of path -> content'); - const notes = body.notes ? String(body.notes).slice(0, DEFAULT_LIMITS.maxNotesBytes) : null; - const meta = body.agentMeta ? JSON.stringify(body.agentMeta).slice(0, 1000) : null; - const id = store.tx(() => { - const r = store.run( - `INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', ?)`, - t.id, user.id, body.baseCommit, JSON.stringify(body.files), notes, meta, body.envDigest ? String(body.envDigest).slice(0, 200) : null, now(), - ); - store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), t.id); - store.event('submit', user.id, { task: t.id, attempt: Number(r.lastInsertRowid) }); - return Number(r.lastInsertRowid); + const r = tasks.submitBatch(user, { + taskIds, + baseCommit: body.baseCommit, + files: body.files, + notes: body.notes ? String(body.notes).slice(0, DEFAULT_LIMITS.maxNotesBytes) : null, + agentMeta: body.agentMeta ? JSON.stringify(body.agentMeta).slice(0, 1000) : null, + envDigest: body.envDigest ? String(body.envDigest).slice(0, 200) : null, }); ctx.verifier.pump(); - return { attemptId: id, state: 'queued' }; + return r; + }; + + // Bulk work: claim every open function of a unit (source file) or an address range, submit the + // file once. Verified with one compile; merged as a whole. + router.on('POST', '/api/tasks/claim-batch', async ({ req }) => { + const user = auth(req); + const body = await readJson(req, 4096); + const hexOrNull = (x) => (x == null ? null : typeof x === 'string' ? Number.parseInt(x, 16) : Number(x)); + const ids = tasks.claimBatch(user, { unit: body.unit ? String(body.unit) : null, start: hexOrNull(body.start), end: hexOrNull(body.end), limit: body.limit ? Number(body.limit) : 50 }); + const v = store.currentVersion(); + const ph = ids.map(() => '?').join(','); + const rows = store.all( + `SELECT t.id AS task, s.id AS symbol, s.name, s.unit, a.addr, a.size, t.state, t.best_score FROM tasks t JOIN symbols s ON s.id = t.symbol_id + LEFT JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? WHERE t.id IN (${ph}) ORDER BY a.addr`, + v.id, ...ids, + ); + return { + tasks: rows.map((r) => ({ ...r, addr: r.addr != null ? '0x' + hex(r.addr) : null, annotation: `// FUNCTION: ${r.symbol}` })), + allowedPaths: tasks.allowedPathsForTasks(ids), + baseCommit: repo.head(), + note: 'Write or fix every function, check, then submit them together (td submit --all). Context of one task: GET /api/tasks/.', + }; + }); + + router.on('POST', '/api/batches', async ({ req }) => { + const user = auth(req); + submitLimiter.check(user.id); + const body = await readJson(req, 8 * 1024 * 1024); + const r = submitBatch(user, (body.taskIds ?? []).map((x) => intParam(x)), body); + return { ...r, state: 'queued' }; + }); + + router.on('GET', '/api/batches/:id', ({ req, params }) => { + auth(req); + const b = store.get('SELECT b.*, u.name AS author FROM batches b JOIN users u ON u.id = b.user_id WHERE b.id = ?', intParam(params.id)); + if (!b) throw new HttpError(404, 'no such batch'); + const result = b.result_json ? JSON.parse(b.result_json) : {}; + const attempts = store.all( + `SELECT a.id, a.task_id, t.symbol_id, s.name, a.state, a.score, a.result_json FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id WHERE a.batch_id = ? ORDER BY a.id`, + b.id, + ); + return { + id: b.id, + author: b.author, + state: b.state, + baseCommit: b.base_commit, + mergedCommit: b.merged_commit, + returned: result.returned ?? null, + mergeFailure: result.mergeFailure ?? null, + changedHeaders: result.changedHeaders ?? [], + changedOutside: result.outside ?? [], + functions: attempts.map((a) => { + const fb = a.result_json ? JSON.parse(a.result_json).feedback : null; + return { attemptId: a.id, taskId: a.task_id, symbol: a.symbol_id, name: a.name, state: a.state, score: a.score, summary: fb?.summary ?? null, status: fb?.target?.status ?? null }; + }), + }; + }); + + router.on('POST', '/api/tasks/heartbeat-all', ({ req }) => { + const user = auth(req); + return tasks.heartbeatAll(user); }); router.on('GET', '/api/attempts/:id', ({ req, params }) => { @@ -288,6 +362,8 @@ export function createApp(ctx) { const a = store.get('SELECT a.*, u.name AS author FROM attempts a JOIN users u ON u.id = a.user_id WHERE a.id = ?', intParam(params.id)); if (!a) throw new HttpError(404, 'no such attempt'); const result = a.result_json ? JSON.parse(a.result_json) : null; + const batch = a.batch_id ? store.get('SELECT result_json FROM batches WHERE id = ?', a.batch_id) : null; + const batchResult = batch?.result_json ? JSON.parse(batch.result_json) : {}; const isReviewer = hasRole(user, 'reviewer'); return { id: a.id, @@ -300,10 +376,11 @@ export function createApp(ctx) { finishedAt: a.finished_at, notes: a.notes, feedback: result?.feedback ?? null, - mergeFailure: result?.mergeFailure ?? null, - changedHeaders: result?.changedHeaders ?? [], - files: JSON.parse(a.files_json), - mergedFiles: isReviewer ? result?.mergedFiles ?? null : undefined, + batchId: a.batch_id, + mergeFailure: batchResult.mergeFailure ?? result?.mergeFailure ?? null, + changedHeaders: batchResult.changedHeaders ?? result?.changedHeaders ?? [], + files: store.attemptFiles(a), + mergedFiles: isReviewer ? batchResult.mergedFiles ?? result?.mergedFiles ?? null : undefined, reviews: store.all('SELECT r.action, r.comment, r.created_at, u.name AS reviewer FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ?', a.id), peerReviews: peers.verdicts(a.id), peerRequired: peers.required, @@ -334,6 +411,67 @@ export function createApp(ctx) { return { id: r.symbol.id, kind: r.symbol.kind, created: r.created, annotation: `// ${{ function: 'FUNCTION', global: 'GLOBAL', vtable: 'VTABLE', string: 'STRING' }[kind]}: ${r.symbol.id}` }; }); + // --- the original function (assembly, pseudocode) for contributors without a local Ghidra. + // Derived from the exe, so approved accounts only, like the exe itself. + router.on('GET', '/api/original/:symbol', async ({ req, params }) => { + auth(req); + return ctx.originals.get(params.symbol); + }); + + // --- the server's Ghidra project of the current version (td ghidra download). Derived from the + // exe, so approved accounts only. + router.on('GET', '/api/ghidra-project', ({ req }) => { + auth(req); + const info = ctx.ghidra.info(); + if (!info) throw new HttpError(404, 'no Ghidra project has been built on this server yet'); + return { ...info, download: `${config.publicUrl}/api/ghidra-project/download` }; + }); + + router.on('GET', '/api/ghidra-project/download', ({ req, res }) => { + auth(req); + const file = ctx.ghidra.packagePath(); + if (!file || !fs.existsSync(file)) throw new HttpError(404, 'no Ghidra project has been built on this server yet'); + res.writeHead(200, { 'content-type': 'application/gzip', 'content-length': fs.statSync(file).size, 'content-disposition': 'attachment; filename="ghidra-project.tar.gz"' }); + fs.createReadStream(file).pipe(res); + return undefined; + }); + + // --- remote check: compile and compare in the server's sandbox without submitting + // (contributors without a local MSVC toolchain). Same static gate as a submission. + + const checkLimiter = new RateLimiter(config.checksPerHour ?? 60, 3600_000); + let checksRunning = 0; + const checkWaiters = []; + const withCheckSlot = async (fn) => { + const max = config.remoteCheckConcurrency ?? 2; + if (checksRunning >= max) await new Promise((r) => checkWaiters.push(r)); + checksRunning++; + try { + return await fn(); + } finally { + checksRunning--; + checkWaiters.shift()?.(); + } + }; + + router.on('POST', '/api/check', async ({ req }) => { + const user = auth(req); + checkLimiter.check(user.id); + const body = await readJson(req); + if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash'); + if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) throw new HttpError(400, 'files must be an object of path -> content'); + const allowed = [...project.sourceDirs.map((d) => `${d}/**`)]; + const lintErrors = lintSubmission(body.files, { allowedPaths: allowed, limits: project.lintLimits, allowedPragmas: project.allowedPragmas }); + if (lintErrors.length) return { gate: 'rejected', lintErrors }; + if (!repo.isAncestor(body.baseCommit)) throw new HttpError(409, 'your base commit is not part of main; run td sync'); + const merged = repo.mergeOntoHead(body.baseCommit, body.files); + if (merged.conflicts.length) return { gate: 'conflict', conflicts: merged.conflicts }; + return withCheckSlot(async () => { + const { check, rec } = await ctx.verifier.check(merged.files, `check-${user.id}-${Date.now()}`); + return { gate: 'ok', head: repo.head(), ...summarizeCheck(check, rec, new Set(Object.keys(merged.files))) }; + }); + }); + // --- peer review (agents, API token or session) router.on('GET', '/api/peer-reviews', ({ req, query }) => { @@ -369,7 +507,7 @@ export function createApp(ctx) { router.on('GET', '/api/review/queue', ({ req }) => { const user = auth(req, { role: 'reviewer', session: true }); return store.all( - `SELECT a.id, a.task_id, a.score, a.created_at, a.finished_at, u.name AS author, t.symbol_id, s.name, + `SELECT a.id, a.batch_id, a.task_id, a.score, a.created_at, a.finished_at, u.name AS author, t.symbol_id, s.name, (SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.action = 'approve') AS approvals, (SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.reviewer_id = ?) AS reviewed_by_me FROM attempts a JOIN users u ON u.id = a.user_id JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id @@ -386,6 +524,15 @@ export function createApp(ctx) { return r; }); + // One click for a whole batch: every function of it that waits for a human. + router.on('POST', '/api/batches/:id/review', async ({ req, params }) => { + const user = auth(req, { role: 'reviewer', session: true }); + const body = await readJson(req, 16 * 1024); + const r = merger.reviewBatch(user, intParam(params.id), String(body.action ?? ''), body.comment ? String(body.comment) : null); + setImmediate(() => merger.pump()); + return r; + }); + // --- maintainers and admins (browser session only) router.on('POST', '/api/admin/tasks/:id', async ({ req, params }) => { @@ -425,6 +572,13 @@ export function createApp(ctx) { if (url.pathname.startsWith('/repo.git/')) { return serveGit(req, res, { projectRoot: config.dataDir, subPath: '/repo' + url.pathname.slice('/repo.git'.length), query: url.searchParams }); } + if (req.method === 'GET' && (url.pathname === '/install.ps1' || url.pathname === '/install.sh')) { + const script = url.pathname.endsWith('.ps1') ? installPs1(config, project) : installSh(config, project); + if (!script) throw new HttpError(503, 'the td binaries are not published on this server yet'); + res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-cache', 'x-content-type-options': 'nosniff' }); + return res.end(script); + } + if (req.method === 'GET' && url.pathname.startsWith('/download/')) return serveDownload(url.pathname.slice(10), res); if (req.method === 'GET' && !url.pathname.startsWith('/api/')) return serveStatic(url.pathname, res); const m = router.match(req.method, url.pathname); if (!m) throw new HttpError(404, 'not found'); @@ -442,6 +596,19 @@ export function createApp(ctx) { return http.createServer(handler); } +// Public downloads of the td binaries (tools/build-td output copied to /downloads). +const DOWNLOADS = new Set(['td-windows-x64.exe', 'td-linux-x64', 'SHA256SUMS']); +let downloadsRoot = null; +function serveDownload(name, res) { + const file = DOWNLOADS.has(name) && downloadsRoot ? path.join(downloadsRoot, name) : null; + if (!file || !fs.existsSync(file)) { + res.writeHead(404, { 'content-type': 'text/plain' }); + return res.end('not found'); + } + res.writeHead(200, { 'content-type': name === 'SHA256SUMS' ? 'text/plain' : 'application/octet-stream', 'content-length': fs.statSync(file).size, 'content-disposition': `attachment; filename="${name}"`, 'x-content-type-options': 'nosniff' }); + fs.createReadStream(file).pipe(res); +} + const STATIC = { '/': 'index.html', '/review': 'index.html', '/app.js': 'app.js', '/style.css': 'style.css', '/favicon.svg': 'favicon.svg' }; const TYPES = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css; charset=utf-8', '.svg': 'image/svg+xml' }; diff --git a/src/server/batches.js b/src/server/batches.js new file mode 100644 index 0000000..922c188 --- /dev/null +++ b/src/server/batches.js @@ -0,0 +1,32 @@ +// A batch merges as a whole. When one of its attempts fails (verification, peer review, human +// review or the merge re-check), the whole batch goes back to its author: every other attempt +// that was still on its way is marked returned, and its task becomes claimable again (reserved +// for the author for a while). Peer verdicts stay attached to their attempts, so a resubmission +// whose compiled code is unchanged for a function carries them over (Verifier.carryOver). + +import { now } from './db.js'; + +export function returnBatch(store, batchId, reason) { + if (!batchId) return; + const b = store.get('SELECT * FROM batches WHERE id = ?', batchId); + if (!b || ['merged', 'returned', 'superseded'].includes(b.state)) return; + const pending = store.all(`SELECT * FROM attempts WHERE batch_id = ? AND state IN ('queued', 'verifying', 'in_review', 'peer_review', 'approved')`, batchId); + for (const a of pending) { + store.run(`UPDATE attempts SET state = 'returned', result_json = json_set(COALESCE(result_json, '{}'), '$.returned', ?) WHERE id = ?`, reason, a.id); + store.run('DELETE FROM peer_leases WHERE attempt_id = ?', a.id); + store.run( + `UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ? AND state IN ('verifying', 'in_review', 'peer_review', 'approved', 'merging')`, + a.user_id, now() + 6 * 3600_000, now(), a.task_id, + ); + } + store.run(`UPDATE batches SET state = 'returned', result_json = json_set(COALESCE(result_json, '{}'), '$.returned', ?) WHERE id = ?`, reason, batchId); + store.event('batch_returned', b.user_id, { batch: batchId, reason }); +} + +/** Attempts of a batch with their task's symbol. */ +export function batchAttempts(store, batchId) { + return store.all( + `SELECT a.*, t.symbol_id FROM attempts a JOIN tasks t ON t.id = a.task_id WHERE a.batch_id = ? ORDER BY a.id`, + batchId, + ); +} diff --git a/src/server/cli.js b/src/server/cli.js index 7fa175c..9792ffa 100644 --- a/src/server/cli.js +++ b/src/server/cli.js @@ -37,6 +37,8 @@ Usage: td-server [options] (all commands accept --data , defaul symbols load import symbols with fixed IDs and per-version addresses symbols units assign functions to translation units (csv: start,end,file) symbols library mark symbols as library code (no tasks) + originals import [--version V] assembly + pseudocode of the original (ExportOriginals.java) + ghidra build | refresh the server's Ghidra project (download, pseudocode); refresh also runs after merges rebuild compile all of main and refresh match status tasks generate create tasks for functions that have none `; @@ -239,6 +241,25 @@ async function main() { } } + if (cmd === 'ghidra' && (sub === 'build' || sub === 'refresh')) { + // The server's Ghidra project (needs ghidraHome in server.json and a JDK 21). + const ctx = openServer(dataDir); + if (!ctx.ghidra.enabled) throw new Error('set ghidraHome in server.json to a Ghidra 11.2.1 installation'); + ctx.store.setMeta('ghidra.commit', ctx.repo.head()); + const info = sub === 'build' ? await ctx.ghidra.build() : await ctx.ghidra.refresh(); + console.log(JSON.stringify(info, null, 2)); + return; + } + + if (cmd === 'originals' && sub === 'import') { + // ghidra_scripts/ExportOriginals.java output, for the current version unless --version is given. + const ctx = openServer(dataDir); + const v = values.version ?? ctx.store.currentVersion()?.id; + const n = await ctx.originals.import(v, rest[0]); + console.log(`imported ${n} functions for ${v}`); + return; + } + if (cmd === 'tasks' && sub === 'generate') { const ctx = openServer(dataDir); console.log(`created ${ctx.tasks.generate()} tasks`); diff --git a/src/server/db.js b/src/server/db.js index ff500b0..d432254 100644 --- a/src/server/db.js +++ b/src/server/db.js @@ -142,6 +142,25 @@ CREATE TABLE IF NOT EXISTS func_status ( confirmed INTEGER NOT NULL DEFAULT 0 ); +-- A batch is one submission: the files once, and one attempt per task (function) it covers. +-- It is verified with one compile and merged as a whole once every attempt is approved; if one +-- attempt fails, the batch goes back to its author (peer verdicts carry over to unchanged code). +CREATE TABLE IF NOT EXISTS batches ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + base_commit TEXT NOT NULL, + files_json TEXT NOT NULL, + notes TEXT, + agent_meta TEXT, + env_digest TEXT, + state TEXT NOT NULL, -- queued, verifying, open, merging, merged, returned, superseded + result_json TEXT, + merged_commit TEXT, + created_at INTEGER NOT NULL, + finished_at INTEGER +); +CREATE INDEX IF NOT EXISTS batches_state ON batches(state); + -- Peer review of attempts that are not byte-identical: other agents read the original's -- assembly and pseudocode and the submitted code, and judge whether the logic is the same. CREATE TABLE IF NOT EXISTS peer_reviews ( @@ -171,6 +190,15 @@ CREATE TABLE IF NOT EXISTS unit_deps ( ); CREATE INDEX IF NOT EXISTS unit_deps_header ON unit_deps(header); +-- The original functions as text (Ghidra export): assembly with symbol names, and pseudocode. +CREATE TABLE IF NOT EXISTS original_code ( + version_id TEXT NOT NULL, + addr INTEGER NOT NULL, + asm TEXT, + c TEXT, + PRIMARY KEY (version_id, addr) +); + CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY, at INTEGER NOT NULL, @@ -189,6 +217,19 @@ export class Store { this.migrateUsers(); this.db.exec(SCHEMA); this.addColumn('func_status', 'confirmed', 'INTEGER NOT NULL DEFAULT 0'); + // Fingerprint of main's compiled code per function (submissions may only change their own tasks). + this.addColumn('func_status', 'code_hash', 'TEXT'); + this.addColumn('attempts', 'batch_id', 'INTEGER REFERENCES batches(id)'); + this.db.exec('CREATE INDEX IF NOT EXISTS attempts_batch ON attempts(batch_id)'); + } + + /** Submitted files of an attempt (kept once per batch). */ + attemptFiles(attempt) { + if (attempt.batch_id) { + const b = this.get('SELECT files_json FROM batches WHERE id = ?', attempt.batch_id); + if (b) return JSON.parse(b.files_json); + } + return JSON.parse(attempt.files_json || '{}'); } /** Adds a column to a table created by an older schema. */ diff --git a/src/server/ghidra.js b/src/server/ghidra.js new file mode 100644 index 0000000..662962d --- /dev/null +++ b/src/server/ghidra.js @@ -0,0 +1,137 @@ +// The server's own Ghidra project of the current target version, kept up to date with the symbol +// database. Contributors download it (td ghidra download) instead of analyzing the exe +// themselves, and the coordinator exports assembly and pseudocode from it for /api/original. +// +// build import + auto-analysis + symbol names (once per version, ~15 min) +// refresh re-apply the names, re-export the originals, pack the project (a few minutes); +// runs on its own after merges, at most every `ghidraRefreshMinutes` (default 30) + +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { run } from '../shared/build.js'; +import { hex } from '../shared/annotations.js'; + +const SCRIPTS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts'); +const PROJECT = 'td'; +const PROGRAM = 'target.exe'; + +export class ServerGhidra { + constructor({ config, store, originals, log = console }) { + Object.assign(this, { config, store, originals, log }); + this.dir = path.join(config.dataDir, 'ghidra'); + this.projectDir = path.join(this.dir, 'project'); + this.busy = null; + this.timer = null; + } + + get enabled() { + return !!this.config.ghidraHome && fs.existsSync(path.join(this.config.ghidraHome, 'support', 'analyzeHeadless')); + } + + /** Metadata of the packed project, or null. */ + info() { + try { + return JSON.parse(fs.readFileSync(path.join(this.dir, 'package.json'), 'utf8')); + } catch { + return null; + } + } + + packagePath() { + const i = this.info(); + return i ? path.join(this.dir, i.file) : null; + } + + async headless(args, timeoutMs) { + fs.mkdirSync(this.projectDir, { recursive: true }); + const exe = path.join(this.config.ghidraHome, 'support', 'analyzeHeadless'); + // Ghidra 11.2 is made for JDK 21; the system default may be newer. + const jdk21 = '/usr/lib/jvm/java-21-openjdk-amd64'; + const javaHome = this.config.ghidraJavaHome ?? (fs.existsSync(jdk21) ? jdk21 : process.env.JAVA_HOME); + const env = { ...process.env, MAXMEM: this.config.ghidraMaxMem ?? '6G', ...(javaHome ? { JAVA_HOME: javaHome, PATH: `${javaHome}/bin:${process.env.PATH}` } : {}) }; + const r = await run(exe, [this.projectDir, PROJECT, ...args, '-scriptPath', SCRIPTS_DIR], { timeoutMs, env }); + if (r.code !== 0) throw new Error(`Ghidra failed (exit ${r.code}): ${r.output.slice(-2000)}`); + return r.output; + } + + writeSymbols(versionId) { + const file = path.join(this.dir, 'symbols.tsv'); + const lines = []; + for (const s of this.store.symbolMap(versionId).values()) { + if (s.addr == null) continue; + lines.push([hex(s.addr), s.kind, s.id, s.name ?? '', s.decorated ?? '', s.library ? 1 : 0].join('\t')); + } + fs.writeFileSync(file, lines.join('\n') + '\n'); + return file; + } + + /** Creates the project from the current version's exe (replacing an old one). */ + async build() { + const v = this.store.currentVersion(); + fs.rmSync(this.projectDir, { recursive: true, force: true }); + fs.mkdirSync(this.dir, { recursive: true }); + const exe = path.join(this.dir, PROGRAM); + fs.copyFileSync(v.file, exe); + this.log.info(`ghidra: analyzing ${v.id} (this takes a while)`); + await this.headless(['-import', exe, '-overwrite', '-postScript', 'ApplySymbols.java', this.writeSymbols(v.id)], 4 * 3600_000); + fs.rmSync(exe, { force: true }); + this.store.setMeta('ghidra.version', v.id); + await this.refresh({ applySymbols: false }); + } + + /** Re-applies the symbol names, re-exports the originals and packs the project. */ + async refresh({ applySymbols = true } = {}) { + const v = this.store.currentVersion(); + if (this.store.getMeta('ghidra.version') !== v.id) return this.build(); + const t0 = Date.now(); + if (applySymbols) await this.headless(['-process', PROGRAM, '-noanalysis', '-postScript', 'ApplySymbols.java', this.writeSymbols(v.id)], 3600_000); + const jsonl = path.join(this.dir, 'originals.jsonl'); + await this.headless(['-process', PROGRAM, '-noanalysis', '-readOnly', '-postScript', 'ExportOriginals.java', jsonl, '30'], 6 * 3600_000); + const n = await this.originals.import(v.id, jsonl); + fs.rmSync(jsonl, { force: true }); + + // Pack the project for download (tar.gz: Windows' tar.exe and Linux tar both unpack it). + const head = this.store.getMeta('ghidra.commit') ?? null; + const name = `ghidra-${v.id}-${Date.now()}.tar.gz`; + const tmp = path.join(this.dir, name + '.part'); + const r = await run('tar', ['-czf', tmp, '-C', this.projectDir, `${PROJECT}.gpr`, `${PROJECT}.rep`], { timeoutMs: 1800_000 }); + if (r.code !== 0) throw new Error(`packing the Ghidra project failed: ${r.output}`); + const old = this.info(); + fs.renameSync(tmp, path.join(this.dir, name)); + const sha256 = await sha256File(path.join(this.dir, name)); + const info = { version: v.id, file: name, sha256, size: fs.statSync(path.join(this.dir, name)).size, builtAt: Date.now(), commit: head, functions: n, ghidra: this.config.ghidraVersion ?? '11.2.1' }; + fs.writeFileSync(path.join(this.dir, 'package.json'), JSON.stringify(info, null, 2)); + if (old?.file && old.file !== name) fs.rmSync(path.join(this.dir, old.file), { force: true }); + this.store.setMeta('ghidra.refreshedAt', Date.now()); + this.log.info(`ghidra: refreshed ${v.id} in ${((Date.now() - t0) / 1000).toFixed(0)}s (${n} functions exported, package ${(info.size / 1048576).toFixed(0)} MB)`); + return info; + } + + /** Called after merges: refresh soon, but at most every ghidraRefreshMinutes. */ + scheduleRefresh(commit) { + if (!this.enabled) return; + if (commit) this.store.setMeta('ghidra.commit', commit); + if (this.timer || this.busy) return; + const every = (this.config.ghidraRefreshMinutes ?? 30) * 60_000; + const last = Number(this.store.getMeta('ghidra.refreshedAt') ?? 0); + const wait = Math.max(0, last + every - Date.now()); + this.timer = setTimeout(() => { + this.timer = null; + this.busy = this.refresh() + .catch((e) => this.log.error(`ghidra refresh failed: ${e.message}`)) + .finally(() => { + this.busy = null; + }); + }, wait); + this.timer.unref?.(); + } +} + +function sha256File(file) { + return new Promise((resolve, reject) => { + const h = crypto.createHash('sha256'); + fs.createReadStream(file).on('data', (d) => h.update(d)).on('end', () => resolve(h.digest('hex'))).on('error', reject); + }); +} diff --git a/src/server/install.js b/src/server/install.js new file mode 100644 index 0000000..39a9d69 --- /dev/null +++ b/src/server/install.js @@ -0,0 +1,111 @@ +// One-line installers. The scripts are generated per request with this server's URL, the project +// name and the SHA-256 of the td binaries in /downloads (built by tools/build-td). +// +// Windows: $env:TD_TOKEN='td_...'; irm https:///install.ps1 | iex +// Linux: curl -fsSL https:///install.sh | TD_TOKEN=td_... sh + +import fs from 'node:fs'; +import path from 'node:path'; + +export function downloadsDir(config) { + return path.join(config.dataDir, 'downloads'); +} + +function checksums(config) { + const file = path.join(downloadsDir(config), 'SHA256SUMS'); + const out = {}; + if (!fs.existsSync(file)) return out; + for (const line of fs.readFileSync(file, 'utf8').split('\n')) { + const m = /^([0-9a-f]{64})\s+(\S+)$/.exec(line.trim()); + if (m) out[m[2]] = m[1]; + } + return out; +} + +const slug = (name) => String(name ?? 'project').replace(/[^A-Za-z0-9_.-]/g, '-'); + +export function installPs1(config, project) { + const sha = checksums(config)['td-windows-x64.exe']; + if (!sha) return null; + return `# Clank Together installer for Windows. Usage (PowerShell, one line): +# $env:TD_TOKEN='td_...'; irm ${config.publicUrl}/install.ps1 | iex +# Installs td (a single program, no Node.js needed), sets up the ${project.name} workspace and +# connects your AI. Optional: $env:TD_DIR (workspace folder), $env:TD_TOOLCHAIN (local compiler root). +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +$server = '${config.publicUrl}' +$token = $env:TD_TOKEN +if (-not $token) { $token = Read-Host 'API token (log in on $server and create one under Account)' } +$dir = if ($env:TD_DIR) { $env:TD_DIR } else { Join-Path $HOME 'clank\\${slug(project.name)}' } +$bin = Join-Path $env:LOCALAPPDATA 'ClankTogether\\bin' +New-Item -ItemType Directory -Force $bin | Out-Null + +if (-not (Get-Command git -ErrorAction SilentlyContinue)) { + Write-Host 'Installing git (winget)...' + winget install --id Git.Git -e --silent --accept-package-agreements --accept-source-agreements | Out-Null + $env:Path += ';' + (Join-Path $env:ProgramFiles 'Git\\cmd') + if (-not (Get-Command git -ErrorAction SilentlyContinue)) { throw 'git is required: install it from https://git-scm.com and run this line again' } +} + +Write-Host 'Downloading td...' +$exe = Join-Path $bin 'td.exe' +Invoke-WebRequest "$server/download/td-windows-x64.exe" -OutFile "$exe.part" -UseBasicParsing +if ((Get-FileHash "$exe.part" -Algorithm SHA256).Hash.ToLower() -ne '${sha}') { Remove-Item "$exe.part"; throw 'td download failed its checksum check' } +Move-Item -Force "$exe.part" $exe + +$userPath = [Environment]::GetEnvironmentVariable('Path', 'User') +if (-not $userPath) { $userPath = '' } +if (($userPath -split ';') -notcontains $bin) { [Environment]::SetEnvironmentVariable('Path', ($userPath.TrimEnd(';') + ';' + $bin), 'User') } +if (($env:Path -split ';') -notcontains $bin) { $env:Path += ';' + $bin } + +$setup = @('setup', $server, $dir, '--token', $token) +if ($env:TD_TOOLCHAIN) { $setup += @('--toolchain', $env:TD_TOOLCHAIN) } +& $exe @setup +if ($LASTEXITCODE -ne 0) { throw 'td setup failed (see above)' } +& $exe connect --workspace $dir +Write-Host '' +Write-Host "Done. Workspace: $dir (td is on your PATH in new terminals)" +`; +} + +export function installSh(config, project) { + const sha = checksums(config)['td-linux-x64']; + if (!sha) return null; + return `#!/bin/sh +# Clank Together installer for Linux. Usage (one line): +# curl -fsSL ${config.publicUrl}/install.sh | TD_TOKEN=td_... sh +# Installs td (a single program, no Node.js needed), sets up the ${project.name} workspace and +# connects your AI. Optional: TD_DIR (workspace folder), TD_TOOLCHAIN (local compiler root). +set -eu +SERVER='${config.publicUrl}' +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) ;; + *) echo "td is available for Linux x86_64 and Windows x64; this is $(uname -s) $(uname -m)" >&2; exit 1 ;; +esac +TOKEN="\${TD_TOKEN:-}" +if [ -z "$TOKEN" ]; then printf 'API token (log in on %s and create one under Account): ' "$SERVER"; read -r TOKEN < /dev/tty; fi +DIR="\${TD_DIR:-$HOME/clank/${slug(project.name)}}" +BIN="$HOME/.local/bin" +mkdir -p "$BIN" +command -v git >/dev/null 2>&1 || { echo 'git is required: install it (e.g. sudo apt install git) and run this line again' >&2; exit 1; } + +echo 'Downloading td...' +curl -fsSL "$SERVER/download/td-linux-x64" -o "$BIN/td.part" +echo "${sha} $BIN/td.part" | sha256sum -c - >/dev/null || { rm -f "$BIN/td.part"; echo 'td download failed its checksum check' >&2; exit 1; } +mv "$BIN/td.part" "$BIN/td" +chmod +x "$BIN/td" +case ":$PATH:" in + *":$BIN:"*) ;; + *) echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$HOME/.profile"; PATH="$BIN:$PATH" ;; +esac + +if [ -n "\${TD_TOOLCHAIN:-}" ]; then + "$BIN/td" setup "$SERVER" "$DIR" --token "$TOKEN" --toolchain "$TD_TOOLCHAIN" +else + "$BIN/td" setup "$SERVER" "$DIR" --token "$TOKEN" +fi +"$BIN/td" connect --workspace "$DIR" +echo +echo "Done. Workspace: $DIR (td is in ~/.local/bin)" +`; +} diff --git a/src/server/merge.js b/src/server/merge.js index 4d84841..4564f8d 100644 --- a/src/server/merge.js +++ b/src/server/merge.js @@ -4,6 +4,7 @@ import { HttpError } from './http.js'; import { now } from './db.js'; import { hasRole } from './accounts.js'; import { STATUS } from '../shared/compare.js'; +import { returnBatch, batchAttempts } from './batches.js'; export class Merger { constructor({ store, config, repo, project, tasks, verifier, getPe, peers, log = console }) { @@ -12,6 +13,7 @@ export class Merger { } start() { + this.store.run(`UPDATE batches SET state = 'open' WHERE state = 'merging'`); this.store.run(`UPDATE tasks SET state = 'approved' WHERE state = 'merging'`); this.timer = setInterval(() => this.pump(), 3000); } @@ -21,91 +23,144 @@ export class Merger { } requiredApprovals(attempt) { - const result = attempt.result_json ? JSON.parse(attempt.result_json) : {}; + const batch = attempt.batch_id ? this.store.get('SELECT result_json FROM batches WHERE id = ?', attempt.batch_id) : null; + const result = batch?.result_json ? JSON.parse(batch.result_json) : {}; const p = this.project.matchPolicy; return result.changedHeaders?.length ? p.approvalsWithHeaderChanges : p.approvals; } + /** A human decision on one attempt (peer-confirmed, not byte-identical). */ review(reviewer, attemptId, action, comment) { + if (!hasRole(reviewer, 'reviewer')) throw new HttpError(403, 'only reviewers can review'); + if (!['approve', 'request_changes', 'reject'].includes(action)) throw new HttpError(400, 'invalid action'); + return this.store.tx(() => this.reviewOne(reviewer, attemptId, action, comment)); + } + + /** The same decision for every attempt of a batch that waits for human review. */ + reviewBatch(reviewer, batchId, action, comment) { if (!hasRole(reviewer, 'reviewer')) throw new HttpError(403, 'only reviewers can review'); if (!['approve', 'request_changes', 'reject'].includes(action)) throw new HttpError(400, 'invalid action'); return this.store.tx(() => { - const attempt = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId); - if (!attempt) throw new HttpError(404, 'no such attempt'); - if (attempt.state !== 'in_review') throw new HttpError(409, `attempt is ${attempt.state}`); - if (attempt.user_id === reviewer.id) throw new HttpError(403, 'you cannot review your own attempt'); - if (this.store.get('SELECT 1 FROM reviews WHERE attempt_id = ? AND reviewer_id = ?', attemptId, reviewer.id)) throw new HttpError(409, 'you already reviewed this attempt'); - - this.store.run('INSERT INTO reviews (attempt_id, reviewer_id, action, comment, created_at) VALUES (?, ?, ?, ?, ?)', attemptId, reviewer.id, action, comment?.slice(0, 4000) ?? null, now()); - this.store.event('review', reviewer.id, { attempt: attemptId, action }); - - if (action === 'approve') { - const approvals = this.store.get(`SELECT COUNT(*) AS n FROM reviews WHERE attempt_id = ? AND action = 'approve'`, attemptId).n; - const needed = this.requiredApprovals(attempt) - approvals; - if (hasRole(reviewer, 'maintainer') || needed <= 0) { - this.store.run(`UPDATE attempts SET state = 'approved' WHERE id = ?`, attemptId); - this.store.run(`UPDATE tasks SET state = 'approved', updated_at = ? WHERE id = ?`, now(), attempt.task_id); - return { state: 'approved' }; - } - return { state: 'in_review', approvalsNeeded: needed }; - } - const newState = action === 'reject' ? 'rejected' : 'needs_cleanup'; - this.store.run('UPDATE attempts SET state = ? WHERE id = ?', newState, attemptId); - this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), attempt.task_id); - return { state: newState }; + const waiting = this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'in_review'`, batchId); + if (!waiting.length) throw new HttpError(409, 'nothing in this batch waits for human review'); + const results = waiting.map((w) => ({ attemptId: w.id, ...this.reviewOne(reviewer, w.id, action, comment) })); + return { batchId, results }; }); } + reviewOne(reviewer, attemptId, action, comment) { + const attempt = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId); + if (!attempt) throw new HttpError(404, 'no such attempt'); + if (attempt.state !== 'in_review') throw new HttpError(409, `attempt is ${attempt.state}`); + if (attempt.user_id === reviewer.id) throw new HttpError(403, 'you cannot review your own attempt'); + if (this.store.get('SELECT 1 FROM reviews WHERE attempt_id = ? AND reviewer_id = ?', attemptId, reviewer.id)) throw new HttpError(409, 'you already reviewed this attempt'); + + this.store.run('INSERT INTO reviews (attempt_id, reviewer_id, action, comment, created_at) VALUES (?, ?, ?, ?, ?)', attemptId, reviewer.id, action, comment?.slice(0, 4000) ?? null, now()); + this.store.event('review', reviewer.id, { attempt: attemptId, action }); + + if (action === 'approve') { + const approvals = this.store.get(`SELECT COUNT(*) AS n FROM reviews WHERE attempt_id = ? AND action = 'approve'`, attemptId).n; + const needed = this.requiredApprovals(attempt) - approvals; + if (hasRole(reviewer, 'maintainer') || needed <= 0) { + this.store.run(`UPDATE attempts SET state = 'approved' WHERE id = ?`, attemptId); + this.store.run(`UPDATE tasks SET state = 'approved', updated_at = ? WHERE id = ?`, now(), attempt.task_id); + return { state: 'approved' }; + } + return { state: 'in_review', approvalsNeeded: needed }; + } + const newState = action === 'reject' ? 'rejected' : 'needs_cleanup'; + this.store.run('UPDATE attempts SET state = ? WHERE id = ?', newState, attemptId); + this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), attempt.task_id); + returnBatch(this.store, attempt.batch_id, `a reviewer asked for changes to ${attempt.task_id}`); + return { state: newState }; + } + + /** Merges the oldest open batch whose attempts are all approved. */ async pump() { if (this.busy) return; - const task = this.store.get(`SELECT * FROM tasks WHERE state = 'approved' ORDER BY updated_at LIMIT 1`); - if (!task) return; + const batch = this.store.get( + `SELECT b.* FROM batches b WHERE b.state = 'open' + AND EXISTS (SELECT 1 FROM attempts a WHERE a.batch_id = b.id) + AND NOT EXISTS (SELECT 1 FROM attempts a WHERE a.batch_id = b.id AND a.state != 'approved') + ORDER BY b.id LIMIT 1`, + ); + if (!batch) return; this.busy = true; try { - await this.merge(task); + await this.merge(batch); } catch (e) { - this.log.error(`merge of task ${task.id} failed: ${e.stack}`); + this.log.error(`merge of batch ${batch.id} failed: ${e.stack}`); try { this.repo.reset(); } catch {} - this.store.run(`UPDATE tasks SET state = 'in_review', updated_at = ? WHERE id = ?`, now(), task.id); - this.store.event('merge_error', null, { task: task.id, error: e.message }); + this.store.run(`UPDATE batches SET state = 'open' WHERE id = ?`, batch.id); + this.store.event('merge_error', null, { batch: batch.id, error: e.message }); } finally { this.busy = false; } } - async merge(task) { - this.store.run(`UPDATE tasks SET state = 'merging', updated_at = ? WHERE id = ?`, now(), task.id); - const attempt = this.store.get(`SELECT * FROM attempts WHERE task_id = ? AND state = 'approved' ORDER BY id DESC LIMIT 1`, task.id); - const author = this.store.get('SELECT name FROM users WHERE id = ?', attempt.user_id).name; - const files = JSON.parse(attempt.files_json); + async merge(batch) { + this.store.run(`UPDATE batches SET state = 'merging' WHERE id = ?`, batch.id); + const attempts = batchAttempts(this.store, batch.id); + for (const a of attempts) this.store.run(`UPDATE tasks SET state = 'merging', updated_at = ? WHERE id = ?`, now(), a.task_id); + const author = this.store.get('SELECT name FROM users WHERE id = ?', batch.user_id).name; + const files = JSON.parse(batch.files_json); - const merged = this.repo.mergeOntoHead(attempt.base_commit, files); const bounce = (summary, details) => { - this.store.run(`UPDATE attempts SET state = 'needs_cleanup', result_json = json_set(COALESCE(result_json, '{}'), '$.mergeFailure', json(?)) WHERE id = ?`, JSON.stringify({ summary, ...details }), attempt.id); - this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), task.id); - this.log.info(`merge of task ${task.id} bounced: ${summary}`); + this.store.run(`UPDATE batches SET result_json = json_set(COALESCE(result_json, '{}'), '$.mergeFailure', json(?)) WHERE id = ?`, JSON.stringify({ summary, ...details }), batch.id); + returnBatch(this.store, batch.id, summary); + this.log.info(`merge of batch ${batch.id} bounced: ${summary}`); }; + const merged = this.repo.mergeOntoHead(batch.base_commit, files); if (merged.conflicts.length) return bounce('conflicts with newer work on main', { conflicts: merged.conflicts }); - // Re-verify on top of the current main: other merges may have landed since review. - const { check, rec } = await this.verifier.check(merged.files, `merge-${attempt.id}`); - const evaln = this.verifier.evaluate({ check, rec, targetId: task.symbol_id, files: merged.files, changedHeaders: Object.keys(merged.files).filter((f) => !/\.(c|cpp|cc|cxx)$/i.test(f)) }); - // Not byte-identical is fine if other agents confirmed the logic; anything worse bounces. - const peerConfirmed = evaln.state === 'peer_review' && this.peers?.confirmed(attempt.id); - if (evaln.state !== 'in_review' && !peerConfirmed) return bounce(`re-verification on current main failed: ${evaln.feedback.summary}`, { feedback: evaln.feedback }); + // Re-verify on top of the current main: other merges may have landed since verification. + const { check, rec, changedHeaders } = await this.verifier.check(merged.files, `merge-${batch.id}`); + const targets = new Set(attempts.map((a) => a.symbol_id)); + const outside = this.verifier.changedOutside(check, merged.files, targets); + const confirmedIds = new Set(); + const results = []; + for (const a of attempts) { + const ev = this.verifier.evaluate({ check, rec, targetId: a.symbol_id, files: merged.files, changedHeaders, outside }); + // Byte-identical, or confirmed by peers and approved by a human; anything worse bounces. + const identical = ev.state === 'approved' || ev.state === 'in_review'; + const peerConfirmed = ev.state === 'peer_review' && this.peers?.confirmed(a.id); + if (!identical && !peerConfirmed) return bounce(`re-verification on current main failed for ${a.symbol_id}: ${ev.feedback.summary}`, { feedback: ev.feedback }); + if (peerConfirmed) confirmedIds.add(a.symbol_id); + results.push({ id: a.symbol_id, name: ev.feedback.target?.demangled ?? null, status: ev.feedback.target?.status, peerConfirmed: !!peerConfirmed }); + } - const reviewers = this.store.all(`SELECT u.name FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ? AND r.action = 'approve'`, attempt.id).map((r) => r.name); - const target = evaln.feedback.target; - const message = `Decompile ${task.symbol_id}${target?.demangled ? ` (${target.demangled})` : ''}\n\nAttempt #${attempt.id} by ${author}, status ${target?.status}.\nReviewed-by: ${reviewers.join(', ')}\n`; + const reviewers = this.store.all( + `SELECT DISTINCT u.name FROM reviews r JOIN users u ON u.id = r.reviewer_id JOIN attempts a ON a.id = r.attempt_id WHERE a.batch_id = ? AND r.action = 'approve'`, + batch.id, + ).map((r) => r.name); + const peers = this.store.all( + `SELECT DISTINCT u.name FROM peer_reviews p JOIN users u ON u.id = p.reviewer_id JOIN attempts a ON a.id = p.attempt_id WHERE a.batch_id = ? AND p.verdict = 'same'`, + batch.id, + ).map((r) => r.name); + const identicalCount = results.filter((r) => !r.peerConfirmed).length; + const title = + results.length === 1 + ? `Decompile ${results[0].id}${results[0].name ? ` (${results[0].name})` : ''}` + : `Decompile ${results.length} functions in ${[...new Set(Object.keys(files))].slice(0, 2).join(', ')}${Object.keys(files).length > 2 ? ', ...' : ''}`; + const lines = results.map((r) => `- ${r.id}${r.name ? ` ${r.name}` : ''}: ${r.peerConfirmed ? 'logic confirmed by peer review' : 'byte-identical'}`); + const message = + `${title}\n\nBatch #${batch.id} by ${author}: ${identicalCount} byte-identical, ${results.length - identicalCount} confirmed by peer review.\n\n${lines.join('\n')}\n` + + (peers.length ? `\nPeer-reviewed-by: ${peers.join(', ')}` : '') + + (reviewers.length ? `\nReviewed-by: ${reviewers.join(', ')}` : '') + + (!reviewers.length && !confirmedIds.size ? '\nMerged automatically: every function is byte-identical to the original.' : '') + + '\n'; const commit = this.repo.commitFiles(merged.files, message, `${author} <${author.replace(/[^A-Za-z0-9_.-]/g, '_')}@users.clank-together.invalid>`); this.store.tx(() => { - this.applyCheckToDatabase(check, rec, commit, attempt.user_id, { confirmedId: peerConfirmed ? task.symbol_id : null }); - this.store.run(`UPDATE attempts SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, attempt.id); - this.store.run(`UPDATE tasks SET state = 'done', lease_user = NULL, updated_at = ? WHERE id = ?`, now(), task.id); - this.store.event('merged', attempt.user_id, { task: task.id, attempt: attempt.id, commit }); + this.applyCheckToDatabase(check, rec, commit, batch.user_id, { confirmedIds }); + for (const a of attempts) { + this.store.run(`UPDATE attempts SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, a.id); + this.store.run(`UPDATE tasks SET state = 'done', lease_user = NULL, updated_at = ? WHERE id = ?`, now(), a.task_id); + } + this.store.run(`UPDATE batches SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, batch.id); + this.store.event('merged', batch.user_id, { batch: batch.id, functions: attempts.length, commit }); }); this.tasks.generate(); if (this.config.pushRemote) { @@ -115,14 +170,15 @@ export class Merger { this.log.error(`push to ${this.config.pushRemote} failed: ${e.message}`); } } - this.log.info(`merged task ${task.id} as ${commit.slice(0, 10)}`); + this.onMerged?.(commit); + this.log.info(`merged batch ${batch.id} (${attempts.length} functions) as ${commit.slice(0, 10)}`); } /** * Records what a check on main learned: per-function status, decorated * names and new symbols for previously unknown references. */ - applyCheckToDatabase(check, rec, commit, userId, { confirmedId = null } = {}) { + applyCheckToDatabase(check, rec, commit, userId, { confirmedIds = new Set() } = {}) { const v = this.store.currentVersion(); const pe = this.getPe(); for (const u of check.units) { @@ -135,10 +191,10 @@ export class Merger { } for (const f of u.functions) { this.store.run( - `INSERT INTO func_status (symbol_id, file, status, score, commit_hash, updated_at, confirmed) VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(symbol_id) DO UPDATE SET file = excluded.file, status = excluded.status, score = excluded.score, commit_hash = excluded.commit_hash, updated_at = excluded.updated_at, + `INSERT INTO func_status (symbol_id, file, status, score, commit_hash, updated_at, confirmed, code_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(symbol_id) DO UPDATE SET file = excluded.file, status = excluded.status, score = excluded.score, commit_hash = excluded.commit_hash, updated_at = excluded.updated_at, code_hash = COALESCE(excluded.code_hash, func_status.code_hash), confirmed = CASE WHEN excluded.confirmed = 1 THEN 1 WHEN excluded.status IN ('exact', 'match_unresolved') THEN 0 ELSE func_status.confirmed END`, - f.id, u.file, f.status, f.score, commit, now(), f.id === confirmedId ? 1 : 0, + f.id, u.file, f.status, f.score, commit, now(), confirmedIds.has(f.id) ? 1 : 0, f.codeHash ?? null, ); if (f.demangled) this.store.run('UPDATE symbols SET name = ? WHERE id = ? AND name IS NULL', f.demangled, f.id); this.store.run('UPDATE symbols SET unit = ? WHERE id = ? AND unit IS NULL', u.file, f.id); diff --git a/src/server/original.js b/src/server/original.js new file mode 100644 index 0000000..33e786d --- /dev/null +++ b/src/server/original.js @@ -0,0 +1,95 @@ +// The original function, for contributors without a local Ghidra: assembly and pseudocode from a +// Ghidra export (ghidra_scripts/ExportOriginals.java, imported with `td-server originals import`), +// and live disassembly by objdump for functions the export does not cover. + +import fs from 'node:fs'; +import readline from 'node:readline'; +import { run } from '../shared/build.js'; +import { hex } from '../shared/annotations.js'; +import { HttpError } from './http.js'; + +export class Originals { + constructor({ store }) { + this.store = store; + } + + /** Imports an ExportOriginals JSON-lines file for a version; returns the number of functions. */ + async import(versionId, file) { + const rl = readline.createInterface({ input: fs.createReadStream(file, 'utf8'), crlfDelay: Infinity }); + let n = 0; + const insert = this.store.db.prepare( + `INSERT INTO original_code (version_id, addr, asm, c) VALUES (?, ?, ?, ?) + ON CONFLICT(version_id, addr) DO UPDATE SET asm = excluded.asm, c = excluded.c`, + ); + this.store.db.exec('BEGIN'); + try { + for await (const line of rl) { + if (!line.trim()) continue; + const r = JSON.parse(line); + insert.run(versionId, Number.parseInt(r.addr, 16), r.asm ?? null, r.c ?? null); + if (++n % 5000 === 0) { + this.store.db.exec('COMMIT'); + this.store.db.exec('BEGIN'); + } + } + this.store.db.exec('COMMIT'); + } catch (e) { + this.store.db.exec('ROLLBACK'); + throw e; + } + return n; + } + + /** {symbol, name, version, addr, size, asm, pseudocode, source} for a symbol ID or a hex address. */ + async get(query) { + const v = this.store.currentVersion(); + if (!v) throw new HttpError(404, 'no current version'); + let sym; + if (/^0x[0-9a-f]+$/i.test(query)) { + sym = this.store.get( + `SELECT s.id, s.name, a.addr, a.size FROM symbol_addresses a JOIN symbols s ON s.id = a.symbol_id WHERE a.version_id = ? AND a.addr = ?`, + v.id, + Number.parseInt(query, 16), + ) ?? { id: null, name: null, addr: Number.parseInt(query, 16), size: null }; + } else { + sym = this.store.get( + `SELECT s.id, s.name, a.addr, a.size FROM symbols s LEFT JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? WHERE s.id = ? OR s.name = ? OR s.decorated = ? LIMIT 1`, + v.id, + query, + query, + query, + ); + } + if (!sym) throw new HttpError(404, `unknown symbol ${query}`); + if (sym.addr == null) throw new HttpError(404, `${sym.id} has no address in ${v.id}`); + const row = this.store.get('SELECT asm, c FROM original_code WHERE version_id = ? AND addr = ?', v.id, sym.addr); + const asm = row?.asm ?? (await this.objdump(v.file, sym.addr, sym.size ?? 256)); + return { + symbol: sym.id, + name: sym.name, + version: v.id, + addr: '0x' + hex(sym.addr), + size: sym.size, + asm, + pseudocode: row?.c ?? null, + source: row ? 'ghidra export' : 'objdump (no pseudocode yet)', + note: 'Pseudocode is Ghidra output, a reading aid; it is not matching code.', + }; + } + + async objdump(file, addr, size) { + const r = await run('objdump', ['-d', '-M', 'intel', '--no-addresses', `--start-address=0x${hex(addr)}`, `--stop-address=0x${hex(addr + Math.min(size, 65536))}`, file], { timeoutMs: 30_000 }); + if (r.code !== 0) { + // Older binutils lack --no-addresses. + const r2 = await run('objdump', ['-d', '-M', 'intel', `--start-address=0x${hex(addr)}`, `--stop-address=0x${hex(addr + Math.min(size, 65536))}`, file], { timeoutMs: 30_000 }); + return cleanObjdump(r2.output); + } + return cleanObjdump(r.output); + } +} + +function cleanObjdump(out) { + const lines = out.split('\n'); + const start = lines.findIndex((l) => /^\s*[0-9a-f]+ <|^Disassembly of section/.test(l)); + return lines.slice(start >= 0 ? start : 0).filter((l) => l.trim() && !/^Disassembly of section/.test(l)).join('\n'); +} diff --git a/src/server/peer.js b/src/server/peer.js index cd9c5ae..557ae14 100644 --- a/src/server/peer.js +++ b/src/server/peer.js @@ -10,6 +10,7 @@ import { HttpError } from './http.js'; import { now } from './db.js'; import { untrusted } from './tasks.js'; import { hex } from '../shared/annotations.js'; +import { returnBatch } from './batches.js'; export const PEER_INSTRUCTIONS = [ 'Decide whether the submitted C++ does the same thing as the original function. It is not byte-identical; your judgment is what counts.', @@ -36,7 +37,7 @@ export class PeerReviews { /** Attempts waiting for peer review that `user` may take. */ available(user, limit = 20) { return this.store.all( - `SELECT a.id AS attempt_id, a.task_id, t.symbol_id, s.name, a.finished_at, + `SELECT a.id AS attempt_id, a.batch_id, a.task_id, t.symbol_id, s.name, a.finished_at, (SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS confirmations, (SELECT COUNT(*) FROM peer_leases l WHERE l.attempt_id = a.id AND l.expires_at > ?) AS in_progress FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id @@ -51,28 +52,56 @@ export class PeerReviews { ).filter((r) => r.confirmations + r.in_progress < this.required); } - /** Leases one review (a given attempt, or the oldest available) and returns its context. */ + /** + * Leases a review (a given attempt, or the oldest available) together with every other function + * of the same batch that waits for this reviewer, and returns the context of all of them. + */ claim(user, attemptId = null) { return this.store.tx(() => { const options = this.available(user, 200); const pick = attemptId ? options.find((o) => o.attempt_id === attemptId) : options[0]; if (!pick) throw new HttpError(attemptId ? 409 : 404, attemptId ? 'this attempt is not open for your review (yours, already reviewed, or enough reviewers)' : 'no attempts are waiting for peer review'); - this.store.run( - `INSERT INTO peer_leases (attempt_id, reviewer_id, expires_at) VALUES (?, ?, ?) - ON CONFLICT(attempt_id, reviewer_id) DO UPDATE SET expires_at = excluded.expires_at`, - pick.attempt_id, user.id, now() + this.leaseMs, - ); - this.store.event('peer_claim', user.id, { attempt: pick.attempt_id }); - return this.context(pick.attempt_id); + const group = pick.batch_id ? options.filter((o) => o.batch_id === pick.batch_id) : [pick]; + for (const o of group) { + this.store.run( + `INSERT INTO peer_leases (attempt_id, reviewer_id, expires_at) VALUES (?, ?, ?) + ON CONFLICT(attempt_id, reviewer_id) DO UPDATE SET expires_at = excluded.expires_at`, + o.attempt_id, user.id, now() + this.leaseMs, + ); + } + this.store.event('peer_claim', user.id, { attempt: pick.attempt_id, functions: group.length }); + return this.context(pick.attempt_id, group.map((o) => o.attempt_id)); }); } + /** One function to review: where the original is and how to read it. */ + functionInfo(attemptId, v) { + const row = this.store.get( + `SELECT a.id, s.id AS symbol, s.name, ad.addr, ad.size FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id + LEFT JOIN symbol_addresses ad ON ad.symbol_id = s.id AND ad.version_id = ? WHERE a.id = ?`, + v.id, + attemptId, + ); + return { + attemptId: row.id, + symbol: row.symbol, + name: row.name, + addr: row.addr != null ? '0x' + hex(row.addr) : null, + size: row.size, + annotation: `// FUNCTION: ${row.symbol}`, + read: [`td_disassemble ${row.symbol}`, `td_decompile ${row.symbol}`], + }; + } + release(user, attemptId) { this.store.run('DELETE FROM peer_leases WHERE attempt_id = ? AND reviewer_id = ?', attemptId, user.id); } - /** What a reviewer sees: the function to compare against and the submitted code, nothing machine-judged. */ - context(attemptId) { + /** + * What a reviewer sees: the functions to compare against and the submitted code, nothing + * machine-judged. `attemptIds`: the functions of the batch this review covers. + */ + context(attemptId, attemptIds = null) { const a = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId); if (!a) throw new HttpError(404, 'no such attempt'); const t = this.store.get('SELECT * FROM tasks WHERE id = ?', a.task_id); @@ -82,12 +111,16 @@ export class PeerReviews { v.id, t.symbol_id, ); - const result = a.result_json ? JSON.parse(a.result_json) : {}; + const batch = a.batch_id ? this.store.get('SELECT * FROM batches WHERE id = ?', a.batch_id) : null; + const batchResult = batch?.result_json ? JSON.parse(batch.result_json) : {}; + const ids = attemptIds ?? (a.batch_id ? this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'peer_review' ORDER BY id`, a.batch_id).map((r) => r.id) : [a.id]); return { attemptId: a.id, required: this.required, confirmations: this.store.get(`SELECT COUNT(*) AS n FROM peer_reviews WHERE attempt_id = ? AND verdict = 'same'`, a.id).n, instructions: PEER_INSTRUCTIONS, + // Every function this review covers (one batch); give one verdict per attemptId. + functions: (ids.length ? ids : [a.id]).map((id) => this.functionInfo(id, v)), original: { symbol: s.id, name: s.name, @@ -98,7 +131,7 @@ export class PeerReviews { }, submission: { annotation: `// FUNCTION: ${s.id}`, - files: result.mergedFiles ?? JSON.parse(a.files_json), + files: batchResult.mergedFiles ?? this.store.attemptFiles(a), notes: untrusted(a.notes), }, leaseExpires: this.store.get('SELECT expires_at FROM peer_leases WHERE attempt_id = ? ORDER BY expires_at DESC LIMIT 1', a.id)?.expires_at ?? null, @@ -135,6 +168,8 @@ export class PeerReviews { `UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, a.user_id, now() + 6 * 3600_000, now(), a.task_id, ); + // The batch merges as a whole, so it goes back with this function; confirmed ones keep their verdicts. + returnBatch(this.store, a.batch_id, `a peer review found a difference in ${a.task_id}`); return { state: 'needs_cleanup', confirmations: taken }; } const confirmations = taken + 1; diff --git a/src/server/tasks.js b/src/server/tasks.js index 9deeaea..9b1cd22 100644 --- a/src/server/tasks.js +++ b/src/server/tasks.js @@ -75,10 +75,46 @@ export class Tasks { return this.store.all(`SELECT id FROM tasks WHERE lease_user = ? AND state = 'leased'`, userId).map((r) => r.id); } + /** How many tasks (functions) one account may hold at once, single claims and batches together. */ + get maxLeased() { + return this.config.maxLeasedTasks ?? 200; + } + + /** + * Claims several tasks at once: every claimable function of a unit (source file) or of an + * address range in the current version, up to `limit`. They are submitted together as a batch. + */ + claimBatch(user, { unit = null, start = null, end = null, limit = 50 } = {}) { + if (!unit && (start == null || end == null)) throw new HttpError(400, 'give a unit (source file) or an address range (start, end)'); + return this.store.tx(() => { + const held = this.leasesOf(user.id).length; + const room = Math.min(Math.max(1, limit | 0), 200, this.maxLeased - held); + if (room <= 0) throw new HttpError(409, `you already hold ${held} tasks; submit or release some first`); + const v = this.store.currentVersion(); + const ph = CLAIMABLE.map(() => '?').join(','); + const where = unit ? 's.unit = ?' : 'a.addr >= ? AND a.addr < ?'; + const args = unit ? [unit] : [start, end]; + const rows = this.store.all( + `SELECT t.id FROM tasks t JOIN symbols s ON s.id = t.symbol_id + JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? + WHERE t.state IN (${ph}) AND ${where} + AND (t.preferred_user IS NULL OR t.preferred_user = ? OR t.preferred_until < ?) + ORDER BY a.addr LIMIT ?`, + v.id, ...CLAIMABLE, ...args, user.id, now(), room, + ); + if (!rows.length) throw new HttpError(404, unit ? `no claimable tasks in ${unit}` : 'no claimable tasks in that range'); + const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000; + const head = this.repo.head(); + for (const r of rows) this.store.run(`UPDATE tasks SET state = 'leased', lease_user = ?, lease_expires = ?, lease_base_commit = ?, updated_at = ? WHERE id = ?`, user.id, expires, head, now(), r.id); + this.store.event('claim_batch', user.id, { tasks: rows.length, unit, start, end }); + return rows.map((r) => r.id); + }); + } + claim(user, taskId) { return this.store.tx(() => { - const max = this.config.maxLeasesPerUser ?? 3; - if (this.leasesOf(user.id).length >= max) throw new HttpError(409, `you already hold ${max} leases; submit or release one first`); + const max = this.maxLeased; + if (this.leasesOf(user.id).length >= max) throw new HttpError(409, `you already hold ${max} tasks; submit or release some first`); let t; if (taskId) { t = this.get(taskId); @@ -114,12 +150,59 @@ export class Tasks { return expires; } + /** Extends every lease the account holds (an agent working through a batch). */ + heartbeatAll(user) { + const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000; + const r = this.store.run(`UPDATE tasks SET lease_expires = ?, updated_at = ? WHERE lease_user = ? AND state = 'leased'`, expires, now(), user.id); + return { leases: Number(r.changes), leaseExpires: expires }; + } + release(user, taskId) { const t = this.requireLease(user, taskId); this.reopen(t.id); this.store.event('release', user.id, { task: t.id }); } + /** + * One submission for one or more leased tasks: the files once, an attempt per task. Verified + * with a single compile and merged as a whole (see Verifier.verifyBatch, Merger). + */ + submitBatch(user, { taskIds, baseCommit, files, notes = null, agentMeta = null, envDigest = null }) { + const ids = [...new Set((taskIds ?? []).map(Number))]; + if (!ids.length) throw new HttpError(400, 'taskIds must list the tasks this submission covers'); + if (ids.length > 200) throw new HttpError(400, 'at most 200 tasks per submission'); + return this.store.tx(() => { + for (const id of ids) this.requireLease(user, id); + const b = this.store.run( + `INSERT INTO batches (user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', ?)`, + user.id, baseCommit, JSON.stringify(files), notes, agentMeta, envDigest, now(), + ); + const batchId = Number(b.lastInsertRowid); + const attempts = []; + for (const id of ids) { + const r = this.store.run( + `INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, batch_id, created_at) VALUES (?, ?, ?, '{}', ?, ?, ?, 'queued', ?, ?)`, + id, user.id, baseCommit, notes, agentMeta, envDigest, batchId, now(), + ); + attempts.push({ taskId: id, attemptId: Number(r.lastInsertRowid) }); + this.store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), id); + } + this.store.event('submit', user.id, { batch: batchId, tasks: ids.length }); + return { batchId, attempts }; + }); + } + + /** Allowed paths of a set of tasks: their unit files plus the project headers. */ + allowedPathsForTasks(taskIds) { + const out = new Set(); + for (const id of taskIds) { + const t = this.get(id); + const sym = this.store.get('SELECT * FROM symbols WHERE id = ?', t.symbol_id); + for (const p of this.allowedPaths(sym)) out.add(p); + } + return [...out]; + } + /** Returns a task to the pool, keeping its attempt history. */ reopen(taskId) { const t = this.get(taskId); @@ -164,8 +247,8 @@ export class Tasks { ); let bestFiles = null; if (t.best_attempt) { - const best = this.store.get('SELECT files_json FROM attempts WHERE id = ?', t.best_attempt); - bestFiles = best ? JSON.parse(best.files_json) : null; + const best = this.store.get('SELECT * FROM attempts WHERE id = ?', t.best_attempt); + bestFiles = best ? this.store.attemptFiles(best) : null; } const neighbours = sym.unit ? this.store.all( diff --git a/src/server/verify.js b/src/server/verify.js index 2d57626..060b15a 100644 --- a/src/server/verify.js +++ b/src/server/verify.js @@ -9,24 +9,35 @@ import { reconcile } from '../shared/reconcile.js'; import { STATUS } from '../shared/compare.js'; import { listSources, isUnit } from '../shared/project.js'; import { now } from './db.js'; +import { returnBatch, batchAttempts } from './batches.js'; const RANK = { [STATUS.EXACT]: 3, [STATUS.UNRESOLVED]: 2, [STATUS.MISMATCH]: 1, [STATUS.MISSING]: 0, [STATUS.NO_ADDRESS]: 0 }; export class Verifier { - constructor({ store, config, repo, project, tasks, getPe, log = console }) { + constructor({ store, config, repo, project, tasks, getPe, peers, log = console }) { this.store = store; this.config = config; this.repo = repo; this.project = project; this.tasks = tasks; this.getPe = getPe; + this.peers = peers; this.log = log; this.running = 0; this.jobsDir = path.join(config.dataDir, 'jobs'); } start() { + this.store.run(`UPDATE batches SET state = 'queued' WHERE state = 'verifying'`); this.store.run(`UPDATE attempts SET state = 'queued' WHERE state = 'verifying'`); + // Attempts from before batches existed become batches of one. + for (const a of this.store.all(`SELECT * FROM attempts WHERE state = 'queued' AND batch_id IS NULL`)) { + const b = this.store.run( + `INSERT INTO batches (user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', ?)`, + a.user_id, a.base_commit, a.files_json, a.notes, a.agent_meta, a.env_digest, a.created_at, + ); + this.store.run('UPDATE attempts SET batch_id = ? WHERE id = ?', Number(b.lastInsertRowid), a.id); + } this.timer = setInterval(() => this.pump(), 2000); this.pump(); } @@ -38,17 +49,23 @@ export class Verifier { pump() { const max = this.config.verifyConcurrency ?? 2; while (this.running < max) { - const a = this.store.tx(() => { - const row = this.store.get(`SELECT * FROM attempts WHERE state = 'queued' ORDER BY id LIMIT 1`); - if (row) this.store.run(`UPDATE attempts SET state = 'verifying' WHERE id = ?`, row.id); + const b = this.store.tx(() => { + const row = this.store.get(`SELECT * FROM batches WHERE state = 'queued' ORDER BY id LIMIT 1`); + if (row) { + this.store.run(`UPDATE batches SET state = 'verifying' WHERE id = ?`, row.id); + this.store.run(`UPDATE attempts SET state = 'verifying' WHERE batch_id = ? AND state = 'queued'`, row.id); + } return row; }); - if (!a) return; + if (!b) return; this.running++; - this.verify(a) + this.verifyBatch(b) .catch((e) => { - this.log.error(`attempt ${a.id}: verifier crashed: ${e.stack}`); - this.finish(a, 'needs_cleanup', 0, { summary: 'internal verifier error; a maintainer has been notified', internalError: true }); + this.log.error(`batch ${b.id}: verifier crashed: ${e.stack}`); + for (const a of batchAttempts(this.store, b.id)) { + if (a.state === 'verifying') this.finish(a, 'needs_cleanup', 0, { summary: 'internal verifier error; a maintainer has been notified', internalError: true }); + } + returnBatch(this.store, b.id, 'internal verifier error'); }) .finally(() => { this.running--; @@ -118,39 +135,91 @@ export class Verifier { return out; } - async verify(attempt) { - const task = this.tasks.get(attempt.task_id); - const sym = this.store.get('SELECT * FROM symbols WHERE id = ?', task.symbol_id); - const submitted = JSON.parse(attempt.files_json); + /** + * Functions of the submitted files whose compiled code differs from main although they are not + * part of the submission: a submission may only change the functions it holds the lease for. + * Unknown on main (no fingerprint yet, e.g. before the first rebuild) is not counted. + */ + changedOutside(check, files, targets) { + const fileSet = new Set(Object.keys(files)); + const seen = new Map(); + for (const u of check.units) for (const f of u.functions) if (fileSet.has(f.file) && !targets.has(f.id) && f.codeHash) seen.set(f.id, f); + const out = []; + for (const [id, f] of seen) { + const main = this.store.get('SELECT code_hash FROM func_status WHERE symbol_id = ?', id); + if (!main) out.push({ id, file: f.file, why: 'new or newly annotated' }); + else if (main.code_hash && main.code_hash !== f.codeHash) out.push({ id, file: f.file, why: 'compiled code changed' }); + } + return out; + } + + /** + * A function that three peers already confirmed keeps that confirmation when its compiled code is + * unchanged in a resubmission (the batch came back because of another function). + */ + carryOver(attempt, target) { + if (!target?.codeHash) return false; + const prev = this.store.all(`SELECT id, result_json FROM attempts WHERE task_id = ? AND id != ? ORDER BY id DESC LIMIT 5`, attempt.task_id, attempt.id); + for (const p of prev) { + const hash = p.result_json ? JSON.parse(p.result_json).feedback?.target?.codeHash : null; + if (hash !== target.codeHash || !this.peers?.confirmed(p.id)) continue; + this.store.run( + `INSERT OR IGNORE INTO peer_reviews (attempt_id, reviewer_id, verdict, reason, agent_meta, created_at) + SELECT ?, reviewer_id, verdict, reason, agent_meta, created_at FROM peer_reviews WHERE attempt_id = ?`, + attempt.id, p.id, + ); + return true; + } + return false; + } + + async verifyBatch(batch) { + const attempts = batchAttempts(this.store, batch.id); + const submitted = JSON.parse(batch.files_json); + const finishAll = (state, feedback) => { + for (const a of attempts) this.finish(a, state, 0, feedback); + this.store.run(`UPDATE batches SET state = 'returned', finished_at = ?, result_json = ? WHERE id = ?`, now(), JSON.stringify({ feedback }), batch.id); + }; // Stage 0: static gate. const lintErrors = lintSubmission(submitted, { - allowedPaths: this.tasks.allowedPaths(sym), + allowedPaths: this.tasks.allowedPathsForTasks(attempts.map((a) => a.task_id)), limits: this.project.lintLimits, allowedPragmas: this.project.allowedPragmas, }); - if (lintErrors.length) return this.finish(attempt, 'rejected', 0, { summary: 'rejected by the static gate', lintErrors }); + if (lintErrors.length) return finishAll('rejected', { summary: 'rejected by the static gate', lintErrors }); + if (!this.repo.isAncestor(batch.base_commit)) return finishAll('rejected', { summary: `base commit ${batch.base_commit} is not part of main; sync and resubmit` }); + const merged = this.repo.mergeOntoHead(batch.base_commit, submitted); + if (merged.conflicts.length) return finishAll('needs_cleanup', { summary: 'your change conflicts with newer work on main; sync, merge and resubmit', conflicts: merged.conflicts }); - if (!this.repo.isAncestor(attempt.base_commit)) { - return this.finish(attempt, 'rejected', 0, { summary: `base commit ${attempt.base_commit} is not part of main; sync and resubmit` }); + // Stages 1-3: one compile for the whole batch, then every function on its own. + const { check, rec, changedHeaders } = await this.check(merged.files, `batch-${batch.id}`); + const targets = new Set(attempts.map((a) => a.symbol_id)); + const outside = this.changedOutside(check, merged.files, targets); + let failed = 0; + for (const a of attempts) { + const ev = this.evaluate({ check, rec, targetId: a.symbol_id, files: merged.files, changedHeaders, outside }); + ev.feedback.headCommit = this.repo.head(); + if (attempts.length > 1) ev.feedback.batch = { id: batch.id, functions: attempts.length }; + if (batch.env_digest && this.config.envDigest && batch.env_digest !== this.config.envDigest) { + ev.feedback.warnings.push('your local toolchain differs from the pinned one; local results may not match the server'); + } + if (ev.state === 'peer_review' && this.carryOver(a, ev.feedback.target)) { + ev.state = 'in_review'; + ev.feedback.summary += '; the compiled code is unchanged since three peers confirmed it, so their verdicts carry over'; + } + if (ev.state === 'needs_cleanup' || ev.state === 'rejected') failed++; + this.finish(a, ev.state, ev.score, ev.feedback); } - const merged = this.repo.mergeOntoHead(attempt.base_commit, submitted); - if (merged.conflicts.length) { - return this.finish(attempt, 'needs_cleanup', 0, { summary: 'your change conflicts with newer work on main; sync, merge and resubmit', conflicts: merged.conflicts }); - } - - // Stages 1-3: compile, compare, regressions. - const { check, rec, changedHeaders } = await this.check(merged.files, `attempt-${attempt.id}`); - const evaluation = this.evaluate({ check, rec, targetId: task.symbol_id, files: merged.files, changedHeaders }); - evaluation.feedback.headCommit = this.repo.head(); - if (attempt.env_digest && this.config.envDigest && attempt.env_digest !== this.config.envDigest) { - evaluation.feedback.warnings.push('your local toolchain differs from the pinned one; local results may not match the server'); - } - return this.finish(attempt, evaluation.state, evaluation.score, evaluation.feedback, { mergedFiles: merged.files, check: summarizeCheck(check), rec, changedHeaders }); + this.store.run( + `UPDATE batches SET state = 'open', finished_at = ?, result_json = ? WHERE id = ?`, + now(), JSON.stringify({ mergedFiles: merged.files, check: summarizeCheck(check), changedHeaders, outside }), batch.id, + ); + if (failed) returnBatch(this.store, batch.id, `${failed} of ${attempts.length} functions need work; see each function's feedback`); } /** Routes a check result: rejected, needs_cleanup or in_review. */ - evaluate({ check, rec, targetId, files, changedHeaders }) { + evaluate({ check, rec, targetId, files, changedHeaders, outside = [] }) { const fileSet = new Set(Object.keys(files)); const policy = this.project.matchPolicy; const feedback = { summary: '', warnings: [] }; @@ -200,12 +269,18 @@ export class Verifier { } else if (regressions.length) { feedback.summary = `your change breaks ${regressions.length} function(s) that matched before`; state = 'needs_cleanup'; + } else if (outside.length) { + feedback.summary = `the submission changes functions it does not hold: ${outside.map((o) => o.id).join(', ')}; claim them as well (td claim --unit) or undo those changes`; + feedback.changedOutside = outside; + state = 'needs_cleanup'; } else if (wrongRefs.length) { feedback.summary = `references point at the wrong symbols: ${wrongRefs.map((c) => c.message).join('; ')}`; state = 'needs_cleanup'; } else if (target.status === STATUS.EXACT || (target.status === STATUS.UNRESOLVED && policy.reviewAt === 'match_unresolved')) { - feedback.summary = target.status === STATUS.EXACT ? 'exact match, sent to human review' : 'bytes match (some referenced data is not yet named), sent to human review'; - state = 'in_review'; + // Byte-identical needs no human: approved directly, merged with its batch. + const auto = policy.autoMergeIdentical !== false; + feedback.summary = (target.status === STATUS.EXACT ? 'byte-identical' : 'byte-identical (some referenced data is not yet named)') + (auto ? ', approved automatically' : ', sent to human review'); + state = auto ? 'approved' : 'in_review'; } else if (target.status === STATUS.MISMATCH || target.status === STATUS.UNRESOLVED) { // Not byte-identical, but it compiles cleanly: other agents judge whether the logic is the same. feedback.summary = `not byte-identical (score ${(score * 100).toFixed(1)}%), sent to peer review: ${policy.peerReviews} other agents compare it with the original`; @@ -221,9 +296,10 @@ export class Verifier { const task = this.tasks.get(attempt.task_id); this.store.tx(() => { this.store.run('UPDATE attempts SET state = ?, score = ?, result_json = ?, finished_at = ? WHERE id = ?', state, score, JSON.stringify({ feedback, ...extra }), now(), attempt.id); - let taskState = state === 'in_review' || state === 'peer_review' ? state : 'needs_cleanup'; + const onTheWay = ['in_review', 'peer_review', 'approved'].includes(state); + let taskState = onTheWay ? state : 'needs_cleanup'; const better = state !== 'rejected' && score > task.best_score; - const cycles = task.cleanup_cycles + (state === 'in_review' || state === 'peer_review' ? 0 : 1); + const cycles = task.cleanup_cycles + (onTheWay ? 0 : 1); if (taskState === 'needs_cleanup' && cycles >= this.project.matchPolicy.maxCleanupCycles) taskState = 'hard'; if (state === 'rejected' && !task.best_attempt) taskState = 'open'; this.store.run( diff --git a/src/server/web/app.js b/src/server/web/app.js index 05489c4..2dcee17 100644 --- a/src/server/web/app.js +++ b/src/server/web/app.js @@ -180,6 +180,9 @@ $('token-form').addEventListener('submit', async (e) => { const t = await post('/api/me/tokens', formData(e.target)); e.target.reset(); $('new-token-value').textContent = t.token; + const origin = location.origin; + $('install-win').textContent = `$env:TD_TOKEN='${t.token}'; irm ${origin}/install.ps1 | iex`; + $('install-linux').textContent = `curl -fsSL ${origin}/install.sh | TD_TOKEN=${t.token} sh`; $('new-token').hidden = false; await loadTokens(); } catch (err) { @@ -187,6 +190,15 @@ $('token-form').addEventListener('submit', async (e) => { } }); +$('copy-install').addEventListener('click', async () => { + try { + await navigator.clipboard.writeText($('install-win').textContent); + $('copy-install').textContent = 'Copied'; + } catch { + $('copy-install').textContent = 'Select the line and copy it'; + } +}); + $('password-form').addEventListener('submit', async (e) => { e.preventDefault(); try { @@ -264,13 +276,41 @@ async function loadQueue() { const q = await api('/api/review/queue'); const body = $('queue-body'); body.replaceChildren(); - if (!q.length) return body.append(el('p', 'Nothing waiting for review.', 'muted')); - body.append( - table( - ['Attempt', 'Symbol', 'Name', 'Author', 'Score', 'Approvals', ''], - q.map((a) => [`#${a.id}`, a.symbol_id, a.name || '', a.author, `${(a.score * 100).toFixed(1)}%`, a.approvals, button(a.reviewed_by_me ? 'Reviewed' : 'Open', () => openAttempt(a.id))]), - ), - ); + if (!q.length) return body.append(el('p', 'Nothing waiting for review. Byte-identical work merges on its own; this queue only holds peer-confirmed functions.', 'muted')); + // One block per batch: the functions three peers confirmed, approved together with one click. + const batches = new Map(); + for (const a of q) { + const key = a.batch_id ?? `a${a.id}`; + if (!batches.has(key)) batches.set(key, []); + batches.get(key).push(a); + } + for (const [key, items] of batches) { + const head = el('div', null, 'batch-head'); + head.append(el('strong', typeof key === 'number' ? `Batch #${key}` : `Attempt #${items[0].id}`), el('span', ` by ${items[0].author} ยท ${items.length} function${items.length > 1 ? 's' : ''} confirmed by peers`, 'muted')); + if (typeof key === 'number') { + head.append( + button( + 'Approve batch', + async () => { + try { + await api(`/api/batches/${key}/review`, { method: 'POST', body: JSON.stringify({ action: 'approve' }) }); + await loadQueue(); + } catch (e) { + alert(e.message); + } + }, + 'primary', + ), + ); + } + body.append(head); + body.append( + table( + ['Attempt', 'Symbol', 'Name', 'Score', 'Approvals', ''], + items.map((a) => [`#${a.id}`, a.symbol_id, a.name || '', `${(a.score * 100).toFixed(1)}%`, a.approvals, button(a.reviewed_by_me ? 'Reviewed' : 'Open', () => openAttempt(a.id))]), + ), + ); + } } async function openAttempt(id) { diff --git a/src/server/web/index.html b/src/server/web/index.html index 4cb7300..ba791df 100644 --- a/src/server/web/index.html +++ b/src/server/web/index.html @@ -48,22 +48,17 @@
-

Join the crew

5 steps +

Join the crew

4 steps
  1. 1
    Register

    Pick a username and password below.

  2. 2
    Get approved

    An admin checks new accounts before they can contribute.

    3. -
  3. 3
    Make a token

    Log in and create an API token for your agent under Account.

    4. -
  4. 4
    Install and set up

    Needs Node.js 22.13 or newer and git.

    - npm install -g https://saty.dev/git/clank/clank-together/archive/main.tar.gz - td setup https://saty.dev wow --token <token> --toolchain <your re/toolchain> - cd wow - td ghidra install - td ghidra build
    5. -
  5. 5
    Connect your AI and start clanking

    Any agent that supports MCP works: Claude Code, Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI and others. td setup prints the exact settings for your workspace. Claude Code:

    - claude mcp add clank-together -- td mcp --workspace <workspace> -

    Other clients take this in their MCP server settings:

    - { "command": "td", "args": ["mcp", "--workspace", "<workspace>"] } -

    Agents without MCP can run the commands directly: td tasks, td claim, td check, td submit, td peer. Then ask your agent to read AGENTS.md and claim a task, or take a peer review.

    6. +
  6. 3
    Create a token

    Log in and create an API token under Account. The page then shows your personal install line.

    7. +
  7. 4
    Paste one line

    It installs td (a single program, nothing else needed besides git), sets up the workspace and connects your AI.

    +

    Windows (PowerShell):

    + $env:TD_TOKEN='<token>'; irm https://saty.dev/install.ps1 | iex +

    Linux:

    + curl -fsSL https://saty.dev/install.sh | TD_TOKEN=<token> sh +

    Any AI that supports MCP works (Claude Code, Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI, ...). Claude Code is connected automatically; for the others the installer prints the settings. Then ask your AI to read AGENTS.md and claim a task, or take a peer review.

@@ -97,6 +92,12 @@

Change password

diff --git a/src/server/web/style.css b/src/server/web/style.css index c2c17be..fdc1e98 100644 --- a/src/server/web/style.css +++ b/src/server/web/style.css @@ -395,3 +395,6 @@ footer code { word-break: break-all; } .gear-a, .gear-b, .meter .fill { animation: none; } .meter .fill { transition: none; } } + +.batch-head { display: flex; flex-wrap: wrap; align-items: center; gap: 10px; margin: 18px 0 4px; } +.batch-head button { margin-left: auto; } diff --git a/src/shared/check.js b/src/shared/check.js index 3b245bf..1bab090 100644 --- a/src/shared/check.js +++ b/src/shared/check.js @@ -6,7 +6,7 @@ import path from 'node:path'; import { CoffObject } from './coff.js'; import { parseListing, normalizePath } from './listing.js'; import { compareObject } from './compare.js'; -import { lintAnnotations } from './annotations.js'; +import { lintAnnotations, hex } from './annotations.js'; import { lintPreprocessed } from './lint.js'; import { compileUnits, prepareJobDir } from './build.js'; import { scanAnnotations } from './project.js'; @@ -69,3 +69,30 @@ export function functionResults(check) { export function jobDirFor(base, name) { return path.join(base, name.replace(/[^A-Za-z0-9_-]/g, '_')); } + +/** `files`: the files the user asked about; annotation problems elsewhere in the repo are not theirs. */ +export function summarizeCheck(result, rec, files = null) { + return { + annotationErrors: files ? result.annotationErrors.filter((e) => files.has(e.file)) : result.annotationErrors, + units: result.units.map((u) => ({ + file: u.file, + compiled: u.compiled, + log: u.compiled ? undefined : u.log, + lintErrors: u.lintErrors, + errors: u.errors, + unannotated: u.unannotated, + functions: u.functions.map((f) => ({ + id: f.id, + name: f.demangled, + addr: f.addr != null ? '0x' + hex(f.addr) : null, + status: f.status, + score: f.score, + size: f.size, + diffRegions: f.diffRegions?.slice(0, 12), + relocations: f.relocations, + })), + })), + referenceConflicts: rec.conflicts, + unknownReferences: rec.unknown.map((x) => ({ symbol: x.decorated, addr: '0x' + hex(x.addr) })), + }; +} diff --git a/src/shared/compare.js b/src/shared/compare.js index 9157776..42ac145 100644 --- a/src/shared/compare.js +++ b/src/shared/compare.js @@ -1,6 +1,7 @@ // Compares compiled functions (COFF object + listing) against the original // executable, using the symbol database to locate each annotated function. +import crypto from 'node:crypto'; import { insnAt, normalizePath } from './listing.js'; import { FUNCTION_KINDS, hex } from './annotations.js'; @@ -309,6 +310,7 @@ export function compareFunction({ fn, addr, origSize, pe, resolve, symbols, proc status, score: Math.round(score * 10000) / 10000, size: { ours: ours.length, original: origSize ?? null }, + codeHash: codeHash(ours, proc), diffRegions: sameLength ? diffRegions(diffs, ours, orig, proc) : [], relocations: relocResults.filter((r) => !r.ok || r.unresolved), // Where the original points is only meaningful when the code lines up. @@ -383,3 +385,23 @@ export function lcsRatio(a, b) { } return (2 * prev[b.length]) / (a.length + b.length); } + +/** + * Fingerprint of our compiled code, to tell whether a function changed between two builds: + * the bytes (relocated fields are zero in an object file) with __LINE__ immediates zeroed, so + * editing lines above a function does not count as changing it. + */ +export function codeHash(bytes, proc) { + const b = Buffer.from(bytes); + const insns = proc?.insns ?? []; + for (let k = 0; k < insns.length; k++) { + const insn = insns[k]; + if (!insn.srcLine) continue; + const end = Math.min(b.length, k + 1 < insns.length ? insns[k + 1].offset : b.length); + for (let p = insn.offset + 1; p + 2 <= end; p++) { + if (p + 4 <= end && b.readUInt32LE(p) === insn.srcLine) b.fill(0, p, p + 4); + else if (p + 2 === end && b.readUInt16LE(p) === insn.srcLine) b.fill(0, p, p + 2); + } + } + return crypto.createHash('sha1').update(b).digest('hex').slice(0, 20); +} diff --git a/template/AGENTS.md b/template/AGENTS.md index 193f757..245e1ee 100644 --- a/template/AGENTS.md +++ b/template/AGENTS.md @@ -7,11 +7,16 @@ reading like code a person would have written. ## Workflow -1. `td_claim_task` gives you a task: a symbol ID (e.g. `fn_00123`), its address, - its size, the file to write it in, and earlier attempts with their feedback. -2. Read the original: `td_disassemble` (ground truth) and `td_decompile` +**Keep working until the user stops you.** After a submission, claim the next +work right away; do not wait for its result. + +1. Claim work. Prefer a whole unit: `td_claim_batch` with a source file gives + you every open function of that file (up to 50); `td_claim_task` gives one. + Each task has a symbol ID (e.g. `fn_00123`), its address and size, and + earlier attempts with their feedback. +2. Read the originals: `td_disassemble` (ground truth) and `td_decompile` (Ghidra's guess, a starting point only). -3. Write the function in the task's file, directly preceded by its annotation: +3. Write each function in its file, directly preceded by its annotation: ```cpp // FUNCTION: fn_00123 void Player::Update(float dt) @@ -19,14 +24,25 @@ reading like code a person would have written. ... } ``` -4. `td_check` compiles and compares. Iterate until the status is `exact` - (or `match_unresolved`, meaning the bytes match but some referenced data - has no symbol yet). -5. `td_submit` the task. The server verifies again in a sandbox: - - **byte-identical** goes straight to human review; +4. `td_check` compiles and compares every function of the file (on the server + if you have no local compiler). Iterate until as many as possible are + `exact` (or `match_unresolved`: the bytes match, some referenced data has + no symbol yet). +5. `td_submit` everything you hold as one batch. The server verifies it in a + sandbox, with one compile for the whole batch: + - **byte-identical** functions are approved and merged **automatically**; - **not byte-identical but compiling cleanly** goes to **peer review**: - three other agents judge whether your logic is the same as the original; + three other agents judge whether your logic is the same as the original, + then a human approves the batch; - compile errors, regressions or wrong references come back to you. + A batch merges as a whole: if one function fails, the batch comes back with + per-function feedback. Fix those functions and submit again; peer verdicts + stay valid for functions whose compiled code did not change. +6. Only change the functions you hold. A submission that changes other + functions in the same file is refused; claim them as well (`td_claim_batch`) + or undo the change. +7. Claim the next work (back to 1). Check earlier batches now and then with + `td_get_batch` and fix what came back. ## Peer review (reviewing other agents' work) diff --git a/test/server.test.js b/test/server.test.js index 8618636..fbe32c2 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -182,8 +182,8 @@ test('evaluate routes results', () => { const rec = { conflicts: [], unknown: [], learned: [] }; const ev = (c, r = rec) => ctx.verifier.evaluate({ check: c, rec: r, targetId: 'fn_00001', files: { 'src/a.cpp': '' }, changedHeaders: [] }); - assert.equal(ev(check(fn(STATUS.EXACT, 1))).state, 'in_review'); - assert.equal(ev(check(fn(STATUS.UNRESOLVED, 1))).state, 'in_review'); + assert.equal(ev(check(fn(STATUS.EXACT, 1))).state, 'approved', 'byte-identical is approved without a human'); + assert.equal(ev(check(fn(STATUS.UNRESOLVED, 1))).state, 'approved'); assert.equal(ev(check(fn(STATUS.MISMATCH, 0.8))).state, 'peer_review'); assert.equal(ev(check(fn(STATUS.MISSING, 0))).state, 'needs_cleanup'); assert.equal(ev(check(null)).state, 'needs_cleanup'); @@ -192,8 +192,8 @@ test('evaluate routes results', () => { // A definitively wrong reference of the target bounces; an ambiguity elsewhere does not. const withRef = { ...fn(STATUS.UNRESOLVED, 1), relocations: [{ symbol: '?Foo@@YAXXZ', ok: true, unresolved: true }] }; assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', hard: true, message: 'Foo is elsewhere' }] }).state, 'needs_cleanup'); - assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Bar@@YAXXZ', hard: true, message: 'Bar is elsewhere' }] }).state, 'in_review'); - assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', message: 'Foo seen twice' }] }).state, 'in_review'); + assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Bar@@YAXXZ', hard: true, message: 'Bar is elsewhere' }] }).state, 'approved'); + assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', message: 'Foo seen twice' }] }).state, 'approved'); }); test('regressions are detected against func_status', () => { @@ -343,3 +343,98 @@ test('peer review: three other agents must agree, by reading only', async () => assert.equal(ctx.peers.verdicts(second)[0].reason.untrusted, true); ctx.store.run(`UPDATE tasks SET state = 'done' WHERE id = ?`, task.id); }); + +test('batches: one compile, auto-approval, peer review, return as a whole, carry-over, one merge', async () => { + const author = await activeUser('bauthor'); + const peers = [await activeUser('bpeer1'), await activeUser('bpeer2'), await activeUser('bpeer3')]; + await ctx.accounts.createUser('breviewer', 'reviewer', PASSWORD); + const reviewer = await login('breviewer'); + delete ctx.merger.pump; // an earlier test stubbed it on the instance; use the real one here + + // Two fresh functions in one unit. + const v = ctx.store.currentVersion(); + const a = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x401300, size: 8, unit: 'src/batch.cpp' }).symbol.id; + const b = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x401340, size: 8, unit: 'src/batch.cpp' }).symbol.id; + ctx.tasks.generate(); + + // The compiler is stubbed: A byte-identical, B not; B's code fingerprint decides carry-over. + let bHash = 'hash-b-1'; + const realCheck = ctx.verifier.check; + ctx.verifier.check = async () => ({ + check: { + annotationErrors: [], + units: [{ + file: 'src/batch.cpp', compiled: true, lintErrors: [], unannotated: [], bindings: {}, inferred: [], includes: [], errors: [], + functions: [ + { id: a, status: STATUS.EXACT, score: 1, file: 'src/batch.cpp', codeHash: 'hash-a', relocations: [], size: { ours: 8, original: 8 } }, + { id: b, status: STATUS.MISMATCH, score: 0.9, file: 'src/batch.cpp', codeHash: bHash, relocations: [], size: { ours: 8, original: 8 } }, + ], + }], + }, + rec: { conflicts: [], unknown: [], learned: [], learnedAddresses: [] }, + changedHeaders: [], + units: ['src/batch.cpp'], + }); + + const claim = await call('POST', '/api/tasks/claim-batch', { unit: 'src/batch.cpp' }, author); + assert.equal(claim.status, 200, JSON.stringify(claim.body)); + assert.deepEqual(claim.body.tasks.map((t) => t.symbol).sort(), [a, b].sort()); + const taskIds = claim.body.tasks.map((t) => t.task); + const source = `// FUNCTION: ${a}\nint fa() { return 1; }\n// FUNCTION: ${b}\nint fb() { return 2; }\n`; + const submit = () => call('POST', '/api/batches', { taskIds, baseCommit: ctx.repo.head(), files: { 'src/batch.cpp': source } }, author); + const batchState = async (id) => (await call('GET', `/api/batches/${id}`, null, author)).body; + const settle = (id) => waitFor(async () => { const st = await batchState(id); return !['queued', 'verifying'].includes(st.state) && st; }); + + const s1 = await submit(); + assert.equal(s1.status, 200, JSON.stringify(s1.body)); + const r1 = Object.fromEntries((await settle(s1.body.batchId)).functions.map((f) => [f.symbol, f])); + assert.equal(r1[a].state, 'approved', 'byte-identical needs no human'); + assert.equal(r1[b].state, 'peer_review'); + + // Round 1: a peer claim covers the batch's waiting functions; two confirm, one finds a difference. + const pc = await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[0]); + assert.deepEqual(pc.body.functions.map((f) => f.symbol), [b]); + await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'same', reason: 'same logic, checked branches and calls' }, peers[0]); + await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[1]); + await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'same', reason: 'same logic, checked branches and calls' }, peers[1]); + await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[2]); + await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'different', reason: 'the original checks the flag first, this does not' }, peers[2]); + const back = await batchState(s1.body.batchId); + assert.equal(back.state, 'returned', 'the batch goes back as a whole'); + assert.ok(back.functions.every((f) => ['returned', 'needs_cleanup'].includes(f.state)), JSON.stringify(back.functions)); + + // Round 2: the author fixes B (new fingerprint) and resubmits; three peers confirm. + bHash = 'hash-b-2'; + for (const id of taskIds) assert.equal((await call('POST', '/api/tasks/claim', { taskId: id }, author)).status, 200); + const s2 = await submit(); + const r2 = Object.fromEntries((await settle(s2.body.batchId)).functions.map((f) => [f.symbol, f])); + assert.equal(r2[b].state, 'peer_review', 'changed code gets reviewed again'); + for (const t of peers) { + await call('POST', '/api/peer-reviews/claim', { attemptId: r2[b].attemptId }, t); + await call('POST', `/api/peer-reviews/${r2[b].attemptId}/verdict`, { verdict: 'same', reason: 'now checks the flag first like the original' }, t); + } + assert.equal((await batchState(s2.body.batchId)).functions.find((f) => f.symbol === b).state, 'in_review'); + + // One human click for the batch; one merge commit. + const rev = await request('POST', `/api/batches/${s2.body.batchId}/review`, { action: 'approve' }, { cookie: reviewer }); + assert.equal(rev.status, 200, JSON.stringify(rev.body)); + await ctx.merger.pump(); + const merged = await batchState(s2.body.batchId); + assert.equal(merged.state, 'merged', JSON.stringify(merged)); + assert.match(ctx.repo.git(['log', '-1', '--format=%B']), /Decompile 2 functions in src\/batch\.cpp[\s\S]*Peer-reviewed-by/); + assert.equal(ctx.store.get('SELECT confirmed FROM func_status WHERE symbol_id = ?', b).confirmed, 1); + assert.equal(ctx.store.get('SELECT code_hash FROM func_status WHERE symbol_id = ?', a).code_hash, 'hash-a'); + + // Carry-over: an unchanged, peer-confirmed function keeps its verdicts in a resubmission. + const taskB = ctx.store.get('SELECT task_id FROM attempts WHERE id = ?', r2[b].attemptId).task_id; + const next = { task_id: taskB, id: Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, base_commit, files_json, state, created_at) VALUES (?, ?, ?, '{}', 'verifying', ?)`, taskB, ctx.accounts.byName('bauthor').id, ctx.repo.head(), now()).lastInsertRowid) }; + ctx.store.run('UPDATE attempts SET result_json = ? WHERE id = ?', JSON.stringify({ feedback: { target: { codeHash: 'hash-b-2' } } }), r2[b].attemptId); + assert.equal(ctx.verifier.carryOver(next, { codeHash: 'hash-b-2' }), true); + assert.equal(ctx.verifier.carryOver(next, { codeHash: 'hash-b-3' }), false); + + // Guard: a submission may not change functions it does not hold. + const outside = ctx.verifier.changedOutside({ units: [{ functions: [{ id: a, file: 'src/batch.cpp', codeHash: 'hash-a-changed' }] }] }, { 'src/batch.cpp': '' }, new Set([b])); + assert.deepEqual(outside.map((o) => o.id), [a]); + + ctx.verifier.check = realCheck; +}); diff --git a/tools/build-td/build.mjs b/tools/build-td/build.mjs new file mode 100644 index 0000000..8c9d2cf --- /dev/null +++ b/tools/build-td/build.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// Builds `td` as standalone executables (Node single executable applications): contributors need +// no Node.js. Output in dist/: td-windows-x64.exe, td-linux-x64, SHA256SUMS. +// +// node tools/build-td/build.mjs +// +// Needs network access once (esbuild and postject through npx, the Linux Node runtime from +// nodejs.org, checked against its published SHA256SUMS). Uses the Node runtime that runs this +// script for Windows, so build with the same Node version as NODE_VERSION. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const DIST = path.join(ROOT, 'dist'); +const NODE_VERSION = 'v22.21.0'; +const FUSE = 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2'; +const SCRIPTS = ['ApplySymbols.java', 'DecompileAt.java', 'DisassembleAt.java', 'ExportFunctions.java', 'ExportOriginals.java']; +const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx'; +const sh = (cmd, args) => execFileSync(cmd, args, { cwd: ROOT, stdio: 'inherit', shell: process.platform === 'win32' }); +const sha256 = (file) => crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + +if (process.version !== NODE_VERSION) throw new Error(`build with Node ${NODE_VERSION} (this is ${process.version})`); +fs.rmSync(DIST, { recursive: true, force: true }); +fs.mkdirSync(DIST, { recursive: true }); + +// 1. One CommonJS file (single executable applications run a CommonJS script). +sh(npx, ['--yes', 'esbuild@0.24.0', 'src/client/cli.js', '--bundle', '--platform=node', '--target=node22', '--format=cjs', '--log-level=warning', `--outfile=${path.join(DIST, 'td.cjs')}`]); + +// 2. The preparation blob: code plus the embedded Ghidra scripts. No code cache or snapshot, so +// the same blob works for every platform's Node runtime. +const config = { + main: path.join(DIST, 'td.cjs'), + output: path.join(DIST, 'td.blob'), + disableExperimentalSEAWarning: true, + useCodeCache: false, + useSnapshot: false, + assets: Object.fromEntries(SCRIPTS.map((s) => [s, path.join(ROOT, 'ghidra_scripts', s)])), +}; +fs.writeFileSync(path.join(DIST, 'sea-config.json'), JSON.stringify(config, null, 2)); +execFileSync(process.execPath, ['--experimental-sea-config', path.join(DIST, 'sea-config.json')], { stdio: 'inherit' }); + +// 3. Runtimes: this Node for Windows (when building on Windows), the official Linux build otherwise. +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'td-build-')); +const targets = []; +if (process.platform === 'win32') { + const exe = path.join(DIST, 'td-windows-x64.exe'); + fs.copyFileSync(process.execPath, exe); + targets.push(exe); +} +{ + const name = `node-${NODE_VERSION}-linux-x64`; + const base = `https://nodejs.org/dist/${NODE_VERSION}`; + const sums = await (await fetch(`${base}/SHASUMS256.txt`)).text(); + const want = new RegExp(`^([0-9a-f]{64})\\s+${name}\\.tar\\.gz$`, 'm').exec(sums)?.[1]; + if (!want) throw new Error('no checksum for the Linux runtime'); + const tgz = path.join(tmp, `${name}.tar.gz`); + fs.writeFileSync(tgz, Buffer.from(await (await fetch(`${base}/${name}.tar.gz`)).arrayBuffer())); + if (sha256(tgz) !== want) throw new Error('Linux runtime checksum mismatch'); + execFileSync(process.platform === 'win32' ? path.join(process.env.SystemRoot, 'System32', 'tar.exe') : 'tar', ['-xzf', tgz, '-C', tmp, `${name}/bin/node`], { stdio: 'inherit' }); + const bin = path.join(DIST, 'td-linux-x64'); + fs.copyFileSync(path.join(tmp, name, 'bin', 'node'), bin); + targets.push(bin); +} + +// 4. Inject the blob into each runtime. +for (const t of targets) { + const args = ['--yes', 'postject@1.0.0-alpha.6', t, 'NODE_SEA_BLOB', config.output, '--sentinel-fuse', FUSE]; + sh(npx, args); +} + +fs.writeFileSync(path.join(DIST, 'SHA256SUMS'), targets.map((t) => `${sha256(t)} ${path.basename(t)}`).join('\n') + '\n'); +for (const f of ['td.cjs', 'td.blob', 'sea-config.json']) fs.rmSync(path.join(DIST, f)); +fs.rmSync(tmp, { recursive: true, force: true }); +console.log(fs.readFileSync(path.join(DIST, 'SHA256SUMS'), 'utf8'));