Batches, auto-merge, standalone td, one-line install, self-update, server Ghidra
- Batches: claim every open function of a unit or address range (td claim --unit/--range), submit the file once; one compile, per-function routing, merged as a whole. A failing function returns the batch; peer verdicts carry over to functions whose compiled code is unchanged (code fingerprints). Submissions may not change functions they do not hold. - Byte-identical functions are approved and merged without human review (matchPolicy.autoMergeIdentical); peer-confirmed ones need one click per batch. - Remote check: td check compiles on the server when there is no local toolchain. - /api/original: assembly and pseudocode of the original from the server's Ghidra export (objdump before an export exists); td disasm/decompile fall back to it. - td as a standalone executable (Node single executable application; tools/build-td), one-line installers (install.ps1 / install.sh) with checksum verification, td connect for any MCP client, td mcp --workspace, self-update (td update, hourly check). - Server-side Ghidra project: built once per version, refreshed after merges, downloadable for contributors (td ghidra download); JDK 21 and the pinned Ghidra in deploy/install.sh. - Agent instructions: work continuously, batches first, peer reviews in between. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RNwtXPYqFu3oKaaXwYwNWW
This commit is contained in:
Clank Together
parent
984614f2f7
commit
125f5c4e7d
27 changed files with 1751 additions and 241 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -2,3 +2,4 @@ node_modules/
|
|||
*.log
|
||||
# Operational notes about the deployment, not part of the tool
|
||||
PORTS-TO-FORWARD.md
|
||||
dist/
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ set -eu
|
|||
|
||||
HOST="${1:?usage: install.sh <public-hostname>}"
|
||||
NODE_VERSION=v22.21.0
|
||||
GHIDRA_VERSION=11.2.1
|
||||
GHIDRA_URL=https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_11.2.1_build/ghidra_11.2.1_PUBLIC_20241105.zip
|
||||
GHIDRA_SHA256=ce4db5117da0fbaf8f33863fec4f40902f754f06b68945a59fb1c0f9b1bc461c
|
||||
SRC=$(cd "$(dirname "$0")/.." && pwd)
|
||||
APP=/opt/togetherdecomp
|
||||
DATA=/var/lib/togetherdecomp
|
||||
|
|
@ -12,7 +15,8 @@ DATA=/var/lib/togetherdecomp
|
|||
echo "== packages"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -q
|
||||
apt-get install -y -q git curl ca-certificates nftables caddy docker.io xz-utils >/dev/null
|
||||
# binutils: objdump disassembles original functions for /api/original before a Ghidra export exists.
|
||||
apt-get install -y -q git curl ca-certificates nftables caddy docker.io xz-utils binutils >/dev/null
|
||||
|
||||
echo "== node $NODE_VERSION"
|
||||
if [ "$(/opt/node/bin/node --version 2>/dev/null || true)" != "$NODE_VERSION" ]; then
|
||||
|
|
@ -25,6 +29,19 @@ if [ "$(/opt/node/bin/node --version 2>/dev/null || true)" != "$NODE_VERSION" ];
|
|||
rm -rf "$tmp"
|
||||
fi
|
||||
|
||||
echo "== ghidra $GHIDRA_VERSION (server-side project and pseudocode)"
|
||||
# The JDK, not just the runtime: Ghidra compiles its .java scripts.
|
||||
apt-get install -y -q openjdk-21-jdk-headless unzip >/dev/null
|
||||
GHIDRA_HOME="/opt/ghidra/ghidra_${GHIDRA_VERSION}_PUBLIC"
|
||||
if [ ! -x "$GHIDRA_HOME/support/analyzeHeadless" ]; then
|
||||
tmp=$(mktemp -d)
|
||||
curl -fsSL "$GHIDRA_URL" -o "$tmp/ghidra.zip"
|
||||
echo "$GHIDRA_SHA256 $tmp/ghidra.zip" | sha256sum -c - >/dev/null
|
||||
mkdir -p /opt/ghidra
|
||||
unzip -q -o "$tmp/ghidra.zip" -d /opt/ghidra
|
||||
rm -rf "$tmp"
|
||||
fi
|
||||
|
||||
echo "== docker sandbox settings"
|
||||
mkdir -p /etc/docker
|
||||
cat > /etc/docker/daemon.json <<'EOF'
|
||||
|
|
@ -53,13 +70,14 @@ if [ ! -f "$DATA/server.json" ]; then
|
|||
fi
|
||||
# Production settings, keeping anything else already configured.
|
||||
UIDGID="$(id -u togetherdecomp):$(id -g togetherdecomp)"
|
||||
runuser -u togetherdecomp -- $NODE -e '
|
||||
runuser -u togetherdecomp -- env GHIDRA_HOME="$GHIDRA_HOME" $NODE -e '
|
||||
const fs = require("fs"); const [file, host, user] = process.argv.slice(1);
|
||||
const c = JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
c.listen = { host: "127.0.0.1", port: 8080 };
|
||||
c.publicUrl = "https://" + host;
|
||||
c.trustProxy = true;
|
||||
c.allowUnsandboxedRunners = false;
|
||||
c.ghidraHome = process.env.GHIDRA_HOME || c.ghidraHome || null;
|
||||
for (const r of Object.values(c.runners || {})) if (r.type === "docker") r.user = user;
|
||||
fs.writeFileSync(file, JSON.stringify(c, null, 2) + "\n");
|
||||
' "$DATA/server.json" "$HOST" "$UIDGID"
|
||||
|
|
|
|||
113
ghidra_scripts/ExportOriginals.java
Normal file
113
ghidra_scripts/ExportOriginals.java
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
// Exports the disassembly and the decompiler output of every function as JSON lines, for the
|
||||
// coordinator's /api/original (contributors then need no local Ghidra):
|
||||
// {"addr":"0x140099b90","asm":"...","c":"..."}
|
||||
// Args: <output.jsonl> [decompile timeout seconds, default 30]
|
||||
// Run on a project built with `td ghidra build` (symbol names applied), with -readOnly.
|
||||
// @category ClankTogether
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import ghidra.app.decompiler.DecompInterface;
|
||||
import ghidra.app.decompiler.DecompileResults;
|
||||
import ghidra.app.script.GhidraScript;
|
||||
import ghidra.program.model.address.Address;
|
||||
import ghidra.program.model.listing.Function;
|
||||
import ghidra.program.model.listing.FunctionIterator;
|
||||
import ghidra.program.model.listing.Instruction;
|
||||
import ghidra.program.model.listing.InstructionIterator;
|
||||
import ghidra.program.model.symbol.Reference;
|
||||
import ghidra.program.model.symbol.Symbol;
|
||||
|
||||
public class ExportOriginals extends GhidraScript {
|
||||
|
||||
@Override
|
||||
protected void run() throws Exception {
|
||||
String[] args = getScriptArgs();
|
||||
int timeout = args.length > 1 ? Integer.parseInt(args[1]) : 30;
|
||||
DecompInterface ifc = new DecompInterface();
|
||||
ifc.openProgram(currentProgram);
|
||||
int n = 0;
|
||||
int failed = 0;
|
||||
try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(args[0]), StandardCharsets.UTF_8))) {
|
||||
FunctionIterator it = currentProgram.getFunctionManager().getFunctions(true);
|
||||
while (it.hasNext() && !monitor.isCancelled()) {
|
||||
Function f = it.next();
|
||||
if (f.isExternal() || f.isThunk()) {
|
||||
continue;
|
||||
}
|
||||
String c;
|
||||
DecompileResults r = ifc.decompileFunction(f, timeout, monitor);
|
||||
if (r.decompileCompleted() && r.getDecompiledFunction() != null) {
|
||||
c = r.getDecompiledFunction().getC();
|
||||
} else {
|
||||
c = null;
|
||||
failed++;
|
||||
}
|
||||
out.write("{\"addr\":\"0x" + Long.toHexString(f.getEntryPoint().getOffset()) + "\",\"asm\":" + json(disassemble(f)) + ",\"c\":" + (c == null ? "null" : json(c)) + "}\n");
|
||||
if (++n % 1000 == 0) {
|
||||
println("exported " + n + " functions");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
ifc.dispose();
|
||||
}
|
||||
println("exported " + n + " functions, " + failed + " without decompiler output");
|
||||
}
|
||||
|
||||
private String disassemble(Function f) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
Address entry = f.getEntryPoint();
|
||||
sb.append("; ").append(f.getName(true)).append(" @ ").append(entry).append(" ").append(f.getSignature().getPrototypeString()).append('\n');
|
||||
InstructionIterator it = currentProgram.getListing().getInstructions(f.getBody(), true);
|
||||
while (it.hasNext()) {
|
||||
Instruction insn = it.next();
|
||||
Address a = insn.getAddress();
|
||||
if (!a.equals(entry)) {
|
||||
for (Symbol s : currentProgram.getSymbolTable().getSymbols(a)) {
|
||||
sb.append(s.getName(true)).append(":\n");
|
||||
}
|
||||
}
|
||||
StringBuilder bytes = new StringBuilder();
|
||||
try {
|
||||
for (byte b : insn.getBytes()) {
|
||||
bytes.append(String.format("%02x ", b & 0xff));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
bytes.append("?? ");
|
||||
}
|
||||
StringBuilder refs = new StringBuilder();
|
||||
for (Reference ref : insn.getReferencesFrom()) {
|
||||
Symbol s = currentProgram.getSymbolTable().getPrimarySymbol(ref.getToAddress());
|
||||
if (s != null) {
|
||||
refs.append(" ; -> ").append(s.getName(true));
|
||||
}
|
||||
}
|
||||
sb.append(String.format(" +0x%04x %08x %-24s %s%s%n", a.subtract(entry), a.getOffset(), bytes.toString().trim(), insn.toString(), refs));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String json(String s) {
|
||||
StringBuilder sb = new StringBuilder("\"");
|
||||
for (int i = 0; i < s.length(); i++) {
|
||||
char ch = s.charAt(i);
|
||||
switch (ch) {
|
||||
case '"': sb.append("\\\""); break;
|
||||
case '\\': sb.append("\\\\"); break;
|
||||
case '\n': sb.append("\\n"); break;
|
||||
case '\r': sb.append("\\r"); break;
|
||||
case '\t': sb.append("\\t"); break;
|
||||
default:
|
||||
if (ch < 0x20) {
|
||||
sb.append(String.format("\\u%04x", (int) ch));
|
||||
} else {
|
||||
sb.append(ch);
|
||||
}
|
||||
}
|
||||
}
|
||||
return sb.append('"').toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -2,11 +2,13 @@
|
|||
// `td` - contributor CLI for Clank Together.
|
||||
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { parseArgs } from 'node:util';
|
||||
import { Workspace } from './workspace.js';
|
||||
import * as ops from './ops.js';
|
||||
import * as ghidra from './ghidra.js';
|
||||
import { startMcp } from './mcp.js';
|
||||
import { maybeUpdate } from './update.js';
|
||||
import { hex } from '../shared/annotations.js';
|
||||
|
||||
const HELP = `td - Clank Together contributor client
|
||||
|
|
@ -18,17 +20,23 @@ Setup
|
|||
[--toolchain <toolchain root>] compiler layout of the project (e.g. re/toolchain)
|
||||
[--vc <VC dir>] [--sdk <SDK dir>] [--image <docker image>]
|
||||
td sync pull main and refresh the symbol database snapshot
|
||||
td ghidra install | build | sync pinned Ghidra: install, (re)build the project, re-apply symbols
|
||||
td ghidra download the server's Ghidra project (names applied, refreshed after merges)
|
||||
td ghidra install | build | sync pinned Ghidra: install it, analyze locally, re-apply symbols
|
||||
td update update td now (it also checks by itself once an hour)
|
||||
td connect [--workspace <dir>] connect your AI (registers with Claude Code, prints MCP settings)
|
||||
td mcp [--workspace <dir>] run the MCP server for your agent (stdio)
|
||||
|
||||
Work
|
||||
td tasks [--state S] [--limit N] list claimable tasks
|
||||
td claim [taskId] claim a task, prints its context
|
||||
td claim --unit <file> [--limit N] claim every open function of a source file (a batch)
|
||||
td claim --range <start>-<end> [--limit N] claim every open function of an address range
|
||||
td task <taskId> show a task's context
|
||||
td check [files...] compile + compare locally (default: changed files)
|
||||
td submit <taskId> [files...] [--notes T] submit for server verification
|
||||
td submit [taskId...] [--notes T] submit everything you hold (or these tasks) as one batch
|
||||
td batch <batchId> state of a submitted batch, per function
|
||||
td result <attemptId> show a verification result
|
||||
td heartbeat <taskId> | td release <taskId>
|
||||
td heartbeat [taskId] | td release <taskId> extend your leases (all without an id)
|
||||
td config [ghidraHome|server <value>] show or change workspace settings
|
||||
td symbol <id|name|0xaddr> look up a symbol
|
||||
td symbol new <kind> <0xaddr> [--name N] get a stable ID for a new global/vtable/string/function
|
||||
|
|
@ -59,6 +67,29 @@ function printCheck(r) {
|
|||
for (const c of r.referenceConflicts) console.log(`conflict: ${c.message ?? c}`);
|
||||
}
|
||||
|
||||
/** How an MCP client starts us: the td binary itself, or node + this script when run from source. */
|
||||
async function launcher() {
|
||||
const sea = await import('node:sea').catch(() => null);
|
||||
return sea?.isSea?.() ? { command: process.execPath, args: [] } : { command: process.execPath, args: [path.resolve(process.argv[1])] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers the MCP server with Claude Code when it is installed, and prints the settings
|
||||
* for every other MCP client.
|
||||
*/
|
||||
async function connect(dir) {
|
||||
const ws = Workspace.find(dir);
|
||||
const l = await launcher();
|
||||
const args = [...l.args, 'mcp', '--workspace', ws.dir];
|
||||
const claude = spawnSync(process.platform === 'win32' ? 'claude.cmd' : 'claude', ['mcp', 'add', '--scope', 'user', 'clank-together', '--', l.command, ...args], { encoding: 'utf8', shell: process.platform === 'win32' });
|
||||
if (claude.status === 0) console.log('Claude Code: connected (MCP server "clank-together"). Start claude and ask it to read AGENTS.md and claim a task.');
|
||||
else console.log('Claude Code: not found, or already connected (claude mcp list shows it).');
|
||||
console.log(`\nOther MCP clients (Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI, ...): add this to their MCP server settings:
|
||||
${JSON.stringify({ mcpServers: { 'clank-together': { command: l.command, args } } }, null, 2)}
|
||||
|
||||
Agents without MCP can run the td commands directly: td tasks, td claim, td check, td submit, td peer ...`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { values, positionals } = parseArgs({
|
||||
allowPositionals: true,
|
||||
|
|
@ -74,8 +105,12 @@ async function main() {
|
|||
state: { type: 'string' },
|
||||
limit: { type: 'string', default: '20' },
|
||||
notes: { type: 'string' },
|
||||
unit: { type: 'string' },
|
||||
range: { type: 'string' },
|
||||
files: { type: 'string' },
|
||||
reason: { type: 'string' },
|
||||
json: { type: 'boolean', default: false },
|
||||
force: { type: 'boolean', default: false },
|
||||
workspace: { type: 'string' },
|
||||
help: { type: 'boolean', short: 'h' },
|
||||
},
|
||||
|
|
@ -83,22 +118,24 @@ async function main() {
|
|||
const [cmd, ...args] = positionals;
|
||||
if (!cmd || values.help) return print(HELP);
|
||||
|
||||
// Self-update (standalone binary only): at most once an hour, never fatal.
|
||||
const serverHint = cmd === 'setup' ? args[0] : (() => { try { return Workspace.find(values.workspace ?? process.cwd()).config.server; } catch { return null; } })();
|
||||
const updated = await maybeUpdate({ server: serverHint, force: cmd === 'update' });
|
||||
if (updated) process.stderr.write(`${updated}\n`);
|
||||
if (cmd === 'update') return print(updated ?? 'td is up to date');
|
||||
|
||||
if (cmd === 'setup') {
|
||||
const [server, dir] = args;
|
||||
if (!server || !dir) throw new Error('usage: td setup <server-url> <dir> --token <API token from the website>');
|
||||
const ws = await ops.setup({ server, dir, token: values.token, exe: values.exe, toolchain: values.toolchain, runner: values.runner, vc: values.vc, sdk: values.sdk, image: values.image });
|
||||
const remote = ws.config.runners?.[ws.project.toolchain]?.type === 'remote';
|
||||
console.log(`\nworkspace ready at ${ws.dir}
|
||||
next:
|
||||
cd ${path.relative(process.cwd(), ws.dir) || '.'}
|
||||
td ghidra install && td ghidra build (one-time, takes a while)
|
||||
td check src/... (self-test once there is code)
|
||||
connect your agent (any MCP client works):
|
||||
Claude Code: claude mcp add clank-together -- node "${path.resolve(process.argv[1])}" mcp --workspace "${ws.dir}"
|
||||
other clients (Cursor, Windsurf, VS Code, Cline, Claude Desktop, ...), in their MCP server settings:
|
||||
${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [path.resolve(process.argv[1]), 'mcp', '--workspace', ws.dir] } } }, null, 2).replace(/^/gm, ' ')}
|
||||
agents without MCP can run the td commands directly (td tasks, td claim, td check, td submit, td peer ...).`);
|
||||
compiling: ${remote ? "on the server (no local compiler needed); add one later with td setup ... --toolchain <dir>" : 'locally'}
|
||||
original code: ${ws.config.ghidraHome ? 'local Ghidra' : "from the server; a local Ghidra is optional (td ghidra install && td ghidra build)"}
|
||||
connect your AI with: td connect --workspace "${ws.dir}"`);
|
||||
return;
|
||||
}
|
||||
if (cmd === 'connect') return connect(values.workspace ?? process.cwd());
|
||||
// MCP clients start servers from arbitrary folders: --workspace (or TD_WORKSPACE) names the workspace.
|
||||
if (cmd === 'mcp') return startMcp(values.workspace ?? process.env.TD_WORKSPACE ?? process.cwd());
|
||||
|
||||
|
|
@ -112,12 +149,20 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat
|
|||
console.log(`#${String(t.id).padEnd(6)} ${t.symbol_id} ${t.addr} size ${String(t.size ?? '?').padEnd(6)} d${t.difficulty} ${t.state.padEnd(13)} best ${(t.best_score * 100).toFixed(0)}% ${t.name ?? ''} ${t.unit ?? ''}`);
|
||||
}
|
||||
return;
|
||||
case 'claim':
|
||||
case 'claim': {
|
||||
// td claim [taskId] | td claim --unit <file> | td claim --range 0x140001000-0x140002000 [--limit N]
|
||||
if (values.unit || values.range) {
|
||||
const [start, end] = values.range ? values.range.split('-') : [];
|
||||
return print(await ops.claimBatch(ws, { unit: values.unit ? ops.relPath(ws, values.unit) : undefined, start, end, limit: Number(values.limit) || 50 }));
|
||||
}
|
||||
return print(await client.post('/api/tasks/claim', { taskId: args[0] ? Number(args[0]) : undefined }));
|
||||
}
|
||||
case 'task':
|
||||
return print(await client.get(`/api/tasks/${args[0]}`));
|
||||
case 'heartbeat':
|
||||
return print(await client.post(`/api/tasks/${args[0]}/heartbeat`));
|
||||
return print(args[0] ? await client.post(`/api/tasks/${args[0]}/heartbeat`) : await client.post('/api/tasks/heartbeat-all'));
|
||||
case 'batch':
|
||||
return print(await client.get(`/api/batches/${args[0]}`));
|
||||
case 'release':
|
||||
return print(await client.post(`/api/tasks/${args[0]}/release`));
|
||||
case 'check': {
|
||||
|
|
@ -125,8 +170,11 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat
|
|||
return values.json ? print(r) : printCheck(r);
|
||||
}
|
||||
case 'submit': {
|
||||
const [taskId, ...files] = args;
|
||||
return print(await ops.submit(ws, Number(taskId), { files: files.map((f) => ops.relPath(ws, f)), notes: values.notes, agentMeta: { client: 'td-cli' } }));
|
||||
// td submit everything you hold, as one batch
|
||||
// td submit <taskId>... [--files a.cpp,b.h]
|
||||
const ids = args.filter((x) => /^\d+$/.test(x)).map(Number);
|
||||
const files = values.files ? values.files.split(',').map((f) => ops.relPath(ws, f)) : [];
|
||||
return print(await ops.submit(ws, ids, { files, notes: values.notes, agentMeta: { client: 'td-cli' } }));
|
||||
}
|
||||
case 'result': {
|
||||
const r = await client.get(`/api/attempts/${args[0]}`);
|
||||
|
|
@ -166,13 +214,15 @@ ${JSON.stringify({ mcpServers: { 'clank-together': { command: 'node', args: [pat
|
|||
if (args[0] === 'install') return print(`Ghidra installed at ${await ghidra.installGhidra(ws)}`);
|
||||
if (args[0] === 'build') return void (await ghidra.buildGhidraProject(ws), print('Ghidra project built'));
|
||||
if (args[0] === 'sync') return void (await ghidra.syncGhidraProject(ws), print('symbols applied'));
|
||||
throw new Error('usage: td ghidra install|build|sync');
|
||||
if (args[0] === 'download') {
|
||||
const r = await ghidra.downloadGhidraProject(ws, { force: values.force });
|
||||
return print(r.updated ? `Ghidra project of ${r.info.version} downloaded${ws.config.ghidraHome ? '' : ' (open it with Ghidra 11.2.1, or run td ghidra install)'}` : 'the Ghidra project is up to date');
|
||||
}
|
||||
throw new Error('usage: td ghidra install|download|build|sync');
|
||||
case 'decompile':
|
||||
case 'disasm': {
|
||||
const s = ws.findSymbol(args[0]);
|
||||
const a = s?.addr ?? (/^0x/i.test(args[0] ?? '') ? Number.parseInt(args[0], 16) : null);
|
||||
if (a == null) throw new Error(`unknown symbol ${args[0]}`);
|
||||
return print(cmd === 'decompile' ? await ghidra.decompile(ws, a) : await ghidra.disassemble(ws, a));
|
||||
// Local Ghidra if configured, otherwise the server's export.
|
||||
return print(await ops.original(ws, args[0], cmd === 'decompile' ? 'c' : 'asm'));
|
||||
}
|
||||
default:
|
||||
print(HELP);
|
||||
|
|
|
|||
|
|
@ -13,7 +13,22 @@ import { hex } from '../shared/annotations.js';
|
|||
import { loadToolchainLock } from '../shared/project.js';
|
||||
import { Api } from './api.js';
|
||||
|
||||
const SCRIPTS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts');
|
||||
const SCRIPT_NAMES = ['ApplySymbols.java', 'DecompileAt.java', 'DisassembleAt.java', 'ExportFunctions.java', 'ExportOriginals.java'];
|
||||
|
||||
/**
|
||||
* The Ghidra scripts: next to the sources when run from a checkout or npm install; extracted
|
||||
* into the workspace when td runs as a single executable (they are embedded as assets).
|
||||
*/
|
||||
async function scriptsDir(ws) {
|
||||
const sea = await import('node:sea').catch(() => null);
|
||||
if (sea?.isSea?.()) {
|
||||
const dir = path.join(ws.tdDir, 'ghidra_scripts');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
for (const name of SCRIPT_NAMES) fs.writeFileSync(path.join(dir, name), sea.getAsset(name, 'utf8'));
|
||||
return dir;
|
||||
}
|
||||
return path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts');
|
||||
}
|
||||
const PROJECT_NAME = 'td';
|
||||
const PROGRAM = 'target.exe';
|
||||
|
||||
|
|
@ -33,7 +48,7 @@ async function runHeadless(ws, args, timeoutMs) {
|
|||
const exe = headless(ws);
|
||||
const projDir = path.join(ws.tdDir, 'ghidra');
|
||||
fs.mkdirSync(projDir, { recursive: true });
|
||||
const full = [projDir, PROJECT_NAME, ...args, '-scriptPath', SCRIPTS_DIR];
|
||||
const full = [projDir, PROJECT_NAME, ...args, '-scriptPath', await scriptsDir(ws)];
|
||||
// .bat files need a shell on Windows.
|
||||
const r = os.platform() === 'win32' ? await run('cmd.exe', ['/d', '/s', '/c', exe, ...full], { timeoutMs }) : await run(exe, full, { timeoutMs });
|
||||
if (r.code !== 0) throw new Error(`Ghidra failed (exit ${r.code}):\n${r.output.slice(-4000)}`);
|
||||
|
|
@ -104,3 +119,28 @@ export function decompile(ws, addr) {
|
|||
export function disassemble(ws, addr) {
|
||||
return query(ws, 'DisassembleAt.java', ['0x' + hex(addr)]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads the server's Ghidra project (built and refreshed on the server, with the shared symbol
|
||||
* names) into the workspace, replacing the local one. Nothing happens when it is up to date.
|
||||
*/
|
||||
export async function downloadGhidraProject(ws, { force = false } = {}) {
|
||||
const client = new Api(ws.config.server, ws.config.token);
|
||||
const info = await client.get('/api/ghidra-project');
|
||||
if (!force && ws.config.ghidraPackage === info.sha256 && fs.existsSync(path.join(ws.tdDir, 'ghidra', `${PROJECT_NAME}.gpr`))) {
|
||||
return { updated: false, info };
|
||||
}
|
||||
const tgz = path.join(ws.tdDir, 'ghidra-project.tar.gz');
|
||||
console.error(`downloading the Ghidra project of ${info.version} (${(info.size / 1048576).toFixed(0)} MB)...`);
|
||||
await client.download(info.download, tgz, info.sha256);
|
||||
const projDir = path.join(ws.tdDir, 'ghidra');
|
||||
fs.rmSync(projDir, { recursive: true, force: true });
|
||||
fs.mkdirSync(projDir, { recursive: true });
|
||||
const tar = os.platform() === 'win32' ? path.join(process.env.SystemRoot ?? 'C:\Windows', 'System32', 'tar.exe') : 'tar';
|
||||
const r = await run(tar, ['-xzf', tgz, '-C', projDir], { timeoutMs: 1800_000 });
|
||||
if (r.code !== 0) throw new Error(`could not unpack the Ghidra project: ${r.output}`);
|
||||
fs.rmSync(tgz, { force: true });
|
||||
ws.config.ghidraPackage = info.sha256;
|
||||
ws.save();
|
||||
return { updated: true, info };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
import readline from 'node:readline';
|
||||
import { Workspace } from './workspace.js';
|
||||
import * as ops from './ops.js';
|
||||
import * as ghidra from './ghidra.js';
|
||||
import { hex } from '../shared/annotations.js';
|
||||
|
||||
const PROTOCOL_VERSIONS = ['2025-06-18', '2025-03-26', '2024-11-05'];
|
||||
|
|
@ -33,9 +32,9 @@ const TOOLS = [
|
|||
},
|
||||
{
|
||||
name: 'td_heartbeat',
|
||||
description: 'Extend the lease on a task you are still working on.',
|
||||
inputSchema: { type: 'object', properties: { taskId: int('task id') }, required: ['taskId'] },
|
||||
run: async (ws, a) => ops.api(ws).post(`/api/tasks/${a.taskId}/heartbeat`),
|
||||
description: 'Extend your leases while you are still working (all of them, or one taskId). Leases expire after two hours otherwise.',
|
||||
inputSchema: { type: 'object', properties: { taskId: int('task id; omit for all') } },
|
||||
run: async (ws, a) => (a.taskId ? ops.api(ws).post(`/api/tasks/${a.taskId}/heartbeat`) : ops.api(ws).post('/api/tasks/heartbeat-all')),
|
||||
},
|
||||
{
|
||||
name: 'td_release_task',
|
||||
|
|
@ -47,13 +46,13 @@ const TOOLS = [
|
|||
name: 'td_decompile',
|
||||
description: 'Ghidra decompiler output for a function in the original executable (by symbol ID, name or hex address). A starting point only; the result must be rewritten into matching C++.',
|
||||
inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] },
|
||||
run: async (ws, a) => ghidra.decompile(ws, resolveAddr(ws, a.symbol)),
|
||||
run: async (ws, a) => ops.original(ws, a.symbol, 'c'),
|
||||
},
|
||||
{
|
||||
name: 'td_disassemble',
|
||||
description: 'Disassembly of a function in the original executable, the ground truth to match.',
|
||||
inputSchema: { type: 'object', properties: { symbol: str('fn_00123, a name, or 0x401000') }, required: ['symbol'] },
|
||||
run: async (ws, a) => ghidra.disassemble(ws, resolveAddr(ws, a.symbol)),
|
||||
run: async (ws, a) => ops.original(ws, a.symbol, 'asm'),
|
||||
},
|
||||
{
|
||||
name: 'td_lookup_symbol',
|
||||
|
|
@ -82,9 +81,24 @@ const TOOLS = [
|
|||
},
|
||||
{
|
||||
name: 'td_submit',
|
||||
description: 'Submit your work on a leased task for server verification. Defaults to all changed files the task allows. Ends the lease.',
|
||||
inputSchema: { type: 'object', properties: { taskId: int('task id'), files: { type: 'array', items: { type: 'string' } }, notes: str('short notes for reviewers and later agents') }, required: ['taskId'] },
|
||||
run: async (ws, a) => ops.submit(ws, a.taskId, { files: a.files, notes: a.notes, agentMeta: { client: 'td-mcp' } }),
|
||||
description: 'Submit your work for server verification, as one batch: every task you hold (default) or the given taskIds, with all changed files they allow. Byte-identical functions are merged automatically; others go to peer review. Ends the leases. Do not wait for the result: claim the next work right away and check results later.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: { taskIds: { type: 'array', items: { type: 'integer' }, description: 'tasks to submit; default: all you hold' }, taskId: int('a single task (same as taskIds: [id])'), files: { type: 'array', items: { type: 'string' } }, notes: str('short notes for reviewers and later agents') },
|
||||
},
|
||||
run: async (ws, a) => ops.submit(ws, a.taskIds ?? (a.taskId ? [a.taskId] : []), { files: a.files, notes: a.notes, agentMeta: { client: 'td-mcp' } }),
|
||||
},
|
||||
{
|
||||
name: 'td_claim_batch',
|
||||
description: 'Claim many functions at once: every open function of a source file (unit), or of an address range of the current version, up to limit (default 50). Work through them in the file, td_check, then td_submit them together. Much cheaper than one function at a time.',
|
||||
inputSchema: { type: 'object', properties: { unit: str('repo-relative source file, e.g. Engine/Source/FrameXML/LoadXML.cpp'), start: str('range start, hex'), end: str('range end, hex (exclusive)'), limit: int('max functions, default 50') } },
|
||||
run: async (ws, a) => ops.claimBatch(ws, a),
|
||||
},
|
||||
{
|
||||
name: 'td_get_batch',
|
||||
description: 'State of a submitted batch, per function (approved, peer_review, in_review, needs_cleanup, merged...).',
|
||||
inputSchema: { type: 'object', properties: { batchId: int('batch id') }, required: ['batchId'] },
|
||||
run: async (ws, a) => ops.api(ws).get(`/api/batches/${a.batchId}`),
|
||||
},
|
||||
{
|
||||
name: 'td_get_result',
|
||||
|
|
@ -132,12 +146,6 @@ const TOOLS = [
|
|||
},
|
||||
];
|
||||
|
||||
function resolveAddr(ws, q) {
|
||||
const s = ws.findSymbol(q);
|
||||
if (s?.addr != null) return s.addr;
|
||||
if (/^0x[0-9a-f]+$/i.test(q)) return Number.parseInt(q, 16);
|
||||
throw new Error(`unknown symbol ${q}`);
|
||||
}
|
||||
|
||||
export function startMcp(dir) {
|
||||
const ws = Workspace.find(dir);
|
||||
|
|
@ -163,7 +171,7 @@ export function startMcp(dir) {
|
|||
protocolVersion: PROTOCOL_VERSIONS.includes(requested) ? requested : PROTOCOL_VERSIONS[0],
|
||||
capabilities: { tools: {} },
|
||||
serverInfo: { name: 'clank-together', version: '0.1.0' },
|
||||
instructions: 'Tools for a collaborative matching decompilation. Read AGENTS.md in the workspace root first. Workflow: td_claim_task -> td_disassemble/td_decompile -> write C++ in the task file with the given annotation -> td_check until exact -> td_submit -> td_get_result. Also review the work of others: td_list_peer_reviews -> td_claim_peer_review -> read the original and the submission (no compiling or diffing) -> td_submit_peer_review.',
|
||||
instructions: INSTRUCTIONS,
|
||||
});
|
||||
}
|
||||
case 'ping':
|
||||
|
|
@ -185,3 +193,17 @@ export function startMcp(dir) {
|
|||
}
|
||||
});
|
||||
}
|
||||
|
||||
// What every connected agent is told when it starts. Work continues until the user stops it.
|
||||
const INSTRUCTIONS = [
|
||||
'Clank Together: a collaborative matching decompilation. Read AGENTS.md in the workspace first.',
|
||||
'Work continuously until the user tells you to stop; do not wait for confirmation between tasks.',
|
||||
'Loop: td_claim_batch (a whole unit: every open function of one source file) or td_claim_task ->',
|
||||
'read the originals (td_disassemble, td_decompile) -> write the C++ with the given annotations ->',
|
||||
'td_check until as many functions as possible are byte-identical -> td_submit (everything you hold) ->',
|
||||
'immediately claim the next work. Do not wait for verification; look at results later with td_get_batch / td_get_result',
|
||||
'and fix what came back (needs_cleanup).',
|
||||
'Between batches, take a peer review when one is waiting: td_list_peer_reviews -> td_claim_peer_review ->',
|
||||
'read the original and the submission only (no compiling or diffing) -> td_submit_peer_review for every function.',
|
||||
'Call td_heartbeat every hour or so while working on a big batch.',
|
||||
].join(' ');
|
||||
|
|
|
|||
|
|
@ -6,12 +6,13 @@ import path from 'node:path';
|
|||
import { execFileSync } from 'node:child_process';
|
||||
import { Api } from './api.js';
|
||||
import { Workspace, sha256File } from './workspace.js';
|
||||
import { checkUnits } from '../shared/check.js';
|
||||
import { checkUnits, summarizeCheck } from '../shared/check.js';
|
||||
import { reconcile } from '../shared/reconcile.js';
|
||||
import { lintSubmission, matchesAny } from '../shared/lint.js';
|
||||
import { isUnit, loadToolchainLock } from '../shared/project.js';
|
||||
import { hex } from '../shared/annotations.js';
|
||||
import { run } from '../shared/build.js';
|
||||
import { rememberServer } from './update.js';
|
||||
|
||||
export function api(ws) {
|
||||
return new Api(ws.config.server, ws.config.token);
|
||||
|
|
@ -43,6 +44,7 @@ export async function setup({ server, dir, token, exe, runner, vc, sdk, image, t
|
|||
}
|
||||
const ws = new Workspace(abs);
|
||||
ws.config.server = server.replace(/\/+$/, '');
|
||||
rememberServer(ws.config.server);
|
||||
ws.config.token = useToken;
|
||||
ws.config.user = me.name;
|
||||
fs.mkdirSync(ws.tdDir, { recursive: true });
|
||||
|
|
@ -60,11 +62,11 @@ export async function setup({ server, dir, token, exe, runner, vc, sdk, image, t
|
|||
log(`target verified: sha256 ${info.version.sha256}`);
|
||||
|
||||
const lock = loadToolchainLock(abs)[info.toolchain] ?? {};
|
||||
ws.config.runners = { [info.toolchain]: runnerConfig({ runner, vc, sdk, image: image ?? lock.image, toolchain, project: ws.project }) };
|
||||
ws.config.runners = { [info.toolchain]: runnerConfig({ runner, vc, sdk, image: image ?? (runner === 'docker' ? lock.image : undefined), toolchain, project: ws.project }) };
|
||||
ws.save();
|
||||
await refreshSymbols(ws);
|
||||
ws.config.envDigest = await computeEnvDigest(ws, info.toolchain);
|
||||
if (lock.clSha256 && ws.config.envDigest !== lock.clSha256) {
|
||||
if (lock.clSha256 && ws.config.envDigest && ws.config.envDigest !== lock.clSha256) {
|
||||
log(`WARNING: your cl.exe (sha256 ${ws.config.envDigest}) is not the pinned one (${lock.clSha256}); results may differ from the server`);
|
||||
}
|
||||
ws.save();
|
||||
|
|
@ -90,7 +92,10 @@ function runnerConfig({ runner, vc, sdk, image, toolchain, project }) {
|
|||
if (type === 'wine') return { type, clPath, wine: 'wine', toolchainRoot: root, jobs, env: { WINEDEBUG: '-all' } };
|
||||
return { type, clPath, toolchainRoot: root, jobs, env: { PATH: `${path.dirname(clPath)};${process.env.PATH ?? ''}` } };
|
||||
}
|
||||
// Nothing local given: compile on the server (remote check).
|
||||
if (!runner && !vc && !image) return { type: 'remote' };
|
||||
const type = runner ?? (os.platform() === 'win32' && vc ? 'native' : 'docker');
|
||||
if (type === 'remote') return { type };
|
||||
if (type === 'docker') {
|
||||
if (!image) throw new Error('no docker image configured; pass --image or set it in toolchain.lock.json');
|
||||
return { type, image };
|
||||
|
|
@ -108,6 +113,7 @@ function runnerConfig({ runner, vc, sdk, image, toolchain, project }) {
|
|||
async function computeEnvDigest(ws, toolchain) {
|
||||
const r = ws.config.runners?.[toolchain];
|
||||
if (!r) return null;
|
||||
if (r.type === 'remote') return null; // the server's compiler is the pinned one
|
||||
if (r.type !== 'docker') return fs.existsSync(r.clPath) ? sha256File(r.clPath) : null;
|
||||
const out = await run('docker', ['run', '--rm', '--network', 'none', r.image, 'sha256sum', '/opt/msvc/bin/cl.exe'], { timeoutMs: 120_000 });
|
||||
return /^[0-9a-f]{64}/.exec(out.output)?.[0] ?? null;
|
||||
|
|
@ -130,7 +136,13 @@ export async function sync(ws) {
|
|||
pulled = false;
|
||||
}
|
||||
const snap = await refreshSymbols(ws);
|
||||
return { head: ws.head(), fastForwarded: pulled, symbols: snap.symbols.length, note: pulled ? null : 'could not fast-forward (local commits or conflicting changes); merge origin/main yourself' };
|
||||
// A newer Ghidra project on the server is only announced (it is a large download).
|
||||
let ghidra = null;
|
||||
try {
|
||||
const g = await api(ws).get('/api/ghidra-project');
|
||||
if (g.sha256 !== ws.config.ghidraPackage) ghidra = `a newer Ghidra project is available (${(g.size / 1048576).toFixed(0)} MB): td ghidra download`;
|
||||
} catch {}
|
||||
return { head: ws.head(), fastForwarded: pulled, symbols: snap.symbols.length, ghidra, note: pulled ? null : 'could not fast-forward (local commits or conflicting changes); merge origin/main yourself' };
|
||||
}
|
||||
|
||||
/** Compiles and compares locally with the same pipeline the server uses. */
|
||||
|
|
@ -139,6 +151,8 @@ export async function check(ws, files) {
|
|||
const targets = files?.length ? files : ws.changedFiles();
|
||||
const units = targets.filter(isUnit);
|
||||
if (!units.length) throw new Error('nothing to check: pass .cpp files or modify some');
|
||||
const runner = ws.config.runners?.[project.toolchain];
|
||||
if (!runner || runner.type === 'remote') return remoteCheck(ws, targets);
|
||||
const result = await checkUnits({
|
||||
repoDir: ws.dir,
|
||||
jobDir: path.join(ws.tdDir, 'build'),
|
||||
|
|
@ -150,50 +164,72 @@ export async function check(ws, files) {
|
|||
runners: ws.config.runners,
|
||||
});
|
||||
const rec = reconcile(result, ws.symbols);
|
||||
return summarize(result, rec, new Set(targets));
|
||||
return summarizeCheck(result, rec, new Set(targets));
|
||||
}
|
||||
|
||||
/** `files`: the files the user asked about; annotation problems elsewhere in the repo are not theirs. */
|
||||
export function summarize(result, rec, files = null) {
|
||||
return {
|
||||
annotationErrors: files ? result.annotationErrors.filter((e) => files.has(e.file)) : result.annotationErrors,
|
||||
units: result.units.map((u) => ({
|
||||
file: u.file,
|
||||
compiled: u.compiled,
|
||||
log: u.compiled ? undefined : u.log,
|
||||
lintErrors: u.lintErrors,
|
||||
errors: u.errors,
|
||||
unannotated: u.unannotated,
|
||||
functions: u.functions.map((f) => ({
|
||||
id: f.id,
|
||||
name: f.demangled,
|
||||
addr: f.addr != null ? '0x' + hex(f.addr) : null,
|
||||
status: f.status,
|
||||
score: f.score,
|
||||
size: f.size,
|
||||
diffRegions: f.diffRegions?.slice(0, 12),
|
||||
relocations: f.relocations,
|
||||
})),
|
||||
})),
|
||||
referenceConflicts: rec.conflicts,
|
||||
unknownReferences: rec.unknown.map((x) => ({ symbol: x.decorated, addr: '0x' + hex(x.addr) })),
|
||||
};
|
||||
/**
|
||||
* Compiles and compares on the server (its sandboxed compiler), for contributors without a
|
||||
* local toolchain. Sends the changed files and the commit they are based on; nothing is submitted.
|
||||
*/
|
||||
async function remoteCheck(ws, targets) {
|
||||
const files = {};
|
||||
for (const f of new Set([...targets, ...ws.changedFiles()])) {
|
||||
const abs = path.join(ws.dir, f);
|
||||
if (fs.existsSync(abs) && /\.(c|cc|cpp|cxx|h|hpp|inl)$/i.test(f)) files[f.replace(/\\/g, '/')] = fs.readFileSync(abs, 'utf8');
|
||||
}
|
||||
const r = await api(ws).post('/api/check', { baseCommit: ws.head(), files });
|
||||
if (r.gate === 'rejected') {
|
||||
const e = new Error('the static gate rejects this code:\n' + r.lintErrors.map((x) => ` ${x.file ?? ''}${x.line ? ':' + x.line : ''} ${x.message}`).join('\n'));
|
||||
e.details = r.lintErrors;
|
||||
throw e;
|
||||
}
|
||||
if (r.gate === 'conflict') throw new Error(`your changes conflict with newer work on main (${r.conflicts.join(', ')}); run td sync and merge`);
|
||||
return { ...r, compiledOn: 'server' };
|
||||
}
|
||||
|
||||
/**
|
||||
* The original function as text. A local Ghidra (td ghidra install/build) answers locally;
|
||||
* otherwise the server's Ghidra export does.
|
||||
*/
|
||||
export async function original(ws, query, kind) {
|
||||
if (ws.config.ghidraHome) {
|
||||
const { decompile, disassemble } = await import('./ghidra.js');
|
||||
const s = ws.findSymbol(query);
|
||||
const addr = s?.addr ?? (/^0x[0-9a-f]+$/i.test(query) ? Number.parseInt(query, 16) : null);
|
||||
if (addr == null) throw new Error(`unknown symbol ${query}`);
|
||||
return kind === 'asm' ? disassemble(ws, addr) : decompile(ws, addr);
|
||||
}
|
||||
const r = await api(ws).get(`/api/original/${encodeURIComponent(query)}`);
|
||||
if (kind === 'asm') return r.asm;
|
||||
return r.pseudocode ?? `// no pseudocode for ${r.symbol ?? query} yet (${r.source}); the assembly is available with td disasm`;
|
||||
}
|
||||
|
||||
/** Submits files for a leased task. Defaults to all changed files the task allows. */
|
||||
export async function submit(ws, taskId, { files, notes, agentMeta } = {}) {
|
||||
const ctx = await api(ws).get(`/api/tasks/${taskId}`);
|
||||
const candidates = files?.length ? files : ws.changedFiles().filter((f) => matchesAny(f, ctx.allowedPaths));
|
||||
if (!candidates.length) throw new Error(`no changed files match the task's allowed paths (${ctx.allowedPaths.join(', ')})`);
|
||||
/**
|
||||
* Submits work. `taskIds` empty = every task you hold, as one batch (a whole unit you claimed
|
||||
* with claim-batch, or several single tasks). Defaults to all changed files those tasks allow.
|
||||
*/
|
||||
export async function submit(ws, taskIds = [], { files, notes, agentMeta } = {}) {
|
||||
const me = await api(ws).get('/api/me');
|
||||
const ids = taskIds.length ? taskIds.map(Number) : me.leases;
|
||||
if (!ids.length) throw new Error('you hold no tasks; claim one (td claim) or a whole unit (td claim --unit <file>)');
|
||||
const allowed = taskIds.length ? [...new Set((await Promise.all(ids.map((id) => api(ws).get(`/api/tasks/${id}`)))).flatMap((c) => c.allowedPaths))] : me.leasedAllowedPaths;
|
||||
const candidates = files?.length ? files : ws.changedFiles().filter((f) => matchesAny(f, allowed));
|
||||
if (!candidates.length) throw new Error(`no changed files match the allowed paths of your tasks (${allowed.join(', ')})`);
|
||||
const contents = {};
|
||||
for (const f of candidates) contents[f.replace(/\\/g, '/')] = fs.readFileSync(path.join(ws.dir, f), 'utf8');
|
||||
const errors = lintSubmission(contents, { allowedPaths: ctx.allowedPaths, limits: ws.project.lintLimits, allowedPragmas: ws.project.allowedPragmas });
|
||||
const errors = lintSubmission(contents, { allowedPaths: allowed, limits: ws.project.lintLimits, allowedPragmas: ws.project.allowedPragmas });
|
||||
if (errors.length) {
|
||||
const e = new Error('submission fails the static gate:\n' + errors.map((x) => ` ${x.file ?? ''}${x.line ? ':' + x.line : ''} ${x.message}`).join('\n'));
|
||||
e.details = errors;
|
||||
throw e;
|
||||
}
|
||||
return api(ws).post(`/api/tasks/${taskId}/submit`, { baseCommit: ws.head(), files: contents, notes, agentMeta, envDigest: ws.config.envDigest });
|
||||
return api(ws).post('/api/batches', { taskIds: ids, baseCommit: ws.head(), files: contents, notes, agentMeta, envDigest: ws.config.envDigest });
|
||||
}
|
||||
|
||||
/** Claims every open function of a unit (source file) or an address range, as one batch. */
|
||||
export function claimBatch(ws, { unit, start, end, limit } = {}) {
|
||||
return api(ws).post('/api/tasks/claim-batch', { unit, start, end, limit });
|
||||
}
|
||||
|
||||
export function relPath(ws, f) {
|
||||
|
|
|
|||
93
src/client/update.js
Normal file
93
src/client/update.js
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
// Self-update of the td binary (only when td runs as the standalone executable; a checkout or
|
||||
// npm install updates through git/npm). At most once an hour td compares its own SHA-256 with
|
||||
// the server's /download/SHA256SUMS; a newer build is downloaded, verified and swapped in for
|
||||
// the next start. TD_NO_UPDATE=1 turns it off.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
const CHECK_EVERY_MS = 3600_000;
|
||||
const STATE_DIR = path.join(os.homedir(), '.clank');
|
||||
const STATE_FILE = path.join(STATE_DIR, 'update.json');
|
||||
const ASSET = process.platform === 'win32' ? 'td-windows-x64.exe' : 'td-linux-x64';
|
||||
|
||||
function readState() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function writeState(s) {
|
||||
try {
|
||||
fs.mkdirSync(STATE_DIR, { recursive: true });
|
||||
fs.writeFileSync(STATE_FILE, JSON.stringify(s));
|
||||
} catch {}
|
||||
}
|
||||
|
||||
/** Remembers the server for updates of a td that is used outside a workspace. */
|
||||
export function rememberServer(server) {
|
||||
writeState({ ...readState(), server });
|
||||
}
|
||||
|
||||
async function ownHash(exe, state) {
|
||||
const st = fs.statSync(exe);
|
||||
const key = `${st.size}:${st.mtimeMs}`;
|
||||
if (state.selfKey === key && state.selfHash) return state.selfHash;
|
||||
const hash = crypto.createHash('sha256').update(fs.readFileSync(exe)).digest('hex');
|
||||
state.selfKey = key;
|
||||
state.selfHash = hash;
|
||||
return hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks for and installs a newer td. Returns a message when an update was installed (it takes
|
||||
* effect on the next start), otherwise null. Never throws: an update must not break a command.
|
||||
*/
|
||||
export async function maybeUpdate({ server, force = false, log = (m) => process.stderr.write(m + '\n') } = {}) {
|
||||
try {
|
||||
if (process.env.TD_NO_UPDATE === '1') return null;
|
||||
const sea = await import('node:sea').catch(() => null);
|
||||
if (!sea?.isSea?.()) return null;
|
||||
const exe = process.execPath;
|
||||
// Leftover from a Windows update (a running .exe can be renamed, not replaced).
|
||||
fs.rmSync(exe + '.old', { force: true });
|
||||
|
||||
const state = readState();
|
||||
const base = (server ?? state.server)?.replace(/\/+$/, '');
|
||||
if (!base) return null;
|
||||
if (!force && state.checkedAt && Date.now() - state.checkedAt < CHECK_EVERY_MS) return null;
|
||||
state.checkedAt = Date.now();
|
||||
|
||||
const res = await fetch(`${base}/download/SHA256SUMS`, { signal: AbortSignal.timeout(10_000) });
|
||||
if (!res.ok) return writeState(state), null;
|
||||
const want = new RegExp(`^([0-9a-f]{64})\\s+${ASSET.replace('.', '\\.')}$`, 'm').exec(await res.text())?.[1];
|
||||
if (!want || want === (await ownHash(exe, state))) return writeState(state), null;
|
||||
|
||||
log(`td: downloading an update from ${base} ...`);
|
||||
const bin = await fetch(`${base}/download/${ASSET}`, { signal: AbortSignal.timeout(600_000) });
|
||||
if (!bin.ok) return writeState(state), null;
|
||||
const data = Buffer.from(await bin.arrayBuffer());
|
||||
if (crypto.createHash('sha256').update(data).digest('hex') !== want) {
|
||||
log('td: the update failed its checksum check and was not installed');
|
||||
return writeState(state), null;
|
||||
}
|
||||
const next = exe + '.new';
|
||||
fs.writeFileSync(next, data, { mode: 0o755 });
|
||||
if (process.platform === 'win32') {
|
||||
fs.renameSync(exe, exe + '.old');
|
||||
fs.renameSync(next, exe);
|
||||
} else {
|
||||
fs.renameSync(next, exe);
|
||||
}
|
||||
delete state.selfKey;
|
||||
delete state.selfHash;
|
||||
writeState(state);
|
||||
return 'td was updated; the new version runs from the next command on';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -11,10 +11,14 @@ import { Tasks } from './tasks.js';
|
|||
import { Verifier } from './verify.js';
|
||||
import { Merger } from './merge.js';
|
||||
import { PeerReviews } from './peer.js';
|
||||
import { Originals } from './original.js';
|
||||
import { downloadsDir, installPs1, installSh } from './install.js';
|
||||
import { ServerGhidra } from './ghidra.js';
|
||||
import { Router, HttpError, readJson, sendJson, RateLimiter } from './http.js';
|
||||
import { PEImage } from '../shared/pe.js';
|
||||
import { loadProject, loadToolchainLock } from '../shared/project.js';
|
||||
import { DEFAULT_LIMITS } from '../shared/lint.js';
|
||||
import { DEFAULT_LIMITS, lintSubmission } from '../shared/lint.js';
|
||||
import { summarizeCheck } from '../shared/check.js';
|
||||
import { hex } from '../shared/annotations.js';
|
||||
|
||||
const WEB_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'web');
|
||||
|
|
@ -30,6 +34,8 @@ export const DEFAULT_CONFIG = {
|
|||
runners: { msvc2010: { type: 'docker', image: 'togetherdecomp/msvc2010:1', memory: '1g', cpus: 1 } },
|
||||
allowUnsandboxedRunners: false,
|
||||
pushRemote: null,
|
||||
ghidraHome: null,
|
||||
ghidraRefreshMinutes: 30,
|
||||
distributeTarget: false,
|
||||
gitCloneUrl: null,
|
||||
gitWebUrl: null,
|
||||
|
|
@ -66,16 +72,20 @@ export function openServer(dataDir, { log = console } = {}) {
|
|||
}
|
||||
|
||||
const tasks = new Tasks({ store, config, repo, project });
|
||||
const verifier = new Verifier({ store, config, repo, project, tasks, getPe, log });
|
||||
const peers = new PeerReviews({ store, project });
|
||||
const verifier = new Verifier({ store, config, repo, project, tasks, getPe, peers, log });
|
||||
const merger = new Merger({ store, config, repo, project, tasks, verifier, getPe, peers, log });
|
||||
const accounts = new Accounts(store);
|
||||
return { config, store, repo, project, tasks, verifier, merger, getPe, accounts, peers };
|
||||
const originals = new Originals({ store });
|
||||
const ghidra = new ServerGhidra({ config, store, originals, log });
|
||||
merger.onMerged = (commit) => ghidra.scheduleRefresh(commit);
|
||||
return { config, store, repo, project, tasks, verifier, merger, getPe, accounts, peers, originals, ghidra };
|
||||
}
|
||||
|
||||
export function createApp(ctx) {
|
||||
const { config, store, repo, project, tasks, merger, accounts, peers } = ctx;
|
||||
const router = new Router();
|
||||
downloadsRoot = downloadsDir(config);
|
||||
const submitLimiter = new RateLimiter(config.submitsPerHour, 3600_000);
|
||||
const symbolLimiter = new RateLimiter(120, 3600_000);
|
||||
const authFailLimiter = new RateLimiter(60, 600_000);
|
||||
|
|
@ -203,7 +213,8 @@ export function createApp(ctx) {
|
|||
|
||||
router.on('GET', '/api/me', ({ req }) => {
|
||||
const user = auth(req);
|
||||
return { name: user.name, role: user.role, via: user.via, leases: tasks.leasesOf(user.id) };
|
||||
const leases = tasks.leasesOf(user.id);
|
||||
return { name: user.name, role: user.role, via: user.via, leases, leasedAllowedPaths: leases.length ? tasks.allowedPathsForTasks(leases) : [] };
|
||||
});
|
||||
|
||||
router.on('POST', '/api/me/password', async ({ req }) => {
|
||||
|
|
@ -265,22 +276,85 @@ export function createApp(ctx) {
|
|||
submitLimiter.check(user.id);
|
||||
const taskId = intParam(params.id);
|
||||
const body = await readJson(req);
|
||||
const t = tasks.requireLease(user, taskId);
|
||||
const r = submitBatch(user, [taskId], body);
|
||||
return { attemptId: r.attempts[0].attemptId, batchId: r.batchId, state: 'queued' };
|
||||
});
|
||||
|
||||
/** Validates a submission body and queues it as one batch for the given tasks. */
|
||||
const submitBatch = (user, taskIds, body) => {
|
||||
if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash');
|
||||
if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) throw new HttpError(400, 'files must be an object of path -> content');
|
||||
const notes = body.notes ? String(body.notes).slice(0, DEFAULT_LIMITS.maxNotesBytes) : null;
|
||||
const meta = body.agentMeta ? JSON.stringify(body.agentMeta).slice(0, 1000) : null;
|
||||
const id = store.tx(() => {
|
||||
const r = store.run(
|
||||
`INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', ?)`,
|
||||
t.id, user.id, body.baseCommit, JSON.stringify(body.files), notes, meta, body.envDigest ? String(body.envDigest).slice(0, 200) : null, now(),
|
||||
);
|
||||
store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), t.id);
|
||||
store.event('submit', user.id, { task: t.id, attempt: Number(r.lastInsertRowid) });
|
||||
return Number(r.lastInsertRowid);
|
||||
const r = tasks.submitBatch(user, {
|
||||
taskIds,
|
||||
baseCommit: body.baseCommit,
|
||||
files: body.files,
|
||||
notes: body.notes ? String(body.notes).slice(0, DEFAULT_LIMITS.maxNotesBytes) : null,
|
||||
agentMeta: body.agentMeta ? JSON.stringify(body.agentMeta).slice(0, 1000) : null,
|
||||
envDigest: body.envDigest ? String(body.envDigest).slice(0, 200) : null,
|
||||
});
|
||||
ctx.verifier.pump();
|
||||
return { attemptId: id, state: 'queued' };
|
||||
return r;
|
||||
};
|
||||
|
||||
// Bulk work: claim every open function of a unit (source file) or an address range, submit the
|
||||
// file once. Verified with one compile; merged as a whole.
|
||||
router.on('POST', '/api/tasks/claim-batch', async ({ req }) => {
|
||||
const user = auth(req);
|
||||
const body = await readJson(req, 4096);
|
||||
const hexOrNull = (x) => (x == null ? null : typeof x === 'string' ? Number.parseInt(x, 16) : Number(x));
|
||||
const ids = tasks.claimBatch(user, { unit: body.unit ? String(body.unit) : null, start: hexOrNull(body.start), end: hexOrNull(body.end), limit: body.limit ? Number(body.limit) : 50 });
|
||||
const v = store.currentVersion();
|
||||
const ph = ids.map(() => '?').join(',');
|
||||
const rows = store.all(
|
||||
`SELECT t.id AS task, s.id AS symbol, s.name, s.unit, a.addr, a.size, t.state, t.best_score FROM tasks t JOIN symbols s ON s.id = t.symbol_id
|
||||
LEFT JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? WHERE t.id IN (${ph}) ORDER BY a.addr`,
|
||||
v.id, ...ids,
|
||||
);
|
||||
return {
|
||||
tasks: rows.map((r) => ({ ...r, addr: r.addr != null ? '0x' + hex(r.addr) : null, annotation: `// FUNCTION: ${r.symbol}` })),
|
||||
allowedPaths: tasks.allowedPathsForTasks(ids),
|
||||
baseCommit: repo.head(),
|
||||
note: 'Write or fix every function, check, then submit them together (td submit --all). Context of one task: GET /api/tasks/<id>.',
|
||||
};
|
||||
});
|
||||
|
||||
router.on('POST', '/api/batches', async ({ req }) => {
|
||||
const user = auth(req);
|
||||
submitLimiter.check(user.id);
|
||||
const body = await readJson(req, 8 * 1024 * 1024);
|
||||
const r = submitBatch(user, (body.taskIds ?? []).map((x) => intParam(x)), body);
|
||||
return { ...r, state: 'queued' };
|
||||
});
|
||||
|
||||
router.on('GET', '/api/batches/:id', ({ req, params }) => {
|
||||
auth(req);
|
||||
const b = store.get('SELECT b.*, u.name AS author FROM batches b JOIN users u ON u.id = b.user_id WHERE b.id = ?', intParam(params.id));
|
||||
if (!b) throw new HttpError(404, 'no such batch');
|
||||
const result = b.result_json ? JSON.parse(b.result_json) : {};
|
||||
const attempts = store.all(
|
||||
`SELECT a.id, a.task_id, t.symbol_id, s.name, a.state, a.score, a.result_json FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id WHERE a.batch_id = ? ORDER BY a.id`,
|
||||
b.id,
|
||||
);
|
||||
return {
|
||||
id: b.id,
|
||||
author: b.author,
|
||||
state: b.state,
|
||||
baseCommit: b.base_commit,
|
||||
mergedCommit: b.merged_commit,
|
||||
returned: result.returned ?? null,
|
||||
mergeFailure: result.mergeFailure ?? null,
|
||||
changedHeaders: result.changedHeaders ?? [],
|
||||
changedOutside: result.outside ?? [],
|
||||
functions: attempts.map((a) => {
|
||||
const fb = a.result_json ? JSON.parse(a.result_json).feedback : null;
|
||||
return { attemptId: a.id, taskId: a.task_id, symbol: a.symbol_id, name: a.name, state: a.state, score: a.score, summary: fb?.summary ?? null, status: fb?.target?.status ?? null };
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
router.on('POST', '/api/tasks/heartbeat-all', ({ req }) => {
|
||||
const user = auth(req);
|
||||
return tasks.heartbeatAll(user);
|
||||
});
|
||||
|
||||
router.on('GET', '/api/attempts/:id', ({ req, params }) => {
|
||||
|
|
@ -288,6 +362,8 @@ export function createApp(ctx) {
|
|||
const a = store.get('SELECT a.*, u.name AS author FROM attempts a JOIN users u ON u.id = a.user_id WHERE a.id = ?', intParam(params.id));
|
||||
if (!a) throw new HttpError(404, 'no such attempt');
|
||||
const result = a.result_json ? JSON.parse(a.result_json) : null;
|
||||
const batch = a.batch_id ? store.get('SELECT result_json FROM batches WHERE id = ?', a.batch_id) : null;
|
||||
const batchResult = batch?.result_json ? JSON.parse(batch.result_json) : {};
|
||||
const isReviewer = hasRole(user, 'reviewer');
|
||||
return {
|
||||
id: a.id,
|
||||
|
|
@ -300,10 +376,11 @@ export function createApp(ctx) {
|
|||
finishedAt: a.finished_at,
|
||||
notes: a.notes,
|
||||
feedback: result?.feedback ?? null,
|
||||
mergeFailure: result?.mergeFailure ?? null,
|
||||
changedHeaders: result?.changedHeaders ?? [],
|
||||
files: JSON.parse(a.files_json),
|
||||
mergedFiles: isReviewer ? result?.mergedFiles ?? null : undefined,
|
||||
batchId: a.batch_id,
|
||||
mergeFailure: batchResult.mergeFailure ?? result?.mergeFailure ?? null,
|
||||
changedHeaders: batchResult.changedHeaders ?? result?.changedHeaders ?? [],
|
||||
files: store.attemptFiles(a),
|
||||
mergedFiles: isReviewer ? batchResult.mergedFiles ?? result?.mergedFiles ?? null : undefined,
|
||||
reviews: store.all('SELECT r.action, r.comment, r.created_at, u.name AS reviewer FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ?', a.id),
|
||||
peerReviews: peers.verdicts(a.id),
|
||||
peerRequired: peers.required,
|
||||
|
|
@ -334,6 +411,67 @@ export function createApp(ctx) {
|
|||
return { id: r.symbol.id, kind: r.symbol.kind, created: r.created, annotation: `// ${{ function: 'FUNCTION', global: 'GLOBAL', vtable: 'VTABLE', string: 'STRING' }[kind]}: ${r.symbol.id}` };
|
||||
});
|
||||
|
||||
// --- the original function (assembly, pseudocode) for contributors without a local Ghidra.
|
||||
// Derived from the exe, so approved accounts only, like the exe itself.
|
||||
router.on('GET', '/api/original/:symbol', async ({ req, params }) => {
|
||||
auth(req);
|
||||
return ctx.originals.get(params.symbol);
|
||||
});
|
||||
|
||||
// --- the server's Ghidra project of the current version (td ghidra download). Derived from the
|
||||
// exe, so approved accounts only.
|
||||
router.on('GET', '/api/ghidra-project', ({ req }) => {
|
||||
auth(req);
|
||||
const info = ctx.ghidra.info();
|
||||
if (!info) throw new HttpError(404, 'no Ghidra project has been built on this server yet');
|
||||
return { ...info, download: `${config.publicUrl}/api/ghidra-project/download` };
|
||||
});
|
||||
|
||||
router.on('GET', '/api/ghidra-project/download', ({ req, res }) => {
|
||||
auth(req);
|
||||
const file = ctx.ghidra.packagePath();
|
||||
if (!file || !fs.existsSync(file)) throw new HttpError(404, 'no Ghidra project has been built on this server yet');
|
||||
res.writeHead(200, { 'content-type': 'application/gzip', 'content-length': fs.statSync(file).size, 'content-disposition': 'attachment; filename="ghidra-project.tar.gz"' });
|
||||
fs.createReadStream(file).pipe(res);
|
||||
return undefined;
|
||||
});
|
||||
|
||||
// --- remote check: compile and compare in the server's sandbox without submitting
|
||||
// (contributors without a local MSVC toolchain). Same static gate as a submission.
|
||||
|
||||
const checkLimiter = new RateLimiter(config.checksPerHour ?? 60, 3600_000);
|
||||
let checksRunning = 0;
|
||||
const checkWaiters = [];
|
||||
const withCheckSlot = async (fn) => {
|
||||
const max = config.remoteCheckConcurrency ?? 2;
|
||||
if (checksRunning >= max) await new Promise((r) => checkWaiters.push(r));
|
||||
checksRunning++;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
checksRunning--;
|
||||
checkWaiters.shift()?.();
|
||||
}
|
||||
};
|
||||
|
||||
router.on('POST', '/api/check', async ({ req }) => {
|
||||
const user = auth(req);
|
||||
checkLimiter.check(user.id);
|
||||
const body = await readJson(req);
|
||||
if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash');
|
||||
if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) throw new HttpError(400, 'files must be an object of path -> content');
|
||||
const allowed = [...project.sourceDirs.map((d) => `${d}/**`)];
|
||||
const lintErrors = lintSubmission(body.files, { allowedPaths: allowed, limits: project.lintLimits, allowedPragmas: project.allowedPragmas });
|
||||
if (lintErrors.length) return { gate: 'rejected', lintErrors };
|
||||
if (!repo.isAncestor(body.baseCommit)) throw new HttpError(409, 'your base commit is not part of main; run td sync');
|
||||
const merged = repo.mergeOntoHead(body.baseCommit, body.files);
|
||||
if (merged.conflicts.length) return { gate: 'conflict', conflicts: merged.conflicts };
|
||||
return withCheckSlot(async () => {
|
||||
const { check, rec } = await ctx.verifier.check(merged.files, `check-${user.id}-${Date.now()}`);
|
||||
return { gate: 'ok', head: repo.head(), ...summarizeCheck(check, rec, new Set(Object.keys(merged.files))) };
|
||||
});
|
||||
});
|
||||
|
||||
// --- peer review (agents, API token or session)
|
||||
|
||||
router.on('GET', '/api/peer-reviews', ({ req, query }) => {
|
||||
|
|
@ -369,7 +507,7 @@ export function createApp(ctx) {
|
|||
router.on('GET', '/api/review/queue', ({ req }) => {
|
||||
const user = auth(req, { role: 'reviewer', session: true });
|
||||
return store.all(
|
||||
`SELECT a.id, a.task_id, a.score, a.created_at, a.finished_at, u.name AS author, t.symbol_id, s.name,
|
||||
`SELECT a.id, a.batch_id, a.task_id, a.score, a.created_at, a.finished_at, u.name AS author, t.symbol_id, s.name,
|
||||
(SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.action = 'approve') AS approvals,
|
||||
(SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.reviewer_id = ?) AS reviewed_by_me
|
||||
FROM attempts a JOIN users u ON u.id = a.user_id JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
|
|
@ -386,6 +524,15 @@ export function createApp(ctx) {
|
|||
return r;
|
||||
});
|
||||
|
||||
// One click for a whole batch: every function of it that waits for a human.
|
||||
router.on('POST', '/api/batches/:id/review', async ({ req, params }) => {
|
||||
const user = auth(req, { role: 'reviewer', session: true });
|
||||
const body = await readJson(req, 16 * 1024);
|
||||
const r = merger.reviewBatch(user, intParam(params.id), String(body.action ?? ''), body.comment ? String(body.comment) : null);
|
||||
setImmediate(() => merger.pump());
|
||||
return r;
|
||||
});
|
||||
|
||||
// --- maintainers and admins (browser session only)
|
||||
|
||||
router.on('POST', '/api/admin/tasks/:id', async ({ req, params }) => {
|
||||
|
|
@ -425,6 +572,13 @@ export function createApp(ctx) {
|
|||
if (url.pathname.startsWith('/repo.git/')) {
|
||||
return serveGit(req, res, { projectRoot: config.dataDir, subPath: '/repo' + url.pathname.slice('/repo.git'.length), query: url.searchParams });
|
||||
}
|
||||
if (req.method === 'GET' && (url.pathname === '/install.ps1' || url.pathname === '/install.sh')) {
|
||||
const script = url.pathname.endsWith('.ps1') ? installPs1(config, project) : installSh(config, project);
|
||||
if (!script) throw new HttpError(503, 'the td binaries are not published on this server yet');
|
||||
res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-cache', 'x-content-type-options': 'nosniff' });
|
||||
return res.end(script);
|
||||
}
|
||||
if (req.method === 'GET' && url.pathname.startsWith('/download/')) return serveDownload(url.pathname.slice(10), res);
|
||||
if (req.method === 'GET' && !url.pathname.startsWith('/api/')) return serveStatic(url.pathname, res);
|
||||
const m = router.match(req.method, url.pathname);
|
||||
if (!m) throw new HttpError(404, 'not found');
|
||||
|
|
@ -442,6 +596,19 @@ export function createApp(ctx) {
|
|||
return http.createServer(handler);
|
||||
}
|
||||
|
||||
// Public downloads of the td binaries (tools/build-td output copied to <dataDir>/downloads).
|
||||
const DOWNLOADS = new Set(['td-windows-x64.exe', 'td-linux-x64', 'SHA256SUMS']);
|
||||
let downloadsRoot = null;
|
||||
function serveDownload(name, res) {
|
||||
const file = DOWNLOADS.has(name) && downloadsRoot ? path.join(downloadsRoot, name) : null;
|
||||
if (!file || !fs.existsSync(file)) {
|
||||
res.writeHead(404, { 'content-type': 'text/plain' });
|
||||
return res.end('not found');
|
||||
}
|
||||
res.writeHead(200, { 'content-type': name === 'SHA256SUMS' ? 'text/plain' : 'application/octet-stream', 'content-length': fs.statSync(file).size, 'content-disposition': `attachment; filename="${name}"`, 'x-content-type-options': 'nosniff' });
|
||||
fs.createReadStream(file).pipe(res);
|
||||
}
|
||||
|
||||
const STATIC = { '/': 'index.html', '/review': 'index.html', '/app.js': 'app.js', '/style.css': 'style.css', '/favicon.svg': 'favicon.svg' };
|
||||
const TYPES = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css; charset=utf-8', '.svg': 'image/svg+xml' };
|
||||
|
||||
|
|
|
|||
32
src/server/batches.js
Normal file
32
src/server/batches.js
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
// A batch merges as a whole. When one of its attempts fails (verification, peer review, human
|
||||
// review or the merge re-check), the whole batch goes back to its author: every other attempt
|
||||
// that was still on its way is marked returned, and its task becomes claimable again (reserved
|
||||
// for the author for a while). Peer verdicts stay attached to their attempts, so a resubmission
|
||||
// whose compiled code is unchanged for a function carries them over (Verifier.carryOver).
|
||||
|
||||
import { now } from './db.js';
|
||||
|
||||
export function returnBatch(store, batchId, reason) {
|
||||
if (!batchId) return;
|
||||
const b = store.get('SELECT * FROM batches WHERE id = ?', batchId);
|
||||
if (!b || ['merged', 'returned', 'superseded'].includes(b.state)) return;
|
||||
const pending = store.all(`SELECT * FROM attempts WHERE batch_id = ? AND state IN ('queued', 'verifying', 'in_review', 'peer_review', 'approved')`, batchId);
|
||||
for (const a of pending) {
|
||||
store.run(`UPDATE attempts SET state = 'returned', result_json = json_set(COALESCE(result_json, '{}'), '$.returned', ?) WHERE id = ?`, reason, a.id);
|
||||
store.run('DELETE FROM peer_leases WHERE attempt_id = ?', a.id);
|
||||
store.run(
|
||||
`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ? AND state IN ('verifying', 'in_review', 'peer_review', 'approved', 'merging')`,
|
||||
a.user_id, now() + 6 * 3600_000, now(), a.task_id,
|
||||
);
|
||||
}
|
||||
store.run(`UPDATE batches SET state = 'returned', result_json = json_set(COALESCE(result_json, '{}'), '$.returned', ?) WHERE id = ?`, reason, batchId);
|
||||
store.event('batch_returned', b.user_id, { batch: batchId, reason });
|
||||
}
|
||||
|
||||
/** Attempts of a batch with their task's symbol. */
|
||||
export function batchAttempts(store, batchId) {
|
||||
return store.all(
|
||||
`SELECT a.*, t.symbol_id FROM attempts a JOIN tasks t ON t.id = a.task_id WHERE a.batch_id = ? ORDER BY a.id`,
|
||||
batchId,
|
||||
);
|
||||
}
|
||||
|
|
@ -37,6 +37,8 @@ Usage: td-server <command> [options] (all commands accept --data <dir>, defaul
|
|||
symbols load <symbols.json> import symbols with fixed IDs and per-version addresses
|
||||
symbols units <csv> assign functions to translation units (csv: start,end,file)
|
||||
symbols library <id...> mark symbols as library code (no tasks)
|
||||
originals import <jsonl> [--version V] assembly + pseudocode of the original (ExportOriginals.java)
|
||||
ghidra build | refresh the server's Ghidra project (download, pseudocode); refresh also runs after merges
|
||||
rebuild compile all of main and refresh match status
|
||||
tasks generate create tasks for functions that have none
|
||||
`;
|
||||
|
|
@ -239,6 +241,25 @@ async function main() {
|
|||
}
|
||||
}
|
||||
|
||||
if (cmd === 'ghidra' && (sub === 'build' || sub === 'refresh')) {
|
||||
// The server's Ghidra project (needs ghidraHome in server.json and a JDK 21).
|
||||
const ctx = openServer(dataDir);
|
||||
if (!ctx.ghidra.enabled) throw new Error('set ghidraHome in server.json to a Ghidra 11.2.1 installation');
|
||||
ctx.store.setMeta('ghidra.commit', ctx.repo.head());
|
||||
const info = sub === 'build' ? await ctx.ghidra.build() : await ctx.ghidra.refresh();
|
||||
console.log(JSON.stringify(info, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (cmd === 'originals' && sub === 'import') {
|
||||
// ghidra_scripts/ExportOriginals.java output, for the current version unless --version is given.
|
||||
const ctx = openServer(dataDir);
|
||||
const v = values.version ?? ctx.store.currentVersion()?.id;
|
||||
const n = await ctx.originals.import(v, rest[0]);
|
||||
console.log(`imported ${n} functions for ${v}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (cmd === 'tasks' && sub === 'generate') {
|
||||
const ctx = openServer(dataDir);
|
||||
console.log(`created ${ctx.tasks.generate()} tasks`);
|
||||
|
|
|
|||
|
|
@ -142,6 +142,25 @@ CREATE TABLE IF NOT EXISTS func_status (
|
|||
confirmed INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- A batch is one submission: the files once, and one attempt per task (function) it covers.
|
||||
-- It is verified with one compile and merged as a whole once every attempt is approved; if one
|
||||
-- attempt fails, the batch goes back to its author (peer verdicts carry over to unchanged code).
|
||||
CREATE TABLE IF NOT EXISTS batches (
|
||||
id INTEGER PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id),
|
||||
base_commit TEXT NOT NULL,
|
||||
files_json TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
agent_meta TEXT,
|
||||
env_digest TEXT,
|
||||
state TEXT NOT NULL, -- queued, verifying, open, merging, merged, returned, superseded
|
||||
result_json TEXT,
|
||||
merged_commit TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
finished_at INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS batches_state ON batches(state);
|
||||
|
||||
-- Peer review of attempts that are not byte-identical: other agents read the original's
|
||||
-- assembly and pseudocode and the submitted code, and judge whether the logic is the same.
|
||||
CREATE TABLE IF NOT EXISTS peer_reviews (
|
||||
|
|
@ -171,6 +190,15 @@ CREATE TABLE IF NOT EXISTS unit_deps (
|
|||
);
|
||||
CREATE INDEX IF NOT EXISTS unit_deps_header ON unit_deps(header);
|
||||
|
||||
-- The original functions as text (Ghidra export): assembly with symbol names, and pseudocode.
|
||||
CREATE TABLE IF NOT EXISTS original_code (
|
||||
version_id TEXT NOT NULL,
|
||||
addr INTEGER NOT NULL,
|
||||
asm TEXT,
|
||||
c TEXT,
|
||||
PRIMARY KEY (version_id, addr)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
at INTEGER NOT NULL,
|
||||
|
|
@ -189,6 +217,19 @@ export class Store {
|
|||
this.migrateUsers();
|
||||
this.db.exec(SCHEMA);
|
||||
this.addColumn('func_status', 'confirmed', 'INTEGER NOT NULL DEFAULT 0');
|
||||
// Fingerprint of main's compiled code per function (submissions may only change their own tasks).
|
||||
this.addColumn('func_status', 'code_hash', 'TEXT');
|
||||
this.addColumn('attempts', 'batch_id', 'INTEGER REFERENCES batches(id)');
|
||||
this.db.exec('CREATE INDEX IF NOT EXISTS attempts_batch ON attempts(batch_id)');
|
||||
}
|
||||
|
||||
/** Submitted files of an attempt (kept once per batch). */
|
||||
attemptFiles(attempt) {
|
||||
if (attempt.batch_id) {
|
||||
const b = this.get('SELECT files_json FROM batches WHERE id = ?', attempt.batch_id);
|
||||
if (b) return JSON.parse(b.files_json);
|
||||
}
|
||||
return JSON.parse(attempt.files_json || '{}');
|
||||
}
|
||||
|
||||
/** Adds a column to a table created by an older schema. */
|
||||
|
|
|
|||
137
src/server/ghidra.js
Normal file
137
src/server/ghidra.js
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
// The server's own Ghidra project of the current target version, kept up to date with the symbol
|
||||
// database. Contributors download it (td ghidra download) instead of analyzing the exe
|
||||
// themselves, and the coordinator exports assembly and pseudocode from it for /api/original.
|
||||
//
|
||||
// build import + auto-analysis + symbol names (once per version, ~15 min)
|
||||
// refresh re-apply the names, re-export the originals, pack the project (a few minutes);
|
||||
// runs on its own after merges, at most every `ghidraRefreshMinutes` (default 30)
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { run } from '../shared/build.js';
|
||||
import { hex } from '../shared/annotations.js';
|
||||
|
||||
const SCRIPTS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'ghidra_scripts');
|
||||
const PROJECT = 'td';
|
||||
const PROGRAM = 'target.exe';
|
||||
|
||||
export class ServerGhidra {
|
||||
constructor({ config, store, originals, log = console }) {
|
||||
Object.assign(this, { config, store, originals, log });
|
||||
this.dir = path.join(config.dataDir, 'ghidra');
|
||||
this.projectDir = path.join(this.dir, 'project');
|
||||
this.busy = null;
|
||||
this.timer = null;
|
||||
}
|
||||
|
||||
get enabled() {
|
||||
return !!this.config.ghidraHome && fs.existsSync(path.join(this.config.ghidraHome, 'support', 'analyzeHeadless'));
|
||||
}
|
||||
|
||||
/** Metadata of the packed project, or null. */
|
||||
info() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(path.join(this.dir, 'package.json'), 'utf8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
packagePath() {
|
||||
const i = this.info();
|
||||
return i ? path.join(this.dir, i.file) : null;
|
||||
}
|
||||
|
||||
async headless(args, timeoutMs) {
|
||||
fs.mkdirSync(this.projectDir, { recursive: true });
|
||||
const exe = path.join(this.config.ghidraHome, 'support', 'analyzeHeadless');
|
||||
// Ghidra 11.2 is made for JDK 21; the system default may be newer.
|
||||
const jdk21 = '/usr/lib/jvm/java-21-openjdk-amd64';
|
||||
const javaHome = this.config.ghidraJavaHome ?? (fs.existsSync(jdk21) ? jdk21 : process.env.JAVA_HOME);
|
||||
const env = { ...process.env, MAXMEM: this.config.ghidraMaxMem ?? '6G', ...(javaHome ? { JAVA_HOME: javaHome, PATH: `${javaHome}/bin:${process.env.PATH}` } : {}) };
|
||||
const r = await run(exe, [this.projectDir, PROJECT, ...args, '-scriptPath', SCRIPTS_DIR], { timeoutMs, env });
|
||||
if (r.code !== 0) throw new Error(`Ghidra failed (exit ${r.code}): ${r.output.slice(-2000)}`);
|
||||
return r.output;
|
||||
}
|
||||
|
||||
writeSymbols(versionId) {
|
||||
const file = path.join(this.dir, 'symbols.tsv');
|
||||
const lines = [];
|
||||
for (const s of this.store.symbolMap(versionId).values()) {
|
||||
if (s.addr == null) continue;
|
||||
lines.push([hex(s.addr), s.kind, s.id, s.name ?? '', s.decorated ?? '', s.library ? 1 : 0].join('\t'));
|
||||
}
|
||||
fs.writeFileSync(file, lines.join('\n') + '\n');
|
||||
return file;
|
||||
}
|
||||
|
||||
/** Creates the project from the current version's exe (replacing an old one). */
|
||||
async build() {
|
||||
const v = this.store.currentVersion();
|
||||
fs.rmSync(this.projectDir, { recursive: true, force: true });
|
||||
fs.mkdirSync(this.dir, { recursive: true });
|
||||
const exe = path.join(this.dir, PROGRAM);
|
||||
fs.copyFileSync(v.file, exe);
|
||||
this.log.info(`ghidra: analyzing ${v.id} (this takes a while)`);
|
||||
await this.headless(['-import', exe, '-overwrite', '-postScript', 'ApplySymbols.java', this.writeSymbols(v.id)], 4 * 3600_000);
|
||||
fs.rmSync(exe, { force: true });
|
||||
this.store.setMeta('ghidra.version', v.id);
|
||||
await this.refresh({ applySymbols: false });
|
||||
}
|
||||
|
||||
/** Re-applies the symbol names, re-exports the originals and packs the project. */
|
||||
async refresh({ applySymbols = true } = {}) {
|
||||
const v = this.store.currentVersion();
|
||||
if (this.store.getMeta('ghidra.version') !== v.id) return this.build();
|
||||
const t0 = Date.now();
|
||||
if (applySymbols) await this.headless(['-process', PROGRAM, '-noanalysis', '-postScript', 'ApplySymbols.java', this.writeSymbols(v.id)], 3600_000);
|
||||
const jsonl = path.join(this.dir, 'originals.jsonl');
|
||||
await this.headless(['-process', PROGRAM, '-noanalysis', '-readOnly', '-postScript', 'ExportOriginals.java', jsonl, '30'], 6 * 3600_000);
|
||||
const n = await this.originals.import(v.id, jsonl);
|
||||
fs.rmSync(jsonl, { force: true });
|
||||
|
||||
// Pack the project for download (tar.gz: Windows' tar.exe and Linux tar both unpack it).
|
||||
const head = this.store.getMeta('ghidra.commit') ?? null;
|
||||
const name = `ghidra-${v.id}-${Date.now()}.tar.gz`;
|
||||
const tmp = path.join(this.dir, name + '.part');
|
||||
const r = await run('tar', ['-czf', tmp, '-C', this.projectDir, `${PROJECT}.gpr`, `${PROJECT}.rep`], { timeoutMs: 1800_000 });
|
||||
if (r.code !== 0) throw new Error(`packing the Ghidra project failed: ${r.output}`);
|
||||
const old = this.info();
|
||||
fs.renameSync(tmp, path.join(this.dir, name));
|
||||
const sha256 = await sha256File(path.join(this.dir, name));
|
||||
const info = { version: v.id, file: name, sha256, size: fs.statSync(path.join(this.dir, name)).size, builtAt: Date.now(), commit: head, functions: n, ghidra: this.config.ghidraVersion ?? '11.2.1' };
|
||||
fs.writeFileSync(path.join(this.dir, 'package.json'), JSON.stringify(info, null, 2));
|
||||
if (old?.file && old.file !== name) fs.rmSync(path.join(this.dir, old.file), { force: true });
|
||||
this.store.setMeta('ghidra.refreshedAt', Date.now());
|
||||
this.log.info(`ghidra: refreshed ${v.id} in ${((Date.now() - t0) / 1000).toFixed(0)}s (${n} functions exported, package ${(info.size / 1048576).toFixed(0)} MB)`);
|
||||
return info;
|
||||
}
|
||||
|
||||
/** Called after merges: refresh soon, but at most every ghidraRefreshMinutes. */
|
||||
scheduleRefresh(commit) {
|
||||
if (!this.enabled) return;
|
||||
if (commit) this.store.setMeta('ghidra.commit', commit);
|
||||
if (this.timer || this.busy) return;
|
||||
const every = (this.config.ghidraRefreshMinutes ?? 30) * 60_000;
|
||||
const last = Number(this.store.getMeta('ghidra.refreshedAt') ?? 0);
|
||||
const wait = Math.max(0, last + every - Date.now());
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = null;
|
||||
this.busy = this.refresh()
|
||||
.catch((e) => this.log.error(`ghidra refresh failed: ${e.message}`))
|
||||
.finally(() => {
|
||||
this.busy = null;
|
||||
});
|
||||
}, wait);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
}
|
||||
|
||||
function sha256File(file) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const h = crypto.createHash('sha256');
|
||||
fs.createReadStream(file).on('data', (d) => h.update(d)).on('end', () => resolve(h.digest('hex'))).on('error', reject);
|
||||
});
|
||||
}
|
||||
111
src/server/install.js
Normal file
111
src/server/install.js
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
// One-line installers. The scripts are generated per request with this server's URL, the project
|
||||
// name and the SHA-256 of the td binaries in <dataDir>/downloads (built by tools/build-td).
|
||||
//
|
||||
// Windows: $env:TD_TOKEN='td_...'; irm https://<host>/install.ps1 | iex
|
||||
// Linux: curl -fsSL https://<host>/install.sh | TD_TOKEN=td_... sh
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
export function downloadsDir(config) {
|
||||
return path.join(config.dataDir, 'downloads');
|
||||
}
|
||||
|
||||
function checksums(config) {
|
||||
const file = path.join(downloadsDir(config), 'SHA256SUMS');
|
||||
const out = {};
|
||||
if (!fs.existsSync(file)) return out;
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
const m = /^([0-9a-f]{64})\s+(\S+)$/.exec(line.trim());
|
||||
if (m) out[m[2]] = m[1];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const slug = (name) => String(name ?? 'project').replace(/[^A-Za-z0-9_.-]/g, '-');
|
||||
|
||||
export function installPs1(config, project) {
|
||||
const sha = checksums(config)['td-windows-x64.exe'];
|
||||
if (!sha) return null;
|
||||
return `# Clank Together installer for Windows. Usage (PowerShell, one line):
|
||||
# $env:TD_TOKEN='td_...'; irm ${config.publicUrl}/install.ps1 | iex
|
||||
# Installs td (a single program, no Node.js needed), sets up the ${project.name} workspace and
|
||||
# connects your AI. Optional: $env:TD_DIR (workspace folder), $env:TD_TOOLCHAIN (local compiler root).
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
$server = '${config.publicUrl}'
|
||||
$token = $env:TD_TOKEN
|
||||
if (-not $token) { $token = Read-Host 'API token (log in on $server and create one under Account)' }
|
||||
$dir = if ($env:TD_DIR) { $env:TD_DIR } else { Join-Path $HOME 'clank\\${slug(project.name)}' }
|
||||
$bin = Join-Path $env:LOCALAPPDATA 'ClankTogether\\bin'
|
||||
New-Item -ItemType Directory -Force $bin | Out-Null
|
||||
|
||||
if (-not (Get-Command git -ErrorAction SilentlyContinue)) {
|
||||
Write-Host 'Installing git (winget)...'
|
||||
winget install --id Git.Git -e --silent --accept-package-agreements --accept-source-agreements | Out-Null
|
||||
$env:Path += ';' + (Join-Path $env:ProgramFiles 'Git\\cmd')
|
||||
if (-not (Get-Command git -ErrorAction SilentlyContinue)) { throw 'git is required: install it from https://git-scm.com and run this line again' }
|
||||
}
|
||||
|
||||
Write-Host 'Downloading td...'
|
||||
$exe = Join-Path $bin 'td.exe'
|
||||
Invoke-WebRequest "$server/download/td-windows-x64.exe" -OutFile "$exe.part" -UseBasicParsing
|
||||
if ((Get-FileHash "$exe.part" -Algorithm SHA256).Hash.ToLower() -ne '${sha}') { Remove-Item "$exe.part"; throw 'td download failed its checksum check' }
|
||||
Move-Item -Force "$exe.part" $exe
|
||||
|
||||
$userPath = [Environment]::GetEnvironmentVariable('Path', 'User')
|
||||
if (-not $userPath) { $userPath = '' }
|
||||
if (($userPath -split ';') -notcontains $bin) { [Environment]::SetEnvironmentVariable('Path', ($userPath.TrimEnd(';') + ';' + $bin), 'User') }
|
||||
if (($env:Path -split ';') -notcontains $bin) { $env:Path += ';' + $bin }
|
||||
|
||||
$setup = @('setup', $server, $dir, '--token', $token)
|
||||
if ($env:TD_TOOLCHAIN) { $setup += @('--toolchain', $env:TD_TOOLCHAIN) }
|
||||
& $exe @setup
|
||||
if ($LASTEXITCODE -ne 0) { throw 'td setup failed (see above)' }
|
||||
& $exe connect --workspace $dir
|
||||
Write-Host ''
|
||||
Write-Host "Done. Workspace: $dir (td is on your PATH in new terminals)"
|
||||
`;
|
||||
}
|
||||
|
||||
export function installSh(config, project) {
|
||||
const sha = checksums(config)['td-linux-x64'];
|
||||
if (!sha) return null;
|
||||
return `#!/bin/sh
|
||||
# Clank Together installer for Linux. Usage (one line):
|
||||
# curl -fsSL ${config.publicUrl}/install.sh | TD_TOKEN=td_... sh
|
||||
# Installs td (a single program, no Node.js needed), sets up the ${project.name} workspace and
|
||||
# connects your AI. Optional: TD_DIR (workspace folder), TD_TOOLCHAIN (local compiler root).
|
||||
set -eu
|
||||
SERVER='${config.publicUrl}'
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Linux-x86_64) ;;
|
||||
*) echo "td is available for Linux x86_64 and Windows x64; this is $(uname -s) $(uname -m)" >&2; exit 1 ;;
|
||||
esac
|
||||
TOKEN="\${TD_TOKEN:-}"
|
||||
if [ -z "$TOKEN" ]; then printf 'API token (log in on %s and create one under Account): ' "$SERVER"; read -r TOKEN < /dev/tty; fi
|
||||
DIR="\${TD_DIR:-$HOME/clank/${slug(project.name)}}"
|
||||
BIN="$HOME/.local/bin"
|
||||
mkdir -p "$BIN"
|
||||
command -v git >/dev/null 2>&1 || { echo 'git is required: install it (e.g. sudo apt install git) and run this line again' >&2; exit 1; }
|
||||
|
||||
echo 'Downloading td...'
|
||||
curl -fsSL "$SERVER/download/td-linux-x64" -o "$BIN/td.part"
|
||||
echo "${sha} $BIN/td.part" | sha256sum -c - >/dev/null || { rm -f "$BIN/td.part"; echo 'td download failed its checksum check' >&2; exit 1; }
|
||||
mv "$BIN/td.part" "$BIN/td"
|
||||
chmod +x "$BIN/td"
|
||||
case ":$PATH:" in
|
||||
*":$BIN:"*) ;;
|
||||
*) echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$HOME/.profile"; PATH="$BIN:$PATH" ;;
|
||||
esac
|
||||
|
||||
if [ -n "\${TD_TOOLCHAIN:-}" ]; then
|
||||
"$BIN/td" setup "$SERVER" "$DIR" --token "$TOKEN" --toolchain "$TD_TOOLCHAIN"
|
||||
else
|
||||
"$BIN/td" setup "$SERVER" "$DIR" --token "$TOKEN"
|
||||
fi
|
||||
"$BIN/td" connect --workspace "$DIR"
|
||||
echo
|
||||
echo "Done. Workspace: $DIR (td is in ~/.local/bin)"
|
||||
`;
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import { HttpError } from './http.js';
|
|||
import { now } from './db.js';
|
||||
import { hasRole } from './accounts.js';
|
||||
import { STATUS } from '../shared/compare.js';
|
||||
import { returnBatch, batchAttempts } from './batches.js';
|
||||
|
||||
export class Merger {
|
||||
constructor({ store, config, repo, project, tasks, verifier, getPe, peers, log = console }) {
|
||||
|
|
@ -12,6 +13,7 @@ export class Merger {
|
|||
}
|
||||
|
||||
start() {
|
||||
this.store.run(`UPDATE batches SET state = 'open' WHERE state = 'merging'`);
|
||||
this.store.run(`UPDATE tasks SET state = 'approved' WHERE state = 'merging'`);
|
||||
this.timer = setInterval(() => this.pump(), 3000);
|
||||
}
|
||||
|
|
@ -21,15 +23,32 @@ export class Merger {
|
|||
}
|
||||
|
||||
requiredApprovals(attempt) {
|
||||
const result = attempt.result_json ? JSON.parse(attempt.result_json) : {};
|
||||
const batch = attempt.batch_id ? this.store.get('SELECT result_json FROM batches WHERE id = ?', attempt.batch_id) : null;
|
||||
const result = batch?.result_json ? JSON.parse(batch.result_json) : {};
|
||||
const p = this.project.matchPolicy;
|
||||
return result.changedHeaders?.length ? p.approvalsWithHeaderChanges : p.approvals;
|
||||
}
|
||||
|
||||
/** A human decision on one attempt (peer-confirmed, not byte-identical). */
|
||||
review(reviewer, attemptId, action, comment) {
|
||||
if (!hasRole(reviewer, 'reviewer')) throw new HttpError(403, 'only reviewers can review');
|
||||
if (!['approve', 'request_changes', 'reject'].includes(action)) throw new HttpError(400, 'invalid action');
|
||||
return this.store.tx(() => this.reviewOne(reviewer, attemptId, action, comment));
|
||||
}
|
||||
|
||||
/** The same decision for every attempt of a batch that waits for human review. */
|
||||
reviewBatch(reviewer, batchId, action, comment) {
|
||||
if (!hasRole(reviewer, 'reviewer')) throw new HttpError(403, 'only reviewers can review');
|
||||
if (!['approve', 'request_changes', 'reject'].includes(action)) throw new HttpError(400, 'invalid action');
|
||||
return this.store.tx(() => {
|
||||
const waiting = this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'in_review'`, batchId);
|
||||
if (!waiting.length) throw new HttpError(409, 'nothing in this batch waits for human review');
|
||||
const results = waiting.map((w) => ({ attemptId: w.id, ...this.reviewOne(reviewer, w.id, action, comment) }));
|
||||
return { batchId, results };
|
||||
});
|
||||
}
|
||||
|
||||
reviewOne(reviewer, attemptId, action, comment) {
|
||||
const attempt = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId);
|
||||
if (!attempt) throw new HttpError(404, 'no such attempt');
|
||||
if (attempt.state !== 'in_review') throw new HttpError(409, `attempt is ${attempt.state}`);
|
||||
|
|
@ -52,60 +71,96 @@ export class Merger {
|
|||
const newState = action === 'reject' ? 'rejected' : 'needs_cleanup';
|
||||
this.store.run('UPDATE attempts SET state = ? WHERE id = ?', newState, attemptId);
|
||||
this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), attempt.task_id);
|
||||
returnBatch(this.store, attempt.batch_id, `a reviewer asked for changes to ${attempt.task_id}`);
|
||||
return { state: newState };
|
||||
});
|
||||
}
|
||||
|
||||
/** Merges the oldest open batch whose attempts are all approved. */
|
||||
async pump() {
|
||||
if (this.busy) return;
|
||||
const task = this.store.get(`SELECT * FROM tasks WHERE state = 'approved' ORDER BY updated_at LIMIT 1`);
|
||||
if (!task) return;
|
||||
const batch = this.store.get(
|
||||
`SELECT b.* FROM batches b WHERE b.state = 'open'
|
||||
AND EXISTS (SELECT 1 FROM attempts a WHERE a.batch_id = b.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM attempts a WHERE a.batch_id = b.id AND a.state != 'approved')
|
||||
ORDER BY b.id LIMIT 1`,
|
||||
);
|
||||
if (!batch) return;
|
||||
this.busy = true;
|
||||
try {
|
||||
await this.merge(task);
|
||||
await this.merge(batch);
|
||||
} catch (e) {
|
||||
this.log.error(`merge of task ${task.id} failed: ${e.stack}`);
|
||||
this.log.error(`merge of batch ${batch.id} failed: ${e.stack}`);
|
||||
try {
|
||||
this.repo.reset();
|
||||
} catch {}
|
||||
this.store.run(`UPDATE tasks SET state = 'in_review', updated_at = ? WHERE id = ?`, now(), task.id);
|
||||
this.store.event('merge_error', null, { task: task.id, error: e.message });
|
||||
this.store.run(`UPDATE batches SET state = 'open' WHERE id = ?`, batch.id);
|
||||
this.store.event('merge_error', null, { batch: batch.id, error: e.message });
|
||||
} finally {
|
||||
this.busy = false;
|
||||
}
|
||||
}
|
||||
|
||||
async merge(task) {
|
||||
this.store.run(`UPDATE tasks SET state = 'merging', updated_at = ? WHERE id = ?`, now(), task.id);
|
||||
const attempt = this.store.get(`SELECT * FROM attempts WHERE task_id = ? AND state = 'approved' ORDER BY id DESC LIMIT 1`, task.id);
|
||||
const author = this.store.get('SELECT name FROM users WHERE id = ?', attempt.user_id).name;
|
||||
const files = JSON.parse(attempt.files_json);
|
||||
async merge(batch) {
|
||||
this.store.run(`UPDATE batches SET state = 'merging' WHERE id = ?`, batch.id);
|
||||
const attempts = batchAttempts(this.store, batch.id);
|
||||
for (const a of attempts) this.store.run(`UPDATE tasks SET state = 'merging', updated_at = ? WHERE id = ?`, now(), a.task_id);
|
||||
const author = this.store.get('SELECT name FROM users WHERE id = ?', batch.user_id).name;
|
||||
const files = JSON.parse(batch.files_json);
|
||||
|
||||
const merged = this.repo.mergeOntoHead(attempt.base_commit, files);
|
||||
const bounce = (summary, details) => {
|
||||
this.store.run(`UPDATE attempts SET state = 'needs_cleanup', result_json = json_set(COALESCE(result_json, '{}'), '$.mergeFailure', json(?)) WHERE id = ?`, JSON.stringify({ summary, ...details }), attempt.id);
|
||||
this.store.run(`UPDATE tasks SET state = 'needs_cleanup', preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, attempt.user_id, now() + 6 * 3600_000, now(), task.id);
|
||||
this.log.info(`merge of task ${task.id} bounced: ${summary}`);
|
||||
this.store.run(`UPDATE batches SET result_json = json_set(COALESCE(result_json, '{}'), '$.mergeFailure', json(?)) WHERE id = ?`, JSON.stringify({ summary, ...details }), batch.id);
|
||||
returnBatch(this.store, batch.id, summary);
|
||||
this.log.info(`merge of batch ${batch.id} bounced: ${summary}`);
|
||||
};
|
||||
const merged = this.repo.mergeOntoHead(batch.base_commit, files);
|
||||
if (merged.conflicts.length) return bounce('conflicts with newer work on main', { conflicts: merged.conflicts });
|
||||
|
||||
// Re-verify on top of the current main: other merges may have landed since review.
|
||||
const { check, rec } = await this.verifier.check(merged.files, `merge-${attempt.id}`);
|
||||
const evaln = this.verifier.evaluate({ check, rec, targetId: task.symbol_id, files: merged.files, changedHeaders: Object.keys(merged.files).filter((f) => !/\.(c|cpp|cc|cxx)$/i.test(f)) });
|
||||
// Not byte-identical is fine if other agents confirmed the logic; anything worse bounces.
|
||||
const peerConfirmed = evaln.state === 'peer_review' && this.peers?.confirmed(attempt.id);
|
||||
if (evaln.state !== 'in_review' && !peerConfirmed) return bounce(`re-verification on current main failed: ${evaln.feedback.summary}`, { feedback: evaln.feedback });
|
||||
// Re-verify on top of the current main: other merges may have landed since verification.
|
||||
const { check, rec, changedHeaders } = await this.verifier.check(merged.files, `merge-${batch.id}`);
|
||||
const targets = new Set(attempts.map((a) => a.symbol_id));
|
||||
const outside = this.verifier.changedOutside(check, merged.files, targets);
|
||||
const confirmedIds = new Set();
|
||||
const results = [];
|
||||
for (const a of attempts) {
|
||||
const ev = this.verifier.evaluate({ check, rec, targetId: a.symbol_id, files: merged.files, changedHeaders, outside });
|
||||
// Byte-identical, or confirmed by peers and approved by a human; anything worse bounces.
|
||||
const identical = ev.state === 'approved' || ev.state === 'in_review';
|
||||
const peerConfirmed = ev.state === 'peer_review' && this.peers?.confirmed(a.id);
|
||||
if (!identical && !peerConfirmed) return bounce(`re-verification on current main failed for ${a.symbol_id}: ${ev.feedback.summary}`, { feedback: ev.feedback });
|
||||
if (peerConfirmed) confirmedIds.add(a.symbol_id);
|
||||
results.push({ id: a.symbol_id, name: ev.feedback.target?.demangled ?? null, status: ev.feedback.target?.status, peerConfirmed: !!peerConfirmed });
|
||||
}
|
||||
|
||||
const reviewers = this.store.all(`SELECT u.name FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ? AND r.action = 'approve'`, attempt.id).map((r) => r.name);
|
||||
const target = evaln.feedback.target;
|
||||
const message = `Decompile ${task.symbol_id}${target?.demangled ? ` (${target.demangled})` : ''}\n\nAttempt #${attempt.id} by ${author}, status ${target?.status}.\nReviewed-by: ${reviewers.join(', ')}\n`;
|
||||
const reviewers = this.store.all(
|
||||
`SELECT DISTINCT u.name FROM reviews r JOIN users u ON u.id = r.reviewer_id JOIN attempts a ON a.id = r.attempt_id WHERE a.batch_id = ? AND r.action = 'approve'`,
|
||||
batch.id,
|
||||
).map((r) => r.name);
|
||||
const peers = this.store.all(
|
||||
`SELECT DISTINCT u.name FROM peer_reviews p JOIN users u ON u.id = p.reviewer_id JOIN attempts a ON a.id = p.attempt_id WHERE a.batch_id = ? AND p.verdict = 'same'`,
|
||||
batch.id,
|
||||
).map((r) => r.name);
|
||||
const identicalCount = results.filter((r) => !r.peerConfirmed).length;
|
||||
const title =
|
||||
results.length === 1
|
||||
? `Decompile ${results[0].id}${results[0].name ? ` (${results[0].name})` : ''}`
|
||||
: `Decompile ${results.length} functions in ${[...new Set(Object.keys(files))].slice(0, 2).join(', ')}${Object.keys(files).length > 2 ? ', ...' : ''}`;
|
||||
const lines = results.map((r) => `- ${r.id}${r.name ? ` ${r.name}` : ''}: ${r.peerConfirmed ? 'logic confirmed by peer review' : 'byte-identical'}`);
|
||||
const message =
|
||||
`${title}\n\nBatch #${batch.id} by ${author}: ${identicalCount} byte-identical, ${results.length - identicalCount} confirmed by peer review.\n\n${lines.join('\n')}\n` +
|
||||
(peers.length ? `\nPeer-reviewed-by: ${peers.join(', ')}` : '') +
|
||||
(reviewers.length ? `\nReviewed-by: ${reviewers.join(', ')}` : '') +
|
||||
(!reviewers.length && !confirmedIds.size ? '\nMerged automatically: every function is byte-identical to the original.' : '') +
|
||||
'\n';
|
||||
const commit = this.repo.commitFiles(merged.files, message, `${author} <${author.replace(/[^A-Za-z0-9_.-]/g, '_')}@users.clank-together.invalid>`);
|
||||
|
||||
this.store.tx(() => {
|
||||
this.applyCheckToDatabase(check, rec, commit, attempt.user_id, { confirmedId: peerConfirmed ? task.symbol_id : null });
|
||||
this.store.run(`UPDATE attempts SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, attempt.id);
|
||||
this.store.run(`UPDATE tasks SET state = 'done', lease_user = NULL, updated_at = ? WHERE id = ?`, now(), task.id);
|
||||
this.store.event('merged', attempt.user_id, { task: task.id, attempt: attempt.id, commit });
|
||||
this.applyCheckToDatabase(check, rec, commit, batch.user_id, { confirmedIds });
|
||||
for (const a of attempts) {
|
||||
this.store.run(`UPDATE attempts SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, a.id);
|
||||
this.store.run(`UPDATE tasks SET state = 'done', lease_user = NULL, updated_at = ? WHERE id = ?`, now(), a.task_id);
|
||||
}
|
||||
this.store.run(`UPDATE batches SET state = 'merged', merged_commit = ? WHERE id = ?`, commit, batch.id);
|
||||
this.store.event('merged', batch.user_id, { batch: batch.id, functions: attempts.length, commit });
|
||||
});
|
||||
this.tasks.generate();
|
||||
if (this.config.pushRemote) {
|
||||
|
|
@ -115,14 +170,15 @@ export class Merger {
|
|||
this.log.error(`push to ${this.config.pushRemote} failed: ${e.message}`);
|
||||
}
|
||||
}
|
||||
this.log.info(`merged task ${task.id} as ${commit.slice(0, 10)}`);
|
||||
this.onMerged?.(commit);
|
||||
this.log.info(`merged batch ${batch.id} (${attempts.length} functions) as ${commit.slice(0, 10)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Records what a check on main learned: per-function status, decorated
|
||||
* names and new symbols for previously unknown references.
|
||||
*/
|
||||
applyCheckToDatabase(check, rec, commit, userId, { confirmedId = null } = {}) {
|
||||
applyCheckToDatabase(check, rec, commit, userId, { confirmedIds = new Set() } = {}) {
|
||||
const v = this.store.currentVersion();
|
||||
const pe = this.getPe();
|
||||
for (const u of check.units) {
|
||||
|
|
@ -135,10 +191,10 @@ export class Merger {
|
|||
}
|
||||
for (const f of u.functions) {
|
||||
this.store.run(
|
||||
`INSERT INTO func_status (symbol_id, file, status, score, commit_hash, updated_at, confirmed) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(symbol_id) DO UPDATE SET file = excluded.file, status = excluded.status, score = excluded.score, commit_hash = excluded.commit_hash, updated_at = excluded.updated_at,
|
||||
`INSERT INTO func_status (symbol_id, file, status, score, commit_hash, updated_at, confirmed, code_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(symbol_id) DO UPDATE SET file = excluded.file, status = excluded.status, score = excluded.score, commit_hash = excluded.commit_hash, updated_at = excluded.updated_at, code_hash = COALESCE(excluded.code_hash, func_status.code_hash),
|
||||
confirmed = CASE WHEN excluded.confirmed = 1 THEN 1 WHEN excluded.status IN ('exact', 'match_unresolved') THEN 0 ELSE func_status.confirmed END`,
|
||||
f.id, u.file, f.status, f.score, commit, now(), f.id === confirmedId ? 1 : 0,
|
||||
f.id, u.file, f.status, f.score, commit, now(), confirmedIds.has(f.id) ? 1 : 0, f.codeHash ?? null,
|
||||
);
|
||||
if (f.demangled) this.store.run('UPDATE symbols SET name = ? WHERE id = ? AND name IS NULL', f.demangled, f.id);
|
||||
this.store.run('UPDATE symbols SET unit = ? WHERE id = ? AND unit IS NULL', u.file, f.id);
|
||||
|
|
|
|||
95
src/server/original.js
Normal file
95
src/server/original.js
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
// The original function, for contributors without a local Ghidra: assembly and pseudocode from a
|
||||
// Ghidra export (ghidra_scripts/ExportOriginals.java, imported with `td-server originals import`),
|
||||
// and live disassembly by objdump for functions the export does not cover.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import readline from 'node:readline';
|
||||
import { run } from '../shared/build.js';
|
||||
import { hex } from '../shared/annotations.js';
|
||||
import { HttpError } from './http.js';
|
||||
|
||||
export class Originals {
|
||||
constructor({ store }) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Imports an ExportOriginals JSON-lines file for a version; returns the number of functions. */
|
||||
async import(versionId, file) {
|
||||
const rl = readline.createInterface({ input: fs.createReadStream(file, 'utf8'), crlfDelay: Infinity });
|
||||
let n = 0;
|
||||
const insert = this.store.db.prepare(
|
||||
`INSERT INTO original_code (version_id, addr, asm, c) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(version_id, addr) DO UPDATE SET asm = excluded.asm, c = excluded.c`,
|
||||
);
|
||||
this.store.db.exec('BEGIN');
|
||||
try {
|
||||
for await (const line of rl) {
|
||||
if (!line.trim()) continue;
|
||||
const r = JSON.parse(line);
|
||||
insert.run(versionId, Number.parseInt(r.addr, 16), r.asm ?? null, r.c ?? null);
|
||||
if (++n % 5000 === 0) {
|
||||
this.store.db.exec('COMMIT');
|
||||
this.store.db.exec('BEGIN');
|
||||
}
|
||||
}
|
||||
this.store.db.exec('COMMIT');
|
||||
} catch (e) {
|
||||
this.store.db.exec('ROLLBACK');
|
||||
throw e;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/** {symbol, name, version, addr, size, asm, pseudocode, source} for a symbol ID or a hex address. */
|
||||
async get(query) {
|
||||
const v = this.store.currentVersion();
|
||||
if (!v) throw new HttpError(404, 'no current version');
|
||||
let sym;
|
||||
if (/^0x[0-9a-f]+$/i.test(query)) {
|
||||
sym = this.store.get(
|
||||
`SELECT s.id, s.name, a.addr, a.size FROM symbol_addresses a JOIN symbols s ON s.id = a.symbol_id WHERE a.version_id = ? AND a.addr = ?`,
|
||||
v.id,
|
||||
Number.parseInt(query, 16),
|
||||
) ?? { id: null, name: null, addr: Number.parseInt(query, 16), size: null };
|
||||
} else {
|
||||
sym = this.store.get(
|
||||
`SELECT s.id, s.name, a.addr, a.size FROM symbols s LEFT JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? WHERE s.id = ? OR s.name = ? OR s.decorated = ? LIMIT 1`,
|
||||
v.id,
|
||||
query,
|
||||
query,
|
||||
query,
|
||||
);
|
||||
}
|
||||
if (!sym) throw new HttpError(404, `unknown symbol ${query}`);
|
||||
if (sym.addr == null) throw new HttpError(404, `${sym.id} has no address in ${v.id}`);
|
||||
const row = this.store.get('SELECT asm, c FROM original_code WHERE version_id = ? AND addr = ?', v.id, sym.addr);
|
||||
const asm = row?.asm ?? (await this.objdump(v.file, sym.addr, sym.size ?? 256));
|
||||
return {
|
||||
symbol: sym.id,
|
||||
name: sym.name,
|
||||
version: v.id,
|
||||
addr: '0x' + hex(sym.addr),
|
||||
size: sym.size,
|
||||
asm,
|
||||
pseudocode: row?.c ?? null,
|
||||
source: row ? 'ghidra export' : 'objdump (no pseudocode yet)',
|
||||
note: 'Pseudocode is Ghidra output, a reading aid; it is not matching code.',
|
||||
};
|
||||
}
|
||||
|
||||
async objdump(file, addr, size) {
|
||||
const r = await run('objdump', ['-d', '-M', 'intel', '--no-addresses', `--start-address=0x${hex(addr)}`, `--stop-address=0x${hex(addr + Math.min(size, 65536))}`, file], { timeoutMs: 30_000 });
|
||||
if (r.code !== 0) {
|
||||
// Older binutils lack --no-addresses.
|
||||
const r2 = await run('objdump', ['-d', '-M', 'intel', `--start-address=0x${hex(addr)}`, `--stop-address=0x${hex(addr + Math.min(size, 65536))}`, file], { timeoutMs: 30_000 });
|
||||
return cleanObjdump(r2.output);
|
||||
}
|
||||
return cleanObjdump(r.output);
|
||||
}
|
||||
}
|
||||
|
||||
function cleanObjdump(out) {
|
||||
const lines = out.split('\n');
|
||||
const start = lines.findIndex((l) => /^\s*[0-9a-f]+ <|^Disassembly of section/.test(l));
|
||||
return lines.slice(start >= 0 ? start : 0).filter((l) => l.trim() && !/^Disassembly of section/.test(l)).join('\n');
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import { HttpError } from './http.js';
|
|||
import { now } from './db.js';
|
||||
import { untrusted } from './tasks.js';
|
||||
import { hex } from '../shared/annotations.js';
|
||||
import { returnBatch } from './batches.js';
|
||||
|
||||
export const PEER_INSTRUCTIONS = [
|
||||
'Decide whether the submitted C++ does the same thing as the original function. It is not byte-identical; your judgment is what counts.',
|
||||
|
|
@ -36,7 +37,7 @@ export class PeerReviews {
|
|||
/** Attempts waiting for peer review that `user` may take. */
|
||||
available(user, limit = 20) {
|
||||
return this.store.all(
|
||||
`SELECT a.id AS attempt_id, a.task_id, t.symbol_id, s.name, a.finished_at,
|
||||
`SELECT a.id AS attempt_id, a.batch_id, a.task_id, t.symbol_id, s.name, a.finished_at,
|
||||
(SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS confirmations,
|
||||
(SELECT COUNT(*) FROM peer_leases l WHERE l.attempt_id = a.id AND l.expires_at > ?) AS in_progress
|
||||
FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
|
|
@ -51,28 +52,56 @@ export class PeerReviews {
|
|||
).filter((r) => r.confirmations + r.in_progress < this.required);
|
||||
}
|
||||
|
||||
/** Leases one review (a given attempt, or the oldest available) and returns its context. */
|
||||
/**
|
||||
* Leases a review (a given attempt, or the oldest available) together with every other function
|
||||
* of the same batch that waits for this reviewer, and returns the context of all of them.
|
||||
*/
|
||||
claim(user, attemptId = null) {
|
||||
return this.store.tx(() => {
|
||||
const options = this.available(user, 200);
|
||||
const pick = attemptId ? options.find((o) => o.attempt_id === attemptId) : options[0];
|
||||
if (!pick) throw new HttpError(attemptId ? 409 : 404, attemptId ? 'this attempt is not open for your review (yours, already reviewed, or enough reviewers)' : 'no attempts are waiting for peer review');
|
||||
const group = pick.batch_id ? options.filter((o) => o.batch_id === pick.batch_id) : [pick];
|
||||
for (const o of group) {
|
||||
this.store.run(
|
||||
`INSERT INTO peer_leases (attempt_id, reviewer_id, expires_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT(attempt_id, reviewer_id) DO UPDATE SET expires_at = excluded.expires_at`,
|
||||
pick.attempt_id, user.id, now() + this.leaseMs,
|
||||
o.attempt_id, user.id, now() + this.leaseMs,
|
||||
);
|
||||
this.store.event('peer_claim', user.id, { attempt: pick.attempt_id });
|
||||
return this.context(pick.attempt_id);
|
||||
}
|
||||
this.store.event('peer_claim', user.id, { attempt: pick.attempt_id, functions: group.length });
|
||||
return this.context(pick.attempt_id, group.map((o) => o.attempt_id));
|
||||
});
|
||||
}
|
||||
|
||||
/** One function to review: where the original is and how to read it. */
|
||||
functionInfo(attemptId, v) {
|
||||
const row = this.store.get(
|
||||
`SELECT a.id, s.id AS symbol, s.name, ad.addr, ad.size FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
LEFT JOIN symbol_addresses ad ON ad.symbol_id = s.id AND ad.version_id = ? WHERE a.id = ?`,
|
||||
v.id,
|
||||
attemptId,
|
||||
);
|
||||
return {
|
||||
attemptId: row.id,
|
||||
symbol: row.symbol,
|
||||
name: row.name,
|
||||
addr: row.addr != null ? '0x' + hex(row.addr) : null,
|
||||
size: row.size,
|
||||
annotation: `// FUNCTION: ${row.symbol}`,
|
||||
read: [`td_disassemble ${row.symbol}`, `td_decompile ${row.symbol}`],
|
||||
};
|
||||
}
|
||||
|
||||
release(user, attemptId) {
|
||||
this.store.run('DELETE FROM peer_leases WHERE attempt_id = ? AND reviewer_id = ?', attemptId, user.id);
|
||||
}
|
||||
|
||||
/** What a reviewer sees: the function to compare against and the submitted code, nothing machine-judged. */
|
||||
context(attemptId) {
|
||||
/**
|
||||
* What a reviewer sees: the functions to compare against and the submitted code, nothing
|
||||
* machine-judged. `attemptIds`: the functions of the batch this review covers.
|
||||
*/
|
||||
context(attemptId, attemptIds = null) {
|
||||
const a = this.store.get('SELECT * FROM attempts WHERE id = ?', attemptId);
|
||||
if (!a) throw new HttpError(404, 'no such attempt');
|
||||
const t = this.store.get('SELECT * FROM tasks WHERE id = ?', a.task_id);
|
||||
|
|
@ -82,12 +111,16 @@ export class PeerReviews {
|
|||
v.id,
|
||||
t.symbol_id,
|
||||
);
|
||||
const result = a.result_json ? JSON.parse(a.result_json) : {};
|
||||
const batch = a.batch_id ? this.store.get('SELECT * FROM batches WHERE id = ?', a.batch_id) : null;
|
||||
const batchResult = batch?.result_json ? JSON.parse(batch.result_json) : {};
|
||||
const ids = attemptIds ?? (a.batch_id ? this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'peer_review' ORDER BY id`, a.batch_id).map((r) => r.id) : [a.id]);
|
||||
return {
|
||||
attemptId: a.id,
|
||||
required: this.required,
|
||||
confirmations: this.store.get(`SELECT COUNT(*) AS n FROM peer_reviews WHERE attempt_id = ? AND verdict = 'same'`, a.id).n,
|
||||
instructions: PEER_INSTRUCTIONS,
|
||||
// Every function this review covers (one batch); give one verdict per attemptId.
|
||||
functions: (ids.length ? ids : [a.id]).map((id) => this.functionInfo(id, v)),
|
||||
original: {
|
||||
symbol: s.id,
|
||||
name: s.name,
|
||||
|
|
@ -98,7 +131,7 @@ export class PeerReviews {
|
|||
},
|
||||
submission: {
|
||||
annotation: `// FUNCTION: ${s.id}`,
|
||||
files: result.mergedFiles ?? JSON.parse(a.files_json),
|
||||
files: batchResult.mergedFiles ?? this.store.attemptFiles(a),
|
||||
notes: untrusted(a.notes),
|
||||
},
|
||||
leaseExpires: this.store.get('SELECT expires_at FROM peer_leases WHERE attempt_id = ? ORDER BY expires_at DESC LIMIT 1', a.id)?.expires_at ?? null,
|
||||
|
|
@ -135,6 +168,8 @@ export class PeerReviews {
|
|||
`UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`,
|
||||
a.user_id, now() + 6 * 3600_000, now(), a.task_id,
|
||||
);
|
||||
// The batch merges as a whole, so it goes back with this function; confirmed ones keep their verdicts.
|
||||
returnBatch(this.store, a.batch_id, `a peer review found a difference in ${a.task_id}`);
|
||||
return { state: 'needs_cleanup', confirmations: taken };
|
||||
}
|
||||
const confirmations = taken + 1;
|
||||
|
|
|
|||
|
|
@ -75,10 +75,46 @@ export class Tasks {
|
|||
return this.store.all(`SELECT id FROM tasks WHERE lease_user = ? AND state = 'leased'`, userId).map((r) => r.id);
|
||||
}
|
||||
|
||||
/** How many tasks (functions) one account may hold at once, single claims and batches together. */
|
||||
get maxLeased() {
|
||||
return this.config.maxLeasedTasks ?? 200;
|
||||
}
|
||||
|
||||
/**
|
||||
* Claims several tasks at once: every claimable function of a unit (source file) or of an
|
||||
* address range in the current version, up to `limit`. They are submitted together as a batch.
|
||||
*/
|
||||
claimBatch(user, { unit = null, start = null, end = null, limit = 50 } = {}) {
|
||||
if (!unit && (start == null || end == null)) throw new HttpError(400, 'give a unit (source file) or an address range (start, end)');
|
||||
return this.store.tx(() => {
|
||||
const held = this.leasesOf(user.id).length;
|
||||
const room = Math.min(Math.max(1, limit | 0), 200, this.maxLeased - held);
|
||||
if (room <= 0) throw new HttpError(409, `you already hold ${held} tasks; submit or release some first`);
|
||||
const v = this.store.currentVersion();
|
||||
const ph = CLAIMABLE.map(() => '?').join(',');
|
||||
const where = unit ? 's.unit = ?' : 'a.addr >= ? AND a.addr < ?';
|
||||
const args = unit ? [unit] : [start, end];
|
||||
const rows = this.store.all(
|
||||
`SELECT t.id FROM tasks t JOIN symbols s ON s.id = t.symbol_id
|
||||
JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ?
|
||||
WHERE t.state IN (${ph}) AND ${where}
|
||||
AND (t.preferred_user IS NULL OR t.preferred_user = ? OR t.preferred_until < ?)
|
||||
ORDER BY a.addr LIMIT ?`,
|
||||
v.id, ...CLAIMABLE, ...args, user.id, now(), room,
|
||||
);
|
||||
if (!rows.length) throw new HttpError(404, unit ? `no claimable tasks in ${unit}` : 'no claimable tasks in that range');
|
||||
const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000;
|
||||
const head = this.repo.head();
|
||||
for (const r of rows) this.store.run(`UPDATE tasks SET state = 'leased', lease_user = ?, lease_expires = ?, lease_base_commit = ?, updated_at = ? WHERE id = ?`, user.id, expires, head, now(), r.id);
|
||||
this.store.event('claim_batch', user.id, { tasks: rows.length, unit, start, end });
|
||||
return rows.map((r) => r.id);
|
||||
});
|
||||
}
|
||||
|
||||
claim(user, taskId) {
|
||||
return this.store.tx(() => {
|
||||
const max = this.config.maxLeasesPerUser ?? 3;
|
||||
if (this.leasesOf(user.id).length >= max) throw new HttpError(409, `you already hold ${max} leases; submit or release one first`);
|
||||
const max = this.maxLeased;
|
||||
if (this.leasesOf(user.id).length >= max) throw new HttpError(409, `you already hold ${max} tasks; submit or release some first`);
|
||||
let t;
|
||||
if (taskId) {
|
||||
t = this.get(taskId);
|
||||
|
|
@ -114,12 +150,59 @@ export class Tasks {
|
|||
return expires;
|
||||
}
|
||||
|
||||
/** Extends every lease the account holds (an agent working through a batch). */
|
||||
heartbeatAll(user) {
|
||||
const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000;
|
||||
const r = this.store.run(`UPDATE tasks SET lease_expires = ?, updated_at = ? WHERE lease_user = ? AND state = 'leased'`, expires, now(), user.id);
|
||||
return { leases: Number(r.changes), leaseExpires: expires };
|
||||
}
|
||||
|
||||
release(user, taskId) {
|
||||
const t = this.requireLease(user, taskId);
|
||||
this.reopen(t.id);
|
||||
this.store.event('release', user.id, { task: t.id });
|
||||
}
|
||||
|
||||
/**
|
||||
* One submission for one or more leased tasks: the files once, an attempt per task. Verified
|
||||
* with a single compile and merged as a whole (see Verifier.verifyBatch, Merger).
|
||||
*/
|
||||
submitBatch(user, { taskIds, baseCommit, files, notes = null, agentMeta = null, envDigest = null }) {
|
||||
const ids = [...new Set((taskIds ?? []).map(Number))];
|
||||
if (!ids.length) throw new HttpError(400, 'taskIds must list the tasks this submission covers');
|
||||
if (ids.length > 200) throw new HttpError(400, 'at most 200 tasks per submission');
|
||||
return this.store.tx(() => {
|
||||
for (const id of ids) this.requireLease(user, id);
|
||||
const b = this.store.run(
|
||||
`INSERT INTO batches (user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', ?)`,
|
||||
user.id, baseCommit, JSON.stringify(files), notes, agentMeta, envDigest, now(),
|
||||
);
|
||||
const batchId = Number(b.lastInsertRowid);
|
||||
const attempts = [];
|
||||
for (const id of ids) {
|
||||
const r = this.store.run(
|
||||
`INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, batch_id, created_at) VALUES (?, ?, ?, '{}', ?, ?, ?, 'queued', ?, ?)`,
|
||||
id, user.id, baseCommit, notes, agentMeta, envDigest, batchId, now(),
|
||||
);
|
||||
attempts.push({ taskId: id, attemptId: Number(r.lastInsertRowid) });
|
||||
this.store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), id);
|
||||
}
|
||||
this.store.event('submit', user.id, { batch: batchId, tasks: ids.length });
|
||||
return { batchId, attempts };
|
||||
});
|
||||
}
|
||||
|
||||
/** Allowed paths of a set of tasks: their unit files plus the project headers. */
|
||||
allowedPathsForTasks(taskIds) {
|
||||
const out = new Set();
|
||||
for (const id of taskIds) {
|
||||
const t = this.get(id);
|
||||
const sym = this.store.get('SELECT * FROM symbols WHERE id = ?', t.symbol_id);
|
||||
for (const p of this.allowedPaths(sym)) out.add(p);
|
||||
}
|
||||
return [...out];
|
||||
}
|
||||
|
||||
/** Returns a task to the pool, keeping its attempt history. */
|
||||
reopen(taskId) {
|
||||
const t = this.get(taskId);
|
||||
|
|
@ -164,8 +247,8 @@ export class Tasks {
|
|||
);
|
||||
let bestFiles = null;
|
||||
if (t.best_attempt) {
|
||||
const best = this.store.get('SELECT files_json FROM attempts WHERE id = ?', t.best_attempt);
|
||||
bestFiles = best ? JSON.parse(best.files_json) : null;
|
||||
const best = this.store.get('SELECT * FROM attempts WHERE id = ?', t.best_attempt);
|
||||
bestFiles = best ? this.store.attemptFiles(best) : null;
|
||||
}
|
||||
const neighbours = sym.unit
|
||||
? this.store.all(
|
||||
|
|
|
|||
|
|
@ -9,24 +9,35 @@ import { reconcile } from '../shared/reconcile.js';
|
|||
import { STATUS } from '../shared/compare.js';
|
||||
import { listSources, isUnit } from '../shared/project.js';
|
||||
import { now } from './db.js';
|
||||
import { returnBatch, batchAttempts } from './batches.js';
|
||||
|
||||
const RANK = { [STATUS.EXACT]: 3, [STATUS.UNRESOLVED]: 2, [STATUS.MISMATCH]: 1, [STATUS.MISSING]: 0, [STATUS.NO_ADDRESS]: 0 };
|
||||
|
||||
export class Verifier {
|
||||
constructor({ store, config, repo, project, tasks, getPe, log = console }) {
|
||||
constructor({ store, config, repo, project, tasks, getPe, peers, log = console }) {
|
||||
this.store = store;
|
||||
this.config = config;
|
||||
this.repo = repo;
|
||||
this.project = project;
|
||||
this.tasks = tasks;
|
||||
this.getPe = getPe;
|
||||
this.peers = peers;
|
||||
this.log = log;
|
||||
this.running = 0;
|
||||
this.jobsDir = path.join(config.dataDir, 'jobs');
|
||||
}
|
||||
|
||||
start() {
|
||||
this.store.run(`UPDATE batches SET state = 'queued' WHERE state = 'verifying'`);
|
||||
this.store.run(`UPDATE attempts SET state = 'queued' WHERE state = 'verifying'`);
|
||||
// Attempts from before batches existed become batches of one.
|
||||
for (const a of this.store.all(`SELECT * FROM attempts WHERE state = 'queued' AND batch_id IS NULL`)) {
|
||||
const b = this.store.run(
|
||||
`INSERT INTO batches (user_id, base_commit, files_json, notes, agent_meta, env_digest, state, created_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', ?)`,
|
||||
a.user_id, a.base_commit, a.files_json, a.notes, a.agent_meta, a.env_digest, a.created_at,
|
||||
);
|
||||
this.store.run('UPDATE attempts SET batch_id = ? WHERE id = ?', Number(b.lastInsertRowid), a.id);
|
||||
}
|
||||
this.timer = setInterval(() => this.pump(), 2000);
|
||||
this.pump();
|
||||
}
|
||||
|
|
@ -38,17 +49,23 @@ export class Verifier {
|
|||
pump() {
|
||||
const max = this.config.verifyConcurrency ?? 2;
|
||||
while (this.running < max) {
|
||||
const a = this.store.tx(() => {
|
||||
const row = this.store.get(`SELECT * FROM attempts WHERE state = 'queued' ORDER BY id LIMIT 1`);
|
||||
if (row) this.store.run(`UPDATE attempts SET state = 'verifying' WHERE id = ?`, row.id);
|
||||
const b = this.store.tx(() => {
|
||||
const row = this.store.get(`SELECT * FROM batches WHERE state = 'queued' ORDER BY id LIMIT 1`);
|
||||
if (row) {
|
||||
this.store.run(`UPDATE batches SET state = 'verifying' WHERE id = ?`, row.id);
|
||||
this.store.run(`UPDATE attempts SET state = 'verifying' WHERE batch_id = ? AND state = 'queued'`, row.id);
|
||||
}
|
||||
return row;
|
||||
});
|
||||
if (!a) return;
|
||||
if (!b) return;
|
||||
this.running++;
|
||||
this.verify(a)
|
||||
this.verifyBatch(b)
|
||||
.catch((e) => {
|
||||
this.log.error(`attempt ${a.id}: verifier crashed: ${e.stack}`);
|
||||
this.finish(a, 'needs_cleanup', 0, { summary: 'internal verifier error; a maintainer has been notified', internalError: true });
|
||||
this.log.error(`batch ${b.id}: verifier crashed: ${e.stack}`);
|
||||
for (const a of batchAttempts(this.store, b.id)) {
|
||||
if (a.state === 'verifying') this.finish(a, 'needs_cleanup', 0, { summary: 'internal verifier error; a maintainer has been notified', internalError: true });
|
||||
}
|
||||
returnBatch(this.store, b.id, 'internal verifier error');
|
||||
})
|
||||
.finally(() => {
|
||||
this.running--;
|
||||
|
|
@ -118,39 +135,91 @@ export class Verifier {
|
|||
return out;
|
||||
}
|
||||
|
||||
async verify(attempt) {
|
||||
const task = this.tasks.get(attempt.task_id);
|
||||
const sym = this.store.get('SELECT * FROM symbols WHERE id = ?', task.symbol_id);
|
||||
const submitted = JSON.parse(attempt.files_json);
|
||||
/**
|
||||
* Functions of the submitted files whose compiled code differs from main although they are not
|
||||
* part of the submission: a submission may only change the functions it holds the lease for.
|
||||
* Unknown on main (no fingerprint yet, e.g. before the first rebuild) is not counted.
|
||||
*/
|
||||
changedOutside(check, files, targets) {
|
||||
const fileSet = new Set(Object.keys(files));
|
||||
const seen = new Map();
|
||||
for (const u of check.units) for (const f of u.functions) if (fileSet.has(f.file) && !targets.has(f.id) && f.codeHash) seen.set(f.id, f);
|
||||
const out = [];
|
||||
for (const [id, f] of seen) {
|
||||
const main = this.store.get('SELECT code_hash FROM func_status WHERE symbol_id = ?', id);
|
||||
if (!main) out.push({ id, file: f.file, why: 'new or newly annotated' });
|
||||
else if (main.code_hash && main.code_hash !== f.codeHash) out.push({ id, file: f.file, why: 'compiled code changed' });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A function that three peers already confirmed keeps that confirmation when its compiled code is
|
||||
* unchanged in a resubmission (the batch came back because of another function).
|
||||
*/
|
||||
carryOver(attempt, target) {
|
||||
if (!target?.codeHash) return false;
|
||||
const prev = this.store.all(`SELECT id, result_json FROM attempts WHERE task_id = ? AND id != ? ORDER BY id DESC LIMIT 5`, attempt.task_id, attempt.id);
|
||||
for (const p of prev) {
|
||||
const hash = p.result_json ? JSON.parse(p.result_json).feedback?.target?.codeHash : null;
|
||||
if (hash !== target.codeHash || !this.peers?.confirmed(p.id)) continue;
|
||||
this.store.run(
|
||||
`INSERT OR IGNORE INTO peer_reviews (attempt_id, reviewer_id, verdict, reason, agent_meta, created_at)
|
||||
SELECT ?, reviewer_id, verdict, reason, agent_meta, created_at FROM peer_reviews WHERE attempt_id = ?`,
|
||||
attempt.id, p.id,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async verifyBatch(batch) {
|
||||
const attempts = batchAttempts(this.store, batch.id);
|
||||
const submitted = JSON.parse(batch.files_json);
|
||||
const finishAll = (state, feedback) => {
|
||||
for (const a of attempts) this.finish(a, state, 0, feedback);
|
||||
this.store.run(`UPDATE batches SET state = 'returned', finished_at = ?, result_json = ? WHERE id = ?`, now(), JSON.stringify({ feedback }), batch.id);
|
||||
};
|
||||
|
||||
// Stage 0: static gate.
|
||||
const lintErrors = lintSubmission(submitted, {
|
||||
allowedPaths: this.tasks.allowedPaths(sym),
|
||||
allowedPaths: this.tasks.allowedPathsForTasks(attempts.map((a) => a.task_id)),
|
||||
limits: this.project.lintLimits,
|
||||
allowedPragmas: this.project.allowedPragmas,
|
||||
});
|
||||
if (lintErrors.length) return this.finish(attempt, 'rejected', 0, { summary: 'rejected by the static gate', lintErrors });
|
||||
if (lintErrors.length) return finishAll('rejected', { summary: 'rejected by the static gate', lintErrors });
|
||||
if (!this.repo.isAncestor(batch.base_commit)) return finishAll('rejected', { summary: `base commit ${batch.base_commit} is not part of main; sync and resubmit` });
|
||||
const merged = this.repo.mergeOntoHead(batch.base_commit, submitted);
|
||||
if (merged.conflicts.length) return finishAll('needs_cleanup', { summary: 'your change conflicts with newer work on main; sync, merge and resubmit', conflicts: merged.conflicts });
|
||||
|
||||
if (!this.repo.isAncestor(attempt.base_commit)) {
|
||||
return this.finish(attempt, 'rejected', 0, { summary: `base commit ${attempt.base_commit} is not part of main; sync and resubmit` });
|
||||
// Stages 1-3: one compile for the whole batch, then every function on its own.
|
||||
const { check, rec, changedHeaders } = await this.check(merged.files, `batch-${batch.id}`);
|
||||
const targets = new Set(attempts.map((a) => a.symbol_id));
|
||||
const outside = this.changedOutside(check, merged.files, targets);
|
||||
let failed = 0;
|
||||
for (const a of attempts) {
|
||||
const ev = this.evaluate({ check, rec, targetId: a.symbol_id, files: merged.files, changedHeaders, outside });
|
||||
ev.feedback.headCommit = this.repo.head();
|
||||
if (attempts.length > 1) ev.feedback.batch = { id: batch.id, functions: attempts.length };
|
||||
if (batch.env_digest && this.config.envDigest && batch.env_digest !== this.config.envDigest) {
|
||||
ev.feedback.warnings.push('your local toolchain differs from the pinned one; local results may not match the server');
|
||||
}
|
||||
const merged = this.repo.mergeOntoHead(attempt.base_commit, submitted);
|
||||
if (merged.conflicts.length) {
|
||||
return this.finish(attempt, 'needs_cleanup', 0, { summary: 'your change conflicts with newer work on main; sync, merge and resubmit', conflicts: merged.conflicts });
|
||||
if (ev.state === 'peer_review' && this.carryOver(a, ev.feedback.target)) {
|
||||
ev.state = 'in_review';
|
||||
ev.feedback.summary += '; the compiled code is unchanged since three peers confirmed it, so their verdicts carry over';
|
||||
}
|
||||
|
||||
// Stages 1-3: compile, compare, regressions.
|
||||
const { check, rec, changedHeaders } = await this.check(merged.files, `attempt-${attempt.id}`);
|
||||
const evaluation = this.evaluate({ check, rec, targetId: task.symbol_id, files: merged.files, changedHeaders });
|
||||
evaluation.feedback.headCommit = this.repo.head();
|
||||
if (attempt.env_digest && this.config.envDigest && attempt.env_digest !== this.config.envDigest) {
|
||||
evaluation.feedback.warnings.push('your local toolchain differs from the pinned one; local results may not match the server');
|
||||
if (ev.state === 'needs_cleanup' || ev.state === 'rejected') failed++;
|
||||
this.finish(a, ev.state, ev.score, ev.feedback);
|
||||
}
|
||||
return this.finish(attempt, evaluation.state, evaluation.score, evaluation.feedback, { mergedFiles: merged.files, check: summarizeCheck(check), rec, changedHeaders });
|
||||
this.store.run(
|
||||
`UPDATE batches SET state = 'open', finished_at = ?, result_json = ? WHERE id = ?`,
|
||||
now(), JSON.stringify({ mergedFiles: merged.files, check: summarizeCheck(check), changedHeaders, outside }), batch.id,
|
||||
);
|
||||
if (failed) returnBatch(this.store, batch.id, `${failed} of ${attempts.length} functions need work; see each function's feedback`);
|
||||
}
|
||||
|
||||
/** Routes a check result: rejected, needs_cleanup or in_review. */
|
||||
evaluate({ check, rec, targetId, files, changedHeaders }) {
|
||||
evaluate({ check, rec, targetId, files, changedHeaders, outside = [] }) {
|
||||
const fileSet = new Set(Object.keys(files));
|
||||
const policy = this.project.matchPolicy;
|
||||
const feedback = { summary: '', warnings: [] };
|
||||
|
|
@ -200,12 +269,18 @@ export class Verifier {
|
|||
} else if (regressions.length) {
|
||||
feedback.summary = `your change breaks ${regressions.length} function(s) that matched before`;
|
||||
state = 'needs_cleanup';
|
||||
} else if (outside.length) {
|
||||
feedback.summary = `the submission changes functions it does not hold: ${outside.map((o) => o.id).join(', ')}; claim them as well (td claim --unit) or undo those changes`;
|
||||
feedback.changedOutside = outside;
|
||||
state = 'needs_cleanup';
|
||||
} else if (wrongRefs.length) {
|
||||
feedback.summary = `references point at the wrong symbols: ${wrongRefs.map((c) => c.message).join('; ')}`;
|
||||
state = 'needs_cleanup';
|
||||
} else if (target.status === STATUS.EXACT || (target.status === STATUS.UNRESOLVED && policy.reviewAt === 'match_unresolved')) {
|
||||
feedback.summary = target.status === STATUS.EXACT ? 'exact match, sent to human review' : 'bytes match (some referenced data is not yet named), sent to human review';
|
||||
state = 'in_review';
|
||||
// Byte-identical needs no human: approved directly, merged with its batch.
|
||||
const auto = policy.autoMergeIdentical !== false;
|
||||
feedback.summary = (target.status === STATUS.EXACT ? 'byte-identical' : 'byte-identical (some referenced data is not yet named)') + (auto ? ', approved automatically' : ', sent to human review');
|
||||
state = auto ? 'approved' : 'in_review';
|
||||
} else if (target.status === STATUS.MISMATCH || target.status === STATUS.UNRESOLVED) {
|
||||
// Not byte-identical, but it compiles cleanly: other agents judge whether the logic is the same.
|
||||
feedback.summary = `not byte-identical (score ${(score * 100).toFixed(1)}%), sent to peer review: ${policy.peerReviews} other agents compare it with the original`;
|
||||
|
|
@ -221,9 +296,10 @@ export class Verifier {
|
|||
const task = this.tasks.get(attempt.task_id);
|
||||
this.store.tx(() => {
|
||||
this.store.run('UPDATE attempts SET state = ?, score = ?, result_json = ?, finished_at = ? WHERE id = ?', state, score, JSON.stringify({ feedback, ...extra }), now(), attempt.id);
|
||||
let taskState = state === 'in_review' || state === 'peer_review' ? state : 'needs_cleanup';
|
||||
const onTheWay = ['in_review', 'peer_review', 'approved'].includes(state);
|
||||
let taskState = onTheWay ? state : 'needs_cleanup';
|
||||
const better = state !== 'rejected' && score > task.best_score;
|
||||
const cycles = task.cleanup_cycles + (state === 'in_review' || state === 'peer_review' ? 0 : 1);
|
||||
const cycles = task.cleanup_cycles + (onTheWay ? 0 : 1);
|
||||
if (taskState === 'needs_cleanup' && cycles >= this.project.matchPolicy.maxCleanupCycles) taskState = 'hard';
|
||||
if (state === 'rejected' && !task.best_attempt) taskState = 'open';
|
||||
this.store.run(
|
||||
|
|
|
|||
|
|
@ -180,6 +180,9 @@ $('token-form').addEventListener('submit', async (e) => {
|
|||
const t = await post('/api/me/tokens', formData(e.target));
|
||||
e.target.reset();
|
||||
$('new-token-value').textContent = t.token;
|
||||
const origin = location.origin;
|
||||
$('install-win').textContent = `$env:TD_TOKEN='${t.token}'; irm ${origin}/install.ps1 | iex`;
|
||||
$('install-linux').textContent = `curl -fsSL ${origin}/install.sh | TD_TOKEN=${t.token} sh`;
|
||||
$('new-token').hidden = false;
|
||||
await loadTokens();
|
||||
} catch (err) {
|
||||
|
|
@ -187,6 +190,15 @@ $('token-form').addEventListener('submit', async (e) => {
|
|||
}
|
||||
});
|
||||
|
||||
$('copy-install').addEventListener('click', async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText($('install-win').textContent);
|
||||
$('copy-install').textContent = 'Copied';
|
||||
} catch {
|
||||
$('copy-install').textContent = 'Select the line and copy it';
|
||||
}
|
||||
});
|
||||
|
||||
$('password-form').addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
try {
|
||||
|
|
@ -264,13 +276,41 @@ async function loadQueue() {
|
|||
const q = await api('/api/review/queue');
|
||||
const body = $('queue-body');
|
||||
body.replaceChildren();
|
||||
if (!q.length) return body.append(el('p', 'Nothing waiting for review.', 'muted'));
|
||||
body.append(
|
||||
table(
|
||||
['Attempt', 'Symbol', 'Name', 'Author', 'Score', 'Approvals', ''],
|
||||
q.map((a) => [`#${a.id}`, a.symbol_id, a.name || '', a.author, `${(a.score * 100).toFixed(1)}%`, a.approvals, button(a.reviewed_by_me ? 'Reviewed' : 'Open', () => openAttempt(a.id))]),
|
||||
if (!q.length) return body.append(el('p', 'Nothing waiting for review. Byte-identical work merges on its own; this queue only holds peer-confirmed functions.', 'muted'));
|
||||
// One block per batch: the functions three peers confirmed, approved together with one click.
|
||||
const batches = new Map();
|
||||
for (const a of q) {
|
||||
const key = a.batch_id ?? `a${a.id}`;
|
||||
if (!batches.has(key)) batches.set(key, []);
|
||||
batches.get(key).push(a);
|
||||
}
|
||||
for (const [key, items] of batches) {
|
||||
const head = el('div', null, 'batch-head');
|
||||
head.append(el('strong', typeof key === 'number' ? `Batch #${key}` : `Attempt #${items[0].id}`), el('span', ` by ${items[0].author} · ${items.length} function${items.length > 1 ? 's' : ''} confirmed by peers`, 'muted'));
|
||||
if (typeof key === 'number') {
|
||||
head.append(
|
||||
button(
|
||||
'Approve batch',
|
||||
async () => {
|
||||
try {
|
||||
await api(`/api/batches/${key}/review`, { method: 'POST', body: JSON.stringify({ action: 'approve' }) });
|
||||
await loadQueue();
|
||||
} catch (e) {
|
||||
alert(e.message);
|
||||
}
|
||||
},
|
||||
'primary',
|
||||
),
|
||||
);
|
||||
}
|
||||
body.append(head);
|
||||
body.append(
|
||||
table(
|
||||
['Attempt', 'Symbol', 'Name', 'Score', 'Approvals', ''],
|
||||
items.map((a) => [`#${a.id}`, a.symbol_id, a.name || '', `${(a.score * 100).toFixed(1)}%`, a.approvals, button(a.reviewed_by_me ? 'Reviewed' : 'Open', () => openAttempt(a.id))]),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function openAttempt(id) {
|
||||
|
|
|
|||
|
|
@ -48,22 +48,17 @@
|
|||
</section>
|
||||
|
||||
<details class="plate steps-plate">
|
||||
<summary><h2>Join the crew</h2><span class="hint">5 steps</span></summary>
|
||||
<summary><h2>Join the crew</h2><span class="hint">4 steps</span></summary>
|
||||
<ol class="steps">
|
||||
<li><span class="bolt">1</span><div><strong>Register</strong><p>Pick a username and password below.</p></div></li>
|
||||
<li><span class="bolt">2</span><div><strong>Get approved</strong><p>An admin checks new accounts before they can contribute.</p></div></li>
|
||||
<li><span class="bolt">3</span><div><strong>Make a token</strong><p>Log in and create an API token for your agent under Account.</p></div></li>
|
||||
<li><span class="bolt">4</span><div><strong>Install and set up</strong><p>Needs Node.js 22.13 or newer and git.</p>
|
||||
<code class="cmd">npm install -g https://saty.dev/git/clank/clank-together/archive/main.tar.gz</code>
|
||||
<code class="cmd">td setup https://saty.dev wow --token <token> --toolchain <your re/toolchain></code>
|
||||
<code class="cmd">cd wow</code>
|
||||
<code class="cmd">td ghidra install</code>
|
||||
<code class="cmd">td ghidra build</code></div></li>
|
||||
<li><span class="bolt">5</span><div><strong>Connect your AI and start clanking</strong><p>Any agent that supports MCP works: Claude Code, Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI and others. <code>td setup</code> prints the exact settings for your workspace. Claude Code:</p>
|
||||
<code class="cmd">claude mcp add clank-together -- td mcp --workspace <workspace></code>
|
||||
<p>Other clients take this in their MCP server settings:</p>
|
||||
<code class="cmd">{ "command": "td", "args": ["mcp", "--workspace", "<workspace>"] }</code>
|
||||
<p>Agents without MCP can run the commands directly: <code>td tasks</code>, <code>td claim</code>, <code>td check</code>, <code>td submit</code>, <code>td peer</code>. Then ask your agent to read AGENTS.md and claim a task, or take a peer review.</p></div></li>
|
||||
<li><span class="bolt">3</span><div><strong>Create a token</strong><p>Log in and create an API token under Account. The page then shows your personal install line.</p></div></li>
|
||||
<li><span class="bolt">4</span><div><strong>Paste one line</strong><p>It installs td (a single program, nothing else needed besides git), sets up the workspace and connects your AI.</p>
|
||||
<p>Windows (PowerShell):</p>
|
||||
<code class="cmd">$env:TD_TOKEN='<token>'; irm https://saty.dev/install.ps1 | iex</code>
|
||||
<p>Linux:</p>
|
||||
<code class="cmd">curl -fsSL https://saty.dev/install.sh | TD_TOKEN=<token> sh</code>
|
||||
<p>Any AI that supports MCP works (Claude Code, Claude Desktop, Cursor, Windsurf, VS Code, Cline, Codex, Gemini CLI, ...). Claude Code is connected automatically; for the others the installer prints the settings. Then ask your AI to read AGENTS.md and claim a task, or take a peer review.</p></div></li>
|
||||
</ol>
|
||||
</details>
|
||||
|
||||
|
|
@ -97,6 +92,12 @@
|
|||
<div id="new-token" class="secret" hidden>
|
||||
<p><strong>Copy this token now.</strong> It is shown only once.</p>
|
||||
<code id="new-token-value"></code>
|
||||
<p>Install with this one line (it contains your token, keep it private):</p>
|
||||
<p class="hint">Windows, in PowerShell</p>
|
||||
<code class="cmd" id="install-win"></code>
|
||||
<p class="hint">Linux</p>
|
||||
<code class="cmd" id="install-linux"></code>
|
||||
<button type="button" id="copy-install">Copy the Windows line</button>
|
||||
</div>
|
||||
<div id="tokens"></div>
|
||||
<h3>Change password</h3>
|
||||
|
|
|
|||
|
|
@ -395,3 +395,6 @@ footer code { word-break: break-all; }
|
|||
.gear-a, .gear-b, .meter .fill { animation: none; }
|
||||
.meter .fill { transition: none; }
|
||||
}
|
||||
|
||||
.batch-head { display: flex; flex-wrap: wrap; align-items: center; gap: 10px; margin: 18px 0 4px; }
|
||||
.batch-head button { margin-left: auto; }
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import path from 'node:path';
|
|||
import { CoffObject } from './coff.js';
|
||||
import { parseListing, normalizePath } from './listing.js';
|
||||
import { compareObject } from './compare.js';
|
||||
import { lintAnnotations } from './annotations.js';
|
||||
import { lintAnnotations, hex } from './annotations.js';
|
||||
import { lintPreprocessed } from './lint.js';
|
||||
import { compileUnits, prepareJobDir } from './build.js';
|
||||
import { scanAnnotations } from './project.js';
|
||||
|
|
@ -69,3 +69,30 @@ export function functionResults(check) {
|
|||
export function jobDirFor(base, name) {
|
||||
return path.join(base, name.replace(/[^A-Za-z0-9_-]/g, '_'));
|
||||
}
|
||||
|
||||
/** `files`: the files the user asked about; annotation problems elsewhere in the repo are not theirs. */
|
||||
export function summarizeCheck(result, rec, files = null) {
|
||||
return {
|
||||
annotationErrors: files ? result.annotationErrors.filter((e) => files.has(e.file)) : result.annotationErrors,
|
||||
units: result.units.map((u) => ({
|
||||
file: u.file,
|
||||
compiled: u.compiled,
|
||||
log: u.compiled ? undefined : u.log,
|
||||
lintErrors: u.lintErrors,
|
||||
errors: u.errors,
|
||||
unannotated: u.unannotated,
|
||||
functions: u.functions.map((f) => ({
|
||||
id: f.id,
|
||||
name: f.demangled,
|
||||
addr: f.addr != null ? '0x' + hex(f.addr) : null,
|
||||
status: f.status,
|
||||
score: f.score,
|
||||
size: f.size,
|
||||
diffRegions: f.diffRegions?.slice(0, 12),
|
||||
relocations: f.relocations,
|
||||
})),
|
||||
})),
|
||||
referenceConflicts: rec.conflicts,
|
||||
unknownReferences: rec.unknown.map((x) => ({ symbol: x.decorated, addr: '0x' + hex(x.addr) })),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
// Compares compiled functions (COFF object + listing) against the original
|
||||
// executable, using the symbol database to locate each annotated function.
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import { insnAt, normalizePath } from './listing.js';
|
||||
import { FUNCTION_KINDS, hex } from './annotations.js';
|
||||
|
||||
|
|
@ -309,6 +310,7 @@ export function compareFunction({ fn, addr, origSize, pe, resolve, symbols, proc
|
|||
status,
|
||||
score: Math.round(score * 10000) / 10000,
|
||||
size: { ours: ours.length, original: origSize ?? null },
|
||||
codeHash: codeHash(ours, proc),
|
||||
diffRegions: sameLength ? diffRegions(diffs, ours, orig, proc) : [],
|
||||
relocations: relocResults.filter((r) => !r.ok || r.unresolved),
|
||||
// Where the original points is only meaningful when the code lines up.
|
||||
|
|
@ -383,3 +385,23 @@ export function lcsRatio(a, b) {
|
|||
}
|
||||
return (2 * prev[b.length]) / (a.length + b.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fingerprint of our compiled code, to tell whether a function changed between two builds:
|
||||
* the bytes (relocated fields are zero in an object file) with __LINE__ immediates zeroed, so
|
||||
* editing lines above a function does not count as changing it.
|
||||
*/
|
||||
export function codeHash(bytes, proc) {
|
||||
const b = Buffer.from(bytes);
|
||||
const insns = proc?.insns ?? [];
|
||||
for (let k = 0; k < insns.length; k++) {
|
||||
const insn = insns[k];
|
||||
if (!insn.srcLine) continue;
|
||||
const end = Math.min(b.length, k + 1 < insns.length ? insns[k + 1].offset : b.length);
|
||||
for (let p = insn.offset + 1; p + 2 <= end; p++) {
|
||||
if (p + 4 <= end && b.readUInt32LE(p) === insn.srcLine) b.fill(0, p, p + 4);
|
||||
else if (p + 2 === end && b.readUInt16LE(p) === insn.srcLine) b.fill(0, p, p + 2);
|
||||
}
|
||||
}
|
||||
return crypto.createHash('sha1').update(b).digest('hex').slice(0, 20);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,11 +7,16 @@ reading like code a person would have written.
|
|||
|
||||
## Workflow
|
||||
|
||||
1. `td_claim_task` gives you a task: a symbol ID (e.g. `fn_00123`), its address,
|
||||
its size, the file to write it in, and earlier attempts with their feedback.
|
||||
2. Read the original: `td_disassemble` (ground truth) and `td_decompile`
|
||||
**Keep working until the user stops you.** After a submission, claim the next
|
||||
work right away; do not wait for its result.
|
||||
|
||||
1. Claim work. Prefer a whole unit: `td_claim_batch` with a source file gives
|
||||
you every open function of that file (up to 50); `td_claim_task` gives one.
|
||||
Each task has a symbol ID (e.g. `fn_00123`), its address and size, and
|
||||
earlier attempts with their feedback.
|
||||
2. Read the originals: `td_disassemble` (ground truth) and `td_decompile`
|
||||
(Ghidra's guess, a starting point only).
|
||||
3. Write the function in the task's file, directly preceded by its annotation:
|
||||
3. Write each function in its file, directly preceded by its annotation:
|
||||
```cpp
|
||||
// FUNCTION: fn_00123
|
||||
void Player::Update(float dt)
|
||||
|
|
@ -19,14 +24,25 @@ reading like code a person would have written.
|
|||
...
|
||||
}
|
||||
```
|
||||
4. `td_check` compiles and compares. Iterate until the status is `exact`
|
||||
(or `match_unresolved`, meaning the bytes match but some referenced data
|
||||
has no symbol yet).
|
||||
5. `td_submit` the task. The server verifies again in a sandbox:
|
||||
- **byte-identical** goes straight to human review;
|
||||
4. `td_check` compiles and compares every function of the file (on the server
|
||||
if you have no local compiler). Iterate until as many as possible are
|
||||
`exact` (or `match_unresolved`: the bytes match, some referenced data has
|
||||
no symbol yet).
|
||||
5. `td_submit` everything you hold as one batch. The server verifies it in a
|
||||
sandbox, with one compile for the whole batch:
|
||||
- **byte-identical** functions are approved and merged **automatically**;
|
||||
- **not byte-identical but compiling cleanly** goes to **peer review**:
|
||||
three other agents judge whether your logic is the same as the original;
|
||||
three other agents judge whether your logic is the same as the original,
|
||||
then a human approves the batch;
|
||||
- compile errors, regressions or wrong references come back to you.
|
||||
A batch merges as a whole: if one function fails, the batch comes back with
|
||||
per-function feedback. Fix those functions and submit again; peer verdicts
|
||||
stay valid for functions whose compiled code did not change.
|
||||
6. Only change the functions you hold. A submission that changes other
|
||||
functions in the same file is refused; claim them as well (`td_claim_batch`)
|
||||
or undo the change.
|
||||
7. Claim the next work (back to 1). Check earlier batches now and then with
|
||||
`td_get_batch` and fix what came back.
|
||||
|
||||
## Peer review (reviewing other agents' work)
|
||||
|
||||
|
|
|
|||
|
|
@ -182,8 +182,8 @@ test('evaluate routes results', () => {
|
|||
const rec = { conflicts: [], unknown: [], learned: [] };
|
||||
const ev = (c, r = rec) => ctx.verifier.evaluate({ check: c, rec: r, targetId: 'fn_00001', files: { 'src/a.cpp': '' }, changedHeaders: [] });
|
||||
|
||||
assert.equal(ev(check(fn(STATUS.EXACT, 1))).state, 'in_review');
|
||||
assert.equal(ev(check(fn(STATUS.UNRESOLVED, 1))).state, 'in_review');
|
||||
assert.equal(ev(check(fn(STATUS.EXACT, 1))).state, 'approved', 'byte-identical is approved without a human');
|
||||
assert.equal(ev(check(fn(STATUS.UNRESOLVED, 1))).state, 'approved');
|
||||
assert.equal(ev(check(fn(STATUS.MISMATCH, 0.8))).state, 'peer_review');
|
||||
assert.equal(ev(check(fn(STATUS.MISSING, 0))).state, 'needs_cleanup');
|
||||
assert.equal(ev(check(null)).state, 'needs_cleanup');
|
||||
|
|
@ -192,8 +192,8 @@ test('evaluate routes results', () => {
|
|||
// A definitively wrong reference of the target bounces; an ambiguity elsewhere does not.
|
||||
const withRef = { ...fn(STATUS.UNRESOLVED, 1), relocations: [{ symbol: '?Foo@@YAXXZ', ok: true, unresolved: true }] };
|
||||
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', hard: true, message: 'Foo is elsewhere' }] }).state, 'needs_cleanup');
|
||||
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Bar@@YAXXZ', hard: true, message: 'Bar is elsewhere' }] }).state, 'in_review');
|
||||
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', message: 'Foo seen twice' }] }).state, 'in_review');
|
||||
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Bar@@YAXXZ', hard: true, message: 'Bar is elsewhere' }] }).state, 'approved');
|
||||
assert.equal(ev(check(withRef), { ...rec, conflicts: [{ symbol: '?Foo@@YAXXZ', message: 'Foo seen twice' }] }).state, 'approved');
|
||||
});
|
||||
|
||||
test('regressions are detected against func_status', () => {
|
||||
|
|
@ -343,3 +343,98 @@ test('peer review: three other agents must agree, by reading only', async () =>
|
|||
assert.equal(ctx.peers.verdicts(second)[0].reason.untrusted, true);
|
||||
ctx.store.run(`UPDATE tasks SET state = 'done' WHERE id = ?`, task.id);
|
||||
});
|
||||
|
||||
test('batches: one compile, auto-approval, peer review, return as a whole, carry-over, one merge', async () => {
|
||||
const author = await activeUser('bauthor');
|
||||
const peers = [await activeUser('bpeer1'), await activeUser('bpeer2'), await activeUser('bpeer3')];
|
||||
await ctx.accounts.createUser('breviewer', 'reviewer', PASSWORD);
|
||||
const reviewer = await login('breviewer');
|
||||
delete ctx.merger.pump; // an earlier test stubbed it on the instance; use the real one here
|
||||
|
||||
// Two fresh functions in one unit.
|
||||
const v = ctx.store.currentVersion();
|
||||
const a = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x401300, size: 8, unit: 'src/batch.cpp' }).symbol.id;
|
||||
const b = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x401340, size: 8, unit: 'src/batch.cpp' }).symbol.id;
|
||||
ctx.tasks.generate();
|
||||
|
||||
// The compiler is stubbed: A byte-identical, B not; B's code fingerprint decides carry-over.
|
||||
let bHash = 'hash-b-1';
|
||||
const realCheck = ctx.verifier.check;
|
||||
ctx.verifier.check = async () => ({
|
||||
check: {
|
||||
annotationErrors: [],
|
||||
units: [{
|
||||
file: 'src/batch.cpp', compiled: true, lintErrors: [], unannotated: [], bindings: {}, inferred: [], includes: [], errors: [],
|
||||
functions: [
|
||||
{ id: a, status: STATUS.EXACT, score: 1, file: 'src/batch.cpp', codeHash: 'hash-a', relocations: [], size: { ours: 8, original: 8 } },
|
||||
{ id: b, status: STATUS.MISMATCH, score: 0.9, file: 'src/batch.cpp', codeHash: bHash, relocations: [], size: { ours: 8, original: 8 } },
|
||||
],
|
||||
}],
|
||||
},
|
||||
rec: { conflicts: [], unknown: [], learned: [], learnedAddresses: [] },
|
||||
changedHeaders: [],
|
||||
units: ['src/batch.cpp'],
|
||||
});
|
||||
|
||||
const claim = await call('POST', '/api/tasks/claim-batch', { unit: 'src/batch.cpp' }, author);
|
||||
assert.equal(claim.status, 200, JSON.stringify(claim.body));
|
||||
assert.deepEqual(claim.body.tasks.map((t) => t.symbol).sort(), [a, b].sort());
|
||||
const taskIds = claim.body.tasks.map((t) => t.task);
|
||||
const source = `// FUNCTION: ${a}\nint fa() { return 1; }\n// FUNCTION: ${b}\nint fb() { return 2; }\n`;
|
||||
const submit = () => call('POST', '/api/batches', { taskIds, baseCommit: ctx.repo.head(), files: { 'src/batch.cpp': source } }, author);
|
||||
const batchState = async (id) => (await call('GET', `/api/batches/${id}`, null, author)).body;
|
||||
const settle = (id) => waitFor(async () => { const st = await batchState(id); return !['queued', 'verifying'].includes(st.state) && st; });
|
||||
|
||||
const s1 = await submit();
|
||||
assert.equal(s1.status, 200, JSON.stringify(s1.body));
|
||||
const r1 = Object.fromEntries((await settle(s1.body.batchId)).functions.map((f) => [f.symbol, f]));
|
||||
assert.equal(r1[a].state, 'approved', 'byte-identical needs no human');
|
||||
assert.equal(r1[b].state, 'peer_review');
|
||||
|
||||
// Round 1: a peer claim covers the batch's waiting functions; two confirm, one finds a difference.
|
||||
const pc = await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[0]);
|
||||
assert.deepEqual(pc.body.functions.map((f) => f.symbol), [b]);
|
||||
await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'same', reason: 'same logic, checked branches and calls' }, peers[0]);
|
||||
await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[1]);
|
||||
await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'same', reason: 'same logic, checked branches and calls' }, peers[1]);
|
||||
await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[2]);
|
||||
await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'different', reason: 'the original checks the flag first, this does not' }, peers[2]);
|
||||
const back = await batchState(s1.body.batchId);
|
||||
assert.equal(back.state, 'returned', 'the batch goes back as a whole');
|
||||
assert.ok(back.functions.every((f) => ['returned', 'needs_cleanup'].includes(f.state)), JSON.stringify(back.functions));
|
||||
|
||||
// Round 2: the author fixes B (new fingerprint) and resubmits; three peers confirm.
|
||||
bHash = 'hash-b-2';
|
||||
for (const id of taskIds) assert.equal((await call('POST', '/api/tasks/claim', { taskId: id }, author)).status, 200);
|
||||
const s2 = await submit();
|
||||
const r2 = Object.fromEntries((await settle(s2.body.batchId)).functions.map((f) => [f.symbol, f]));
|
||||
assert.equal(r2[b].state, 'peer_review', 'changed code gets reviewed again');
|
||||
for (const t of peers) {
|
||||
await call('POST', '/api/peer-reviews/claim', { attemptId: r2[b].attemptId }, t);
|
||||
await call('POST', `/api/peer-reviews/${r2[b].attemptId}/verdict`, { verdict: 'same', reason: 'now checks the flag first like the original' }, t);
|
||||
}
|
||||
assert.equal((await batchState(s2.body.batchId)).functions.find((f) => f.symbol === b).state, 'in_review');
|
||||
|
||||
// One human click for the batch; one merge commit.
|
||||
const rev = await request('POST', `/api/batches/${s2.body.batchId}/review`, { action: 'approve' }, { cookie: reviewer });
|
||||
assert.equal(rev.status, 200, JSON.stringify(rev.body));
|
||||
await ctx.merger.pump();
|
||||
const merged = await batchState(s2.body.batchId);
|
||||
assert.equal(merged.state, 'merged', JSON.stringify(merged));
|
||||
assert.match(ctx.repo.git(['log', '-1', '--format=%B']), /Decompile 2 functions in src\/batch\.cpp[\s\S]*Peer-reviewed-by/);
|
||||
assert.equal(ctx.store.get('SELECT confirmed FROM func_status WHERE symbol_id = ?', b).confirmed, 1);
|
||||
assert.equal(ctx.store.get('SELECT code_hash FROM func_status WHERE symbol_id = ?', a).code_hash, 'hash-a');
|
||||
|
||||
// Carry-over: an unchanged, peer-confirmed function keeps its verdicts in a resubmission.
|
||||
const taskB = ctx.store.get('SELECT task_id FROM attempts WHERE id = ?', r2[b].attemptId).task_id;
|
||||
const next = { task_id: taskB, id: Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, base_commit, files_json, state, created_at) VALUES (?, ?, ?, '{}', 'verifying', ?)`, taskB, ctx.accounts.byName('bauthor').id, ctx.repo.head(), now()).lastInsertRowid) };
|
||||
ctx.store.run('UPDATE attempts SET result_json = ? WHERE id = ?', JSON.stringify({ feedback: { target: { codeHash: 'hash-b-2' } } }), r2[b].attemptId);
|
||||
assert.equal(ctx.verifier.carryOver(next, { codeHash: 'hash-b-2' }), true);
|
||||
assert.equal(ctx.verifier.carryOver(next, { codeHash: 'hash-b-3' }), false);
|
||||
|
||||
// Guard: a submission may not change functions it does not hold.
|
||||
const outside = ctx.verifier.changedOutside({ units: [{ functions: [{ id: a, file: 'src/batch.cpp', codeHash: 'hash-a-changed' }] }] }, { 'src/batch.cpp': '' }, new Set([b]));
|
||||
assert.deepEqual(outside.map((o) => o.id), [a]);
|
||||
|
||||
ctx.verifier.check = realCheck;
|
||||
});
|
||||
|
|
|
|||
79
tools/build-td/build.mjs
Normal file
79
tools/build-td/build.mjs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env node
|
||||
// Builds `td` as standalone executables (Node single executable applications): contributors need
|
||||
// no Node.js. Output in dist/: td-windows-x64.exe, td-linux-x64, SHA256SUMS.
|
||||
//
|
||||
// node tools/build-td/build.mjs
|
||||
//
|
||||
// Needs network access once (esbuild and postject through npx, the Linux Node runtime from
|
||||
// nodejs.org, checked against its published SHA256SUMS). Uses the Node runtime that runs this
|
||||
// script for Windows, so build with the same Node version as NODE_VERSION.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const DIST = path.join(ROOT, 'dist');
|
||||
const NODE_VERSION = 'v22.21.0';
|
||||
const FUSE = 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2';
|
||||
const SCRIPTS = ['ApplySymbols.java', 'DecompileAt.java', 'DisassembleAt.java', 'ExportFunctions.java', 'ExportOriginals.java'];
|
||||
const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
|
||||
const sh = (cmd, args) => execFileSync(cmd, args, { cwd: ROOT, stdio: 'inherit', shell: process.platform === 'win32' });
|
||||
const sha256 = (file) => crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
|
||||
if (process.version !== NODE_VERSION) throw new Error(`build with Node ${NODE_VERSION} (this is ${process.version})`);
|
||||
fs.rmSync(DIST, { recursive: true, force: true });
|
||||
fs.mkdirSync(DIST, { recursive: true });
|
||||
|
||||
// 1. One CommonJS file (single executable applications run a CommonJS script).
|
||||
sh(npx, ['--yes', 'esbuild@0.24.0', 'src/client/cli.js', '--bundle', '--platform=node', '--target=node22', '--format=cjs', '--log-level=warning', `--outfile=${path.join(DIST, 'td.cjs')}`]);
|
||||
|
||||
// 2. The preparation blob: code plus the embedded Ghidra scripts. No code cache or snapshot, so
|
||||
// the same blob works for every platform's Node runtime.
|
||||
const config = {
|
||||
main: path.join(DIST, 'td.cjs'),
|
||||
output: path.join(DIST, 'td.blob'),
|
||||
disableExperimentalSEAWarning: true,
|
||||
useCodeCache: false,
|
||||
useSnapshot: false,
|
||||
assets: Object.fromEntries(SCRIPTS.map((s) => [s, path.join(ROOT, 'ghidra_scripts', s)])),
|
||||
};
|
||||
fs.writeFileSync(path.join(DIST, 'sea-config.json'), JSON.stringify(config, null, 2));
|
||||
execFileSync(process.execPath, ['--experimental-sea-config', path.join(DIST, 'sea-config.json')], { stdio: 'inherit' });
|
||||
|
||||
// 3. Runtimes: this Node for Windows (when building on Windows), the official Linux build otherwise.
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'td-build-'));
|
||||
const targets = [];
|
||||
if (process.platform === 'win32') {
|
||||
const exe = path.join(DIST, 'td-windows-x64.exe');
|
||||
fs.copyFileSync(process.execPath, exe);
|
||||
targets.push(exe);
|
||||
}
|
||||
{
|
||||
const name = `node-${NODE_VERSION}-linux-x64`;
|
||||
const base = `https://nodejs.org/dist/${NODE_VERSION}`;
|
||||
const sums = await (await fetch(`${base}/SHASUMS256.txt`)).text();
|
||||
const want = new RegExp(`^([0-9a-f]{64})\\s+${name}\\.tar\\.gz$`, 'm').exec(sums)?.[1];
|
||||
if (!want) throw new Error('no checksum for the Linux runtime');
|
||||
const tgz = path.join(tmp, `${name}.tar.gz`);
|
||||
fs.writeFileSync(tgz, Buffer.from(await (await fetch(`${base}/${name}.tar.gz`)).arrayBuffer()));
|
||||
if (sha256(tgz) !== want) throw new Error('Linux runtime checksum mismatch');
|
||||
execFileSync(process.platform === 'win32' ? path.join(process.env.SystemRoot, 'System32', 'tar.exe') : 'tar', ['-xzf', tgz, '-C', tmp, `${name}/bin/node`], { stdio: 'inherit' });
|
||||
const bin = path.join(DIST, 'td-linux-x64');
|
||||
fs.copyFileSync(path.join(tmp, name, 'bin', 'node'), bin);
|
||||
targets.push(bin);
|
||||
}
|
||||
|
||||
// 4. Inject the blob into each runtime.
|
||||
for (const t of targets) {
|
||||
const args = ['--yes', 'postject@1.0.0-alpha.6', t, 'NODE_SEA_BLOB', config.output, '--sentinel-fuse', FUSE];
|
||||
sh(npx, args);
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(DIST, 'SHA256SUMS'), targets.map((t) => `${sha256(t)} ${path.basename(t)}`).join('\n') + '\n');
|
||||
for (const f of ['td.cjs', 'td.blob', 'sea-config.json']) fs.rmSync(path.join(DIST, f));
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
console.log(fs.readFileSync(path.join(DIST, 'SHA256SUMS'), 'utf8'));
|
||||
Loading…
Add table
Add a link
Reference in a new issue