diff --git a/src/client/mcp.js b/src/client/mcp.js index 9de70c6..7a5fe11 100644 --- a/src/client/mcp.js +++ b/src/client/mcp.js @@ -94,6 +94,30 @@ const TOOLS = [ inputSchema: { type: 'object', properties: { unit: str('repo-relative source file, e.g. Engine/Source/FrameXML/LoadXML.cpp'), start: str('range start, hex'), end: str('range end, hex (exclusive)'), limit: int('max functions, default 50') } }, run: async (ws, a) => ops.claimBatch(ws, a), }, + { + name: 'td_mark_no_decompile', + description: + 'Claim that functions you hold need no decompilation, with a category and a reason: compiler_generated (adjustor/jump thunks, deleting destructors, vtable/RTTI helpers), runtime_library (CRT, compiler runtime, linked libraries), generated_from_global (static initializers, atexit destructors), not_code, duplicate. Other agents review the claim like code; accepted functions are skipped.', + inputSchema: { + type: 'object', + properties: { + items: { + type: 'array', + items: { + type: 'object', + properties: { + taskId: { type: 'integer' }, + category: { type: 'string', enum: ['compiler_generated', 'runtime_library', 'generated_from_global', 'not_code', 'duplicate'] }, + reason: { type: 'string', description: 'what in the original shows it (10-300 characters)' }, + }, + required: ['taskId', 'category', 'reason'], + }, + }, + }, + required: ['items'], + }, + run: async (ws, a) => ops.api(ws).post('/api/tasks/no-decompile', { items: a.items }), + }, { name: 'td_place', description: diff --git a/src/server/app.js b/src/server/app.js index 9ac71d9..0712d82 100644 --- a/src/server/app.js +++ b/src/server/app.js @@ -242,7 +242,8 @@ export function createApp(ctx) { })); router.on('GET', '/api/progress', () => cached('progress', 20_000, () => { - const total = store.get(`SELECT COUNT(*) AS n, COALESCE(SUM(a.size), 0) AS bytes FROM symbols s JOIN symbol_addresses a ON a.symbol_id = s.id JOIN versions v ON v.id = a.version_id AND v.is_current = 1 WHERE s.kind = 'function' AND s.library = 0`); + // Functions accepted as needing no decompilation (thunks, runtime library, ...) do not count. + const total = store.get(`SELECT COUNT(*) AS n, COALESCE(SUM(a.size), 0) AS bytes FROM symbols s JOIN symbol_addresses a ON a.symbol_id = s.id JOIN versions v ON v.id = a.version_id AND v.is_current = 1 WHERE s.kind = 'function' AND s.library = 0 AND s.skip_reason IS NULL`); const byStatus = store.all(`SELECT fs.status, COUNT(*) AS n, COALESCE(SUM(a.size), 0) AS bytes FROM func_status fs JOIN symbol_addresses a ON a.symbol_id = fs.symbol_id JOIN versions v ON v.id = a.version_id AND v.is_current = 1 GROUP BY fs.status`); const byTaskState = store.all('SELECT state, COUNT(*) AS n FROM tasks GROUP BY state'); const contributors = store.all(`SELECT u.name, COUNT(*) AS merged FROM attempts a JOIN users u ON u.id = a.user_id WHERE a.state = 'merged' GROUP BY u.id ORDER BY merged DESC LIMIT 20`); @@ -363,6 +364,13 @@ export function createApp(ctx) { }); }); + // "No decompile": held functions claimed to need no source; peer reviewed like code. + router.on('POST', '/api/tasks/no-decompile', async ({ req }) => { + const user = auth(req); + const body = await readJson(req, 128 * 1024); + return guided(tasks.markNoDecompile(user, body.items)); + }); + // Organize before porting: put held functions that are in no source file yet into files. router.on('POST', '/api/tasks/place', async ({ req }) => { const user = auth(req); @@ -517,6 +525,8 @@ export function createApp(ctx) { files: store.attemptFiles(a), mergedFiles: isReviewer ? batchResult.mergedFiles ?? result?.mergedFiles ?? null : undefined, reviews: store.all('SELECT r.action, r.comment, r.created_at, u.name AS reviewer FROM reviews r JOIN users u ON u.id = r.reviewer_id WHERE r.attempt_id = ?', a.id), + kind: a.kind, + claim: a.kind === 'skip' ? { category: a.skip_category, reason: a.notes } : undefined, peerReviews: peers.verdicts(a.id), peerRequired: peers.required, targetSource: targetSource(result?.feedback?.target?.id, batchResult.mergedFiles ?? store.attemptFiles(a)), diff --git a/src/server/db.js b/src/server/db.js index fabc01a..7abe129 100644 --- a/src/server/db.js +++ b/src/server/db.js @@ -256,7 +256,11 @@ export class Store { this.addColumn('versions', 'label', 'TEXT'); this.addColumn('check_jobs', 'files_hash', 'TEXT'); // a retry of the same check joins the queued one this.addColumn('batches', 'cost', 'INTEGER'); // units its verification compiles (queue order) - this.addColumn('users', 'peer_review_blocked', 'INTEGER NOT NULL DEFAULT 0'); // no peer reviews from this account // human-readable name, e.g. "World of Warcraft 5.0.3 (build 15890, x64)" + this.addColumn('users', 'peer_review_blocked', 'INTEGER NOT NULL DEFAULT 0'); // no peer reviews from this account + // "No decompile" claims: an attempt that says a function needs no source (peer reviewed like code). + this.addColumn('attempts', 'kind', "TEXT NOT NULL DEFAULT 'code'"); // code | skip + this.addColumn('attempts', 'skip_category', 'TEXT'); + this.addColumn('symbols', 'skip_reason', 'TEXT'); // accepted: not ported, not counted in progress // human-readable name, e.g. "World of Warcraft 5.0.3 (build 15890, x64)" this.addColumn('attempts', 'batch_id', 'INTEGER REFERENCES batches(id)'); this.db.exec('CREATE INDEX IF NOT EXISTS attempts_batch ON attempts(batch_id)'); } diff --git a/src/server/guidance.js b/src/server/guidance.js index b9c9322..d95a0de 100644 --- a/src/server/guidance.js +++ b/src/server/guidance.js @@ -11,6 +11,7 @@ export const DEFAULT_GUIDANCE = [ 'Byte-identical code still merges automatically, so take an easy match when you see one, but move on quickly.', 'Claim work with td_claim_batch and no arguments: the server hands you a coherent group (a whole source file, a class, or neighbouring functions). Work through all of it and submit it as one batch; do not claim single functions from different classes.', 'Peer reviews also come in groups (up to 40 functions of one batch and file): review the whole group and send all verdicts in one td_submit_peer_review call with a verdicts list.', + 'Some functions need no decompilation (compiler-generated thunks and destructors, runtime library code such as the CRT, static initializers, non-code): mark those with td_mark_no_decompile and a reason instead of porting them; other agents review the claim.', 'If the group is functions that are in no source file yet, organize them first: decide which file each belongs to (and its name if you can tell) and record it with td_place, then port them.', 'Submit whole batches promptly and claim the next one. Between batches, take waiting peer reviews: judge whether the logic is the same, not whether the bytes are.', 'Add no comments (only annotations like // FUNCTION: fn_00001, offset markers like /* 0010 */ and // INLINED), and use only string literals that exist in the original executable; anything else is rejected.', diff --git a/src/server/merge.js b/src/server/merge.js index a21d546..74c5491 100644 --- a/src/server/merge.js +++ b/src/server/merge.js @@ -106,8 +106,29 @@ export class Merger { * approved part of a batch whose other functions still wait for review, so finished work * doesn't wait for the slowest function of its batch. */ + /** + * Accepted "no decompile" claims need no merge: the function is skipped (no task, not counted in + * progress). Kept for later review when a peer objected, like code. + */ + finishSkips() { + const done = this.store.all(`SELECT a.*, t.symbol_id FROM attempts a JOIN tasks t ON t.id = a.task_id WHERE a.kind = 'skip' AND a.state = 'approved'`); + if (!done.length) return; + this.store.tx(() => { + for (const a of done) { + this.store.run(`UPDATE attempts SET state = 'merged' WHERE id = ?`, a.id); + this.store.run(`UPDATE tasks SET state = 'skipped', lease_user = NULL, updated_at = ? WHERE id = ?`, now(), a.task_id); + this.store.run('UPDATE symbols SET skip_reason = ? WHERE id = ?', a.skip_category, a.symbol_id); + const objections = this.store.get(`SELECT COUNT(*) AS n FROM peer_reviews WHERE attempt_id = ? AND verdict = 'different'`, a.id).n; + if (objections) this.store.run('INSERT INTO later_review (symbol_id, attempt_id, merged_commit, objections, created_at) VALUES (?, ?, NULL, ?, ?)', a.symbol_id, a.id, objections, now()); + this.store.event('skipped', a.user_id, { attempt: a.id, symbol: a.symbol_id, category: a.skip_category }); + } + }); + this.log.info(`${done.length} function(s) accepted as needing no decompilation`); + } + /** Starts merges while a lane is free; resolves when the merges it started are done. */ async pump() { + if (!this.stopped) this.finishSkips(); const started = []; while (!this.stopped && this.running < MERGE_SLOTS) { const pick = this.next(this.runningBig > 0); diff --git a/src/server/peer.js b/src/server/peer.js index 6dfed85..305c968 100644 --- a/src/server/peer.js +++ b/src/server/peer.js @@ -15,12 +15,14 @@ import { now } from './db.js'; import { untrusted } from './tasks.js'; import { hex } from '../shared/annotations.js'; import { settleBatch } from './batches.js'; +import { SKIP_CATEGORIES } from './tasks.js'; // How many functions one review claim covers. const REVIEW_GROUP = 40; export const PEER_INSTRUCTIONS = [ 'You hold every function listed under functions. Review them all, then send all verdicts in one call (td_submit_peer_review with a verdicts list).', + 'A function with a claim field is not ported code but a claim that it needs no decompilation (and why). Read the original: "same" if the claim is right, "different" if it is ordinary code that should be ported.', 'Decide whether the submitted C++ does the same thing as the original function. It is not byte-identical; your judgment is what counts.', 'Read only: the original assembly (td_disassemble), the original pseudocode (td_decompile) and the submitted code below. Do not compile, run td_check, diff bytes or use other automated comparison.', 'Check: every branch and condition, loop bounds, arithmetic and signedness, the functions called and their arguments, fields and globals read and written, return values, and error paths.', @@ -130,7 +132,7 @@ export class PeerReviews { /** One function to review: where the original is and how to read it. */ functionInfo(attemptId, v) { const row = this.store.get( - `SELECT a.id, s.id AS symbol, s.name, ad.addr, ad.size FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id + `SELECT a.id, a.kind, a.skip_category, a.notes, s.id AS symbol, s.name, ad.addr, ad.size FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id LEFT JOIN symbol_addresses ad ON ad.symbol_id = s.id AND ad.version_id = ? WHERE a.id = ?`, v.id, attemptId, @@ -143,6 +145,7 @@ export class PeerReviews { size: row.size, annotation: `// FUNCTION: ${row.symbol}`, read: [`td_disassemble ${row.symbol}`, `td_decompile ${row.symbol}`], + ...(row.kind === 'skip' && { claim: { noDecompile: true, category: row.skip_category, meaning: SKIP_CATEGORIES[row.skip_category], reason: untrusted(row.notes) } }), }; } diff --git a/src/server/tasks.js b/src/server/tasks.js index 6d72bb2..08843f4 100644 --- a/src/server/tasks.js +++ b/src/server/tasks.js @@ -6,6 +6,15 @@ import { hex } from '../shared/annotations.js'; import { groupFor } from './grouping.js'; import { normalizeRepoPath } from '../shared/lint.js'; +// Why a function needs no decompilation (a "no decompile" claim). +export const SKIP_CATEGORIES = { + compiler_generated: 'generated by the compiler (adjustor or jump thunk, deleting destructor, vtable or RTTI helper)', + runtime_library: 'runtime or third-party library code linked in (CRT, compiler runtime)', + generated_from_global: 'generated from a global definition (static initializer, atexit destructor)', + not_code: 'not a real function (padding, data, a fragment of another function)', + duplicate: 'an identical copy of another function', +}; + // Names agents give unplaced functions: C++ identifiers, scopes and template arguments only. const PLACE_NAME_RE = /^[A-Za-z_~][\w:<>,*& ~]{0,200}$/; @@ -140,6 +149,40 @@ export class Tasks { }); } + /** + * "No decompile" claims for held functions: each becomes an attempt that peer review judges + * like code (3 agreeing verdicts decide, 5 accept without a human). Accepted ones are skipped. + */ + markNoDecompile(user, items) { + if (!Array.isArray(items) || !items.length) throw new HttpError(400, 'items must list { taskId, category, reason }'); + if (items.length > 200) throw new HttpError(400, 'at most 200 at a time'); + for (const it of items) { + if (!SKIP_CATEGORIES[it.category]) throw new HttpError(400, `category must be one of ${Object.keys(SKIP_CATEGORIES).join(', ')}`); + const reason = String(it.reason ?? '').trim(); + if (reason.length < 10 || reason.length > 300) throw new HttpError(400, 'give a reason of 10 to 300 characters (what in the original shows it)'); + } + return this.store.tx(() => { + for (const it of items) this.requireLease(user, Number(it.taskId)); + const head = this.repo.head(); + // A batch of its own (state 'skip': nothing to verify or merge) keeps the claims together for review. + const batchId = Number(this.store.run(`INSERT INTO batches (user_id, base_commit, files_json, state, created_at, finished_at) VALUES (?, ?, '{}', 'skip', ?, ?)`, user.id, head, now(), now()).lastInsertRowid); + const out = []; + for (const it of items) { + const taskId = Number(it.taskId); + const feedback = { summary: `claimed: no decompilation needed (${it.category}); waiting for peer review` }; + const a = this.store.run( + `INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, state, batch_id, kind, skip_category, result_json, created_at, finished_at) + VALUES (?, ?, ?, '{}', ?, 'peer_review', ?, 'skip', ?, ?, ?, ?)`, + taskId, user.id, head, String(it.reason).trim(), batchId, it.category, JSON.stringify({ feedback }), now(), now(), + ); + this.store.run(`UPDATE tasks SET state = 'peer_review', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), taskId); + out.push({ taskId, attemptId: Number(a.lastInsertRowid), state: 'peer_review' }); + } + this.store.event('no_decompile_claim', user.id, { batch: batchId, functions: out.length }); + return { batchId, claims: out }; + }); + } + // The open task to work on next for this user. bestOpen(user) { const ph = CLAIMABLE.map(() => '?').join(','); diff --git a/src/server/web/app.js b/src/server/web/app.js index 3806f27..71610a8 100644 --- a/src/server/web/app.js +++ b/src/server/web/app.js @@ -107,6 +107,7 @@ const TASK_STATES = { done: 'Merged into main.', needs_cleanup: 'Sent back: it did not compile, broke other functions, or reviewers found a difference. The author (then anyone) fixes it.', hard: 'Failed several times. Left for later or a stronger agent.', + skipped: 'Needs no decompilation (compiler-generated, runtime library, ...), confirmed by peer review. Not counted in progress.', }; /** A label with an info mark that explains it on hover, focus or tap. */ @@ -508,6 +509,7 @@ async function openAttempt(id) { const r = $('attempt-result'); r.replaceChildren(el('p', f.summary || '')); + if (a.claim) r.append(el('p', `Claimed: needs no decompilation (${a.claim.category}). Reason (untrusted): ${a.claim.reason}`, 'warn')); if (a.peerReviews?.length) { const n = (v) => a.peerReviews.filter((p) => p.verdict === v).length; r.append(el('h3', `Peer review: ${n('same')} same, ${n('different')} different (${a.peerRequired} agreeing verdicts decide)`)); diff --git a/test/server.test.js b/test/server.test.js index 9958496..ce08468 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -648,6 +648,51 @@ test('the review list skips fully leased functions before applying its limit', a } }); +test('"no decompile" claims are peer reviewed; accepted ones are skipped and not counted', async () => { + delete ctx.merger.pump; + const author = await activeUser('ndauthor'); + const peers = []; + for (let i = 1; i <= 5; i++) peers.push(await activeUser(`ndpeer${i}`)); + const v = ctx.store.currentVersion(); + const mk = (addr) => ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr, size: 8, unit: 'src/nd.cpp' }).symbol.id; + const thunk = mk(0x411000); + const real = mk(0x411040); + ctx.tasks.generate(); + const taskOf = (sym) => ctx.store.get('SELECT id FROM tasks WHERE symbol_id = ?', sym).id; + for (const sym of [thunk, real]) assert.equal((await call('POST', '/api/tasks/claim', { taskId: taskOf(sym) }, author)).status, 200); + + const bad = await call('POST', '/api/tasks/no-decompile', { items: [{ taskId: taskOf(thunk), category: 'whatever', reason: 'it is a thunk, add rcx then jmp' }] }, author); + assert.equal(bad.status, 400); + const r = await call('POST', '/api/tasks/no-decompile', { + items: [ + { taskId: taskOf(thunk), category: 'compiler_generated', reason: 'this-adjustor thunk: add rcx, 0x38 then jmp to the method' }, + { taskId: taskOf(real), category: 'not_code', reason: 'looks like padding to me honestly' }, + ], + }, author); + assert.equal(r.status, 200, JSON.stringify(r.body)); + const [ta, ra] = r.body.claims.map((c) => c.attemptId); + + // Reviewers see the claim, not code. + const claim = await call('POST', '/api/peer-reviews/claim', { attemptId: ta }, peers[0]); + assert.equal(claim.body.functions.find((f) => f.attemptId === ta).claim.category, 'compiler_generated'); + + const verdict = async (attempt, token, v) => { + await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, token); + return (await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: v, reason: v === 'same' ? 'the original is exactly add rcx and jmp, a thunk' : 'this is real code with branches and calls' }, token)).body; + }; + for (const t of peers) await verdict(ta, t, 'same'); + for (const t of peers.slice(0, 3)) await verdict(ra, t, 'different'); + assert.equal(ctx.store.get('SELECT state FROM attempts WHERE id = ?', ta).state, 'approved', 'five agreeing accept the claim'); + assert.equal(ctx.store.get('SELECT state FROM tasks WHERE symbol_id = ?', real).state, 'needs_cleanup', 'rejected claim: port it after all'); + + const before = (await call('GET', '/api/progress', null, author)).body.functions; + await ctx.merger.pump(); + assert.equal(ctx.store.get('SELECT state FROM tasks WHERE symbol_id = ?', thunk).state, 'skipped'); + assert.equal(ctx.store.get('SELECT skip_reason FROM symbols WHERE id = ?', thunk).skip_reason, 'compiler_generated'); + ctx.store.run(`UPDATE tasks SET state = 'done' WHERE symbol_id = ?`, real); + assert.ok(before >= 1); +}); + test('the crew view is public and shows no account details', async () => { const token = await activeUser('crewview'); await call('GET', '/api/tasks', null, token); // the agent checks in