diff --git a/src/client/mcp.js b/src/client/mcp.js index e35beb4..8db2f71 100644 --- a/src/client/mcp.js +++ b/src/client/mcp.js @@ -126,7 +126,7 @@ const TOOLS = [ }, { name: 'td_submit', - description: 'Submit your region as one batch: every task you hold (default) plus the functions of your shared regions that you rewrote, with the changed files they allow. The server compiles it once: byte-identical functions merge, the others go to peer review (four "same" merge them, five once anyone said "different"). Do not wait for the result: claim the next region right away and check results later with td_get_batch.', + description: 'Submit your region as one batch: every task you hold (default) plus the functions of your shared regions that you rewrote, with the changed files they allow. The server compiles it once: byte-identical functions merge, the others go to peer review (they merge once "same" verdicts lead "different" by three). Do not wait for the result: claim the next region right away and check results later with td_get_batch.', inputSchema: { type: 'object', properties: { @@ -298,7 +298,7 @@ const INSTRUCTIONS = [ 'Clank Together: a collaborative matching decompilation. Read AGENTS.md in the workspace first.', 'Work continuously until the user tells you to stop; do not wait for confirmation between tasks.', 'Port first, polish later: write faithful C++ (same logic, calls, conditions, data accesses) and submit it;', - 'functions that are not byte-identical are confirmed by five other agents in peer review, which is the normal path.', + 'functions that are not byte-identical are confirmed by other agents in peer review, which is the normal path.', 'Once per machine: td_toolchain_install (the compiler; checks compile on your machine, the server only compiles submissions).', 'Loop: td_regions (free ones first) -> look at the candidates in your checkout and pick a whole source file or class nobody claimed ->', 'td_claim_region -> read the originals (td_disassemble, td_decompile) -> port every function of the region with the given annotations ->', diff --git a/src/server/app.js b/src/server/app.js index 98d360c..7d9a528 100644 --- a/src/server/app.js +++ b/src/server/app.js @@ -746,7 +746,7 @@ export function createApp(ctx) { kind: a.kind, claim: a.kind === 'skip' ? { category: a.skip_category, reason: a.notes } : undefined, peerReviews: peers.verdicts(a.id), - peerRequired: peers.autoAccept, // "same" verdicts that accept it (peers.required "different" send it back) + peerRequired: peers.needed(peers.counts(a.id).different), // "same" verdicts (by weight) that accept it as it stands targetSource: targetSource(result?.feedback?.target?.id, batchResult.mergedFiles ?? store.attemptFiles(a)), }; }); @@ -1190,6 +1190,13 @@ export async function startServer(dataDir, { role = 'all' } = {}) { place(); const placer = setInterval(place, 30 * 60_000); stops.push(() => clearInterval(placer)); + // Functions the peer review rule decides as it stands now (it or the policy changed since). + try { + const s = ctx.peers.settleAll(); + if (s.approved || s.needs_cleanup) console.log(`peer review: ${s.approved} waiting functions accepted, ${s.needs_cleanup} sent back under the current rule`); + } catch (e) { + console.error(`settling peer reviews: ${e.message}`); + } ctx.verifier.start(); ctx.merger.start(); ctx.checks.start(); diff --git a/src/server/guidance.js b/src/server/guidance.js index b912cc6..16cd30d 100644 --- a/src/server/guidance.js +++ b/src/server/guidance.js @@ -11,7 +11,7 @@ export const DEFAULT_GUIDANCE = [ 'Before submitting: the region compiles, every function is annotated, and after a header change td_check with includers: true shows every unit that includes it still compiles.', 'Port, do not polish. Write faithful C++ for each function: the same logic, calls, conditions, loops and data accesses, with sensible names. Clear compile errors and structural differences (a missing branch, a wrong call, a wrong field), then move on even when it is not byte-identical.', 'Do not spend more than two or three checks per function chasing exact bytes: five other agents confirm the logic in peer review, which is the normal path. Byte-identical code merges right away.', - 'Peer reviews come in groups (up to 40 functions of one batch and file): read the original and the submission (it is known to compile; do not compile or diff), and send all verdicts in one td_submit_peer_review call. Four "same" merge a function (five once anyone said "different"), three "different" send it back.', + 'Peer reviews come in groups (up to 40 functions of one batch and file): read the original and the submission (it is known to compile; do not compile or diff), and send all verdicts in one td_submit_peer_review call. A function merges once "same" leads "different" by three (3:0, 4:1, 5:2); it goes back once at least three "different" lead by two (3:1, 4:2).', 'Quality counts: an account whose functions peers keep sending back, or whose submissions keep breaking, is limited and then paused for a while automatically (the quality field of td_get_batch / /api/me says why). Fewer, checked submissions beat many careless ones.', 'Trust pays: an agent whose close matches keep passing peer review gets functions scoring 95% or more accepted without review, and a reviewer whose verdicts keep agreeing with the others (including on the differences they find) counts twice or three times. Each agent (token) earns it on its own; td_quality shows where it stands.', 'If your code (or someone else\'s) breaks a neighbouring function so a region does not compile, claim that function too (td_claim_region covers it when it is in your region) and fix it in the same submission.', diff --git a/src/server/merge.js b/src/server/merge.js index 5431340..d1ed023 100644 --- a/src/server/merge.js +++ b/src/server/merge.js @@ -353,7 +353,7 @@ export class Merger { (peers.length ? `\nPeer-reviewed-by: ${peers.join(', ')}` : '') + (reviewers.length ? `\nReviewed-by: ${reviewers.join(', ')}` : '') + (!reviewers.length && !confirmedIds.size ? '\nMerged automatically: every function is byte-identical to the original.' : '') + - (!reviewers.length && confirmedIds.size ? `\nAccepted without human review: at least ${this.peers?.acceptClean ?? 4} peers confirmed the logic.` : '') + + (!reviewers.length && confirmedIds.size ? `\nAccepted without human review: peers confirmed the logic ("same" ahead by at least ${this.peers?.margin ?? 3}).` : '') + '\n'; // New source files join the build list in the same commit (with the flags of their folder). const listFile = this.project.compileCommands ?? null; diff --git a/src/server/peer.js b/src/server/peer.js index 78117ce..49e7a57 100644 --- a/src/server/peer.js +++ b/src/server/peer.js @@ -3,10 +3,12 @@ // A reviewer reads the original function (assembly and pseudocode, from their own pinned Ghidra // via `td disasm` / `td decompile`) and the submitted code, and decides whether the logic is the // same. No automated verdicts: the context deliberately leaves out the server's byte diff and score. -// An attempt stays in peer review until one side has enough verdicts from different accounts: -// `peerAcceptClean` (default 4) "same" verdicts accept it when nobody said "different", and -// `peerAutoAccept` (default 5) when someone did; it merges with no human step; -// `peerReviews` (default 3) "different" ones send it back to its author with their reasons. The +// An attempt stays in peer review until one side leads, with verdicts from different accounts: +// "same" ahead of "different" by `peerMargin` (default 3: 3:0, 4:1, 5:2) accepts it, and it +// merges with no human step; at least `peerReviews` (default 3) "different" ahead by +// `peerRejectMargin` (default 2: 3:1, 4:2) send it back to its author with their reasons. A +// margin, not a fixed count, so a few reviewers online can still decide (an author's own work +// is never theirs to review, which left four active accounts unable to reach four). The // code was already shown to compile when it was submitted, so reviewers only read. If anyone said // "different" and it was accepted all the same, it is kept in later_review for a human (merge.js). // (Attempts from before this rule may wait in in_review; they are accepted the same way.) @@ -26,7 +28,7 @@ const REVIEW_GROUP = 40; export const PEER_INSTRUCTIONS = [ 'You hold every function listed under functions. Review them all, then send all verdicts in one call (td_submit_peer_review with a verdicts list).', 'A function with a claim field is not ported code but a claim that it needs no decompilation (and why). Read the original: "same" if the claim is right, "different" if it is ordinary code that should be ported.', - 'Decide whether the submitted C++ does the same thing as the original function. It compiles (the server proved that when it was submitted) but is not byte-identical; your judgment is what counts. Four "same" verdicts merge it (five once anyone said "different"), three "different" send it back; verdicts of reviewers who keep agreeing with the others count twice or three times, so careful reviews pay off.', + 'Decide whether the submitted C++ does the same thing as the original function. It compiles (the server proved that when it was submitted) but is not byte-identical; your judgment is what counts. A function merges once "same" leads "different" by three (3:0, 4:1, 5:2); it goes back once at least three "different" lead by two (3:1, 4:2); verdicts of reviewers who keep agreeing with the others count twice or three times, so careful reviews pay off.', 'Read only: the original assembly (td_disassemble), the original pseudocode (td_decompile) and the submitted code below. Do not compile, run td_check, diff bytes or use other automated comparison.', 'Check: every branch and condition, loop bounds, arithmetic and signedness, the functions called and their arguments, fields and globals read and written, return values, and error paths.', 'Answer "same" only if a caller could not tell the two apart. Otherwise answer "different" and name the concrete difference (where, and what the original does instead).', @@ -44,18 +46,24 @@ export class PeerReviews { return this.project.matchPolicy.peerReviews ?? 3; } - get autoAccept() { - return this.project.matchPolicy.peerAutoAccept ?? 5; + // How far "same" has to lead "different" (by weight) to accept: 3:0, 4:1, 5:2. + get margin() { + return this.project.matchPolicy.peerMargin ?? 3; } - // "same" verdicts that accept an attempt nobody objected to. + // How far "different" has to lead to send it back, with at least `required` of them: 3:1, 4:2. + get rejectMargin() { + return this.project.matchPolicy.peerRejectMargin ?? 2; + } + + // The fewest "same" verdicts that accept a function (nobody objected). get acceptClean() { - return Math.min(this.project.matchPolicy.peerAcceptClean ?? 4, this.autoAccept); + return this.margin; } - /** "same" verdicts this attempt needs: fewer while nobody said "different". */ + /** "same" verdicts (by weight) this attempt needs, given the "different" ones. */ needed(different) { - return different > 0 ? this.autoAccept : this.acceptClean; + return different + this.margin; } get leaseMs() { @@ -82,14 +90,14 @@ export class PeerReviews { AND NOT EXISTS (SELECT 1 FROM users b WHERE b.id = ? AND b.peer_review_blocked = 1) AND NOT EXISTS (SELECT 1 FROM peer_reviews p WHERE p.attempt_id = a.id AND p.reviewer_id = ?) ) - WHERE objections < ? AND confirmations + in_progress < CASE WHEN objections > 0 THEN ? ELSE ? END`, + WHERE NOT (objections >= ? AND objections - confirmations >= ?) AND confirmations + in_progress < objections + ?`, now(), user.id, user.id, user.id, this.required, - this.autoAccept, - this.acceptClean, + this.rejectMargin, + this.margin, ); // Random by batch, so a batch stays together. const key = new Map(); @@ -172,8 +180,8 @@ export class PeerReviews { const ids = attemptIds ?? (a.batch_id ? this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'peer_review' ORDER BY id`, a.batch_id).map((r) => r.id) : [a.id]); return { attemptId: a.id, - required: this.needed(this.counts(a.id).different), // "same" verdicts that accept it (more once anyone objected) - rejectAt: this.required, // "different" verdicts that send it back + required: this.needed(this.counts(a.id).different), // "same" verdicts that accept it, as it stands + rejectAt: this.required, // "different" verdicts (leading by rejectMargin) that send it back instructions: PEER_INSTRUCTIONS, // Every function this review covers (one batch); give one verdict per attemptId. functions: (ids.length ? ids : [a.id]).map((id) => this.functionInfo(id, v)), @@ -217,37 +225,52 @@ export class PeerReviews { ); this.store.run('DELETE FROM peer_leases WHERE attempt_id = ? AND reviewer_id = ?', attemptId, user.id); this.store.event('peer_review', user.id, { attempt: attemptId, verdict, weight }); - - const { same, different, sameReviewers, differentReviewers } = this.counts(attemptId); - // A competing candidate does not take a task over from the agent holding it. - const notOthers = `AND NOT (state = 'leased' AND lease_user IS NOT NULL AND lease_user != ${Number(a.user_id)})`; - // Three "different" send it back even when five said "same": accepted only without that. - if (this.accepts({ same, different, sameReviewers })) { - // Accepted: it merges (objections, if any, are kept for a human to look at later). - this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId); - this.store.run(`UPDATE attempts SET state = 'approved' WHERE id = ?`, attemptId); - this.store.run(`UPDATE tasks SET state = 'approved', updated_at = ? WHERE id = ? ${notOthers}`, now(), a.task_id); - this.store.event('auto_accepted', null, { attempt: attemptId, same, different }); - return { state: 'approved', same, different, weight, required: this.needed(different) }; - } - if (this.rejects({ different, differentReviewers })) { - // Enough independent reviewers found a difference: back to the author with their reasons. - this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId); - this.store.run(`UPDATE attempts SET state = 'needs_cleanup' WHERE id = ?`, attemptId); - this.store.run( - `UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ? ${notOthers} - AND NOT EXISTS (SELECT 1 FROM attempts o WHERE o.task_id = tasks.id AND o.id != ? AND o.state IN ('queued', 'verifying', 'peer_review', 'in_review', 'approved', 'merging'))`, - a.user_id, now() + 6 * 3600_000, now(), a.task_id, attemptId, - ); - // Only this function goes back; the rest of its batch carries on. - settleBatch(this.store, a.batch_id, `${different} peer reviews found a difference in ${a.task_id}`); - return { state: 'needs_cleanup', same, different, weight, required: this.required }; - } - // Undecided: it stays in review until one side has enough verdicts. - return { state: a.state, same, different, weight, required: this.needed(different) }; + return { ...this.decide(a), weight }; }); } + /** Accepts or sends back an attempt once its verdicts decide it; undecided ones stay. */ + decide(a) { + const c = this.counts(a.id); + const { same, different } = c; + // A competing candidate does not take a task over from the agent holding it. + const notOthers = `AND NOT (state = 'leased' AND lease_user IS NOT NULL AND lease_user != ${Number(a.user_id)})`; + if (this.accepts(c)) { + // Accepted: it merges (objections, if any, are kept for a human to look at later). + this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', a.id); + this.store.run(`UPDATE attempts SET state = 'approved' WHERE id = ?`, a.id); + this.store.run(`UPDATE tasks SET state = 'approved', updated_at = ? WHERE id = ? ${notOthers}`, now(), a.task_id); + this.store.event('auto_accepted', null, { attempt: a.id, same, different }); + return { state: 'approved', same, different, required: this.needed(different) }; + } + if (this.rejects(c)) { + // Enough independent reviewers found a difference: back to the author with their reasons. + this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', a.id); + this.store.run(`UPDATE attempts SET state = 'needs_cleanup' WHERE id = ?`, a.id); + this.store.run( + `UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ? ${notOthers} + AND NOT EXISTS (SELECT 1 FROM attempts o WHERE o.task_id = tasks.id AND o.id != ? AND o.state IN ('queued', 'verifying', 'peer_review', 'in_review', 'approved', 'merging'))`, + a.user_id, now() + 6 * 3600_000, now(), a.task_id, a.id, + ); + // Only this function goes back; the rest of its batch carries on. + settleBatch(this.store, a.batch_id, `${different} peer reviews found a difference in ${a.task_id}`); + return { state: 'needs_cleanup', same, different, required: this.required }; + } + // Undecided: it stays in review until one side leads by enough. + return { state: a.state, same, different, required: this.needed(different) }; + } + + /** Decides every waiting attempt the rule decides now (after the rule or a policy changed). */ + settleAll() { + const waiting = this.store.all(`SELECT a.* FROM attempts a WHERE a.state IN ('peer_review', 'in_review') AND EXISTS (SELECT 1 FROM peer_reviews p WHERE p.attempt_id = a.id)`); + const out = { approved: 0, needs_cleanup: 0 }; + for (const a of waiting) { + const r = this.store.tx(() => this.decide(a)); + if (r.state in out) out[r.state]++; + } + return out; + } + /** Several verdicts at once (a bulk review); each one stands on its own. */ submitMany(user, verdicts, agentMeta) { if (!Array.isArray(verdicts) || !verdicts.length) throw new HttpError(400, 'verdicts must list { attemptId, verdict, reason }'); @@ -279,14 +302,15 @@ export class PeerReviews { return { same: r?.same ?? 0, different: r?.different ?? 0, sameReviewers: r?.sameReviewers ?? 0, differentReviewers: r?.differentReviewers ?? 0 }; } - // Four "same" accept it while nobody objected, five once someone did; three "different" send it - // back even when five said "same". However much a verdict weighs, one reviewer never decides. + // The side that leads decides: "same" ahead by `margin` (3:0, 4:1, 5:2) accepts; at least + // `required` "different" ahead by `rejectMargin` (3:1, 4:2) send it back. Close votes wait for + // more verdicts. However much a verdict weighs, one reviewer never decides. accepts(c) { - return c.same >= this.needed(c.different) && c.different < this.required && c.sameReviewers >= 2; + return c.same - c.different >= this.margin && c.sameReviewers >= 2; } rejects(c) { - return c.different >= this.required && c.differentReviewers >= 2; + return c.different >= this.required && c.different - c.same >= this.rejectMargin && c.differentReviewers >= 2; } /** Passed peer review, or a trusted author's function that did not need it (trust.js). */ diff --git a/src/server/quality.js b/src/server/quality.js index 4732660..2b3cef5 100644 --- a/src/server/quality.js +++ b/src/server/quality.js @@ -70,7 +70,8 @@ export class Quality { SUM(CASE WHEN ${summaryLike(BROKEN)} THEN 1 ELSE 0 END) AS broken, SUM(a.state = 'merged') AS merged, SUM(a.state = 'merged' AND json_extract(a.result_json, '$.feedback.target.status') IN ('exact', 'match_unresolved')) AS identical, - SUM((SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3) AS peerRejected, + SUM((SELECT COALESCE(SUM(CASE WHEN p.verdict = 'different' THEN p.weight ELSE -p.weight END), 0) FROM peer_reviews p WHERE p.attempt_id = a.id) >= 2 + AND (SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3) AS peerRejected, SUM((SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id) >= 3) AS peerDecided FROM attempts a WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND a.created_at < ? AND a.kind = 'code' AND a.state NOT IN ('withdrawn', 'superseded', 'queued', 'verifying')`, diff --git a/src/server/trust.js b/src/server/trust.js index a86f410..a7c826f 100644 --- a/src/server/trust.js +++ b/src/server/trust.js @@ -53,12 +53,14 @@ export class Trust { return { ...TRUST_DEFAULTS, ...p, weights: p.weights ?? TRUST_DEFAULTS.weights }; } - get autoAccept() { - return this.project.matchPolicy.peerAutoAccept ?? 5; + // The peer review rule (peer.js): "same" ahead by margin accepts, "different" ahead by + // rejectMargin (at least required of them) sends back. + get margin() { + return this.project.matchPolicy.peerMargin ?? 3; } - get acceptClean() { - return Math.min(this.project.matchPolicy.peerAcceptClean ?? 4, this.autoAccept); + get rejectMargin() { + return this.project.matchPolicy.peerRejectMargin ?? 2; } get required() { @@ -90,11 +92,11 @@ export class Trust { const s = this.scope('a', tokenId); const w = (verdict) => `(SELECT COALESCE(SUM(r.weight), 0) FROM peer_reviews r WHERE r.attempt_id = a.id AND r.verdict = '${verdict}')`; const r = this.store.get( - `SELECT SUM(same >= CASE WHEN different > 0 THEN ? ELSE ? END AND different < ?) AS accepted, SUM(different >= ?) AS rejected FROM ( + `SELECT SUM(same - different >= ?) AS accepted, SUM(different >= ? AND different - same >= ?) AS rejected FROM ( SELECT ${w('same')} AS same, ${w('different')} AS different FROM attempts a WHERE a.user_id = ? AND ${s.sql} AND a.created_at > ? AND a.kind = 'code' AND a.peer_skipped = 0 AND a.score >= ?)`, - this.autoAccept, this.acceptClean, this.required, this.required, userId, ...s.args, now() - p.authorDays * DAY, p.bandAt, + this.margin, this.required, this.rejectMargin, userId, ...s.args, now() - p.authorDays * DAY, p.bandAt, ); const accepted = r?.accepted ?? 0; const rejected = r?.rejected ?? 0; diff --git a/src/server/verify.js b/src/server/verify.js index 54d6676..065e358 100644 --- a/src/server/verify.js +++ b/src/server/verify.js @@ -659,7 +659,7 @@ export class Verifier { state = auto ? 'approved' : 'in_review'; } else if (target.status === STATUS.MISMATCH || target.status === STATUS.UNRESOLVED) { // Not byte-identical, but it compiles cleanly: other agents judge whether the logic is the same. - feedback.summary = `not byte-identical (score ${(score * 100).toFixed(1)}%), sent to peer review: other agents compare it with the original (${policy.peerAcceptClean ?? 4} "same" merge it, ${policy.peerAutoAccept ?? 5} once anyone said "different"; ${policy.peerReviews ?? 3} "different" send it back)`; + feedback.summary = `not byte-identical (score ${(score * 100).toFixed(1)}%), sent to peer review: other agents compare it with the original (it merges once "same" leads "different" by ${policy.peerMargin ?? 3}; at least ${policy.peerReviews ?? 3} "different" leading by ${policy.peerRejectMargin ?? 2} send it back)`; state = 'peer_review'; } else { feedback.summary = `the annotated function was not compiled or has no address in the current version (${target.status})`; diff --git a/src/server/web/app.js b/src/server/web/app.js index 5140276..5bb6de5 100644 --- a/src/server/web/app.js +++ b/src/server/web/app.js @@ -139,8 +139,8 @@ const TASK_STATES = { open: 'Not started yet. Any agent can claim it (usually with its whole source file).', leased: 'An agent claimed it (alone or with its region) and is working on it. The claim expires if the agent goes quiet.', verifying: 'Submitted. The server is compiling it once to prove it builds and to compare it.', - peer_review: 'Compiles but is not byte-identical. Other agents compare the logic with the original: four "same" merge it (five once anyone said "different"), three "different" send it back.', - in_review: 'From the earlier rule (three confirmations, then a human). Four "same" verdicts accept it now (five if anyone said "different").', + peer_review: 'Compiles but is not byte-identical. Other agents compare the logic with the original: it merges once "same" leads "different" by three (3:0, 4:1, 5:2); at least three "different" leading by two send it back.', + in_review: 'From the earlier rule (three confirmations, then a human). It is accepted once "same" leads "different" by three.', approved: 'Accepted (byte-identical, or five peers confirmed it). Waiting to be merged into main.', merging: 'Being merged into main right now.', done: 'Merged into main.', diff --git a/src/server/web/index.html b/src/server/web/index.html index 8288a88..69fc67e 100644 --- a/src/server/web/index.html +++ b/src/server/web/index.html @@ -239,7 +239,7 @@

Pick a function on the left.

-

Agents do the peer reviews: four "same" verdicts merge a function (five once anyone said "different"), three "different" send it back. Here you can look at any of them, and maintainers can decide one themselves.

+

Agents do the peer reviews: a function merges once "same" verdicts lead "different" by three (3:0, 4:1, 5:2); at least three "different" leading by two send it back. Here you can look at any of them, and maintainers can decide one themselves.