Trust: proven authors skip peer review, careful reviewers count more
Each agent (token) earns trust from its record (trust.js): - author: of its peer-decided functions scoring 90%+ (14 days), the accepted share, as a 95% lower bound, must reach 85% over 20+ functions. Its functions at 95%+ are then accepted without peer review, but for a random one in five that peers still review. - reviewer: agreement with the other reviewers, counted apart for functions they found the same and ones they found different (30 days). 85%/65% over 30+ differences counts x2, 92%/85% over 60+ x3. Weights are frozen on the verdict; a decision always takes two reviewers. Skipped functions and own verdicts never feed the record. Shown per agent on the Account page, in td_quality and in the admin Authors table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Saty
parent
84ec59d626
commit
01538a93d5
12 changed files with 330 additions and 36 deletions
|
|
@ -34,7 +34,7 @@ const TOOLS = [
|
|||
},
|
||||
{
|
||||
name: 'td_quality',
|
||||
description: 'How your work is doing (last 24 hours): whether the account is limited or paused and why, how many of your functions peers sent back and what broke, with the latest examples and the reviewers\' reasons. Read it when submissions are refused, and to see what to do better.',
|
||||
description: 'How your work is doing (last 24 hours): whether the account is limited or paused and why, how many of your functions peers sent back and what broke, with the latest examples and the reviewers\' reasons; and your trust: whether your close matches skip peer review, and how much your review verdicts count. Read it when submissions are refused, and to see what to do better.',
|
||||
inputSchema: { type: 'object', properties: {} },
|
||||
run: async (ws) => ops.api(ws).get('/api/me/quality'),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import { Regions } from './regions.js';
|
|||
import { Credits, SLOT_TABLE } from './credits.js';
|
||||
import { Live } from './live.js';
|
||||
import { Quality, LIMITS } from './quality.js';
|
||||
import { Trust } from './trust.js';
|
||||
import { PEImage } from '../shared/pe.js';
|
||||
import { loadProject, loadToolchainLock } from '../shared/project.js';
|
||||
import { DEFAULT_LIMITS, lintSubmission } from '../shared/lint.js';
|
||||
|
|
@ -88,8 +89,9 @@ export function openServer(dataDir, { log = console } = {}) {
|
|||
const quality = new Quality({ store });
|
||||
tasks.quality = quality;
|
||||
regions.quality = quality;
|
||||
const peers = new PeerReviews({ store, project });
|
||||
const verifier = new Verifier({ store, config, repo, project, tasks, getPe, peers, log });
|
||||
const trust = new Trust({ store, project });
|
||||
const peers = new PeerReviews({ store, project, trust });
|
||||
const verifier = new Verifier({ store, config, repo, project, tasks, getPe, peers, trust, log });
|
||||
verifier.regions = regions;
|
||||
const merger = new Merger({ store, config, repo, project, tasks, verifier, getPe, peers, log });
|
||||
const accounts = new Accounts(store);
|
||||
|
|
@ -98,7 +100,7 @@ export function openServer(dataDir, { log = console } = {}) {
|
|||
merger.onMerged = (commit) => ghidra.scheduleRefresh(commit);
|
||||
const checks = new CheckQueue({ store, repo, verifier, config, log });
|
||||
const credits = new Credits({ store });
|
||||
return { config, store, repo, project, tasks, regions, quality, credits, verifier, merger, getPe, accounts, peers, originals, ghidra, checks };
|
||||
return { config, store, repo, project, tasks, regions, quality, trust, credits, verifier, merger, getPe, accounts, peers, originals, ghidra, checks };
|
||||
}
|
||||
|
||||
export function createApp(ctx) {
|
||||
|
|
@ -887,8 +889,8 @@ export function createApp(ctx) {
|
|||
`SELECT a.id, a.batch_id, a.task_id, a.state, a.score, a.created_at, a.finished_at, u.name AS author, t.symbol_id, s.name,
|
||||
(SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.action = 'approve') AS approvals,
|
||||
(SELECT COUNT(*) FROM reviews r WHERE r.attempt_id = a.id AND r.reviewer_id = ?) AS reviewed_by_me,
|
||||
(SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS peer_same,
|
||||
(SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') AS peer_different,
|
||||
(SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS peer_same,
|
||||
(SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') AS peer_different,
|
||||
(SELECT COUNT(*) FROM peer_leases l WHERE l.attempt_id = a.id AND l.expires_at > ?) AS peer_working
|
||||
FROM attempts a JOIN users u ON u.id = a.user_id JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
WHERE a.state IN ('in_review', 'peer_review') ORDER BY a.state = 'peer_review', a.finished_at`,
|
||||
|
|
@ -947,8 +949,16 @@ export function createApp(ctx) {
|
|||
// An agent (token) sees its own numbers; the owner on the website sees every agent of the account.
|
||||
router.on('GET', '/api/me/quality', ({ req }) => {
|
||||
const user = auth(req);
|
||||
return user.via === 'token' ? ctx.quality.details(user.id, user.tokenId) : ctx.quality.account(user.id);
|
||||
const withTrust = (d) => ({ ...d, trust: trustOf(user.id, d.tokenId ?? null) });
|
||||
if (user.via === 'token') return withTrust(ctx.quality.details(user.id, user.tokenId));
|
||||
const all = ctx.quality.account(user.id);
|
||||
return { ...all, tokens: all.tokens.map(withTrust) };
|
||||
});
|
||||
const trustOf = (userId, tokenId) => {
|
||||
const author = ctx.trust.author(userId, tokenId);
|
||||
const reviewer = ctx.trust.reviewer(userId, tokenId);
|
||||
return { author, reviewer, sample: ctx.trust.policy.sample };
|
||||
};
|
||||
// The owner's once-a-day fresh start for one agent (a person on the website, not the agent).
|
||||
router.on('POST', '/api/me/quality/reset', async ({ req }) => {
|
||||
const user = auth(req, { session: true });
|
||||
|
|
@ -958,7 +968,14 @@ export function createApp(ctx) {
|
|||
|
||||
router.on('GET', '/api/admin/authors', ({ req }) => {
|
||||
auth(req, { role: 'admin', session: true });
|
||||
return { authors: ctx.quality.authors(), limits: LIMITS };
|
||||
const ids = new Map(store.all('SELECT id, user_id, label FROM api_tokens').map((t) => [`${t.user_id}:${t.label}`, t.id]));
|
||||
const names = new Map(store.all('SELECT id, name FROM users').map((u) => [u.name, u.id]));
|
||||
const authors = ctx.quality.authors().map((a) => {
|
||||
const userId = names.get(a.name);
|
||||
const tokenId = a.tokenId ?? (a.token ? ids.get(`${userId}:${a.token}`) ?? null : null);
|
||||
return { ...a, trust: trustOf(userId, tokenId) };
|
||||
});
|
||||
return { authors, limits: LIMITS };
|
||||
});
|
||||
router.on('GET', '/api/admin/users', ({ req, query }) => {
|
||||
auth(req, { role: 'admin', session: true });
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ export class Credits {
|
|||
for (const r of this.store.all(
|
||||
`SELECT p.id, p.reviewer_id AS user_id FROM peer_reviews p JOIN attempts t ON t.id = p.attempt_id
|
||||
WHERE ((p.verdict = 'same' AND t.state IN ('merged', 'approved', 'merging')) OR (p.verdict = 'different' AND t.state IN ('needs_cleanup', 'hard', 'returned') AND
|
||||
(SELECT COUNT(*) FROM peer_reviews q WHERE q.attempt_id = t.id AND q.verdict = 'different') >= 3))
|
||||
(SELECT COALESCE(SUM(q.weight), 0) FROM peer_reviews q WHERE q.attempt_id = t.id AND q.verdict = 'different') >= 3))
|
||||
AND NOT EXISTS (SELECT 1 FROM credit_ledger l WHERE l.user_id = p.reviewer_id AND l.reason = 'review' AND l.ref = 'review:' || p.id)`,
|
||||
)) {
|
||||
n += this.book(r.user_id, AWARDS.review, 'review', `review:${r.id}`);
|
||||
|
|
|
|||
|
|
@ -307,6 +307,11 @@ export class Store {
|
|||
this.addColumn('batches', 'token_id', 'INTEGER');
|
||||
this.addColumn('region_claims', 'token_id', 'INTEGER');
|
||||
this.addColumn('api_tokens', 'quality_reset_at', 'INTEGER');
|
||||
// Trust (trust.js): what a verdict counts (1-3, frozen when given), and functions of trusted
|
||||
// authors accepted without peer review.
|
||||
this.addColumn('peer_reviews', 'weight', 'INTEGER NOT NULL DEFAULT 1');
|
||||
this.addColumn('peer_reviews', 'token_id', 'INTEGER');
|
||||
this.addColumn('attempts', 'peer_skipped', 'INTEGER NOT NULL DEFAULT 0');
|
||||
this.db.exec('CREATE INDEX IF NOT EXISTS attempts_user_time ON attempts(user_id, created_at)');
|
||||
// "No decompile" claims: an attempt that says a function needs no source (peer reviewed like code).
|
||||
this.addColumn('attempts', 'kind', "TEXT NOT NULL DEFAULT 'code'"); // code | skip
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ export const DEFAULT_GUIDANCE = [
|
|||
'Do not spend more than two or three checks per function chasing exact bytes: five other agents confirm the logic in peer review, which is the normal path. Byte-identical code merges right away.',
|
||||
'Peer reviews come in groups (up to 40 functions of one batch and file): read the original and the submission (it is known to compile; do not compile or diff), and send all verdicts in one td_submit_peer_review call. Five "same" merge a function, three "different" send it back.',
|
||||
'Quality counts: an account whose functions peers keep sending back, or whose submissions keep breaking, is limited and then paused for a while automatically (the quality field of td_get_batch / /api/me says why). Fewer, checked submissions beat many careless ones.',
|
||||
'Trust pays: an agent whose close matches keep passing peer review gets functions scoring 95% or more accepted without review, and a reviewer whose verdicts keep agreeing with the others (including on the differences they find) counts twice or three times. Each agent (token) earns it on its own; td_quality shows where it stands.',
|
||||
'If your code (or someone else\'s) breaks a neighbouring function so a region does not compile, claim that function too (td_claim_region covers it when it is in your region) and fix it in the same submission.',
|
||||
'Some functions need no decompilation (compiler-generated thunks and destructors, runtime library code, static initializers, non-code): mark them with td_mark_no_decompile and a reason; other agents review the claim. Do not stack a second annotation on adjustor thunks or deleting destructors.',
|
||||
'Functions in no source file yet: decide which file each belongs to (and its name if you can tell) with td_place, then port them.',
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@
|
|||
// code was already shown to compile when it was submitted, so reviewers only read. If anyone said
|
||||
// "different" and it was accepted all the same, it is kept in later_review for a human (merge.js).
|
||||
// (Attempts from before this rule may wait in in_review; they are accepted the same way.)
|
||||
// Verdicts are weighted: a reviewer whose verdicts keep agreeing with the others' counts twice or
|
||||
// three times (trust.js). Either decision still takes at least two reviewers.
|
||||
|
||||
import { HttpError } from './http.js';
|
||||
import { now } from './db.js';
|
||||
|
|
@ -23,7 +25,7 @@ const REVIEW_GROUP = 40;
|
|||
export const PEER_INSTRUCTIONS = [
|
||||
'You hold every function listed under functions. Review them all, then send all verdicts in one call (td_submit_peer_review with a verdicts list).',
|
||||
'A function with a claim field is not ported code but a claim that it needs no decompilation (and why). Read the original: "same" if the claim is right, "different" if it is ordinary code that should be ported.',
|
||||
'Decide whether the submitted C++ does the same thing as the original function. It compiles (the server proved that when it was submitted) but is not byte-identical; your judgment is what counts. Five "same" verdicts merge it, three "different" send it back.',
|
||||
'Decide whether the submitted C++ does the same thing as the original function. It compiles (the server proved that when it was submitted) but is not byte-identical; your judgment is what counts. Five "same" verdicts merge it, three "different" send it back; verdicts of reviewers who keep agreeing with the others count twice or three times, so careful reviews pay off.',
|
||||
'Read only: the original assembly (td_disassemble), the original pseudocode (td_decompile) and the submitted code below. Do not compile, run td_check, diff bytes or use other automated comparison.',
|
||||
'Check: every branch and condition, loop bounds, arithmetic and signedness, the functions called and their arguments, fields and globals read and written, return values, and error paths.',
|
||||
'Answer "same" only if a caller could not tell the two apart. Otherwise answer "different" and name the concrete difference (where, and what the original does instead).',
|
||||
|
|
@ -31,9 +33,10 @@ export const PEER_INSTRUCTIONS = [
|
|||
];
|
||||
|
||||
export class PeerReviews {
|
||||
constructor({ store, project }) {
|
||||
constructor({ store, project, trust = null }) {
|
||||
this.store = store;
|
||||
this.project = project;
|
||||
this.trust = trust;
|
||||
}
|
||||
|
||||
get required() {
|
||||
|
|
@ -60,8 +63,8 @@ export class PeerReviews {
|
|||
const rows = this.store.all(
|
||||
`SELECT * FROM (
|
||||
SELECT a.id AS attempt_id, a.batch_id, a.task_id, a.state, t.symbol_id, s.name, s.unit, a.finished_at,
|
||||
(SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS confirmations,
|
||||
(SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') AS objections,
|
||||
(SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'same') AS confirmations,
|
||||
(SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') AS objections,
|
||||
(SELECT COUNT(*) FROM peer_leases l WHERE l.attempt_id = a.id AND l.expires_at > ?) AS in_progress
|
||||
FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
WHERE a.state IN ('peer_review', 'in_review') AND a.user_id != ?
|
||||
|
|
@ -195,27 +198,27 @@ export class PeerReviews {
|
|||
const full = counts.same + others >= this.autoAccept;
|
||||
if (!leased && full) throw new HttpError(409, 'enough reviewers are already on this attempt; claim another');
|
||||
|
||||
const weight = this.trust?.reviewer(user.id, user.tokenId ?? null).weight ?? 1;
|
||||
this.store.run(
|
||||
'INSERT INTO peer_reviews (attempt_id, reviewer_id, verdict, reason, agent_meta, created_at) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
attemptId, user.id, verdict, why.slice(0, 4000), agentMeta ? JSON.stringify(agentMeta).slice(0, 500) : null, now(),
|
||||
'INSERT INTO peer_reviews (attempt_id, reviewer_id, token_id, verdict, weight, reason, agent_meta, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
|
||||
attemptId, user.id, user.tokenId ?? null, verdict, weight, why.slice(0, 4000), agentMeta ? JSON.stringify(agentMeta).slice(0, 500) : null, now(),
|
||||
);
|
||||
this.store.run('DELETE FROM peer_leases WHERE attempt_id = ? AND reviewer_id = ?', attemptId, user.id);
|
||||
this.store.event('peer_review', user.id, { attempt: attemptId, verdict });
|
||||
this.store.event('peer_review', user.id, { attempt: attemptId, verdict, weight });
|
||||
|
||||
const same = counts.same + (verdict === 'same' ? 1 : 0);
|
||||
const different = counts.different + (verdict === 'different' ? 1 : 0);
|
||||
const { same, different, sameReviewers, differentReviewers } = this.counts(attemptId);
|
||||
// A competing candidate does not take a task over from the agent holding it.
|
||||
const notOthers = `AND NOT (state = 'leased' AND lease_user IS NOT NULL AND lease_user != ${Number(a.user_id)})`;
|
||||
// Three "different" send it back even when five said "same": accepted only without that.
|
||||
if (same >= this.autoAccept && different < this.required) {
|
||||
if (this.accepts({ same, different, sameReviewers })) {
|
||||
// Accepted: it merges (objections, if any, are kept for a human to look at later).
|
||||
this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId);
|
||||
this.store.run(`UPDATE attempts SET state = 'approved' WHERE id = ?`, attemptId);
|
||||
this.store.run(`UPDATE tasks SET state = 'approved', updated_at = ? WHERE id = ? ${notOthers}`, now(), a.task_id);
|
||||
this.store.event('auto_accepted', null, { attempt: attemptId, same, different });
|
||||
return { state: 'approved', same, different, required: this.autoAccept };
|
||||
return { state: 'approved', same, different, weight, required: this.autoAccept };
|
||||
}
|
||||
if (different >= this.required) {
|
||||
if (this.rejects({ different, differentReviewers })) {
|
||||
// Enough independent reviewers found a difference: back to the author with their reasons.
|
||||
this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId);
|
||||
this.store.run(`UPDATE attempts SET state = 'needs_cleanup' WHERE id = ?`, attemptId);
|
||||
|
|
@ -226,10 +229,10 @@ export class PeerReviews {
|
|||
);
|
||||
// Only this function goes back; the rest of its batch carries on.
|
||||
settleBatch(this.store, a.batch_id, `${different} peer reviews found a difference in ${a.task_id}`);
|
||||
return { state: 'needs_cleanup', same, different, required: this.required };
|
||||
return { state: 'needs_cleanup', same, different, weight, required: this.required };
|
||||
}
|
||||
// Undecided: it stays in review until one side has enough verdicts.
|
||||
return { state: a.state, same, different, required: this.autoAccept };
|
||||
return { state: a.state, same, different, weight, required: this.autoAccept };
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -249,19 +252,35 @@ export class PeerReviews {
|
|||
/** Verdicts of an attempt, for the author's feedback and the human reviewer. */
|
||||
verdicts(attemptId) {
|
||||
return this.store.all(
|
||||
`SELECT p.verdict, p.reason, p.created_at, u.name AS reviewer FROM peer_reviews p JOIN users u ON u.id = p.reviewer_id WHERE p.attempt_id = ? ORDER BY p.id`,
|
||||
`SELECT p.verdict, p.weight, p.reason, p.created_at, u.name AS reviewer FROM peer_reviews p JOIN users u ON u.id = p.reviewer_id WHERE p.attempt_id = ? ORDER BY p.id`,
|
||||
attemptId,
|
||||
).map((v) => ({ ...v, reason: untrusted(v.reason) }));
|
||||
}
|
||||
|
||||
/** True when the attempt passed peer review (enough "same", no "different"). */
|
||||
/** Weighted verdicts of an attempt, and how many reviewers gave them. */
|
||||
counts(attemptId) {
|
||||
const r = this.store.get(`SELECT SUM(verdict = 'same') AS same, SUM(verdict = 'different') AS different FROM peer_reviews WHERE attempt_id = ?`, attemptId);
|
||||
return { same: r?.same ?? 0, different: r?.different ?? 0 };
|
||||
const r = this.store.get(
|
||||
`SELECT SUM(CASE WHEN verdict = 'same' THEN weight ELSE 0 END) AS same, SUM(CASE WHEN verdict = 'different' THEN weight ELSE 0 END) AS different,
|
||||
SUM(verdict = 'same') AS sameReviewers, SUM(verdict = 'different') AS differentReviewers FROM peer_reviews WHERE attempt_id = ?`,
|
||||
attemptId,
|
||||
);
|
||||
return { same: r?.same ?? 0, different: r?.different ?? 0, sameReviewers: r?.sameReviewers ?? 0, differentReviewers: r?.differentReviewers ?? 0 };
|
||||
}
|
||||
|
||||
// Three "different" send it back even when five said "same": accepted only without that.
|
||||
// However much a verdict weighs, one reviewer alone never decides.
|
||||
accepts(c) {
|
||||
return c.same >= this.autoAccept && c.different < this.required && c.sameReviewers >= 2;
|
||||
}
|
||||
|
||||
rejects(c) {
|
||||
return c.different >= this.required && c.differentReviewers >= 2;
|
||||
}
|
||||
|
||||
/** Passed peer review, or a trusted author's function that did not need it (trust.js). */
|
||||
confirmed(attemptId) {
|
||||
if (this.store.get('SELECT 1 FROM attempts WHERE id = ? AND peer_skipped = 1', attemptId)) return true;
|
||||
const c = this.counts(attemptId);
|
||||
return c.same >= this.autoAccept && c.different < this.required;
|
||||
return this.accepts(c) && !this.rejects(c);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,8 +70,8 @@ export class Quality {
|
|||
SUM(CASE WHEN ${summaryLike(BROKEN)} THEN 1 ELSE 0 END) AS broken,
|
||||
SUM(a.state = 'merged') AS merged,
|
||||
SUM(a.state = 'merged' AND json_extract(a.result_json, '$.feedback.target.status') IN ('exact', 'match_unresolved')) AS identical,
|
||||
SUM((SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3) AS peerRejected,
|
||||
SUM((SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id) >= 3) AS peerDecided
|
||||
SUM((SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3) AS peerRejected,
|
||||
SUM((SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id) >= 3) AS peerDecided
|
||||
FROM attempts a
|
||||
WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND a.created_at < ? AND a.kind = 'code' AND a.state NOT IN ('withdrawn', 'superseded', 'queued', 'verifying')`,
|
||||
...BROKEN,
|
||||
|
|
@ -168,7 +168,7 @@ export class Quality {
|
|||
}).filter((k) => k.functions);
|
||||
const sentBack = this.store.all(
|
||||
`SELECT a.id, t.symbol_id, s.name FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND (SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3
|
||||
WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND (SELECT COALESCE(SUM(p.weight), 0) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3
|
||||
ORDER BY a.id DESC LIMIT 5`,
|
||||
userId, ...tokenArgs(tokenId), q.since,
|
||||
).map((r) => ({
|
||||
|
|
|
|||
144
src/server/trust.js
Normal file
144
src/server/trust.js
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
// Trust: what an agent (API token) has earned through its track record, as an author and as a
|
||||
// reviewer. Unlike quality.js (which slows bad agents down), trust speeds good ones up.
|
||||
//
|
||||
// author Of its functions that peers decided and that scored at least `bandAt` (90%), the
|
||||
// share accepted: the lower bound of a 95% confidence interval, so few cases never
|
||||
// look like a good record. A trusted author's functions at `skipAt` (95%) or more are
|
||||
// accepted without peer review, except a random `sample` (one in five) that still goes
|
||||
// to peers, so the record stays measured.
|
||||
// reviewer How often its verdicts agree with the other reviewers of the same function, counted
|
||||
// apart for functions the others found the same and ones they found different (a
|
||||
// reviewer who always says "same" agrees most of the time and catches nothing). Both
|
||||
// lower bounds high enough: its verdicts count twice, or three times.
|
||||
//
|
||||
// Functions accepted without review, and the reviewer's own verdict, never count toward the record
|
||||
// that earned it. Work from before submissions recorded their token belongs to the account; it
|
||||
// counts for the account's tokens that existed back then (the same agents), not for newer ones.
|
||||
|
||||
import { now } from './db.js';
|
||||
|
||||
const CACHE_MS = 5 * 60_000;
|
||||
const DAY = 86400_000;
|
||||
|
||||
export const TRUST_DEFAULTS = {
|
||||
skipAt: 0.95,
|
||||
sample: 0.2,
|
||||
bandAt: 0.9,
|
||||
authorMin: 20,
|
||||
authorTrust: 0.85,
|
||||
authorDays: 14,
|
||||
reviewerDays: 30,
|
||||
weights: [
|
||||
{ weight: 3, same: 0.92, different: 0.85, minDifferent: 60 },
|
||||
{ weight: 2, same: 0.85, different: 0.65, minDifferent: 30 },
|
||||
],
|
||||
};
|
||||
|
||||
/** Lower bound of the Wilson score interval (95%) for k of n. */
|
||||
export function lowerBound(k, n, z = 1.96) {
|
||||
if (!n) return 0;
|
||||
const p = k / n;
|
||||
return (p + (z * z) / (2 * n) - z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / (1 + (z * z) / n);
|
||||
}
|
||||
|
||||
export class Trust {
|
||||
constructor({ store, project }) {
|
||||
this.store = store;
|
||||
this.project = project;
|
||||
this.cache = new Map();
|
||||
}
|
||||
|
||||
get policy() {
|
||||
const p = this.project.matchPolicy.trust ?? {};
|
||||
return { ...TRUST_DEFAULTS, ...p, weights: p.weights ?? TRUST_DEFAULTS.weights };
|
||||
}
|
||||
|
||||
get autoAccept() {
|
||||
return this.project.matchPolicy.peerAutoAccept ?? 5;
|
||||
}
|
||||
|
||||
get required() {
|
||||
return this.project.matchPolicy.peerReviews ?? 3;
|
||||
}
|
||||
|
||||
cached(key, fn) {
|
||||
const hit = this.cache.get(key);
|
||||
if (hit && now() - hit.at < CACHE_MS) return hit.value;
|
||||
const value = fn();
|
||||
this.cache.set(key, { at: now(), value });
|
||||
return value;
|
||||
}
|
||||
|
||||
// Rows of this agent: its token, plus the account's untracked work if the token is that old.
|
||||
scope(alias, tokenId) {
|
||||
if (tokenId == null) return { sql: `${alias}.token_id IS NULL`, args: [] };
|
||||
const started = this.store.get('SELECT MIN(created_at) AS at FROM batches WHERE token_id IS NOT NULL')?.at ?? now();
|
||||
const created = this.store.get('SELECT created_at FROM api_tokens WHERE id = ?', tokenId)?.created_at ?? now();
|
||||
return created < started ? { sql: `(${alias}.token_id = ? OR ${alias}.token_id IS NULL)`, args: [tokenId] } : { sql: `${alias}.token_id = ?`, args: [tokenId] };
|
||||
}
|
||||
|
||||
/** The agent's record as an author, and whether it skips peer review (and from what score). */
|
||||
author(userId, tokenId = null) {
|
||||
return this.cached(`a:${userId}:${tokenId ?? '-'}`, () => {
|
||||
const p = this.policy;
|
||||
const s = this.scope('a', tokenId);
|
||||
const w = (verdict) => `(SELECT COALESCE(SUM(r.weight), 0) FROM peer_reviews r WHERE r.attempt_id = a.id AND r.verdict = '${verdict}')`;
|
||||
const r = this.store.get(
|
||||
`SELECT SUM(same >= ? AND different < ?) AS accepted, SUM(different >= ?) AS rejected FROM (
|
||||
SELECT ${w('same')} AS same, ${w('different')} AS different FROM attempts a
|
||||
WHERE a.user_id = ? AND ${s.sql} AND a.created_at > ? AND a.kind = 'code' AND a.peer_skipped = 0
|
||||
AND json_extract(a.result_json, '$.feedback.target.score') >= ?)`,
|
||||
this.autoAccept, this.required, this.required, userId, ...s.args, now() - p.authorDays * DAY, p.bandAt,
|
||||
);
|
||||
const accepted = r?.accepted ?? 0;
|
||||
const rejected = r?.rejected ?? 0;
|
||||
const score = lowerBound(accepted, accepted + rejected);
|
||||
const mode = this.store.get('SELECT submit_mode FROM users WHERE id = ?', userId)?.submit_mode ?? 'auto';
|
||||
const trusted = mode !== 'paused' && accepted + rejected >= p.authorMin && score >= p.authorTrust;
|
||||
return { accepted, rejected, score, trusted, skipAt: trusted ? p.skipAt : null, needs: { decided: p.authorMin, score: p.authorTrust, bandAt: p.bandAt } };
|
||||
});
|
||||
}
|
||||
|
||||
/** The agent's record as a reviewer, and the weight its verdicts carry (1, 2 or 3). */
|
||||
reviewer(userId, tokenId = null) {
|
||||
return this.cached(`r:${userId}:${tokenId ?? '-'}`, () => {
|
||||
const p = this.policy;
|
||||
const s = this.scope('p', tokenId);
|
||||
const rows = this.store.all(
|
||||
`SELECT p.verdict,
|
||||
(SELECT COUNT(*) FROM peer_reviews q WHERE q.attempt_id = p.attempt_id AND q.reviewer_id != p.reviewer_id) AS others,
|
||||
(SELECT COUNT(*) FROM peer_reviews q WHERE q.attempt_id = p.attempt_id AND q.reviewer_id != p.reviewer_id AND q.verdict = 'same') AS othersSame
|
||||
FROM peer_reviews p WHERE p.reviewer_id = ? AND ${s.sql} AND p.created_at > ?`,
|
||||
userId, ...s.args, now() - p.reviewerDays * DAY,
|
||||
);
|
||||
const n = { same: 0, sameAgreed: 0, different: 0, differentAgreed: 0, reviews: rows.length };
|
||||
for (const r of rows) {
|
||||
// The majority of the other reviewers (by head count, so weights never feed themselves).
|
||||
if (r.others < 2 || r.othersSame * 2 === r.others) continue;
|
||||
if (r.othersSame * 2 > r.others) {
|
||||
n.same++;
|
||||
if (r.verdict === 'same') n.sameAgreed++;
|
||||
} else {
|
||||
n.different++;
|
||||
if (r.verdict === 'different') n.differentAgreed++;
|
||||
}
|
||||
}
|
||||
const sameScore = lowerBound(n.sameAgreed, n.same);
|
||||
const differentScore = lowerBound(n.differentAgreed, n.different);
|
||||
const blocked = !!this.store.get('SELECT 1 FROM users WHERE id = ? AND peer_review_blocked = 1', userId);
|
||||
const tier = blocked ? null : p.weights.find((t) => n.different >= t.minDifferent && sameScore >= t.same && differentScore >= t.different);
|
||||
return { ...n, sameScore, differentScore, weight: tier?.weight ?? 1, tiers: p.weights };
|
||||
});
|
||||
}
|
||||
|
||||
/** Whether this function of a trusted author is accepted without peer review. */
|
||||
skips(attempt, score) {
|
||||
const a = this.author(attempt.user_id, attempt.token_id ?? null);
|
||||
if (!a.trusted || score < a.skipAt) return false;
|
||||
return Math.random() >= this.policy.sample;
|
||||
}
|
||||
|
||||
forget() {
|
||||
this.cache.clear();
|
||||
}
|
||||
}
|
||||
|
|
@ -38,7 +38,8 @@ export function bestResults(check) {
|
|||
}
|
||||
|
||||
export class Verifier {
|
||||
constructor({ store, config, repo, project, tasks, getPe, peers, log = console }) {
|
||||
constructor({ store, config, repo, project, tasks, getPe, peers, trust = null, log = console }) {
|
||||
this.trust = trust;
|
||||
this.store = store;
|
||||
this.config = config;
|
||||
this.repo = repo;
|
||||
|
|
@ -448,8 +449,8 @@ export class Verifier {
|
|||
const hash = p.result_json ? JSON.parse(p.result_json).feedback?.target?.codeHash : null;
|
||||
if (hash !== target.codeHash || !this.peers?.confirmed(p.id)) continue;
|
||||
this.store.run(
|
||||
`INSERT OR IGNORE INTO peer_reviews (attempt_id, reviewer_id, verdict, reason, agent_meta, created_at)
|
||||
SELECT ?, reviewer_id, verdict, reason, agent_meta, created_at FROM peer_reviews WHERE attempt_id = ?`,
|
||||
`INSERT OR IGNORE INTO peer_reviews (attempt_id, reviewer_id, token_id, verdict, weight, reason, agent_meta, created_at)
|
||||
SELECT ?, reviewer_id, token_id, verdict, weight, reason, agent_meta, created_at FROM peer_reviews WHERE attempt_id = ?`,
|
||||
attempt.id, p.id,
|
||||
);
|
||||
return true;
|
||||
|
|
@ -534,6 +535,14 @@ export class Verifier {
|
|||
ev.state = 'in_review';
|
||||
ev.feedback.summary += '; the compiled code is unchanged since three peers confirmed it, so their verdicts carry over';
|
||||
}
|
||||
// A trusted author's function that is close to the original: accepted without peer review
|
||||
// (but for a random sample that keeps the author's record measured; trust.js).
|
||||
if (ev.state === 'peer_review' && a.kind !== 'skip' && this.trust?.skips(a, ev.score)) {
|
||||
ev.state = 'approved';
|
||||
ev.feedback.summary = `not byte-identical (score ${(ev.score * 100).toFixed(1)}%), accepted without peer review: this agent's functions keep passing peer review (trust)`;
|
||||
ev.feedback.trusted = true;
|
||||
this.store.run('UPDATE attempts SET peer_skipped = 1 WHERE id = ?', a.id);
|
||||
}
|
||||
if (ev.state === 'needs_cleanup' || ev.state === 'rejected') failed++;
|
||||
this.finish(a, ev.state, ev.score, ev.feedback);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -700,6 +700,36 @@ function renderQuality(q) {
|
|||
box.append(el('p', 'Each agent is judged on its own. Half of its functions sent back by peers (20+ reviewed) or 60% broken (50+ submitted) limits it; 70% sent back (30+) or 85% broken (80+) pauses it. Both lift by themselves as its last 24 hours get better.', 'hint'));
|
||||
}
|
||||
|
||||
// Short form for tables: skips peer review / verdict weight.
|
||||
function trustCell(t) {
|
||||
if (!t) return '-';
|
||||
const parts = [];
|
||||
if (t.author.trusted) parts.push(el('div', `skips review at ${Math.round(t.author.skipAt * 100)}%+`, 'status active'));
|
||||
if (t.reviewer.weight > 1) parts.push(el('div', `verdicts count ×${t.reviewer.weight}`, 'status active'));
|
||||
parts.push(el('div', `author ${Math.round(t.author.score * 100)}% · reviewer ${Math.round(t.reviewer.sameScore * 100)}/${Math.round(t.reviewer.differentScore * 100)}%`, 'hint'));
|
||||
return parts;
|
||||
}
|
||||
|
||||
// What the agent has earned: skipping peer review as an author, more weight as a reviewer.
|
||||
function trustBlock(t) {
|
||||
const pct = (x) => `${Math.round(x * 100)}%`;
|
||||
const box = el('div', null, 'trust');
|
||||
const au = t.author;
|
||||
const rv = t.reviewer;
|
||||
const authorLine = au.trusted
|
||||
? `Trusted author: its functions scoring ${pct(au.skipAt)} or more are accepted without peer review (one in ${Math.round(1 / t.sample)} still goes to peers, to keep the record measured).`
|
||||
: `Not yet a trusted author. Of its functions scoring ${pct(au.needs.bandAt)}+ that peers decided (14 days), ${fmt(au.accepted)} were accepted and ${fmt(au.rejected)} sent back: ${pct(au.score)} at the least (needs ${pct(au.needs.score)} over ${au.needs.decided}+ functions).`;
|
||||
const tiers = rv.tiers.map((x) => `×${x.weight}: ${pct(x.same)} / ${pct(x.different)} over ${x.minDifferent}+ differences`).join(', ');
|
||||
const reviewerLine = `Reviewer: agrees with the other reviewers ${pct(rv.sameScore)} of the time when they found it the same (${fmt(rv.same)}), ${pct(rv.differentScore)} when they found a difference (${fmt(rv.different)}), at the least (30 days). ${rv.weight > 1 ? `Its verdicts count ×${rv.weight}.` : `Its verdicts count once (${tiers}).`}`;
|
||||
box.append(
|
||||
el('div', null, 'quality-head'),
|
||||
el('p', authorLine, au.trusted ? 'good' : 'hint'),
|
||||
el('p', reviewerLine, rv.weight > 1 ? 'good' : 'hint'),
|
||||
);
|
||||
box.firstChild.append(el('span', 'Trust', 'agent-subtitle'), ...(au.trusted ? [el('span', 'skips review', 'status active')] : []), ...(rv.weight > 1 ? [el('span', `reviews ×${rv.weight}`, 'status active')] : []));
|
||||
return box;
|
||||
}
|
||||
|
||||
function agentCard(q) {
|
||||
const pct = (x) => `${Math.round(x * 100)}%`;
|
||||
const card = el('div', null, `agent-card ${q.level}`);
|
||||
|
|
@ -723,6 +753,7 @@ function agentCard(q) {
|
|||
['Broken or refused', q.submitted ? `${pct(q.brokenRate)} of ${fmt(q.submitted)}${trend(q.brokenRate, prev.submitted ? prev.brokenRate : null)}` : '-'],
|
||||
]),
|
||||
);
|
||||
if (q.trust) card.append(trustBlock(q.trust));
|
||||
if (q.kinds.length) card.append(el('p', `What broke: ${q.kinds.map((k) => `${fmt(k.functions)} ${k.label}`).join(' · ')}`, 'muted'));
|
||||
if (q.sentBack.length) {
|
||||
const d = el('details', null, 'fold');
|
||||
|
|
@ -842,10 +873,10 @@ function renderAuthors(authors) {
|
|||
$('authors').replaceChildren(
|
||||
authors.length
|
||||
? table(
|
||||
['Author', 'Agent', 'Level', 'Sent back by peers', 'Broken or refused', 'Why', 'Submissions (account)'],
|
||||
['Author', 'Agent', 'Level', 'Trust', 'Sent back by peers', 'Broken or refused', 'Why', 'Submissions (account)'],
|
||||
authors.map((a) => {
|
||||
const modes = el('div', null, 'row-actions');
|
||||
for (const [mode, label] of [['auto', 'Auto'], ['trusted', 'Trust'], ['paused', 'Pause']]) {
|
||||
for (const [mode, label] of [['auto', 'Auto'], ['trusted', 'Never limit'], ['paused', 'Pause']]) {
|
||||
const b = button(label, async () => {
|
||||
if (mode === 'paused' && !confirm(`Pause submissions from ${a.name}? Their agent can still review and read, but not submit or claim regions.`)) return;
|
||||
await adminAction(a.name, `submit-${mode}`);
|
||||
|
|
@ -857,6 +888,7 @@ function renderAuthors(authors) {
|
|||
a.name,
|
||||
a.token ?? el('span', 'before agents were told apart', 'hint'),
|
||||
[chip(a.level), ...(a.mode !== 'auto' ? [el('div', `set: ${a.mode}`, 'hint')] : [])],
|
||||
trustCell(a.trust),
|
||||
a.peerDecided ? `${pct(a.peerRate)} of ${a.peerDecided}` : '-',
|
||||
a.submitted ? `${pct(a.brokenRate)} of ${a.submitted}` : '-',
|
||||
el('div', a.reasons.join('; ') || '', 'hint'),
|
||||
|
|
|
|||
|
|
@ -752,3 +752,8 @@ details.fold-plate[open] > summary { margin-bottom: 12px; }
|
|||
.agent-card.limited { border-left-color: var(--warn); }
|
||||
.agent-card.paused { border-left-color: var(--danger); }
|
||||
.agent-title { font: 700 15px var(--font-display); text-transform: uppercase; letter-spacing: 0.06em; }
|
||||
|
||||
.trust { margin: 8px 0; padding-top: 6px; border-top: 1px dashed var(--line); }
|
||||
.trust p { margin: 4px 0; }
|
||||
.trust .good { color: var(--ok); }
|
||||
.agent-subtitle { font: 700 12px var(--font-display); text-transform: uppercase; letter-spacing: 0.08em; color: var(--muted); }
|
||||
|
|
|
|||
|
|
@ -1336,3 +1336,65 @@ test('each agent (token) is judged on its own; the owner resets one agent once a
|
|||
assert.equal((await request('POST', '/api/me/quality/reset', { tokenId: 999999 }, { cookie })).status, 404);
|
||||
ctx.store.run(`UPDATE attempts SET state = 'rejected' WHERE user_id = ?`, author.id);
|
||||
});
|
||||
|
||||
test('trust: careful reviewers count more, trusted authors skip peer review, one reviewer never decides', async () => {
|
||||
const task = ctx.store.get('SELECT id FROM tasks LIMIT 1').id;
|
||||
const head = ctx.repo.head();
|
||||
const mk = (n, prefix) => Promise.all(Array.from({ length: n }, (_, i) => activeUser(`${prefix}${i}`)));
|
||||
const reviewerToken = await activeUser('trustrev');
|
||||
const reviewer = ctx.accounts.userByApiToken(reviewerToken);
|
||||
const authorToken = await activeUser('trustauthor');
|
||||
const author = ctx.accounts.userByApiToken(authorToken);
|
||||
await mk(3, 'trustpeer');
|
||||
const peers = [0, 1, 2].map((i) => ctx.accounts.byName(`trustpeer${i}`));
|
||||
const attempt = (userId, tokenId, score, state = 'merged') =>
|
||||
Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, token_id, base_commit, files_json, state, result_json, created_at) VALUES (?, ?, ?, ?, '{}', ?, ?, ?)`,
|
||||
task, userId, tokenId, head, state, JSON.stringify({ feedback: { target: { score } } }), now()).lastInsertRowid);
|
||||
const verdict = (a, userId, v, tokenId = null) =>
|
||||
ctx.store.run(`INSERT INTO peer_reviews (attempt_id, reviewer_id, token_id, verdict, reason, created_at) VALUES (?, ?, ?, ?, 'the loop bound and the call match the original', ?)`, a, userId, tokenId, v, now());
|
||||
|
||||
// A reviewer who agrees with the others both ways: 200 the same, 40 differences found too.
|
||||
for (let i = 0; i < 240; i++) {
|
||||
const v = i < 200 ? 'same' : 'different';
|
||||
const a = attempt(author.id, null, 0.8, 'rejected');
|
||||
verdict(a, peers[0].id, v);
|
||||
verdict(a, peers[1].id, v);
|
||||
verdict(a, reviewer.id, v, reviewer.tokenId);
|
||||
}
|
||||
ctx.trust.forget();
|
||||
assert.equal(ctx.trust.reviewer(reviewer.id, reviewer.tokenId).weight, 2);
|
||||
assert.equal(ctx.trust.reviewer(peers[2].id).weight, 1, 'no record, weight 1');
|
||||
|
||||
// A weighty "different" alone does not send a function back; a second reviewer does.
|
||||
const open = attempt(author.id, author.tokenId, 0.9, 'peer_review');
|
||||
const r1 = ctx.peers.submit(reviewer, open, 'different', 'the original checks the flag before the loop', null);
|
||||
assert.equal(r1.weight, 2);
|
||||
assert.equal(r1.state, 'peer_review');
|
||||
assert.equal(ctx.peers.submit(peers[2], open, 'different', 'the original returns early when the list is empty', null).state, 'needs_cleanup');
|
||||
|
||||
// An author whose close matches keep passing: trusted; close functions skip review.
|
||||
ctx.store.run(`UPDATE attempts SET state = 'superseded' WHERE user_id = ?`, author.id);
|
||||
ctx.store.run(`DELETE FROM peer_reviews WHERE attempt_id IN (SELECT id FROM attempts WHERE user_id = ?)`, author.id);
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const a = attempt(author.id, author.tokenId, 0.97);
|
||||
for (const p of [...peers, reviewer]) verdict(a, p.id, 'same');
|
||||
verdict(a, ctx.accounts.byName('rsreviewer1').id, 'same');
|
||||
}
|
||||
ctx.trust.forget();
|
||||
const t = ctx.trust.author(author.id, author.tokenId);
|
||||
assert.equal(t.trusted, true, JSON.stringify(t));
|
||||
ctx.project.matchPolicy.trust = { sample: 0 };
|
||||
try {
|
||||
assert.equal(ctx.trust.skips({ user_id: author.id, token_id: author.tokenId }, 0.96), true);
|
||||
assert.equal(ctx.trust.skips({ user_id: author.id, token_id: author.tokenId }, 0.9), false, 'below skipAt: peers review it');
|
||||
assert.equal(ctx.trust.skips({ user_id: author.id, token_id: null }, 0.99), false, 'another agent of the account earns its own');
|
||||
} finally {
|
||||
delete ctx.project.matchPolicy.trust;
|
||||
}
|
||||
const skipped = attempt(author.id, author.tokenId, 0.97, 'approved');
|
||||
ctx.store.run('UPDATE attempts SET peer_skipped = 1 WHERE id = ?', skipped);
|
||||
assert.equal(ctx.peers.confirmed(skipped), true);
|
||||
const stats = (await call('GET', '/api/me/quality', null, authorToken)).body;
|
||||
assert.equal(stats.trust.author.trusted, true);
|
||||
ctx.store.run(`UPDATE attempts SET state = 'superseded' WHERE user_id = ?`, author.id);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue