Trust: proven authors skip peer review, careful reviewers count more
Each agent (token) earns trust from its record (trust.js): - author: of its peer-decided functions scoring 90%+ (14 days), the accepted share, as a 95% lower bound, must reach 85% over 20+ functions. Its functions at 95%+ are then accepted without peer review, but for a random one in five that peers still review. - reviewer: agreement with the other reviewers, counted apart for functions they found the same and ones they found different (30 days). 85%/65% over 30+ differences counts x2, 92%/85% over 60+ x3. Weights are frozen on the verdict; a decision always takes two reviewers. Skipped functions and own verdicts never feed the record. Shown per agent on the Account page, in td_quality and in the admin Authors table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
01538a93d5
commit
7120dcaebc
10 changed files with 69 additions and 35 deletions
|
|
@ -308,7 +308,7 @@ test('changing the password logs out browser sessions', async () => {
|
|||
await login('erin', 'a brand new password');
|
||||
});
|
||||
|
||||
test('peer review: five "same" accept, three "different" send back, by reading only', async () => {
|
||||
test('peer review: four "same" accept, three "different" send back, by reading only', async () => {
|
||||
const author = await activeUser('pauthor');
|
||||
const r1 = await activeUser('peer1');
|
||||
const r2 = await activeUser('peer2');
|
||||
|
|
@ -332,14 +332,14 @@ test('peer review: five "same" accept, three "different" send back, by reading o
|
|||
assert.ok(!(await call('GET', '/api/peer-reviews', null, author)).body.some((x) => x.attempt_id === attempt), 'authors do not see their own attempts');
|
||||
const claim = await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r1);
|
||||
assert.equal(claim.status, 200);
|
||||
assert.equal(claim.body.required, 5);
|
||||
assert.equal(claim.body.required, 4);
|
||||
assert.equal(claim.body.rejectAt, 3);
|
||||
assert.ok(claim.body.instructions.some((i) => /Do not compile/.test(i)));
|
||||
assert.equal(JSON.stringify(claim.body).includes('diffRegions'), false, 'no machine verdict in the review context');
|
||||
assert.equal(claim.body.original.read[0], `td_disassemble ${task.symbol_id}`);
|
||||
|
||||
for (const r of [r2, r3, r4, r5]) await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r);
|
||||
assert.equal((await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r6)).status, 409, 'no more reviewers than needed');
|
||||
for (const r of [r2, r3, r4]) await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r);
|
||||
assert.equal((await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r5)).status, 409, 'no more reviewers than needed');
|
||||
|
||||
assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'short' }, r1)).status, 400);
|
||||
assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'all branches and calls agree' }, author)).status, 403);
|
||||
|
|
@ -347,9 +347,7 @@ test('peer review: five "same" accept, three "different" send back, by reading o
|
|||
assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'same loop bounds and the same return' }, r1)).status, 409, 'one verdict per account');
|
||||
assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'checked every condition and field access' }, r2)).body.state, 'peer_review');
|
||||
assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'identical behavior, only register use differs' }, r3)).body.state, 'peer_review', 'three are not enough to accept');
|
||||
assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'same calls in the same order, same result' }, r4)).body.state, 'peer_review');
|
||||
const done = await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'same fields read and written, same return' }, r5);
|
||||
assert.equal(done.body.state, 'approved', 'five accept it, with no human step');
|
||||
assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'same calls in the same order, same result' }, r4)).body.state, 'approved', 'four accept it while nobody objected, with no human step');
|
||||
assert.equal(ctx.tasks.get(task.id).state, 'approved');
|
||||
assert.ok(ctx.peers.confirmed(attempt));
|
||||
|
||||
|
|
@ -431,7 +429,7 @@ test('a header batch is verified on its own units and merged with every includer
|
|||
}
|
||||
});
|
||||
|
||||
test('five agreeing peers accept a function without human review, also over an objection', async () => {
|
||||
test('agreeing peers accept a function without human review (four, or five over an objection)', async () => {
|
||||
await activeUser('aaauthor');
|
||||
const author = ctx.accounts.byName('aaauthor');
|
||||
const peers = [];
|
||||
|
|
@ -452,8 +450,7 @@ test('five agreeing peers accept a function without human review, also over an o
|
|||
const clean = make(0x401900);
|
||||
for (let i = 0; i < 3; i++) await review(clean.attempt, peers[i], 'same');
|
||||
assert.equal(ctx.store.get('SELECT state FROM attempts WHERE id = ?', clean.attempt).state, 'peer_review', 'three are not enough');
|
||||
assert.equal((await review(clean.attempt, peers[3], 'same')).state, 'peer_review', 'it stays open for more reviewers');
|
||||
assert.equal((await review(clean.attempt, peers[4], 'same')).state, 'approved', 'five accept it');
|
||||
assert.equal((await review(clean.attempt, peers[3], 'same')).state, 'approved', 'four accept it');
|
||||
assert.equal(ctx.store.get('SELECT state FROM tasks WHERE id = ?', clean.task).state, 'approved');
|
||||
|
||||
const contested = make(0x401940);
|
||||
|
|
@ -470,9 +467,9 @@ test('five agreeing peers accept a function without human review, also over an o
|
|||
});
|
||||
|
||||
test('reviewer stats and pausing a reviewer', async () => {
|
||||
// The auto-accept test left verdicts from aapeer1..5; aapeer1 objected on the contested function.
|
||||
// The auto-accept test left verdicts from aapeer1..6; aapeer1 objected on the contested function.
|
||||
const stats = Object.fromEntries(reviewerStats(ctx.store).map((r) => [r.name, r]));
|
||||
assert.ok(stats.aapeer2.reviews >= 2);
|
||||
assert.ok(stats.aapeer2.reviews >= 2, JSON.stringify(stats.aapeer2));
|
||||
assert.equal(stats.aapeer1.different, 1);
|
||||
assert.ok(stats.aapeer2.peerCompared >= 1);
|
||||
|
||||
|
|
@ -1398,3 +1395,23 @@ test('trust: careful reviewers count more, trusted authors skip peer review, one
|
|||
assert.equal(stats.trust.author.trusted, true);
|
||||
ctx.store.run(`UPDATE attempts SET state = 'superseded' WHERE user_id = ?`, author.id);
|
||||
});
|
||||
|
||||
test('four "same" accept a function nobody objected to; five once someone did', async () => {
|
||||
const task = ctx.store.get('SELECT id FROM tasks LIMIT 1').id;
|
||||
const author = ctx.accounts.byName('trustauthor');
|
||||
const revs = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await activeUser(`fourrev${i}`);
|
||||
revs.push(ctx.accounts.byName(`fourrev${i}`));
|
||||
}
|
||||
const open = () => Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, base_commit, files_json, state, created_at) VALUES (?, ?, ?, '{}', 'peer_review', ?)`, task, author.id, ctx.repo.head(), now()).lastInsertRowid);
|
||||
const why = 'the branches, the call and the return value match the original';
|
||||
const clean = open();
|
||||
for (let i = 0; i < 3; i++) assert.equal(ctx.peers.submit(revs[i], clean, 'same', why, null).state, 'peer_review');
|
||||
assert.equal(ctx.peers.submit(revs[3], clean, 'same', why, null).state, 'approved');
|
||||
const disputed = open();
|
||||
ctx.peers.submit(revs[0], disputed, 'different', 'the original checks the flag before the loop', null);
|
||||
for (let i = 1; i < 4; i++) assert.equal(ctx.peers.submit(revs[i], disputed, 'same', why, null).state, 'peer_review');
|
||||
assert.equal(ctx.peers.submit(revs[4], disputed, 'same', why, null).state, 'peer_review', 'four "same" against one "different": not yet');
|
||||
ctx.store.run(`UPDATE attempts SET state = 'superseded' WHERE id IN (?, ?)`, clean, disputed);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue