diff --git a/src/server/app.js b/src/server/app.js
index b3bce05..525a23c 100644
--- a/src/server/app.js
+++ b/src/server/app.js
@@ -24,6 +24,7 @@ import { Router, HttpError, readJson, sendJson, RateLimiter } from './http.js';
import { Regions } from './regions.js';
import { Credits, SLOT_TABLE } from './credits.js';
import { Live } from './live.js';
+import { Quality, LIMITS } from './quality.js';
import { PEImage } from '../shared/pe.js';
import { loadProject, loadToolchainLock } from '../shared/project.js';
import { DEFAULT_LIMITS, lintSubmission } from '../shared/lint.js';
@@ -84,6 +85,9 @@ export function openServer(dataDir, { log = console } = {}) {
const tasks = new Tasks({ store, config, repo, project });
const regions = new Regions({ store, config, repo, tasks });
tasks.regions = regions;
+ const quality = new Quality({ store });
+ tasks.quality = quality;
+ regions.quality = quality;
const peers = new PeerReviews({ store, project });
const verifier = new Verifier({ store, config, repo, project, tasks, getPe, peers, log });
verifier.regions = regions;
@@ -94,7 +98,7 @@ export function openServer(dataDir, { log = console } = {}) {
merger.onMerged = (commit) => ghidra.scheduleRefresh(commit);
const checks = new CheckQueue({ store, repo, verifier, config, log });
const credits = new Credits({ store });
- return { config, store, repo, project, tasks, regions, credits, verifier, merger, getPe, accounts, peers, originals, ghidra, checks };
+ return { config, store, repo, project, tasks, regions, quality, credits, verifier, merger, getPe, accounts, peers, originals, ghidra, checks };
}
export function createApp(ctx) {
@@ -312,6 +316,10 @@ export function createApp(ctx) {
leasedAllowedPaths: leases.length ? tasks.allowedPathsForTasks(leases) : [],
regions: ctx.regions.mine(user).map((r) => ({ kind: r.kind, key: r.key, expires: r.expires_at })),
regionTasks: shared,
+ quality: (() => {
+ const q = ctx.quality.of(user.id);
+ return { level: q.level, reasons: q.reasons, ...(q.level !== 'ok' ? { note: Quality.advice(q) } : {}) };
+ })(),
regionAllowedPaths: shared.length ? tasks.allowedPathsForTasks(shared.slice(0, 500).map((t) => t.taskId)) : [],
};
});
@@ -928,6 +936,10 @@ export function createApp(ctx) {
return tasks.get(t.id);
});
+ router.on('GET', '/api/admin/authors', ({ req }) => {
+ auth(req, { role: 'admin', session: true });
+ return { authors: ctx.quality.authors(), limits: LIMITS };
+ });
router.on('GET', '/api/admin/users', ({ req, query }) => {
auth(req, { role: 'admin', session: true });
return { roles: ROLES, users: accounts.listUsers(query.get('status') ?? undefined), work: claimedWork(), reviewers: reviewerStats(store) };
@@ -965,8 +977,20 @@ export function createApp(ctx) {
'block-reviews': (name, admin) => setReviewBlock(name, 1, admin),
'allow-reviews': (name, admin) => setReviewBlock(name, 0, admin),
delete: (name, admin) => accounts.deleteUser(name, admin.id),
+ // Submissions: auto (limited by recent quality), trusted (never limited) or paused.
+ 'submit-auto': (name) => setSubmitMode(name, 'auto'),
+ 'submit-trusted': (name) => setSubmitMode(name, 'trusted'),
+ 'submit-paused': (name) => setSubmitMode(name, 'paused'),
'reset-password': (name, admin) => accounts.resetPassword(name, admin.id),
};
+ const setSubmitMode = (name, mode) => {
+ const u = accounts.byName(name);
+ if (!u) throw new HttpError(404, 'no such user');
+ store.run('UPDATE users SET submit_mode = ? WHERE id = ?', mode, u.id);
+ ctx.quality.forget(u.id);
+ store.event('submit_mode', null, { name, mode });
+ return { name, mode };
+ };
const setReviewBlock = (name, blocked, admin) => {
const u = accounts.byName(name);
if (!u) throw new HttpError(404, 'no such user');
diff --git a/src/server/db.js b/src/server/db.js
index bf6f482..38280e4 100644
--- a/src/server/db.js
+++ b/src/server/db.js
@@ -300,6 +300,7 @@ export class Store {
this.addColumn('check_jobs', 'files_hash', 'TEXT'); // a retry of the same check joins the queued one
this.addColumn('batches', 'cost', 'INTEGER'); // units its verification compiles (queue order)
this.addColumn('users', 'peer_review_blocked', 'INTEGER NOT NULL DEFAULT 0'); // no peer reviews from this account
+ this.addColumn('users', 'submit_mode', "TEXT NOT NULL DEFAULT 'auto'"); // auto (quality.js) | trusted | paused
// "No decompile" claims: an attempt that says a function needs no source (peer reviewed like code).
this.addColumn('attempts', 'kind', "TEXT NOT NULL DEFAULT 'code'"); // code | skip
this.addColumn('attempts', 'skip_category', 'TEXT');
diff --git a/src/server/guidance.js b/src/server/guidance.js
index 96bacd1..b2662ef 100644
--- a/src/server/guidance.js
+++ b/src/server/guidance.js
@@ -12,6 +12,7 @@ export const DEFAULT_GUIDANCE = [
'Port, do not polish. Write faithful C++ for each function: the same logic, calls, conditions, loops and data accesses, with sensible names. Clear compile errors and structural differences (a missing branch, a wrong call, a wrong field), then move on even when it is not byte-identical.',
'Do not spend more than two or three checks per function chasing exact bytes: five other agents confirm the logic in peer review, which is the normal path. Byte-identical code merges right away.',
'Peer reviews come in groups (up to 40 functions of one batch and file): read the original and the submission (it is known to compile; do not compile or diff), and send all verdicts in one td_submit_peer_review call. Five "same" merge a function, three "different" send it back.',
+ 'Quality counts: an account whose functions peers keep sending back, or whose submissions keep breaking, is limited and then paused for a while automatically (the quality field of td_get_batch / /api/me says why). Fewer, checked submissions beat many careless ones.',
'If your code (or someone else\'s) breaks a neighbouring function so a region does not compile, claim that function too (td_claim_region covers it when it is in your region) and fix it in the same submission.',
'Some functions need no decompilation (compiler-generated thunks and destructors, runtime library code, static initializers, non-code): mark them with td_mark_no_decompile and a reason; other agents review the claim. Do not stack a second annotation on adjustor thunks or deleting destructors.',
'Functions in no source file yet: decide which file each belongs to (and its name if you can tell) with td_place, then port them.',
diff --git a/src/server/quality.js b/src/server/quality.js
new file mode 100644
index 0000000..6e68502
--- /dev/null
+++ b/src/server/quality.js
@@ -0,0 +1,107 @@
+// How good an author's recent work is, and what that allows. An agent that keeps submitting broken
+// code, or code peers keep sending back, slows down by itself instead of filling the queues:
+//
+// ok normal limits
+// limited one batch waiting at a time, six submissions an hour, one region at a time
+// paused no submissions or region claims
+//
+// The level follows the last 24 hours, so it recovers by itself once the work gets better (or the
+// bad stretch ages out). Admins can set an account to 'trusted' (never limited) or 'paused'.
+
+import { now } from './db.js';
+
+const WINDOW_MS = 24 * 3600_000;
+const CACHE_MS = 2 * 60_000;
+
+// Feedback summaries of submissions that broke something or were refused before compiling.
+const BROKEN = [
+ 'compilation failed',
+ 'your change breaks %',
+ 'rejected%',
+ 'declares functions that are defined nowhere%',
+ 'removes the annotation of%',
+ 'annotation errors',
+ 'base commit % is not part of main%',
+];
+
+export const LIMITS = {
+ limited: { queuedBatches: 1, submitsPerHour: 6, regions: 1 },
+};
+
+export class Quality {
+ constructor({ store }) {
+ this.store = store;
+ this.cache = new Map();
+ }
+
+ /** The author's recent numbers and the level they lead to (cached briefly). */
+ of(userId) {
+ const hit = this.cache.get(userId);
+ if (hit && now() - hit.at < CACHE_MS) return hit.value;
+ const value = this.compute(userId);
+ this.cache.set(userId, { at: now(), value });
+ return value;
+ }
+
+ compute(userId) {
+ const since = now() - WINDOW_MS;
+ const mode = this.store.get('SELECT submit_mode FROM users WHERE id = ?', userId)?.submit_mode ?? 'auto';
+ const brokenSql = BROKEN.map(() => "json_extract(a.result_json, '$.feedback.summary') LIKE ?").join(' OR ');
+ const r = this.store.get(
+ `SELECT COUNT(*) AS submitted,
+ SUM(CASE WHEN ${brokenSql} THEN 1 ELSE 0 END) AS broken,
+ SUM((SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3) AS peerRejected,
+ SUM((SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id) >= 3) AS peerDecided
+ FROM attempts a
+ WHERE a.user_id = ? AND a.created_at > ? AND a.kind = 'code' AND a.state NOT IN ('withdrawn', 'superseded', 'queued', 'verifying')`,
+ ...BROKEN,
+ userId,
+ since,
+ );
+ const submitted = r?.submitted ?? 0;
+ const broken = r?.broken ?? 0;
+ const decided = r?.peerDecided ?? 0;
+ const rejected = r?.peerRejected ?? 0;
+ const peerRate = decided ? rejected / decided : 0;
+ const brokenRate = submitted ? broken / submitted : 0;
+ const reasons = [];
+ let level = 'ok';
+ const raise = (to, why) => {
+ if (to === 'paused' || level === 'ok') level = to;
+ reasons.push(why);
+ };
+ if (decided >= 30 && peerRate >= 0.7) raise('paused', `${Math.round(peerRate * 100)}% of your reviewed functions were sent back by peers (${rejected} of ${decided})`);
+ else if (decided >= 20 && peerRate >= 0.5) raise('limited', `${Math.round(peerRate * 100)}% of your reviewed functions were sent back by peers (${rejected} of ${decided})`);
+ if (submitted >= 80 && brokenRate >= 0.85) raise('paused', `${Math.round(brokenRate * 100)}% of your submitted functions did not compile, broke other functions or were refused (${broken} of ${submitted})`);
+ else if (submitted >= 50 && brokenRate >= 0.6) raise('limited', `${Math.round(brokenRate * 100)}% of your submitted functions did not compile, broke other functions or were refused (${broken} of ${submitted})`);
+ if (mode === 'trusted') level = 'ok';
+ if (mode === 'paused') {
+ level = 'paused';
+ reasons.unshift('a maintainer paused submissions from this account');
+ }
+ return { level, mode, reasons, submitted, broken, peerDecided: decided, peerRejected: rejected, peerRate, brokenRate, windowHours: 24 };
+ }
+
+ forget(userId) {
+ this.cache.delete(userId);
+ }
+
+ /** What an agent is told when its level limits it. */
+ static advice(q) {
+ const what = q.level === 'paused' ? 'Submissions and region claims are paused for this account' : 'This account is limited to one batch waiting at a time, six submissions an hour and one region';
+ return `${what}: ${q.reasons.join('; ')}. It lifts by itself as your recent work gets better (last 24 hours). Port each function faithfully from the disassembly, td_check it until it compiles and matches structurally, and submit only what you checked.`;
+ }
+
+ /** Every author with work in the window, for the admin view. */
+ authors() {
+ const ids = this.store.all(`SELECT DISTINCT user_id FROM attempts WHERE created_at > ?`, now() - WINDOW_MS).map((r) => r.user_id);
+ const out = [];
+ for (const id of ids) {
+ const u = this.store.get('SELECT name, status FROM users WHERE id = ?', id);
+ if (!u) continue;
+ out.push({ name: u.name, status: u.status, ...this.of(id) });
+ }
+ const rank = { paused: 0, limited: 1, ok: 2 };
+ return out.sort((a, b) => rank[a.level] - rank[b.level] || b.peerRate - a.peerRate || b.submitted - a.submitted);
+ }
+}
diff --git a/src/server/regions.js b/src/server/regions.js
index 0e25f4d..e7dd189 100644
--- a/src/server/regions.js
+++ b/src/server/regions.js
@@ -108,6 +108,12 @@ export class Regions {
if (!['unit', 'class'].includes(kind)) throw new HttpError(400, "kind must be 'unit' or 'class'");
key = String(key ?? '').trim();
if (!key || key.length > 300) throw new HttpError(400, 'key must name the source file (kind unit) or the class (kind class)');
+ const q = this.quality?.of(user.id);
+ if (q?.level === 'paused') throw new HttpError(429, this.quality.constructor.advice(q), { retryAfterSeconds: 3600 });
+ if (q?.level === 'limited') {
+ const other = this.mine(user).filter((x) => !(x.kind === kind && x.key === key));
+ if (other.length) throw new HttpError(409, `${this.quality.constructor.advice(q)} Finish or release ${other[0].key} first.`);
+ }
return this.store.tx(() => {
const members = this.members(kind, key);
if (!members.length) throw new HttpError(404, `no functions in ${kind} ${key}`);
diff --git a/src/server/tasks.js b/src/server/tasks.js
index 706458f..cc17e8c 100644
--- a/src/server/tasks.js
+++ b/src/server/tasks.js
@@ -307,6 +307,13 @@ export class Tasks {
if (ids.length > this.maxLeased) throw new HttpError(400, `at most ${this.maxLeased} tasks per submission`);
return this.store.tx(() => {
const own = new Set(ids.filter((id) => this.requireSubmittable(user, id).own));
+ // Authors whose recent work keeps breaking or being sent back slow down (quality.js).
+ const q = this.quality?.of(user.id);
+ if (q?.level === 'paused') throw new HttpError(429, this.quality.constructor.advice(q), { retryAfterSeconds: 3600, quality: q.level });
+ if (q?.level === 'limited') {
+ const lastHour = this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND created_at > ? AND state != 'skip'`, user.id, now() - 3600_000).n;
+ if (lastHour >= 6) throw new HttpError(429, this.quality.constructor.advice(q), { retryAfterSeconds: 600, quality: q.level });
+ }
// An agent stuck in a loop (every submission refused before it compiled: not synced, conflicts,
// comments) pauses until it has fixed that, instead of filling the queue every few seconds.
const recent = this.store.all(
@@ -320,7 +327,7 @@ export class Tasks {
throw new HttpError(429, `your last ${recent.length} submissions were all refused before compiling (latest: ${refused[0].why}). Run td_sync, merge, fix that and check locally (td_check) before submitting again`, { retryAfterSeconds: 600 });
}
// Keep the verification queue moving: a few batches waiting per account at a time.
- const max = this.config.maxQueuedBatches ?? 8;
+ const max = q?.level === 'limited' ? 1 : this.config.maxQueuedBatches ?? 8;
const waiting = this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND state IN ('queued', 'verifying')`, user.id).n;
if (waiting >= max) {
throw new HttpError(429, `you have ${waiting} batches waiting for verification (the limit is ${max}); keep your claims, do peer reviews meanwhile (td_list_peer_reviews), and submit again when one is done (td_get_batch shows it)`, { retryAfterSeconds: 120 });
diff --git a/src/server/web/app.js b/src/server/web/app.js
index d57358b..bd9b402 100644
--- a/src/server/web/app.js
+++ b/src/server/web/app.js
@@ -752,8 +752,41 @@ async function adminAction(name, action, body) {
let adminData = null;
async function loadAdmin() {
- adminData = await api('/api/admin/users');
+ const [users, authors] = await Promise.all([api('/api/admin/users'), api('/api/admin/authors').catch(() => ({ authors: [] }))]);
+ adminData = users;
renderAdmin();
+ renderAuthors(authors.authors);
+}
+
+function renderAuthors(authors) {
+ const pct = (a) => `${Math.round(a * 100)}%`;
+ const chip = (level) => el('span', level, `status ${level === 'ok' ? 'active' : level === 'limited' ? 'pending' : 'revoked'}`);
+ $('authors').replaceChildren(
+ authors.length
+ ? table(
+ ['Author', 'Level', 'Sent back by peers', 'Broken or refused', 'Why', 'Submissions'],
+ authors.map((a) => {
+ const modes = el('div', null, 'row-actions');
+ for (const [mode, label] of [['auto', 'Auto'], ['trusted', 'Trust'], ['paused', 'Pause']]) {
+ const b = button(label, async () => {
+ if (mode === 'paused' && !confirm(`Pause submissions from ${a.name}? Their agent can still review and read, but not submit or claim regions.`)) return;
+ await adminAction(a.name, `submit-${mode}`);
+ }, mode === 'paused' ? 'danger' : '');
+ b.disabled = a.mode === mode;
+ modes.append(b);
+ }
+ return [
+ a.name,
+ [chip(a.level), ...(a.mode !== 'auto' ? [el('div', `set: ${a.mode}`, 'hint')] : [])],
+ a.peerDecided ? `${pct(a.peerRate)} of ${a.peerDecided}` : '-',
+ a.submitted ? `${pct(a.brokenRate)} of ${a.submitted}` : '-',
+ el('div', a.reasons.join('; ') || '', 'hint'),
+ modes,
+ ];
+ }),
+ )
+ : el('p', 'No submissions in the last 24 hours.', 'muted'),
+ );
}
function renderAdmin() {
diff --git a/src/server/web/index.html b/src/server/web/index.html
index 62c396b..a2b901c 100644
--- a/src/server/web/index.html
+++ b/src/server/web/index.html
@@ -346,6 +346,11 @@
+
+ Authors
+ The last 24 hours of each author's submissions. Authors whose functions peers keep sending back, or whose submissions keep breaking (compile errors, regressions, refusals), are limited (one batch waiting, six submissions an hour, one region) or paused automatically; it lifts by itself as their work gets better. Trusted accounts are never limited; paused ones cannot submit until you set them back to auto.
+
+
Peer reviewers
Agreement with the other reviewers of the same function and with human decisions, and how many verdicts came within 10 seconds of a verdict on another batch (a sign of not reading). Pausing someone stops new peer reviews from them; their verdicts so far stay.
diff --git a/test/server.test.js b/test/server.test.js
index adccb34..6339a87 100644
--- a/test/server.test.js
+++ b/test/server.test.js
@@ -1251,3 +1251,45 @@ test('three "different" send a function back even when five said "same"', async
assert.equal(r.body.state, 'needs_cleanup', 'five "same" do not outvote three "different"');
ctx.store.run(`UPDATE tasks SET state = 'done' WHERE id = ?`, task);
});
+
+test('authors whose work peers keep sending back are limited, then paused; trusted ones are not', async () => {
+ const token = await activeUser('qaauthor');
+ const author = ctx.accounts.byName('qaauthor');
+ const reviewers = [];
+ for (let i = 1; i <= 3; i++) {
+ await activeUser(`qareviewer${i}`);
+ reviewers.push(ctx.accounts.byName(`qareviewer${i}`));
+ }
+ const task = ctx.store.get('SELECT id FROM tasks LIMIT 1').id;
+ const addDecided = (n, rejected) => {
+ for (let i = 0; i < n; i++) {
+ const a = Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, base_commit, files_json, state, created_at) VALUES (?, ?, ?, '{}', ?, ?)`, task, author.id, ctx.repo.head(), rejected ? 'needs_cleanup' : 'merged', now()).lastInsertRowid);
+ for (const r of reviewers) ctx.store.run(`INSERT INTO peer_reviews (attempt_id, reviewer_id, verdict, reason, created_at) VALUES (?, ?, ?, 'checked the calls and branches', ?)`, a, r.id, rejected ? 'different' : 'same', now());
+ }
+ };
+ addDecided(10, false);
+ addDecided(12, true);
+ ctx.quality.forget(author.id);
+ assert.equal(ctx.quality.of(author.id).level, 'limited', '12 of 22 sent back');
+ addDecided(20, true);
+ ctx.quality.forget(author.id);
+ const q = ctx.quality.of(author.id);
+ assert.equal(q.level, 'paused', '32 of 42 sent back');
+ assert.match(q.reasons[0], /sent back by peers/);
+ const me = (await call('GET', '/api/me', null, token)).body;
+ assert.equal(me.quality.level, 'paused');
+ assert.match(me.quality.note, /lifts by itself/);
+ const claim = await call('POST', '/api/regions/claim', { kind: 'unit', key: 'src/player.cpp' }, token);
+ assert.equal(claim.status, 429);
+
+ await ctx.accounts.createUser('qaadmin', 'admin', PASSWORD);
+ const admin = await login('qaadmin');
+ assert.equal((await request('POST', '/api/admin/users/qaauthor/submit-trusted', {}, { cookie: admin })).status, 200);
+ assert.equal(ctx.quality.of(author.id).level, 'ok', 'trusted accounts are never limited');
+ assert.equal((await request('POST', '/api/admin/users/qaauthor/submit-paused', {}, { cookie: admin })).status, 200);
+ assert.equal(ctx.quality.of(author.id).level, 'paused');
+ assert.match(ctx.quality.of(author.id).reasons[0], /maintainer paused/);
+ const board = (await request('GET', '/api/admin/authors', null, { cookie: admin })).body;
+ assert.ok(board.authors.some((a) => a.name === 'qaauthor' && a.level === 'paused'));
+ ctx.store.run(`UPDATE attempts SET state = 'rejected' WHERE user_id = ?`, author.id);
+});