Quality is judged per agent (API token)
Submissions and region claims record their token; each agent has its own level, limits, stats and daily reset, so one bad agent no longer holds up the account's others. Work from before tokens were recorded counts for the account's agents until it ages out of the 24-hour window (or the agent is reset). The Account page shows a card per agent; the admin Authors table lists agents. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Saty
parent
787c5a3ca6
commit
8e6d8ad1fe
8 changed files with 208 additions and 110 deletions
|
|
@ -317,7 +317,14 @@ export function createApp(ctx) {
|
|||
regions: ctx.regions.mine(user).map((r) => ({ kind: r.kind, key: r.key, expires: r.expires_at })),
|
||||
regionTasks: shared,
|
||||
quality: (() => {
|
||||
const q = ctx.quality.of(user.id);
|
||||
// An agent sees its own level; the owner on the website the worst of their agents.
|
||||
if (user.via !== 'token') {
|
||||
const rank = { ok: 0, limited: 1, paused: 2 };
|
||||
const all = ctx.store.all('SELECT id FROM api_tokens WHERE user_id = ? AND revoked_at IS NULL', user.id).map((t) => ctx.quality.of(user.id, t.id));
|
||||
const worst = all.reduce((w, q) => (rank[q.level] > rank[w] ? q.level : w), 'ok');
|
||||
return { level: worst, agents: all.length, limitedAgents: all.filter((q) => q.level !== 'ok').length };
|
||||
}
|
||||
const q = ctx.quality.of(user.id, user.tokenId);
|
||||
return { level: q.level, reasons: q.reasons, ...(q.level !== 'ok' ? { note: Quality.advice(q) } : {}) };
|
||||
})(),
|
||||
regionAllowedPaths: shared.length ? tasks.allowedPathsForTasks(shared.slice(0, 500).map((t) => t.taskId)) : [],
|
||||
|
|
@ -937,14 +944,16 @@ export function createApp(ctx) {
|
|||
});
|
||||
|
||||
// How the account's agents are doing: level, numbers, what broke, recent examples (quality.js).
|
||||
// An agent (token) sees its own numbers; the owner on the website sees every agent of the account.
|
||||
router.on('GET', '/api/me/quality', ({ req }) => {
|
||||
const user = auth(req);
|
||||
return ctx.quality.details(user.id);
|
||||
return user.via === 'token' ? ctx.quality.details(user.id, user.tokenId) : ctx.quality.account(user.id);
|
||||
});
|
||||
// The owner's once-a-day fresh start (a person on the website, not the agent).
|
||||
router.on('POST', '/api/me/quality/reset', ({ req }) => {
|
||||
// The owner's once-a-day fresh start for one agent (a person on the website, not the agent).
|
||||
router.on('POST', '/api/me/quality/reset', async ({ req }) => {
|
||||
const user = auth(req, { session: true });
|
||||
return ctx.quality.reset(user.id);
|
||||
const body = await readJson(req, 1024);
|
||||
return ctx.quality.reset(user.id, body.tokenId == null ? null : intParam(body.tokenId));
|
||||
});
|
||||
|
||||
router.on('GET', '/api/admin/authors', ({ req }) => {
|
||||
|
|
|
|||
|
|
@ -302,6 +302,12 @@ export class Store {
|
|||
this.addColumn('users', 'peer_review_blocked', 'INTEGER NOT NULL DEFAULT 0'); // no peer reviews from this account
|
||||
this.addColumn('users', 'submit_mode', "TEXT NOT NULL DEFAULT 'auto'"); // auto (quality.js) | trusted | paused
|
||||
this.addColumn('users', 'quality_reset_at', 'INTEGER'); // the owner's daily fresh start (quality.js)
|
||||
// Which agent (API token) submitted or claimed what: quality is judged per agent.
|
||||
this.addColumn('attempts', 'token_id', 'INTEGER');
|
||||
this.addColumn('batches', 'token_id', 'INTEGER');
|
||||
this.addColumn('region_claims', 'token_id', 'INTEGER');
|
||||
this.addColumn('api_tokens', 'quality_reset_at', 'INTEGER');
|
||||
this.db.exec('CREATE INDEX IF NOT EXISTS attempts_user_time ON attempts(user_id, created_at)');
|
||||
// "No decompile" claims: an attempt that says a function needs no source (peer reviewed like code).
|
||||
this.addColumn('attempts', 'kind', "TEXT NOT NULL DEFAULT 'code'"); // code | skip
|
||||
this.addColumn('attempts', 'skip_category', 'TEXT');
|
||||
|
|
|
|||
|
|
@ -1,15 +1,19 @@
|
|||
// How good an author's recent work is, and what that allows. An agent that keeps submitting broken
|
||||
// code, or code peers keep sending back, slows down by itself instead of filling the queues:
|
||||
// How good an agent's recent work is, and what that allows. Each API token is one agent, judged on
|
||||
// its own: an agent that keeps submitting broken code, or code peers keep sending back, slows down
|
||||
// by itself instead of filling the queues, and the account's other agents go on as they are.
|
||||
//
|
||||
// ok normal limits
|
||||
// limited one batch waiting at a time, six submissions an hour, one region at a time
|
||||
// paused no submissions or region claims
|
||||
//
|
||||
// The level follows the last 24 hours, so it recovers by itself once the work gets better (or the
|
||||
// bad stretch ages out). A limited or paused user may reset once a day: only work after the reset
|
||||
// counts, so an agent that got better is free at once, and one that did not is limited again when
|
||||
// its new work fills the same thresholds. Admins can set an account to 'trusted' (never limited)
|
||||
// or 'paused' (no reset).
|
||||
// bad stretch ages out). The owner may reset a limited or paused agent once a day: only its work
|
||||
// after the reset counts, so an agent that got better is free at once, and one that did not is
|
||||
// limited again when its new work fills the same thresholds. Admins set a whole account to
|
||||
// 'trusted' (never limited) or 'paused' (no reset).
|
||||
//
|
||||
// Work from before submissions recorded their token belongs to the account as a whole; it counts
|
||||
// for every agent of the account until it is out of the window (or that agent is reset).
|
||||
|
||||
import { now } from './db.js';
|
||||
import { HttpError } from './http.js';
|
||||
|
|
@ -30,6 +34,10 @@ const KINDS = [
|
|||
['stale', 'not synced with main', ['base commit % is not part of main%']],
|
||||
];
|
||||
const BROKEN = KINDS.flatMap(([, , patterns]) => patterns);
|
||||
const summaryLike = (patterns) => patterns.map(() => "json_extract(a.result_json, '$.feedback.summary') LIKE ?").join(' OR ');
|
||||
// Attempts of one agent: its token, or (tokenId null) the account's work without a token.
|
||||
const byToken = (tokenId) => (tokenId == null ? 'a.token_id IS NULL' : 'a.token_id = ?');
|
||||
const tokenArgs = (tokenId) => (tokenId == null ? [] : [tokenId]);
|
||||
|
||||
export const LIMITS = {
|
||||
limited: { queuedBatches: 1, submitsPerHour: 6, regions: 1 },
|
||||
|
|
@ -41,29 +49,34 @@ export class Quality {
|
|||
this.cache = new Map();
|
||||
}
|
||||
|
||||
/** The author's recent numbers and the level they lead to (cached briefly). */
|
||||
of(userId) {
|
||||
const hit = this.cache.get(userId);
|
||||
/** The agent's (token's) recent numbers and the level they lead to (cached briefly). */
|
||||
of(userId, tokenId = null) {
|
||||
const key = `${userId}:${tokenId ?? '-'}`;
|
||||
const hit = this.cache.get(key);
|
||||
if (hit && now() - hit.at < CACHE_MS) return hit.value;
|
||||
const value = this.compute(userId);
|
||||
this.cache.set(userId, { at: now(), value });
|
||||
const value = this.compute(userId, tokenId);
|
||||
this.cache.set(key, { at: now(), value });
|
||||
return value;
|
||||
}
|
||||
|
||||
// Counts over [since, until).
|
||||
numbers(userId, since, until = now() + 1) {
|
||||
const brokenSql = BROKEN.map(() => "json_extract(a.result_json, '$.feedback.summary') LIKE ?").join(' OR ');
|
||||
forget(userId) {
|
||||
for (const k of this.cache.keys()) if (k.startsWith(`${userId}:`)) this.cache.delete(k);
|
||||
}
|
||||
|
||||
// Counts over [since, until) for one agent.
|
||||
numbers(userId, tokenId, since, until = now() + 1) {
|
||||
const r = this.store.get(
|
||||
`SELECT COUNT(*) AS submitted,
|
||||
SUM(CASE WHEN ${brokenSql} THEN 1 ELSE 0 END) AS broken,
|
||||
SUM(CASE WHEN ${summaryLike(BROKEN)} THEN 1 ELSE 0 END) AS broken,
|
||||
SUM(a.state = 'merged') AS merged,
|
||||
SUM(a.state = 'merged' AND json_extract(a.result_json, '$.feedback.target.status') IN ('exact', 'match_unresolved')) AS identical,
|
||||
SUM((SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3) AS peerRejected,
|
||||
SUM((SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id) >= 3) AS peerDecided
|
||||
FROM attempts a
|
||||
WHERE a.user_id = ? AND a.created_at >= ? AND a.created_at < ? AND a.kind = 'code' AND a.state NOT IN ('withdrawn', 'superseded', 'queued', 'verifying')`,
|
||||
WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND a.created_at < ? AND a.kind = 'code' AND a.state NOT IN ('withdrawn', 'superseded', 'queued', 'verifying')`,
|
||||
...BROKEN,
|
||||
userId,
|
||||
...tokenArgs(tokenId),
|
||||
since,
|
||||
until,
|
||||
);
|
||||
|
|
@ -81,24 +94,43 @@ export class Quality {
|
|||
};
|
||||
}
|
||||
|
||||
compute(userId) {
|
||||
const u = this.store.get('SELECT submit_mode, quality_reset_at FROM users WHERE id = ?', userId);
|
||||
const mode = u?.submit_mode ?? 'auto';
|
||||
const resetAt = u?.quality_reset_at ?? null;
|
||||
const since = Math.max(now() - WINDOW_MS, resetAt ?? 0);
|
||||
const n = this.numbers(userId, since);
|
||||
// The level some numbers lead to, with the reasons.
|
||||
static judge(n, prefix = '') {
|
||||
const reasons = [];
|
||||
let level = 'ok';
|
||||
const raise = (to, why) => {
|
||||
if (to === 'paused' || level === 'ok') level = to;
|
||||
reasons.push(why);
|
||||
reasons.push(prefix + why);
|
||||
};
|
||||
const peer = `${Math.round(n.peerRate * 100)}% of your reviewed functions were sent back by peers (${n.peerRejected} of ${n.peerDecided})`;
|
||||
const broke = `${Math.round(n.brokenRate * 100)}% of your submitted functions did not compile, broke other functions or were refused (${n.broken} of ${n.submitted})`;
|
||||
const peer = `${Math.round(n.peerRate * 100)}% of the reviewed functions were sent back by peers (${n.peerRejected} of ${n.peerDecided})`;
|
||||
const broke = `${Math.round(n.brokenRate * 100)}% of the submitted functions did not compile, broke other functions or were refused (${n.broken} of ${n.submitted})`;
|
||||
if (n.peerDecided >= 30 && n.peerRate >= 0.7) raise('paused', peer);
|
||||
else if (n.peerDecided >= 20 && n.peerRate >= 0.5) raise('limited', peer);
|
||||
if (n.submitted >= 80 && n.brokenRate >= 0.85) raise('paused', broke);
|
||||
else if (n.submitted >= 50 && n.brokenRate >= 0.6) raise('limited', broke);
|
||||
return { level, reasons };
|
||||
}
|
||||
|
||||
compute(userId, tokenId = null) {
|
||||
const u = this.store.get('SELECT submit_mode, quality_reset_at FROM users WHERE id = ?', userId);
|
||||
const mode = u?.submit_mode ?? 'auto';
|
||||
const resetAt = tokenId == null ? u?.quality_reset_at ?? null : this.store.get('SELECT quality_reset_at FROM api_tokens WHERE id = ?', tokenId)?.quality_reset_at ?? null;
|
||||
const since = Math.max(now() - WINDOW_MS, resetAt ?? 0);
|
||||
const n = this.numbers(userId, tokenId, since);
|
||||
let { level, reasons } = Quality.judge(n);
|
||||
// The account's work from before tokens were recorded still counts for its agents, until it
|
||||
// ages out of the window or the agent is reset.
|
||||
let legacy = null;
|
||||
if (tokenId != null) {
|
||||
// An account-wide reset (by the owner) also gives the agents a fresh start from that work.
|
||||
const l = this.numbers(userId, null, Math.max(since, u?.quality_reset_at ?? 0));
|
||||
if (l.submitted) {
|
||||
legacy = l;
|
||||
const j = Quality.judge(l, 'this account before work was tracked per agent: ');
|
||||
if (j.level === 'paused' || (j.level === 'limited' && level === 'ok')) level = j.level;
|
||||
reasons = [...reasons, ...j.reasons];
|
||||
}
|
||||
}
|
||||
if (mode === 'trusted') level = 'ok';
|
||||
if (mode === 'paused') {
|
||||
level = 'paused';
|
||||
|
|
@ -106,41 +138,39 @@ export class Quality {
|
|||
}
|
||||
const nextReset = resetAt ? resetAt + RESET_EVERY_MS : 0;
|
||||
const canReset = level !== 'ok' && mode === 'auto' && now() >= nextReset;
|
||||
return { level, mode, reasons, ...n, since, resetAt, canReset, nextResetAt: canReset || level === 'ok' ? null : nextReset || null, windowHours: 24 };
|
||||
return { tokenId, level, mode, reasons, ...n, legacy, since, resetAt, canReset, nextResetAt: canReset || level === 'ok' ? null : nextReset || null, windowHours: 24 };
|
||||
}
|
||||
|
||||
forget(userId) {
|
||||
this.cache.delete(userId);
|
||||
}
|
||||
|
||||
/** A fresh start for a limited or paused author: only work after now counts. Once a day. */
|
||||
reset(userId) {
|
||||
const q = this.compute(userId);
|
||||
/** A fresh start for one limited or paused agent: only its work after now counts. Once a day. */
|
||||
reset(userId, tokenId = null) {
|
||||
if (tokenId != null && !this.store.get('SELECT 1 FROM api_tokens WHERE id = ? AND user_id = ?', tokenId, userId)) throw new HttpError(404, 'no such token of yours');
|
||||
const q = this.compute(userId, tokenId);
|
||||
if (q.mode === 'paused') throw new HttpError(403, 'a maintainer paused this account; ask them');
|
||||
if (q.level === 'ok') throw new HttpError(409, 'nothing to reset: this account is not limited');
|
||||
if (!q.canReset) throw new HttpError(429, `one reset per 24 hours; the next is possible at ${new Date(q.nextResetAt).toISOString()}`, { retryAfterSeconds: Math.ceil((q.nextResetAt - now()) / 1000) });
|
||||
this.store.run('UPDATE users SET quality_reset_at = ? WHERE id = ?', now(), userId);
|
||||
this.store.event('quality_reset', userId, { was: q.level, peerRate: q.peerRate, brokenRate: q.brokenRate });
|
||||
if (q.level === 'ok') throw new HttpError(409, 'nothing to reset: this agent is not limited');
|
||||
if (!q.canReset) throw new HttpError(429, `one reset per agent per 24 hours; the next is possible at ${new Date(q.nextResetAt).toISOString()}`, { retryAfterSeconds: Math.ceil((q.nextResetAt - now()) / 1000) });
|
||||
if (tokenId == null) this.store.run('UPDATE users SET quality_reset_at = ? WHERE id = ?', now(), userId);
|
||||
else this.store.run('UPDATE api_tokens SET quality_reset_at = ? WHERE id = ?', now(), tokenId);
|
||||
this.store.event('quality_reset', userId, { token: tokenId, was: q.level, peerRate: q.peerRate, brokenRate: q.brokenRate });
|
||||
this.forget(userId);
|
||||
return this.details(userId);
|
||||
return this.details(userId, tokenId);
|
||||
}
|
||||
|
||||
/** Everything an author (or their agent) needs to see how the work is going and what to fix. */
|
||||
details(userId) {
|
||||
const q = this.of(userId);
|
||||
const previous = this.numbers(userId, q.since - WINDOW_MS, q.since);
|
||||
/** Everything an owner (or the agent) needs to see how one agent is doing and what to fix. */
|
||||
details(userId, tokenId = null) {
|
||||
const q = this.of(userId, tokenId);
|
||||
const previous = this.numbers(userId, tokenId, q.since - WINDOW_MS, q.since);
|
||||
const kinds = KINDS.map(([key, label, patterns]) => {
|
||||
const n = this.store.get(
|
||||
`SELECT COUNT(*) AS n FROM attempts a WHERE a.user_id = ? AND a.created_at >= ? AND a.kind = 'code' AND (${patterns.map(() => "json_extract(a.result_json, '$.feedback.summary') LIKE ?").join(' OR ')})`,
|
||||
userId, q.since, ...patterns,
|
||||
`SELECT COUNT(*) AS n FROM attempts a WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND a.kind = 'code' AND (${summaryLike(patterns)})`,
|
||||
userId, ...tokenArgs(tokenId), q.since, ...patterns,
|
||||
).n;
|
||||
return { kind: key, label, functions: n };
|
||||
}).filter((k) => k.functions);
|
||||
const sentBack = this.store.all(
|
||||
`SELECT a.id, t.symbol_id, s.name, a.created_at FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
WHERE a.user_id = ? AND a.created_at >= ? AND (SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3
|
||||
`SELECT a.id, t.symbol_id, s.name FROM attempts a JOIN tasks t ON t.id = a.task_id JOIN symbols s ON s.id = t.symbol_id
|
||||
WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND (SELECT COUNT(*) FROM peer_reviews p WHERE p.attempt_id = a.id AND p.verdict = 'different') >= 3
|
||||
ORDER BY a.id DESC LIMIT 5`,
|
||||
userId, q.since,
|
||||
userId, ...tokenArgs(tokenId), q.since,
|
||||
).map((r) => ({
|
||||
attemptId: r.id,
|
||||
symbol: r.symbol_id,
|
||||
|
|
@ -149,28 +179,46 @@ export class Quality {
|
|||
}));
|
||||
const broken = this.store.all(
|
||||
`SELECT a.id, t.symbol_id, json_extract(a.result_json, '$.feedback.summary') AS summary FROM attempts a JOIN tasks t ON t.id = a.task_id
|
||||
WHERE a.user_id = ? AND a.created_at >= ? AND a.kind = 'code' AND (${BROKEN.map(() => "json_extract(a.result_json, '$.feedback.summary') LIKE ?").join(' OR ')})
|
||||
WHERE a.user_id = ? AND ${byToken(tokenId)} AND a.created_at >= ? AND a.kind = 'code' AND (${summaryLike(BROKEN)})
|
||||
ORDER BY a.id DESC LIMIT 5`,
|
||||
userId, q.since, ...BROKEN,
|
||||
userId, ...tokenArgs(tokenId), q.since, ...BROKEN,
|
||||
).map((r) => ({ attemptId: r.id, symbol: r.symbol_id, summary: String(r.summary).slice(0, 300) }));
|
||||
return { ...q, previous, kinds, sentBack, broken, ...(q.level !== 'ok' ? { note: Quality.advice(q) } : {}), thresholds: { limited: 'half sent back by peers (20+ reviewed) or 60% broken (50+ submitted)', paused: '70% sent back (30+ reviewed) or 85% broken (80+ submitted)' } };
|
||||
const label = tokenId == null ? null : this.store.get('SELECT label FROM api_tokens WHERE id = ?', tokenId)?.label ?? null;
|
||||
return {
|
||||
...q,
|
||||
label,
|
||||
previous,
|
||||
kinds,
|
||||
sentBack,
|
||||
broken,
|
||||
...(q.level !== 'ok' ? { note: Quality.advice(q) } : {}),
|
||||
thresholds: { limited: 'half sent back by peers (20+ reviewed) or 60% broken (50+ submitted)', paused: '70% sent back (30+ reviewed) or 85% broken (80+ submitted)' },
|
||||
};
|
||||
}
|
||||
|
||||
/** Every agent (token) of an account, plus the account's work from before tokens were recorded. */
|
||||
account(userId) {
|
||||
const tokens = this.store.all('SELECT id FROM api_tokens WHERE user_id = ? AND revoked_at IS NULL ORDER BY id', userId).map((t) => this.details(userId, t.id));
|
||||
const legacy = this.numbers(userId, null, now() - WINDOW_MS);
|
||||
return { tokens, legacy: legacy.submitted ? { ...legacy, ...Quality.judge(legacy) } : null };
|
||||
}
|
||||
|
||||
/** What an agent is told when its level limits it. */
|
||||
static advice(q) {
|
||||
const what = q.level === 'paused' ? 'Submissions and region claims are paused for this account' : 'This account is limited to one batch waiting at a time, six submissions an hour and one region';
|
||||
const reset = q.canReset ? ' The account owner can reset this once a day on the website (Account page).' : '';
|
||||
return `${what}: ${q.reasons.join('; ')}. It lifts by itself as your recent work gets better (last 24 hours).${reset} Port each function faithfully from the disassembly, td_check it until it compiles and matches structurally, and submit only what you checked.`;
|
||||
const what = q.level === 'paused' ? 'Submissions and region claims are paused for this agent' : 'This agent is limited to one batch waiting at a time, six submissions an hour and one region';
|
||||
const reset = q.canReset ? ' The account owner can reset it once a day on the website (Account page).' : '';
|
||||
return `${what}: ${q.reasons.join('; ')}. It lifts by itself as the recent work gets better (last 24 hours).${reset} Port each function faithfully from the disassembly, td_check it until it compiles and matches structurally, and submit only what you checked.`;
|
||||
}
|
||||
|
||||
/** Every author with work in the window, for the admin view. */
|
||||
/** Every agent with work in the window, for the admin view. */
|
||||
authors() {
|
||||
const ids = this.store.all(`SELECT DISTINCT user_id FROM attempts WHERE created_at > ?`, now() - WINDOW_MS).map((r) => r.user_id);
|
||||
const pairs = this.store.all(`SELECT DISTINCT user_id, token_id FROM attempts WHERE created_at > ?`, now() - WINDOW_MS);
|
||||
const out = [];
|
||||
for (const id of ids) {
|
||||
const u = this.store.get('SELECT name, status FROM users WHERE id = ?', id);
|
||||
for (const p of pairs) {
|
||||
const u = this.store.get('SELECT name, status FROM users WHERE id = ?', p.user_id);
|
||||
if (!u) continue;
|
||||
out.push({ name: u.name, status: u.status, ...this.of(id) });
|
||||
const label = p.token_id == null ? null : this.store.get('SELECT label FROM api_tokens WHERE id = ?', p.token_id)?.label ?? '(revoked token)';
|
||||
out.push({ name: u.name, status: u.status, token: label, ...this.of(p.user_id, p.token_id) });
|
||||
}
|
||||
const rank = { paused: 0, limited: 1, ok: 2 };
|
||||
return out.sort((a, b) => rank[a.level] - rank[b.level] || b.peerRate - a.peerRate || b.submitted - a.submitted);
|
||||
|
|
|
|||
|
|
@ -108,10 +108,11 @@ export class Regions {
|
|||
if (!['unit', 'class'].includes(kind)) throw new HttpError(400, "kind must be 'unit' or 'class'");
|
||||
key = String(key ?? '').trim();
|
||||
if (!key || key.length > 300) throw new HttpError(400, 'key must name the source file (kind unit) or the class (kind class)');
|
||||
const q = this.quality?.of(user.id);
|
||||
const q = this.quality?.of(user.id, user.tokenId ?? null);
|
||||
if (q?.level === 'paused') throw new HttpError(429, this.quality.constructor.advice(q), { retryAfterSeconds: 3600 });
|
||||
if (q?.level === 'limited') {
|
||||
const other = this.mine(user).filter((x) => !(x.kind === kind && x.key === key));
|
||||
const token = user.tokenId ?? null;
|
||||
const other = this.mine(user).filter((x) => !(x.kind === kind && x.key === key) && (x.token_id ?? null) === token);
|
||||
if (other.length) throw new HttpError(409, `${this.quality.constructor.advice(q)} Finish or release ${other[0].key} first.`);
|
||||
}
|
||||
return this.store.tx(() => {
|
||||
|
|
@ -121,9 +122,9 @@ export class Regions {
|
|||
if (!left.length) throw new HttpError(409, `${key} is finished: every function is on main`);
|
||||
const expires = now() + this.leaseMs;
|
||||
this.store.run(
|
||||
`INSERT INTO region_claims (user_id, kind, key, created_at, expires_at) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(user_id, kind, key) DO UPDATE SET expires_at = excluded.expires_at`,
|
||||
user.id, kind, key, now(), expires,
|
||||
`INSERT INTO region_claims (user_id, token_id, kind, key, created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(user_id, kind, key) DO UPDATE SET expires_at = excluded.expires_at, token_id = excluded.token_id`,
|
||||
user.id, user.tokenId ?? null, kind, key, now(), expires,
|
||||
);
|
||||
const held = this.tasks.leasesOf(user.id).length;
|
||||
let room = this.tasks.maxLeased - held;
|
||||
|
|
@ -187,7 +188,7 @@ export class Regions {
|
|||
|
||||
/** The regions `user` holds. */
|
||||
mine(user) {
|
||||
return this.store.all('SELECT kind, key, created_at, expires_at FROM region_claims WHERE user_id = ? AND expires_at > ? ORDER BY created_at', user.id, now());
|
||||
return this.store.all('SELECT kind, key, token_id, created_at, expires_at FROM region_claims WHERE user_id = ? AND expires_at > ? ORDER BY created_at', user.id, now());
|
||||
}
|
||||
|
||||
heartbeat(user) {
|
||||
|
|
|
|||
|
|
@ -308,10 +308,13 @@ export class Tasks {
|
|||
return this.store.tx(() => {
|
||||
const own = new Set(ids.filter((id) => this.requireSubmittable(user, id).own));
|
||||
// Authors whose recent work keeps breaking or being sent back slow down (quality.js).
|
||||
const q = this.quality?.of(user.id);
|
||||
const q = this.quality?.of(user.id, user.tokenId ?? null);
|
||||
const token = user.tokenId ?? null;
|
||||
const mine = token == null ? 'token_id IS NULL' : 'token_id = ?';
|
||||
const mineArgs = token == null ? [] : [token];
|
||||
if (q?.level === 'paused') throw new HttpError(429, this.quality.constructor.advice(q), { retryAfterSeconds: 3600, quality: q.level });
|
||||
if (q?.level === 'limited') {
|
||||
const lastHour = this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND created_at > ? AND state != 'skip'`, user.id, now() - 3600_000).n;
|
||||
const lastHour = this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND ${mine} AND created_at > ? AND state != 'skip'`, user.id, ...mineArgs, now() - 3600_000).n;
|
||||
if (lastHour >= 6) throw new HttpError(429, this.quality.constructor.advice(q), { retryAfterSeconds: 600, quality: q.level });
|
||||
}
|
||||
// An agent stuck in a loop (every submission refused before it compiled: not synced, conflicts,
|
||||
|
|
@ -327,22 +330,25 @@ export class Tasks {
|
|||
throw new HttpError(429, `your last ${recent.length} submissions were all refused before compiling (latest: ${refused[0].why}). Run td_sync, merge, fix that and check locally (td_check) before submitting again`, { retryAfterSeconds: 600 });
|
||||
}
|
||||
// Keep the verification queue moving: a few batches waiting per account at a time.
|
||||
// A limited agent: one batch of its own waiting; the account's other agents are not held up by it.
|
||||
const max = q?.level === 'limited' ? 1 : this.config.maxQueuedBatches ?? 8;
|
||||
const waiting = this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND state IN ('queued', 'verifying')`, user.id).n;
|
||||
const waiting = q?.level === 'limited'
|
||||
? this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND ${mine} AND state IN ('queued', 'verifying')`, user.id, ...mineArgs).n
|
||||
: this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND state IN ('queued', 'verifying')`, user.id).n;
|
||||
if (waiting >= max) {
|
||||
throw new HttpError(429, `you have ${waiting} batches waiting for verification (the limit is ${max}); keep your claims, do peer reviews meanwhile (td_list_peer_reviews), and submit again when one is done (td_get_batch shows it)`, { retryAfterSeconds: 120 });
|
||||
}
|
||||
const b = this.store.run(
|
||||
`INSERT INTO batches (user_id, base_commit, files_json, notes, agent_meta, env_digest, state, result_json, created_at) VALUES (?, ?, ?, ?, ?, ?, 'queued', ?, ?)`,
|
||||
user.id, baseCommit, JSON.stringify(files), notes, agentMeta, envDigest, confirm.length ? JSON.stringify({ confirm: confirm.map(String).slice(0, 500) }) : null, now(),
|
||||
`INSERT INTO batches (user_id, token_id, base_commit, files_json, notes, agent_meta, env_digest, state, result_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', ?, ?)`,
|
||||
user.id, token, baseCommit, JSON.stringify(files), notes, agentMeta, envDigest, confirm.length ? JSON.stringify({ confirm: confirm.map(String).slice(0, 500) }) : null, now(),
|
||||
);
|
||||
const batchId = Number(b.lastInsertRowid);
|
||||
this.store.run('UPDATE batches SET cost = ? WHERE id = ?', this.compileCost(Object.keys(files)), batchId);
|
||||
const attempts = [];
|
||||
for (const id of ids) {
|
||||
const r = this.store.run(
|
||||
`INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, batch_id, competing, created_at) VALUES (?, ?, ?, '{}', ?, ?, ?, 'queued', ?, ?, ?)`,
|
||||
id, user.id, baseCommit, notes, agentMeta, envDigest, batchId, own.has(id) ? 0 : 1, now(),
|
||||
`INSERT INTO attempts (task_id, user_id, token_id, base_commit, files_json, notes, agent_meta, env_digest, state, batch_id, competing, created_at) VALUES (?, ?, ?, ?, '{}', ?, ?, ?, 'queued', ?, ?, ?)`,
|
||||
id, user.id, token, baseCommit, notes, agentMeta, envDigest, batchId, own.has(id) ? 0 : 1, now(),
|
||||
);
|
||||
attempts.push({ taskId: id, attemptId: Number(r.lastInsertRowid), ...(own.has(id) ? {} : { competing: true }) });
|
||||
// A competing candidate leaves the task as its holder has it.
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ async function refresh() {
|
|||
$('account-link').title = `${me.name} · ${me.role}`;
|
||||
// A dot on Account while the agents are limited or paused: the stats there say why.
|
||||
$('account-dot').hidden = !me.quality || me.quality.level === 'ok';
|
||||
$('account-dot').title = me.quality?.level === 'ok' ? '' : `your agents are ${me.quality?.level}`;
|
||||
$('account-dot').title = me.quality?.level === 'ok' ? '' : `${me.quality?.limitedAgents ?? 1} of your agents ${me.quality?.limitedAgents === 1 ? 'is' : 'are'} limited or paused`;
|
||||
api('/api/credits/me').then((w) => setCredits(w.balance, false)).catch(() => {});
|
||||
}
|
||||
connectLive(true);
|
||||
|
|
@ -684,32 +684,46 @@ async function loadAccount() {
|
|||
renderQuality(await api('/api/me/quality').catch(() => null));
|
||||
}
|
||||
|
||||
// The account's own quality numbers: level, what broke, recent examples, the daily reset.
|
||||
// How each agent (API token) of the account is doing: level, what broke, recent examples, the
|
||||
// daily reset per agent. Work from before agents were told apart is shown while it still counts.
|
||||
function renderQuality(q) {
|
||||
const box = $('my-quality');
|
||||
box.replaceChildren();
|
||||
if (!q) return;
|
||||
if (!q.tokens.length) box.append(el('p', 'No agent tokens yet: create one below, and its stats show up here.', 'muted'));
|
||||
for (const t of q.tokens) box.append(agentCard(t));
|
||||
if (q.legacy) {
|
||||
const l = el('div', null, 'agent-card');
|
||||
l.append(el('div', 'Before agents were told apart (last 24 hours)', 'agent-title'), el('p', `${fmt(q.legacy.submitted)} functions submitted; ${q.legacy.peerDecided ? `${Math.round(q.legacy.peerRate * 100)}% of ${q.legacy.peerDecided} sent back by peers` : 'none reviewed'}; ${Math.round(q.legacy.brokenRate * 100)}% broken or refused.${q.legacy.level !== 'ok' ? ' This still counts for your agents until it ages out, or until you reset them.' : ''}`, 'hint'));
|
||||
box.append(l);
|
||||
}
|
||||
box.append(el('p', 'Each agent is judged on its own. Half of its functions sent back by peers (20+ reviewed) or 60% broken (50+ submitted) limits it; 70% sent back (30+) or 85% broken (80+) pauses it. Both lift by themselves as its last 24 hours get better.', 'hint'));
|
||||
}
|
||||
|
||||
function agentCard(q) {
|
||||
const pct = (x) => `${Math.round(x * 100)}%`;
|
||||
const card = el('div', null, `agent-card ${q.level}`);
|
||||
const head = el('div', null, 'quality-head');
|
||||
head.append(el('span', q.level === 'ok' ? 'all good' : q.level, `status ${q.level === 'ok' ? 'active' : q.level === 'limited' ? 'pending' : 'revoked'}`));
|
||||
if (q.resetAt && q.resetAt > Date.now() - 24 * 3600_000) head.append(el('span', ` counting since your reset ${ago(q.resetAt)}`, 'hint'));
|
||||
box.append(head);
|
||||
if (q.reasons.length) box.append(el('p', q.reasons.join('; '), 'warn'));
|
||||
const trend = (now_, before, better) => {
|
||||
head.append(el('span', q.label ?? 'agent', 'agent-title'), el('span', q.level === 'ok' ? 'all good' : q.level, `status ${q.level === 'ok' ? 'active' : q.level === 'limited' ? 'pending' : 'revoked'}`));
|
||||
if (q.resetAt && q.resetAt > Date.now() - 24 * 3600_000) head.append(el('span', `counting since your reset ${ago(q.resetAt)}`, 'hint'));
|
||||
card.append(head);
|
||||
if (q.reasons.length) card.append(el('p', q.reasons.join('; '), 'warn'));
|
||||
const prev = q.previous ?? {};
|
||||
const trend = (now_, before) => {
|
||||
if (before === null) return '';
|
||||
const d = now_ - before;
|
||||
if (Math.abs(d) < 0.02) return ' (same as the day before)';
|
||||
return (better ? d < 0 : d > 0) ? ` (${d < 0 ? 'down' : 'up'} from ${pct(before)}: better)` : ` (${d < 0 ? 'down' : 'up'} from ${pct(before)}: worse)`;
|
||||
return ` (${d < 0 ? 'down' : 'up'} from ${pct(before)}: ${d < 0 ? 'better' : 'worse'})`;
|
||||
};
|
||||
const prev = q.previous ?? {};
|
||||
const rows = [
|
||||
['Functions submitted', fmt(q.submitted)],
|
||||
['Merged', `${fmt(q.merged)} (${fmt(q.identical)} byte-identical)`],
|
||||
['Sent back by peers', q.peerDecided ? `${pct(q.peerRate)} of ${fmt(q.peerDecided)} reviewed${trend(q.peerRate, prev.peerDecided ? prev.peerRate : null, true)}` : 'none reviewed yet'],
|
||||
['Broken or refused', q.submitted ? `${pct(q.brokenRate)} of ${fmt(q.submitted)}${trend(q.brokenRate, prev.submitted ? prev.brokenRate : null, true)}` : '-'],
|
||||
];
|
||||
box.append(table(['Last 24 hours', ''], rows));
|
||||
if (q.kinds.length) box.append(el('p', `What broke: ${q.kinds.map((k) => `${fmt(k.functions)} ${k.label}`).join(' · ')}`, 'muted'));
|
||||
card.append(
|
||||
table(['Last 24 hours', ''], [
|
||||
['Functions submitted', fmt(q.submitted)],
|
||||
['Merged', `${fmt(q.merged)} (${fmt(q.identical)} byte-identical)`],
|
||||
['Sent back by peers', q.peerDecided ? `${pct(q.peerRate)} of ${fmt(q.peerDecided)} reviewed${trend(q.peerRate, prev.peerDecided ? prev.peerRate : null)}` : 'none reviewed yet'],
|
||||
['Broken or refused', q.submitted ? `${pct(q.brokenRate)} of ${fmt(q.submitted)}${trend(q.brokenRate, prev.submitted ? prev.brokenRate : null)}` : '-'],
|
||||
]),
|
||||
);
|
||||
if (q.kinds.length) card.append(el('p', `What broke: ${q.kinds.map((k) => `${fmt(k.functions)} ${k.label}`).join(' · ')}`, 'muted'));
|
||||
if (q.sentBack.length) {
|
||||
const d = el('details', null, 'fold');
|
||||
d.append(el('summary', `Latest functions peers sent back (${q.sentBack.length})`));
|
||||
|
|
@ -717,21 +731,22 @@ function renderQuality(q) {
|
|||
d.append(el('p', `${s.symbol} ${s.name ?? ''}`, 'mono'));
|
||||
for (const r of s.reasons) d.append(el('p', `· ${r?.text ?? ''}`, 'hint'));
|
||||
}
|
||||
box.append(d);
|
||||
card.append(d);
|
||||
}
|
||||
if (q.broken.length) {
|
||||
const d = el('details', null, 'fold');
|
||||
d.append(el('summary', `Latest broken submissions (${q.broken.length})`), ...q.broken.map((b) => el('p', `${b.symbol}: ${b.summary}`, 'hint')));
|
||||
box.append(d);
|
||||
card.append(d);
|
||||
}
|
||||
box.append(el('p', `Limits: ${q.thresholds.limited} limits the account; ${q.thresholds.paused} pauses it. Both lift by themselves as the last 24 hours get better.`, 'hint'));
|
||||
if (q.level !== 'ok') {
|
||||
const row = el('div', null, 'actions');
|
||||
const reset = button('Reset (start counting from now)', async () => {
|
||||
if (!confirm('Start counting from now? Only work after this counts. If your agent keeps doing the same, it is limited again once the new work fills the same thresholds. You get one reset per 24 hours.')) return;
|
||||
const reset = button('Reset this agent', async () => {
|
||||
if (!confirm(`Start counting from now for "${q.label ?? 'this agent'}"? Only its work after this counts. If it keeps doing the same, it is limited again once its new work fills the same thresholds. One reset per agent per 24 hours.`)) return;
|
||||
try {
|
||||
renderQuality(await post('/api/me/quality/reset'));
|
||||
toast('Reset: counting from now. Fix what your agent does wrong before letting it go on.', { icon: '🔄' });
|
||||
await post('/api/me/quality/reset', { tokenId: q.tokenId });
|
||||
toast('Reset: counting from now. Fix what the agent does wrong before letting it go on.', { icon: '🔄' });
|
||||
renderQuality(await api('/api/me/quality'));
|
||||
refresh();
|
||||
} catch (e) {
|
||||
toast(e.message);
|
||||
}
|
||||
|
|
@ -739,8 +754,9 @@ function renderQuality(q) {
|
|||
reset.disabled = !q.canReset;
|
||||
row.append(reset);
|
||||
if (!q.canReset) row.append(el('span', q.mode === 'paused' ? ' A maintainer paused this account; ask them.' : ` Next reset possible ${q.nextResetAt ? new Date(q.nextResetAt).toLocaleString() : 'later'}.`, 'hint'));
|
||||
box.append(row);
|
||||
card.append(row);
|
||||
}
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderTokens(tokens) {
|
||||
|
|
@ -826,7 +842,7 @@ function renderAuthors(authors) {
|
|||
$('authors').replaceChildren(
|
||||
authors.length
|
||||
? table(
|
||||
['Author', 'Level', 'Sent back by peers', 'Broken or refused', 'Why', 'Submissions'],
|
||||
['Author', 'Agent', 'Level', 'Sent back by peers', 'Broken or refused', 'Why', 'Submissions (account)'],
|
||||
authors.map((a) => {
|
||||
const modes = el('div', null, 'row-actions');
|
||||
for (const [mode, label] of [['auto', 'Auto'], ['trusted', 'Trust'], ['paused', 'Pause']]) {
|
||||
|
|
@ -839,6 +855,7 @@ function renderAuthors(authors) {
|
|||
}
|
||||
return [
|
||||
a.name,
|
||||
a.token ?? el('span', 'before agents were told apart', 'hint'),
|
||||
[chip(a.level), ...(a.mode !== 'auto' ? [el('div', `set: ${a.mode}`, 'hint')] : [])],
|
||||
a.peerDecided ? `${pct(a.peerRate)} of ${a.peerDecided}` : '-',
|
||||
a.submitted ? `${pct(a.brokenRate)} of ${a.submitted}` : '-',
|
||||
|
|
|
|||
|
|
@ -747,3 +747,8 @@ details.fold-plate[open] > summary { margin-bottom: 12px; }
|
|||
.nav-dot { display: inline-block; width: 8px; height: 8px; margin-left: 6px; border-radius: 50%; background: var(--warn); vertical-align: 2px; box-shadow: 0 0 6px var(--warn); }
|
||||
.quality-head { display: flex; align-items: center; gap: 8px; margin-bottom: 6px; }
|
||||
.my-quality .fold p { margin: 4px 0; }
|
||||
|
||||
.agent-card { margin: 10px 0; padding: 10px 14px; border: 1px solid var(--line); border-left: 4px solid var(--ok); border-radius: 3px; background: var(--well); }
|
||||
.agent-card.limited { border-left-color: var(--warn); }
|
||||
.agent-card.paused { border-left-color: var(--danger); }
|
||||
.agent-title { font: 700 15px var(--font-display); text-transform: uppercase; letter-spacing: 0.06em; }
|
||||
|
|
|
|||
|
|
@ -1294,9 +1294,12 @@ test('authors whose work peers keep sending back are limited, then paused; trust
|
|||
ctx.store.run(`UPDATE attempts SET state = 'rejected' WHERE user_id = ?`, author.id);
|
||||
});
|
||||
|
||||
test('a limited author may reset once a day; the same bad work limits them again', async () => {
|
||||
test('each agent (token) is judged on its own; the owner resets one agent once a day', async () => {
|
||||
const token = await activeUser('rsauthor');
|
||||
const author = ctx.accounts.byName('rsauthor');
|
||||
const tid = ctx.accounts.userByApiToken(token).tokenId;
|
||||
const other = ctx.accounts.createApiToken(author.id, 'second agent').token;
|
||||
const otherTid = ctx.accounts.userByApiToken(other).tokenId;
|
||||
const reviewers = [];
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
await activeUser(`rsreviewer${i}`);
|
||||
|
|
@ -1305,7 +1308,7 @@ test('a limited author may reset once a day; the same bad work limits them again
|
|||
const task = ctx.store.get('SELECT id FROM tasks LIMIT 1').id;
|
||||
const sentBack = (n, at = now()) => {
|
||||
for (let i = 0; i < n; i++) {
|
||||
const a = Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, base_commit, files_json, state, created_at) VALUES (?, ?, ?, '{}', 'needs_cleanup', ?)`, task, author.id, ctx.repo.head(), at).lastInsertRowid);
|
||||
const a = Number(ctx.store.run(`INSERT INTO attempts (task_id, user_id, token_id, base_commit, files_json, state, created_at) VALUES (?, ?, ?, ?, '{}', 'needs_cleanup', ?)`, task, author.id, tid, ctx.repo.head(), at).lastInsertRowid);
|
||||
for (const r of reviewers) ctx.store.run(`INSERT INTO peer_reviews (attempt_id, reviewer_id, verdict, reason, created_at) VALUES (?, ?, 'different', 'the original checks the flag first', ?)`, a, r.id, at);
|
||||
}
|
||||
};
|
||||
|
|
@ -1316,17 +1319,20 @@ test('a limited author may reset once a day; the same bad work limits them again
|
|||
assert.ok(stats.canReset);
|
||||
assert.equal(stats.sentBack.length, 5);
|
||||
assert.equal(stats.sentBack[0].reasons[0].untrusted, true, 'reviewer text is marked as data');
|
||||
assert.equal((await call('POST', '/api/me/quality/reset', {}, token)).status, 403, 'the person resets, not the agent token');
|
||||
assert.equal((await call('GET', '/api/me/quality', null, other)).body.level, 'ok', 'the account\'s other agent is not held up');
|
||||
assert.equal((await call('POST', '/api/me/quality/reset', { tokenId: tid }, token)).status, 403, 'the person resets, not the agent token');
|
||||
|
||||
const cookie = await login('rsauthor');
|
||||
const r = await request('POST', '/api/me/quality/reset', {}, { cookie });
|
||||
const overview = (await request('GET', '/api/me/quality', null, { cookie })).body;
|
||||
assert.deepEqual(overview.tokens.map((t) => [t.tokenId, t.level]).sort(), [[tid, 'limited'], [otherTid, 'ok']].sort());
|
||||
const r = await request('POST', '/api/me/quality/reset', { tokenId: tid }, { cookie });
|
||||
assert.equal(r.status, 200, JSON.stringify(r.body));
|
||||
assert.equal(r.body.level, 'ok', 'a fresh start');
|
||||
assert.equal((await request('POST', '/api/me/quality/reset', {}, { cookie })).status, 409, 'nothing to reset now');
|
||||
assert.equal((await request('POST', '/api/me/quality/reset', { tokenId: tid }, { cookie })).status, 409, 'nothing to reset now');
|
||||
sentBack(25);
|
||||
ctx.quality.forget(author.id);
|
||||
assert.equal(ctx.quality.of(author.id).level, 'limited', 'no better: limited again');
|
||||
const again = await request('POST', '/api/me/quality/reset', {}, { cookie });
|
||||
assert.equal(again.status, 429, 'one reset per day');
|
||||
assert.equal(ctx.quality.of(author.id, tid).level, 'limited', 'no better: limited again');
|
||||
assert.equal((await request('POST', '/api/me/quality/reset', { tokenId: tid }, { cookie })).status, 429, 'one reset per agent per day');
|
||||
assert.equal((await request('POST', '/api/me/quality/reset', { tokenId: 999999 }, { cookie })).status, 404);
|
||||
ctx.store.run(`UPDATE attempts SET state = 'rejected' WHERE user_id = ?`, author.id);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue