From 10f840b18fe065d255dcac097dfb574e2c535c8f Mon Sep 17 00:00:00 2001 From: Saty <5752427+SatyPardus@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:32:20 +0200 Subject: [PATCH] Regions, competing candidates, five "same" merge, checks compile locally - Agents list regions (source files or classes: open, done, in flight, who claimed them), claim one as a whole and submit it in one batch. - Several agents may claim a region; each may submit any function of it. Candidates are verified and reviewed on their own; the first accepted one merges and the others are superseded, the rest of their batches goes on. A done task stays done (trigger). - Peer review: five "same" accept (no human step), three "different" send back. - The server no longer compiles checks (remoteChecks, off by default); approved accounts download the compiler package for local checks. - Tables for game credits and rounds. Co-Authored-By: Claude Opus 5.5 --- src/server/app.js | 66 ++++++++++++++- src/server/db.js | 45 ++++++++++ src/server/merge.js | 23 +++++ src/server/peer.js | 64 ++++++-------- src/server/regions.js | 189 ++++++++++++++++++++++++++++++++++++++++++ src/server/tasks.js | 25 ++++-- src/server/verify.js | 19 ++++- test/server.test.js | 132 +++++++++++++++++++++++------ 8 files changed, 488 insertions(+), 75 deletions(-) create mode 100644 src/server/regions.js diff --git a/src/server/app.js b/src/server/app.js index 9d5d199..9973ef1 100644 --- a/src/server/app.js +++ b/src/server/app.js @@ -21,6 +21,7 @@ import { withGuidance } from './guidance.js'; import { reviewerStats } from './reviewers.js'; import { inferUnits } from './grouping.js'; import { Router, HttpError, readJson, sendJson, RateLimiter } from './http.js'; +import { Regions } from './regions.js'; import { PEImage } from '../shared/pe.js'; import { loadProject, loadToolchainLock } from '../shared/project.js'; import { DEFAULT_LIMITS, lintSubmission } from '../shared/lint.js'; @@ -35,6 +36,7 @@ export const DEFAULT_CONFIG = { leaseMinutes: 120, maxLeasesPerUser: 3, verifyConcurrency: 3, + remoteChecks: false, // checks compile on the contributor's machine (td toolchain install) buildTimeoutMs: 300000, submitsPerHour: 30, runners: { msvc2010: { type: 'docker', image: 'togetherdecomp/msvc2010:1', memory: '1g', cpus: 1 } }, @@ -78,15 +80,18 @@ export function openServer(dataDir, { log = console } = {}) { } const tasks = new Tasks({ store, config, repo, project }); + const regions = new Regions({ store, config, repo, tasks }); + tasks.regions = regions; const peers = new PeerReviews({ store, project }); const verifier = new Verifier({ store, config, repo, project, tasks, getPe, peers, log }); + verifier.regions = regions; const merger = new Merger({ store, config, repo, project, tasks, verifier, getPe, peers, log }); const accounts = new Accounts(store); const originals = new Originals({ store }); const ghidra = new ServerGhidra({ config, store, originals, log }); merger.onMerged = (commit) => ghidra.scheduleRefresh(commit); const checks = new CheckQueue({ store, repo, verifier, config, log }); - return { config, store, repo, project, tasks, verifier, merger, getPe, accounts, peers, originals, ghidra, checks }; + return { config, store, repo, project, tasks, regions, verifier, merger, getPe, accounts, peers, originals, ghidra, checks }; } export function createApp(ctx) { @@ -496,7 +501,58 @@ export function createApp(ctx) { router.on('POST', '/api/tasks/heartbeat-all', ({ req }) => { const user = auth(req); - return tasks.heartbeatAll(user); + return { ...tasks.heartbeatAll(user), regions: ctx.regions.heartbeat(user) }; + }); + + // --- regions: a source file or class, claimed and worked on as a whole (regions.js) + + router.on('GET', '/api/regions', ({ req, query }) => { + const user = auth(req); + return guided(ctx.regions.list({ user, limit: intParam(query.get('limit') ?? '50'), free: query.get('free') === '1', prefix: query.get('prefix') || null })); + }); + + router.on('GET', '/api/regions/mine', ({ req }) => { + const user = auth(req); + return guided({ regions: ctx.regions.mine(user) }); + }); + + router.on('POST', '/api/regions/claim', async ({ req }) => { + const user = auth(req); + const body = await readJson(req, 4096); + reviewFirst(user); + const r = ctx.regions.claim(user, { kind: body.kind ?? 'unit', key: body.key }); + return guided({ + ...r, + baseCommit: repo.head(), + note: r.alsoClaimedBy.length + ? `Also claimed by ${r.alsoClaimedBy.join(', ')}: you may submit any function of it; the first accepted candidate of each function merges. Prefer an unclaimed region (td_regions free) unless you mean to help here.` + : 'Port every function of the region, compile and check locally, then submit them together (td submit --all).', + }); + }); + + router.on('POST', '/api/regions/release', async ({ req }) => { + const user = auth(req); + const body = await readJson(req, 4096); + return ctx.regions.release(user, { kind: body.kind ?? 'unit', key: String(body.key ?? '') }); + }); + + // --- the compiler for local checks: only for approved accounts (the Microsoft files are not for + // the public). The maintainer puts it at /toolchain/toolchain.tar.gz (td-server toolchain). + + const toolchainFile = () => path.join(config.dataDir, 'toolchain', 'toolchain.tar.gz'); + router.on('GET', '/api/toolchain', ({ req }) => { + auth(req); + const f = toolchainFile(); + if (!fs.existsSync(f)) return { available: false, note: 'no compiler package on this server yet; ask a maintainer' }; + const meta = JSON.parse(fs.readFileSync(path.join(path.dirname(f), 'toolchain.json'), 'utf8')); + return { available: true, ...meta, toolchain: project.toolchain, compiler: project.compile?.compiler ?? null }; + }); + router.on('GET', '/api/toolchain/download', ({ req, res }) => { + auth(req); + const f = toolchainFile(); + if (!fs.existsSync(f)) throw new HttpError(404, 'no compiler package on this server yet'); + res.writeHead(200, { 'content-type': 'application/gzip', 'content-length': fs.statSync(f).size, 'content-disposition': 'attachment; filename="toolchain.tar.gz"', 'cache-control': 'private, no-store' }); + fs.createReadStream(f).pipe(res); }); // The submitted definition of one function (annotations through the closing brace). @@ -598,6 +654,10 @@ export function createApp(ctx) { router.on('POST', '/api/check', async ({ req, res }) => { const user = auth(req); + // Checks run on the contributor's machine; the server compiles only submissions and merges. + if (!config.remoteChecks) { + throw new HttpError(410, 'the server no longer compiles checks: compile locally. Run "td toolchain install" once (downloads the compiler), then td_check works on your machine. Submit with td_submit when your region compiles and matches.'); + } checkLimiter.check(user.id); const body = await readJson(req); if (typeof body.baseCommit !== 'string' || !/^[0-9a-f]{40}$/.test(body.baseCommit)) throw new HttpError(400, 'baseCommit must be a full commit hash'); @@ -925,7 +985,7 @@ export async function startServer(dataDir, { role = 'all' } = {}) { } let server = null; if (role !== 'worker') { - const sweeper = setInterval(() => { try { ctx.tasks.sweepExpired(); } catch (e) { console.error(`lease sweep: ${e.message}`); } }, 60_000); + const sweeper = setInterval(() => { try { ctx.tasks.sweepExpired(); ctx.regions.sweepExpired(); } catch (e) { console.error(`lease sweep: ${e.message}`); } }, 60_000); stops.push(() => clearInterval(sweeper)); server = createApp(ctx); await new Promise((r) => server.listen(ctx.config.listen.port, ctx.config.listen.host, r)); diff --git a/src/server/db.js b/src/server/db.js index 7abe129..8967850 100644 --- a/src/server/db.js +++ b/src/server/db.js @@ -238,6 +238,49 @@ CREATE TABLE IF NOT EXISTS events ( type TEXT NOT NULL, data TEXT ); + +-- A region (a source file or a class) an agent registered to work on as a whole. Several agents +-- may hold the same region; each may then submit any of its functions, and the first accepted +-- candidate of a function wins (the others are superseded). +CREATE TABLE IF NOT EXISTS region_claims ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + kind TEXT NOT NULL CHECK (kind IN ('unit', 'class')), + key TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + UNIQUE (user_id, kind, key) +); +CREATE INDEX IF NOT EXISTS region_claims_key ON region_claims(kind, key); + +-- Game credits: earned for work that reached main and for reviews that agreed with the outcome, +-- spent and won in the games. Play money only. The balance is the sum. +CREATE TABLE IF NOT EXISTS credit_ledger ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + amount INTEGER NOT NULL, + reason TEXT NOT NULL, -- merged, identical, review, game:, grant + ref TEXT, -- what it was for (attempt, review or round), once per reason + created_at INTEGER NOT NULL, + UNIQUE (user_id, reason, ref) +); +CREATE INDEX IF NOT EXISTS credit_ledger_user ON credit_ledger(user_id); + +CREATE TABLE IF NOT EXISTS game_rounds ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + game TEXT NOT NULL, + bet INTEGER NOT NULL, + payout INTEGER NOT NULL, + detail TEXT, + created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS game_rounds_at ON game_rounds(created_at); + +-- A function on main stays done: a competing candidate that finishes later must not revive it. +CREATE TRIGGER IF NOT EXISTS tasks_done_stays_done BEFORE UPDATE OF state ON tasks + WHEN OLD.state IN ('done', 'skipped') AND NEW.state NOT IN ('done', 'skipped', 'open') + BEGIN SELECT RAISE(IGNORE); END; `; export const now = () => Date.now(); @@ -262,6 +305,8 @@ export class Store { this.addColumn('attempts', 'skip_category', 'TEXT'); this.addColumn('symbols', 'skip_reason', 'TEXT'); // accepted: not ported, not counted in progress // human-readable name, e.g. "World of Warcraft 5.0.3 (build 15890, x64)" this.addColumn('attempts', 'batch_id', 'INTEGER REFERENCES batches(id)'); + // A candidate for a function another agent holds (region claims): it does not move the task. + this.addColumn('attempts', 'competing', 'INTEGER NOT NULL DEFAULT 0'); this.db.exec('CREATE INDEX IF NOT EXISTS attempts_batch ON attempts(batch_id)'); } diff --git a/src/server/merge.js b/src/server/merge.js index 99ec971..f0f9138 100644 --- a/src/server/merge.js +++ b/src/server/merge.js @@ -371,6 +371,7 @@ export class Merger { } } this.store.event('merged', batch.user_id, { batch: batch.id, functions: attempts.length, commit }); + this.supersedeCompetitors(attempts, batch.id, commit); if (split) { this.rebaseRemainder(split, merged.files, commit); settleBatch(this.store, split.parent.id, 'its approved functions merged; the others were sent back'); @@ -388,6 +389,28 @@ export class Merger { this.log.info(`merged batch ${batch.id} (${attempts.length} functions) as ${commit.slice(0, 10)}`); } + /** + * Other candidates of the functions that just merged (region claims let several agents submit + * one function): they are superseded, and the rest of their batches goes on without them. + */ + supersedeCompetitors(merged, batchId, commit) { + const batches = new Set(); + for (const a of merged) { + const losers = this.store.all( + `SELECT id, batch_id FROM attempts WHERE task_id = ? AND id != ? AND state IN ('queued', 'verifying', 'peer_review', 'in_review', 'approved')`, + a.task_id, + a.id, + ); + for (const l of losers) { + const feedback = { summary: `another candidate for ${a.symbol_id} was accepted and merged first (batch ${batchId}, ${commit.slice(0, 10)}); this one is no longer needed` }; + this.store.run(`UPDATE attempts SET state = 'superseded', finished_at = ?, result_json = json_set(COALESCE(result_json, '{}'), '$.feedback', json(?)) WHERE id = ?`, now(), JSON.stringify(feedback), l.id); + this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', l.id); + if (l.batch_id) batches.add(l.batch_id); + } + } + for (const b of batches) settleBatch(this.store, b, 'its other functions were merged from competing submissions'); + } + /** * What a merge changes about one function's source, for the commit message: an annotation on * code that was already there (an identical copy, or code marked INLINED) shows up in the diff diff --git a/src/server/peer.js b/src/server/peer.js index 82fb2eb..2f31200 100644 --- a/src/server/peer.js +++ b/src/server/peer.js @@ -3,12 +3,12 @@ // A reviewer reads the original function (assembly and pseudocode, from their own pinned Ghidra // via `td disasm` / `td decompile`) and the submitted code, and decides whether the logic is the // same. No automated verdicts: the context deliberately leaves out the server's byte diff and score. -// An attempt stays in peer review until `peerReviews` (default 3) different accounts agree: that -// many "same" verdicts send it on to human review, that many "different" ones send it back to its -// author with their reasons. With mixed verdicts, further reviewers are let in until one side -// gets there. A function stays open for more reviewers while it waits for a human: with -// `peerAutoAccept` (default 5) "same" verdicts it is accepted without human review. If anyone said -// "different", it is merged all the same and kept in later_review for a human (merge.js). +// An attempt stays in peer review until one side has enough verdicts from different accounts: +// `peerAutoAccept` (default 5) "same" verdicts accept it, and it merges with no human step; +// `peerReviews` (default 3) "different" ones send it back to its author with their reasons. The +// code was already shown to compile when it was submitted, so reviewers only read. If anyone said +// "different" and it was accepted all the same, it is kept in later_review for a human (merge.js). +// (Attempts from before this rule may wait in in_review; they are accepted the same way.) import { HttpError } from './http.js'; import { now } from './db.js'; @@ -23,7 +23,7 @@ const REVIEW_GROUP = 40; export const PEER_INSTRUCTIONS = [ 'You hold every function listed under functions. Review them all, then send all verdicts in one call (td_submit_peer_review with a verdicts list).', 'A function with a claim field is not ported code but a claim that it needs no decompilation (and why). Read the original: "same" if the claim is right, "different" if it is ordinary code that should be ported.', - 'Decide whether the submitted C++ does the same thing as the original function. It is not byte-identical; your judgment is what counts.', + 'Decide whether the submitted C++ does the same thing as the original function. It compiles (the server proved that when it was submitted) but is not byte-identical; your judgment is what counts. Five "same" verdicts merge it, three "different" send it back.', 'Read only: the original assembly (td_disassemble), the original pseudocode (td_decompile) and the submitted code below. Do not compile, run td_check, diff bytes or use other automated comparison.', 'Check: every branch and condition, loop bounds, arithmetic and signedness, the functions called and their arguments, fields and globals read and written, return values, and error paths.', 'Answer "same" only if a caller could not tell the two apart. Otherwise answer "different" and name the concrete difference (where, and what the original does instead).', @@ -68,16 +68,13 @@ export class PeerReviews { AND NOT EXISTS (SELECT 1 FROM users b WHERE b.id = ? AND b.peer_review_blocked = 1) AND NOT EXISTS (SELECT 1 FROM peer_reviews p WHERE p.attempt_id = a.id AND p.reviewer_id = ?) ) - WHERE (state = 'peer_review' AND MAX(confirmations, objections) + in_progress < ?) - OR (state = 'in_review' AND ? > ? AND confirmations + in_progress < ?)`, + WHERE objections < ? AND confirmations + in_progress < ?`, now(), user.id, user.id, user.id, this.required, this.autoAccept, - this.required, - this.autoAccept, ); // Random by batch, so a batch stays together. const key = new Map(); @@ -89,11 +86,6 @@ export class PeerReviews { return rows.sort((a, b) => a.shuffle - b.shuffle).slice(0, limit); } - // A function waiting for a human takes further reviewers toward auto-acceptance. - openForAutoAccept(r) { - return this.autoAccept > this.required && r.confirmations + r.in_progress < this.autoAccept; - } - /** * Leases a review (a given attempt, or the oldest available) together with every other function * of the same batch that waits for this reviewer, and returns the context of all of them. @@ -165,7 +157,8 @@ export class PeerReviews { const ids = attemptIds ?? (a.batch_id ? this.store.all(`SELECT id FROM attempts WHERE batch_id = ? AND state = 'peer_review' ORDER BY id`, a.batch_id).map((r) => r.id) : [a.id]); return { attemptId: a.id, - required: this.required, + required: this.autoAccept, // "same" verdicts that accept it + rejectAt: this.required, // "different" verdicts that send it back instructions: PEER_INSTRUCTIONS, // Every function this review covers (one batch); give one verdict per attemptId. functions: (ids.length ? ids : [a.id]).map((id) => this.functionInfo(id, v)), @@ -199,7 +192,7 @@ export class PeerReviews { const leased = this.store.get('SELECT 1 FROM peer_leases WHERE attempt_id = ? AND reviewer_id = ? AND expires_at > ?', attemptId, user.id, now()); const counts = this.counts(attemptId); const others = this.store.get('SELECT COUNT(*) AS n FROM peer_leases WHERE attempt_id = ? AND reviewer_id != ? AND expires_at > ?', attemptId, user.id, now()).n; - const full = a.state === 'in_review' ? !this.openForAutoAccept({ objections: counts.different, confirmations: counts.same, in_progress: others }) : Math.max(counts.same, counts.different) + others >= this.required; + const full = counts.same + others >= this.autoAccept; if (!leased && full) throw new HttpError(409, 'enough reviewers are already on this attempt; claim another'); this.store.run( @@ -211,38 +204,31 @@ export class PeerReviews { const same = counts.same + (verdict === 'same' ? 1 : 0); const different = counts.different + (verdict === 'different' ? 1 : 0); - if (a.state === 'in_review') { - // Already waiting for a human: enough agreeing peers accept it (objections are kept for later). - if (same >= this.autoAccept) { - this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId); - this.store.run(`UPDATE attempts SET state = 'approved' WHERE id = ?`, attemptId); - this.store.run(`UPDATE tasks SET state = 'approved', updated_at = ? WHERE id = ?`, now(), a.task_id); - this.store.event('auto_accepted', null, { attempt: attemptId, same, different }); - return { state: 'approved', same, different, required: this.required }; - } - return { state: 'in_review', same, different, required: this.required }; + // A competing candidate does not take a task over from the agent holding it. + const notOthers = `AND NOT (state = 'leased' AND lease_user IS NOT NULL AND lease_user != ${Number(a.user_id)})`; + if (same >= this.autoAccept) { + // Accepted: it merges (objections, if any, are kept for a human to look at later). + this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId); + this.store.run(`UPDATE attempts SET state = 'approved' WHERE id = ?`, attemptId); + this.store.run(`UPDATE tasks SET state = 'approved', updated_at = ? WHERE id = ? ${notOthers}`, now(), a.task_id); + this.store.event('auto_accepted', null, { attempt: attemptId, same, different }); + return { state: 'approved', same, different, required: this.autoAccept }; } if (different >= this.required) { // Enough independent reviewers found a difference: back to the author with their reasons. this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId); this.store.run(`UPDATE attempts SET state = 'needs_cleanup' WHERE id = ?`, attemptId); this.store.run( - `UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ?`, - a.user_id, now() + 6 * 3600_000, now(), a.task_id, + `UPDATE tasks SET state = 'needs_cleanup', cleanup_cycles = cleanup_cycles + 1, preferred_user = ?, preferred_until = ?, updated_at = ? WHERE id = ? ${notOthers} + AND NOT EXISTS (SELECT 1 FROM attempts o WHERE o.task_id = tasks.id AND o.id != ? AND o.state IN ('queued', 'verifying', 'peer_review', 'in_review', 'approved', 'merging'))`, + a.user_id, now() + 6 * 3600_000, now(), a.task_id, attemptId, ); // Only this function goes back; the rest of its batch carries on. settleBatch(this.store, a.batch_id, `${different} peer reviews found a difference in ${a.task_id}`); return { state: 'needs_cleanup', same, different, required: this.required }; } - if (same >= this.required) { - // Leases toward auto-acceptance stay. - if (!(this.autoAccept > this.required)) this.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', attemptId); - this.store.run(`UPDATE attempts SET state = 'in_review' WHERE id = ?`, attemptId); - this.store.run(`UPDATE tasks SET state = 'in_review', updated_at = ? WHERE id = ?`, now(), a.task_id); - return { state: 'in_review', same, different, required: this.required }; - } // Undecided: it stays in review until one side has enough verdicts. - return { state: 'peer_review', same, different, required: this.required }; + return { state: a.state, same, different, required: this.autoAccept }; }); } @@ -274,6 +260,6 @@ export class PeerReviews { } confirmed(attemptId) { - return this.counts(attemptId).same >= this.required; + return this.counts(attemptId).same >= this.autoAccept; } } diff --git a/src/server/regions.js b/src/server/regions.js new file mode 100644 index 0000000..020dee7 --- /dev/null +++ b/src/server/regions.js @@ -0,0 +1,189 @@ +// Regions: a source file (module) or a class, claimed and worked on as a whole. +// +// An agent lists the regions (what is open, done, in flight, and who already works there), looks +// at the candidates in its own checkout, and registers the one it takes. That leases every open +// function of the region to it in one step. A region may be claimed by several agents; each may +// then submit any function of it that is not on main yet. Competing candidates of a function are +// verified and reviewed independently, and the first accepted one merges (merge.js supersedes the +// others), so two agents' work combines per function. + +import { HttpError } from './http.js'; +import { now } from './db.js'; +import { className } from './grouping.js'; +import { CLAIMABLE } from './tasks.js'; + +const IN_FLIGHT = ['verifying', 'peer_review', 'in_review', 'approved', 'merging']; +const FINISHED = ['done', 'skipped']; + +export class Regions { + constructor({ store, config, repo, tasks }) { + this.store = store; + this.config = config; + this.repo = repo; + this.tasks = tasks; + } + + get leaseMs() { + return (this.config.leaseMinutes ?? 120) * 60_000; + } + + /** + * Every source file with functions still to do: counts per state and who holds it. Sorted for + * picking: unclaimed and unblocked first, then the most open functions. + */ + list({ limit = 100, user = null, free = false, prefix = null } = {}) { + const rows = this.store.all( + `SELECT s.unit AS key, t.state, t.lease_user, COUNT(*) AS n, SUM(COALESCE(a.size, 0)) AS bytes + FROM tasks t JOIN symbols s ON s.id = t.symbol_id + LEFT JOIN symbol_addresses a ON a.symbol_id = s.id AND a.version_id = ? + WHERE s.unit IS NOT NULL ${prefix ? 'AND s.unit LIKE ?' : ''} + GROUP BY s.unit, t.state, t.lease_user`, + this.store.currentVersion()?.id ?? '', + ...(prefix ? [`${prefix}%`] : []), + ); + const claims = this.activeClaims('unit'); + const byKey = new Map(); + for (const r of rows) { + const g = byKey.get(r.key) ?? { kind: 'unit', key: r.key, functions: 0, bytes: 0, open: 0, openBytes: 0, done: 0, inFlight: 0, heldByOthers: 0, heldByYou: 0 }; + g.functions += r.n; + g.bytes += r.bytes; + if (CLAIMABLE.includes(r.state)) (g.open += r.n), (g.openBytes += r.bytes); + else if (FINISHED.includes(r.state)) g.done += r.n; + else if (IN_FLIGHT.includes(r.state)) g.inFlight += r.n; + else if (r.state === 'leased') { + if (user && r.lease_user === user.id) g.heldByYou += r.n; + else g.heldByOthers += r.n; + } + byKey.set(r.key, g); + } + const out = []; + for (const g of byKey.values()) { + if (!g.open && !g.heldByOthers && !g.heldByYou) continue; + const c = claims.get(g.key) ?? []; + g.claimedBy = c.map((x) => x.name); + g.claimedByYou = !!user && c.some((x) => x.user_id === user.id); + // Blocked: someone else holds functions of it, or has some on their way to main (their + // submission would conflict with a rewrite of the file). + g.blocked = g.heldByOthers > 0 || g.inFlight > 0; + g.free = !g.blocked && !g.claimedBy.length; + if (free && !g.free) continue; + out.push(g); + } + out.sort((a, b) => Number(b.free) - Number(a.free) || a.claimedBy.length - b.claimedBy.length || b.open - a.open || a.key.localeCompare(b.key)); + return { total: out.length, regions: out.slice(0, Math.min(Math.max(1, limit | 0), 1000)) }; + } + + activeClaims(kind) { + const out = new Map(); + for (const r of this.store.all( + `SELECT rc.user_id, rc.key, u.name FROM region_claims rc JOIN users u ON u.id = rc.user_id WHERE rc.kind = ? AND rc.expires_at > ?`, + kind, + now(), + )) { + if (!out.has(r.key)) out.set(r.key, []); + out.get(r.key).push(r); + } + return out; + } + + /** Task ids of a region, with their state and holder. */ + members(kind, key) { + if (kind === 'unit') { + return this.store.all(`SELECT t.id, t.state, t.lease_user, t.symbol_id, s.name FROM tasks t JOIN symbols s ON s.id = t.symbol_id WHERE s.unit = ? ORDER BY t.id`, key); + } + const like = `${key.replace(/[%_]/g, (c) => `\\${c}`)}::%`; + return this.store + .all(`SELECT t.id, t.state, t.lease_user, t.symbol_id, s.name FROM tasks t JOIN symbols s ON s.id = t.symbol_id WHERE s.name LIKE ? ESCAPE '\\' ORDER BY t.id`, like) + .filter((r) => className(r.name) === key); + } + + /** + * Registers a region for `user` and leases its open functions to them. Functions another agent + * already holds stay theirs; the claim still lets this agent submit its own candidates for them. + */ + claim(user, { kind = 'unit', key }) { + if (!['unit', 'class'].includes(kind)) throw new HttpError(400, "kind must be 'unit' or 'class'"); + key = String(key ?? '').trim(); + if (!key || key.length > 300) throw new HttpError(400, 'key must name the source file (kind unit) or the class (kind class)'); + return this.store.tx(() => { + const members = this.members(kind, key); + if (!members.length) throw new HttpError(404, `no functions in ${kind} ${key}`); + const left = members.filter((m) => !FINISHED.includes(m.state)); + if (!left.length) throw new HttpError(409, `${key} is finished: every function is on main`); + const expires = now() + this.leaseMs; + this.store.run( + `INSERT INTO region_claims (user_id, kind, key, created_at, expires_at) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(user_id, kind, key) DO UPDATE SET expires_at = excluded.expires_at`, + user.id, kind, key, now(), expires, + ); + const held = this.tasks.leasesOf(user.id).length; + let room = this.tasks.maxLeased - held; + const head = this.repo.head(); + const leased = []; + for (const m of members) { + if (room <= 0) break; + if (!CLAIMABLE.includes(m.state)) continue; + const t = this.store.get('SELECT preferred_user, preferred_until FROM tasks WHERE id = ?', m.id); + if (t.preferred_user && t.preferred_user !== user.id && t.preferred_until > now()) continue; + this.store.run(`UPDATE tasks SET state = 'leased', lease_user = ?, lease_expires = ?, lease_base_commit = ?, updated_at = ? WHERE id = ?`, user.id, expires, head, now(), m.id); + leased.push(m.id); + room--; + } + const others = (this.activeClaims(kind).get(key) ?? []).filter((c) => c.user_id !== user.id).map((c) => c.name); + this.store.event('region_claim', user.id, { kind, key, leased: leased.length, shared: others.length }); + return { + region: { kind, key }, + leased, + functions: members.map((m) => ({ + taskId: m.id, + symbol: m.symbol_id, + name: m.name, + state: leased.includes(m.id) ? 'leased' : m.state, + yours: leased.includes(m.id) || (m.state === 'leased' && m.lease_user === user.id), + shared: m.state === 'leased' && m.lease_user !== user.id, + })), + alsoClaimedBy: others, + leaseExpires: expires, + allowedPaths: this.tasks.allowedPathsForTasks(members.slice(0, 500).map((m) => m.id)), + }; + }); + } + + release(user, { kind = 'unit', key }) { + return this.store.tx(() => { + const r = this.store.run('DELETE FROM region_claims WHERE user_id = ? AND kind = ? AND key = ?', user.id, kind, key); + let released = 0; + for (const m of this.members(kind, key)) { + if (m.state === 'leased' && m.lease_user === user.id) { + this.tasks.reopen(m.id); + released++; + } + } + if (r.changes) this.store.event('region_release', user.id, { kind, key, released }); + return { released }; + }); + } + + /** The regions `user` holds. */ + mine(user) { + return this.store.all('SELECT kind, key, created_at, expires_at FROM region_claims WHERE user_id = ? AND expires_at > ? ORDER BY created_at', user.id, now()); + } + + heartbeat(user) { + return Number(this.store.run('UPDATE region_claims SET expires_at = ? WHERE user_id = ? AND expires_at > ?', now() + this.leaseMs, user.id, now()).changes); + } + + /** Whether `user` holds a live region claim that covers this task's function. */ + covers(user, task) { + const sym = this.store.get('SELECT unit, name FROM symbols WHERE id = ?', task.symbol_id); + if (!sym) return false; + const t = now(); + if (sym.unit && this.store.get(`SELECT 1 FROM region_claims WHERE user_id = ? AND kind = 'unit' AND key = ? AND expires_at > ?`, user.id, sym.unit, t)) return true; + const cls = className(sym.name); + return !!cls && !!this.store.get(`SELECT 1 FROM region_claims WHERE user_id = ? AND kind = 'class' AND key = ? AND expires_at > ?`, user.id, cls, t); + } + + sweepExpired() { + return Number(this.store.run('DELETE FROM region_claims WHERE expires_at < ?', now()).changes); + } +} diff --git a/src/server/tasks.js b/src/server/tasks.js index fb54855..1c70656 100644 --- a/src/server/tasks.js +++ b/src/server/tasks.js @@ -254,6 +254,20 @@ export class Tasks { return t; } + /** + * A task `user` may submit: one they hold, or (a competing candidate) one in a region they + * claimed that is not on main yet. Returns the task and whether it is theirs. + */ + requireSubmittable(user, taskId) { + const t = this.get(taskId); + if (t.state === 'leased' && t.lease_user === user.id) return { task: t, own: true }; + if (this.regions?.covers(user, t)) { + if (['done', 'skipped', 'merging'].includes(t.state)) throw new HttpError(409, `task ${taskId} (${t.symbol_id}) is ${t.state === 'merging' ? 'being merged' : 'already on main'}; leave it out`); + return { task: t, own: CLAIMABLE.includes(t.state) }; + } + throw new HttpError(409, `you do not hold task ${taskId} (${t.symbol_id}) and no region you claimed covers it`); + } + heartbeat(user, taskId) { this.requireLease(user, taskId); const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000; @@ -292,7 +306,7 @@ export class Tasks { if (!ids.length) throw new HttpError(400, 'taskIds must list the tasks this submission covers'); if (ids.length > this.maxLeased) throw new HttpError(400, `at most ${this.maxLeased} tasks per submission`); return this.store.tx(() => { - for (const id of ids) this.requireLease(user, id); + const own = new Set(ids.filter((id) => this.requireSubmittable(user, id).own)); // Keep the verification queue moving: a few batches waiting per account at a time. const max = this.config.maxQueuedBatches ?? 8; const waiting = this.store.get(`SELECT COUNT(*) AS n FROM batches WHERE user_id = ? AND state IN ('queued', 'verifying')`, user.id).n; @@ -308,11 +322,12 @@ export class Tasks { const attempts = []; for (const id of ids) { const r = this.store.run( - `INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, batch_id, created_at) VALUES (?, ?, ?, '{}', ?, ?, ?, 'queued', ?, ?)`, - id, user.id, baseCommit, notes, agentMeta, envDigest, batchId, now(), + `INSERT INTO attempts (task_id, user_id, base_commit, files_json, notes, agent_meta, env_digest, state, batch_id, competing, created_at) VALUES (?, ?, ?, '{}', ?, ?, ?, 'queued', ?, ?, ?)`, + id, user.id, baseCommit, notes, agentMeta, envDigest, batchId, own.has(id) ? 0 : 1, now(), ); - attempts.push({ taskId: id, attemptId: Number(r.lastInsertRowid) }); - this.store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), id); + attempts.push({ taskId: id, attemptId: Number(r.lastInsertRowid), ...(own.has(id) ? {} : { competing: true }) }); + // A competing candidate leaves the task as its holder has it. + if (own.has(id)) this.store.run(`UPDATE tasks SET state = 'verifying', lease_user = NULL, lease_expires = NULL, updated_at = ? WHERE id = ?`, now(), id); } this.store.event('submit', user.id, { batch: batchId, tasks: ids.length }); return { batchId, attempts }; diff --git a/src/server/verify.js b/src/server/verify.js index f4ad594..5038cb2 100644 --- a/src/server/verify.js +++ b/src/server/verify.js @@ -314,6 +314,11 @@ export class Verifier { } workedOnByOthers(symbolId, userId) { + // A region this user claimed is shared work: changes to its functions are theirs to make too. + if (this.regions) { + const t = this.store.get('SELECT * FROM tasks WHERE symbol_id = ?', symbolId); + if (t && this.regions.covers({ id: userId }, t)) return false; + } return !!this.store.get( `SELECT 1 FROM tasks t WHERE t.symbol_id = ? AND ( (t.state = 'leased' AND t.lease_user != ?) @@ -343,7 +348,12 @@ export class Verifier { } async verifyBatch(batch) { - const attempts = batchAttempts(this.store, batch.id); + // Superseded while it waited (a competing candidate merged first): nothing to judge. + const attempts = batchAttempts(this.store, batch.id).filter((a) => a.state === 'verifying' || a.state === 'queued'); + if (!attempts.length) { + this.store.run(`UPDATE batches SET state = 'superseded', finished_at = ? WHERE id = ?`, now(), batch.id); + return; + } const submitted = JSON.parse(batch.files_json); const finishAll = (state, feedback) => { for (const a of attempts) this.finish(a, state, 0, feedback); @@ -407,6 +417,7 @@ export class Verifier { const expires = now() + (this.config.leaseMinutes ?? 120) * 60_000; this.store.tx(() => { this.store.run(`UPDATE attempts SET state = 'withdrawn', finished_at = ?, result_json = ? WHERE id = ?`, now(), JSON.stringify({ feedback: { summary: 'not written: no annotation for it in the submission; it stays in your claim' } }), attempt.id); + if (attempt.competing) return; // the task is its holder's this.store.run(`UPDATE tasks SET state = 'leased', lease_user = ?, lease_expires = ?, updated_at = ? WHERE id = ?`, attempt.user_id, expires, now(), attempt.task_id); }); } @@ -492,6 +503,12 @@ export class Verifier { const onTheWay = ['in_review', 'peer_review', 'approved'].includes(state); let taskState = onTheWay ? state : 'needs_cleanup'; const better = state !== 'rejected' && score > task.best_score; + // A competing candidate records a better score, but the task stays as its holder has it. + if (attempt.competing) { + if (better) this.store.run('UPDATE tasks SET best_score = ?, best_attempt = ?, updated_at = ? WHERE id = ?', score, attempt.id, now(), task.id); + this.store.event('verified', attempt.user_id, { attempt: attempt.id, state, score, competing: true }); + return; + } const cycles = task.cleanup_cycles + (onTheWay ? 0 : 1); if (taskState === 'needs_cleanup' && cycles >= this.project.matchPolicy.maxCleanupCycles) taskState = 'hard'; if (state === 'rejected' && !task.best_attempt) taskState = 'open'; diff --git a/test/server.test.js b/test/server.test.js index f7cd5ab..7c445df 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -20,7 +20,7 @@ const quiet = { info() {}, error() {}, warn() {} }; before(async () => { dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'td-test-')); - fs.writeFileSync(path.join(dataDir, 'server.json'), JSON.stringify({ listen: { host: '127.0.0.1', port: 0 }, verifyConcurrency: 1 })); + fs.writeFileSync(path.join(dataDir, 'server.json'), JSON.stringify({ listen: { host: '127.0.0.1', port: 0 }, verifyConcurrency: 1, remoteChecks: true })); const repoDir = path.join(dataDir, 'repo'); fs.cpSync(path.join(ROOT, 'template'), repoDir, { recursive: true }); const git = (...a) => execFileSync('git', a, { cwd: repoDir, stdio: 'pipe' }); @@ -297,12 +297,14 @@ test('changing the password logs out browser sessions', async () => { await login('erin', 'a brand new password'); }); -test('peer review: three other agents must agree, by reading only', async () => { +test('peer review: five "same" accept, three "different" send back, by reading only', async () => { const author = await activeUser('pauthor'); const r1 = await activeUser('peer1'); const r2 = await activeUser('peer2'); const r3 = await activeUser('peer3'); const r4 = await activeUser('peer4'); + const r5 = await activeUser('peer5'); + const r6 = await activeUser('peer6'); ctx.merger.pump = async () => {}; const authorId = ctx.accounts.byName('pauthor').id; @@ -319,23 +321,25 @@ test('peer review: three other agents must agree, by reading only', async () => assert.ok(!(await call('GET', '/api/peer-reviews', null, author)).body.some((x) => x.attempt_id === attempt), 'authors do not see their own attempts'); const claim = await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r1); assert.equal(claim.status, 200); - assert.equal(claim.body.required, 3); + assert.equal(claim.body.required, 5); + assert.equal(claim.body.rejectAt, 3); assert.ok(claim.body.instructions.some((i) => /Do not compile/.test(i))); assert.equal(JSON.stringify(claim.body).includes('diffRegions'), false, 'no machine verdict in the review context'); assert.equal(claim.body.original.read[0], `td_disassemble ${task.symbol_id}`); - await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r2); - await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r3); - assert.equal((await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r4)).status, 409, 'no more reviewers than needed'); + for (const r of [r2, r3, r4, r5]) await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r); + assert.equal((await call('POST', '/api/peer-reviews/claim', { attemptId: attempt }, r6)).status, 409, 'no more reviewers than needed'); assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'short' }, r1)).status, 400); assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'all branches and calls agree' }, author)).status, 403); assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'all branches and calls agree' }, r1)).body.state, 'peer_review'); assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'same loop bounds and the same return' }, r1)).status, 409, 'one verdict per account'); assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'checked every condition and field access' }, r2)).body.state, 'peer_review'); - const done = await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'identical behavior, only register use differs' }, r3); - assert.equal(done.body.state, 'in_review'); - assert.equal(ctx.tasks.get(task.id).state, 'in_review'); + assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'identical behavior, only register use differs' }, r3)).body.state, 'peer_review', 'three are not enough to accept'); + assert.equal((await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'same calls in the same order, same result' }, r4)).body.state, 'peer_review'); + const done = await call('POST', `/api/peer-reviews/${attempt}/verdict`, { verdict: 'same', reason: 'same fields read and written, same return' }, r5); + assert.equal(done.body.state, 'approved', 'five accept it, with no human step'); + assert.equal(ctx.tasks.get(task.id).state, 'approved'); assert.ok(ctx.peers.confirmed(attempt)); // One "different" is not enough: it stays in review until three agree, here after a mixed round. @@ -436,16 +440,16 @@ test('five agreeing peers accept a function without human review, also over an o const clean = make(0x401900); for (let i = 0; i < 3; i++) await review(clean.attempt, peers[i], 'same'); - assert.equal(ctx.store.get('SELECT state FROM attempts WHERE id = ?', clean.attempt).state, 'in_review', 'three send it to a human'); - assert.equal((await review(clean.attempt, peers[3], 'same')).state, 'in_review', 'it stays open for more reviewers'); + assert.equal(ctx.store.get('SELECT state FROM attempts WHERE id = ?', clean.attempt).state, 'peer_review', 'three are not enough'); + assert.equal((await review(clean.attempt, peers[3], 'same')).state, 'peer_review', 'it stays open for more reviewers'); assert.equal((await review(clean.attempt, peers[4], 'same')).state, 'approved', 'five accept it'); assert.equal(ctx.store.get('SELECT state FROM tasks WHERE id = ?', clean.task).state, 'approved'); const contested = make(0x401940); await review(contested.attempt, peers[0], 'different'); for (let i = 1; i <= 3; i++) await review(contested.attempt, peers[i], 'same'); - assert.equal(ctx.store.get('SELECT state FROM attempts WHERE id = ?', contested.attempt).state, 'in_review'); - assert.equal((await review(contested.attempt, peers[4], 'same')).state, 'in_review'); + assert.equal(ctx.store.get('SELECT state FROM attempts WHERE id = ?', contested.attempt).state, 'peer_review'); + assert.equal((await review(contested.attempt, peers[4], 'same')).state, 'peer_review'); assert.equal((await review(contested.attempt, peers[5], 'same')).state, 'approved', 'five "same" accept it over one objection (flagged for later at merge)'); for (const x of [clean, contested]) { @@ -623,7 +627,7 @@ test('the review list skips fully leased functions before applying its limit', a const author = ctx.accounts.byName('rlauthor'); const viewer = ctx.accounts.byName('rlviewer'); const others = []; - for (let i = 1; i <= 3; i++) { + for (let i = 1; i <= 5; i++) { await activeUser(`rlpeer${i}`); others.push(ctx.accounts.byName(`rlpeer${i}`)); } @@ -639,7 +643,7 @@ test('the review list skips fully leased functions before applying its limit', a const free = make(0x410040); for (const o of others) ctx.store.run('INSERT INTO peer_leases (attempt_id, reviewer_id, expires_at) VALUES (?, ?, ?)', full.attempt, o.id, now() + 3600_000); const ids = ctx.peers.available(viewer, 200).map((r) => r.attempt_id); - assert.ok(!ids.includes(full.attempt), 'three reviewers on it already'); + assert.ok(!ids.includes(full.attempt), 'five reviewers on it already'); assert.ok(ids.includes(free.attempt)); for (const x of [full, free]) { ctx.store.run('DELETE FROM peer_leases WHERE attempt_id = ?', x.attempt); @@ -872,7 +876,8 @@ test('approving a batch leaves its functions in peer review alone unless asked', test('split: the approved part of a batch merges while the rest waits for review', async () => { const author = await activeUser('sauthor'); - const peers = [await activeUser('speer1'), await activeUser('speer2'), await activeUser('speer3')]; + const peers = []; + for (let i = 1; i <= 5; i++) peers.push(await activeUser(`speer${i}`)); await ctx.accounts.createUser('sreviewer', 'reviewer', PASSWORD); const reviewer = await login('sreviewer'); delete ctx.merger.pump; @@ -917,13 +922,14 @@ test('split: the approved part of a batch merges while the rest waits for review assert.equal(rest.base_commit, head1); assert.equal(JSON.parse(rest.files_json)['src/split.cpp'], source); - // D passes review later and merges on top. + // D passes review later (five "same", no human step) and merges on top. const att = (await batchState(s.body.batchId)).functions.find((f) => f.symbol === d).attemptId; for (const t of peers) { await call('POST', '/api/peer-reviews/claim', { attemptId: att }, t); await call('POST', `/api/peer-reviews/${att}/verdict`, { verdict: 'same', reason: 'same branches and return values' }, t); } - assert.equal((await request('POST', `/api/batches/${s.body.batchId}/review`, { action: 'approve' }, { cookie: reviewer })).status, 200); + assert.equal(ctx.store.get('SELECT state FROM attempts WHERE id = ?', att).state, 'approved'); + void reviewer; await ctx.merger.pump(); assert.equal(ctx.store.get('SELECT state FROM batches WHERE id = ?', s.body.batchId).state, 'merged'); assert.equal(ctx.repo.show(ctx.repo.head(), 'src/split.cpp'), source); @@ -933,7 +939,8 @@ test('split: the approved part of a batch merges while the rest waits for review test('batches: one compile, auto-approval, peer review, one function sent back, carry-over', async () => { const author = await activeUser('bauthor'); - const peers = [await activeUser('bpeer1'), await activeUser('bpeer2'), await activeUser('bpeer3')]; + const peers = []; + for (let i = 1; i <= 5; i++) peers.push(await activeUser(`bpeer${i}`)); await ctx.accounts.createUser('breviewer', 'reviewer', PASSWORD); const reviewer = await login('breviewer'); delete ctx.merger.pump; // an earlier test stubbed it on the instance; use the real one here @@ -982,7 +989,7 @@ test('batches: one compile, auto-approval, peer review, one function sent back, // Round 1: a peer claim covers the batch's waiting functions; all three find a difference. const pc = await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, peers[0]); assert.deepEqual(pc.body.functions.map((f) => f.symbol), [b]); - for (const t of peers) { + for (const t of peers.slice(0, 3)) { await call('POST', '/api/peer-reviews/claim', { attemptId: r1[b].attemptId }, t); await call('POST', `/api/peer-reviews/${r1[b].attemptId}/verdict`, { verdict: 'different', reason: 'the original checks the flag first, this does not' }, t); } @@ -995,9 +1002,9 @@ test('batches: one compile, auto-approval, peer review, one function sent back, assert.equal(ctx.store.get('SELECT state FROM tasks WHERE symbol_id = ?', a).state, 'done'); assert.equal((await batchState(s1.body.batchId)).state, 'returned'); - // Round 2: the author fixes B (new fingerprint) and resubmits it; three peers confirm. + // Round 2: the author fixes B (new fingerprint) and resubmits it; five peers confirm. bHash = 'hash-b-2'; - const objector = await activeUser('bpeer4'); + const objector = await activeUser('bpeer6'); const taskOfB = claim.body.tasks.find((t) => t.symbol === b).task; assert.equal((await call('POST', '/api/tasks/claim', { taskId: taskOfB }, author)).status, 200); const s2 = await call('POST', '/api/batches', { taskIds: [taskOfB], baseCommit: ctx.repo.head(), files: { 'src/batch.cpp': source } }, author); @@ -1009,11 +1016,8 @@ test('batches: one compile, auto-approval, peer review, one function sent back, await call('POST', '/api/peer-reviews/claim', { attemptId: r2[b].attemptId }, t); await call('POST', `/api/peer-reviews/${r2[b].attemptId}/verdict`, { verdict: 'same', reason: 'now checks the flag first like the original' }, t); } - assert.equal((await batchState(s2.body.batchId)).functions.find((f) => f.symbol === b).state, 'in_review'); - - // One human click for the batch; one merge commit. - const rev = await request('POST', `/api/batches/${s2.body.batchId}/review`, { action: 'approve' }, { cookie: reviewer }); - assert.equal(rev.status, 200, JSON.stringify(rev.body)); + assert.equal((await batchState(s2.body.batchId)).functions.find((f) => f.symbol === b).state, 'approved', 'five "same" accept it over one objection'); + void reviewer; // Nothing on main changed since verification: the merge reuses that compile. let compiles = 0; const counted = ctx.verifier.check; @@ -1041,3 +1045,77 @@ test('batches: one compile, auto-approval, peer review, one function sent back, ctx.verifier.check = realCheck; }); + +test('regions: two agents claim one file, the first accepted candidate merges, the other is superseded', async () => { + const first = await activeUser('rgfirst'); + const second = await activeUser('rgsecond'); + delete ctx.merger.pump; + const v = ctx.store.currentVersion(); + const e = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x402700, size: 8, unit: 'src/region.cpp' }).symbol.id; + const f = ctx.store.ensureSymbol({ kind: 'function', versionId: v.id, addr: 0x402740, size: 8, unit: 'src/region.cpp' }).symbol.id; + ctx.tasks.generate(); + + // Stubbed compiler: whoever's file has "return 7" in fe matches; fe of the other candidate does not. + const realCheck = ctx.verifier.check; + ctx.verifier.check = async (files) => { + const text = files['src/region.cpp'] ?? ''; + const functions = []; + if (text.includes('int fe()')) functions.push({ id: e, status: text.includes('return 7') ? STATUS.EXACT : STATUS.MISMATCH, score: text.includes('return 7') ? 1 : 0.8, file: 'src/region.cpp', codeHash: text.includes('return 7') ? 'e-good' : 'e-other', relocations: [], size: { ours: 8, original: 8 } }); + if (text.includes('int ff()')) functions.push({ id: f, status: STATUS.MISMATCH, score: 0.9, file: 'src/region.cpp', codeHash: 'f-1', relocations: [], size: { ours: 8, original: 8 } }); + return { check: { annotationErrors: [], units: [{ file: 'src/region.cpp', compiled: true, lintErrors: [], unannotated: [], bindings: {}, inferred: [], includes: [], errors: [], functions }] }, rec: { conflicts: [], unknown: [], learned: [], learnedAddresses: [] }, changedHeaders: [], units: ['src/region.cpp'] }; + }; + + const list = (await call('GET', '/api/regions?prefix=src/region', null, first)).body; + const region = list.regions.find((r) => r.key === 'src/region.cpp'); + assert.deepEqual([region.open, region.free], [2, true], 'unclaimed and open'); + + const c1 = await call('POST', '/api/regions/claim', { kind: 'unit', key: 'src/region.cpp' }, first); + assert.equal(c1.status, 200, JSON.stringify(c1.body)); + assert.equal(c1.body.leased.length, 2, 'the whole region at once'); + const c2 = await call('POST', '/api/regions/claim', { kind: 'unit', key: 'src/region.cpp' }, second); + assert.equal(c2.status, 200); + assert.deepEqual(c2.body.alsoClaimedBy, ['rgfirst']); + assert.ok(c2.body.functions.every((x) => x.shared), 'held by the first agent, shared with the second'); + assert.equal((await call('GET', '/api/regions?prefix=src/region', null, first)).body.regions[0].free, false); + + const taskOf = (sym) => c1.body.functions.find((x) => x.symbol === sym).taskId; + const head = ctx.repo.head(); + const batchState = async (id, token) => (await call('GET', `/api/batches/${id}`, null, token)).body; + const settle = (id, token) => waitFor(async () => { const st = await batchState(id, token); return !['queued', 'verifying'].includes(st.state) && st; }); + + // The second agent's candidate of fe matches; the first agent's does not. + const s2 = await call('POST', '/api/batches', { taskIds: [taskOf(e)], baseCommit: head, files: { 'src/region.cpp': `// FUNCTION: ${e}\nint fe() { return 7; }\n` } }, second); + assert.equal(s2.status, 200, JSON.stringify(s2.body)); + assert.equal(s2.body.attempts[0].competing, true); + assert.equal(ctx.tasks.get(taskOf(e)).state, 'leased', 'a competing candidate leaves the holder\'s lease alone'); + const s1 = await call('POST', '/api/batches', { taskIds: [taskOf(e), taskOf(f)], baseCommit: head, files: { 'src/region.cpp': `// FUNCTION: ${e}\nint fe() { return 6; }\n\n// FUNCTION: ${f}\nint ff() { return 8; }\n` } }, first); + assert.equal(s1.status, 200, JSON.stringify(s1.body)); + await settle(s2.body.batchId, second); + await settle(s1.body.batchId, first); + + await ctx.merger.pump(); + assert.equal((await batchState(s2.body.batchId, second)).state, 'merged'); + assert.equal(ctx.tasks.get(taskOf(e)).state, 'done'); + const loser = (await batchState(s1.body.batchId, first)).functions.find((x) => x.symbol === e); + assert.equal(loser.state, 'superseded', 'the other candidate of the merged function is not needed any more'); + assert.equal((await batchState(s1.body.batchId, first)).functions.find((x) => x.symbol === f).state, 'peer_review', 'the rest of its batch goes on'); + + // A done function stays done whatever a late candidate does. + ctx.store.run(`UPDATE tasks SET state = 'needs_cleanup' WHERE id = ?`, taskOf(e)); + assert.equal(ctx.tasks.get(taskOf(e)).state, 'done'); + + assert.equal((await call('POST', '/api/regions/release', { kind: 'unit', key: 'src/region.cpp' }, second)).status, 200); + ctx.store.run(`UPDATE attempts SET state = 'rejected' WHERE batch_id = ?`, s1.body.batchId); + ctx.store.run(`UPDATE tasks SET state = 'done' WHERE id = ?`, taskOf(f)); + ctx.store.run('DELETE FROM region_claims'); + ctx.verifier.check = realCheck; +}); + +test('the server does not compile checks unless remoteChecks is on', async () => { + const user = await activeUser('rcuser'); + ctx.config.remoteChecks = false; + const r = await call('POST', '/api/check', { baseCommit: ctx.repo.head(), files: {} }, user); + assert.equal(r.status, 410); + assert.match(r.body.error, /td toolchain install/); + ctx.config.remoteChecks = true; +});