Worker drains on stop instead of re-queuing work; update.sh no longer waits for it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RNwtXPYqFu3oKaaXwYwNWW

deploy/togetherdecomp-worker.service

@@ -13,7 +13,8 @@ ExecStart=/opt/node/bin/node --disable-warning=ExperimentalWarning /opt/together
 Restart=on-failure
 # Restarting re-queues whatever was being verified.
 RestartSec=5
-TimeoutStopSec=15
+# Stopping waits for the running verification or merge to finish.
+TimeoutStopSec=1200
 Environment=NODE_ENV=production
 
 # Hardening. The process needs: its data dir, the docker socket, git.

deploy/update.sh

@@ -13,7 +13,6 @@ set -eu
 SRC=$(cd "${1:?usage: update.sh <source-dir>}" && pwd)
 BASE=/opt/togetherdecomp-releases
 LINK=/opt/togetherdecomp
-JOBS=/var/lib/togetherdecomp/jobs
 NODE=/opt/node/bin/node
 REL="$BASE/$(date +%Y%m%d-%H%M%S)"
 
@@ -66,17 +65,10 @@ else
   systemctl restart togetherdecomp.service
   sleep 2
   systemctl is-active --quiet togetherdecomp.service || { journalctl -u togetherdecomp -n 20 --no-pager; exit 1; }
-  # A restart re-queues what the worker is doing. Remote checks are cheap to redo (the waiting agent
+  # The worker finishes its running verification, merge or check before it stops (no work is
-  # still gets its result); a batch verification or merge is waited for, up to 15 minutes.
+  # redone), then starts on the new release. That can take minutes, so don't wait for it here.
-  echo "== restarting the worker once no batch is being verified or merged"
+  echo "== the worker restarts once its running work is done"
-  waited=0
+  systemctl restart --no-block togetherdecomp-worker.service
-  while ls "$JOBS" 2>/dev/null | grep -qE '^(batch|merge)-' && [ "$waited" -lt 900 ]; do
-    sleep 5
-    waited=$((waited + 5))
-  done
-  systemctl restart togetherdecomp-worker.service
-  sleep 2
-  systemctl is-active --quiet togetherdecomp-worker.service || { journalctl -u togetherdecomp-worker -n 20 --no-pager; exit 1; }
 fi
 
 # Keep the last three releases for a quick rollback (ln -sfn <release> /opt/togetherdecomp).

src/server/app.js

@@ -720,14 +720,21 @@ export async function startServer(dataDir, { role = 'all' } = {}) {
     await new Promise((r) => server.listen(ctx.config.listen.port, ctx.config.listen.host, r));
     console.log(`clank together coordinator listening on http://${ctx.config.listen.host}:${ctx.config.listen.port} (public url ${ctx.config.publicUrl})`);
   }
-  // Quick exit: requests still open (an agent waiting on a remote check can wait for minutes) are
+  // The web side exits quickly: requests still open (an agent waiting on a remote check can wait
-  // cut off after a moment; agents retry, and the work itself lives in the database and the worker.
+  // for minutes) are cut off after a moment; agents retry. The worker drains: it takes no new work
+  // and exits once its running verifications, merges and checks are done, so nothing is redone.
+  const busy = () => ctx.runsWorker && (ctx.verifier.running > 0 || ctx.merger.busy || ctx.checks.running > 0);
+  const exitWhenIdle = () => (busy() ? setTimeout(exitWhenIdle, 1000) : process.exit(0));
   const shutdown = () => {
     for (const stop of stops) stop();
-    if (!server) process.exit(0);
+    if (busy()) console.log('stopping: finishing the running work first');
-    server.close(() => process.exit(0));
+    if (!server) return exitWhenIdle();
+    server.close();
     server.closeIdleConnections();
-    setTimeout(() => process.exit(0), 3000).unref();
+    setTimeout(() => {
+      server.closeAllConnections();
+      exitWhenIdle();
+    }, 3000).unref();
   };
   process.on('SIGINT', shutdown);
   process.on('SIGTERM', shutdown);