Less traffic: compressed td downloads, symbol table only when changed, no update retry loop

- /download serves a gzip copy (made when binaries are published) to
  clients that accept it: td's self-update and the Linux installer
  (curl --compressed). 86 MB -> 32 MB for Windows, 123 MB -> 41 MB for Linux.
- /api/symbols carries an ETag; td keeps it with its snapshot and gets 304
  when nothing changed (it fetched 1.2 MB on every sync).
- td's self-update gave up after 10 minutes, so a slow connection downloaded
  most of the binary every hour forever. It now only gives up on a download
  that stalls for a minute.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RNwtXPYqFu3oKaaXwYwNWW

src/client/api.js

@@ -36,6 +36,16 @@ export class Api {
   get(p) {
     return this.request('GET', p);
   }
+
+  /** GET that sends the ETag of a cached copy: null when unchanged (304), else { data, etag }. */
+  async getIfChanged(p, etag) {
+    const res = await fetch(this.server + p, {
+      headers: { ...(this.token ? { authorization: `Bearer ${this.token}` } : {}), ...(etag ? { 'if-none-match': etag } : {}) },
+    });
+    if (res.status === 304) return null;
+    if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
+    return { data: await res.json(), etag: res.headers.get('etag') };
+  }
   post(p, body = {}) {
     return this.request('POST', p, body);
   }

src/client/ops.js

@@ -120,10 +120,15 @@ async function computeEnvDigest(ws, toolchain) {
 }
 
 export async function refreshSymbols(ws) {
-  const snap = await api(ws).get('/api/symbols');
-  fs.writeFileSync(ws.symbolsFile, JSON.stringify(snap));
+  // The symbol table is large and changes in bursts; download it only when it changed.
+  const etagFile = ws.symbolsFile + '.etag';
+  const known = fs.existsSync(ws.symbolsFile) && fs.existsSync(etagFile) ? fs.readFileSync(etagFile, 'utf8') : null;
+  const r = await api(ws).getIfChanged('/api/symbols', known);
+  if (!r) return JSON.parse(fs.readFileSync(ws.symbolsFile, 'utf8'));
+  fs.writeFileSync(ws.symbolsFile, JSON.stringify(r.data));
+  if (r.etag) fs.writeFileSync(etagFile, r.etag);
   ws._symbols = null;
-  return snap;
+  return r.data;
 }
 
 /** Fast-forwards the clone to the server's main and refreshes the symbol snapshot. */

src/client/update.js

@@ -9,6 +9,7 @@ import path from 'node:path';
 import crypto from 'node:crypto';
 
 const CHECK_EVERY_MS = 3600_000;
+const STALL_MS = 60_000;
 const STATE_DIR = path.join(os.homedir(), '.clank');
 const STATE_FILE = path.join(STATE_DIR, 'update.json');
 const ASSET = process.platform === 'win32' ? 'td-windows-x64.exe' : 'td-linux-x64';
@@ -68,9 +69,19 @@ export async function maybeUpdate({ server, force = false, log = (m) => process.
     if (!want || want === (await ownHash(exe, state))) return writeState(state), null;
 
     log(`td: downloading an update from ${base} ...`);
-    const bin = await fetch(`${base}/download/${ASSET}`, { signal: AbortSignal.timeout(600_000) });
-    if (!bin.ok) return writeState(state), null;
-    const data = Buffer.from(await bin.arrayBuffer());
+    // Slow connections take as long as they need; only a download that stalls is given up.
+    const stall = new AbortController();
+    let timer = setTimeout(() => stall.abort(), STALL_MS);
+    const bin = await fetch(`${base}/download/${ASSET}`, { signal: stall.signal });
+    if (!bin.ok) return clearTimeout(timer), writeState(state), null;
+    const parts = [];
+    for await (const chunk of bin.body) {
+      parts.push(chunk);
+      clearTimeout(timer);
+      timer = setTimeout(() => stall.abort(), STALL_MS);
+    }
+    clearTimeout(timer);
+    const data = Buffer.concat(parts);
     if (crypto.createHash('sha256').update(data).digest('hex') !== want) {
       log('td: the update failed its checksum check and was not installed');
       return writeState(state), null;

src/server/app.js

@@ -2,6 +2,7 @@
 
 import fs from 'node:fs';
 import path from 'node:path';
+import crypto from 'node:crypto';
 import http from 'node:http';
 import { fileURLToPath } from 'node:url';
 import { Store, now } from './db.js';
@@ -178,11 +179,20 @@ export function createApp(ctx) {
     return undefined;
   });
 
-  router.on('GET', '/api/symbols', ({ query }) => {
+  // The whole symbol table (over a megabyte): sent only when it changed since the client's copy.
+  router.on('GET', '/api/symbols', ({ req, res, query }) => {
     const v = query.get('version') ? store.get('SELECT * FROM versions WHERE id = ?', query.get('version')) : store.currentVersion();
     if (!v) throw new HttpError(404, 'unknown version');
     const rows = [...store.symbolMap(v.id).values()].map((s) => [s.id, s.kind, s.addr, s.size, s.name, s.decorated, s.library ? 1 : 0, s.unit]);
-    return { version: v.id, head: repo.head(), columns: ['id', 'kind', 'addr', 'size', 'name', 'decorated', 'library', 'unit'], symbols: rows };
+    const body = JSON.stringify({ version: v.id, columns: ['id', 'kind', 'addr', 'size', 'name', 'decorated', 'library', 'unit'], symbols: rows });
+    const etag = `"${crypto.createHash('sha1').update(body).digest('hex')}"`;
+    const headers = { etag, 'cache-control': 'no-cache', 'x-content-type-options': 'nosniff' };
+    if (req.headers['if-none-match'] === etag) {
+      res.writeHead(304, headers);
+      return void res.end();
+    }
+    res.writeHead(200, { ...headers, 'content-type': 'application/json; charset=utf-8' });
+    res.end(body);
   });
 
   // Public: who is on the project and what their agents are doing right now. No token labels or
@@ -714,7 +724,7 @@ export function createApp(ctx) {
         res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-cache', 'x-content-type-options': 'nosniff' });
         return res.end(script);
       }
-      if (req.method === 'GET' && url.pathname.startsWith('/download/')) return serveDownload(url.pathname.slice(10), res);
+      if (req.method === 'GET' && url.pathname.startsWith('/download/')) return serveDownload(url.pathname.slice(10), req, res);
       if (req.method === 'GET' && !url.pathname.startsWith('/api/')) return serveStatic(url.pathname, res);
       const m = router.match(req.method, url.pathname);
       if (!m) throw new HttpError(404, 'not found');
@@ -735,14 +745,26 @@ export function createApp(ctx) {
 // Public downloads of the td binaries (tools/build-td output copied to <dataDir>/downloads).
 const DOWNLOADS = new Set(['td-windows-x64.exe', 'td-linux-x64', 'SHA256SUMS']);
 let downloadsRoot = null;
-function serveDownload(name, res) {
+function serveDownload(name, req, res) {
   const file = DOWNLOADS.has(name) && downloadsRoot ? path.join(downloadsRoot, name) : null;
   if (!file || !fs.existsSync(file)) {
     res.writeHead(404, { 'content-type': 'text/plain' });
     return res.end('not found');
   }
-  res.writeHead(200, { 'content-type': name === 'SHA256SUMS' ? 'text/plain' : 'application/octet-stream', 'content-length': fs.statSync(file).size, 'content-disposition': `attachment; filename="${name}"`, 'x-content-type-options': 'nosniff' });
-  fs.createReadStream(file).pipe(res);
+  // A compressed copy (made when the binaries are published) for clients that accept gzip: td's
+  // self-update and the Linux installer; about a third of the size.
+  const gz = file + '.gz';
+  const zipped = /\bgzip\b/.test(String(req.headers['accept-encoding'] ?? '')) && fs.existsSync(gz) && fs.statSync(gz).mtimeMs >= fs.statSync(file).mtimeMs;
+  const sent = zipped ? gz : file;
+  res.writeHead(200, {
+    'content-type': name === 'SHA256SUMS' ? 'text/plain' : 'application/octet-stream',
+    'content-length': fs.statSync(sent).size,
+    'content-disposition': `attachment; filename="${name}"`,
+    'x-content-type-options': 'nosniff',
+    vary: 'accept-encoding',
+    ...(zipped ? { 'content-encoding': 'gzip' } : {}),
+  });
+  fs.createReadStream(sent).pipe(res);
 }
 
 const STATIC = { '/': 'index.html', '/review': 'index.html', '/app.js': 'app.js', '/style.css': 'style.css', '/favicon.svg': 'favicon.svg' };

src/server/install.js

@@ -99,7 +99,7 @@ mkdir -p "$BIN"
 command -v git >/dev/null 2>&1 || { echo 'git is required: install it (e.g. sudo apt install git) and run this line again' >&2; exit 1; }
 
 echo 'Downloading td...'
-curl -fsSL "$SERVER/download/td-linux-x64" -o "$BIN/td.part"
+curl -fsSL --compressed "$SERVER/download/td-linux-x64" -o "$BIN/td.part"
 echo "${sha}  $BIN/td.part" | sha256sum -c - >/dev/null || { rm -f "$BIN/td.part"; echo 'td download failed its checksum check' >&2; exit 1; }
 mv "$BIN/td.part" "$BIN/td"
 chmod +x "$BIN/td"

test/server.test.js

@@ -554,6 +554,32 @@ test('a function that no longer passes at merge goes back alone; the rest merges
   }
 });
 
+test('the symbol table is only sent when it changed; downloads come compressed when accepted', async () => {
+  const token = await activeUser('trafficker');
+  const auth = { authorization: `Bearer ${token}` };
+  const first = await fetch(base + '/api/symbols', { headers: auth });
+  assert.equal(first.status, 200);
+  const etag = first.headers.get('etag');
+  assert.ok(etag);
+  await first.json();
+  const again = await fetch(base + '/api/symbols', { headers: { ...auth, 'if-none-match': etag } });
+  assert.equal(again.status, 304, 'unchanged: nothing sent');
+
+  const dir = path.join(ctx.config.dataDir, 'downloads');
+  fs.mkdirSync(dir, { recursive: true });
+  const bin = Buffer.alloc(200_000, 7);
+  fs.writeFileSync(path.join(dir, 'td-linux-x64'), bin);
+  fs.writeFileSync(path.join(dir, 'td-linux-x64.gz'), (await import('node:zlib')).gzipSync(bin));
+  const zipped = await fetch(base + '/download/td-linux-x64', { headers: { 'accept-encoding': 'gzip' } });
+  assert.equal(zipped.headers.get('content-encoding'), 'gzip');
+  assert.ok(Number(zipped.headers.get('content-length')) < 10_000);
+  assert.deepEqual(Buffer.from(await zipped.arrayBuffer()), bin, 'fetch unpacks it to the same bytes');
+  const plain = await fetch(base + '/download/td-linux-x64', { headers: { 'accept-encoding': 'identity' } });
+  assert.equal(plain.headers.get('content-encoding'), null);
+  assert.equal(Number(plain.headers.get('content-length')), bin.length);
+  await plain.arrayBuffer();
+});
+
 test('the crew view is public and shows no account details', async () => {
   const token = await activeUser('crewview');
   await call('GET', '/api/tasks', null, token); // the agent checks in

tools/deploy.sh

@@ -29,6 +29,8 @@ ssh "$HOST" "set -e
       install -o togetherdecomp -g togetherdecomp -m 0644 /tmp/\$f \$D/\$f.new && mv \$D/\$f.new \$D/\$f && rm -f /tmp/\$f
     done
     chmod 0755 \$D/td-linux-x64
+    # Compressed copies for clients that accept gzip (td self-update, the Linux installer).
+    for f in td-windows-x64.exe td-linux-x64; do gzip -9 -k -f \$D/\$f && chown togetherdecomp:togetherdecomp \$D/\$f.gz; done
     echo 'td binaries published'
   fi
   TD_KEEP_WORKER='${TD_KEEP_WORKER:-}' sh \"\$d/deploy/update.sh\" \"\$d\"