diff --git a/src/client/api.js b/src/client/api.js index a2b1238..8778d19 100644 --- a/src/client/api.js +++ b/src/client/api.js @@ -36,6 +36,16 @@ export class Api { get(p) { return this.request('GET', p); } + + /** GET that sends the ETag of a cached copy: null when unchanged (304), else { data, etag }. */ + async getIfChanged(p, etag) { + const res = await fetch(this.server + p, { + headers: { ...(this.token ? { authorization: `Bearer ${this.token}` } : {}), ...(etag ? { 'if-none-match': etag } : {}) }, + }); + if (res.status === 304) return null; + if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); + return { data: await res.json(), etag: res.headers.get('etag') }; + } post(p, body = {}) { return this.request('POST', p, body); } diff --git a/src/client/ops.js b/src/client/ops.js index 28a4495..cf41659 100644 --- a/src/client/ops.js +++ b/src/client/ops.js @@ -120,10 +120,15 @@ async function computeEnvDigest(ws, toolchain) { } export async function refreshSymbols(ws) { - const snap = await api(ws).get('/api/symbols'); - fs.writeFileSync(ws.symbolsFile, JSON.stringify(snap)); + // The symbol table is large and changes in bursts; download it only when it changed. + const etagFile = ws.symbolsFile + '.etag'; + const known = fs.existsSync(ws.symbolsFile) && fs.existsSync(etagFile) ? fs.readFileSync(etagFile, 'utf8') : null; + const r = await api(ws).getIfChanged('/api/symbols', known); + if (!r) return JSON.parse(fs.readFileSync(ws.symbolsFile, 'utf8')); + fs.writeFileSync(ws.symbolsFile, JSON.stringify(r.data)); + if (r.etag) fs.writeFileSync(etagFile, r.etag); ws._symbols = null; - return snap; + return r.data; } /** Fast-forwards the clone to the server's main and refreshes the symbol snapshot. */ diff --git a/src/client/update.js b/src/client/update.js index 870e907..428642d 100644 --- a/src/client/update.js +++ b/src/client/update.js @@ -9,6 +9,7 @@ import path from 'node:path'; import crypto from 'node:crypto'; const CHECK_EVERY_MS = 3600_000; +const STALL_MS = 60_000; const STATE_DIR = path.join(os.homedir(), '.clank'); const STATE_FILE = path.join(STATE_DIR, 'update.json'); const ASSET = process.platform === 'win32' ? 'td-windows-x64.exe' : 'td-linux-x64'; @@ -68,9 +69,19 @@ export async function maybeUpdate({ server, force = false, log = (m) => process. if (!want || want === (await ownHash(exe, state))) return writeState(state), null; log(`td: downloading an update from ${base} ...`); - const bin = await fetch(`${base}/download/${ASSET}`, { signal: AbortSignal.timeout(600_000) }); - if (!bin.ok) return writeState(state), null; - const data = Buffer.from(await bin.arrayBuffer()); + // Slow connections take as long as they need; only a download that stalls is given up. + const stall = new AbortController(); + let timer = setTimeout(() => stall.abort(), STALL_MS); + const bin = await fetch(`${base}/download/${ASSET}`, { signal: stall.signal }); + if (!bin.ok) return clearTimeout(timer), writeState(state), null; + const parts = []; + for await (const chunk of bin.body) { + parts.push(chunk); + clearTimeout(timer); + timer = setTimeout(() => stall.abort(), STALL_MS); + } + clearTimeout(timer); + const data = Buffer.concat(parts); if (crypto.createHash('sha256').update(data).digest('hex') !== want) { log('td: the update failed its checksum check and was not installed'); return writeState(state), null; diff --git a/src/server/app.js b/src/server/app.js index 2fa5822..3725db4 100644 --- a/src/server/app.js +++ b/src/server/app.js @@ -2,6 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; +import crypto from 'node:crypto'; import http from 'node:http'; import { fileURLToPath } from 'node:url'; import { Store, now } from './db.js'; @@ -178,11 +179,20 @@ export function createApp(ctx) { return undefined; }); - router.on('GET', '/api/symbols', ({ query }) => { + // The whole symbol table (over a megabyte): sent only when it changed since the client's copy. + router.on('GET', '/api/symbols', ({ req, res, query }) => { const v = query.get('version') ? store.get('SELECT * FROM versions WHERE id = ?', query.get('version')) : store.currentVersion(); if (!v) throw new HttpError(404, 'unknown version'); const rows = [...store.symbolMap(v.id).values()].map((s) => [s.id, s.kind, s.addr, s.size, s.name, s.decorated, s.library ? 1 : 0, s.unit]); - return { version: v.id, head: repo.head(), columns: ['id', 'kind', 'addr', 'size', 'name', 'decorated', 'library', 'unit'], symbols: rows }; + const body = JSON.stringify({ version: v.id, columns: ['id', 'kind', 'addr', 'size', 'name', 'decorated', 'library', 'unit'], symbols: rows }); + const etag = `"${crypto.createHash('sha1').update(body).digest('hex')}"`; + const headers = { etag, 'cache-control': 'no-cache', 'x-content-type-options': 'nosniff' }; + if (req.headers['if-none-match'] === etag) { + res.writeHead(304, headers); + return void res.end(); + } + res.writeHead(200, { ...headers, 'content-type': 'application/json; charset=utf-8' }); + res.end(body); }); // Public: who is on the project and what their agents are doing right now. No token labels or @@ -714,7 +724,7 @@ export function createApp(ctx) { res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-cache', 'x-content-type-options': 'nosniff' }); return res.end(script); } - if (req.method === 'GET' && url.pathname.startsWith('/download/')) return serveDownload(url.pathname.slice(10), res); + if (req.method === 'GET' && url.pathname.startsWith('/download/')) return serveDownload(url.pathname.slice(10), req, res); if (req.method === 'GET' && !url.pathname.startsWith('/api/')) return serveStatic(url.pathname, res); const m = router.match(req.method, url.pathname); if (!m) throw new HttpError(404, 'not found'); @@ -735,14 +745,26 @@ export function createApp(ctx) { // Public downloads of the td binaries (tools/build-td output copied to /downloads). const DOWNLOADS = new Set(['td-windows-x64.exe', 'td-linux-x64', 'SHA256SUMS']); let downloadsRoot = null; -function serveDownload(name, res) { +function serveDownload(name, req, res) { const file = DOWNLOADS.has(name) && downloadsRoot ? path.join(downloadsRoot, name) : null; if (!file || !fs.existsSync(file)) { res.writeHead(404, { 'content-type': 'text/plain' }); return res.end('not found'); } - res.writeHead(200, { 'content-type': name === 'SHA256SUMS' ? 'text/plain' : 'application/octet-stream', 'content-length': fs.statSync(file).size, 'content-disposition': `attachment; filename="${name}"`, 'x-content-type-options': 'nosniff' }); - fs.createReadStream(file).pipe(res); + // A compressed copy (made when the binaries are published) for clients that accept gzip: td's + // self-update and the Linux installer; about a third of the size. + const gz = file + '.gz'; + const zipped = /\bgzip\b/.test(String(req.headers['accept-encoding'] ?? '')) && fs.existsSync(gz) && fs.statSync(gz).mtimeMs >= fs.statSync(file).mtimeMs; + const sent = zipped ? gz : file; + res.writeHead(200, { + 'content-type': name === 'SHA256SUMS' ? 'text/plain' : 'application/octet-stream', + 'content-length': fs.statSync(sent).size, + 'content-disposition': `attachment; filename="${name}"`, + 'x-content-type-options': 'nosniff', + vary: 'accept-encoding', + ...(zipped ? { 'content-encoding': 'gzip' } : {}), + }); + fs.createReadStream(sent).pipe(res); } const STATIC = { '/': 'index.html', '/review': 'index.html', '/app.js': 'app.js', '/style.css': 'style.css', '/favicon.svg': 'favicon.svg' }; diff --git a/src/server/install.js b/src/server/install.js index 81abfca..f8b3956 100644 --- a/src/server/install.js +++ b/src/server/install.js @@ -99,7 +99,7 @@ mkdir -p "$BIN" command -v git >/dev/null 2>&1 || { echo 'git is required: install it (e.g. sudo apt install git) and run this line again' >&2; exit 1; } echo 'Downloading td...' -curl -fsSL "$SERVER/download/td-linux-x64" -o "$BIN/td.part" +curl -fsSL --compressed "$SERVER/download/td-linux-x64" -o "$BIN/td.part" echo "${sha} $BIN/td.part" | sha256sum -c - >/dev/null || { rm -f "$BIN/td.part"; echo 'td download failed its checksum check' >&2; exit 1; } mv "$BIN/td.part" "$BIN/td" chmod +x "$BIN/td" diff --git a/test/server.test.js b/test/server.test.js index a6cebcc..e76e919 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -554,6 +554,32 @@ test('a function that no longer passes at merge goes back alone; the rest merges } }); +test('the symbol table is only sent when it changed; downloads come compressed when accepted', async () => { + const token = await activeUser('trafficker'); + const auth = { authorization: `Bearer ${token}` }; + const first = await fetch(base + '/api/symbols', { headers: auth }); + assert.equal(first.status, 200); + const etag = first.headers.get('etag'); + assert.ok(etag); + await first.json(); + const again = await fetch(base + '/api/symbols', { headers: { ...auth, 'if-none-match': etag } }); + assert.equal(again.status, 304, 'unchanged: nothing sent'); + + const dir = path.join(ctx.config.dataDir, 'downloads'); + fs.mkdirSync(dir, { recursive: true }); + const bin = Buffer.alloc(200_000, 7); + fs.writeFileSync(path.join(dir, 'td-linux-x64'), bin); + fs.writeFileSync(path.join(dir, 'td-linux-x64.gz'), (await import('node:zlib')).gzipSync(bin)); + const zipped = await fetch(base + '/download/td-linux-x64', { headers: { 'accept-encoding': 'gzip' } }); + assert.equal(zipped.headers.get('content-encoding'), 'gzip'); + assert.ok(Number(zipped.headers.get('content-length')) < 10_000); + assert.deepEqual(Buffer.from(await zipped.arrayBuffer()), bin, 'fetch unpacks it to the same bytes'); + const plain = await fetch(base + '/download/td-linux-x64', { headers: { 'accept-encoding': 'identity' } }); + assert.equal(plain.headers.get('content-encoding'), null); + assert.equal(Number(plain.headers.get('content-length')), bin.length); + await plain.arrayBuffer(); +}); + test('the crew view is public and shows no account details', async () => { const token = await activeUser('crewview'); await call('GET', '/api/tasks', null, token); // the agent checks in diff --git a/tools/deploy.sh b/tools/deploy.sh index 66d9c77..7975852 100644 --- a/tools/deploy.sh +++ b/tools/deploy.sh @@ -29,6 +29,8 @@ ssh "$HOST" "set -e install -o togetherdecomp -g togetherdecomp -m 0644 /tmp/\$f \$D/\$f.new && mv \$D/\$f.new \$D/\$f && rm -f /tmp/\$f done chmod 0755 \$D/td-linux-x64 + # Compressed copies for clients that accept gzip (td self-update, the Linux installer). + for f in td-windows-x64.exe td-linux-x64; do gzip -9 -k -f \$D/\$f && chown togetherdecomp:togetherdecomp \$D/\$f.gz; done echo 'td binaries published' fi TD_KEEP_WORKER='${TD_KEEP_WORKER:-}' sh \"\$d/deploy/update.sh\" \"\$d\"