diff --git a/README.md b/README.md index 63c6773..547579f 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ attempt gets first pick on the follow-up for 6 hours. ## Maintainer bootstrap +Run `npm ci` in the coordinator checkout before starting the server or running tests. + ```sh # on the server (as the service user): td-server --data /var/lib/togetherdecomp td-server version add 1.0 game.exe # register the target @@ -131,6 +133,13 @@ Submitted overrides apply only to the current query. Reverse include traversal handles cycles without losing transitive dependents; uncommitted working-tree text is never used as the baseline. +The original-string gate uses each source unit's configured MSVC execution charset +(`/execution-charset:.949`, for example) when checking readable Unicode literals. +Numeric byte escapes remain literal bytes, and wide strings remain UTF-16LE. +Unrepresentable characters are rejected rather than accepted as replacement `?` +bytes. Configure source UTF-8 and the target execution code page separately; this +does not permit new strings absent from the original executable. + ## Building td `node tools/build-td/build.mjs` (Node 22.21) writes `dist/td-windows-x64.exe`, `dist/td-linux-x64` and diff --git a/deploy/install.sh b/deploy/install.sh index ca9135e..99816f7 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -61,7 +61,8 @@ usermod -aG docker togetherdecomp # Code updates go through update.sh (releases behind a symlink); here it only lays the code down. if [ ! -e "$APP" ]; then mkdir -p /opt/togetherdecomp-releases/initial - cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" /opt/togetherdecomp-releases/initial/ + cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" "$SRC/package-lock.json" /opt/togetherdecomp-releases/initial/ + (cd /opt/togetherdecomp-releases/initial && PATH="/opt/node/bin:$PATH" /opt/node/bin/npm ci --omit=dev --ignore-scripts) chown -R root:root /opt/togetherdecomp-releases && chmod -R go-w,a+rX /opt/togetherdecomp-releases ln -s /opt/togetherdecomp-releases/initial "$APP" else diff --git a/deploy/rodecomp/Dockerfile b/deploy/rodecomp/Dockerfile index 7fa07c2..28a7b92 100644 --- a/deploy/rodecomp/Dockerfile +++ b/deploy/rodecomp/Dockerfile @@ -3,7 +3,8 @@ FROM node:24-bookworm-slim RUN apt-get update && apt-get install -y --no-install-recommends git binutils ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker WORKDIR /app -COPY package.json ./ +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts COPY src ./src COPY ghidra_scripts ./ghidra_scripts COPY template ./template diff --git a/deploy/rodecomp/Dockerfile.client b/deploy/rodecomp/Dockerfile.client index 49fefe7..77f6ffc 100644 --- a/deploy/rodecomp/Dockerfile.client +++ b/deploy/rodecomp/Dockerfile.client @@ -1,6 +1,7 @@ FROM node:22.21.0-bookworm-slim WORKDIR /app -COPY package.json ./ +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts COPY src ./src COPY ghidra_scripts ./ghidra_scripts COPY tools/build-td ./tools/build-td diff --git a/deploy/update.sh b/deploy/update.sh index ee518fd..0ebd12a 100644 --- a/deploy/update.sh +++ b/deploy/update.sh @@ -32,7 +32,8 @@ OLD=$(readlink -f "$LINK" 2>/dev/null || true) echo "== preparing $REL" mkdir -p "$REL" -cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" "$REL/" +cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" "$SRC/package-lock.json" "$REL/" +(cd "$REL" && PATH="$(dirname "$NODE"):$PATH" "$(dirname "$NODE")/npm" ci --omit=dev --ignore-scripts) chown -R root:root "$REL" chmod -R go-w,a+rX "$REL" for f in $(find "$REL/src" -name '*.js'); do diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..f10bc7a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,41 @@ +{ + "name": "clank-together", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "clank-together", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "bin": { + "td": "src/client/cli.js", + "td-server": "src/server/cli.js" + }, + "engines": { + "node": ">=22.13.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + } + } +} diff --git a/package.json b/package.json index de0166e..bcd5b92 100644 --- a/package.json +++ b/package.json @@ -15,5 +15,8 @@ "test": "node --test \"test/**/*.test.js\"", "server": "node src/server/cli.js start", "client": "node src/client/cli.js" + }, + "dependencies": { + "iconv-lite": "0.6.3" } } diff --git a/src/server/verify.js b/src/server/verify.js index 96c392d..a82514c 100644 --- a/src/server/verify.js +++ b/src/server/verify.js @@ -4,11 +4,11 @@ import fs from 'node:fs'; import path from 'node:path'; import { lintSubmission } from '../shared/lint.js'; -import { freeTextErrorsAcross } from '../shared/freetext.js'; +import { freeTextErrorsAcross, executionCharset } from '../shared/freetext.js'; import { checkUnits } from '../shared/check.js'; import { reconcile } from '../shared/reconcile.js'; import { STATUS, sameOriginal } from '../shared/compare.js'; -import { listSources, isUnit } from '../shared/project.js'; +import { listSources, isUnit, unitSettings } from '../shared/project.js'; import { changedNames, unitsMentioning } from './affected.js'; import { undefinedDeclarations } from './undefined.js'; import { sourceIncluders } from './includers.js'; @@ -232,8 +232,9 @@ export class Verifier { freeTextGate(files, baseCommit) { const exe = this.getPe().buf; const inOriginal = (bytes) => exe.indexOf(bytes) !== -1; - const pooled = Object.keys(files).map((f) => this.repo.show(baseCommit, f)).filter((t) => t !== null).join('\n'); - return freeTextErrorsAcross(files, pooled, inOriginal); + const baseFiles = Object.fromEntries(this.repo.showMany(baseCommit, Object.keys(files))); + const charsetForFile = (file) => executionCharset(unitSettings(this.project, file).cflags); + return freeTextErrorsAcross(files, '', inOriginal, { baseFiles, charsetForFile }); } /** diff --git a/src/shared/freetext.js b/src/shared/freetext.js index 6f8051a..c6ddfa8 100644 --- a/src/shared/freetext.js +++ b/src/shared/freetext.js @@ -4,6 +4,8 @@ // executable (text the original authors wrote). Identifier names are the only free text left. // Comments and strings already on main are left alone: the rules apply to what a submission adds. +import iconv from 'iconv-lite'; + const ANNOTATION_RE = /^\/\/\s*(FUNCTION|STUB|GLOBAL|VTABLE|STRING|SIZE|LIBRARY|OFFSET)\s*:\s*[\w.]+(\s+<[\w\s:<>,*&]{1,120}>)?\s*$/; const OFFSET_RE = /^\/\*\s*(?:0x)?(?:[0-9A-Fa-f]{1,8}|-{2,8})\s*\*\/$/; const BLANK_RE = /^(?:\/\/[\s\-=*\/]*|\/\*[\s\-=*]*\*\/)$/; @@ -23,7 +25,7 @@ export function allowedComment(text) { * Comments and string literals of C++ source. Adjacent literals ("a" "b") form one string. * @returns {{comments: {text: string, line: number}[], strings: {pieces: Buffer[], text: string, wide: boolean, line: number, include: string|null, linkage: boolean}[]}} */ -export function scanText(src) { +export function scanText(src, executionCharset = null) { const comments = []; const literals = []; let line = 1; @@ -59,14 +61,14 @@ export function scanText(src) { } const wide = c === 'L' && (n === '"' || n === "'") && !/\w/.test(src[i - 1] ?? ''); if (c === '"' || (wide && n === '"')) { - const { value, end } = readQuoted(src, wide ? i + 1 : i); + const { value, segments, end } = readQuoted(src, wide ? i + 1 : i); const prev = literals[literals.length - 1]; if (prev && prev.end === lastCodeEnd(src, i) && prev.include === null) { - prev.pieces.push(value); + prev.pieces.push({ value, segments }); prev.wide ||= wide; prev.end = end; } else { - literals.push({ pieces: [value], wide, line, include: INCLUDE_RE.test(src.slice(lineBegin, i)) ? value : null, linkage: lastCode === 'extern', end }); + literals.push({ pieces: [{ value, segments }], wide, line, include: INCLUDE_RE.test(src.slice(lineBegin, i)) ? value : null, linkage: lastCode === 'extern', end }); } i = end; continue; @@ -87,8 +89,8 @@ export function scanText(src) { i++; } const strings = literals.map((l) => ({ - pieces: l.pieces.map((p) => encode(p, l.wide)), - text: l.pieces.join(''), + pieces: l.pieces.map((p) => encode(p, l.wide, executionCharset)), + text: l.pieces.map((p) => p.value).join(''), wide: l.wide, line: l.line, include: l.include, @@ -110,46 +112,76 @@ const SIMPLE_ESCAPES = { n: '\n', t: '\t', r: '\r', a: '\x07', b: '\b', f: '\f', function readQuoted(src, at) { const q = src[at]; let out = ''; + const segments = []; + const append = (text, raw = false) => { + out += text; + const previous = segments[segments.length - 1]; + if (previous && previous.raw === raw) previous.text += text; + else segments.push({ text, raw }); + }; let j = at + 1; while (j < src.length && src[j] !== q && src[j] !== '\n') { if (src[j] !== '\\') { - out += src[j++]; + append(src[j++]); continue; } const e = src[j + 1]; if (e in SIMPLE_ESCAPES) { - out += SIMPLE_ESCAPES[e]; + append(SIMPLE_ESCAPES[e]); j += 2; } else if (e === 'x') { const m = /^[0-9A-Fa-f]+/.exec(src.slice(j + 2, j + 10)); - out += String.fromCharCode(m ? parseInt(m[0], 16) & 0xffff : 0); + append(String.fromCharCode(m ? parseInt(m[0], 16) & 0xffff : 0), true); j += 2 + (m ? m[0].length : 0); } else if (e === 'u' || e === 'U') { const len = e === 'u' ? 4 : 8; const cp = parseInt(src.slice(j + 2, j + 2 + len), 16); - out += cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : ''; + append(cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : ''); j += 2 + len; } else if (/[0-7]/.test(e ?? '')) { const m = /^[0-7]{1,3}/.exec(src.slice(j + 1)); - out += String.fromCharCode(parseInt(m[0], 8)); + append(String.fromCharCode(parseInt(m[0], 8)), true); j += 1 + m[0].length; } else if (e === '\n') { j += 2; // line continuation } else if (e === '\r' && src[j + 2] === '\n') { j += 3; } else { - out += e ?? ''; + append(e ?? ''); j += 2; } } - return { value: out, end: j + 1 }; + return { value: out, segments, end: j + 1 }; } // The bytes the compiler emits for a literal (without the terminator). -function encode(s, wide) { - if (wide) return Buffer.from(s, 'utf16le'); - // Narrow literals are in the ANSI code page; characters beyond Latin-1 keep their UTF-8 bytes. - return [...s].every((ch) => ch.codePointAt(0) <= 0xff) ? Buffer.from(s, 'latin1') : Buffer.from(s, 'utf8'); +function encode(piece, wide, executionCharset) { + if (wide) return Buffer.from(piece.value, 'utf16le'); + if (!executionCharset) { + return [...piece.value].every((ch) => ch.codePointAt(0) <= 0xff) ? Buffer.from(piece.value, 'latin1') : Buffer.from(piece.value, 'utf8'); + } + const parts = []; + for (const segment of piece.segments) { + if (segment.raw) { + parts.push(Buffer.from(segment.text, 'latin1')); + continue; + } + const bytes = iconv.encode(segment.text, executionCharset); + if (iconv.decode(bytes, executionCharset) !== segment.text) return null; + parts.push(bytes); + } + return Buffer.concat(parts); +} + +export function executionCharset(cflags) { + let charset = null; + for (const flag of cflags ?? []) { + if (/^[/-]utf-8$/i.test(flag)) charset = 'utf8'; + const match = /^[/-]execution-charset:(.+)$/i.exec(flag); + if (match) charset = match[1].replace(/^\.(\d+)$/, 'cp$1'); + } + if (charset && !iconv.encodingExists(charset)) throw new Error(`unsupported execution character set: ${charset}`); + return charset; } /** @@ -159,8 +191,8 @@ function encode(s, wide) { * @param {(bytes: Buffer) => boolean} inOriginal whether the bytes occur in the original executable * @returns {{line: number, message: string}[]} the messages never repeat the text itself */ -export function freeTextErrors(text, baseText, inOriginal) { - return checkText(scanText(text), baseText === null ? { comments: [], strings: [] } : scanText(baseText), inOriginal); +export function freeTextErrors(text, baseText, inOriginal, charset = null) { + return checkText(scanText(text, charset), baseText === null ? { comments: [], strings: [] } : scanText(baseText, charset), inOriginal); } /** @@ -168,13 +200,13 @@ export function freeTextErrors(text, baseText, inOriginal) { * comments and strings may move from one submitted file to another, each base occurrence once. * @returns {{file: string, line: number, message: string}[]} */ -export function freeTextErrorsAcross(files, pooledBase, inOriginal) { - const base = scanText(pooledBase); - const pool = { comments: counts(base.comments.map((c) => c.text.trim())), strings: counts(base.strings.map(stringKey)) }; +export function freeTextErrorsAcross(files, pooledBase, inOriginal, { charsetForFile = () => null, baseFiles = null } = {}) { + const bases = baseFiles ? Object.entries(baseFiles).filter(([, text]) => text !== null).map(([file, text]) => scanText(text, charsetForFile(file))) : [scanText(pooledBase)]; + const pool = { comments: counts(bases.flatMap((base) => base.comments.map((c) => c.text.trim()))), strings: counts(bases.flatMap((base) => base.strings.map(stringKey))) }; const errors = []; for (const [file, text] of Object.entries(files)) { if (typeof text !== 'string') continue; - for (const e of checkText(scanText(text), pool, inOriginal)) errors.push({ file, ...e }); + for (const e of checkText(scanText(text, charsetForFile(file)), pool, inOriginal)) errors.push({ file, ...e }); } return errors; } @@ -190,6 +222,10 @@ function checkText(now, base, inOriginal) { const baseStrings = base.strings instanceof Map ? base.strings : counts(base.strings.map(stringKey)); for (const s of now.strings) { + if (s.pieces.some((piece) => piece === null)) { + errors.push({ line: s.line, message: 'this string literal cannot be represented in the configured execution character set' }); + continue; + } if (take(baseStrings, stringKey(s))) continue; if (s.include !== null) { if (!INCLUDE_PATH_RE.test(s.include) || s.include.startsWith('/')) { @@ -205,7 +241,7 @@ function checkText(now, base, inOriginal) { return errors; } -const stringKey = (s) => `${s.wide ? 'L' : ''}${s.include !== null ? '#' : ''}${Buffer.concat(s.pieces).toString('hex')}`; +const stringKey = (s) => `${s.wide ? 'L' : ''}${s.include !== null ? '#' : ''}${s.pieces.some((p) => p === null) ? `unencodable:${s.text}` : Buffer.concat(s.pieces).toString('hex')}`; function counts(list) { const m = new Map(); diff --git a/test/freetext.test.js b/test/freetext.test.js index e7f4372..bedf04a 100644 --- a/test/freetext.test.js +++ b/test/freetext.test.js @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { scanText, freeTextErrors, freeTextErrorsAcross, allowedComment } from '../src/shared/freetext.js'; +import { scanText, freeTextErrors, freeTextErrorsAcross, allowedComment, executionCharset } from '../src/shared/freetext.js'; // Stands in for the original executable's bytes. const original = Buffer.concat([Buffer.from('Interface\\GlueXML\0', 'latin1'), Buffer.from('hello world\0%d\0', 'latin1'), Buffer.from('Wide\0', 'utf16le')]); @@ -63,3 +63,26 @@ test('pieces of a concatenated literal may each come from the original', () => { assert.deepEqual(freeTextErrors('const char* s = "%d" "hello world";\n', null, inOriginal), []); assert.equal(freeTextErrors('const char* s = "%d" "made up";\n', null, inOriginal).length, 1); }); + +test('execution charset matches Hangul and universal escapes while preserving numeric byte escapes', () => { + const original = Buffer.from('c0afc0fa005769646500', 'hex'); + const found = (bytes) => original.includes(bytes); + assert.equal(executionCharset(['/source-charset:utf-8', '/execution-charset:.949']), 'cp949'); + assert.equal(executionCharset(['/execution-charset:.949', '/utf-8']), 'utf8'); + assert.deepEqual(freeTextErrors('const char* s = "유저";', null, found, 'cp949'), []); + assert.deepEqual(freeTextErrors(String.raw`const char* s = "\uC720\uC800";`, null, found, 'cp949'), []); + assert.deepEqual(freeTextErrors(String.raw`const char* s = "\xC0\xAF\xC0\xFA";`, null, found, 'cp949'), []); + assert.deepEqual(freeTextErrors(String.raw`const char* s = "유\xC0\xFA";`, null, found, 'cp949'), []); + assert.equal(freeTextErrors('const char* s = "유저";', null, found, 'utf8').length, 1); + assert.equal(freeTextErrors('const char* s = "새 문자열";', null, found, 'cp949').length, 1); + assert.equal(freeTextErrors('const char* s = "😀";', null, (b) => b.equals(Buffer.from('?')), 'cp949').length, 1, 'unrepresentable Unicode cannot pass through replacement bytes'); + assert.deepEqual(scanText('const wchar_t* s = L"유저";', 'cp949').strings[0].pieces, [Buffer.from('유저', 'utf16le')]); +}); + +test('mixed execution charsets preserve pooled original byte identity across moves', () => { + const baseFiles = { 'a.cpp': String.raw`const char* s = "\xC0\xAF\xC0\xFA";`, 'b.cpp': null }; + const options = { baseFiles, charsetForFile: (file) => file === 'a.cpp' ? 'utf8' : 'cp949' }; + assert.deepEqual(freeTextErrorsAcross({ 'a.cpp': '', 'b.cpp': 'const char* s = "유저";' }, '', () => false, options), []); + const twice = freeTextErrorsAcross({ 'a.cpp': baseFiles['a.cpp'], 'b.cpp': 'const char* s = "유저";' }, '', () => false, options); + assert.equal(twice.length, 1, 'one original occurrence does not exempt an extra copy'); +});