Validate original literals using the configured execution charset

README.md

@@ -68,6 +68,8 @@ attempt gets first pick on the follow-up for 6 hours.
 
 ## Maintainer bootstrap
 
+Run `npm ci` in the coordinator checkout before starting the server or running tests.
+
 ```sh
 # on the server (as the service user):  td-server <cmd> --data /var/lib/togetherdecomp
 td-server version add 1.0 game.exe              # register the target
@@ -131,6 +133,13 @@ Submitted overrides apply only to the current query. Reverse include traversal
 handles cycles without losing transitive dependents; uncommitted working-tree
 text is never used as the baseline.
 
+The original-string gate uses each source unit's configured MSVC execution charset
+(`/execution-charset:.949`, for example) when checking readable Unicode literals.
+Numeric byte escapes remain literal bytes, and wide strings remain UTF-16LE.
+Unrepresentable characters are rejected rather than accepted as replacement `?`
+bytes. Configure source UTF-8 and the target execution code page separately; this
+does not permit new strings absent from the original executable.
+
 ## Building td
 
 `node tools/build-td/build.mjs` (Node 22.21) writes `dist/td-windows-x64.exe`, `dist/td-linux-x64` and

deploy/install.sh

@@ -61,7 +61,8 @@ usermod -aG docker togetherdecomp
 # Code updates go through update.sh (releases behind a symlink); here it only lays the code down.
 if [ ! -e "$APP" ]; then
   mkdir -p /opt/togetherdecomp-releases/initial
-  cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" /opt/togetherdecomp-releases/initial/
+  cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" "$SRC/package-lock.json" /opt/togetherdecomp-releases/initial/
+  (cd /opt/togetherdecomp-releases/initial && PATH="/opt/node/bin:$PATH" /opt/node/bin/npm ci --omit=dev --ignore-scripts)
   chown -R root:root /opt/togetherdecomp-releases && chmod -R go-w,a+rX /opt/togetherdecomp-releases
   ln -s /opt/togetherdecomp-releases/initial "$APP"
 else

deploy/rodecomp/Dockerfile

@@ -3,7 +3,8 @@ FROM node:24-bookworm-slim
 RUN apt-get update && apt-get install -y --no-install-recommends git binutils ca-certificates && rm -rf /var/lib/apt/lists/*
 COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
 WORKDIR /app
-COPY package.json ./
+COPY package.json package-lock.json ./
+RUN npm ci --omit=dev --ignore-scripts
 COPY src ./src
 COPY ghidra_scripts ./ghidra_scripts
 COPY template ./template

deploy/rodecomp/Dockerfile.client

@@ -1,6 +1,7 @@
 FROM node:22.21.0-bookworm-slim
 WORKDIR /app
-COPY package.json ./
+COPY package.json package-lock.json ./
+RUN npm ci --omit=dev --ignore-scripts
 COPY src ./src
 COPY ghidra_scripts ./ghidra_scripts
 COPY tools/build-td ./tools/build-td

deploy/update.sh

@@ -32,7 +32,8 @@ OLD=$(readlink -f "$LINK" 2>/dev/null || true)
 
 echo "== preparing $REL"
 mkdir -p "$REL"
-cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" "$REL/"
+cp -a "$SRC/src" "$SRC/template" "$SRC/ghidra_scripts" "$SRC/toolchain" "$SRC/deploy" "$SRC/package.json" "$SRC/package-lock.json" "$REL/"
+(cd "$REL" && PATH="$(dirname "$NODE"):$PATH" "$(dirname "$NODE")/npm" ci --omit=dev --ignore-scripts)
 chown -R root:root "$REL"
 chmod -R go-w,a+rX "$REL"
 for f in $(find "$REL/src" -name '*.js'); do

package-lock.json

@@ -0,0 +1,41 @@
+{
+  "name": "clank-together",
+  "version": "0.1.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "clank-together",
+      "version": "0.1.0",
+      "license": "MIT",
+      "dependencies": {
+        "iconv-lite": "0.6.3"
+      },
+      "bin": {
+        "td": "src/client/cli.js",
+        "td-server": "src/server/cli.js"
+      },
+      "engines": {
+        "node": ">=22.13.0"
+      }
+    },
+    "node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+      "license": "MIT"
+    }
+  }
+}

package.json

@@ -15,5 +15,8 @@
     "test": "node --test \"test/**/*.test.js\"",
     "server": "node src/server/cli.js start",
     "client": "node src/client/cli.js"
+  },
+  "dependencies": {
+    "iconv-lite": "0.6.3"
   }
 }

src/server/verify.js

@@ -4,11 +4,11 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { lintSubmission } from '../shared/lint.js';
-import { freeTextErrorsAcross } from '../shared/freetext.js';
+import { freeTextErrorsAcross, executionCharset } from '../shared/freetext.js';
 import { checkUnits } from '../shared/check.js';
 import { reconcile } from '../shared/reconcile.js';
 import { STATUS, sameOriginal } from '../shared/compare.js';
-import { listSources, isUnit } from '../shared/project.js';
+import { listSources, isUnit, unitSettings } from '../shared/project.js';
 import { changedNames, unitsMentioning } from './affected.js';
 import { undefinedDeclarations } from './undefined.js';
 import { sourceIncluders } from './includers.js';
@@ -232,8 +232,9 @@ export class Verifier {
   freeTextGate(files, baseCommit) {
     const exe = this.getPe().buf;
     const inOriginal = (bytes) => exe.indexOf(bytes) !== -1;
-    const pooled = Object.keys(files).map((f) => this.repo.show(baseCommit, f)).filter((t) => t !== null).join('\n');
-    return freeTextErrorsAcross(files, pooled, inOriginal);
+    const baseFiles = Object.fromEntries(this.repo.showMany(baseCommit, Object.keys(files)));
+    const charsetForFile = (file) => executionCharset(unitSettings(this.project, file).cflags);
+    return freeTextErrorsAcross(files, '', inOriginal, { baseFiles, charsetForFile });
   }
 
   /**

src/shared/freetext.js

@@ -4,6 +4,8 @@
 // executable (text the original authors wrote). Identifier names are the only free text left.
 // Comments and strings already on main are left alone: the rules apply to what a submission adds.
 
+import iconv from 'iconv-lite';
+
 const ANNOTATION_RE = /^\/\/\s*(FUNCTION|STUB|GLOBAL|VTABLE|STRING|SIZE|LIBRARY|OFFSET)\s*:\s*[\w.]+(\s+<[\w\s:<>,*&]{1,120}>)?\s*$/;
 const OFFSET_RE = /^\/\*\s*(?:0x)?(?:[0-9A-Fa-f]{1,8}|-{2,8})\s*\*\/$/;
 const BLANK_RE = /^(?:\/\/[\s\-=*\/]*|\/\*[\s\-=*]*\*\/)$/;
@@ -23,7 +25,7 @@ export function allowedComment(text) {
  * Comments and string literals of C++ source. Adjacent literals ("a" "b") form one string.
  * @returns {{comments: {text: string, line: number}[], strings: {pieces: Buffer[], text: string, wide: boolean, line: number, include: string|null, linkage: boolean}[]}}
  */
-export function scanText(src) {
+export function scanText(src, executionCharset = null) {
   const comments = [];
   const literals = [];
   let line = 1;
@@ -59,14 +61,14 @@ export function scanText(src) {
     }
     const wide = c === 'L' && (n === '"' || n === "'") && !/\w/.test(src[i - 1] ?? '');
     if (c === '"' || (wide && n === '"')) {
-      const { value, end } = readQuoted(src, wide ? i + 1 : i);
+      const { value, segments, end } = readQuoted(src, wide ? i + 1 : i);
       const prev = literals[literals.length - 1];
       if (prev && prev.end === lastCodeEnd(src, i) && prev.include === null) {
-        prev.pieces.push(value);
+        prev.pieces.push({ value, segments });
         prev.wide ||= wide;
         prev.end = end;
       } else {
-        literals.push({ pieces: [value], wide, line, include: INCLUDE_RE.test(src.slice(lineBegin, i)) ? value : null, linkage: lastCode === 'extern', end });
+        literals.push({ pieces: [{ value, segments }], wide, line, include: INCLUDE_RE.test(src.slice(lineBegin, i)) ? value : null, linkage: lastCode === 'extern', end });
       }
       i = end;
       continue;
@@ -87,8 +89,8 @@ export function scanText(src) {
     i++;
   }
   const strings = literals.map((l) => ({
-    pieces: l.pieces.map((p) => encode(p, l.wide)),
-    text: l.pieces.join(''),
+    pieces: l.pieces.map((p) => encode(p, l.wide, executionCharset)),
+    text: l.pieces.map((p) => p.value).join(''),
     wide: l.wide,
     line: l.line,
     include: l.include,
@@ -110,46 +112,76 @@ const SIMPLE_ESCAPES = { n: '\n', t: '\t', r: '\r', a: '\x07', b: '\b', f: '\f',
 function readQuoted(src, at) {
   const q = src[at];
   let out = '';
+  const segments = [];
+  const append = (text, raw = false) => {
+    out += text;
+    const previous = segments[segments.length - 1];
+    if (previous && previous.raw === raw) previous.text += text;
+    else segments.push({ text, raw });
+  };
   let j = at + 1;
   while (j < src.length && src[j] !== q && src[j] !== '\n') {
     if (src[j] !== '\\') {
-      out += src[j++];
+      append(src[j++]);
       continue;
     }
     const e = src[j + 1];
     if (e in SIMPLE_ESCAPES) {
-      out += SIMPLE_ESCAPES[e];
+      append(SIMPLE_ESCAPES[e]);
       j += 2;
     } else if (e === 'x') {
       const m = /^[0-9A-Fa-f]+/.exec(src.slice(j + 2, j + 10));
-      out += String.fromCharCode(m ? parseInt(m[0], 16) & 0xffff : 0);
+      append(String.fromCharCode(m ? parseInt(m[0], 16) & 0xffff : 0), true);
       j += 2 + (m ? m[0].length : 0);
     } else if (e === 'u' || e === 'U') {
       const len = e === 'u' ? 4 : 8;
       const cp = parseInt(src.slice(j + 2, j + 2 + len), 16);
-      out += cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : '';
+      append(cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : '');
       j += 2 + len;
     } else if (/[0-7]/.test(e ?? '')) {
       const m = /^[0-7]{1,3}/.exec(src.slice(j + 1));
-      out += String.fromCharCode(parseInt(m[0], 8));
+      append(String.fromCharCode(parseInt(m[0], 8)), true);
       j += 1 + m[0].length;
     } else if (e === '\n') {
       j += 2; // line continuation
     } else if (e === '\r' && src[j + 2] === '\n') {
       j += 3;
     } else {
-      out += e ?? '';
+      append(e ?? '');
       j += 2;
     }
   }
-  return { value: out, end: j + 1 };
+  return { value: out, segments, end: j + 1 };
 }
 
 // The bytes the compiler emits for a literal (without the terminator).
-function encode(s, wide) {
-  if (wide) return Buffer.from(s, 'utf16le');
-  // Narrow literals are in the ANSI code page; characters beyond Latin-1 keep their UTF-8 bytes.
-  return [...s].every((ch) => ch.codePointAt(0) <= 0xff) ? Buffer.from(s, 'latin1') : Buffer.from(s, 'utf8');
+function encode(piece, wide, executionCharset) {
+  if (wide) return Buffer.from(piece.value, 'utf16le');
+  if (!executionCharset) {
+    return [...piece.value].every((ch) => ch.codePointAt(0) <= 0xff) ? Buffer.from(piece.value, 'latin1') : Buffer.from(piece.value, 'utf8');
+  }
+  const parts = [];
+  for (const segment of piece.segments) {
+    if (segment.raw) {
+      parts.push(Buffer.from(segment.text, 'latin1'));
+      continue;
+    }
+    const bytes = iconv.encode(segment.text, executionCharset);
+    if (iconv.decode(bytes, executionCharset) !== segment.text) return null;
+    parts.push(bytes);
+  }
+  return Buffer.concat(parts);
+}
+
+export function executionCharset(cflags) {
+  let charset = null;
+  for (const flag of cflags ?? []) {
+    if (/^[/-]utf-8$/i.test(flag)) charset = 'utf8';
+    const match = /^[/-]execution-charset:(.+)$/i.exec(flag);
+    if (match) charset = match[1].replace(/^\.(\d+)$/, 'cp$1');
+  }
+  if (charset && !iconv.encodingExists(charset)) throw new Error(`unsupported execution character set: ${charset}`);
+  return charset;
 }
 
 /**
@@ -159,8 +191,8 @@ function encode(s, wide) {
  * @param {(bytes: Buffer) => boolean} inOriginal  whether the bytes occur in the original executable
  * @returns {{line: number, message: string}[]}  the messages never repeat the text itself
  */
-export function freeTextErrors(text, baseText, inOriginal) {
-  return checkText(scanText(text), baseText === null ? { comments: [], strings: [] } : scanText(baseText), inOriginal);
+export function freeTextErrors(text, baseText, inOriginal, charset = null) {
+  return checkText(scanText(text, charset), baseText === null ? { comments: [], strings: [] } : scanText(baseText, charset), inOriginal);
 }
 
 /**
@@ -168,13 +200,13 @@ export function freeTextErrors(text, baseText, inOriginal) {
  * comments and strings may move from one submitted file to another, each base occurrence once.
  * @returns {{file: string, line: number, message: string}[]}
  */
-export function freeTextErrorsAcross(files, pooledBase, inOriginal) {
-  const base = scanText(pooledBase);
-  const pool = { comments: counts(base.comments.map((c) => c.text.trim())), strings: counts(base.strings.map(stringKey)) };
+export function freeTextErrorsAcross(files, pooledBase, inOriginal, { charsetForFile = () => null, baseFiles = null } = {}) {
+  const bases = baseFiles ? Object.entries(baseFiles).filter(([, text]) => text !== null).map(([file, text]) => scanText(text, charsetForFile(file))) : [scanText(pooledBase)];
+  const pool = { comments: counts(bases.flatMap((base) => base.comments.map((c) => c.text.trim()))), strings: counts(bases.flatMap((base) => base.strings.map(stringKey))) };
   const errors = [];
   for (const [file, text] of Object.entries(files)) {
     if (typeof text !== 'string') continue;
-    for (const e of checkText(scanText(text), pool, inOriginal)) errors.push({ file, ...e });
+    for (const e of checkText(scanText(text, charsetForFile(file)), pool, inOriginal)) errors.push({ file, ...e });
   }
   return errors;
 }
@@ -190,6 +222,10 @@ function checkText(now, base, inOriginal) {
 
   const baseStrings = base.strings instanceof Map ? base.strings : counts(base.strings.map(stringKey));
   for (const s of now.strings) {
+    if (s.pieces.some((piece) => piece === null)) {
+      errors.push({ line: s.line, message: 'this string literal cannot be represented in the configured execution character set' });
+      continue;
+    }
     if (take(baseStrings, stringKey(s))) continue;
     if (s.include !== null) {
       if (!INCLUDE_PATH_RE.test(s.include) || s.include.startsWith('/')) {
@@ -205,7 +241,7 @@ function checkText(now, base, inOriginal) {
   return errors;
 }
 
-const stringKey = (s) => `${s.wide ? 'L' : ''}${s.include !== null ? '#' : ''}${Buffer.concat(s.pieces).toString('hex')}`;
+const stringKey = (s) => `${s.wide ? 'L' : ''}${s.include !== null ? '#' : ''}${s.pieces.some((p) => p === null) ? `unencodable:${s.text}` : Buffer.concat(s.pieces).toString('hex')}`;
 
 function counts(list) {
   const m = new Map();

test/freetext.test.js

@@ -1,6 +1,6 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
-import { scanText, freeTextErrors, freeTextErrorsAcross, allowedComment } from '../src/shared/freetext.js';
+import { scanText, freeTextErrors, freeTextErrorsAcross, allowedComment, executionCharset } from '../src/shared/freetext.js';
 
 // Stands in for the original executable's bytes.
 const original = Buffer.concat([Buffer.from('Interface\\GlueXML\0', 'latin1'), Buffer.from('hello world\0%d\0', 'latin1'), Buffer.from('Wide\0', 'utf16le')]);
@@ -63,3 +63,26 @@ test('pieces of a concatenated literal may each come from the original', () => {
   assert.deepEqual(freeTextErrors('const char* s = "%d" "hello world";\n', null, inOriginal), []);
   assert.equal(freeTextErrors('const char* s = "%d" "made up";\n', null, inOriginal).length, 1);
 });
+
+test('execution charset matches Hangul and universal escapes while preserving numeric byte escapes', () => {
+  const original = Buffer.from('c0afc0fa005769646500', 'hex');
+  const found = (bytes) => original.includes(bytes);
+  assert.equal(executionCharset(['/source-charset:utf-8', '/execution-charset:.949']), 'cp949');
+  assert.equal(executionCharset(['/execution-charset:.949', '/utf-8']), 'utf8');
+  assert.deepEqual(freeTextErrors('const char* s = "유저";', null, found, 'cp949'), []);
+  assert.deepEqual(freeTextErrors(String.raw`const char* s = "\uC720\uC800";`, null, found, 'cp949'), []);
+  assert.deepEqual(freeTextErrors(String.raw`const char* s = "\xC0\xAF\xC0\xFA";`, null, found, 'cp949'), []);
+  assert.deepEqual(freeTextErrors(String.raw`const char* s = "유\xC0\xFA";`, null, found, 'cp949'), []);
+  assert.equal(freeTextErrors('const char* s = "유저";', null, found, 'utf8').length, 1);
+  assert.equal(freeTextErrors('const char* s = "새 문자열";', null, found, 'cp949').length, 1);
+  assert.equal(freeTextErrors('const char* s = "😀";', null, (b) => b.equals(Buffer.from('?')), 'cp949').length, 1, 'unrepresentable Unicode cannot pass through replacement bytes');
+  assert.deepEqual(scanText('const wchar_t* s = L"유저";', 'cp949').strings[0].pieces, [Buffer.from('유저', 'utf16le')]);
+});
+
+test('mixed execution charsets preserve pooled original byte identity across moves', () => {
+  const baseFiles = { 'a.cpp': String.raw`const char* s = "\xC0\xAF\xC0\xFA";`, 'b.cpp': null };
+  const options = { baseFiles, charsetForFile: (file) => file === 'a.cpp' ? 'utf8' : 'cp949' };
+  assert.deepEqual(freeTextErrorsAcross({ 'a.cpp': '', 'b.cpp': 'const char* s = "유저";' }, '', () => false, options), []);
+  const twice = freeTextErrorsAcross({ 'a.cpp': baseFiles['a.cpp'], 'b.cpp': 'const char* s = "유저";' }, '', () => false, options);
+  assert.equal(twice.length, 1, 'one original occurrence does not exempt an extra copy');
+});