Translations keyed by labels instead of English text

tr('pnis.delivery_2_to_6_weeks') instead of tr('Delivery: 2 to 6 weeks'): the
English lives once, in src/i18n/en.js (inlined into the page at build, so no
extra request), and it/pt/fr map the same labels. Changing English wording no
longer renames a key in every dictionary.

- 2,899 labels, made from where each text is used and its first words; short
  shared words are common.*
- plurals are one label with {one, other}: trn('label', n)
- static markup: data-i18n="label", data-i18n-aria-label, data-i18n-title
- saves from 1.33 and older stored English text in a few places; tr() still
  resolves it
- i18n-check: undefined/unused labels, markup drift, plural shape; i18n-add.js
  works on labels and --rename updates src/ too
- the old-branch port tool gives old English-keyed code and dictionaries the
  same labels
This commit is contained in:
Frederic Jung 2026-10-27 11:05:09 +01:00
parent d0ba85ef8f
commit e817beed1a
No known key found for this signature in database
94 changed files with 13681 additions and 10633 deletions

138
scripts/legacy/labels.js Normal file
View file

@ -0,0 +1,138 @@
// Turns English-keyed translation calls into label-keyed ones, for code written before labels (1.33 and
// older): tr('Delivery: about a month') → tr('pnis.delivery_about_a_month'), trn('{n} POP','{n} POPs',n)
// → trn('common.n_pop',n), and data-i18n markup → data-i18n="page.…". It is how src/ got its labels, and
// transforms.js runs it on old branches so their work lines up with src/ (see port-branch.js).
//
// Labels for text that already has one come from `known` (the English dictionary, src/i18n/en.js);
// new text gets a label made from its first words and where it is used, the same way every time.
const vm = require('node:vm');
const LIT = String.raw`'(?:[^'\\\n]|\\.)*'|"(?:[^"\\\n]|\\.)*"|\x60(?:[^\x60\\$]|\\.|\$(?!\{))*\x60`;
const CALL = new RegExp(String.raw`(?<![\w$.])(tr|trn|N_)\(\s*(${LIT})(?:\s*,\s*(${LIT}))?`, 'g');
const CTX = vm.createContext({});
const literal = (s) => vm.runInContext(s, CTX);
const hash = (t) => { let h = 2166136261; for (let i = 0; i < t.length; i++) { h ^= t.charCodeAt(i); h = Math.imul(h, 16777619); } return (h >>> 0).toString(16).padStart(8, '0'); };
// "<b>Laser shortage!</b> {n} days…" → "laser_shortage_n_days"
function slug(text) {
const words = text.replace(/<[^>]*>/g, ' ').normalize('NFKD').replace(/[̀-ͯ]/g, '').toLowerCase()
.replace(/[^a-z0-9]+/g, ' ').trim().split(' ').filter(Boolean);
let s = '';
for (const w of words) { if (s && (s + '_' + w).length > 36) break; s = s ? s + '_' + w : w; if (s.split('_').length >= 6) break; }
return s || 'x';
}
// "game/70-rules/30-pnis.js" → "pnis"; the shell → "page".
const areaOf = (file) => file === 'index.html' ? 'page' : file.split('/').pop().replace(/^\d+-/, '').replace(/\.[a-z]+$/, '').replace(/-/g, '_');
const folderOf = (file) => file === 'index.html' ? 'page' : file.split('/').slice(0, -1).pop().replace(/^\d+-/, '').replace(/-/g, '_');
// A plural is keyed by both forms.
const pkey = (one, other) => `${one}\u0000${other}`;
const ENT = { amp: '&', lt: '<', gt: '>', quot: '"', '#39': "'", apos: "'", nbsp: ' ' };
const decode = (s) => s.replace(/&(amp|lt|gt|quot|#39|apos|nbsp);/g, (m, e) => ENT[e]);
// Every translatable text in the partials: [{file, kind:'call'|'markup'|'attr', key, one, other, …}].
function scan(parts) {
const out = [];
for (const [file, text] of parts) {
if (file.endsWith('.js') || file === 'index.html') {
for (const m of text.matchAll(CALL)) {
if (file === 'index.html' && !inScript(text, m.index)) continue;
if (m[1] === 'trn') {
if (!m[3]) throw new Error(`${file}: trn() needs two literal forms: ${m[0].slice(0, 60)}`);
const one = literal(m[2]), other = literal(m[3]);
out.push({ file, kind: 'call', fn: 'trn', key: pkey(one, other), one, other, at: m.index + m[0].indexOf(m[2]), end: m.index + m[0].length });
} else {
const t = literal(m[2]);
out.push({ file, kind: 'call', fn: m[1], key: t, text: t, at: m.index + m[0].indexOf(m[2]), end: m.index + m[0].indexOf(m[2]) + m[2].length });
}
}
}
if (file === 'index.html') {
for (const m of text.matchAll(/<([a-z][a-z0-9]*)\b([^>]*?)\sdata-i18n>([^<]*)<\/\1>/g)) {
out.push({ file, kind: 'markup', key: decode(m[3]), text: decode(m[3]), at: m.index + m[0].indexOf(' data-i18n>'), end: m.index + m[0].indexOf(' data-i18n>') + ' data-i18n'.length });
}
for (const m of text.matchAll(/<[a-z][a-z0-9]*\b[^>]*>/g)) {
if (inScript(text, m.index)) continue;
for (const a of m[0].matchAll(/\s(aria-label|title)="([^"]*)"/g)) {
const pos = m.index + a.index + a[0].length;
out.push({ file, kind: 'attr', attr: a[1], key: decode(a[2]), text: decode(a[2]), at: pos, end: pos });
}
}
}
}
return out;
}
// Inside one of the shell's <script> elements (the language loader), where text is code, not markup.
function inScript(html, i) {
const open = html.lastIndexOf('<script', i), close = html.lastIndexOf('</script>', i);
return open > close;
}
// Labels for every text in the scan: known ones from `known` (Map key → label), new ones made here.
function assign(items, known) {
const labels = new Map(known), taken = new Set(known.values());
const files = new Map();
for (const it of items) { if (!files.has(it.key)) files.set(it.key, new Set()); files.get(it.key).add(it.file); }
for (const it of items) {
if (labels.has(it.key)) continue;
const fs = [...files.get(it.key)];
const folders = new Set(fs.map(folderOf));
// Short, plain words ("Off", "Close", "Customers") are shared vocabulary wherever they appear.
const short = !/[{<]/.test(it.one ?? it.text) && slug(it.one ?? it.text).split('_').length <= 2;
const area = short ? 'common' : fs.length === 1 ? areaOf(fs[0]) : folders.size === 1 ? [...folders][0] : 'common';
const base = `${area}.${slug(it.one ?? it.text)}`;
let l = base;
if (taken.has(l)) l = `${base}_${hash(it.key).slice(0, 4)}`;
if (taken.has(l)) l = `${base}_${hash(it.key)}`;
labels.set(it.key, l);
taken.add(l);
}
return labels;
}
// Rewrites the partials to use labels. Returns {parts, en}: en maps each label to its English
// (a string, or {one, other} for plurals), for every text these partials use.
function labelize(parts, known = new Map()) {
const items = scan(parts);
const labels = assign(items, known);
const en = new Map();
const byFile = new Map();
for (const it of items) {
const l = labels.get(it.key);
en.set(l, it.fn === 'trn' ? { one: it.one, other: it.other } : it.text);
if (!byFile.has(it.file)) byFile.set(it.file, []);
byFile.get(it.file).push({ ...it, label: l });
}
const out = parts.map(([file, text]) => {
const list = (byFile.get(file) || []).sort((a, b) => b.at - a.at);
for (const it of list) {
const rep = it.kind === 'call' ? `'${it.label}'`
: it.kind === 'markup' ? ` data-i18n="${it.label}"`
: ` data-i18n-${it.attr}="${it.label}"`;
text = text.slice(0, it.at) + rep + text.slice(it.end);
}
return [file, text];
});
return { parts: out, en, labels };
}
// The `known` map from an English dictionary object {label: text | {one, other}}.
function knownFrom(enDict) {
const k = new Map();
for (const [l, v] of Object.entries(enDict || {})) k.set(typeof v === 'string' ? v : pkey(v.one, v.other), l);
return k;
}
// A dictionary keyed by English text (it.js before labels) → keyed by label, using the labels made above.
function relabelDict(old, labels) {
const out = {};
for (const [key, l] of labels) {
if (key.includes('\u0000')) {
const [one, other] = key.split('\u0000');
if (typeof old[one] === 'string' && typeof old[other] === 'string') out[l] = { one: old[one], other: old[other] };
} else if (typeof old[key] === 'string') out[l] = old[key];
}
return out;
}
module.exports = { labelize, knownFrom, relabelDict, scan, slug, pkey, CALL, LIT };

View file

@ -11,7 +11,8 @@
// How: your branch's site/index.html and the one you started from are both split with the same rules
// that made src/ (split-index-html.js + transforms.js), and each partial gets a three-way merge
// (git merge-file) into the src/ file from main. Your dictionary changes (site/i18n/*.js) are applied
// entry by entry onto main's sorted dictionaries, so appended lines never conflict.
// entry by entry onto main's sorted dictionaries, so appended lines never conflict. Text keyed by English
// (before 1.34) gets the labels src/ uses, and new text gets new labels in src/i18n/en.js (labels.js).
//
// It also works the other way round, merging commits made on the old layout into a branch that has src/.
// Options: --base REF (the pre-split commit the old-layout side started from; default: merge-base of HEAD
@ -21,7 +22,8 @@ const os = require('node:os');
const path = require('node:path');
const { execFileSync, spawnSync } = require('node:child_process');
const { split } = require('./split-index-html');
const { transform } = require('./transforms');
const { transformFull } = require('./transforms');
const { relabelDict } = require('./labels');
const D = require('../lib/dict');
const ROOT = path.join(__dirname, '..', '..');
@ -48,7 +50,7 @@ if (!theirPage) die(`${theirs} has no site/index.html; nothing to port`);
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'tubes-port-'));
const parts = (html, label) => {
try { return new Map(transform(split(html))); }
try { const r = transformFull(split(html)); return Object.assign(new Map(r.parts), { labels: r.labels }); }
catch (e) { die(`could not split ${label}'s site/index.html the way src/ was made: ${e.message}`); }
};
const B = parts(basePage, 'the base'), T = parts(theirPage, 'your branch');
@ -67,12 +69,17 @@ for (const [file, t] of T) {
if (r.status !== 0) conflicts.push(`src/${file}`);
}
// English merged cleanly: put it back in canonical order.
if (!conflicts.includes('src/i18n/en.js') && fs.existsSync(D.fileOf('en'))) { const e = D.read('en'); D.write('en', e.dict, e.header); }
// Dictionaries: your additions, edits and removals, applied onto main's sorted files.
for (const lang of D.langs()) {
const rel = `site/i18n/${lang}.js`;
const tb = show(base, rel), tt = show(theirs, rel);
if (!tt || tb === tt) continue;
const db = tb ? D.parse(tb, lang) : {}, dt = D.parse(tt, lang);
// Old dictionaries are keyed by English text: give them the same labels the code got.
const old = (t, labels) => { const d = D.parse(t, lang); return Object.keys(d).some((k) => /\s/.test(k)) ? relabelDict(d, labels) : d; };
const db = tb ? old(tb, B.labels) : {}, dt = old(tt, T.labels);
const main = target ? show(target, rel) : null;
// During a merge the working file may hold conflict markers, so start from main's own copy.
const { header, dict } = main ? { header: main.slice(0, main.indexOf('(window.I18N')), dict: D.parse(main, lang) } : D.read(lang);

View file

@ -8,6 +8,8 @@
// 2. Coordinate tables that lived on one very long line (CITY_LL, CITY_NUDGE, LANDS_LL) get one entry per
// line, CITY_LL and CITY_NUDGE sorted by id: adding a city no longer edits a 4,000-character line
// that everyone else is editing too.
// 3. Translations are keyed by labels, not by English text (labels.js): tr('pnis.delivery_month') with the
// English in src/i18n/en.js, inlined into the page by an `<!-- @include i18n -->` line in the shell.
// [file, first action in it]. A group runs until the next group's first action.
const ACT_GROUPS = [
@ -82,7 +84,7 @@ function oneLine(text, re, { sort = false, indent = ' ' } = {}) {
}).join('\n');
}
function transform(parts) {
function tidy(parts) {
const out = [];
for (const [file, text] of parts) {
if (file === 'game/90-actions/00-actions.js') out.push(...splitActions(text));
@ -95,4 +97,24 @@ function transform(parts) {
return out;
}
module.exports = { transform, entries };
const EN_HEADER = `// English: the text of every label the game uses (in tr(), trn() and N_() calls in src/, and data-i18n in src/index.html).
// Inlined into the page at build. One entry per line, sorted by label; plurals are {"one":…,"other":…}.
// Add or change text with scripts/i18n-add.js, check with npm run i18n:check.
`;
// All the tidying, plus labels. `known` maps English text to the labels src/ already has, so text that
// exists today keeps its label (default: the current src/i18n/en.js). Returns the parts and the labels used.
function transformFull(parts, known) {
const D = require('../lib/dict');
const { labelize, knownFrom } = require('./labels');
if (!known) known = knownFrom(require('node:fs').existsSync(D.fileOf('en')) ? D.read('en').dict : {});
const { parts: labelled, en, labels } = labelize(tidy(parts), known);
const out = labelled.map(([file, text]) => file === 'index.html'
? [file, text.replace('<!-- @include game -->', '<!-- @include i18n -->\n<!-- @include game -->')] : [file, text]);
out.push(['i18n/en.js', D.format('en', EN_HEADER, Object.fromEntries(en))]);
return { parts: out, labels };
}
const transform = (parts, known) => transformFull(parts, known).parts;
module.exports = { transform, transformFull, tidy, entries };