Translations keyed by labels instead of English text
tr('pnis.delivery_2_to_6_weeks') instead of tr('Delivery: 2 to 6 weeks'): the
English lives once, in src/i18n/en.js (inlined into the page at build, so no
extra request), and it/pt/fr map the same labels. Changing English wording no
longer renames a key in every dictionary.
- 2,899 labels, made from where each text is used and its first words; short
shared words are common.*
- plurals are one label with {one, other}: trn('label', n)
- static markup: data-i18n="label", data-i18n-aria-label, data-i18n-title
- saves from 1.33 and older stored English text in a few places; tr() still
resolves it
- i18n-check: undefined/unused labels, markup drift, plural shape; i18n-add.js
works on labels and --rename updates src/ too
- the old-branch port tool gives old English-keyed code and dictionaries the
same labels
This commit is contained in:
parent
d0ba85ef8f
commit
e817beed1a
94 changed files with 13681 additions and 10633 deletions
138
scripts/legacy/labels.js
Normal file
138
scripts/legacy/labels.js
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
// Turns English-keyed translation calls into label-keyed ones, for code written before labels (1.33 and
|
||||
// older): tr('Delivery: about a month') → tr('pnis.delivery_about_a_month'), trn('{n} POP','{n} POPs',n)
|
||||
// → trn('common.n_pop',n), and data-i18n markup → data-i18n="page.…". It is how src/ got its labels, and
|
||||
// transforms.js runs it on old branches so their work lines up with src/ (see port-branch.js).
|
||||
//
|
||||
// Labels for text that already has one come from `known` (the English dictionary, src/i18n/en.js);
|
||||
// new text gets a label made from its first words and where it is used, the same way every time.
|
||||
const vm = require('node:vm');
|
||||
|
||||
const LIT = String.raw`'(?:[^'\\\n]|\\.)*'|"(?:[^"\\\n]|\\.)*"|\x60(?:[^\x60\\$]|\\.|\$(?!\{))*\x60`;
|
||||
const CALL = new RegExp(String.raw`(?<![\w$.])(tr|trn|N_)\(\s*(${LIT})(?:\s*,\s*(${LIT}))?`, 'g');
|
||||
const CTX = vm.createContext({});
|
||||
const literal = (s) => vm.runInContext(s, CTX);
|
||||
|
||||
const hash = (t) => { let h = 2166136261; for (let i = 0; i < t.length; i++) { h ^= t.charCodeAt(i); h = Math.imul(h, 16777619); } return (h >>> 0).toString(16).padStart(8, '0'); };
|
||||
// "<b>Laser shortage!</b> {n} days…" → "laser_shortage_n_days"
|
||||
function slug(text) {
|
||||
const words = text.replace(/<[^>]*>/g, ' ').normalize('NFKD').replace(/[̀-ͯ]/g, '').toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, ' ').trim().split(' ').filter(Boolean);
|
||||
let s = '';
|
||||
for (const w of words) { if (s && (s + '_' + w).length > 36) break; s = s ? s + '_' + w : w; if (s.split('_').length >= 6) break; }
|
||||
return s || 'x';
|
||||
}
|
||||
// "game/70-rules/30-pnis.js" → "pnis"; the shell → "page".
|
||||
const areaOf = (file) => file === 'index.html' ? 'page' : file.split('/').pop().replace(/^\d+-/, '').replace(/\.[a-z]+$/, '').replace(/-/g, '_');
|
||||
const folderOf = (file) => file === 'index.html' ? 'page' : file.split('/').slice(0, -1).pop().replace(/^\d+-/, '').replace(/-/g, '_');
|
||||
// A plural is keyed by both forms.
|
||||
const pkey = (one, other) => `${one}\u0000${other}`;
|
||||
|
||||
const ENT = { amp: '&', lt: '<', gt: '>', quot: '"', '#39': "'", apos: "'", nbsp: ' ' };
|
||||
const decode = (s) => s.replace(/&(amp|lt|gt|quot|#39|apos|nbsp);/g, (m, e) => ENT[e]);
|
||||
|
||||
// Every translatable text in the partials: [{file, kind:'call'|'markup'|'attr', key, one, other, …}].
|
||||
function scan(parts) {
|
||||
const out = [];
|
||||
for (const [file, text] of parts) {
|
||||
if (file.endsWith('.js') || file === 'index.html') {
|
||||
for (const m of text.matchAll(CALL)) {
|
||||
if (file === 'index.html' && !inScript(text, m.index)) continue;
|
||||
if (m[1] === 'trn') {
|
||||
if (!m[3]) throw new Error(`${file}: trn() needs two literal forms: ${m[0].slice(0, 60)}`);
|
||||
const one = literal(m[2]), other = literal(m[3]);
|
||||
out.push({ file, kind: 'call', fn: 'trn', key: pkey(one, other), one, other, at: m.index + m[0].indexOf(m[2]), end: m.index + m[0].length });
|
||||
} else {
|
||||
const t = literal(m[2]);
|
||||
out.push({ file, kind: 'call', fn: m[1], key: t, text: t, at: m.index + m[0].indexOf(m[2]), end: m.index + m[0].indexOf(m[2]) + m[2].length });
|
||||
}
|
||||
}
|
||||
}
|
||||
if (file === 'index.html') {
|
||||
for (const m of text.matchAll(/<([a-z][a-z0-9]*)\b([^>]*?)\sdata-i18n>([^<]*)<\/\1>/g)) {
|
||||
out.push({ file, kind: 'markup', key: decode(m[3]), text: decode(m[3]), at: m.index + m[0].indexOf(' data-i18n>'), end: m.index + m[0].indexOf(' data-i18n>') + ' data-i18n'.length });
|
||||
}
|
||||
for (const m of text.matchAll(/<[a-z][a-z0-9]*\b[^>]*>/g)) {
|
||||
if (inScript(text, m.index)) continue;
|
||||
for (const a of m[0].matchAll(/\s(aria-label|title)="([^"]*)"/g)) {
|
||||
const pos = m.index + a.index + a[0].length;
|
||||
out.push({ file, kind: 'attr', attr: a[1], key: decode(a[2]), text: decode(a[2]), at: pos, end: pos });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
// Inside one of the shell's <script> elements (the language loader), where text is code, not markup.
|
||||
function inScript(html, i) {
|
||||
const open = html.lastIndexOf('<script', i), close = html.lastIndexOf('</script>', i);
|
||||
return open > close;
|
||||
}
|
||||
|
||||
// Labels for every text in the scan: known ones from `known` (Map key → label), new ones made here.
|
||||
function assign(items, known) {
|
||||
const labels = new Map(known), taken = new Set(known.values());
|
||||
const files = new Map();
|
||||
for (const it of items) { if (!files.has(it.key)) files.set(it.key, new Set()); files.get(it.key).add(it.file); }
|
||||
for (const it of items) {
|
||||
if (labels.has(it.key)) continue;
|
||||
const fs = [...files.get(it.key)];
|
||||
const folders = new Set(fs.map(folderOf));
|
||||
// Short, plain words ("Off", "Close", "Customers") are shared vocabulary wherever they appear.
|
||||
const short = !/[{<]/.test(it.one ?? it.text) && slug(it.one ?? it.text).split('_').length <= 2;
|
||||
const area = short ? 'common' : fs.length === 1 ? areaOf(fs[0]) : folders.size === 1 ? [...folders][0] : 'common';
|
||||
const base = `${area}.${slug(it.one ?? it.text)}`;
|
||||
let l = base;
|
||||
if (taken.has(l)) l = `${base}_${hash(it.key).slice(0, 4)}`;
|
||||
if (taken.has(l)) l = `${base}_${hash(it.key)}`;
|
||||
labels.set(it.key, l);
|
||||
taken.add(l);
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
// Rewrites the partials to use labels. Returns {parts, en}: en maps each label to its English
|
||||
// (a string, or {one, other} for plurals), for every text these partials use.
|
||||
function labelize(parts, known = new Map()) {
|
||||
const items = scan(parts);
|
||||
const labels = assign(items, known);
|
||||
const en = new Map();
|
||||
const byFile = new Map();
|
||||
for (const it of items) {
|
||||
const l = labels.get(it.key);
|
||||
en.set(l, it.fn === 'trn' ? { one: it.one, other: it.other } : it.text);
|
||||
if (!byFile.has(it.file)) byFile.set(it.file, []);
|
||||
byFile.get(it.file).push({ ...it, label: l });
|
||||
}
|
||||
const out = parts.map(([file, text]) => {
|
||||
const list = (byFile.get(file) || []).sort((a, b) => b.at - a.at);
|
||||
for (const it of list) {
|
||||
const rep = it.kind === 'call' ? `'${it.label}'`
|
||||
: it.kind === 'markup' ? ` data-i18n="${it.label}"`
|
||||
: ` data-i18n-${it.attr}="${it.label}"`;
|
||||
text = text.slice(0, it.at) + rep + text.slice(it.end);
|
||||
}
|
||||
return [file, text];
|
||||
});
|
||||
return { parts: out, en, labels };
|
||||
}
|
||||
|
||||
// The `known` map from an English dictionary object {label: text | {one, other}}.
|
||||
function knownFrom(enDict) {
|
||||
const k = new Map();
|
||||
for (const [l, v] of Object.entries(enDict || {})) k.set(typeof v === 'string' ? v : pkey(v.one, v.other), l);
|
||||
return k;
|
||||
}
|
||||
|
||||
// A dictionary keyed by English text (it.js before labels) → keyed by label, using the labels made above.
|
||||
function relabelDict(old, labels) {
|
||||
const out = {};
|
||||
for (const [key, l] of labels) {
|
||||
if (key.includes('\u0000')) {
|
||||
const [one, other] = key.split('\u0000');
|
||||
if (typeof old[one] === 'string' && typeof old[other] === 'string') out[l] = { one: old[one], other: old[other] };
|
||||
} else if (typeof old[key] === 'string') out[l] = old[key];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
module.exports = { labelize, knownFrom, relabelDict, scan, slug, pkey, CALL, LIT };
|
||||
|
|
@ -11,7 +11,8 @@
|
|||
// How: your branch's site/index.html and the one you started from are both split with the same rules
|
||||
// that made src/ (split-index-html.js + transforms.js), and each partial gets a three-way merge
|
||||
// (git merge-file) into the src/ file from main. Your dictionary changes (site/i18n/*.js) are applied
|
||||
// entry by entry onto main's sorted dictionaries, so appended lines never conflict.
|
||||
// entry by entry onto main's sorted dictionaries, so appended lines never conflict. Text keyed by English
|
||||
// (before 1.34) gets the labels src/ uses, and new text gets new labels in src/i18n/en.js (labels.js).
|
||||
//
|
||||
// It also works the other way round, merging commits made on the old layout into a branch that has src/.
|
||||
// Options: --base REF (the pre-split commit the old-layout side started from; default: merge-base of HEAD
|
||||
|
|
@ -21,7 +22,8 @@ const os = require('node:os');
|
|||
const path = require('node:path');
|
||||
const { execFileSync, spawnSync } = require('node:child_process');
|
||||
const { split } = require('./split-index-html');
|
||||
const { transform } = require('./transforms');
|
||||
const { transformFull } = require('./transforms');
|
||||
const { relabelDict } = require('./labels');
|
||||
const D = require('../lib/dict');
|
||||
|
||||
const ROOT = path.join(__dirname, '..', '..');
|
||||
|
|
@ -48,7 +50,7 @@ if (!theirPage) die(`${theirs} has no site/index.html; nothing to port`);
|
|||
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'tubes-port-'));
|
||||
const parts = (html, label) => {
|
||||
try { return new Map(transform(split(html))); }
|
||||
try { const r = transformFull(split(html)); return Object.assign(new Map(r.parts), { labels: r.labels }); }
|
||||
catch (e) { die(`could not split ${label}'s site/index.html the way src/ was made: ${e.message}`); }
|
||||
};
|
||||
const B = parts(basePage, 'the base'), T = parts(theirPage, 'your branch');
|
||||
|
|
@ -67,12 +69,17 @@ for (const [file, t] of T) {
|
|||
if (r.status !== 0) conflicts.push(`src/${file}`);
|
||||
}
|
||||
|
||||
// English merged cleanly: put it back in canonical order.
|
||||
if (!conflicts.includes('src/i18n/en.js') && fs.existsSync(D.fileOf('en'))) { const e = D.read('en'); D.write('en', e.dict, e.header); }
|
||||
|
||||
// Dictionaries: your additions, edits and removals, applied onto main's sorted files.
|
||||
for (const lang of D.langs()) {
|
||||
const rel = `site/i18n/${lang}.js`;
|
||||
const tb = show(base, rel), tt = show(theirs, rel);
|
||||
if (!tt || tb === tt) continue;
|
||||
const db = tb ? D.parse(tb, lang) : {}, dt = D.parse(tt, lang);
|
||||
// Old dictionaries are keyed by English text: give them the same labels the code got.
|
||||
const old = (t, labels) => { const d = D.parse(t, lang); return Object.keys(d).some((k) => /\s/.test(k)) ? relabelDict(d, labels) : d; };
|
||||
const db = tb ? old(tb, B.labels) : {}, dt = old(tt, T.labels);
|
||||
const main = target ? show(target, rel) : null;
|
||||
// During a merge the working file may hold conflict markers, so start from main's own copy.
|
||||
const { header, dict } = main ? { header: main.slice(0, main.indexOf('(window.I18N')), dict: D.parse(main, lang) } : D.read(lang);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
// 2. Coordinate tables that lived on one very long line (CITY_LL, CITY_NUDGE, LANDS_LL) get one entry per
|
||||
// line, CITY_LL and CITY_NUDGE sorted by id: adding a city no longer edits a 4,000-character line
|
||||
// that everyone else is editing too.
|
||||
// 3. Translations are keyed by labels, not by English text (labels.js): tr('pnis.delivery_month') with the
|
||||
// English in src/i18n/en.js, inlined into the page by an `<!-- @include i18n -->` line in the shell.
|
||||
|
||||
// [file, first action in it]. A group runs until the next group's first action.
|
||||
const ACT_GROUPS = [
|
||||
|
|
@ -82,7 +84,7 @@ function oneLine(text, re, { sort = false, indent = ' ' } = {}) {
|
|||
}).join('\n');
|
||||
}
|
||||
|
||||
function transform(parts) {
|
||||
function tidy(parts) {
|
||||
const out = [];
|
||||
for (const [file, text] of parts) {
|
||||
if (file === 'game/90-actions/00-actions.js') out.push(...splitActions(text));
|
||||
|
|
@ -95,4 +97,24 @@ function transform(parts) {
|
|||
return out;
|
||||
}
|
||||
|
||||
module.exports = { transform, entries };
|
||||
const EN_HEADER = `// English: the text of every label the game uses (in tr(), trn() and N_() calls in src/, and data-i18n in src/index.html).
|
||||
// Inlined into the page at build. One entry per line, sorted by label; plurals are {"one":…,"other":…}.
|
||||
// Add or change text with scripts/i18n-add.js, check with npm run i18n:check.
|
||||
`;
|
||||
|
||||
// All the tidying, plus labels. `known` maps English text to the labels src/ already has, so text that
|
||||
// exists today keeps its label (default: the current src/i18n/en.js). Returns the parts and the labels used.
|
||||
function transformFull(parts, known) {
|
||||
const D = require('../lib/dict');
|
||||
const { labelize, knownFrom } = require('./labels');
|
||||
if (!known) known = knownFrom(require('node:fs').existsSync(D.fileOf('en')) ? D.read('en').dict : {});
|
||||
const { parts: labelled, en, labels } = labelize(tidy(parts), known);
|
||||
const out = labelled.map(([file, text]) => file === 'index.html'
|
||||
? [file, text.replace('<!-- @include game -->', '<!-- @include i18n -->\n<!-- @include game -->')] : [file, text]);
|
||||
out.push(['i18n/en.js', D.format('en', EN_HEADER, Object.fromEntries(en))]);
|
||||
return { parts: out, labels };
|
||||
}
|
||||
|
||||
const transform = (parts, known) => transformFull(parts, known).parts;
|
||||
|
||||
module.exports = { transform, transformFull, tidy, entries };
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue