Faster tests, split everywhere: one sharded CI for pushes and releases

- The harness builds the DOM without the page's 2 MB of script text and
  runs the scripts straight in the window: a page load in tests takes
  about half the time (jsdom's parser was slow on the script text).
- That also took the game's source out of the DOM, which showed three
  tests that matched a phrase in the page source rather than the screen;
  they now check the panel, the routing window and the dictionary.
- CI plans its split once and runs the tests on 6 runners; every run on
  main records each file's time and caches it, so the parts stay even
  without anyone refreshing tests/durations.json.
- auto-release.yml and release.yml call the same sharded tests instead
  of running all 170 files on one runner each (11 and 15 min); a release
  is tested once, before it is tagged.
- Audit: five tests that duplicated a gameplay rule or checked nothing
  are gone; the two slowest checks sample a third of the cities, and a
  year of play to reach one due date became a moved due date.
This commit is contained in:
Frederic Jung 2026-11-07 07:24:09 +01:00
parent 7a6eb29013
commit 0342d90657
No known key found for this signature in database
17 changed files with 244 additions and 109 deletions

View file

@ -0,0 +1,18 @@
// A node:test reporter that writes how long each test file took, as JSON {"tests/x.test.js": seconds}, to its
// destination. scripts/test.js adds it next to the usual output with --timings FILE; CI merges the parts' files
// into tests/durations.json, which the next run splits its parts by.
const path = require('node:path');
module.exports = async function* timings(source) {
const sec = {};
for await (const { type, data } of source) {
// Top-level tests only: their durations include their subtests.
if ((type === 'test:pass' || type === 'test:fail') && data.nesting === 0 && data.file && data.details) {
const f = path.relative(process.cwd(), data.file).split(path.sep).join('/');
sec[f] = (sec[f] || 0) + (data.details.duration_ms || 0) / 1000;
}
}
const out = {};
for (const f of Object.keys(sec).sort()) out[f] = Math.round(sec[f] * 10) / 10;
yield JSON.stringify(out, null, 1) + '\n';
};

View file

@ -5,6 +5,7 @@
// machine has cores, so absolute numbers vary, but their sizes relative to each other are what count.
//
// npm run test:timings
// node scripts/test-timings.js --merge DIR (CI) fold the parts' --timings files into it
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
@ -12,6 +13,24 @@ const { spawn } = require('node:child_process');
const root = path.join(__dirname, '..');
const dir = path.join(root, 'tests');
// --merge DIR: fold the timings CI's parts wrote (scripts/test.js --timings) into tests/durations.json. Files that are
// gone drop out; files no part ran keep their old time.
const mi = process.argv.indexOf('--merge');
if (mi >= 0) {
const from = process.argv[mi + 1], dur = path.join(dir, 'durations.json');
let out = {};
try { out = JSON.parse(fs.readFileSync(dur, 'utf8')); } catch (e) { /* none yet */ }
let n = 0;
for (const f of fs.readdirSync(from, { recursive: true }).filter((f) => String(f).endsWith('.json'))) {
for (const [k, v] of Object.entries(JSON.parse(fs.readFileSync(path.join(from, String(f)), 'utf8')))) if (v > 0) { out[k] = v; n++; }
}
const have = new Set(fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).map((f) => 'tests/' + f));
const sorted = Object.fromEntries(Object.keys(out).filter((k) => have.has(k)).sort().map((k) => [k, out[k]]));
fs.writeFileSync(dur, JSON.stringify(sorted, null, 1) + '\n');
console.log(`merged ${n} timings; tests/durations.json has ${Object.keys(sorted).length} of ${have.size} files`);
process.exit(0);
}
const files = fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).sort().map((f) => 'tests/' + f);
const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism());
const out = {};

View file

@ -5,14 +5,18 @@
//
// npm test everything
// npm test -- tests/peering.test.js just these files (other arguments go to node --test)
// npm test -- --shard 2/4 one of four parts of about equal time (CI runs the four at once)
// npm test -- --shard 2/4 one of four parts of about equal time
// npm test -- --shard 2/4 --list which files that part holds, without running them
// node scripts/test.js --plan 6 the split into 6 parts as JSON (CI's plan job: one list per runner)
// npm test -- --timings out.json also write each file's seconds to out.json (CI merges these into
// the durations the next run splits by)
// TEST_JOBS=1 npm test one file at a time, as it used to be
// npm test -- --stress 8 tests/x.test.js hunt a flaky test: 8 copies of these files at once, on a machine kept
// busier than CI ever is, and how many runs failed (and why)
//
// Parts are balanced by tests/durations.json, each file's typical seconds (refresh: npm run test:timings).
// A file missing from it counts as a typical one, so a stale list only makes the parts a little uneven.
// Parts are balanced by tests/durations.json, each file's typical seconds. CI keeps it fresh in its cache from every
// run on main (.github/workflows/ci.yml); npm run test:timings rewrites the committed copy. A file missing from it
// counts as a typical one, so a stale list only makes the parts a little uneven.
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
@ -31,6 +35,12 @@ if (ti >= 0) {
if (!(stress >= 1)) { console.error('--stress wants a count, like --stress 8'); process.exit(2); }
args.splice(ti, 2);
}
let timings = null;
const tmi = args.indexOf('--timings');
if (tmi >= 0) { timings = args[tmi + 1]; args.splice(tmi, 2); if (!timings) { console.error('--timings wants a file'); process.exit(2); } }
let plan = 0;
const pi = args.indexOf('--plan');
if (pi >= 0) { plan = parseInt(args[pi + 1], 10); args.splice(pi, 2); if (!(plan >= 1)) { console.error('--plan wants a count, like --plan 6'); process.exit(2); } }
let shard = null;
const si = args.indexOf('--shard');
if (si >= 0) {
@ -58,6 +68,10 @@ function part(list, i, n) {
}
return out[i - 1].sort();
}
if (plan) {
console.log(JSON.stringify(Array.from({ length: plan }, (_, i) => part(files, i + 1, plan).join(' '))));
process.exit(0);
}
if (shard && files.length) {
files = part(files, shard[0], shard[1]);
console.log(`# part ${shard[0]}/${shard[1]}: ${files.length} files`);
@ -84,5 +98,7 @@ if (stress) {
});
return;
}
const r = spawnSync(process.execPath, ['--test', `--test-concurrency=${jobs}`, ...args, ...files], { stdio: 'inherit', cwd: root });
const rep = timings ? ['--test-reporter=spec', '--test-reporter-destination=stdout',
`--test-reporter=${path.join(__dirname, 'lib', 'timings-reporter.js')}`, `--test-reporter-destination=${timings}`] : [];
const r = spawnSync(process.execPath, ['--test', `--test-concurrency=${jobs}`, ...rep, ...args, ...files], { stdio: 'inherit', cwd: root });
process.exit(r.status ?? 1);