Merge #247: CI: tests in four parts on four runners, every core in use

CI: tests in four parts on four runners, every core in use
This commit is contained in:
Frederic Jung 2026-10-31 14:27:07 +01:00 • committed by GitHub
commit 2decbdf69c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 204 additions and 7 deletions

View file

@ -8,8 +8,34 @@ on:
permissions:
contents: read
# A newer push to the same pull request makes the older run pointless: cancel it instead of queueing behind it.
# Runs on main always finish.
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test:
# The tests in four parts of about equal time, each on its own runner, each running as many files at once as
# it has cores (scripts/test.js; the split comes from tests/durations.json).
part:
name: test ${{ matrix.part }}/4
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
part: [1, 2, 3, 4]
steps:
- uses: actions/checkout@v7
- uses: actions/setup-node@v7
with:
node-version: 24
cache: npm
- run: npm ci
- run: npm test -- --shard ${{ matrix.part }}/4
build:
name: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
@ -18,10 +44,20 @@ jobs:
node-version: 24
cache: npm
- run: npm ci
- run: npm test
- name: Build (not deployed — only releases deploy)
run: npm run build
# The one check to look at, named as before: green only when every part and the build are.
test:
needs: [part, build]
if: always()
runs-on: ubuntu-latest
steps:
- name: Every part and the build passed
run: |
echo "parts: ${{ needs.part.result }}, build: ${{ needs.build.result }}"
test "${{ needs.part.result }}" = success && test "${{ needs.build.result }}" = success
translations:
runs-on: ubuntu-latest
steps:

View file

@ -11,6 +11,7 @@
},
"scripts": {
"test": "node scripts/test.js",
"test:timings": "node scripts/test-timings.js",
"build": "node scripts/build.js",
"i18n:check": "node scripts/i18n-check.js",
"i18n:fix": "node scripts/i18n-check.js --fix",

39
scripts/test-timings.js Normal file
View file

@ -0,0 +1,39 @@
#!/usr/bin/env node
// Times each tests/*.test.js on its own and writes tests/durations.json (seconds per file), which
// `npm test -- --shard i/n` uses to split the files into parts of about equal time.
// Run it after adding or growing slow tests; it takes a few minutes. Files run as many at once as the
// machine has cores, so absolute numbers vary, but their sizes relative to each other are what count.
//
// npm run test:timings
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawn } = require('node:child_process');
const root = path.join(__dirname, '..');
const dir = path.join(root, 'tests');
const files = fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).sort().map((f) => 'tests/' + f);
const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism());
const out = {};
let next = 0, failed = 0;
function run() {
if (next >= files.length) return Promise.resolve();
const f = files[next++], t = performance.now();
return new Promise((res) => {
const p = spawn(process.execPath, ['--test', f], { cwd: root, stdio: 'ignore' });
p.on('close', (code) => {
out[f] = Math.round((performance.now() - t) / 100) / 10;
if (code) { failed++; console.log(`✗ ${f} failed (timed anyway)`); }
console.log(`${String(out[f]).padStart(6)}s ${f}`);
res();
});
}).then(run);
}
Promise.all(Array.from({ length: jobs }, run)).then(() => {
const sorted = Object.fromEntries(Object.keys(out).sort().map((k) => [k, out[k]]));
fs.writeFileSync(path.join(dir, 'durations.json'), JSON.stringify(sorted, null, 1) + '\n');
const sum = Object.values(out).reduce((a, b) => a + b, 0);
console.log(`${files.length} files, ${Math.round(sum)} s in all; wrote tests/durations.json${failed ? ` (${failed} failed)` : ''}`);
});

View file

@ -1,16 +1,62 @@
#!/usr/bin/env node
// Runs every tests/*.test.js, one file at a time. New test files are picked up on their own, so adding
// one never touches package.json (and never conflicts with someone else adding theirs).
// Runs every tests/*.test.js, each file in its own process, as many at once as the machine has cores. New test
// files are picked up on their own, so adding one never touches package.json (and never conflicts with someone
// else adding theirs).
//
// npm test everything
// npm test -- tests/peering.test.js just these files (other arguments go to node --test)
// npm test -- --shard 2/4 one of four parts of about equal time (CI runs the four at once)
// npm test -- --shard 2/4 --list which files that part holds, without running them
// TEST_JOBS=1 npm test one file at a time, as it used to be
//
// Parts are balanced by tests/durations.json, each file's typical seconds (refresh: npm run test:timings).
// A file missing from it counts as a typical one, so a stale list only makes the parts a little uneven.
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const dir = path.join(__dirname, '..', 'tests');
const root = path.join(__dirname, '..');
const dir = path.join(root, 'tests');
const args = process.argv.slice(2);
const files = args.some((a) => a.endsWith('.test.js')) ? [] :
const list = args.includes('--list');
if (list) args.splice(args.indexOf('--list'), 1);
let shard = null;
const si = args.indexOf('--shard');
if (si >= 0) {
const m = /^(\d+)\/(\d+)$/.exec(args[si + 1] || '');
if (!m || +m[1] < 1 || +m[1] > +m[2]) { console.error('--shard wants i/n, like 2/4'); process.exit(2); }
shard = [+m[1], +m[2]];
args.splice(si, 2);
}
let files = args.some((a) => a.endsWith('.test.js')) ? [] :
fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).sort().map((f) => path.join('tests', f));
const r = spawnSync(process.execPath, ['--test', '--test-concurrency=1', ...args, ...files], { stdio: 'inherit', cwd: path.join(__dirname, '..') });
// Longest first, each onto the part with the least so far: parts within a file's length of each other.
function part(list, i, n) {
let dur = {};
try { dur = JSON.parse(fs.readFileSync(path.join(dir, 'durations.json'), 'utf8')); } catch (e) { /* none yet: by count */ }
const known = list.map((f) => dur[f.replace(/\\/g, '/')]).filter((s) => s > 0).sort((a, b) => a - b);
const typical = known.length ? known[known.length >> 1] : 1;
const w = (f) => dur[f.replace(/\\/g, '/')] > 0 ? dur[f.replace(/\\/g, '/')] : typical;
const load = Array(n).fill(0), out = Array.from({ length: n }, () => []);
for (const f of list.slice().sort((a, b) => w(b) - w(a) || a.localeCompare(b))) {
let k = 0;
for (let j = 1; j < n; j++) if (load[j] < load[k]) k = j;
load[k] += w(f); out[k].push(f);
}
return out[i - 1].sort();
}
if (shard && files.length) {
files = part(files, shard[0], shard[1]);
console.log(`# part ${shard[0]}/${shard[1]}: ${files.length} files`);
if (list) { for (const f of files) console.log(f); process.exit(0); }
// node --test with no files would run every test it can find: an empty part has nothing to do.
if (!files.length) process.exit(0);
}
const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism());
const r = spawnSync(process.execPath, ['--test', `--test-concurrency=${jobs}`, ...args, ...files], { stdio: 'inherit', cwd: root });
process.exit(r.status ?? 1);

75
tests/durations.json Normal file
View file

@ -0,0 +1,75 @@
{
"tests/any-home-city.test.js": 12.0,
"tests/architect.test.js": 20.1,
"tests/automation.test.js": 13.8,
"tests/backbone-core.test.js": 27.5,
"tests/badges-variants.test.js": 20.2,
"tests/build.test.js": 1.3,
"tests/capacity-deals.test.js": 12.6,
"tests/chaos-monkey.test.js": 30.3,
"tests/city-size.test.js": 8.5,
"tests/compute-idempotent.test.js": 10.4,
"tests/ddos-floods.test.js": 22.8,
"tests/ddos-model.test.js": 31.7,
"tests/eyeballs.test.js": 13.0,
"tests/fiber-broker.test.js": 9.7,
"tests/find-city.test.js": 9.6,
"tests/game.test.js": 42.8,
"tests/gameplay-rules.test.js": 42.9,
"tests/gangnam.test.js": 12.3,
"tests/glowball-sakura-lager.test.js": 15.8,
"tests/i18n.test.js": 59.9,
"tests/islands-and-arctic.test.js": 13.7,
"tests/issues.test.js": 103.0,
"tests/language-picker.test.js": 12.5,
"tests/latency.test.js": 11.1,
"tests/levels.test.js": 16.3,
"tests/link-metrics.test.js": 22.0,
"tests/link-trial.test.js": 6.4,
"tests/load-time.test.js": 102.7,
"tests/metric-panel.test.js": 8.1,
"tests/modes.test.js": 41.1,
"tests/more-cables.test.js": 14.3,
"tests/moscow-premium.test.js": 12.0,
"tests/networks-list.test.js": 6.4,
"tests/new-game-speed.test.js": 10.1,
"tests/optic-counts.test.js": 9.7,
"tests/own-asn.test.js": 7.9,
"tests/pages.test.js": 12.4,
"tests/peering-desk.test.js": 13.9,
"tests/peering.test.js": 27.6,
"tests/perf.test.js": 10.9,
"tests/persist.test.js": 16.1,
"tests/pni-all.test.js": 12.5,
"tests/pni-global.test.js": 17.9,
"tests/pni-list.test.js": 15.4,
"tests/pnis-and-sold-out.test.js": 11.6,
"tests/puzzle-monkey.test.js": 7.9,
"tests/puzzles.test.js": 73.3,
"tests/regions.test.js": 80.3,
"tests/release-story.test.js": 2.8,
"tests/report-link.test.js": 12.0,
"tests/router-art.test.js": 7.8,
"tests/routers-per-pop.test.js": 8.1,
"tests/routing-reference.test.js": 11.6,
"tests/rules-locked.test.js": 8.4,
"tests/rules.test.js": 29.4,
"tests/sales-trial.test.js": 33.3,
"tests/save-size.test.js": 16.1,
"tests/save-slots.test.js": 48.5,
"tests/scenarios-playtest.test.js": 113.2,
"tests/sea-routing.test.js": 42.6,
"tests/services.test.js": 15.7,
"tests/specialist.test.js": 14.3,
"tests/speed.test.js": 144.6,
"tests/standing-orders.test.js": 66.8,
"tests/subsea-upgrade-timing.test.js": 13.5,
"tests/supply.test.js": 11.0,
"tests/tier1-cone.test.js": 19.8,
"tests/useless-subsea-upgrades.test.js": 7.2,
"tests/utilities-caches.test.js": 9.5,
"tests/utilities.test.js": 15.6,
"tests/weekly-architect.test.js": 28.7,
"tests/weekly-puzzle.test.js": 145.0,
"tests/weekly-scenario.test.js": 54.4
}