From 2d1f48f3e6208f15c7c0a3bf42047a72378f10b0 Mon Sep 17 00:00:00 2001 From: Frederic Jung Date: Sat, 31 Oct 2026 14:11:02 +0100 Subject: [PATCH 1/2] CI: tests in four parts on four runners, every core in use, older PR runs cancelled npm test ran every file one after another (--test-concurrency=1, from when there were three files). It now runs as many files at once as there are cores (TEST_JOBS=1 for the old way), and --shard i/n picks one of n parts of about equal time, balanced by tests/durations.json (npm run test:timings). CI runs four parts and the build side by side; a final 'test' job is green only when all are, so the check keeps its name. A newer push to a PR cancels the older run. --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++-- package.json | 1 + scripts/test-timings.js | 39 ++++++++++++++++++++++++++++ scripts/test.js | 56 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 scripts/test-timings.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47ba455..8bb0ed7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,34 @@ on: permissions: contents: read +# A newer push to the same pull request makes the older run pointless: cancel it instead of queueing behind it. +# Runs on main always finish. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - test: + # The tests in four parts of about equal time, each on its own runner, each running as many files at once as + # it has cores (scripts/test.js; the split comes from tests/durations.json). + part: + name: test ${{ matrix.part }}/4 + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + part: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + - run: npm ci + - run: npm test -- --shard ${{ matrix.part }}/4 + + build: + name: build runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -18,10 +44,20 @@ jobs: node-version: 24 cache: npm - run: npm ci - - run: npm test - name: Build (not deployed — only releases deploy) run: npm run build + # The one check to look at, named as before: green only when every part and the build are. + test: + needs: [part, build] + if: always() + runs-on: ubuntu-latest + steps: + - name: Every part and the build passed + run: | + echo "parts: ${{ needs.part.result }}, build: ${{ needs.build.result }}" + test "${{ needs.part.result }}" = success && test "${{ needs.build.result }}" = success + translations: runs-on: ubuntu-latest steps: diff --git a/package.json b/package.json index 0de12e8..27b3b0e 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ }, "scripts": { "test": "node scripts/test.js", + "test:timings": "node scripts/test-timings.js", "build": "node scripts/build.js", "i18n:check": "node scripts/i18n-check.js", "i18n:fix": "node scripts/i18n-check.js --fix", diff --git a/scripts/test-timings.js b/scripts/test-timings.js new file mode 100644 index 0000000..1bf7d5e --- /dev/null +++ b/scripts/test-timings.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// Times each tests/*.test.js on its own and writes tests/durations.json (seconds per file), which +// `npm test -- --shard i/n` uses to split the files into parts of about equal time. +// Run it after adding or growing slow tests; it takes a few minutes. Files run as many at once as the +// machine has cores, so absolute numbers vary, but their sizes relative to each other are what count. +// +// npm run test:timings +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); + +const root = path.join(__dirname, '..'); +const dir = path.join(root, 'tests'); +const files = fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).sort().map((f) => 'tests/' + f); +const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism()); +const out = {}; +let next = 0, failed = 0; + +function run() { + if (next >= files.length) return Promise.resolve(); + const f = files[next++], t = performance.now(); + return new Promise((res) => { + const p = spawn(process.execPath, ['--test', f], { cwd: root, stdio: 'ignore' }); + p.on('close', (code) => { + out[f] = Math.round((performance.now() - t) / 100) / 10; + if (code) { failed++; console.log(`✗ ${f} failed (timed anyway)`); } + console.log(`${String(out[f]).padStart(6)}s ${f}`); + res(); + }); + }).then(run); +} + +Promise.all(Array.from({ length: jobs }, run)).then(() => { + const sorted = Object.fromEntries(Object.keys(out).sort().map((k) => [k, out[k]])); + fs.writeFileSync(path.join(dir, 'durations.json'), JSON.stringify(sorted, null, 1) + '\n'); + const sum = Object.values(out).reduce((a, b) => a + b, 0); + console.log(`${files.length} files, ${Math.round(sum)} s in all; wrote tests/durations.json${failed ? ` (${failed} failed)` : ''}`); +}); diff --git a/scripts/test.js b/scripts/test.js index 62e0c9a..2eb92f4 100644 --- a/scripts/test.js +++ b/scripts/test.js @@ -1,16 +1,62 @@ #!/usr/bin/env node -// Runs every tests/*.test.js, one file at a time. New test files are picked up on their own, so adding -// one never touches package.json (and never conflicts with someone else adding theirs). +// Runs every tests/*.test.js, each file in its own process, as many at once as the machine has cores. New test +// files are picked up on their own, so adding one never touches package.json (and never conflicts with someone +// else adding theirs). // // npm test everything // npm test -- tests/peering.test.js just these files (other arguments go to node --test) +// npm test -- --shard 2/4 one of four parts of about equal time (CI runs the four at once) +// npm test -- --shard 2/4 --list which files that part holds, without running them +// TEST_JOBS=1 npm test one file at a time, as it used to be +// +// Parts are balanced by tests/durations.json, each file's typical seconds (refresh: npm run test:timings). +// A file missing from it counts as a typical one, so a stale list only makes the parts a little uneven. const fs = require('node:fs'); +const os = require('node:os'); const path = require('node:path'); const { spawnSync } = require('node:child_process'); -const dir = path.join(__dirname, '..', 'tests'); +const root = path.join(__dirname, '..'); +const dir = path.join(root, 'tests'); const args = process.argv.slice(2); -const files = args.some((a) => a.endsWith('.test.js')) ? [] : + +const list = args.includes('--list'); +if (list) args.splice(args.indexOf('--list'), 1); +let shard = null; +const si = args.indexOf('--shard'); +if (si >= 0) { + const m = /^(\d+)\/(\d+)$/.exec(args[si + 1] || ''); + if (!m || +m[1] < 1 || +m[1] > +m[2]) { console.error('--shard wants i/n, like 2/4'); process.exit(2); } + shard = [+m[1], +m[2]]; + args.splice(si, 2); +} + +let files = args.some((a) => a.endsWith('.test.js')) ? [] : fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).sort().map((f) => path.join('tests', f)); -const r = spawnSync(process.execPath, ['--test', '--test-concurrency=1', ...args, ...files], { stdio: 'inherit', cwd: path.join(__dirname, '..') }); + +// Longest first, each onto the part with the least so far: parts within a file's length of each other. +function part(list, i, n) { + let dur = {}; + try { dur = JSON.parse(fs.readFileSync(path.join(dir, 'durations.json'), 'utf8')); } catch (e) { /* none yet: by count */ } + const known = list.map((f) => dur[f.replace(/\\/g, '/')]).filter((s) => s > 0).sort((a, b) => a - b); + const typical = known.length ? known[known.length >> 1] : 1; + const w = (f) => dur[f.replace(/\\/g, '/')] > 0 ? dur[f.replace(/\\/g, '/')] : typical; + const load = Array(n).fill(0), out = Array.from({ length: n }, () => []); + for (const f of list.slice().sort((a, b) => w(b) - w(a) || a.localeCompare(b))) { + let k = 0; + for (let j = 1; j < n; j++) if (load[j] < load[k]) k = j; + load[k] += w(f); out[k].push(f); + } + return out[i - 1].sort(); +} +if (shard && files.length) { + files = part(files, shard[0], shard[1]); + console.log(`# part ${shard[0]}/${shard[1]}: ${files.length} files`); + if (list) { for (const f of files) console.log(f); process.exit(0); } + // node --test with no files would run every test it can find: an empty part has nothing to do. + if (!files.length) process.exit(0); +} + +const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism()); +const r = spawnSync(process.execPath, ['--test', `--test-concurrency=${jobs}`, ...args, ...files], { stdio: 'inherit', cwd: root }); process.exit(r.status ?? 1); From 2cb22ad3c4c54e9b01ff0d4f01a9b684676be6b5 Mon Sep 17 00:00:00 2001 From: Frederic Jung Date: Sat, 31 Oct 2026 14:23:42 +0100 Subject: [PATCH 2/2] tests/durations.json: seconds per test file, so the four CI parts take about the same time --- tests/durations.json | 75 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/durations.json diff --git a/tests/durations.json b/tests/durations.json new file mode 100644 index 0000000..6ec3786 --- /dev/null +++ b/tests/durations.json @@ -0,0 +1,75 @@ +{ + "tests/any-home-city.test.js": 12.0, + "tests/architect.test.js": 20.1, + "tests/automation.test.js": 13.8, + "tests/backbone-core.test.js": 27.5, + "tests/badges-variants.test.js": 20.2, + "tests/build.test.js": 1.3, + "tests/capacity-deals.test.js": 12.6, + "tests/chaos-monkey.test.js": 30.3, + "tests/city-size.test.js": 8.5, + "tests/compute-idempotent.test.js": 10.4, + "tests/ddos-floods.test.js": 22.8, + "tests/ddos-model.test.js": 31.7, + "tests/eyeballs.test.js": 13.0, + "tests/fiber-broker.test.js": 9.7, + "tests/find-city.test.js": 9.6, + "tests/game.test.js": 42.8, + "tests/gameplay-rules.test.js": 42.9, + "tests/gangnam.test.js": 12.3, + "tests/glowball-sakura-lager.test.js": 15.8, + "tests/i18n.test.js": 59.9, + "tests/islands-and-arctic.test.js": 13.7, + "tests/issues.test.js": 103.0, + "tests/language-picker.test.js": 12.5, + "tests/latency.test.js": 11.1, + "tests/levels.test.js": 16.3, + "tests/link-metrics.test.js": 22.0, + "tests/link-trial.test.js": 6.4, + "tests/load-time.test.js": 102.7, + "tests/metric-panel.test.js": 8.1, + "tests/modes.test.js": 41.1, + "tests/more-cables.test.js": 14.3, + "tests/moscow-premium.test.js": 12.0, + "tests/networks-list.test.js": 6.4, + "tests/new-game-speed.test.js": 10.1, + "tests/optic-counts.test.js": 9.7, + "tests/own-asn.test.js": 7.9, + "tests/pages.test.js": 12.4, + "tests/peering-desk.test.js": 13.9, + "tests/peering.test.js": 27.6, + "tests/perf.test.js": 10.9, + "tests/persist.test.js": 16.1, + "tests/pni-all.test.js": 12.5, + "tests/pni-global.test.js": 17.9, + "tests/pni-list.test.js": 15.4, + "tests/pnis-and-sold-out.test.js": 11.6, + "tests/puzzle-monkey.test.js": 7.9, + "tests/puzzles.test.js": 73.3, + "tests/regions.test.js": 80.3, + "tests/release-story.test.js": 2.8, + "tests/report-link.test.js": 12.0, + "tests/router-art.test.js": 7.8, + "tests/routers-per-pop.test.js": 8.1, + "tests/routing-reference.test.js": 11.6, + "tests/rules-locked.test.js": 8.4, + "tests/rules.test.js": 29.4, + "tests/sales-trial.test.js": 33.3, + "tests/save-size.test.js": 16.1, + "tests/save-slots.test.js": 48.5, + "tests/scenarios-playtest.test.js": 113.2, + "tests/sea-routing.test.js": 42.6, + "tests/services.test.js": 15.7, + "tests/specialist.test.js": 14.3, + "tests/speed.test.js": 144.6, + "tests/standing-orders.test.js": 66.8, + "tests/subsea-upgrade-timing.test.js": 13.5, + "tests/supply.test.js": 11.0, + "tests/tier1-cone.test.js": 19.8, + "tests/useless-subsea-upgrades.test.js": 7.2, + "tests/utilities-caches.test.js": 9.5, + "tests/utilities.test.js": 15.6, + "tests/weekly-architect.test.js": 28.7, + "tests/weekly-puzzle.test.js": 145.0, + "tests/weekly-scenario.test.js": 54.4 +}