CI: tests in four parts on four runners, every core in use, older PR runs cancelled
npm test ran every file one after another (--test-concurrency=1, from when there were three files). It now runs as many files at once as there are cores (TEST_JOBS=1 for the old way), and --shard i/n picks one of n parts of about equal time, balanced by tests/durations.json (npm run test:timings). CI runs four parts and the build side by side; a final 'test' job is green only when all are, so the check keeps its name. A newer push to a PR cancels the older run.
This commit is contained in:
parent
7756d89d41
commit
2d1f48f3e6
4 changed files with 129 additions and 7 deletions
39
scripts/test-timings.js
Normal file
39
scripts/test-timings.js
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env node
|
||||
// Times each tests/*.test.js on its own and writes tests/durations.json (seconds per file), which
|
||||
// `npm test -- --shard i/n` uses to split the files into parts of about equal time.
|
||||
// Run it after adding or growing slow tests; it takes a few minutes. Files run as many at once as the
|
||||
// machine has cores, so absolute numbers vary, but their sizes relative to each other are what count.
|
||||
//
|
||||
// npm run test:timings
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawn } = require('node:child_process');
|
||||
|
||||
const root = path.join(__dirname, '..');
|
||||
const dir = path.join(root, 'tests');
|
||||
const files = fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).sort().map((f) => 'tests/' + f);
|
||||
const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism());
|
||||
const out = {};
|
||||
let next = 0, failed = 0;
|
||||
|
||||
function run() {
|
||||
if (next >= files.length) return Promise.resolve();
|
||||
const f = files[next++], t = performance.now();
|
||||
return new Promise((res) => {
|
||||
const p = spawn(process.execPath, ['--test', f], { cwd: root, stdio: 'ignore' });
|
||||
p.on('close', (code) => {
|
||||
out[f] = Math.round((performance.now() - t) / 100) / 10;
|
||||
if (code) { failed++; console.log(`✗ ${f} failed (timed anyway)`); }
|
||||
console.log(`${String(out[f]).padStart(6)}s ${f}`);
|
||||
res();
|
||||
});
|
||||
}).then(run);
|
||||
}
|
||||
|
||||
Promise.all(Array.from({ length: jobs }, run)).then(() => {
|
||||
const sorted = Object.fromEntries(Object.keys(out).sort().map((k) => [k, out[k]]));
|
||||
fs.writeFileSync(path.join(dir, 'durations.json'), JSON.stringify(sorted, null, 1) + '\n');
|
||||
const sum = Object.values(out).reduce((a, b) => a + b, 0);
|
||||
console.log(`${files.length} files, ${Math.round(sum)} s in all; wrote tests/durations.json${failed ? ` (${failed} failed)` : ''}`);
|
||||
});
|
||||
|
|
@ -1,16 +1,62 @@
|
|||
#!/usr/bin/env node
|
||||
// Runs every tests/*.test.js, one file at a time. New test files are picked up on their own, so adding
|
||||
// one never touches package.json (and never conflicts with someone else adding theirs).
|
||||
// Runs every tests/*.test.js, each file in its own process, as many at once as the machine has cores. New test
|
||||
// files are picked up on their own, so adding one never touches package.json (and never conflicts with someone
|
||||
// else adding theirs).
|
||||
//
|
||||
// npm test everything
|
||||
// npm test -- tests/peering.test.js just these files (other arguments go to node --test)
|
||||
// npm test -- --shard 2/4 one of four parts of about equal time (CI runs the four at once)
|
||||
// npm test -- --shard 2/4 --list which files that part holds, without running them
|
||||
// TEST_JOBS=1 npm test one file at a time, as it used to be
|
||||
//
|
||||
// Parts are balanced by tests/durations.json, each file's typical seconds (refresh: npm run test:timings).
|
||||
// A file missing from it counts as a typical one, so a stale list only makes the parts a little uneven.
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
|
||||
const dir = path.join(__dirname, '..', 'tests');
|
||||
const root = path.join(__dirname, '..');
|
||||
const dir = path.join(root, 'tests');
|
||||
const args = process.argv.slice(2);
|
||||
const files = args.some((a) => a.endsWith('.test.js')) ? [] :
|
||||
|
||||
const list = args.includes('--list');
|
||||
if (list) args.splice(args.indexOf('--list'), 1);
|
||||
let shard = null;
|
||||
const si = args.indexOf('--shard');
|
||||
if (si >= 0) {
|
||||
const m = /^(\d+)\/(\d+)$/.exec(args[si + 1] || '');
|
||||
if (!m || +m[1] < 1 || +m[1] > +m[2]) { console.error('--shard wants i/n, like 2/4'); process.exit(2); }
|
||||
shard = [+m[1], +m[2]];
|
||||
args.splice(si, 2);
|
||||
}
|
||||
|
||||
let files = args.some((a) => a.endsWith('.test.js')) ? [] :
|
||||
fs.readdirSync(dir).filter((f) => f.endsWith('.test.js')).sort().map((f) => path.join('tests', f));
|
||||
const r = spawnSync(process.execPath, ['--test', '--test-concurrency=1', ...args, ...files], { stdio: 'inherit', cwd: path.join(__dirname, '..') });
|
||||
|
||||
// Longest first, each onto the part with the least so far: parts within a file's length of each other.
|
||||
function part(list, i, n) {
|
||||
let dur = {};
|
||||
try { dur = JSON.parse(fs.readFileSync(path.join(dir, 'durations.json'), 'utf8')); } catch (e) { /* none yet: by count */ }
|
||||
const known = list.map((f) => dur[f.replace(/\\/g, '/')]).filter((s) => s > 0).sort((a, b) => a - b);
|
||||
const typical = known.length ? known[known.length >> 1] : 1;
|
||||
const w = (f) => dur[f.replace(/\\/g, '/')] > 0 ? dur[f.replace(/\\/g, '/')] : typical;
|
||||
const load = Array(n).fill(0), out = Array.from({ length: n }, () => []);
|
||||
for (const f of list.slice().sort((a, b) => w(b) - w(a) || a.localeCompare(b))) {
|
||||
let k = 0;
|
||||
for (let j = 1; j < n; j++) if (load[j] < load[k]) k = j;
|
||||
load[k] += w(f); out[k].push(f);
|
||||
}
|
||||
return out[i - 1].sort();
|
||||
}
|
||||
if (shard && files.length) {
|
||||
files = part(files, shard[0], shard[1]);
|
||||
console.log(`# part ${shard[0]}/${shard[1]}: ${files.length} files`);
|
||||
if (list) { for (const f of files) console.log(f); process.exit(0); }
|
||||
// node --test with no files would run every test it can find: an empty part has nothing to do.
|
||||
if (!files.length) process.exit(0);
|
||||
}
|
||||
|
||||
const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism());
|
||||
const r = spawnSync(process.execPath, ['--test', `--test-concurrency=${jobs}`, ...args, ...files], { stdio: 'inherit', cwd: root });
|
||||
process.exit(r.status ?? 1);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue