From 151b7786edb2023c72263d0227a282ed75d1494b Mon Sep 17 00:00:00 2001 From: Frederic Jung Date: Thu, 5 Nov 2026 11:38:26 +0100 Subject: [PATCH] Tests wait for state instead of sleeping, on free ports The relay, parity and benchmark server tests slept fixed times before asserting, which a busy CI runner overran. They now wait for what they need (the relay reports what it holds), take a port from the OS, and use fresh HTTP connections. npm test -- --stress N hunts flakes. --- .claude/skills/write-tests/SKILL.md | 14 ++++ scripts/agent/relay.js | 2 + scripts/test.js | 28 +++++++- tests/agent-relay.test.js | 105 +++++++++++++++------------- tests/gpu-bench-server.test.js | 8 ++- tests/mcp-parity.test.js | 9 +-- tests/net.js | 33 +++++++++ tests/noc-window.test.js | 4 +- tests/save-size.test.js | 4 +- tests/save-slots.test.js | 4 +- 10 files changed, 152 insertions(+), 59 deletions(-) create mode 100644 tests/net.js diff --git a/.claude/skills/write-tests/SKILL.md b/.claude/skills/write-tests/SKILL.md index 42937b9..bfdc934 100644 --- a/.claude/skills/write-tests/SKILL.md +++ b/.claude/skills/write-tests/SKILL.md @@ -54,3 +54,17 @@ pull requests collide. Run one file with `npm test -- tests/route-servers.test.j - **UI:** render with `renderAll()` and query `dom.window.document`; check phone layouts by setting `window.innerWidth = 390` before rendering. - Run `npm test`; the whole suite must pass, not just the new test. + +## Servers, sockets and anything async + +CI runs four parts at once, each with its files in parallel, so a test can wait seconds for things that take +milliseconds on your machine. Never sleep a fixed time and then assert: wait for the state with `tests/net.js`. + +- `until(() => condition)` resolves true as soon as it holds (false after 20 s); assert on its result. +- `freePort()` asks the OS for a free port; never a fixed or random one. +- `kill(child)` stops a spawned server and resolves once it is gone. +- Listen and connect on `127.0.0.1`, not `localhost` (it can resolve to `::1` only on a CI runner). +- For HTTP to a test server, pass `agent: false`: a kept-alive socket can be reset by the server's idle timer. +- A fixed wait is fine only to prove that something does *not* happen. + +Hunt a suspected flake before pushing: `npm test -- --stress 8 tests/x.test.js` runs 8 copies at once, busier than CI. diff --git a/scripts/agent/relay.js b/scripts/agent/relay.js index a99e1da..a596f9a 100644 --- a/scripts/agent/relay.js +++ b/scripts/agent/relay.js @@ -106,6 +106,8 @@ function createRelay({ server } = {}) { return { wss, path: AGENT_PATH, + // What the relay holds right now, so tests wait for a state instead of sleeping. + status: () => ({ page: !!(page && page.readyState === 1), tools: [...tools].filter((t) => t.readyState === 1).length, calls: pending.size }), close() { for (const c of [page, ...tools]) { try { c.close(4002, 'relay shutdown'); } catch {} } try { wss.close(); } catch {} }, }; } diff --git a/scripts/test.js b/scripts/test.js index 2eb92f4..e847053 100644 --- a/scripts/test.js +++ b/scripts/test.js @@ -8,13 +8,15 @@ // npm test -- --shard 2/4 one of four parts of about equal time (CI runs the four at once) // npm test -- --shard 2/4 --list which files that part holds, without running them // TEST_JOBS=1 npm test one file at a time, as it used to be +// npm test -- --stress 8 tests/x.test.js hunt a flaky test: 8 copies of these files at once, on a machine kept +// busier than CI ever is, and how many runs failed (and why) // // Parts are balanced by tests/durations.json, each file's typical seconds (refresh: npm run test:timings). // A file missing from it counts as a typical one, so a stale list only makes the parts a little uneven. const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -const { spawnSync } = require('node:child_process'); +const { spawn, spawnSync } = require('node:child_process'); const root = path.join(__dirname, '..'); const dir = path.join(root, 'tests'); @@ -22,6 +24,13 @@ const args = process.argv.slice(2); const list = args.includes('--list'); if (list) args.splice(args.indexOf('--list'), 1); +let stress = 0; +const ti = args.indexOf('--stress'); +if (ti >= 0) { + stress = parseInt(args[ti + 1], 10); + if (!(stress >= 1)) { console.error('--stress wants a count, like --stress 8'); process.exit(2); } + args.splice(ti, 2); +} let shard = null; const si = args.indexOf('--shard'); if (si >= 0) { @@ -58,5 +67,22 @@ if (shard && files.length) { } const jobs = Math.max(1, parseInt(process.env.TEST_JOBS, 10) || os.availableParallelism()); +// Stress: N runs of the same files at once, each its own node --test, so they fight for the CPU the way the four +// CI parts and their own parallel files do (and worse). A test that only passes on a quiet machine fails here. +if (stress) { + if (!args.some((a) => a.endsWith('.test.js'))) { console.error('--stress wants the test files to run, like tests/agent-relay.test.js'); process.exit(2); } + const runs = Array.from({ length: stress }, (_, i) => new Promise((res) => { + const c = spawn(process.execPath, ['--test', '--test-reporter=dot', ...args], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'] }); + let out = ''; c.stdout.on('data', (d) => { out += d; }); c.stderr.on('data', (d) => { out += d; }); + c.on('close', (code) => res({ i: i + 1, code, out })); + })); + Promise.all(runs).then((rs) => { + const bad = rs.filter((r) => r.code !== 0); + for (const r of bad) console.log(`\n--- run ${r.i} failed ---\n${r.out.slice(-4000)}`); + console.log(`\n# stress: ${rs.length - bad.length} of ${rs.length} runs passed, ${stress} at once on ${os.availableParallelism()} cores`); + process.exit(bad.length ? 1 : 0); + }); + return; +} const r = spawnSync(process.execPath, ['--test', `--test-concurrency=${jobs}`, ...args, ...files], { stdio: 'inherit', cwd: root }); process.exit(r.status ?? 1); diff --git a/tests/agent-relay.test.js b/tests/agent-relay.test.js index 21074f9..d24148e 100644 --- a/tests/agent-relay.test.js +++ b/tests/agent-relay.test.js @@ -18,19 +18,19 @@ const ROOT = path.join(__dirname, '..'); const { createRelay } = require('../scripts/agent/relay'); const { createRelaySession } = require('../scripts/lib/relay-client'); const SERVER = path.join(ROOT, 'scripts', 'mcp.js'); -const PORT = 8799 + (process.pid % 500); // dodge a busy port across suites +const { wait, until, freePort, kill } = require('./net'); +let PORT; // a free one from the OS, before the first test +test.before(async () => { PORT = await freePort(); }); const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'tubes-relay-')); -const wait = (ms) => new Promise((r) => setTimeout(r, ms)); -// Waits until ok() holds, up to ms: a busy CI runner can take seconds to load a page, so no fixed delay is enough. -const until = async (ok, ms = 20000) => { const t0 = Date.now(); while (!ok() && Date.now() - t0 < ms) await wait(50); return ok(); }; // The relay in-process, riding a real http.Server (the same createRelay the servers host): no child to leak, and // 'listening' means the first dial cannot race the bind. +let CUR = null; // the relay started last function startRelay(port = PORT) { const srv = http.createServer((req, res) => { res.writeHead(404); res.end(); }); const relay = createRelay({ server: srv }); - return { + return CUR = { relay, srv, listening: () => new Promise((res, rej) => { srv.once('error', rej); srv.listen(port, '127.0.0.1', res); }), close: () => new Promise((r) => { relay.close(); srv.close(() => r()); }), @@ -46,6 +46,12 @@ function page(opts = {}) { return a; } const dial = (a, port = PORT) => a.ev(`window.__AGENT_RELAY_TEST__=true;API_RELAY_HOST='127.0.0.1:'+${port};apiAgentSync()`); +// Waits until the relay holds this page and the page knows how many tools are there (the clock follows that count). +async function joined(a) { + assert.ok(await until(() => { const s = CUR.relay.status(); return s.page && a.ev('API_RELAY.tools') === s.tools; }), 'the page joined the relay'); +} +// Waits until the page has heard the relay's tool count: its clock runs or holds by it. +const heard = async (a, n) => assert.ok(await until(() => a.ev('API_RELAY.tools') === n), `the page sees ${n} tool(s)`); // Kill the page's dial and retry timer before a test ends, or its backoff keeps the process alive. const stop = (...ps) => { for (const a of ps) if (a) a.ev("window.__AGENT_RELAY_TEST__=false;apiAgentSet('on',false)"); }; @@ -74,7 +80,8 @@ function toolClient(port = PORT, headers) { close: () => { try { ws.close(); } catch {} }, }; } -async function tool() { const t = toolClient(); await t.open(); t.auth('tool'); await wait(150); return t; } +// A tool that has been let in: the relay's first presence message comes right after the role is granted. +async function tool(port = PORT) { const t = toolClient(port); await t.open(); t.auth('tool'); assert.ok(await until(() => t.events.length > 0), 'the relay let the tool in'); return t; } async function withRelay(fn) { const r = startRelay(); @@ -104,7 +111,7 @@ function mcp(argv, onErr) { child.stdin.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n'); return r; }, call: async (name, args) => (await send({ method: 'tools/call', params: { name, arguments: args || {} } })).result, json: async (name, args) => JSON.parse((await send({ method: 'tools/call', params: { name, arguments: args || {} } })).result.content[0].text), - kill: async () => { child.kill('SIGTERM'); await wait(200); child.kill('SIGKILL'); }, + kill: () => kill(child), }; } @@ -128,8 +135,8 @@ test('a live page answers tool calls through the relay, from a fixed list of too const a = page(); try { a.ev("ACT.start({});S.cash=500000;compute()"); - await wait(600); - assert.ok(t.events.some((e) => e.page === true), 'the tools see the page join'); + assert.ok(await until(() => t.events.some((e) => e.page === true)), 'the tools see the page join'); + await joined(a); const q = await t.call('act', { action: 'buildPop', args: { city: 'ams' }, dryRun: true }); assert.equal(q.value.outcome, 'quote'); assert.equal(q.value.quote.setup, 20000); @@ -156,7 +163,7 @@ test('in the player\'s tab the player owns the game, their windows, their settin const a = page({ ms: 0 }); try { a.ev("ACT.start({});S.cash=1e7;S.nextFail=S.day+40;S.nextEvent=S.day+50;compute()"); - await wait(500); + await joined(a); // The dice stay hidden, whole or by path, and the schema says so. const st = (await t.call('state')).value; for (const k of ['rng', 'nextFail', 'nextEvent']) assert.ok(!(k in st), `${k} is held back`); @@ -211,7 +218,7 @@ test('the agent\'s clock moves only with advance; a running clock only with time const a = page({ ms: 0 }); try { a.ev("ACT.start({});S.cash=1e7;compute()"); - await wait(500); + await joined(a); assert.equal(a.ev('UI.speed'), 0, 'a tool is connected, but the agent\'s clock stands still'); a.ev('last=performance.now();for(let i=1;i<=20;i++)frame(last+i*250)'); assert.equal(a.ev('S.day'), 0, 'the loop runs no days'); @@ -247,20 +254,19 @@ test('the page client reconnects after the relay restarts, and stops when the pl const a = page(); try { a.ev("ACT.start({});S.cash=500000;compute()"); - await wait(500); - assert.ok(t.events.some((e) => e.page === true)); + assert.ok(await until(() => t.events.some((e) => e.page === true))); await r.close(); - await wait(300); + assert.ok(await until(() => a.ev('!API_RELAY.ws||API_RELAY.ws.readyState!==1')), 'the page saw the relay go'); a.ev("apiAgentSet('on',false)"); const r2 = startRelay(); await r2.listening(); try { - await wait(1200); // past the first backoff retry + await wait(1200); // past the first backoff retry: it must not dial assert.equal(a.ev('API_RELAY.on'), false, 'the client is off'); + assert.equal(r2.relay.status().page, false, 'and never dialled'); a.ev("apiAgentSet('on',true)"); - await wait(1200); + await joined(a); const t2 = await tool(); - await wait(300); assert.ok(t2.events.some((e) => e.page === true), 'the page reconnected after the relay came back'); assert.equal((await t2.call('state', { path: 'cash' })).value, 500000, 'and answers calls again'); t2.close(); @@ -272,21 +278,21 @@ test('a paced clock holds while no tool is connected, and runs when one arrives' const a = page({ seed: 5 }); try { a.ev("ACT.start({});S.nextFail=S.day+99;S.nextEvent=S.day+99;compute()"); - await wait(500); + await joined(a); assert.equal(a.ev('UI.speed'), 0, 'no tool connected: the clock holds'); assert.equal(a.ev('UI.clockLocked'), true, 'agent mode stays on'); a.ev('last=performance.now();frame(last);frame(last+250);frame(last+500)'); assert.equal(a.ev('S.day'), 0, 'no day ran'); const t = await tool(); - await wait(300); + await heard(a, 1); assert.ok(a.ev('UI.speed') > 0, 'a tool connecting runs it'); t.close(); - await wait(400); + await heard(a, 0); assert.equal(a.ev('UI.speed'), 0, 'the tool leaving holds it again'); a.ev('S.uiSpeed=4;gameResume()'); assert.equal(a.ev('UI.speed'), 0, 'gameResume holds with no tool connected'); const t2 = await tool(); - await wait(300); + await heard(a, 1); assert.ok(a.ev('UI.speed') > 0, 'and a tool arriving runs it again'); // An unrelated preference change does not drop the connection or hold the clock. a.ev('window.__w0=API_RELAY.ws'); @@ -301,9 +307,11 @@ test('a paced clock holds while no tool is connected, and runs when one arrives' test('a second page replaces the first: one game per relay, the newest tab wins', () => withRelay(async () => { const a = page({ seed: 5 }); a.ev("ACT.start({});S.cash=111000;compute()"); + await joined(a); // in that order: two dials at once may land either way const b = page({ seed: 6 }); b.ev("ACT.start({});S.cash=222000;compute()"); - await wait(700); + assert.ok(await until(() => a.ev('API_RELAY.on') === false), 'the first page yields to the second'); + await joined(b); const t = await tool(); try { assert.equal((await t.call('state', { path: 'cash' })).value, 222000, 'the newest page answers'); @@ -313,7 +321,7 @@ test('a second page replaces the first: one game per relay, the newest tab wins' test('the MCP server drives the player\'s page in browser mode, over stdio', () => withRelay(async () => { const a = page({ seed: 9, ms: 0 }); a.ev("ACT.start({});S.cash=1000000;compute()"); - await wait(500); + await joined(a); const m = mcp(['--browser', '--relay-host', 'localhost', '--relay-port', String(PORT)]); try { const init = await m.init(); @@ -343,7 +351,7 @@ test('a game page older than the code stops the server rather than mislead', () a.ev("ACT.start({});compute()"); // A stale tab whose commands keep their names but change shape: only a deep compare of the sections catches it. a.ev('const __c=TUBES.call;TUBES.call=(t,x,c)=>{const v=__c(t,x,c);if(t==="schema"&&!x.section)v.commands.buildPop.args=[{name:"city",required:true}];return v;};'); - await wait(400); + await joined(a); const m = mcp(['--browser', '--relay-port', String(PORT), '--page-wait-ms', '8000']); try { const init = await m.init(); @@ -368,10 +376,11 @@ test('the MCP server answers the handshake at once and waits for the game page', try { const t0 = Date.now(); await m.init(); - assert.ok(Date.now() - t0 < 3000, 'initialize does not wait for a browser tab'); + // Far below the page wait (20 s), with room for a slow start of the process on a busy runner. + assert.ok(Date.now() - t0 < 10000, 'initialize does not wait for a browser tab'); const early = await m.call('status'); assert.equal(early.isError, true, 'a tool before the page arrives is an error, not a hang'); - await wait(300); + await until(() => /waiting for the game page/.test(m.stderr())); assert.match(m.stderr(), /waiting for the game page/); a = page({ seed: 4 }); a.ev("ACT.start({});S.cash=444000;compute()"); @@ -381,19 +390,18 @@ test('the MCP server answers the handshake at once and waits for the game page', } finally { stop(a); await m.kill(); } })); -const free = () => new Promise((res, rej) => { const s = http.createServer(); s.on('error', rej); s.listen(0, '127.0.0.1', () => { const p = s.address().port; s.close(() => res(p)); }); }); test('dev.js binds loopback and parses --port; the page dials its own host', async () => { - const port = await free(); + const port = await freePort(); const child = spawn(process.execPath, [path.join(ROOT, 'scripts', 'dev.js'), '--port', String(port)], { stdio: ['ignore', 'ignore', 'pipe'] }); let serr = ''; child.stderr.on('data', (d) => { serr += d; }); let a = null; try { - await wait(700); - assert.match(serr, new RegExp(`agent relay on ws://127\\.0\\.0\\.1:${port}/agent`), 'dev.js binds loopback and honours --port'); - const t = toolClient(port); - await t.open(); t.auth('tool'); await wait(150); + const banner = new RegExp(`agent relay on ws://127\\.0\\.0\\.1:${port}/agent`); + await until(() => banner.test(serr)); // dev.js builds the page first: seconds on a busy runner + assert.match(serr, banner, 'dev.js binds loopback and honours --port'); + const t = await tool(port); assert.ok(t.events.some((e) => e.page === false), 'the relay answers on 127.0.0.1'); t.close(); a = load({ url: `http://127.0.0.1:${port}/`, agent: true, storage: { 'sot-agent': JSON.stringify({ on: true }) } }); @@ -403,7 +411,7 @@ test('dev.js binds loopback and parses --port; the page dials its own host', asy assert.equal(a.ev('apiRelayUrl()'), `ws://127.0.0.1:${port}/agent`, 'its own host and port, the relay path'); } finally { stop(a); - child.kill('SIGTERM'); await wait(150); child.kill('SIGKILL'); + await kill(child); } }); @@ -416,11 +424,11 @@ test('calls neither page nor relay can answer resolve as errors, not hangs', asy const a = page(); pages.push(a); a.ev('ACT.start({});S.cash=500000;compute()'); - await wait(400); + await joined(a); // The page dies with a call in flight; it is silenced first, so the call is provably pending when it closes. a.ev('API_RELAY.ws.onmessage=()=>{}'); const p = t.call('act', { action: 'buildPop', args: { city: 'ams' } }); - await wait(100); + assert.ok(await until(() => r.relay.status().calls === 1), 'the call reached the relay'); a.ev('API_RELAY.ws.close(1000)'); assert.match((await p).error || '', /disconnected before answering/i); stop(a); @@ -429,11 +437,11 @@ test('calls neither page nor relay can answer resolve as errors, not hangs', asy const b = page({ seed: 12 }); pages.push(b); b.ev('ACT.start({});S.cash=222000;compute()'); - await wait(400); + await joined(b); b.ev('API_RELAY.ws.onmessage=()=>{}'); const s = createRelaySession({ host: '127.0.0.1', port: PORT }); const p2 = s.call('state', { path: 'cash' }); - await wait(150); + assert.ok(await until(() => r.relay.status().calls === 1), 'the call reached the relay'); await r.close(); assert.match((await p2).refused || '', /connection closed/i); await s.close(); @@ -447,13 +455,13 @@ test('a page that hangs on a call answers within --call-ms, not minutes later', const a = page({ seed: 4 }); try { a.ev('ACT.start({});compute()'); - await wait(400); + await joined(a); a.ev('API_RELAY.ws.onmessage=()=>{}'); // connected, alive, and never answering const s = createRelaySession({ host: '127.0.0.1', port: PORT, callMs: 300 }); const t0 = Date.now(); const r = await s.call('state', { path: 'cash' }); assert.match(r.refused || '', /no answer from the game page in 0\.3 s/); - assert.ok(Date.now() - t0 < 2000, 'the configured timeout, not the default'); + assert.ok(Date.now() - t0 < 10000, 'the configured timeout, not the default (30 s)'); assert.equal(createRelaySession({}).config.callMs, 30000, 'the default is 30 s'); await s.close(); } finally { stop(a); } @@ -473,9 +481,8 @@ test('the relay refuses a cross-site Origin and takes a same-origin page or a to const same = toolClient(PORT, { Origin: `http://127.0.0.1:${PORT}` }); await same.open(); same.auth('page'); - await wait(150); const t = await tool(); - await wait(100); + await until(() => same.events.some((e) => e.tools === 1)); assert.ok(same.events.some((e) => e.tools === 1), 'the same-origin page connects and sees the tool'); t.close(); same.close(); })); @@ -486,16 +493,16 @@ function bannerOf(argv) { let serr = ''; return new Promise((res) => { const done = (v) => { clearTimeout(t); child.kill('SIGKILL'); res(v); }; - const t = setTimeout(() => done({ addr: null, serr }), 5000); + const t = setTimeout(() => done({ addr: null, serr }), 30000); // dev.js builds the page first child.stderr.on('data', (d) => { serr += d; const m = /agent relay on ws:\/\/(\S+)\/agent/.exec(serr); if (m) done({ addr: m[1], serr }); }); }); } test('dev.js: a bare --host binds loopback, never every interface; binding wider is the operator\'s choice, and said', async () => { - const p1 = await free(); + const p1 = await freePort(); const dev = await bannerOf([path.join(ROOT, 'scripts', 'dev.js'), '--host', '--port', String(p1)]); assert.equal(dev.addr, `127.0.0.1:${p1}`, `the socket is bound to loopback (${dev.serr})`); - const p2 = await free(); + const p2 = await freePort(); const wide = await bannerOf([path.join(ROOT, 'scripts', 'dev.js'), '--host', '0.0.0.0', '--port', String(p2)]); assert.equal(wide.addr, `0.0.0.0:${p2}`, 'the banner is the real socket, not the flag echoed back'); assert.match(wide.serr, /bound to 0\.0\.0\.0, so anything that can reach this address can drive/); @@ -504,7 +511,7 @@ test('dev.js: a bare --host binds loopback, never every interface; binding wider test('browser mode over HTTP: a client connects to the server, the server to the player\'s page', () => withRelay(async () => { const a = page({ seed: 11, ms: 0 }); a.ev("ACT.start({});S.cash=555000;compute()"); - await wait(400); + await joined(a); const child = spawn(process.execPath, [SERVER, '--browser', '--relay-port', String(PORT), '--http', '127.0.0.1:0'], { stdio: ['ignore', 'ignore', 'pipe'] }); let err = ''; try { @@ -514,11 +521,13 @@ test('browser mode over HTTP: a client connects to the server, the server to the }); const post = (body) => new Promise((res, rej) => { const data = JSON.stringify({ jsonrpc: '2.0', id: 1, ...body }); - const req = http.request({ host: '127.0.0.1', port, path: '/mcp', method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream', 'mcp-protocol-version': '2025-06-18', 'content-length': Buffer.byteLength(data) } }, (r) => { + // A fresh connection each time (agent: false): a kept-alive one can be closed by the server's idle timer just + // as the next request goes out on it, which a slow runner hits now and then (ECONNRESET). + const req = http.request({ agent: false, host: '127.0.0.1', port, path: '/mcp', method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream', 'mcp-protocol-version': '2025-06-18', 'content-length': Buffer.byteLength(data) } }, (r) => { let t = ''; r.setEncoding('utf8'); r.on('data', (d) => { t += d; }); r.on('end', () => { const line = t.split('\n').find((l) => l.startsWith('data: ')); res(JSON.parse(line ? line.slice(6) : t)); }); }); - req.on('error', rej); req.end(data); + req.on('error', (e) => rej(new Error(`${e.message}; the server said: ${err}`))); req.end(data); }); const init = await post({ method: 'initialize', params: { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'test', version: '1' } } }); assert.match(init.result.instructions, /player's own browser tab/); @@ -526,5 +535,5 @@ test('browser mode over HTTP: a client connects to the server, the server to the assert.equal(JSON.parse(r.result.content[0].text), 555000, 'the player\'s game, through HTTP and the relay'); const no = await post({ method: 'tools/call', params: { name: 'new_game', arguments: {} } }); assert.ok(no.error || (no.result && no.result.isError), 'no new_game in the player\'s tab'); - } finally { stop(a); child.kill('SIGTERM'); await wait(150); child.kill('SIGKILL'); } + } finally { stop(a); await kill(child); } })); diff --git a/tests/gpu-bench-server.test.js b/tests/gpu-bench-server.test.js index 7767842..eaaa6ff 100644 --- a/tests/gpu-bench-server.test.js +++ b/tests/gpu-bench-server.test.js @@ -8,6 +8,8 @@ const path = require('node:path'); const http = require('node:http'); const { spawn } = require('node:child_process'); +const { until, freePort, kill } = require('./net'); + const ROOT = path.join(__dirname, '..'); function req(port, { method = 'GET', p = '/', host, headers = {}, body } = {}) { @@ -20,11 +22,11 @@ function req(port, { method = 'GET', p = '/', host, headers = {}, body } = {}) { } test('the benchmark server answers only its own page', async (t) => { - const port = 20000 + Math.floor(Math.random() * 20000), out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'bench-')), 'out.md'); + const port = await freePort(), out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'bench-')), 'out.md'); const srv = spawn(process.execPath, [path.join(ROOT, 'scripts', 'gpu-bench.js'), '--port', String(port), '--host', '127.0.0.1', '--out', out], { stdio: ['ignore', 'pipe', 'pipe'] }); let log = ''; srv.stdout.on('data', (c) => { log += c; }); srv.stderr.on('data', (c) => { log += c; }); - t.after(() => srv.kill()); - for (let i = 0; i < 100 && !/Ctrl\+C/.test(log); i++) await new Promise((r) => setTimeout(r, 100)); + t.after(() => kill(srv)); + await until(() => /Ctrl\+C|Error/.test(log) || srv.exitCode !== null); assert.match(log, /Ctrl\+C/, 'the server started'); // DNS rebinding: a request for another host name gets nothing. diff --git a/tests/mcp-parity.test.js b/tests/mcp-parity.test.js index e47d135..7154b07 100644 --- a/tests/mcp-parity.test.js +++ b/tests/mcp-parity.test.js @@ -12,8 +12,9 @@ const { createRelay } = require('../scripts/agent/relay'); const { createSession, TOOLS } = require('../scripts/lib/mcp-game'); const { createBrowserSession } = require('../scripts/lib/mcp-browser'); -const PORT = 9399 + (process.pid % 400); -const wait = (ms) => new Promise((r) => setTimeout(r, ms)); +const { freePort } = require('./net'); +let PORT; +test.before(async () => { PORT = await freePort(); }); const HEADLESS_JS = fs.readFileSync(path.join(__dirname, '..', 'scripts', 'lib', 'mcp-headless.js'), 'utf8'); const SCRIPT = [ @@ -66,8 +67,8 @@ test('the same scripted session gives the same outcomes headless and through the a.ev(HEADLESS_JS); a.ev("__mcpHeadless.newGame({city:'fra',name:'Scratch Networks',preset:'oncall'})"); a.ev(`window.__AGENT_RELAY_TEST__=true;API_RELAY_HOST='127.0.0.1:${PORT}';apiAgentSync()`); - await wait(400); - assert.deepEqual(await s.ready({ waitMs: 5000 }), { ok: true }, 'the tab serves this checkout\'s contract'); + // ready() waits for the page itself, generously: jsdom on a busy runner can take seconds to dial. + assert.deepEqual(await s.ready({ waitMs: 20000 }), { ok: true }, 'the tab serves this checkout\'s contract'); const there = await play(s); for (let i = 0; i < SCRIPT.length; i++) assert.deepEqual(there[i], here[i], `step ${i}: ${SCRIPT[i][0]} ${JSON.stringify(SCRIPT[i][1] || {})}`); assert.ok(here.some(([n, v]) => n === 'act' && v.outcome === 'decision'), 'the script meets a question'); diff --git a/tests/net.js b/tests/net.js new file mode 100644 index 0000000..5aca9a9 --- /dev/null +++ b/tests/net.js @@ -0,0 +1,33 @@ +// Helpers for tests that run servers and sockets. On a busy CI runner (four parts at once, each file in its own +// process) anything can take seconds, so a test waits for a state, never for a fixed time: a fixed sleep is either +// too short (a red build that means nothing) or longer than needed on every run. +const http = require('node:http'); + +const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Resolves true as soon as ok() holds, false after ms. Assert on the result, so a timeout fails with your message. +async function until(ok, ms = 20000) { + const t0 = Date.now(); + while (!(await ok())) { if (Date.now() - t0 > ms) return false; await wait(10); } + return true; +} + +// A port nobody is using on 127.0.0.1, from the OS. A fixed or random one can be taken by another suite, or by +// the OS itself for an outgoing connection (the ephemeral range starts at 32768 on Linux). +const freePort = () => new Promise((res, rej) => { + const s = http.createServer(); + s.on('error', rej); + s.listen(0, '127.0.0.1', () => { const p = s.address().port; s.close(() => res(p)); }); +}); + +// Stops a child process: asks first, then insists, and resolves once it has really gone (so its port is free). +function kill(child, ms = 2000) { + return new Promise((res) => { + if (!child || child.exitCode !== null || child.signalCode !== null) return res(); + const t = setTimeout(() => child.kill('SIGKILL'), ms); + child.once('exit', () => { clearTimeout(t); res(); }); + child.kill('SIGTERM'); + }); +} + +module.exports = { wait, until, freePort, kill }; diff --git a/tests/noc-window.test.js b/tests/noc-window.test.js index 3534bcd..0262b42 100644 --- a/tests/noc-window.test.js +++ b/tests/noc-window.test.js @@ -1,9 +1,11 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const { load, browser } = require('./harness'); +const { until } = require('./net'); // A game window and its NOC window in one fake browser: shared localStorage, broadcast channels and locks. -const settle = async (ev, cond, what) => { for (let i = 0; i < 400 && !ev(cond); i++) await new Promise((r) => setTimeout(r, 5)); assert.ok(ev(cond), what || cond); }; +// Waits for the page's async work (storage, channels) to land: up to 20 s, as a busy runner may need. +const settle = async (ev, cond, what) => assert.ok(await until(() => ev(cond)), what || cond); const wait = (ms = 30) => new Promise((r) => setTimeout(r, ms)); const START = "UI.pitched=true;ACT.pickStart({city:'fra'});ACT.start({});closeModal();S.cash=1e8;for(const g of GOALS)S.done[g.id]=0;S.lvl=3;S.nextFail=S.nextEvent=1e9;UI.speed=0;S.rules=makeRules('freeze');" + "ACT.launchBB({city:'fra'});S.pops.fra.bb.d=300;S.pops.fra.rgen=1;ACT.buyTransit({city:'fra',net:'cog',tier:1});ACT.buyIx({city:'fra',tier:0,signed:true});S.pni.push({id:'p1',city:'fra',net:'drz',tier:0,ready:0});compute();save(true)"; diff --git a/tests/save-size.test.js b/tests/save-size.test.js index 1607493..1d048ba 100644 --- a/tests/save-size.test.js +++ b/tests/save-size.test.js @@ -1,6 +1,7 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const { load } = require('./harness'); +const { until } = require('./net'); // Make the browser's storage refuse the save: 'QuotaExceededError' (too big) or 'SecurityError' (not allowed). const refuse = name => `window.__err=${JSON.stringify(name)};{const P=Object.getPrototypeOf(localStorage),set=P.setItem; @@ -73,7 +74,8 @@ test('#176 an older, unpacked save still loads the same', () => { }); // Compressing runs in the background: wait for it, without depending on how long it takes. -const settle = async (ev, cond) => { for (let i = 0; i < 400 && !ev(cond); i++) await new Promise(r => setTimeout(r, 5)); assert.ok(ev(cond), cond); }; +// Waits for the page's async work (storage, channels) to land: up to 20 s, as a busy runner may need. +const settle = async (ev, cond, what) => assert.ok(await until(() => ev(cond)), what || cond); const huge = "for(let i=0;i<12000;i++){const slots=1+i%4;S.biz.push({id:'h'+i,city:'fra',city2:'ams',name:'Huge Carrier '+i,type:'wwave',bw:1,rate:4000+i,mood:100,since:i,link:'df1',slots,what:wwWhat(S.links.find(l=>l.id==='df1'),slots)});}compute()"; test('#176 small saves stay plain JSON, written at once', () => { diff --git a/tests/save-slots.test.js b/tests/save-slots.test.js index 74fbdcf..c656e38 100644 --- a/tests/save-slots.test.js +++ b/tests/save-slots.test.js @@ -1,10 +1,12 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const { load, browser, flush } = require('./harness'); +const { until } = require('./net'); // Two windows of one browser share localStorage, its storage events, a BroadcastChannel and Web Locks; a reload is a // new window with the same tab (sessionStorage). -const settle = async (ev, cond, what) => { for (let i = 0; i < 400 && !ev(cond); i++) await new Promise((r) => setTimeout(r, 5)); assert.ok(ev(cond), what || cond); }; +// Waits for the page's async work (storage, channels) to land: up to 20 s, as a busy runner may need. +const settle = async (ev, cond, what) => assert.ok(await until(() => ev(cond)), what || cond); const play = (ev, name, city = 'fra', days = 5) => ev(`UI.pitched=true;UI.nm=${JSON.stringify(name)};if($('nmIn'))$('nmIn').value=UI.nm;ACT.pickStart({city:'${city}'});ACT.start({});ACT.launchBB({city:'${city}'});for(let i=0;i<${days};i++)dayTick();save(true)`); const idx = (ev) => JSON.parse(ev("JSON.stringify(slotIdx())")); const card = (ev) => ev("document.getElementById('modalCard').textContent");