From e5deb90968638e9a4761147b3eefe0e7600e2f68 Mon Sep 17 00:00:00 2001 From: Frederic Jung Date: Mon, 26 Oct 2026 08:40:20 +0100 Subject: [PATCH] Routing reference test runs on today's routing loop; perf bench reads exported saves The reference test from #106 now patches the loop as it looks after #96 (business-first passes) and also covers prefer, prepend and protect customers. It catches an exit index that ignores RouteScript preferences, which the shortcut test in speed.test.js cannot see. npm run perf -- --save accepts files from the in-game Export save, so scripts/benchmark-performance.js and its README section are not needed. The performance skill documents --save, both equivalence tests and how to handle noisy runs. --- .claude/skills/performance/SKILL.md | 14 +++++++++ README.md | 16 ---------- scripts/benchmark-performance.js | 43 --------------------------- scripts/perf-bench.js | 5 +++- tests/helpers/reference-allocation.js | 4 +-- tests/routing-reference.test.js | 3 +- 6 files changed, 22 insertions(+), 63 deletions(-) delete mode 100644 scripts/benchmark-performance.js diff --git a/.claude/skills/performance/SKILL.md b/.claude/skills/performance/SKILL.md index d5dacd5..2fb10a2 100644 --- a/.claude/skills/performance/SKILL.md +++ b/.claude/skills/performance/SKILL.md @@ -33,6 +33,18 @@ and says whether both versions route the save's traffic identically. Without `--base` it just times this checkout. `--runs N`, `--md FILE`, `--json FILE` and `--summary FILE` are there for CI and for digging. +`--save FILE` swaps the fixture for another save, such as a laggy save from a bug report. Both a plain +save and a file from the in-game menu's Export save work: + +```bash +npm run perf -- --save ~/Downloads/series-of-tubes-save.json --base /tmp/base.html +``` + +Routing speed-ups must also keep the two equivalence tests green: `tests/speed.test.js` (the fast +path against the same code with its shortcuts switched off) and `tests/routing-reference.test.js` +(the indexed exit choice against a plain scan of every exit, across preferences, RouteScript, +incidents and failures). + ## Demonstrate - **Every pull request**: the Performance workflow runs the same comparison and posts the table as a @@ -46,6 +58,8 @@ Without `--base` it just times this checkout. `--runs N`, `--md FILE`, `--json F ## Read the table - Below about 2 ms, differences are noise. The verdicts already ignore them. +- A busy machine can swing a single row by ±20% on identical code. When a verdict looks surprising + for what the change touches, rerun with `--runs 15` before reporting it, and report both runs. - ⚠️ slower (15% or more) or 🔴 slower (50% or more) needs a reason in the PR, or a fix. A new feature may cost time; a refactor or a fix should not. - "Routing results differ" is expected when a change is meant to change gameplay (new cities, diff --git a/README.md b/README.md index 8e0b322..712a37e 100644 --- a/README.md +++ b/README.md @@ -160,19 +160,3 @@ Issues and pull requests are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md). ## License [MIT](LICENSE) © 2026 Frederic Jung - -## Benchmark an exported save - -To measure simulation updates, actions, the closed mobile Networks sheet and -opening Connections with your own save: - -```bash -node scripts/benchmark-performance.js /path/to/save.json -git show origin/main:site/index.html > /tmp/tubes-baseline.html -node scripts/benchmark-performance.js /path/to/save.json /tmp/tubes-baseline.html -``` - -The optional baseline compares timings and asserts that the complete serialized -simulation output matches. Saves may be plain state or exports wrapped in `S`. -These are headless JavaScript/DOM timings, not phone interaction or browser -painting measurements. For the seeded network benchmark used by CI, use `npm run perf`. diff --git a/scripts/benchmark-performance.js b/scripts/benchmark-performance.js deleted file mode 100644 index 5b0f805..0000000 --- a/scripts/benchmark-performance.js +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env node -// Headless CPU/DOM benchmark, not a browser frame-rate measurement. -// node scripts/benchmark-performance.js SAVE.json [BASELINE.html] -// git show main:site/index.html > /tmp/tubes-baseline.html -const fs = require('node:fs'); -const path = require('node:path'); -const assert = require('node:assert/strict'); -const { load } = require('../tests/harness'); - -const [saveFile, baseline] = process.argv.slice(2); -if (!saveFile) { - console.error('Usage: node scripts/benchmark-performance.js SAVE.json [BASELINE.html]'); - process.exit(1); -} -const input = JSON.parse(fs.readFileSync(saveFile, 'utf8')); -const save = input.S || input; -function benchmark(file) { - const { ev, dom, errors } = load({ file }); - try { - ev('ACT.start({})'); - dom.window.benchmarkSave = JSON.parse(JSON.stringify(save)); - ev('S=migrate(window.benchmarkSave);UI.speed=0;innerWidth=390;innerHeight=844;compute()'); - const result = JSON.parse(ev('JSON.stringify(R)')); - const measure = (setup, run, repeats) => { - ev(setup); - const samples = JSON.parse(ev(`(()=>{const out=[];for(let i=0;i<${repeats};i++){const start=performance.now();${run};out.push(performance.now()-start);}return JSON.stringify(out);})()`)); - const sorted = [...samples].sort((a, b) => a - b); - return { medianMs: +sorted[Math.floor(sorted.length / 2)].toFixed(1), samplesMs: samples.map(v => +v.toFixed(1)) }; - }; - const timings = { - compute: measure('', 'compute()', 5), - closedNetworks: measure("UI.left='nets';UI.leftOpen=false;UI.sel=null", 'renderLeft()', 3), - openConnections: measure("UI._dh=null;UI.dtab='conn'", 'ACT.connOpen()', 1), - action: measure("$('dash').hidden=true;UI.leftOpen=false;UI.left='nets'", 'after()', 3), - }; - assert.deepEqual(errors, []); - return { result, timings }; - } finally { dom.window.close(); } -} -const original = baseline ? benchmark(path.resolve(baseline)) : null; -const current = benchmark(path.join(__dirname, '..', 'site', 'index.html')); -if (original) assert.deepEqual(current.result, original.result, 'Simulation output must match the baseline'); -console.log(JSON.stringify({ baseline: original?.timings, current: current.timings, ...(original && { identicalSimulation: true }) }, null, 2)); diff --git a/scripts/perf-bench.js b/scripts/perf-bench.js index fdb6ab2..ff30811 100644 --- a/scripts/perf-bench.js +++ b/scripts/perf-bench.js @@ -4,6 +4,7 @@ // node scripts/perf-bench.js time this checkout // node scripts/perf-bench.js --base OLD.html compare OLD.html (e.g. main) with this checkout // node scripts/perf-bench.js --base OLD.html --md out.md --summary "$GITHUB_STEP_SUMMARY" +// node scripts/perf-bench.js --save my-export.json --base OLD.html a player's save (plain or exported from the menu) // // Both versions load the same save in one process and are timed in alternating turns, so a busy machine slows // them equally. Each number is the median of several runs. It also says whether both versions route the save's @@ -19,7 +20,9 @@ const ROOT = path.join(__dirname, '..'); const HEAD = opt('--head', path.join(ROOT, 'site', 'index.html')); const BASE = opt('--base', null); const RUNS = +opt('--runs', 7); -const SAVE = fs.readFileSync(opt('--save', path.join(ROOT, 'tests', 'fixtures', 'big-network.json')), 'utf8'); +// A plain save, or a file from the in-game "Export save", which wraps it as {app, version, exported, S}. +const RAW = JSON.parse(fs.readFileSync(opt('--save', path.join(ROOT, 'tests', 'fixtures', 'big-network.json')), 'utf8')); +const SAVE = JSON.stringify(RAW && RAW.app === 'series-of-tubes' && RAW.S ? RAW.S : RAW); // What we time. Each body runs inside the game page; panels flip between two selections so nothing can be // skipped as "unchanged". diff --git a/tests/helpers/reference-allocation.js b/tests/helpers/reference-allocation.js index db0508a..a5c557c 100644 --- a/tests/helpers/reference-allocation.js +++ b/tests/helpers/reference-allocation.js @@ -1,9 +1,9 @@ // Deliberately scan all exits for every flow, without byRank, local-city // shortcuts or first-group selection. Stable sorting preserves E order on ties. module.exports = ` - for(const rank of RANKS){ + for(const pass of PASSES||[null])for(const rank of RANKS){ for(f of flows){ - if(f.rem<=1e-6||(rank===0&&!f.cache))continue; + if(f.rem<=1e-6||(rank===0&&!f.cache)||(pass&&!pass(f)))continue; Dv=SDi[f.ci];Pv=SPi[f.ci];PP=walked[f.ci]||(walked[f.ci]=[]);vis++; const candidates=E.filter(e=>e.used= 0 && end > start, 'allocation loop boundaries must exist'); ev(`window.referenceCompute=eval('('+${JSON.stringify(source.slice(0,start)+referenceAllocation+source.slice(end))}+')')`); @@ -35,6 +35,7 @@ test('indexed routing matches the simple allocation loop across network changes' "S.pni.forEach((p,i)=>p.pref=i%3===0?'high':i%3===1?'low':'normal')", "S.te={on:true,script:''}", "S.te.script='prefer pni for goggle\\nspill at 80%\\navoid transit for netchill'", + "S.te.script='prefer transit Cogentle\\nprepend transit Cogentle 2\\nprotect customers'", "S.ev.rpki={nets:['ggl','nfx'],until:S.day+3};S.ev.depeer=S.day+3", "S.ev.ddos={city:'fra',until:S.day+3,gbps:300,vec:'syn',entry:{k:'edge'}}", "S.links[0].eqDown=S.day+2;S.links[1].cut=S.day+2;S.links[2].loss={until:S.day+2,f:0.5}",