Routing reference test runs on today's routing loop; perf bench reads exported saves
The reference test from #106 now patches the loop as it looks after #96 (business-first passes) and also covers prefer, prepend and protect customers. It catches an exit index that ignores RouteScript preferences, which the shortcut test in speed.test.js cannot see. npm run perf -- --save accepts files from the in-game Export save, so scripts/benchmark-performance.js and its README section are not needed. The performance skill documents --save, both equivalence tests and how to handle noisy runs.
This commit is contained in:
parent
2366bb9a8f
commit
e5deb90968
6 changed files with 22 additions and 63 deletions
|
|
@ -33,6 +33,18 @@ and says whether both versions route the save's traffic identically.
|
||||||
Without `--base` it just times this checkout. `--runs N`, `--md FILE`, `--json FILE` and
|
Without `--base` it just times this checkout. `--runs N`, `--md FILE`, `--json FILE` and
|
||||||
`--summary FILE` are there for CI and for digging.
|
`--summary FILE` are there for CI and for digging.
|
||||||
|
|
||||||
|
`--save FILE` swaps the fixture for another save, such as a laggy save from a bug report. Both a plain
|
||||||
|
save and a file from the in-game menu's Export save work:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run perf -- --save ~/Downloads/series-of-tubes-save.json --base /tmp/base.html
|
||||||
|
```
|
||||||
|
|
||||||
|
Routing speed-ups must also keep the two equivalence tests green: `tests/speed.test.js` (the fast
|
||||||
|
path against the same code with its shortcuts switched off) and `tests/routing-reference.test.js`
|
||||||
|
(the indexed exit choice against a plain scan of every exit, across preferences, RouteScript,
|
||||||
|
incidents and failures).
|
||||||
|
|
||||||
## Demonstrate
|
## Demonstrate
|
||||||
|
|
||||||
- **Every pull request**: the Performance workflow runs the same comparison and posts the table as a
|
- **Every pull request**: the Performance workflow runs the same comparison and posts the table as a
|
||||||
|
|
@ -46,6 +58,8 @@ Without `--base` it just times this checkout. `--runs N`, `--md FILE`, `--json F
|
||||||
## Read the table
|
## Read the table
|
||||||
|
|
||||||
- Below about 2 ms, differences are noise. The verdicts already ignore them.
|
- Below about 2 ms, differences are noise. The verdicts already ignore them.
|
||||||
|
- A busy machine can swing a single row by ±20% on identical code. When a verdict looks surprising
|
||||||
|
for what the change touches, rerun with `--runs 15` before reporting it, and report both runs.
|
||||||
- ⚠️ slower (15% or more) or 🔴 slower (50% or more) needs a reason in the PR, or a fix. A new
|
- ⚠️ slower (15% or more) or 🔴 slower (50% or more) needs a reason in the PR, or a fix. A new
|
||||||
feature may cost time; a refactor or a fix should not.
|
feature may cost time; a refactor or a fix should not.
|
||||||
- "Routing results differ" is expected when a change is meant to change gameplay (new cities,
|
- "Routing results differ" is expected when a change is meant to change gameplay (new cities,
|
||||||
|
|
|
||||||
16
README.md
16
README.md
|
|
@ -160,19 +160,3 @@ Issues and pull requests are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||||
## License
|
## License
|
||||||
|
|
||||||
[MIT](LICENSE) © 2026 Frederic Jung
|
[MIT](LICENSE) © 2026 Frederic Jung
|
||||||
|
|
||||||
## Benchmark an exported save
|
|
||||||
|
|
||||||
To measure simulation updates, actions, the closed mobile Networks sheet and
|
|
||||||
opening Connections with your own save:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
node scripts/benchmark-performance.js /path/to/save.json
|
|
||||||
git show origin/main:site/index.html > /tmp/tubes-baseline.html
|
|
||||||
node scripts/benchmark-performance.js /path/to/save.json /tmp/tubes-baseline.html
|
|
||||||
```
|
|
||||||
|
|
||||||
The optional baseline compares timings and asserts that the complete serialized
|
|
||||||
simulation output matches. Saves may be plain state or exports wrapped in `S`.
|
|
||||||
These are headless JavaScript/DOM timings, not phone interaction or browser
|
|
||||||
painting measurements. For the seeded network benchmark used by CI, use `npm run perf`.
|
|
||||||
|
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
#!/usr/bin/env node
|
|
||||||
// Headless CPU/DOM benchmark, not a browser frame-rate measurement.
|
|
||||||
// node scripts/benchmark-performance.js SAVE.json [BASELINE.html]
|
|
||||||
// git show main:site/index.html > /tmp/tubes-baseline.html
|
|
||||||
const fs = require('node:fs');
|
|
||||||
const path = require('node:path');
|
|
||||||
const assert = require('node:assert/strict');
|
|
||||||
const { load } = require('../tests/harness');
|
|
||||||
|
|
||||||
const [saveFile, baseline] = process.argv.slice(2);
|
|
||||||
if (!saveFile) {
|
|
||||||
console.error('Usage: node scripts/benchmark-performance.js SAVE.json [BASELINE.html]');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
const input = JSON.parse(fs.readFileSync(saveFile, 'utf8'));
|
|
||||||
const save = input.S || input;
|
|
||||||
function benchmark(file) {
|
|
||||||
const { ev, dom, errors } = load({ file });
|
|
||||||
try {
|
|
||||||
ev('ACT.start({})');
|
|
||||||
dom.window.benchmarkSave = JSON.parse(JSON.stringify(save));
|
|
||||||
ev('S=migrate(window.benchmarkSave);UI.speed=0;innerWidth=390;innerHeight=844;compute()');
|
|
||||||
const result = JSON.parse(ev('JSON.stringify(R)'));
|
|
||||||
const measure = (setup, run, repeats) => {
|
|
||||||
ev(setup);
|
|
||||||
const samples = JSON.parse(ev(`(()=>{const out=[];for(let i=0;i<${repeats};i++){const start=performance.now();${run};out.push(performance.now()-start);}return JSON.stringify(out);})()`));
|
|
||||||
const sorted = [...samples].sort((a, b) => a - b);
|
|
||||||
return { medianMs: +sorted[Math.floor(sorted.length / 2)].toFixed(1), samplesMs: samples.map(v => +v.toFixed(1)) };
|
|
||||||
};
|
|
||||||
const timings = {
|
|
||||||
compute: measure('', 'compute()', 5),
|
|
||||||
closedNetworks: measure("UI.left='nets';UI.leftOpen=false;UI.sel=null", 'renderLeft()', 3),
|
|
||||||
openConnections: measure("UI._dh=null;UI.dtab='conn'", 'ACT.connOpen()', 1),
|
|
||||||
action: measure("$('dash').hidden=true;UI.leftOpen=false;UI.left='nets'", 'after()', 3),
|
|
||||||
};
|
|
||||||
assert.deepEqual(errors, []);
|
|
||||||
return { result, timings };
|
|
||||||
} finally { dom.window.close(); }
|
|
||||||
}
|
|
||||||
const original = baseline ? benchmark(path.resolve(baseline)) : null;
|
|
||||||
const current = benchmark(path.join(__dirname, '..', 'site', 'index.html'));
|
|
||||||
if (original) assert.deepEqual(current.result, original.result, 'Simulation output must match the baseline');
|
|
||||||
console.log(JSON.stringify({ baseline: original?.timings, current: current.timings, ...(original && { identicalSimulation: true }) }, null, 2));
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
// node scripts/perf-bench.js time this checkout
|
// node scripts/perf-bench.js time this checkout
|
||||||
// node scripts/perf-bench.js --base OLD.html compare OLD.html (e.g. main) with this checkout
|
// node scripts/perf-bench.js --base OLD.html compare OLD.html (e.g. main) with this checkout
|
||||||
// node scripts/perf-bench.js --base OLD.html --md out.md --summary "$GITHUB_STEP_SUMMARY"
|
// node scripts/perf-bench.js --base OLD.html --md out.md --summary "$GITHUB_STEP_SUMMARY"
|
||||||
|
// node scripts/perf-bench.js --save my-export.json --base OLD.html a player's save (plain or exported from the menu)
|
||||||
//
|
//
|
||||||
// Both versions load the same save in one process and are timed in alternating turns, so a busy machine slows
|
// Both versions load the same save in one process and are timed in alternating turns, so a busy machine slows
|
||||||
// them equally. Each number is the median of several runs. It also says whether both versions route the save's
|
// them equally. Each number is the median of several runs. It also says whether both versions route the save's
|
||||||
|
|
@ -19,7 +20,9 @@ const ROOT = path.join(__dirname, '..');
|
||||||
const HEAD = opt('--head', path.join(ROOT, 'site', 'index.html'));
|
const HEAD = opt('--head', path.join(ROOT, 'site', 'index.html'));
|
||||||
const BASE = opt('--base', null);
|
const BASE = opt('--base', null);
|
||||||
const RUNS = +opt('--runs', 7);
|
const RUNS = +opt('--runs', 7);
|
||||||
const SAVE = fs.readFileSync(opt('--save', path.join(ROOT, 'tests', 'fixtures', 'big-network.json')), 'utf8');
|
// A plain save, or a file from the in-game "Export save", which wraps it as {app, version, exported, S}.
|
||||||
|
const RAW = JSON.parse(fs.readFileSync(opt('--save', path.join(ROOT, 'tests', 'fixtures', 'big-network.json')), 'utf8'));
|
||||||
|
const SAVE = JSON.stringify(RAW && RAW.app === 'series-of-tubes' && RAW.S ? RAW.S : RAW);
|
||||||
|
|
||||||
// What we time. Each body runs inside the game page; panels flip between two selections so nothing can be
|
// What we time. Each body runs inside the game page; panels flip between two selections so nothing can be
|
||||||
// skipped as "unchanged".
|
// skipped as "unchanged".
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
// Deliberately scan all exits for every flow, without byRank, local-city
|
// Deliberately scan all exits for every flow, without byRank, local-city
|
||||||
// shortcuts or first-group selection. Stable sorting preserves E order on ties.
|
// shortcuts or first-group selection. Stable sorting preserves E order on ties.
|
||||||
module.exports = `
|
module.exports = `
|
||||||
for(const rank of RANKS){
|
for(const pass of PASSES||[null])for(const rank of RANKS){
|
||||||
for(f of flows){
|
for(f of flows){
|
||||||
if(f.rem<=1e-6||(rank===0&&!f.cache))continue;
|
if(f.rem<=1e-6||(rank===0&&!f.cache)||(pass&&!pass(f)))continue;
|
||||||
Dv=SDi[f.ci];Pv=SPi[f.ci];PP=walked[f.ci]||(walked[f.ci]=[]);vis++;
|
Dv=SDi[f.ci];Pv=SPi[f.ci];PP=walked[f.ci]||(walked[f.ci]=[]);vis++;
|
||||||
const candidates=E.filter(e=>e.used<e.cap&&e.ok(f.n)&&
|
const candidates=E.filter(e=>e.used<e.cap&&e.ok(f.n)&&
|
||||||
(TR?eff(e,f.n):base(e))===rank&&Dv[CIDX[e.city]]<Infinity)
|
(TR?eff(e,f.n):base(e))===rank&&Dv[CIDX[e.city]]<Infinity)
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ function network(ev) {
|
||||||
// Share the model and path sender, but replace indexed exit selection with
|
// Share the model and path sender, but replace indexed exit selection with
|
||||||
// the original full scan and stable sort for every flow.
|
// the original full scan and stable sort for every flow.
|
||||||
const source = ev('compute.toString()');
|
const source = ev('compute.toString()');
|
||||||
const start = source.indexOf(' for(const rank of RANKS){');
|
const start = source.indexOf(' for(const pass of PASSES||[null])for(const rank of RANKS){');
|
||||||
const end = source.indexOf(' for(const i of NORD)', start);
|
const end = source.indexOf(' for(const i of NORD)', start);
|
||||||
assert.ok(start >= 0 && end > start, 'allocation loop boundaries must exist');
|
assert.ok(start >= 0 && end > start, 'allocation loop boundaries must exist');
|
||||||
ev(`window.referenceCompute=eval('('+${JSON.stringify(source.slice(0,start)+referenceAllocation+source.slice(end))}+')')`);
|
ev(`window.referenceCompute=eval('('+${JSON.stringify(source.slice(0,start)+referenceAllocation+source.slice(end))}+')')`);
|
||||||
|
|
@ -35,6 +35,7 @@ test('indexed routing matches the simple allocation loop across network changes'
|
||||||
"S.pni.forEach((p,i)=>p.pref=i%3===0?'high':i%3===1?'low':'normal')",
|
"S.pni.forEach((p,i)=>p.pref=i%3===0?'high':i%3===1?'low':'normal')",
|
||||||
"S.te={on:true,script:''}",
|
"S.te={on:true,script:''}",
|
||||||
"S.te.script='prefer pni for goggle\\nspill at 80%\\navoid transit for netchill'",
|
"S.te.script='prefer pni for goggle\\nspill at 80%\\navoid transit for netchill'",
|
||||||
|
"S.te.script='prefer transit Cogentle\\nprepend transit Cogentle 2\\nprotect customers'",
|
||||||
"S.ev.rpki={nets:['ggl','nfx'],until:S.day+3};S.ev.depeer=S.day+3",
|
"S.ev.rpki={nets:['ggl','nfx'],until:S.day+3};S.ev.depeer=S.day+3",
|
||||||
"S.ev.ddos={city:'fra',until:S.day+3,gbps:300,vec:'syn',entry:{k:'edge'}}",
|
"S.ev.ddos={city:'fra',until:S.day+3,gbps:300,vec:'syn',entry:{k:'edge'}}",
|
||||||
"S.links[0].eqDown=S.day+2;S.links[1].cut=S.day+2;S.links[2].loss={until:S.day+2,f:0.5}",
|
"S.links[0].eqDown=S.day+2;S.links[1].cut=S.day+2;S.links[2].loss={until:S.day+2,f:0.5}",
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue