From 72f31e809b676ef300957c90a067248013b98979 Mon Sep 17 00:00:00 2001 From: jvanvik Date: Mon, 9 Nov 2026 20:35:18 +0100 Subject: [PATCH] Server-run engine: every call answers, and a crashing load gets its own worker A load whose networks did not all check out could hang its session for good, a worker could spin on a job it could never run, a session whose load crashed its worker never got a worker of its own, and an unload followed at once by a load (a language change) lost the session's place or never answered. Fixed: - Worker: a load call is checked whole before it touches anything. One that names no network, or one twice, is refused (ERR) and every realm stays as it was. A network without a save now answers err (broken) like one whose save does not load, and the others load. A network marked loading with no load under way answers NO_REALM instead of waiting for ever. - Worker: it goes on only while something ran. If a job were ever still waiting when nothing could run, it is answered (ERR, logged) rather than left pending, and the worker no longer busy-loops on it. - Engine: a session is on its worker from the moment its load is sent. A worker exit counts against the sessions whose load was under way (else against every session it held), so a save that crashes its worker gets a worker of its own after two exits and is given up (GAVE_UP) after a third, while the sessions that only shared the worker lose their realms but not their place in the pool. Loads under way count for placement. - Engine: an unload no longer drops the placement of a session whose next load is under way, so an unload and a load sent back to back stay on one worker; a worker of its own is retired only when no call is pending on it (a pending call there used to never settle). - Engine: a message that cannot be sent rejects (ERR) and leaves nothing pending. - Realm: a what-if puts UI.hotMap back too (teSlop's trial left its own). - Realm: a network in a language this server has no dictionary for plays in English, as the page does, instead of loading broken. Tests (tests/mp-sim-engine.test.js): a malformed load changes nothing and every call of the session answers within 3 s, with the workers idle after; an unload and a load back to back keep the session on its worker; a session whose load takes its worker down twice gets a worker of its own, then GAVE_UP, while the bystander is never blamed; an ask leaves UI.hotMap as it was; a network in an unknown language loads in English. Each fails on the engine before this change (the worker's alone: the hang; the engine's alone: the placement and the exits; the realm's alone: teSlop's hotMap and the language). The two drafts no test played are played now (tests/mp-sim-acts.test.js): a premium border crossing into Hong Kong (buildLink with cnPrem) and the AI's own script deployed as it wrote it while it drops most of the traffic (teDeploy with slop, on a small network with one capped transit port), each compared byte for byte with the page's clicks, and both read in the DRAFTS check (nothing is left unplayed). Runs, each file alone under an 8G cap: mp-sim-engine 16/16 (11.6 s), mp-sim-contract 8/8, mp-sim-acts 5 + 1 todo (10.9 s), mp-sim-determinism 6/6 (53.4 s). --- scripts/lib/mp-sim-engine.js | 67 ++++++++++++++++------- scripts/lib/mp-sim-realm.js | 10 ++-- scripts/lib/mp-sim-worker.js | 38 ++++++++++--- tests/durations.json | 2 +- tests/mp-sim-acts.test.js | 29 ++++++++-- tests/mp-sim-engine.test.js | 103 ++++++++++++++++++++++++++++++++--- 6 files changed, 204 insertions(+), 45 deletions(-) diff --git a/scripts/lib/mp-sim-engine.js b/scripts/lib/mp-sim-engine.js index 6a19f3c..06dd79c 100644 --- a/scripts/lib/mp-sim-engine.js +++ b/scripts/lib/mp-sim-engine.js @@ -8,8 +8,9 @@ // {t:'loaded', nid, err}], one per network, in order. env: the checkpoint envelope {s, uidN, rt, last, // day} (s: the save text; rt: what the realm needs besides it to play on exactly as before, opaque // here: route memory, R's routing point, key order, the sales desk's memory); begin: {name, home, -// preset, asn, mp, tut} builds a new network, checkpointed at once (saved). A network whose save -// does not load (or that cannot begin) answers err: broken (4.7). +// preset, asn, mp, tut} builds a new network, checkpointed at once (saved). A network that comes +// without a save, whose save does not load, or that cannot begin answers err: broken (4.7), and +// the others load. A load of a network that is loaded replaces its realm (a failed one leaves none). // engine.day(code, {d, speed, last, alone:[nid], nets:[nid], mute}) → {t:'dayDone', code, d, nets:{[nid]: {day, lost, // pause?, err?, retried?, ms}}}. Each network in sorted order; one that failed twice answers err and // its day as it was, and is broken from then on (no more days; actions answer 'stopped'). @@ -24,15 +25,21 @@ // engine.stats() → {workers, realms, sessions} (not a promise); engine.close() → stops the workers // engine.simId the build id this engine's game belongs to (SIM_ID of the multiplayer page from the same src/) // -// Rejections: a call for a network that is not loaded (code 'NO_REALM'), a malformed call, and every call pending on -// a worker that exits (out of memory, a crash): Error('worker exited'), code 'EXIT'; that worker's sessions are gone and -// the host loads them again (4.7). A session whose worker exited twice within an hour gets a worker of its own; when that -// one exits too, its loads reject with code 'GAVE_UP' until the engine is made again (the server restarts). +// Rejections: a call for a network that is not loaded (code 'NO_REALM'), a malformed call (code 'ERR': a load that +// names no network, or one twice, changes nothing), and every call pending on a worker that exits (out of memory, a +// crash): Error('worker exited'), code 'EXIT'; that worker's sessions are gone and the host loads them again (4.7). Every +// call is answered or rejected; none waits for ever. An exit counts against the sessions whose load was under way on +// that worker (a save that crashes or overflows it), or, when none was, against every session it held. A session with +// two exits against it within an hour gets a worker of its own; when that one exits too, its loads reject with code +// 'GAVE_UP' until the engine is made again (the server restarts). A session stays on its worker while a load of it is +// under way, so an unload and a load sent back to back (a language change) land on the same worker. // Epochs: every realm load mints one (a load, a begin, the fresh realm after a timeout); loaded, pulled, actDone and // subbed carry the realm's current one. Versions are the realm's: only a pull (or an action's own pull) makes one. // // Tests only, with createEngine({test: {seed, now, step} | true}): engine.eval(code, nid, src) runs code in a realm -// ({value} | {error} | {timeout, epoch}); engine.mem() reports each worker's heap; engine.kill(i) ends worker i. +// ({value} | {error} | {timeout, epoch}); engine.mem() reports each worker's heap; engine.kill(i) ends pool worker i, +// engine.killOf(code) the worker of that session; engine.workerOf(code) is its pool index (-1: none, or its own), +// engine.dedicated(code) whether it has a worker of its own. 'use strict'; const path = require('node:path'); const vm = require('node:vm'); @@ -59,11 +66,13 @@ function createEngine(opts = {}) { const W = { root: opts.root, page: opts.page, test, strict: opts.strict !== undefined ? !!opts.strict : process.env.SIM_STRICT === '1', timeouts: opts.timeouts }; let seq = 0, closed = false; const pool = [], own = new Map(), placed = new Map(), exits = new Map(), gaveUp = new Set(); - // A session's realms, counted where they live (placement by realm count). - const count = (w) => [...w.sessions.values()].reduce((a, b) => a + b.size, 0); + // A worker's realms (sessions: code → its loaded nids), and for placement the networks of its loads under way too + // (loading: code → how many). + const size = (w) => [...w.sessions.values()].reduce((a, b) => a + b.size, 0); + const count = (w) => size(w) + [...w.loading.values()].reduce((a, b) => a + b, 0); function spawn(dedicated = false) { - const w = { worker: null, pending: new Map(), sessions: new Map(), dedicated, dead: false }; + const w = { worker: null, pending: new Map(), sessions: new Map(), loading: new Map(), dedicated, dead: false }; w.worker = new Worker(FILE, { workerData: { mpSim: W }, resourceLimits: { maxOldGenerationSizeMb: heapMb } }); w.worker.on('message', (m) => { if (m.log) { log(m.log[0], m.log[1]); return; } @@ -85,18 +94,22 @@ function createEngine(opts = {}) { function died(w, c) { if (w.dead) return; w.dead = true; + // Whose exit it is: the sessions whose load was under way (a save that crashes or overflows its worker), else every + // session it held. The others lose their realms (the host loads them again) without an exit against them. + const held = new Set([...w.sessions.keys(), ...w.loading.keys()]), blamed = new Set(w.loading.size ? w.loading.keys() : held); const e = Object.assign(new Error('worker exited'), { code: 'EXIT', exitCode: c }); for (const p of w.pending.values()) p.no(e); w.pending.clear(); const t = Date.now(); - for (const code of w.sessions.keys()) { - placed.delete(code); + for (const code of held) { + if (placed.get(code) === w) placed.delete(code); + if (!blamed.has(code)) continue; if (w.dedicated) { gaveUp.add(code); own.delete(code); log('error', `mp-sim: session ${code} took down its own worker too; not loading it again until a restart`); continue; } exits.set(code, (exits.get(code) || []).filter((x) => t - x < HOUR).concat(t)); } - w.sessions.clear(); + w.sessions.clear(); w.loading.clear(); if (!closed) { - if (!closed) log('warn', `mp-sim: a worker exited (code ${c}); its sessions load again from their checkpoints`); + log('warn', `mp-sim: a worker exited (code ${c}); its sessions load again from their checkpoints`); const i = pool.indexOf(w); if (i >= 0) pool[i] = spawn(); } @@ -119,7 +132,12 @@ function createEngine(opts = {}) { if (closed) return Promise.reject(Object.assign(new Error('engine closed'), { code: 'CLOSED' })); if (!w || w.dead) return Promise.reject(Object.assign(new Error('worker exited'), { code: 'EXIT' })); const id = ++seq; - return new Promise((ok, no) => { if (!w.pending.size) w.worker.ref(); w.pending.set(id, { ok, no }); w.worker.postMessage({ id, ...m }); }); + return new Promise((ok, no) => { + if (!w.pending.size) w.worker.ref(); + w.pending.set(id, { ok, no }); + // A message that cannot be sent (not cloneable) is a malformed call, and nothing waits for its answer. + try { w.worker.postMessage({ id, ...m }); } catch (e) { w.pending.delete(id); if (!w.pending.size) w.worker.unref(); no(Object.assign(new Error(String((e && e.message) || e)), { code: 'ERR' })); } + }); } const noRealm = (code, nid) => Promise.reject(Object.assign(new Error(`no realm ${code}/${nid}`), { code: 'NO_REALM' })); const to = (code, nid, m) => { const w = workerOf(code, false); return w ? send(w, { code, nid, ...m }) : noRealm(code, nid); }; @@ -128,14 +146,18 @@ function createEngine(opts = {}) { const engine = { simId: null, load(code, nets) { - if (!Array.isArray(nets) || !nets.length) return Promise.reject(new Error('load: give the networks to load')); + if (!Array.isArray(nets) || !nets.length) return Promise.reject(Object.assign(new Error('load: give the networks to load'), { code: 'ERR' })); let w; try { w = workerOf(code, true); } catch (e) { return Promise.reject(e); } + // Under way: the session is on this worker from now (an exit meanwhile counts against it, and its placement holds). + const n = nets.length, done = () => { const k = (w.loading.get(code) || 0) - n; if (k > 0) w.loading.set(code, k); else w.loading.delete(code); }; + w.loading.set(code, (w.loading.get(code) || 0) + n); return send(w, { t: 'load', code, nets }).then((r) => { + done(); const s = mine(w, code); for (const l of r) if (!l.err) s.add(l.nid); else s.delete(l.nid); return r; - }); + }, (e) => { done(); throw e; }); }, day(code, { d, speed = 1, last = 1, alone = [], nets = [], mute = false } = {}) { const w = workerOf(code, false); @@ -155,14 +177,17 @@ function createEngine(opts = {}) { if (!w) return Promise.resolve({ t: 'unloaded', code, ...(nid !== undefined ? { nid } : {}) }); return send(w, { t: 'unload', code, ...(nid !== undefined ? { nid } : {}) }).then((r) => { const s = w.sessions.get(code); - if (s) { if (nid === undefined) s.clear(); else s.delete(nid); if (!s.size) { w.sessions.delete(code); placed.delete(code); } } - if (w.dedicated && !w.sessions.size) { own.delete(code); w.dead = true; w.worker.terminate(); } + if (s) { if (nid === undefined) s.clear(); else s.delete(nid); } + // Gone from here, unless a load of it sent after this unload is under way. + if ((!s || !s.size) && !w.loading.has(code)) { w.sessions.delete(code); if (placed.get(code) === w) placed.delete(code); } + // A worker of its own with nothing on it and nothing pending is done (with a call pending it would never answer). + if (w.dedicated && !w.dead && !w.sessions.size && !w.loading.size && !w.pending.size) { if (own.get(code) === w) own.delete(code); w.dead = true; w.worker.terminate(); } return r; }); }, stats() { const ws = [...pool, ...own.values()].filter((w) => !w.dead); - return { workers: ws.length, realms: ws.reduce((a, w) => a + count(w), 0), sessions: ws.reduce((a, w) => a + w.sessions.size, 0) }; + return { workers: ws.length, realms: ws.reduce((a, w) => a + size(w), 0), sessions: ws.reduce((a, w) => a + w.sessions.size, 0) }; }, close() { closed = true; @@ -175,7 +200,9 @@ function createEngine(opts = {}) { engine.eval = (code, nid, src, ms) => to(code, nid, { t: 'eval', src: String(src), ...(ms ? { ms } : {}) }); engine.mem = () => Promise.all(pool.filter((w) => !w.dead).map((w) => send(w, { t: 'mem' }))); engine.kill = (i = 0) => { const w = pool[i]; if (w && !w.dead) w.worker.terminate(); }; + engine.killOf = (code) => { const w = placed.get(code); if (w && !w.dead) w.worker.terminate(); }; engine.workerOf = (code) => pool.indexOf(placed.get(code)); + engine.dedicated = (code) => !!(placed.get(code) && placed.get(code).dedicated); } { const page = require('./page'); engine.simId = page.simId(page.assemble({ src: path.join(opts.root || page.ROOT, 'src'), mp: true }).html); } return engine; diff --git a/scripts/lib/mp-sim-realm.js b/scripts/lib/mp-sim-realm.js index 1e081f0..cb1a8d1 100644 --- a/scripts/lib/mp-sim-realm.js +++ b/scripts/lib/mp-sim-realm.js @@ -365,12 +365,12 @@ function srvAct(inp){const m=JSON.parse(inp),act=String(m.act),ent={rid:String(m /* ---- asks (5.9): what-ifs that open a window and put everything back ---- */ function srvAsk(inp){const m=JSON.parse(inp),k=SIMR.mark,act=String(m.act),out={}; if(!SIM_ASK.has(act)||!srvOwn(ACT,act))return JSON.stringify({err:'not a what-if the server answers'}); - const args=m.args&&typeof m.args==='object'&&!Array.isArray(m.args)?m.args:{},td=UI.teDraft,sl=UI.slop,fx0=SIMR.fx.slice(),fl=SIMR.floats; + const args=m.args&&typeof m.args==='object'&&!Array.isArray(m.args)?m.args:{},td=UI.teDraft,sl=UI.slop,fx0=SIMR.fx.slice(),fl=SIMR.floats,hot=UI.hotMap; srvView(m.ctx);$('modal').hidden=!(m.ctx&&m.ctx.modal);SIMR.modal=null; try{ACT[act](args);}catch(e){out.err='the game threw: '+String(e&&e.message||e).slice(0,200);srvLog('ask '+act+' threw: '+String(e&&e.stack||e));} if(SIMR.modal)out.modal=SIMR.modal; if(act==='teCheck'||act==='teSlop'){out.ui={teDraft:typeof UI.teDraft==='string'?UI.teDraft:null};if(act==='teSlop')out.ui.slop=typeof UI.slop==='string'?UI.slop:null;} - R=k.R;uidN=k.uidN;S.rng=k.rng; + R=k.R;UI.hotMap=hot;uidN=k.uidN;S.rng=k.rng; if(JSON.stringify(S)!==k.s){srvLog('ask '+act+' changed the game: put back');srvRestore('{}');} else Object.assign(UI,k.ui); SIMR.mark=null;UI.teDraft=td;UI.slop=sl;SIMR.fx=fx0;SIMR.floats=fl;SIMR.modal=null;$('modal').hidden=true; @@ -456,9 +456,11 @@ class Realm { let n = 0, w0 = 0; // The realm's console, rate-limited: 20 lines, then one a second. ctx.__hostLog = (lvl, txt) => { const t = Date.now(); if (n > 20 && t - w0 < 1000) return; if (t - w0 >= 1000) { w0 = t; n = Math.min(n, 20); } n++; log(lvl, String(txt).slice(0, 2000)); }; - this.lang = LANG_RE.test(lang) && lang !== 'en' && dictScript(lang, code.root) ? lang : 'en'; + // A language this server has no dictionary for plays in English (the page falls back the same way), not broken. + let d = null; + if (LANG_RE.test(lang) && lang !== 'en') { try { d = dictScript(lang, code.root); } catch (e) { log('warn', `no dictionary for ${lang}: English`); } } + this.lang = d ? lang : 'en'; new vm.Script(prelude({ lang: this.lang, test: !!test, ...(test || {}) }), { filename: 'srv-prelude.js' }).runInContext(ctx, { timeout: loadMs }); - const d = this.lang === 'en' ? null : dictScript(this.lang, code.root); if (d) d.runInContext(ctx, { timeout: loadMs }); code.game.runInContext(ctx, { timeout: loadMs }); code.epilogue.runInContext(ctx, { timeout: loadMs }); diff --git a/scripts/lib/mp-sim-worker.js b/scripts/lib/mp-sim-worker.js index e01860b..de5cfd4 100644 --- a/scripts/lib/mp-sim-worker.js +++ b/scripts/lib/mp-sim-worker.js @@ -4,7 +4,9 @@ // Order. Messages run one at a time, in arrival order per network. A session's day is one task per network, in sorted // network order, with a turn for other messages between two networks: an action for network X lands between days of // other networks, never inside X's own day. The networks of a day are fixed when it starts. Loads, new networks and -// unloads of a session wait for its day; actions for a network that is still loading wait for it. +// unloads of a session wait for its day; actions for a network that is still loading wait for it. Every message is +// answered: a load that names no network (or one twice) is refused whole and changes nothing; a network in it that +// has no save, or whose save does not load, answers err (broken) and the others load. // // Failures. Every realm call has a time limit (a day 2 s, an action 1 s, a what-if 3 s, a pull or a save 1 s). Before // a day or an action the realm is marked (its S as text, uidN, the dice, and for a day its rt: route memory and routing @@ -90,6 +92,7 @@ function createCore(opts = {}) { function loadOne(c, n) { const e = realms.get(key(c, n.nid)); try { + if (!n.begin && !(n.env && typeof n.env === 'object' && typeof n.env.s === 'string')) throw new Error('no save to load'); e.realm = build(n.nid, n.lang); e.lang = e.realm.lang; const epoch = epochOf(); @@ -204,8 +207,10 @@ function createCore(opts = {}) { return true; } for (let k = 0; k < i; k++) if (jobs[k].code === j.code && SESSION_OPS.has(jobs[k].t) && jobs[k].t !== 'day') return false; + // A network that is loading waits for its load; one marked loading with no load under way (none can be: a guard) + // waits for nothing, so start() answers NO_REALM. const e = realms.get(key(j.code, j.nid)); - return !e || !e.loading; + return !e || !e.loading || !(s && s.load); } function reply(j, r) { post({ id: j.id, ok: true, reply: r === undefined ? null : r }); } function fail(j, x) { post({ id: j.id, ok: false, error: { message: String((x && x.message) || x), code: (x && x.code) || 'ERR' } }); } @@ -222,11 +227,17 @@ function createCore(opts = {}) { function start(j) { if (j.t === 'mem') { const h = v8.getHeapStatistics(), mu = process.memoryUsage(); return reply(j, { heapUsed: h.used_heap_size, heapTotal: h.total_heap_size, external: h.external_memory, rss: mu.rss, realms: realms.size }); } if (j.t === 'load') { - const s = session(j.code); - for (const n of j.nets) { - if (!n || typeof n.nid !== 'string' || (!n.begin && !(n.env && typeof n.env.s === 'string'))) throw new Error('load: each network needs a nid and an env with its save text, or begin'); - realms.set(key(j.code, n.nid), { code: j.code, nid: n.nid, lang: n.lang || 'en', prefs: n.prefs || {}, loading: true, realm: null, epoch: '', v: 0, day: 0, lost: false, broken: false, mark: null, dayMark: null }); + // The whole call first, with no side effects: a malformed one is refused and leaves every realm as it was. A + // network without a save to load (or a begin) is only that network's problem: loadOne answers it broken. + const nets = Array.isArray(j.nets) ? j.nets : [], seen = new Set(); + if (!nets.length) throw new Error('load: give the networks to load'); + for (const n of nets) { + if (!n || typeof n !== 'object' || typeof n.nid !== 'string' || !n.nid) throw new Error('load: each network needs a nid'); + if (seen.has(n.nid)) throw new Error(`load: ${n.nid.slice(0, 40)} is there twice`); + seen.add(n.nid); } + const s = session(j.code); + for (const n of nets) realms.set(key(j.code, n.nid), { code: j.code, nid: n.nid, lang: n.lang || 'en', prefs: n.prefs || {}, loading: true, realm: null, epoch: '', v: 0, day: 0, lost: false, broken: false, mark: null, dayMark: null }); s.load = { j, i: 0, out: [] }; return; } @@ -242,7 +253,8 @@ function createCore(opts = {}) { return reply(j, { t: 'unloaded', code: j.code, ...(j.nid !== undefined ? { nid: j.nid } : {}) }); } const e = realms.get(key(j.code, j.nid)); - if (!e) throw Object.assign(new Error(`no realm ${key(j.code, j.nid)}`), { code: 'NO_REALM' }); + if (e && e.loading) { realms.delete(key(j.code, j.nid)); log('error', `[${j.nid}] marked loading with no load under way: dropped`); } + if (!e || e.loading) throw Object.assign(new Error(`no realm ${key(j.code, j.nid)}`), { code: 'NO_REALM' }); const h = H[j.t]; if (!h) throw new Error(`unknown message ${j.t}`); return reply(j, h(j, e)); @@ -263,12 +275,20 @@ function createCore(opts = {}) { if (D.i >= D.order.length) { s.day = null; reply(D.j, { t: 'dayDone', code: D.j.code, d: D.j.d, nets: D.out }); } return true; } + /* Turns: a message and a step of a session's load or day, in alternation. The worker goes on only while something ran: + what a job waits for (a load, a day, an earlier job of its session) is itself a job or a step that runs, so a pass + where nothing could run means nothing will until a message comes (handle() kicks). If a job were still waiting + then, nothing could ever free it: it is answered, not left pending, and the worker does not spin on it. */ let flip = false; function pump() { pumping = false; flip = !flip; - if (flip) { if (!runJob()) runStep(); } else if (!runStep()) runJob(); - if (jobs.length || [...sessions.values()].some((s) => s.load || s.day)) kick(); + if (flip ? runJob() || runStep() : runStep() || runJob()) return kick(); + if (!jobs.length) return; + const j = jobs.shift(); + log('error', `mp-sim worker: ${j.t} for ${key(j.code, j.nid)} waited for nothing that could come; answered`); + fail(j, new Error(`${j.t} could not run`)); + kick(); } function kick() { if (!pumping) { pumping = true; setImmediate(pump); } } return { diff --git a/tests/durations.json b/tests/durations.json index d729fb0..ad5e894 100644 --- a/tests/durations.json +++ b/tests/durations.json @@ -35,7 +35,7 @@ "tests/mp-sim-acts.test.js": 14.0, "tests/mp-sim-contract.test.js": 7.0, "tests/mp-sim-determinism.test.js": 40.0, - "tests/mp-sim-engine.test.js": 9.0, + "tests/mp-sim-engine.test.js": 11.4, "tests/networks-list.test.js": 6.4, "tests/new-game-speed.test.js": 10.1, "tests/optic-counts.test.js": 9.7, diff --git a/tests/mp-sim-acts.test.js b/tests/mp-sim-acts.test.js index de16bda..4f6d48f 100644 --- a/tests/mp-sim-acts.test.js +++ b/tests/mp-sim-acts.test.js @@ -47,6 +47,17 @@ function pieCo() { P.ev("ACT.buildPop({city:'sin'});ACT.pieFound();ACT.pieFactory();ACT.piePad({city:'sin'});S.pie.stock=400;closeModal();S.rng=99;uidN=1000;S=migrate(JSON.parse(JSON.stringify(S)));compute();"); return P; } +// A small network with one transit port, Auto-TE and a script slot: a cap on that transit drops most of its traffic +// (teDeploy's other path, when the AI's script is the one deployed). +function oneUp() { + const P = page(); + P.ev("ACT.start({});if(UI.pendGame)ACT.pitchSkip();S.cash=1e9;S.lvl=6;for(const g of GOALS)S.done[g.id]=0;ACT.launchBB({city:'fra'});S.pops.fra.bb.d=3000"); + P.ev("ACT.buyTransit({city:'fra',net:NETS.filter(n=>sellsAt(n,'fra')&&n.price)[0].id,tier:1});ACT.coreUp();ACT.coreUp();S.te={on:true,day:S.day,script:''};closeModal();S.rng=99;uidN=1000;S=migrate(JSON.parse(JSON.stringify(S)));compute();"); + P.cap = `cap transit ${P.ev('NETS.find(n=>n.id===S.transit[0].net).name')} at 5 gbps`; + return P; +} +// A Hong Kong and a Shanghai POP and a mainland license: a link between them crosses the border, and may cross it premium. +const CROSS = "S.cnLic={state:'ok'};for(const c of ['hkg','sha'])if(!S.pops[c])S.pops[c]={day:0,bb:null,boost:0,down:0,routers:1,gen:false};S=migrate(JSON.parse(JSON.stringify(S)));compute();"; const loadInto = async (code, nid, P) => checked((await E.load(code, [{ nid, env: { s: P.ev('JSON.stringify(S)'), uidN: +P.ev('uidN'), rt: null }, lang: 'en', prefs: {} }]))[0]); test('two-click buttons: the first click arms SRV_ARM\'s key (or waits for ok), and confirmed:true on the server is both clicks', async (t) => { @@ -112,9 +123,8 @@ test('repayLoan: armed by its id, repaid by the second click (a network small en /* The UI values an act reads, recorded while it runs (its redraw, after(), and windows left out: they read the page's own state to draw it). Every page-held value it reads must be one of its DRAFTS; each of its DRAFTS is read on one of - the paths played here, except the few named with the reason they are not played. */ + the paths played here. */ const PAGE_HELD = [...new Set([...Object.keys(A.DRAFT_TYPES), 'lmode', 'dfMode', 'drag', 'snap', 'hover', 'vp', 'nf', 'dtab'])]; -const UNPLAYED = { cnPrem: 'buildLink reads it only on a link into Hong Kong across the border', slop: 'teDeploy reads it only when the script cost a tenth of the traffic' }; test('DRAFTS: the page-held UI values each act reads, all of them', (t) => { const P = rich(); t.after(() => P.close()); @@ -128,11 +138,13 @@ test('DRAFTS: the page-held UI values each act reads, all of them', (t) => { ['buildLink', `openLinkModal('fra','${near}')`, { tier: 1 }], ['buildLink', `openLinkModal('${sib[0]}','${sib[1]}');UI.route='same';`, { tier: 1 }], ['buildLink', `openLinkModal('lon','nyc');UI.via='land';`, { tier: 1 }], + ['buildLink', CROSS + "openLinkModal('hkg','sha');ACT.cnPrem({v:1});", { tier: 1 }], ['orderDf', "UI.pending={a:'fra',b:'muc'};UI.dfSrc='lease';UI.dfSys='lite';", {}], ['orderDf', "UI.pending={a:'fra',b:'muc'};UI.dfSrc='build';UI.dfSys='lite';", {}], ['orderCable', "UI.pending={a:'lon',b:'nyc'};UI.seaDesign='express';UI.seaterminal=true;UI.seaengineering=false;", {}], ['decomGo', "UI.decom=new Set(['bet']);", {}], ['teDeploy', "S.te={on:true,day:S.day,script:''};UI.slop=undefined;", { script: 'prefer cog at fra' }], + (() => { const Q = oneUp(); return ['teDeploy', `UI.slop=${JSON.stringify(Q.cap)};`, { script: Q.cap }, Q]; })(), ['pieLaunch', 'UI.pieIns=true;UI.pieLas=false;', { rk: 'falcon', cr: 'short' }, pieCo()], ]; const reads = {}; @@ -151,7 +163,7 @@ test('DRAFTS: the page-held UI values each act reads, all of them', (t) => { } for (const act of Object.keys(A.DRAFTS)) { assert.ok(reads[act], `${act} was played`); - for (const k of A.DRAFTS[act]) if (!UNPLAYED[k]) assert.ok(reads[act].has(k), `${act} reads ${k}`); + for (const k of A.DRAFTS[act]) assert.ok(reads[act].has(k), `${act} reads ${k}`); } assert.deepEqual(P.errors, []); }); @@ -160,7 +172,7 @@ test('drafts and texts: the server\'s run of each draft and text act leaves the const P = rich(); t.after(() => P.close()); const { ev } = P; - ev("S.te={on:true,day:S.day,script:''};closeModal();UI.sel=null;compute();"); + ev("S.te={on:true,day:S.day,script:''};closeModal();UI.sel=null;compute();" + CROSS); const code = 'DRAFT5', nid = 'drft0001'; await loadInto(code, nid, P); const near = ev("Object.keys(CITY).filter(c=>!S.pops[c]&&homeOk(c)).sort((a,b)=>kmBetween('ams',a)-kmBetween('ams',b))[0]"); @@ -168,6 +180,7 @@ test('drafts and texts: the server\'s run of each draft and text act leaves the // [act, args, what the page did before the click (its drafts), the drafts sent] const plays = [ ['buildLink', { tier: 1 }, `openLinkModal('fra','${near}')`, () => pick(ev, ['pending', 'route', 'via', 'cnPrem'])], + ['buildLink', { tier: 1 }, "openLinkModal('hkg','sha');ACT.cnPrem({v:1});", () => pick(ev, ['pending', 'route', 'via', 'cnPrem'])], ['orderDf', {}, "UI.pending={a:'fra',b:'muc'};UI.dfSrc='lease';UI.dfSys='lite';", () => pick(ev, ['pending', 'dfSrc', 'dfSys'])], ['orderCable', {}, "UI.pending={a:'lon',b:'nyc'};UI.seaDesign='express';UI.seaterminal=true;UI.seaengineering=false;", () => pick(ev, ['pending', 'seaDesign', 'seaterminal', 'seaengineering'])], ['decomGo', {}, "UI.decom=new Set(['bet']);", () => ({ decom: JSON.parse(ev('JSON.stringify([...UI.decom])')) })], @@ -177,6 +190,7 @@ test('drafts and texts: the server\'s run of each draft and text act leaves the ['renameGo', { name: 'Tubular Bells' }, '', null], ]; await play(code, nid, ev, plays); + assert.equal(ev("S.links.filter(l=>l.premium).length"), 1, 'the crossing into Hong Kong went premium'); // A what-if with its text: the window, and the network as it was. const K = checked(await E.ask(code, nid, msg('teCheck', { script }))); assert.match(K.modal.h, /term/); @@ -191,6 +205,13 @@ test('drafts and texts: the server\'s run of each draft and text act leaves the ['pieLaunch', { rk: 'falcon', cr: 'short' }, 'UI.pieIns=true;UI.pieLas=false;', () => pick(Q.ev, ['pieIns', 'pieLas'])], ]); assert.deepEqual(Q.errors, []); + // The AI's script, deployed as it wrote it, drops most of the traffic: its own apology, not the fat-finger one. + const T = oneUp(); + t.after(() => T.close()); + await loadInto('SL0PPY', 'slop0001', T); + await play('SL0PPY', 'slop0001', T.ev, [['teDeploy', { script: T.cap }, `UI.slop=${JSON.stringify(T.cap)};`, () => ({ slop: true })]]); + assert.equal(T.ev("S.log.slice(0,3).some(x=>/AI apologises/.test(x.t))"), true, 'the slop path'); + assert.deepEqual(T.errors, []); }); async function play(code, nid, ev, plays) { for (const [act, args, before, ui] of plays) { diff --git a/tests/mp-sim-engine.test.js b/tests/mp-sim-engine.test.js index 49c853e..270d2f9 100644 --- a/tests/mp-sim-engine.test.js +++ b/tests/mp-sim-engine.test.js @@ -3,7 +3,8 @@ // contract's shapes (tests/lib/mp-sim-shape.js); the capture table turns each kind of effect into an fx; failures go back // to the mark (a throw in place, a timeout into a fresh realm, an exiting worker rejects everything pending); asks put // everything back; actions report only the UI they changed; a session's day holds back loads; unwatched networks stay -// bounded; and realms do not share their intrinsics. +// bounded; realms do not share their intrinsics; every call is answered, whatever a load holds; and a session whose load +// takes its worker down gets a worker of its own, then is given up, while the others keep their places. const test = require('node:test'); const assert = require('node:assert/strict'); const { createEngine, snapWith } = require('../scripts/lib/mp-sim-engine'); @@ -18,6 +19,9 @@ const on = (X) => ({ }); const { day, ev } = on(E); const updOf = (r) => JSON.parse(r.upd); +const wait = (ms) => new Promise((ok) => setTimeout(ok, ms)); +// A call's answer within ms, or a failure that says it never came (a hung call would hang the test instead). +const within = (p, ms = 3000) => { let t; return Promise.race([p, new Promise((ok, no) => { t = setTimeout(() => no(new Error(`no answer within ${ms} ms`)), ms); })]).finally(() => clearTimeout(t)); }; const HOST = { clock: fixture('clock'), roster: fixture('roster'), since: { days: 0, again: 0 } }; test('a new network: begin, a snapshot, its welcome, actions, days, a what-if, views, prefs, a save and a card', async () => { @@ -166,8 +170,8 @@ test('an ask leaves S (the same object), R, the dice and uidN as they were, also checked((await E.load(code, [{ nid, env: { s: bigScenic(), uidN: 1000, rt: null }, lang: 'en', prefs: {} }]))[0]); await ev(code, nid, "const a=ddosAttack(true);a.until=S.day+5;S.te={on:true,day:S.day,script:''};compute();return 1;"); checked(await E.pull(code, nid, { snap: true })); checked(await E.pull(code, nid, {})); - const before = "window.__S0=S;window.__R0=R;window.__rng=S.rng;window.__u=uidN;window.__j=JSON.stringify(S);return 1;"; - const same = 'return [S===__S0,R===__R0,S.rng===__rng,uidN===__u,JSON.stringify(S)===__j,R.ddos.some(m=>m.attack)&&R.ddos.every(m=>!m.attack||m.attack===ddosFind(m.attack.id))]'; + const before = "window.__S0=S;window.__R0=R;window.__rng=S.rng;window.__u=uidN;window.__j=JSON.stringify(S);window.__hot=UI.hotMap;return 1;"; + const same = 'return [S===__S0,R===__R0,S.rng===__rng,uidN===__u,JSON.stringify(S)===__j,R.ddos.some(m=>m.attack)&&R.ddos.every(m=>!m.attack||m.attack===ddosFind(m.attack.id)),UI.hotMap===__hot]'; const lid = await ev(code, nid, 'return S.links.find(l=>!linkDown(l)).id'); const script = await ev(code, nid, 'return TE_EXAMPLE'); for (const [act, args] of [['teSlop', {}], ['teCheck', { script }], ['linkCheck', { id: lid }], ['teSlop', {}]]) { @@ -176,7 +180,7 @@ test('an ask leaves S (the same object), R, the dice and uidN as they were, also assert.ok(r.modal && r.modal.h.length > 20, `${act} answers with its window`); if (act !== 'linkCheck') assert.equal(typeof r.ui.teDraft, 'string', `${act}: the draft`); if (act === 'teSlop') assert.equal(r.ui.slop, r.ui.teDraft, 'the AI\'s script is the draft'); - assert.deepEqual(await ev(code, nid, same), [true, true, true, true, true, true], act); + assert.deepEqual(await ev(code, nid, same), [true, true, true, true, true, true, true], act); } assert.equal(checked(await E.pull(code, nid, {})).upd, undefined, 'and nothing to send'); await E.unload(code); @@ -268,11 +272,12 @@ test('alone, an auto-pause stops the realm\'s clock as the page\'s does: a secon }); test('a network whose save does not load is broken; the others of the session load', async () => { - const code = 'BR0K3N', ok = 'okok0001', bad = 'badd0002'; - const L = await E.load(code, [{ nid: ok, env: { s: bigScenic(), uidN: 1000, rt: null }, lang: 'de', prefs: {} }, { nid: bad, env: { s: '{"v":1,"pops":', uidN: 1 }, lang: 'en', prefs: {} }]); + const code = 'BR0K3N', ok = 'okok0001', bad = 'badd0002', zz = 'zzzz0003'; + const L = await E.load(code, [{ nid: ok, env: { s: bigScenic(), uidN: 1000, rt: null }, lang: 'de', prefs: {} }, { nid: bad, env: { s: '{"v":1,"pops":', uidN: 1 }, lang: 'en', prefs: {} }, { ...begin(zz), lang: 'zz' }]); L.forEach(checked); - assert.deepEqual([L[0].err, typeof L[1].err], [undefined, 'string']); + assert.deepEqual([L[0].err, typeof L[1].err, L[2].err], [undefined, 'string', undefined]); assert.deepEqual(Object.keys(L[1]).sort(), ['err', 'nid', 't']); + assert.equal(await ev(code, zz, 'return LANG'), 'en', 'a language without a dictionary here plays in English'); // The network's language is the realm's: the log speaks German. checked(await E.act(code, ok, msg('buildPop', { city: 'dus' }))); assert.match(await ev(code, ok, 'return LANG+":"+S.log[0].t'), /^de:/); @@ -280,6 +285,90 @@ test('a network whose save does not load is broken; the others of the session lo await E.unload(code); }); +test('a load call with a malformed network changes nothing, one without a save is broken, and every call still answers', async () => { + const code = 'MA1F0R', a = 'mala0001', b = 'malb0002'; + const [A] = await E.load(code, [begin(a)]); + checked(A); + const env = { s: A.saved.txt, uidN: A.saved.uidN, rt: A.saved.rt }; + // A network without a nid, or one twice: the call is refused whole, and a keeps the realm it had. + await assert.rejects(within(E.load(code, [{ nid: a, env, lang: 'en', prefs: {} }, { env, lang: 'en', prefs: {} }])), (e) => e.code === 'ERR'); + assert.equal(checked(await within(E.pull(code, a, {}))).epoch, A.epoch, 'a as it was'); + await assert.rejects(within(E.load(code, [begin(b), begin(b)])), (e) => e.code === 'ERR' && /twice/.test(e.message)); + assert.equal(checked(await within(E.act(code, a, msg('launchBB', { city: 'ams' })))).epoch, A.epoch); + // a again with b, whose checkpoint the host could not read: b is broken, a loads (a new epoch). + const L = await within(E.load(code, [{ nid: a, env, lang: 'en', prefs: {} }, { nid: b, env: null, lang: 'en', prefs: {} }])); + L.forEach(checked); + assert.deepEqual([L[0].err, L[1].err], [undefined, 'no save to load']); + assert.notEqual(L[0].epoch, A.epoch); + // Every call of the session answers at once: a plays, b has no realm, unloads and loads go through. + assert.equal(checked(await within(E.act(code, a, msg('launchBB', { city: 'fra' })))).outcome, 'applied'); + assert.equal(checked(await within(E.day(code, { d: 1, nets: [a, b] }))).nets[a].day, 1); + await assert.rejects(within(E.pull(code, b, {})), (e) => e.code === 'NO_REALM'); + checked(await within(E.unload(code, a))); + checked((await within(E.load(code, [begin(a)])))[0]); + checked(await within(E.unload(code))); + // Nothing is left waiting, so the workers are idle (a job that can never run once kept one at 100% of a core). + await wait(100); + const c0 = process.cpuUsage(); + await wait(1000); + const c = process.cpuUsage(c0), ms = (c.user + c.system) / 1000; + assert.ok(ms < 400, `${ms.toFixed(0)} ms of CPU over 1 s of quiet`); +}); + +test('an unload and a load sent back to back (a language change) keep the session on its worker', async () => { + const code = 'GERM4N', nid = 'lang0001'; + checked((await E.load(code, [begin(nid)]))[0]); + const w = E.workerOf(code); + const u = E.unload(code), l = E.load(code, [{ ...begin(nid), lang: 'de' }]); + checked(await within(u)); + checked((await within(l))[0]); + assert.equal(E.workerOf(code), w); + assert.equal(checked(await within(E.pull(code, nid, {}))).nid, nid, 'the network answers where it is'); + checked((await E.load(code, [begin('lang0002')]))[0]); + assert.equal(E.workerOf(code), w, 'and the session\'s next network goes there too'); + checked(await within(E.pull(code, nid, {}))); + await E.unload(code); +}); + +test('a session whose load takes its worker down: twice, a worker of its own; once more, given up; the others keep their places', async () => { + const W = createEngine({ workers: 2, test: TEST, timeouts: ROOMY }); + try { + const code = 'P01S0N', nid = 'pois0001', by = { BYST01: 'byst0001', BYST02: 'byst0002' }; + for (const [c, n] of Object.entries(by)) checked((await W.load(c, [begin(n)]))[0]); + assert.deepEqual([W.workerOf('BYST01'), W.workerOf('BYST02')], [0, 1]); + // Its load crashes the worker (as a save that overflows the heap would): the worker it shares with BYST01. + const lost = []; + for (let i = 0; i < 2; i++) { + const p = W.load(code, [begin(nid)]); + assert.deepEqual([W.workerOf(code), W.dedicated(code)], [0, false], `attempt ${i + 1}: in the pool`); + W.killOf(code); + await assert.rejects(p, (e) => e.code === 'EXIT'); + for (const [c, n] of Object.entries(by)) { + if (!(await W.pull(c, n, {}).then(() => false, (e) => e.code === 'NO_REALM'))) continue; + lost.push(c); + checked((await W.load(c, [begin(n)]))[0]); + assert.equal(W.dedicated(c), false, `${c} is not blamed: it goes back to the pool`); + } + } + assert.deepEqual(lost, ['BYST01', 'BYST01'], 'the bystander on that worker lost its realm each time, the other never'); + // Twice within the hour: a worker of its own. An unload and a load back to back stay there, and answer. + checked((await W.load(code, [begin(nid)]))[0]); + assert.deepEqual([W.dedicated(code), W.stats().workers], [true, 3]); + const u = W.unload(code), l = W.load(code, [begin(nid)]); + checked(await within(u)); + checked((await within(l))[0]); + checked(await within(W.pull(code, nid, {}))); + assert.equal(W.dedicated(code), true); + // Once more, on its own worker: given up until a restart; its worker is not replaced, and the others play on. + const p = W.load(code, [begin('pois0002')]); + W.killOf(code); + await assert.rejects(p, (e) => e.code === 'EXIT'); + await assert.rejects(W.load(code, [begin(nid)]), (e) => e.code === 'GAVE_UP'); + for (const [c, n] of Object.entries(by)) checked(await within(W.pull(c, n, {}))); + assert.deepEqual([W.stats().workers, W.dedicated('BYST01')], [2, false]); + } finally { await W.close(); } +}); + test('a polluted prototype stays in its own realm', async () => { const code = 'P0WWVT', a = 'poll0001', b = 'poll0002'; (await E.load(code, [begin(a), begin(b)])).forEach(checked);