seriesoftubes/scripts/lib/mp-sim-engine.js
jvanvik 72f31e809b Server-run engine: every call answers, and a crashing load gets its own worker
A load whose networks did not all check out could hang its session for good, a
worker could spin on a job it could never run, a session whose load crashed its
worker never got a worker of its own, and an unload followed at once by a load
(a language change) lost the session's place or never answered. Fixed:

- Worker: a load call is checked whole before it touches anything. One that
  names no network, or one twice, is refused (ERR) and every realm stays as it
  was. A network without a save now answers err (broken) like one whose save
  does not load, and the others load. A network marked loading with no load
  under way answers NO_REALM instead of waiting for ever.
- Worker: it goes on only while something ran. If a job were ever still
  waiting when nothing could run, it is answered (ERR, logged) rather than left
  pending, and the worker no longer busy-loops on it.
- Engine: a session is on its worker from the moment its load is sent. A worker
  exit counts against the sessions whose load was under way (else against every
  session it held), so a save that crashes its worker gets a worker of its own
  after two exits and is given up (GAVE_UP) after a third, while the sessions
  that only shared the worker lose their realms but not their place in the pool.
  Loads under way count for placement.
- Engine: an unload no longer drops the placement of a session whose next load
  is under way, so an unload and a load sent back to back stay on one worker; a
  worker of its own is retired only when no call is pending on it (a pending
  call there used to never settle).
- Engine: a message that cannot be sent rejects (ERR) and leaves nothing pending.
- Realm: a what-if puts UI.hotMap back too (teSlop's trial left its own).
- Realm: a network in a language this server has no dictionary for plays in
  English, as the page does, instead of loading broken.

Tests (tests/mp-sim-engine.test.js): a malformed load changes nothing and every
call of the session answers within 3 s, with the workers idle after; an unload
and a load back to back keep the session on its worker; a session whose load
takes its worker down twice gets a worker of its own, then GAVE_UP, while the
bystander is never blamed; an ask leaves UI.hotMap as it was; a network in an
unknown language loads in English. Each fails on the engine before this change
(the worker's alone: the hang; the engine's alone: the placement and the exits;
the realm's alone: teSlop's hotMap and the language).

The two drafts no test played are played now (tests/mp-sim-acts.test.js): a
premium border crossing into Hong Kong (buildLink with cnPrem) and the AI's own
script deployed as it wrote it while it drops most of the traffic (teDeploy with
slop, on a small network with one capped transit port), each compared byte for
byte with the page's clicks, and both read in the DRAFTS check (nothing is left
unplayed).

Runs, each file alone under an 8G cap: mp-sim-engine 16/16 (11.6 s),
mp-sim-contract 8/8, mp-sim-acts 5 + 1 todo (10.9 s), mp-sim-determinism 6/6
(53.4 s).
2026-11-09 20:47:31 +01:00

211 lines
14 KiB
JavaScript

// The server-run engine (the server-sim design, sections 4.2-4.3): a small pool of worker threads (mp-sim-worker.js),
// each holding the vm realms (mp-sim-realm.js) of the sessions placed on it, behind a promise API. The session host
// (scripts/lib/mp-sim.js) keeps the clock, presence and votes, and asks this for days, actions, what-ifs and pulls.
// tests/lib/mp-sim-fake-engine.js has the same API with canned answers.
//
// const engine = createEngine({workers: 2, heapMb: 1536, root, log});
// engine.load(code, [{nid, env?, begin?, lang, prefs, rng?}]) → [{t:'loaded', nid, epoch, day, lost, saved?} |
// {t:'loaded', nid, err}], one per network, in order. env: the checkpoint envelope {s, uidN, rt, last,
// day} (s: the save text; rt: what the realm needs besides it to play on exactly as before, opaque
// here: route memory, R's routing point, key order, the sales desk's memory); begin: {name, home,
// preset, asn, mp, tut} builds a new network, checkpointed at once (saved). A network that comes
// without a save, whose save does not load, or that cannot begin answers err: broken (4.7), and
// the others load. A load of a network that is loaded replaces its realm (a failed one leaves none).
// engine.day(code, {d, speed, last, alone:[nid], nets:[nid], mute}) → {t:'dayDone', code, d, nets:{[nid]: {day, lost,
// pause?, err?, retried?, ms}}}. Each network in sorted order; one that failed twice answers err and
// its day as it was, and is broken from then on (no more days; actions answer 'stopped').
// engine.act(code, nid, msg) → {t:'actDone', nid, epoch, rid, outcome, why?, v, upd} (upd: the update with the result,
// version v; null when stopped) | {t:'err', code, nid, rid, what:'timeout'|'throw', msg}
// engine.ask(code, nid, msg) → {t:'askDone', nid, rid, modal?, ui?, err?} | {t:'err', …}
// engine.pull(code, nid, {snap, views}) → {t:'pulled', nid, epoch, v, upd?, snap?, subs?}. snap is the realm's half
// of a snapshot: snapWith(snap, {clock, roster, since}) makes the stream event.
// engine.sub(code, nid, {views}) → {t:'subbed', nid, epoch, v, subs}; engine.greet(code, nid) → a pulled
// engine.prefs(code, nid, {prefs}) → undefined; engine.save(code, nid) → {t:'saved', nid, txt, uidN, rt, day, lost}
// engine.card(code, nid) → {t:'card', nid, card}; engine.unload(code, nid?) → {t:'unloaded', code, nid?}
// engine.stats() → {workers, realms, sessions} (not a promise); engine.close() → stops the workers
// engine.simId the build id this engine's game belongs to (SIM_ID of the multiplayer page from the same src/)
//
// Rejections: a call for a network that is not loaded (code 'NO_REALM'), a malformed call (code 'ERR': a load that
// names no network, or one twice, changes nothing), and every call pending on a worker that exits (out of memory, a
// crash): Error('worker exited'), code 'EXIT'; that worker's sessions are gone and the host loads them again (4.7). Every
// call is answered or rejected; none waits for ever. An exit counts against the sessions whose load was under way on
// that worker (a save that crashes or overflows it), or, when none was, against every session it held. A session with
// two exits against it within an hour gets a worker of its own; when that one exits too, its loads reject with code
// 'GAVE_UP' until the engine is made again (the server restarts). A session stays on its worker while a load of it is
// under way, so an unload and a load sent back to back (a language change) land on the same worker.
// Epochs: every realm load mints one (a load, a begin, the fresh realm after a timeout); loaded, pulled, actDone and
// subbed carry the realm's current one. Versions are the realm's: only a pull (or an action's own pull) makes one.
//
// Tests only, with createEngine({test: {seed, now, step} | true}): engine.eval(code, nid, src) runs code in a realm
// ({value} | {error} | {timeout, epoch}); engine.mem() reports each worker's heap; engine.kill(i) ends pool worker i,
// engine.killOf(code) the worker of that session; engine.workerOf(code) is its pool index (-1: none, or its own),
// engine.dedicated(code) whether it has a worker of its own.
'use strict';
const path = require('node:path');
const vm = require('node:vm');
const { Worker } = require('node:worker_threads');
const FILE = path.join(__dirname, 'mp-sim-worker.js');
const HOUR = 3600e3;
/* The stream event of a snapshot: the realm's half (S, r, keep, epoch, v, now) with the host's (clock, roster,
since), joined as text: the realm's 270 KB are never parsed again. */
function snapWith(snap, { clock, roster, since }) {
if (typeof snap !== 'string' || !snap.endsWith('}')) throw new Error('snapWith: not a snapshot');
return snap.slice(0, -1) + ',"clock":' + JSON.stringify(clock) + ',"roster":' + JSON.stringify(roster) + ',"since":' + JSON.stringify(since) + '}';
}
function createEngine(opts = {}) {
if (!vm.constants || vm.constants.DONT_CONTEXTIFY === undefined) {
throw new Error(`server-run play needs Node 22.8 or newer (vm.constants.DONT_CONTEXTIFY); this is ${process.version}`);
}
const N = Math.max(1, Math.floor(+opts.workers || 2));
const heapMb = Math.max(64, Math.floor(+opts.heapMb || 1536));
const log = typeof opts.log === 'function' ? opts.log : () => {};
const test = opts.test || null;
const W = { root: opts.root, page: opts.page, test, strict: opts.strict !== undefined ? !!opts.strict : process.env.SIM_STRICT === '1', timeouts: opts.timeouts };
let seq = 0, closed = false;
const pool = [], own = new Map(), placed = new Map(), exits = new Map(), gaveUp = new Set();
// A worker's realms (sessions: code → its loaded nids), and for placement the networks of its loads under way too
// (loading: code → how many).
const size = (w) => [...w.sessions.values()].reduce((a, b) => a + b.size, 0);
const count = (w) => size(w) + [...w.loading.values()].reduce((a, b) => a + b, 0);
function spawn(dedicated = false) {
const w = { worker: null, pending: new Map(), sessions: new Map(), loading: new Map(), dedicated, dead: false };
w.worker = new Worker(FILE, { workerData: { mpSim: W }, resourceLimits: { maxOldGenerationSizeMb: heapMb } });
w.worker.on('message', (m) => {
if (m.log) { log(m.log[0], m.log[1]); return; }
if (m.fatal) { log('error', 'mp-sim worker could not start: ' + m.fatal); return; }
if (m.ready) return;
const p = w.pending.get(m.id);
if (!p) return;
w.pending.delete(m.id);
// An idle worker does not keep the process alive; one with calls pending does.
if (!w.pending.size) w.worker.unref();
if (m.ok) p.ok(m.reply);
else p.no(Object.assign(new Error(m.error.message), { code: m.error.code }));
});
w.worker.on('error', (e) => log('error', 'mp-sim worker failed: ' + (e && e.message)));
w.worker.on('exit', (c) => died(w, c));
w.worker.unref();
return w;
}
function died(w, c) {
if (w.dead) return;
w.dead = true;
// Whose exit it is: the sessions whose load was under way (a save that crashes or overflows its worker), else every
// session it held. The others lose their realms (the host loads them again) without an exit against them.
const held = new Set([...w.sessions.keys(), ...w.loading.keys()]), blamed = new Set(w.loading.size ? w.loading.keys() : held);
const e = Object.assign(new Error('worker exited'), { code: 'EXIT', exitCode: c });
for (const p of w.pending.values()) p.no(e);
w.pending.clear();
const t = Date.now();
for (const code of held) {
if (placed.get(code) === w) placed.delete(code);
if (!blamed.has(code)) continue;
if (w.dedicated) { gaveUp.add(code); own.delete(code); log('error', `mp-sim: session ${code} took down its own worker too; not loading it again until a restart`); continue; }
exits.set(code, (exits.get(code) || []).filter((x) => t - x < HOUR).concat(t));
}
w.sessions.clear(); w.loading.clear();
if (!closed) {
log('warn', `mp-sim: a worker exited (code ${c}); its sessions load again from their checkpoints`);
const i = pool.indexOf(w);
if (i >= 0) pool[i] = spawn();
}
}
for (let i = 0; i < N; i++) pool.push(spawn());
// The worker of a session: where it is, else the pool's least loaded one (or one of its own, after two exits).
function workerOf(code, create) {
let w = placed.get(code);
if (w && !w.dead) return w;
if (!create) return null;
if (gaveUp.has(code)) throw Object.assign(new Error(`session ${code} kept failing its worker`), { code: 'GAVE_UP' });
if ((exits.get(code) || []).filter((x) => Date.now() - x < HOUR).length >= 2) {
w = spawn(true); own.set(code, w);
} else w = pool.reduce((a, b) => (count(b) < count(a) ? b : a));
placed.set(code, w);
return w;
}
function send(w, m) {
if (closed) return Promise.reject(Object.assign(new Error('engine closed'), { code: 'CLOSED' }));
if (!w || w.dead) return Promise.reject(Object.assign(new Error('worker exited'), { code: 'EXIT' }));
const id = ++seq;
return new Promise((ok, no) => {
if (!w.pending.size) w.worker.ref();
w.pending.set(id, { ok, no });
// A message that cannot be sent (not cloneable) is a malformed call, and nothing waits for its answer.
try { w.worker.postMessage({ id, ...m }); } catch (e) { w.pending.delete(id); if (!w.pending.size) w.worker.unref(); no(Object.assign(new Error(String((e && e.message) || e)), { code: 'ERR' })); }
});
}
const noRealm = (code, nid) => Promise.reject(Object.assign(new Error(`no realm ${code}/${nid}`), { code: 'NO_REALM' }));
const to = (code, nid, m) => { const w = workerOf(code, false); return w ? send(w, { code, nid, ...m }) : noRealm(code, nid); };
const mine = (w, code) => { let s = w.sessions.get(code); if (!s) w.sessions.set(code, s = new Set()); return s; };
const engine = {
simId: null,
load(code, nets) {
if (!Array.isArray(nets) || !nets.length) return Promise.reject(Object.assign(new Error('load: give the networks to load'), { code: 'ERR' }));
let w;
try { w = workerOf(code, true); } catch (e) { return Promise.reject(e); }
// Under way: the session is on this worker from now (an exit meanwhile counts against it, and its placement holds).
const n = nets.length, done = () => { const k = (w.loading.get(code) || 0) - n; if (k > 0) w.loading.set(code, k); else w.loading.delete(code); };
w.loading.set(code, (w.loading.get(code) || 0) + n);
return send(w, { t: 'load', code, nets }).then((r) => {
done();
const s = mine(w, code);
for (const l of r) if (!l.err) s.add(l.nid); else s.delete(l.nid);
return r;
}, (e) => { done(); throw e; });
},
day(code, { d, speed = 1, last = 1, alone = [], nets = [], mute = false } = {}) {
const w = workerOf(code, false);
if (!w) return Promise.reject(Object.assign(new Error(`no session ${code} here`), { code: 'NO_REALM' }));
return send(w, { t: 'day', code, d, speed, last, alone, nets, mute });
},
act(code, nid, msg) { return to(code, nid, { t: 'act', msg }); },
ask(code, nid, msg) { return to(code, nid, { t: 'ask', msg }); },
pull(code, nid, { snap = false, views } = {}) { return to(code, nid, { t: 'pull', snap: !!snap, ...(views ? { views } : {}) }); },
sub(code, nid, { views = [] } = {}) { return to(code, nid, { t: 'sub', views }); },
prefs(code, nid, { prefs } = {}) { return to(code, nid, { t: 'prefs', prefs }).then(() => undefined); },
greet(code, nid) { return to(code, nid, { t: 'greet' }); },
save(code, nid) { return to(code, nid, { t: 'save' }); },
card(code, nid) { return to(code, nid, { t: 'card' }); },
unload(code, nid) {
const w = workerOf(code, false);
if (!w) return Promise.resolve({ t: 'unloaded', code, ...(nid !== undefined ? { nid } : {}) });
return send(w, { t: 'unload', code, ...(nid !== undefined ? { nid } : {}) }).then((r) => {
const s = w.sessions.get(code);
if (s) { if (nid === undefined) s.clear(); else s.delete(nid); }
// Gone from here, unless a load of it sent after this unload is under way.
if ((!s || !s.size) && !w.loading.has(code)) { w.sessions.delete(code); if (placed.get(code) === w) placed.delete(code); }
// A worker of its own with nothing on it and nothing pending is done (with a call pending it would never answer).
if (w.dedicated && !w.dead && !w.sessions.size && !w.loading.size && !w.pending.size) { if (own.get(code) === w) own.delete(code); w.dead = true; w.worker.terminate(); }
return r;
});
},
stats() {
const ws = [...pool, ...own.values()].filter((w) => !w.dead);
return { workers: ws.length, realms: ws.reduce((a, w) => a + size(w), 0), sessions: ws.reduce((a, w) => a + w.sessions.size, 0) };
},
close() {
closed = true;
const ws = [...pool, ...own.values()];
return Promise.all(ws.map((w) => (w.dead ? null : w.worker.terminate()))).then(() => undefined);
},
snapWith,
};
if (test) {
engine.eval = (code, nid, src, ms) => to(code, nid, { t: 'eval', src: String(src), ...(ms ? { ms } : {}) });
engine.mem = () => Promise.all(pool.filter((w) => !w.dead).map((w) => send(w, { t: 'mem' })));
engine.kill = (i = 0) => { const w = pool[i]; if (w && !w.dead) w.worker.terminate(); };
engine.killOf = (code) => { const w = placed.get(code); if (w && !w.dead) w.worker.terminate(); };
engine.workerOf = (code) => pool.indexOf(placed.get(code));
engine.dedicated = (code) => !!(placed.get(code) && placed.get(code).dedicated);
}
{ const page = require('./page'); engine.simId = page.simId(page.assemble({ src: path.join(opts.root || page.ROOT, 'src'), mp: true }).html); }
return engine;
}
module.exports = { createEngine, snapWith };