// The server-run engine (the server-sim design, sections 4.2-4.3): a small pool of worker threads (mp-sim-worker.js), // each holding the vm realms (mp-sim-realm.js) of the sessions placed on it, behind a promise API. The session host // (scripts/lib/mp-sim.js) keeps the clock, presence and votes, and asks this for days, actions, what-ifs and pulls. // tests/lib/mp-sim-fake-engine.js has the same API with canned answers. // // const engine = createEngine({workers: 2, heapMb: 1536, root, log, maxOwn: 4, onLost}); // engine.load(code, [{nid, env?, begin?, lang, prefs, rng?}]) → [{t:'loaded', nid, epoch, day, lost, saved?} | // {t:'loaded', nid, err}], one per network, in order. env: the checkpoint envelope {s, uidN, rt, last, // day} (s: the save text; rt: what the realm needs besides it to play on exactly as before, opaque // here: route memory, R's routing point, key order, the sales desk's memory); begin: {name, home, // preset, asn, mp, tut} builds a new network, checkpointed at once (saved). A network that comes // without a save, whose save does not load, or that cannot begin answers err: broken (4.7), and // the others load. A load of a network that is loaded replaces its realm (a failed one leaves none). // engine.day(code, {d, speed, last, alone:[nid], nets:[nid], mute}) → {t:'dayDone', code, d, nets:{[nid]: {day, lost, // pause?, err?, retried?, ms}}}. Each network in sorted order; one that failed twice answers err and // its day as it was, and is broken from then on (no more days; actions answer 'stopped'). // engine.act(code, nid, msg) → {t:'actDone', nid, epoch, rid, outcome, why?, v, upd, ms} (upd: the update with the // result, version v; null when stopped) | {t:'err', code, nid, rid, what:'timeout'|'throw', msg} // engine.ask(code, nid, msg) → {t:'askDone', nid, rid, modal?, ui?, err?, ms} (neither modal nor err: the what-if // opened nothing) | {t:'err', …} // engine.pull(code, nid, {snap, views}) → {t:'pulled', nid, epoch, v, upd?, snap?, subs?, ms}. snap is the realm's // half of a snapshot: snapWith(snap, {clock, roster, since}) makes the stream event. // ms: the worker's time for the call (the mark, the realm, the update), without the wait behind other calls: what // the host's rate limit of worker time counts (section 10). // engine.sub(code, nid, {views}) → {t:'subbed', nid, epoch, v, subs}; engine.greet(code, nid) → a pulled // engine.prefs(code, nid, {prefs}) → undefined (or the {t:'err'} of a realm that failed); // engine.save(code, nid) → {t:'saved', nid, txt, uidN, rt, day, lost}; engine.card(code, nid) → {t:'card', nid, card} // engine.unload(code, nid?) → {t:'unloaded', code, nid?}; engine.has(code, nid) → whether that network has a realm // engine.stats() → {workers, realms, sessions} (not a promise); engine.mem() → each pool worker's heap and rss // engine.close() → stops the workers // engine.simId the build id this engine's game belongs to (SIM_ID of the multiplayer page from the same src/). The // game text is read once, here: every worker, also one started after an exit, runs that game, and // a src/ that does not compile makes createEngine throw. // // Rejections: a call for a network that is not loaded (code 'NO_REALM'), a malformed call (code 'ERR': a load that // names no network, or one twice, changes nothing), every call pending on a worker that exits (out of memory, a crash): // Error('worker exited'), code 'EXIT', and every call while no worker can start (code 'SIM_DOWN', 503 for the host). // Every call is answered or rejected; none waits for ever. // Lost networks. A worker that exits takes the realms of every session on it, also of those with no call pending, and // the worker drops a network whose realm cannot be built again after a failure. Such a network answers NO_REALM: for a // network the host believes loaded, NO_REALM means "load it again from its checkpoint" (4.7). opts.onLost(code, nids, // why) says it as it happens ('exit', or the worker's reason). // Blame. An exit counts against the session the worker was running (a call or a step that crashed or overflowed it), // else against those whose load was under way, else against every session it held; a worker that never started // blames nobody. A session with two exits against it within an hour gets a worker of its own (at most opts.maxOwn of // them, 4 by default: past that it is given up); when that one exits too, its loads reject with code 'GAVE_UP' until // the engine is made again (the server restarts). A worker that exits before it is ready is started again after a // pause that doubles; after 5 such starts in a row the engine is down (SIM_DOWN). A session stays on its worker while // a load of it is under way, so an unload and a load sent back to back (a language change) land on the same worker. // Epochs: every realm load mints one (a load, a begin, the fresh realm after a timeout); loaded, pulled, actDone and // subbed carry the realm's current one. Versions are the realm's: only a pull (or an action's own pull) makes one. // // Tests only, with createEngine({test: {seed, now, step} | true}): engine.eval(code, nid, src) runs code in a realm // ({value} | {error} | {timeout, epoch}); engine.kill(i) ends pool worker i, engine.killOf(code) the worker of that // session; engine.workerOf(code) is its pool index (-1: none, or its own), engine.dedicated(code) whether it has a // worker of its own, engine.running(i) the session pool worker i is running as far as the engine knows. // opts.test.startFail: no worker starts; opts.startBackoffMs: the first pause before starting one again (1000). 'use strict'; const path = require('node:path'); const vm = require('node:vm'); const { Worker } = require('node:worker_threads'); const page = require('./page'); const { gameSource } = require('./mp-sim-realm'); const FILE = path.join(__dirname, 'mp-sim-worker.js'); const HOUR = 3600e3; /* The stream event of a snapshot: the realm's half (S, r, keep, epoch, v, now) with the host's (clock, roster, since), joined as text: the realm's 270 KB are never parsed again. */ function snapWith(snap, { clock, roster, since }) { if (typeof snap !== 'string' || !snap.endsWith('}')) throw new Error('snapWith: not a snapshot'); return snap.slice(0, -1) + ',"clock":' + JSON.stringify(clock) + ',"roster":' + JSON.stringify(roster) + ',"since":' + JSON.stringify(since) + '}'; } function createEngine(opts = {}) { if (!vm.constants || vm.constants.DONT_CONTEXTIFY === undefined) { throw new Error(`server-run play needs Node 22.8 or newer (vm.constants.DONT_CONTEXTIFY); this is ${process.version}`); } const N = Math.max(1, Math.floor(+opts.workers || 2)); const heapMb = Math.max(64, Math.floor(+opts.heapMb || 1536)); const maxOwn = Math.max(0, Math.floor(opts.maxOwn !== undefined ? +opts.maxOwn || 0 : 4)); const backoff = Math.max(1, +opts.startBackoffMs || 1000); const log = typeof opts.log === 'function' ? opts.log : () => {}; const lostTo = (code, nids, why) => { if (typeof opts.onLost === 'function' && nids.length) try { opts.onLost(code, nids, why); } catch (e) { log('error', 'mp-sim: onLost threw: ' + (e && e.message)); } }; const test = opts.test || null; // The game, read once, here: a src/ that does not compile throws now, and every worker runs the game simId names. const root = opts.root || page.ROOT, src = gameSource({ root, page: opts.page }); try { new vm.Script(src, { filename: 'game.js' }); } catch (e) { throw new Error(`mp-sim: the game in ${path.join(root, 'src')} does not compile: ${e.message}`); } const simId = page.simId(page.assemble({ src: path.join(root, 'src'), mp: true }).html); const W = { root: opts.root, page: opts.page, src, simId, test, strict: opts.strict !== undefined ? !!opts.strict : process.env.SIM_STRICT === '1', timeouts: opts.timeouts }; let seq = 0, closed = false, down = null, fails = 0; const DOWN = () => Object.assign(new Error(`server-run play is down: ${down}`), { code: 'SIM_DOWN' }); const CLOSED = () => Object.assign(new Error('engine closed'), { code: 'CLOSED' }); // Why no call can go out now: closed, or down (else null). const shut = () => (closed ? CLOSED() : down ? DOWN() : null); const pool = [], own = new Map(), placed = new Map(), exits = new Map(), gaveUp = new Set(); // A worker's realms (sessions: code → its loaded nids), and for placement the networks of its loads under way too // (loading: code → how many). const size = (w) => [...w.sessions.values()].reduce((a, b) => a + b.size, 0); const count = (w) => size(w) + [...w.loading.values()].reduce((a, b) => a + b, 0); function spawn(dedicated = false) { const w = { worker: null, pending: new Map(), sessions: new Map(), loading: new Map(), dedicated, dead: false, ready: false, fatal: null, running: null }; w.worker = new Worker(FILE, { workerData: { mpSim: W }, resourceLimits: { maxOldGenerationSizeMb: heapMb } }); w.worker.on('message', (m) => { if (m.log) { log(m.log[0], m.log[1]); return; } if (m.fatal) { w.fatal = String(m.fatal).split('\n')[0].slice(0, 300); return; } if (m.ready) { w.ready = true; fails = 0; return; } // The session it runs from now until its next reply: whose doing an exit then is. if ('run' in m) { w.running = m.run; return; } if (m.lost) { const s = w.sessions.get(m.lost.code); if (s) s.delete(m.lost.nid); lostTo(m.lost.code, [m.lost.nid], m.lost.why); return; } w.running = null; const p = w.pending.get(m.id); if (!p) return; w.pending.delete(m.id); // An idle worker does not keep the process alive; one with calls pending does. if (!w.pending.size) w.worker.unref(); if (m.ok) p.ok(m.reply); else p.no(Object.assign(new Error(m.error.message), { code: m.error.code })); }); w.worker.on('error', (e) => log('error', 'mp-sim worker failed: ' + (e && e.message))); w.worker.on('exit', (c) => died(w, c)); w.worker.unref(); return w; } function died(w, c) { if (w.dead) return; w.dead = true; // Whose exit it is: the session it was running (a call or a step that crashed or overflowed it), else those whose // load was under way, else every session it held; nobody's when it never started. The others lose their realms // (the host loads them again) without an exit against them. const held = new Set([...w.sessions.keys(), ...w.loading.keys()]); const blamed = new Set(!w.ready ? [] : w.running && held.has(w.running) ? [w.running] : w.loading.size ? w.loading.keys() : held); const e = Object.assign(new Error('worker exited'), { code: 'EXIT', exitCode: c }); for (const p of w.pending.values()) p.no(e); w.pending.clear(); const t = Date.now(); for (const code of held) { if (placed.get(code) === w) placed.delete(code); lostTo(code, [...(w.sessions.get(code) || [])], 'exit'); if (!blamed.has(code)) continue; if (w.dedicated) { gaveUp.add(code); own.delete(code); log('error', `mp-sim: session ${code} took down its own worker too; not loading it again until a restart`); continue; } exits.set(code, (exits.get(code) || []).filter((x) => t - x < HOUR).concat(t)); } w.sessions.clear(); w.loading.clear(); for (const [k, x] of own) if (x === w) own.delete(k); if (closed) return; const i = pool.indexOf(w); if (!w.ready) { // It never started (the game does not run in a worker): not again at once, but after a pause that doubles; after // 5 such starts in a row, server-run play is down until the engine is made again. if (++fails >= 5) { if (!down) { down = w.fatal || `a worker exited before it started (code ${c})`; log('error', `mp-sim: no worker can start (${down}); server-run play is down until a restart`); } return; } log('warn', `mp-sim: a worker could not start (${w.fatal || 'code ' + c}); trying again in ${backoff * 2 ** (fails - 1)} ms`); if (i >= 0) setTimeout(() => { if (!closed && !down && pool[i] === w) pool[i] = spawn(); }, backoff * 2 ** (fails - 1)).unref(); return; } log('warn', `mp-sim: a worker exited (code ${c}); its sessions load again from their checkpoints`); if (i >= 0) pool[i] = spawn(); } for (let i = 0; i < N; i++) pool.push(spawn()); // The worker of a session: where it is, else the pool's least loaded one (or one of its own, after two exits). function workerOf(code, create) { let w = placed.get(code); if (w && !w.dead) return w; if (!create) return null; if (down) throw DOWN(); if (gaveUp.has(code)) throw Object.assign(new Error(`session ${code} kept failing its worker`), { code: 'GAVE_UP' }); if ((exits.get(code) || []).filter((x) => Date.now() - x < HOUR).length >= 2) { if (own.size >= maxOwn) { gaveUp.add(code); log('error', `mp-sim: session ${code} keeps failing its worker, and ${own.size} sessions have workers of their own already; not loading it again until a restart`); throw Object.assign(new Error(`session ${code} kept failing its worker`), { code: 'GAVE_UP' }); } w = spawn(true); own.set(code, w); } else { const live = pool.filter((x) => !x.dead); if (!live.length) throw Object.assign(new Error('no worker is up yet: try again'), { code: 'SIM_DOWN' }); w = live.reduce((a, b) => (count(b) < count(a) ? b : a)); } placed.set(code, w); return w; } function send(w, m) { if (closed) return Promise.reject(CLOSED()); if (!w || w.dead) return Promise.reject(Object.assign(new Error('worker exited'), { code: 'EXIT' })); const id = ++seq; return new Promise((ok, no) => { if (!w.pending.size) w.worker.ref(); w.pending.set(id, { ok, no }); // A message that cannot be sent (not cloneable) is a malformed call, and nothing waits for its answer. try { w.worker.postMessage({ id, ...m }); } catch (e) { w.pending.delete(id); if (!w.pending.size) w.worker.unref(); no(Object.assign(new Error(String((e && e.message) || e)), { code: 'ERR' })); } }); } const noRealm = (code, nid) => Promise.reject(Object.assign(new Error(`no realm ${code}/${nid}`), { code: 'NO_REALM' })); const to = (code, nid, m) => { const x = shut(); if (x) return Promise.reject(x); const w = workerOf(code, false); return w ? send(w, { code, nid, ...m }) : noRealm(code, nid); }; const mine = (w, code) => { let s = w.sessions.get(code); if (!s) w.sessions.set(code, s = new Set()); return s; }; const engine = { simId, load(code, nets) { { const x = shut(); if (x) return Promise.reject(x); } if (!Array.isArray(nets) || !nets.length) return Promise.reject(Object.assign(new Error('load: give the networks to load'), { code: 'ERR' })); let w; try { w = workerOf(code, true); } catch (e) { return Promise.reject(e); } // Under way: the session is on this worker from now (an exit meanwhile counts against it, and its placement holds). const n = nets.length, done = () => { const k = (w.loading.get(code) || 0) - n; if (k > 0) w.loading.set(code, k); else w.loading.delete(code); }; w.loading.set(code, (w.loading.get(code) || 0) + n); return send(w, { t: 'load', code, nets }).then((r) => { done(); const s = mine(w, code); for (const l of r) if (!l.err) s.add(l.nid); else s.delete(l.nid); return r; }, (e) => { done(); throw e; }); }, day(code, { d, speed = 1, last = 1, alone = [], nets = [], mute = false } = {}) { { const x = shut(); if (x) return Promise.reject(x); } const w = workerOf(code, false); if (!w) return Promise.reject(Object.assign(new Error(`no session ${code} here`), { code: 'NO_REALM' })); return send(w, { t: 'day', code, d, speed, last, alone, nets, mute }); }, act(code, nid, msg) { return to(code, nid, { t: 'act', msg }); }, ask(code, nid, msg) { return to(code, nid, { t: 'ask', msg }); }, pull(code, nid, { snap = false, views } = {}) { return to(code, nid, { t: 'pull', snap: !!snap, ...(views ? { views } : {}) }); }, sub(code, nid, { views = [] } = {}) { return to(code, nid, { t: 'sub', views }); }, prefs(code, nid, { prefs } = {}) { return to(code, nid, { t: 'prefs', prefs }).then((r) => (r && r.t === 'err' ? r : undefined)); }, greet(code, nid) { return to(code, nid, { t: 'greet' }); }, save(code, nid) { return to(code, nid, { t: 'save' }); }, card(code, nid) { return to(code, nid, { t: 'card' }); }, unload(code, nid) { if (closed) return Promise.reject(CLOSED()); const w = workerOf(code, false); if (!w) return Promise.resolve({ t: 'unloaded', code, ...(nid !== undefined ? { nid } : {}) }); return send(w, { t: 'unload', code, ...(nid !== undefined ? { nid } : {}) }).then((r) => { const s = w.sessions.get(code); if (s) { if (nid === undefined) s.clear(); else s.delete(nid); } // Gone from here, unless a load of it sent after this unload is under way. if ((!s || !s.size) && !w.loading.has(code)) { w.sessions.delete(code); if (placed.get(code) === w) placed.delete(code); } // A worker of its own with nothing on it and nothing pending is done (with a call pending it would never answer). if (w.dedicated && !w.dead && !w.sessions.size && !w.loading.size && !w.pending.size) { if (own.get(code) === w) own.delete(code); w.dead = true; w.worker.terminate(); } return r; }); }, has(code, nid) { const w = placed.get(code), s = w && !w.dead && w.sessions.get(code); return !!(s && s.has(nid)); }, mem() { return Promise.all(pool.filter((w) => !w.dead).map((w) => send(w, { t: 'mem' }))); }, stats() { const ws = [...pool, ...own.values()].filter((w) => !w.dead); return { workers: ws.length, realms: ws.reduce((a, w) => a + size(w), 0), sessions: ws.reduce((a, w) => a + w.sessions.size, 0) }; }, close() { closed = true; const ws = [...pool, ...own.values()]; return Promise.all(ws.map((w) => (w.dead ? null : w.worker.terminate()))).then(() => undefined); }, snapWith, }; if (test) { engine.eval = (code, nid, src, ms) => to(code, nid, { t: 'eval', src: String(src), ...(ms ? { ms } : {}) }); engine.kill = (i = 0) => { const w = pool[i]; if (w && !w.dead) w.worker.terminate(); }; engine.killOf = (code) => { const w = placed.get(code); if (w && !w.dead) w.worker.terminate(); }; engine.workerOf = (code) => pool.indexOf(placed.get(code)); engine.dedicated = (code) => !!(placed.get(code) && placed.get(code).dedicated); engine.running = (i = 0) => (pool[i] ? pool[i].running : null); } return engine; } module.exports = { createEngine, snapWith };