// A worker thread of the server-run engine (the server-sim design, section 4.2): it compiles the game once, holds the // realms of the sessions placed on it (every network of a session in one worker), and answers the host's messages. // // Order. Messages run one at a time, in arrival order per network. A session's day is one task per network, in sorted // network order, with a turn for other messages between two networks: an action for network X lands between days of // other networks, never inside X's own day. The networks of a day are fixed when it starts. Loads, new networks and // unloads of a session wait for its day; actions for a network that is still loading wait for it. Every message is // answered: a load that names no network (or one twice), or a day whose lists are not lists, is refused whole and // changes nothing; a network in a load that has no save, or whose save does not load, answers err (broken) and the // others load. Nothing a message holds, and nothing a step throws, takes the worker down. // // Failures. Every realm call has a time limit (a day 2 s, an action 1 s, a what-if 3 s, a pull or a save 1 s). Before // a day or an action the realm is marked (its S as text, uidN, the dice, and for a day its rt: route memory and routing // point), and the worker keeps the mark. A mark changes nothing: one that fails leaves the realm as it was, and one // that runs out of time has the realm built again from its own save, never from an older mark (that would undo what // came after it). A call that throws has run its finally blocks: the realm goes back to the mark in place. A call that // times out has not (vm skips them), and one after which the realm's built-ins changed (a prototype written through) // left them changed: such a realm is never called again for play. A fresh one is built from the mark, with a new // epoch, and takes over the fx the mark held. A failed day runs once more; failing again, the network is broken at its // state before the day. When not even a fresh realm can be built, the network is dropped: its calls answer NO_REALM, // and the engine is told ({lost}), so the host loads it again from its checkpoint (4.7). // // The worker → host replies are the shapes of tests/lib/mp-sim-shape.js checkWorker(), sent as {id, ok, reply} (or // {id, ok:false, error}) for the engine to match its promises; logs go up as {log:[level, text]}, a dropped network as // {lost:{code, nid, why}}, and before each job or step that runs a realm {run: code}, the session it runs (the engine // forgets it at the next reply): a worker that dies then is that session's doing. 'use strict'; const { isMainThread, parentPort, workerData } = require('node:worker_threads'); const crypto = require('node:crypto'); const v8 = require('node:v8'); const { compileCode, Realm, isTimeout, parts } = require('./mp-sim-realm'); const TIMEOUTS = { day: 2000, act: 1000, ask: 3000, pull: 1000, save: 1000, mark: 1000, small: 1000, load: 30000 }; const now = () => performance.now(); const MSG = (e) => String((e && (e.stack || e.message)) || e).slice(0, 2000); const LINE = (e) => String((e && e.message) || e).split('\n')[0].slice(0, 300); const msOf = (t0) => +(now() - t0).toFixed(2); /* opts: {root, page, src, simId, test: {seed, now, step} | true | null, strict, timeouts, log(level, text), post(message)}. */ function createCore(opts = {}) { const code = compileCode({ root: opts.root, page: opts.page, src: opts.src, simId: opts.simId }); const T = { ...TIMEOUTS, ...(opts.timeouts || {}) }; const test = opts.test ? (typeof opts.test === 'object' ? opts.test : {}) : null; const log = opts.log || (() => {}); const post = opts.post || (() => {}); const realms = new Map(), sessions = new Map(), jobs = []; let epochN = 0, pumping = false, turn = 0; const key = (c, n) => c + '/' + n; const epochOf = () => crypto.randomBytes(4).toString('hex') + (++epochN).toString(36); const session = (c) => { let s = sessions.get(c); if (!s) sessions.set(c, s = { code: c, day: null, load: null }); return s; }; const running = (c) => post({ run: c }); /* ---- realms ---- */ function build(nid, lang) { return new Realm({ code, lang, test: test && { seed: test.seed, now: test.now, step: test.step }, loadMs: T.load, log: (lvl, txt) => log(lvl === 'log' || lvl === 'info' || lvl === 'debug' ? 'info' : 'warn', `[${nid}] ${txt}`) }); } // A realm call: [header, ...parts]; a throw inside the realm comes back as {srvErr}, a timeout throws. A realm whose // built-ins changed is marked polluted: the caller replaces it. const call = (e, name, input, ms) => { const r = e.realm.call(name, input, ms); if (r[0] && r[0].polluted) e.polluted = true; return r; }; // What a fresh realm takes from one that is done (its save, rt, R, fx), whatever its built-ins are now. const peek = (e, name, input, ms) => (e.realm ? parts(e.realm.raw(name, input, ms, false)) : [{ srvErr: 'gone' }]); /* A fresh realm in place of one that is done (timed out, polluted, or could not go back in place), from {s, uidN, rt, live}: same network, new epoch, versions from 0, the fx the old one held (the mark's, or after a read all of them). R comes from the old realm, which still answers after a timeout: the marked R (mark) or its current one, packed whole; when it cannot (or is polluted), or that R does not load, R is routed again from rt's routing point. The old realm is never called again, even when this throws. */ function rebuild(e, from, mark = true) { if (!from || typeof from.s !== 'string') { e.realm = null; throw new Error('rebuilding the realm failed: nothing to build from'); } let rpack = null, fx = null; if (!e.polluted) try { const s = e.realm && e.realm.raw('srvRPack', { mark }, T.load, false); if (typeof s === 'string' && s.startsWith('{"o"')) rpack = s; } catch (x) { /* gone */ } try { const [h] = peek(e, 'srvFxOut', { mark }, T.small); if (h && Array.isArray(h.fx)) fx = h; } catch (x) { /* gone */ } e.realm = null; const epoch = epochOf(); const load = (rp) => { const r = build(e.nid, e.lang); const [h] = r.call('srvLoad', { save: from.s, uidN: from.uidN, rt: from.rt, rpack: rp, fx, live: !!from.live, prefs: e.prefs, epoch, strict: !!opts.strict, clock: from.clock }, T.load); if (h.srvErr) throw new Error(h.srvErr); return [r, h]; }; let r, h; try { [r, h] = load(rpack); } catch (x) { if (!rpack) throw new Error('rebuilding the realm failed: ' + LINE(x)); log('warn', `[${e.nid}] R as it was did not load (${LINE(x)}): routed again`); rpack = null; try { [r, h] = load(null); } catch (y) { throw new Error('rebuilding the realm failed: ' + LINE(y)); } } e.realm = r; e.polluted = false; e.epoch = epoch; e.v = 0; e.day = h.day; e.lost = h.lost; log('warn', `[${e.nid}] realm rebuilt (epoch ${epoch}${rpack ? '' : ', R routed again'})`); } // The network cannot go on: dropped (later calls answer NO_REALM), and the engine tells the host (4.7). function lose(e, why) { e.realm = null; e.broken = true; if (realms.get(key(e.code, e.nid)) === e) realms.delete(key(e.code, e.nid)); log('error', `[${e.nid}] dropped, its realm could not be built again: ${why}`); post({ lost: { code: e.code, nid: e.nid, why: String(why).slice(0, 300) } }); } // A recovery: true when the network has a realm to play on again, else it is dropped. const recover = (e, f) => { try { f(); return true; } catch (x) { lose(e, LINE(x)); return false; } }; const gone = (e) => (e.realm ? '' : '; the network was dropped: load it again'); // The rt of the old realm's mark, if it still answers; else the last day's, whose routing point belongs to that day's // S: with another mark only its route memory counts (a close R). function markRt(e) { try { const [h] = peek(e, 'srvRt', { mark: true }, 200); if (h && !h.srvErr && h.load) return h; } catch (x) { /* gone */ } return e.dayMark ? (e.dayMark === e.mark ? e.dayMark.rt : { ...e.dayMark.rt, pt: undefined, rk: undefined }) : null; } // The restore point of the next day or action. A failed mark throws {mark:true} and changed nothing applied; one that // ran out of time has the realm built again from its own save first (or the network dropped). function mark(e, input) { e.mark = null; if (input.day) e.dayMark = null; let h, s; try { [h, s] = call(e, 'srvMark', input, T.mark); } catch (x) { if (!isTimeout(x)) throw x; recover(e, () => rebuildAfterRead(e)); throw Object.assign(new Error(`marking the network took longer than ${T.mark} ms`), { mark: true, timeout: true }); } if (h.srvErr) { if (e.polluted) recover(e, () => rebuildAfterRead(e)); throw Object.assign(new Error('marking the network failed: ' + LINE(h.srvErr)), { mark: true }); } const m = { s, uidN: h.uidN, rt: h.rt || null, day: h.day, clock: h.clock, live: true }; if (input.day) e.dayMark = m; e.mark = m; return m; } // After a throw: back to the mark in place; if even that fails (or the realm is polluted), a fresh realm from the mark. function restore(e, input) { if (!e.polluted) { try { const [h] = call(e, 'srvRestore', input, T.small); if (!h.srvErr) return; log('warn', `[${e.nid}] restore failed: ${h.srvErr}`); } catch (x) { log('warn', `[${e.nid}] restore failed: ${LINE(x)}`); } } rebuild(e, { ...e.mark, rt: input.day ? e.mark.rt : markRt(e) }); } // A call that changed nothing applied and timed out (or left the realm polluted): the fresh realm starts from the // realm's own save. When even that cannot be read, this throws: never an older mark. function rebuildAfterRead(e) { let from = null; try { const [h, txt] = peek(e, 'srvSave', undefined, T.save); if (!h.srvErr && typeof txt === 'string') from = { s: txt, uidN: h.uidN, rt: h.rt, clock: h.clock }; } catch (x) { /* gone */ } if (!from) { e.realm = null; throw new Error('rebuilding the realm failed: its save could not be read'); } rebuild(e, from, false); } /* ---- the handlers: each returns the reply (or throws, which the engine turns into a rejection) ---- */ const errReply = (m, what, msg) => ({ t: 'err', code: m.code, nid: m.nid, ...(m.msg && m.msg.rid ? { rid: m.msg.rid } : {}), what, msg: String(msg).slice(0, 500) }); function loadOne(c, n) { const e = realms.get(key(c, n.nid)); try { if (!n.begin && !(n.env && typeof n.env === 'object' && typeof n.env.s === 'string')) throw new Error('no save to load'); e.realm = build(n.nid, n.lang); e.lang = e.realm.lang; const epoch = epochOf(); const [h] = n.begin ? call(e, 'srvBegin', { begin: n.begin, prefs: n.prefs, epoch, strict: !!opts.strict }, T.load) : call(e, 'srvLoad', { save: n.env.s, uidN: n.env.uidN, rt: n.env.rt, prefs: n.prefs, rng: n.rng, epoch, strict: !!opts.strict }, T.load); if (h.srvErr) throw new Error(h.srvErr); Object.assign(e, { epoch, v: 0, day: h.day, lost: h.lost, loading: false }); const out = { t: 'loaded', nid: n.nid, epoch, day: h.day, lost: h.lost }; if (n.begin) out.saved = saved(e); return out; } catch (x) { if (realms.get(key(c, n.nid)) === e) realms.delete(key(c, n.nid)); log('warn', `[${n.nid}] did not load: ${MSG(x)}`); return { t: 'loaded', nid: n.nid, err: isTimeout(x) ? 'the network took too long to load' : LINE(x) }; } } function saved(e) { const [h, txt] = call(e, 'srvSave', undefined, T.save); if (h.srvErr) { if (e.polluted) recover(e, () => rebuildAfterRead(e)); throw new Error(h.srvErr); } return { t: 'saved', nid: e.nid, txt, uidN: h.uidN, rt: h.rt, day: h.day, lost: h.lost }; } // One network's day, with its mark, one retry, and broken after a second failure. function dayOne(e, m) { const t0 = now(), d0 = e.day; const input = { speed: m.speed, last: m.last, alone: m.alone.includes(e.nid), mute: !!m.mute }; let err = null; for (let i = 0; i < 2 && e.realm; i++) { try { mark(e, { day: true }); const [h] = call(e, 'srvDay', input, T.day); if (h.srvErr) throw new Error(h.srvErr); e.day = h.day; e.lost = h.lost; return { day: h.day, lost: h.lost, ...(h.pause ? { pause: true } : {}), ...(i ? { retried: true } : {}), ms: msOf(t0) }; } catch (x) { err = x; log('warn', `[${e.nid}] day ${d0 + 1} failed (${x.mark ? 'mark' : isTimeout(x) ? 'timeout' : e.polluted ? 'polluted' : 'throw'}${i ? ', again' : ''}): ${MSG(x)}`); // A failed mark changed nothing (the realm is as it was, built again from its own save, or dropped). Else back // to this attempt's mark: in place after a throw, a fresh realm after a timeout or a pollution. if (!x.mark && !recover(e, () => { if (isTimeout(x) || e.polluted) rebuild(e, e.dayMark); else restore(e, { day: true }); })) break; } } e.broken = true; return { day: e.day, lost: !!e.lost, err: (isTimeout(err) ? 'timeout: ' : '') + LINE(err) + gone(e), retried: true, ms: msOf(t0) }; } // An action or a what-if: its mark, the call, and the way back when it fails. function marked(m, e, name, input, ms, what) { try { mark(e, input); } catch (x) { if (!x.mark) throw x; return [errReply(m, x.timeout ? 'timeout' : 'throw', x.message + gone(e))]; } let r; try { r = call(e, name, m.msg, ms); } catch (x) { if (!isTimeout(x)) throw x; recover(e, () => rebuild(e, { ...e.mark, rt: markRt(e) })); return [errReply(m, 'timeout', `the ${what} took longer than ${ms} ms` + gone(e))]; } if (r[0].srvErr) { recover(e, () => (e.polluted ? rebuild(e, { ...e.mark, rt: markRt(e) }) : restore(e, {}))); return [errReply(m, 'throw', r[0].srvErr + gone(e))]; } return r; } const H = { act(m, e) { if (e.broken || e.lost) return { t: 'actDone', nid: m.nid, epoch: e.epoch, rid: m.msg.rid, outcome: 'stopped', v: e.v, upd: null }; const t0 = now(), [h, upd] = marked(m, e, 'srvAct', { keys: true }, T.act, 'action'); if (h.t === 'err') return h; e.v = h.v; if (h.outcome === 'stopped') e.lost = true; return { t: 'actDone', nid: m.nid, epoch: e.epoch, rid: m.msg.rid, outcome: h.outcome, ...(h.why ? { why: h.why } : {}), v: h.v, upd: upd || null, ms: msOf(t0) }; }, ask(m, e) { if (e.broken || e.lost) return { t: 'askDone', nid: m.nid, rid: m.msg.rid, err: 'stopped' }; const t0 = now(), [h] = marked(m, e, 'srvAsk', {}, T.ask, 'what-if'); if (h.t === 'err') return h; return { t: 'askDone', nid: m.nid, rid: m.msg.rid, ...h, ms: msOf(t0) }; }, pull(m, e) { return pulled(m, e, 'srvPull', { snap: !!m.snap, views: m.views }); }, greet(m, e) { return pulled(m, e, 'srvGreet', {}); }, sub(m, e) { return guarded(m, e, () => { const [h] = call(e, 'srvSub', { views: m.views || [] }, T.pull); if (h.srvErr) return errReply(m, 'throw', h.srvErr); return { t: 'subbed', nid: m.nid, epoch: e.epoch, v: h.v, subs: h.subs }; }); }, save(m, e) { return guarded(m, e, () => saved(e)); }, card(m, e) { return guarded(m, e, () => { const [h] = call(e, 'srvCard', undefined, T.small); if (h.srvErr) return errReply(m, 'throw', h.srvErr); return { t: 'card', nid: m.nid, card: h.card || null }; }); }, prefs(m, e) { if (m.prefs) e.prefs = { ...e.prefs, ...m.prefs }; return guarded(m, e, () => { const [h] = call(e, 'srvPrefs', { prefs: m.prefs }, T.small); return h.srvErr ? errReply(m, 'throw', h.srvErr) : null; }); }, // Tests only: a piece of code in the realm, its value back. eval(m, e) { if (!test) throw new Error('eval is for tests'); try { return { value: e.realm.eval(m.src, m.ms || 5000) }; } catch (x) { if (isTimeout(x)) { recover(e, () => rebuildAfterRead(e)); return { timeout: true, epoch: e.epoch }; } return { error: MSG(x) }; } }, }; function pulled(m, e, name, input) { return guarded(m, e, () => { const t0 = now(), [h, ...p] = call(e, name, input, T.pull); if (h.srvErr) return errReply(m, 'throw', h.srvErr); e.v = h.v; const o = { t: 'pulled', nid: m.nid, epoch: e.epoch, v: h.v }; (h.parts || []).forEach((k, i) => { o[k] = p[i]; }); if (h.subs) o.subs = h.subs; o.ms = msOf(t0); return o; }); } // A call that changes nothing applied: after a timeout (or a pollution) the realm is built again from its own save. function guarded(m, e, f) { let r; try { r = f(); } catch (x) { if (!isTimeout(x)) { if (e.polluted && e.realm) recover(e, () => rebuildAfterRead(e)); throw x; } recover(e, () => rebuildAfterRead(e)); return errReply(m, 'timeout', `${m.t} took too long` + gone(e)); } if (e.polluted && e.realm) recover(e, () => rebuildAfterRead(e)); return r; } /* ---- the queue ---- */ const SESSION_OPS = new Set(['load', 'unload', 'day']); function runnable(j, i) { const s = sessions.get(j.code); if (SESSION_OPS.has(j.t)) { if (s && (s.day || s.load)) return false; for (let k = 0; k < i; k++) if (jobs[k].code === j.code) return false; return true; } for (let k = 0; k < i; k++) if (jobs[k].code === j.code && SESSION_OPS.has(jobs[k].t) && jobs[k].t !== 'day') return false; // A network that is loading waits for its load; one marked loading with no load under way (none can be: a guard) // waits for nothing, so start() answers NO_REALM. const e = realms.get(key(j.code, j.nid)); return !e || !e.loading || !(s && s.load); } function reply(j, r) { post({ id: j.id, ok: true, reply: r === undefined ? null : r }); } function fail(j, x) { post({ id: j.id, ok: false, error: { message: String((x && x.message) || x), code: (x && x.code) || 'ERR' } }); } function runJob() { for (let i = 0; i < jobs.length; i++) { const j = jobs[i]; if (!runnable(j, i)) continue; jobs.splice(i, 1); try { start(j); } catch (x) { fail(j, x); } return true; } return false; } const strings = (a) => [...new Set(a.filter((n) => typeof n === 'string'))]; function start(j) { if (j.t === 'mem') { const h = v8.getHeapStatistics(), mu = process.memoryUsage(); return reply(j, { heapUsed: h.used_heap_size, heapTotal: h.total_heap_size, external: h.external_memory, rss: mu.rss, realms: realms.size }); } if (j.t === 'load') { // The whole call first, with no side effects: a malformed one is refused and leaves every realm as it was. A // network without a save to load (or a begin) is only that network's problem: loadOne answers it broken. const nets = Array.isArray(j.nets) ? j.nets : [], seen = new Set(); if (!nets.length) throw new Error('load: give the networks to load'); for (const n of nets) { if (!n || typeof n !== 'object' || typeof n.nid !== 'string' || !n.nid) throw new Error('load: each network needs a nid'); if (seen.has(n.nid)) throw new Error(`load: ${n.nid.slice(0, 40)} is there twice`); seen.add(n.nid); } const s = session(j.code); for (const n of nets) realms.set(key(j.code, n.nid), { code: j.code, nid: n.nid, lang: n.lang || 'en', prefs: n.prefs || {}, loading: true, realm: null, epoch: '', v: 0, day: 0, lost: false, broken: false, polluted: false, mark: null, dayMark: null }); s.load = { j, i: 0, out: [] }; return; } if (j.t === 'day') { // The lists as lists (of network ids), or the day is refused whole; speed and last the realm checks itself. for (const k of ['nets', 'alone']) if (j[k] !== undefined && !Array.isArray(j[k])) throw new Error(`day: ${k} must be a list`); j.alone = strings(j.alone || []); const s = session(j.code); const order = strings(j.nets || []).sort().filter((n) => { const e = realms.get(key(j.code, n)); return e && !e.loading && !e.broken && !e.lost; }); s.day = { j, order, i: 0, out: {} }; return; } if (j.t === 'unload') { for (const [k, e] of [...realms]) if (e.code === j.code && (j.nid === undefined || e.nid === j.nid)) realms.delete(k); if (![...realms.values()].some((e) => e.code === j.code)) sessions.delete(j.code); return reply(j, { t: 'unloaded', code: j.code, ...(j.nid !== undefined ? { nid: j.nid } : {}) }); } const e = realms.get(key(j.code, j.nid)); if (e && e.loading) { realms.delete(key(j.code, j.nid)); log('error', `[${j.nid}] marked loading with no load under way: dropped`); } if (!e || e.loading || !e.realm) throw Object.assign(new Error(`no realm ${key(j.code, j.nid)}`), { code: 'NO_REALM' }); const h = H[j.t]; if (!h) throw new Error(`unknown message ${j.t}`); running(j.code); return reply(j, h(j, e)); } // One step of a session's load (one network) or day (one network). A throw answers the whole call, never the worker. function runStep() { const ss = [...sessions.values()].filter((s) => s.load || s.day); if (!ss.length) return false; const s = ss[turn++ % ss.length]; if (s.load) { const L = s.load; try { const n = L.j.nets[L.i++]; if (n) { running(s.code); L.out.push(loadOne(L.j.code, n)); } if (L.i >= L.j.nets.length) { s.load = null; reply(L.j, L.out); } } catch (x) { s.load = null; for (const n of L.j.nets.slice(L.i)) { const e = n && realms.get(key(L.j.code, n.nid)); if (e && e.loading) realms.delete(key(L.j.code, n.nid)); } log('error', `mp-sim worker: a load of ${s.code} failed: ${MSG(x)}`); fail(L.j, x); } return true; } const D = s.day; try { const nid = D.order[D.i++]; const e = nid && realms.get(key(D.j.code, nid)); if (e && !e.broken && !e.lost && e.realm) { running(s.code); try { D.out[nid] = dayOne(e, D.j); } catch (x) { e.broken = true; log('error', `[${nid}] the day could not run: ${MSG(x)}`); D.out[nid] = { day: e.day, lost: !!e.lost, err: 'the day could not run: ' + LINE(x), retried: false, ms: 0 }; } } if (D.i >= D.order.length) { s.day = null; reply(D.j, { t: 'dayDone', code: D.j.code, d: D.j.d, nets: D.out }); } } catch (x) { s.day = null; log('error', `mp-sim worker: a day of ${s.code} failed: ${MSG(x)}`); fail(D.j, x); } return true; } /* Turns: a message and a step of a session's load or day, in alternation. The worker goes on only while something ran: what a job waits for (a load, a day, an earlier job of its session) is itself a job or a step that runs, so a pass where nothing could run means nothing will until a message comes (handle() kicks). If a job were still waiting then, nothing could ever free it: it is answered, not left pending, and the worker does not spin on it. */ let flip = false; function pump() { pumping = false; flip = !flip; try { if (flip ? runJob() || runStep() : runStep() || runJob()) return kick(); if (!jobs.length) return; const j = jobs.shift(); log('error', `mp-sim worker: ${j.t} for ${key(j.code, j.nid)} waited for nothing that could come; answered`); fail(j, new Error(`${j.t} could not run`)); } catch (x) { log('error', `mp-sim worker: ${MSG(x)}`); } kick(); } function kick() { if (!pumping) { pumping = true; setImmediate(pump); } } return { handle(m) { jobs.push(m); kick(); }, stats: () => ({ realms: realms.size, sessions: sessions.size, jobs: jobs.length }), simId: code.simId, }; } if (!isMainThread && workerData && workerData.mpSim) { const W = workerData.mpSim; const post = (m) => parentPort.postMessage(m); const log = (level, text) => post({ log: [level, text] }); process.on('unhandledRejection', (e) => log('warn', 'unhandled rejection in a realm: ' + MSG(e))); let core; try { // Tests only: a worker that cannot start (the engine backs off, then stops trying). if (W.test && W.test.startFail) throw new Error('injected: the game does not start'); core = createCore({ ...W, log, post }); } catch (e) { post({ fatal: MSG(e) }); process.exit(1); } parentPort.on('message', (m) => core.handle(m)); post({ ready: true, simId: core.simId }); } module.exports = { createCore, TIMEOUTS };