A network loaded from its checkpoint drifted from the one that never stopped:
after a checkpoint on day 1, 2 or 5 its R differed at once (bizDem at Ashburn
6426 against 26, then cityDem, citySrv, tNet), and from the next day S too
(cash, eyeball growth). Three causes, each shown by a probe:
- R after a day is the R of the day's last routing, made before the day's end
changed S (customers who left or signed, moods, bills, transit history) and
routed from the memory of the R before it. srvLoad routed the saved S again
from R's own memory: another R, which the next day reads (sfOf for eyeball
growth, the peering share). Every compute() in the realm now keeps, for the
R it makes, S as it routed it (one JSON per top-level key, 0.8 ms big) and
the memory it started from. rt carries that routing point (pt: the memory,
the keys that changed since, the keys S did not have then), and srvLoad
routes that S again from it: the same R, byte for byte. The old restore test
checkpointed after day 10, where a failure or event routed again at the end
of the day and nothing changed after it: it passed by luck.
- The sales desk's tried offers (UI.slTried, an offer that did not fit waits a
week) live outside S. A fresh realm forgot them and the desk tried other
offers. rt carries them (sl); a day mark keeps them and the in-place restore
after a throw puts them back (before, a parsed S dropped them).
- A save and a reload give some keys back in another key order (a customer's
defaults go last, the intern's week tally in its own order), and the week's
report lists the tally in that order. At a checkpoint rt carries the live
text of each such key (ord, srvGate's round trip, about 8 ms), taken on load
only where it holds what sanitize() made of the save. A fresh realm after a
timeout takes the mark's own text the same way (live).
The plain game drifts across a save and a reload too (jsdom: from the first
day after it; the page routes a reloaded S with an empty R and keeps no desk
memory): a game property, so the engine carries what is missing.
The envelope keeps its shape {s, uidN, rt, last, day}; rt keeps R's own route
memory, so an older checkpoint loads as before (close, not exact). Big network:
the checkpoint grows from about 42-49 KB gzip to 54-79 KB (rt 7-29 KB; the
whole R would be 375 KB), srvSave from 2-2.5 ms to 7-8 ms, a load by one
routing. A day with its mark costs the same (big 39.8 -> 37.8 ms, busy
90.0 -> 88.8 ms, within noise: the day mark joins per-key JSON now, which pays
for the routing point).
Tests: the restore test checkpoints after days 1, 2, 5 and 10 of the scenic big
network and of the same network busy (intern on every order, the desk signing
on its own, failures, events, an attack), those after days 2 and 10 right after
an action and a what-if (sent to the copies too), each compared byte for byte
at once and 10 days later (SIM_LONG: after every day to 30, 60 checkpoints). It
fails on the old engine, and without any one of the three parts. The retry test
plays busy networks (fails without the desk memory in srvRestore). The market
checkpoint compares bytes too; canon() is gone.
292 lines
16 KiB
JavaScript
292 lines
16 KiB
JavaScript
// A worker thread of the server-run engine (the server-sim design, section 4.2): it compiles the game once, holds the
|
|
// realms of the sessions placed on it (every network of a session in one worker), and answers the host's messages.
|
|
//
|
|
// Order. Messages run one at a time, in arrival order per network. A session's day is one task per network, in sorted
|
|
// network order, with a turn for other messages between two networks: an action for network X lands between days of
|
|
// other networks, never inside X's own day. The networks of a day are fixed when it starts. Loads, new networks and
|
|
// unloads of a session wait for its day; actions for a network that is still loading wait for it.
|
|
//
|
|
// Failures. Every realm call has a time limit (a day 2 s, an action 1 s, a what-if 3 s, a pull or a save 1 s). Before
|
|
// a day or an action the realm is marked (its S as text, uidN, the dice, and for a day its rt: route memory and routing
|
|
// point), and the worker keeps the mark. A call that throws has run its finally blocks: the realm goes back to the mark
|
|
// in place. A call that times out has not (vm skips them), so that realm is never used again: a fresh one is built from
|
|
// the mark, with a new epoch. A failed day runs once more; failing again, the network is broken at its state before
|
|
// the day.
|
|
//
|
|
// The worker → host replies are the shapes of tests/lib/mp-sim-shape.js checkWorker(), sent as {id, ok, reply} (or
|
|
// {id, ok:false, error}) for the engine to match its promises; logs go up as {log:[level, text]}.
|
|
'use strict';
|
|
const { isMainThread, parentPort, workerData } = require('node:worker_threads');
|
|
const crypto = require('node:crypto');
|
|
const v8 = require('node:v8');
|
|
const { compileCode, Realm, isTimeout } = require('./mp-sim-realm');
|
|
|
|
const TIMEOUTS = { day: 2000, act: 1000, ask: 3000, pull: 1000, save: 1000, mark: 1000, small: 1000, load: 30000 };
|
|
const now = () => performance.now();
|
|
const MSG = (e) => String((e && (e.stack || e.message)) || e).slice(0, 2000);
|
|
|
|
/* opts: {root, page, test: {seed, now, step} | true | null, strict, timeouts, log(level, text), post(message)}. */
|
|
function createCore(opts = {}) {
|
|
const code = compileCode({ root: opts.root, page: opts.page });
|
|
const T = { ...TIMEOUTS, ...(opts.timeouts || {}) };
|
|
const test = opts.test ? (typeof opts.test === 'object' ? opts.test : {}) : null;
|
|
const log = opts.log || (() => {});
|
|
const post = opts.post || (() => {});
|
|
const realms = new Map(), sessions = new Map(), jobs = [];
|
|
let epochN = 0, pumping = false, turn = 0;
|
|
const key = (c, n) => c + '/' + n;
|
|
const epochOf = () => crypto.randomBytes(4).toString('hex') + (++epochN).toString(36);
|
|
const session = (c) => { let s = sessions.get(c); if (!s) sessions.set(c, s = { code: c, day: null, load: null }); return s; };
|
|
|
|
/* ---- realms ---- */
|
|
function build(nid, lang) {
|
|
return new Realm({ code, lang, test: test && { seed: test.seed, now: test.now, step: test.step }, loadMs: T.load,
|
|
log: (lvl, txt) => log(lvl === 'log' || lvl === 'info' || lvl === 'debug' ? 'info' : 'warn', `[${nid}] ${txt}`) });
|
|
}
|
|
// A realm call: [header, ...parts]; a throw inside the realm comes back as {srvErr}, a timeout throws.
|
|
const call = (e, name, input, ms) => e.realm.call(name, input, ms);
|
|
/* A fresh realm in place of one that timed out, from {s, uidN, rt, live}: same network, new epoch, versions from 0. R
|
|
comes from the old realm, which still answers after a timeout: the marked R (mark) or its current one, packed whole.
|
|
When it cannot, R is routed again from rt's routing point. live: s is a mark, the realm's own S as text. */
|
|
function rebuild(e, from, mark = true) {
|
|
let rpack = null;
|
|
try { const s = e.realm && e.realm.raw('srvRPack', { mark }, T.load); if (typeof s === 'string' && s.startsWith('{"o"')) rpack = s; } catch (x) { /* gone */ }
|
|
const r = build(e.nid, e.lang), epoch = epochOf();
|
|
const [h] = r.call('srvLoad', { save: from.s, uidN: from.uidN, rt: from.rt, rpack, live: !!from.live, prefs: e.prefs, epoch, strict: !!opts.strict, clock: from.clock }, T.load);
|
|
if (h.srvErr) throw new Error('rebuilding the realm failed: ' + h.srvErr);
|
|
e.realm = r; e.epoch = epoch; e.v = 0; e.day = h.day; e.lost = h.lost;
|
|
log('warn', `[${e.nid}] realm rebuilt after a timeout (epoch ${epoch}${rpack ? '' : ', R routed again'})`);
|
|
}
|
|
// The rt of the old realm's mark, if it still answers; else the last day's, whose routing point belongs to that day's
|
|
// S: with another mark only its route memory counts (a close R).
|
|
function markRt(e) {
|
|
try { const [h] = call(e, 'srvRt', { mark: true }, 200); if (h && !h.srvErr && h.load) return h; } catch (x) { /* gone */ }
|
|
return e.dayMark ? (e.dayMark === e.mark ? e.dayMark.rt : { ...e.dayMark.rt, pt: undefined }) : null;
|
|
}
|
|
function mark(e, input) {
|
|
const [h, s] = call(e, 'srvMark', input, T.mark);
|
|
if (h.srvErr) throw new Error('mark failed: ' + h.srvErr);
|
|
const m = { s, uidN: h.uidN, rt: h.rt || null, day: h.day, clock: h.clock, live: true };
|
|
if (input.day) e.dayMark = m;
|
|
e.mark = m;
|
|
return m;
|
|
}
|
|
// After a throw: back to the mark in place; if even that fails, a fresh realm from the mark.
|
|
function restore(e, input) {
|
|
try { const [h] = call(e, 'srvRestore', input, T.small); if (!h.srvErr) return; log('warn', `[${e.nid}] restore failed: ${h.srvErr}`); }
|
|
catch (x) { log('warn', `[${e.nid}] restore timed out`); }
|
|
rebuild(e, { ...e.mark, rt: input.day ? e.mark.rt : markRt(e) });
|
|
}
|
|
// A read-only call that timed out: the realm's state is whole (nothing applied was interrupted), so the fresh realm
|
|
// starts from its save when it still answers, else from the last mark.
|
|
function rebuildAfterRead(e) {
|
|
let from = null;
|
|
try { const [h, txt] = call(e, 'srvSave', undefined, T.save); if (!h.srvErr) from = { s: txt, uidN: h.uidN, rt: h.rt, clock: h.clock }; } catch (x) { /* gone */ }
|
|
if (from) rebuild(e, from, false); else rebuild(e, { ...e.mark, rt: markRt(e) });
|
|
}
|
|
|
|
/* ---- the handlers: each returns the reply (or throws, which the engine turns into a rejection) ---- */
|
|
const errReply = (m, what, msg) => ({ t: 'err', code: m.code, nid: m.nid, ...(m.msg && m.msg.rid ? { rid: m.msg.rid } : {}), what, msg: String(msg).slice(0, 500) });
|
|
function loadOne(c, n) {
|
|
const e = realms.get(key(c, n.nid));
|
|
try {
|
|
e.realm = build(n.nid, n.lang);
|
|
e.lang = e.realm.lang;
|
|
const epoch = epochOf();
|
|
const [h] = n.begin
|
|
? call(e, 'srvBegin', { begin: n.begin, prefs: n.prefs, epoch, strict: !!opts.strict }, T.load)
|
|
: call(e, 'srvLoad', { save: n.env.s, uidN: n.env.uidN, rt: n.env.rt, prefs: n.prefs, rng: n.rng, epoch, strict: !!opts.strict }, T.load);
|
|
if (h.srvErr) throw new Error(h.srvErr);
|
|
Object.assign(e, { epoch, v: 0, day: h.day, lost: h.lost, loading: false });
|
|
const out = { t: 'loaded', nid: n.nid, epoch, day: h.day, lost: h.lost };
|
|
if (n.begin) out.saved = saved(e);
|
|
return out;
|
|
} catch (x) {
|
|
realms.delete(key(c, n.nid));
|
|
log('warn', `[${n.nid}] did not load: ${MSG(x)}`);
|
|
return { t: 'loaded', nid: n.nid, err: isTimeout(x) ? 'the network took too long to load' : String((x && x.message) || x).split('\n')[0].slice(0, 300) };
|
|
}
|
|
}
|
|
function saved(e) {
|
|
const [h, txt] = call(e, 'srvSave', undefined, T.save);
|
|
if (h.srvErr) throw new Error(h.srvErr);
|
|
return { t: 'saved', nid: e.nid, txt, uidN: h.uidN, rt: h.rt, day: h.day, lost: h.lost };
|
|
}
|
|
// One network's day, with its mark, one retry, and broken after a second failure.
|
|
function dayOne(e, m) {
|
|
const t0 = now(), d0 = e.day;
|
|
const input = { speed: m.speed, last: m.last, alone: (m.alone || []).includes(e.nid), mute: !!m.mute };
|
|
let retried = false, err = null;
|
|
for (let i = 0; i < 2; i++) {
|
|
try {
|
|
mark(e, { day: true });
|
|
const [h] = call(e, 'srvDay', input, T.day);
|
|
if (h.srvErr) throw Object.assign(new Error(h.srvErr), { inRealm: true });
|
|
e.day = h.day; e.lost = h.lost;
|
|
return { day: h.day, lost: h.lost, ...(h.pause ? { pause: true } : {}), ...(retried ? { retried: true } : {}), ms: +(now() - t0).toFixed(2) };
|
|
} catch (x) {
|
|
err = x; retried = true;
|
|
log('warn', `[${e.nid}] day ${d0 + 1} failed (${isTimeout(x) ? 'timeout' : 'throw'}${i ? ', again' : ''}): ${MSG(x)}`);
|
|
try { if (isTimeout(x) || !e.mark) rebuild(e, e.dayMark || e.mark); else restore(e, { day: true }); }
|
|
catch (y) { log('error', `[${e.nid}] could not go back before the day: ${MSG(y)}`); break; }
|
|
}
|
|
}
|
|
e.broken = true;
|
|
return { day: e.day, lost: !!e.lost, err: (isTimeout(err) ? 'timeout: ' : '') + String((err && err.message) || err).split('\n')[0].slice(0, 300), retried: true, ms: +(now() - t0).toFixed(2) };
|
|
}
|
|
const H = {
|
|
act(m, e) {
|
|
if (e.broken || e.lost) return { t: 'actDone', nid: m.nid, epoch: e.epoch, rid: m.msg.rid, outcome: 'stopped', v: e.v, upd: null };
|
|
try {
|
|
mark(e, { keys: true });
|
|
const [h, upd] = call(e, 'srvAct', m.msg, T.act);
|
|
if (h.srvErr) { restore(e, {}); return errReply(m, 'throw', h.srvErr); }
|
|
e.v = h.v;
|
|
if (h.outcome === 'stopped') e.lost = true;
|
|
return { t: 'actDone', nid: m.nid, epoch: e.epoch, rid: m.msg.rid, outcome: h.outcome, ...(h.why ? { why: h.why } : {}), v: h.v, upd: upd || null };
|
|
} catch (x) {
|
|
if (!isTimeout(x)) throw x;
|
|
rebuild(e, { ...e.mark, rt: markRt(e) });
|
|
return errReply(m, 'timeout', `the action took longer than ${T.act} ms`);
|
|
}
|
|
},
|
|
ask(m, e) {
|
|
if (e.broken || e.lost) return { t: 'askDone', nid: m.nid, rid: m.msg.rid, err: 'stopped' };
|
|
try {
|
|
mark(e, {});
|
|
const [h] = call(e, 'srvAsk', m.msg, T.ask);
|
|
if (h.srvErr) { restore(e, {}); return errReply(m, 'throw', h.srvErr); }
|
|
return { t: 'askDone', nid: m.nid, rid: m.msg.rid, ...h };
|
|
} catch (x) {
|
|
if (!isTimeout(x)) throw x;
|
|
rebuild(e, { ...e.mark, rt: markRt(e) });
|
|
return errReply(m, 'timeout', `the what-if took longer than ${T.ask} ms`);
|
|
}
|
|
},
|
|
pull(m, e) { return pulled(m, e, 'srvPull', { snap: !!m.snap, views: m.views }); },
|
|
greet(m, e) { return pulled(m, e, 'srvGreet', {}); },
|
|
sub(m, e) {
|
|
return guarded(m, e, () => { const [h] = call(e, 'srvSub', { views: m.views || [] }, T.pull); if (h.srvErr) return errReply(m, 'throw', h.srvErr); return { t: 'subbed', nid: m.nid, epoch: e.epoch, v: h.v, subs: h.subs }; });
|
|
},
|
|
save(m, e) { return guarded(m, e, () => saved(e)); },
|
|
card(m, e) { return guarded(m, e, () => { const [h] = call(e, 'srvCard', undefined, T.small); if (h.srvErr) return errReply(m, 'throw', h.srvErr); return { t: 'card', nid: m.nid, card: h.card || null }; }); },
|
|
prefs(m, e) { if (m.prefs) e.prefs = { ...e.prefs, ...m.prefs }; return guarded(m, e, () => { call(e, 'srvPrefs', { prefs: m.prefs }, T.small); return null; }); },
|
|
// Tests only: a piece of code in the realm, its value back.
|
|
eval(m, e) {
|
|
if (!test) throw new Error('eval is for tests');
|
|
try { return { value: e.realm.eval(m.src, m.ms || 5000) }; }
|
|
catch (x) { if (isTimeout(x)) { rebuildAfterRead(e); return { timeout: true, epoch: e.epoch }; } return { error: MSG(x) }; }
|
|
},
|
|
};
|
|
function pulled(m, e, name, input) {
|
|
return guarded(m, e, () => {
|
|
const [h, ...p] = call(e, name, input, T.pull);
|
|
if (h.srvErr) return errReply(m, 'throw', h.srvErr);
|
|
e.v = h.v;
|
|
const o = { t: 'pulled', nid: m.nid, epoch: e.epoch, v: h.v };
|
|
(h.parts || []).forEach((k, i) => { o[k] = p[i]; });
|
|
if (h.subs) o.subs = h.subs;
|
|
return o;
|
|
});
|
|
}
|
|
// A call that changes nothing applied: after a timeout the realm is built again from its own save.
|
|
function guarded(m, e, f) {
|
|
try { return f(); } catch (x) { if (!isTimeout(x)) throw x; rebuildAfterRead(e); return errReply(m, 'timeout', `${m.t} took too long`); }
|
|
}
|
|
|
|
/* ---- the queue ---- */
|
|
const SESSION_OPS = new Set(['load', 'unload', 'day']);
|
|
function runnable(j, i) {
|
|
const s = sessions.get(j.code);
|
|
if (SESSION_OPS.has(j.t)) {
|
|
if (s && (s.day || s.load)) return false;
|
|
for (let k = 0; k < i; k++) if (jobs[k].code === j.code) return false;
|
|
return true;
|
|
}
|
|
for (let k = 0; k < i; k++) if (jobs[k].code === j.code && SESSION_OPS.has(jobs[k].t) && jobs[k].t !== 'day') return false;
|
|
const e = realms.get(key(j.code, j.nid));
|
|
return !e || !e.loading;
|
|
}
|
|
function reply(j, r) { post({ id: j.id, ok: true, reply: r === undefined ? null : r }); }
|
|
function fail(j, x) { post({ id: j.id, ok: false, error: { message: String((x && x.message) || x), code: (x && x.code) || 'ERR' } }); }
|
|
function runJob() {
|
|
for (let i = 0; i < jobs.length; i++) {
|
|
const j = jobs[i];
|
|
if (!runnable(j, i)) continue;
|
|
jobs.splice(i, 1);
|
|
try { start(j); } catch (x) { fail(j, x); }
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
function start(j) {
|
|
if (j.t === 'mem') { const h = v8.getHeapStatistics(), mu = process.memoryUsage(); return reply(j, { heapUsed: h.used_heap_size, heapTotal: h.total_heap_size, external: h.external_memory, rss: mu.rss, realms: realms.size }); }
|
|
if (j.t === 'load') {
|
|
const s = session(j.code);
|
|
for (const n of j.nets) {
|
|
if (!n || typeof n.nid !== 'string' || (!n.begin && !(n.env && typeof n.env.s === 'string'))) throw new Error('load: each network needs a nid and an env with its save text, or begin');
|
|
realms.set(key(j.code, n.nid), { code: j.code, nid: n.nid, lang: n.lang || 'en', prefs: n.prefs || {}, loading: true, realm: null, epoch: '', v: 0, day: 0, lost: false, broken: false, mark: null, dayMark: null });
|
|
}
|
|
s.load = { j, i: 0, out: [] };
|
|
return;
|
|
}
|
|
if (j.t === 'day') {
|
|
const s = session(j.code);
|
|
const order = [...new Set(j.nets || [])].sort().filter((n) => { const e = realms.get(key(j.code, n)); return e && !e.loading && !e.broken && !e.lost; });
|
|
s.day = { j, order, i: 0, out: {} };
|
|
return;
|
|
}
|
|
if (j.t === 'unload') {
|
|
for (const [k, e] of [...realms]) if (e.code === j.code && (j.nid === undefined || e.nid === j.nid)) realms.delete(k);
|
|
if (![...realms.values()].some((e) => e.code === j.code)) sessions.delete(j.code);
|
|
return reply(j, { t: 'unloaded', code: j.code, ...(j.nid !== undefined ? { nid: j.nid } : {}) });
|
|
}
|
|
const e = realms.get(key(j.code, j.nid));
|
|
if (!e) throw Object.assign(new Error(`no realm ${key(j.code, j.nid)}`), { code: 'NO_REALM' });
|
|
const h = H[j.t];
|
|
if (!h) throw new Error(`unknown message ${j.t}`);
|
|
return reply(j, h(j, e));
|
|
}
|
|
// One step of a session's load (one network) or day (one network).
|
|
function runStep() {
|
|
const ss = [...sessions.values()].filter((s) => s.load || s.day);
|
|
if (!ss.length) return false;
|
|
const s = ss[turn++ % ss.length];
|
|
if (s.load) {
|
|
const L = s.load, n = L.j.nets[L.i++];
|
|
if (n) L.out.push(loadOne(L.j.code, n));
|
|
if (L.i >= L.j.nets.length) { s.load = null; reply(L.j, L.out); }
|
|
return true;
|
|
}
|
|
const D = s.day, nid = D.order[D.i++];
|
|
if (nid) { const e = realms.get(key(D.j.code, nid)); if (e && !e.broken && !e.lost) D.out[nid] = dayOne(e, D.j); }
|
|
if (D.i >= D.order.length) { s.day = null; reply(D.j, { t: 'dayDone', code: D.j.code, d: D.j.d, nets: D.out }); }
|
|
return true;
|
|
}
|
|
let flip = false;
|
|
function pump() {
|
|
pumping = false;
|
|
flip = !flip;
|
|
if (flip) { if (!runJob()) runStep(); } else if (!runStep()) runJob();
|
|
if (jobs.length || [...sessions.values()].some((s) => s.load || s.day)) kick();
|
|
}
|
|
function kick() { if (!pumping) { pumping = true; setImmediate(pump); } }
|
|
return {
|
|
handle(m) { jobs.push(m); kick(); },
|
|
stats: () => ({ realms: realms.size, sessions: sessions.size, jobs: jobs.length }),
|
|
simId: code.simId,
|
|
};
|
|
}
|
|
|
|
if (!isMainThread && workerData && workerData.mpSim) {
|
|
const W = workerData.mpSim;
|
|
const post = (m) => parentPort.postMessage(m);
|
|
const log = (level, text) => post({ log: [level, text] });
|
|
process.on('unhandledRejection', (e) => log('warn', 'unhandled rejection in a realm: ' + MSG(e)));
|
|
let core;
|
|
try { core = createCore({ ...W, log, post }); } catch (e) { post({ fatal: MSG(e) }); process.exit(1); }
|
|
parentPort.on('message', (m) => core.handle(m));
|
|
post({ ready: true, simId: core.simId });
|
|
}
|
|
|
|
module.exports = { createCore, TIMEOUTS };
|