'use strict'; /* deploy/update.sh against a scratch "server": a bare origin, a checkout standing in for /opt/upstream, and stub * systemctl / curl / npm / upstream-admin / sleep on the PATH. Nothing real is touched. It checks the script's logic * (order of steps, refusals, rollback, exit codes), not systemd, a real Node or a real network. */ const test = require('node:test'); const assert = require('node:assert/strict'); const fs = require('fs'); const os = require('os'); const path = require('path'); const { spawnSync } = require('child_process'); const SCRIPT = path.join(__dirname, '..', 'deploy', 'update.sh'); const haveGit = spawnSync('git', ['--version']).status === 0; const opts = { skip: !haveGit && 'git not installed' }; const STUBS = { systemctl: `echo "systemctl $* [tree=$(git -C "$UPSTREAM_DIR" rev-parse HEAD)]" >> "$STUB_DIR/calls.log" [ "$1" = start ] && [ -e "$STUB_DIR/restart_fail" ] && exit 1 [ "$1" = stop ] && [ -e "$STUB_DIR/stop_fail" ] && exit 1 if [ "$1" = start ] && [ -e "$STUB_DIR/kill_once" ]; then rm -f "$STUB_DIR/kill_once"; kill -TERM $PPID; fi exit 0`, curl: `echo "curl $*" >> "$STUB_DIR/calls.log" [ -e "$UPSTREAM_DIR/BROKEN" ] && exit 22 exit 0`, sleep: 'exit 0', npm: `echo "npm $* [cwd=$PWD]" >> "$STUB_DIR/calls.log" case $1 in ci) [ -e "$STUB_DIR/npm_fail" ] && exit 1 ;; test) [ -e "$PWD/TESTFAIL" ] && exit 1 ;; esac exit 0`, 'upstream-admin': `echo "upstream-admin $*" >> "$STUB_DIR/calls.log" case $1 in backup) [ -e "$STUB_DIR/backup_fail" ] && { echo "error: disk full" >&2; exit 1; } mkdir -p "$2"; f="$2/upstream-20261011T041500Z.db"; echo snap > "$f" echo "backup written: $f (5 bytes, verified)" ;; verify-backup) [ -e "$STUB_DIR/verify_fail" ] && exit 1; [ -f "$2" ] || exit 3 ;; esac exit 0`, }; const GIT_ENV = { GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null', GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@example.com', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@example.com' }; function git(cwd, ...args) { const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: Object.assign({}, process.env, GIT_ENV) }); assert.equal(r.status, 0, 'git ' + args.join(' ') + ': ' + r.stderr); return r.stdout.trim(); } // origin (bare) + a working clone `dev` to push from + `app` = the checkout on the "server" function fixture() { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'upstream-update-')); const p = (n) => path.join(root, n); for (const d of ['origin.git', 'dev', 'app', 'bin', 'stub']) fs.mkdirSync(p(d)); git(p('origin.git'), 'init', '--bare', '-b', 'main', '-q'); git(p('dev'), 'init', '-b', 'main', '-q'); git(p('dev'), 'remote', 'add', 'origin', p('origin.git')); fs.writeFileSync(p('dev/package.json'), '{"name":"x"}\n'); fs.writeFileSync(p('dev/package-lock.json'), '{"v":1}\n'); fs.writeFileSync(p('dev/.gitignore'), 'node_modules/\n'); fs.writeFileSync(p('dev/server.js'), '// v1\n'); git(p('dev'), 'add', '-A'); git(p('dev'), 'commit', '-q', '-m', 'v1'); git(p('dev'), 'push', '-q', 'origin', 'main'); git(root, 'clone', '-q', p('origin.git'), p('app')); fs.mkdirSync(p('app/node_modules')); for (const [name, body] of Object.entries(STUBS)) { fs.writeFileSync(p('bin/' + name), '#!/bin/sh\n' + body + '\n', { mode: 0o755 }); } const f = { root, app: p('app'), dev: p('dev'), stub: p('stub'), backups: p('backups'), old: git(p('app'), 'rev-parse', 'HEAD'), head: () => git(p('app'), 'rev-parse', 'HEAD'), calls: () => (fs.existsSync(p('stub/calls.log')) ? fs.readFileSync(p('stub/calls.log'), 'utf8').split('\n').filter(Boolean) : []), // a new commit on origin; files = { path: content | null } publish(files, msg) { for (const [n, c] of Object.entries(files)) { if (c === null) fs.rmSync(path.join(p('dev'), n), { force: true }); else { fs.mkdirSync(path.dirname(path.join(p('dev'), n)), { recursive: true }); fs.writeFileSync(path.join(p('dev'), n), c); } } git(p('dev'), 'add', '-A'); git(p('dev'), 'commit', '-q', '-m', msg || 'next'); git(p('dev'), 'push', '-q', 'origin', 'main'); return git(p('dev'), 'rev-parse', 'HEAD'); }, run(args, env) { const r = spawnSync('sh', [SCRIPT].concat(args || []), { encoding: 'utf8', env: Object.assign({}, process.env, GIT_ENV, { PATH: p('bin') + path.delimiter + process.env.PATH, STUB_DIR: p('stub'), UPSTREAM_DIR: p('app'), UPSTREAM_ADMIN: p('bin/upstream-admin'), UPSTREAM_ALLOW_NONROOT: '1', BACKUP_DIR: p('backups'), LOCK_DIR: p('lock'), UPSTREAM_ENV_FILE: p('none.env'), HEALTH_TIMEOUT: '2', }, env || {}), }); return { code: r.status, out: r.stdout, err: r.stderr, all: r.stdout + r.stderr }; }, }; return f; } const idx = (calls, prefix) => calls.findIndex((c) => c.startsWith(prefix)); test('already up to date: exit 0, no backup, no restart', opts, () => { const f = fixture(); const r = f.run(); assert.equal(r.code, 0, r.all); assert.match(r.out, /already up to date/); assert.deepEqual(f.calls(), []); assert.equal(fs.existsSync(path.join(f.root, 'lock')), false, 'lock released'); }); test('update: snapshot first, fast-forward, no npm ci when the lock is unchanged, restart, health check, old -> new printed', opts, () => { const f = fixture(); const next = f.publish({ 'server.js': '// v2\n' }, 'second'); const r = f.run(); assert.equal(r.code, 0, r.all); assert.equal(f.head(), next); assert.match(r.out, new RegExp(f.old.slice(0, 9) + ' -> ' + next.slice(0, 9))); assert.match(r.out, /second/, 'the commit subjects that went out are listed'); const c = f.calls(); assert.ok(idx(c, 'systemctl stop upstream') >= 0 && idx(c, 'upstream-admin backup') > idx(c, 'systemctl stop'), 'the snapshot comes from the stopped database'); assert.ok(idx(c, 'upstream-admin verify-backup') > idx(c, 'upstream-admin backup')); assert.ok(idx(c, 'systemctl start upstream') > idx(c, 'upstream-admin verify-backup'), 'start only after the snapshot'); assert.ok(idx(c, 'curl') > idx(c, 'systemctl start')); assert.match(c[idx(c, 'systemctl stop')], new RegExp('tree=' + f.old), 'mixed-code window: the service is stopped while the tree is still the OLD one'); assert.match(c[idx(c, 'systemctl start')], new RegExp('tree=' + next), '... and started only when the tree is the NEW one'); assert.equal(c.filter((x) => x.startsWith('systemctl')).length, 2, 'one stop, one start'); assert.equal(c.some((x) => x.startsWith('npm')), false, 'no npm ci, no npm test'); assert.match(c.find((x) => x.startsWith('upstream-admin backup')), new RegExp('backup ' + f.backups.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + ' --keep 14')); }); test('npm ci --omit=dev only when package-lock changed; npm test only with --test', opts, () => { const f = fixture(); f.publish({ 'package-lock.json': '{"v":2}\n' }); let r = f.run(); assert.equal(r.code, 0, r.all); assert.ok(f.calls().some((x) => x.startsWith('npm ci --omit=dev --ignore-scripts')), 'no install scripts as root'); assert.ok(idx(f.calls(), 'npm ci') > idx(f.calls(), 'systemctl stop') && idx(f.calls(), 'npm ci') < idx(f.calls(), 'systemctl start'), 'deps change only while the service is stopped'); assert.equal(f.calls().some((x) => x.startsWith('npm test')), false); const g = fixture(); g.publish({ 'server.js': '// v2\n' }); r = g.run(['--test']); assert.equal(r.code, 0, r.all); const c = g.calls(); assert.ok(idx(c, 'npm test') >= 0 && idx(c, 'npm test') < idx(c, 'systemctl stop'), 'tests run before the service is touched'); const cwd = (/\[cwd=(.*)\]$/.exec(c[idx(c, 'npm test')]) || [])[1]; assert.ok(cwd && !cwd.startsWith(g.app), 'tests run in an isolated copy, never in the live tree: ' + cwd); assert.equal(fs.existsSync(cwd), false, 'the copy is removed again'); assert.deepEqual(c.filter((x) => x.startsWith('npm ci')).length, 1, 'only the isolated copy got an npm ci (lock unchanged)'); assert.match(c[idx(c, 'npm ci')], /--ignore-scripts/); }); test('a dirty tree is refused before anything happens (tracked change and untracked file)', opts, () => { for (const dirty of [(f) => fs.writeFileSync(path.join(f.app, 'server.js'), '// hotfix\n'), (f) => fs.writeFileSync(path.join(f.app, 'stray.txt'), 'x')]) { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); dirty(f); const r = f.run(); assert.equal(r.code, 2); assert.match(r.err, /dirty/); assert.equal(f.head(), f.old); assert.deepEqual(f.calls(), []); } // ignored files (node_modules, data) are not dirt const g = fixture(); g.publish({ 'server.js': '// v2\n' }); fs.writeFileSync(path.join(g.app, 'node_modules', 'x.js'), '1'); assert.equal(g.run().code, 0); }); test('refuses: credentials in the origin URL, detached HEAD, a rewritten history, a second run at the same time', opts, () => { let f = fixture(); git(f.app, 'remote', 'set-url', 'origin', 'https://jvanvik:glpat-secret@git.example.com/jvanvik/Upstream.git'); let r = f.run(); assert.equal(r.code, 2); assert.match(r.err, /credentials/); assert.ok(!r.all.includes('glpat-secret'), 'the token is never echoed'); assert.deepEqual(f.calls(), []); f = fixture(); git(f.app, 'checkout', '-q', '--detach'); r = f.run(); assert.equal(r.code, 2); assert.match(r.err, /detached HEAD/); f = fixture(); // origin gets a different history (force push) git(f.dev, 'reset', '-q', '--hard', 'HEAD'); fs.writeFileSync(path.join(f.dev, 'server.js'), '// other\n'); git(f.dev, 'commit', '-q', '--amend', '-a', '-m', 'rewritten'); git(f.dev, 'push', '-q', '-f', 'origin', 'main'); r = f.run(); assert.equal(r.code, 2); assert.match(r.err, /not a fast-forward/); assert.equal(f.head(), f.old); assert.deepEqual(f.calls(), []); f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.mkdirSync(path.join(f.root, 'lock')); r = f.run(); assert.equal(r.code, 2); assert.match(r.err, /another update/); assert.equal(f.head(), f.old); }); test('--dry-run fetches and reports, changes nothing', opts, () => { const f = fixture(); f.publish({ 'server.js': '// v2\n', 'server/migrations/005_x.sql': 'select 1;\n' }, 'with a migration'); const r = f.run(['--dry-run']); assert.equal(r.code, 0, r.all); assert.match(r.out, /NEW MIGRATIONS.*005_x\.sql/); assert.match(r.out, /dry run/); assert.equal(f.head(), f.old); assert.deepEqual(f.calls(), []); }); test('no verified snapshot, no update', opts, () => { let f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.writeFileSync(path.join(f.stub, 'backup_fail'), ''); let r = f.run(); assert.equal(r.code, 2); assert.match(r.out, /no verified snapshot/); assert.equal(f.head(), f.old); assert.equal(f.calls().filter((x) => x.startsWith('systemctl')).map((x) => x.split(' ')[1]).join(), 'stop,start', 'stopped for the snapshot, started again when there is none'); assert.equal(f.calls().some((x) => x.startsWith('npm')), false); f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.writeFileSync(path.join(f.stub, 'verify_fail'), ''); r = f.run(); assert.equal(r.code, 2); assert.match(r.out, /no verified snapshot/); assert.equal(f.head(), f.old); }); test('new version does not answer /healthz: rolled back to the old rev, restarted, exit 1; migrations are called out with the snapshot', opts, () => { const f = fixture(); f.publish({ BROKEN: 'x', 'server/migrations/005_x.sql': 'select 1;\n' }, 'bad release'); const r = f.run(); assert.equal(r.code, 1, r.all); assert.match(r.out, /FAILED: no 200 from/); assert.match(r.out, /rolled back: .* answers \/healthz again/); assert.match(r.out, /migrations \(005_x\.sql \).*restore the snapshot .*upstream-20261011T041500Z\.db/); assert.equal(f.head(), f.old); assert.equal(fs.existsSync(path.join(f.app, 'BROKEN')), false, 'the tree is the old one again'); assert.equal(f.calls().filter((x) => x.startsWith('systemctl start')).length, 2, 'start for the new code, start for the old'); assert.equal(f.calls().filter((x) => x.startsWith('systemctl stop')).length, 2, 'the broken new process is stopped before the tree is put back'); assert.match(r.out, /005_x\.sql/); }); test('rollback across migration 005_site_garrison.sql warns about stranded garrisons', opts, () => { const f = fixture(); f.publish({ BROKEN: 'x', 'server/migrations/005_site_garrison.sql': 'select 1;\n' }, 'garrison'); const dry = f.run(['--dry-run']); assert.match(dry.out, /005_site_garrison\.sql adds garrisons in landing sites/); const r = f.run(); assert.equal(r.code, 1, r.all); assert.match(r.out, /site_stationed.*neither shows, nor powers, nor sends home/); }); test('rollback restores dependencies when the lock file had changed', opts, () => { const f = fixture(); f.publish({ BROKEN: 'x', 'package-lock.json': '{"v":2}\n' }); const r = f.run(); assert.equal(r.code, 1, r.all); assert.equal(f.calls().filter((x) => x.startsWith('npm ci --omit=dev')).length, 2, 'ci for the new lock, ci again for the old one'); assert.equal(f.head(), f.old); }); test('a failing npm ci or a red --test rolls back; a red suite never restarts the service for the new code', opts, () => { let f = fixture(); f.publish({ 'package-lock.json': '{"v":2}\n' }); fs.writeFileSync(path.join(f.stub, 'npm_fail'), ''); let r = f.run(); // npm ci fails for the new AND the old lock (the stub fails always), so the rollback itself cannot recover assert.equal(r.code, 3, r.all); assert.equal(f.calls().filter((x) => x.startsWith('systemctl start')).length, 0, 'no start on top of a failed recovery'); assert.match(r.out, /npm ci for the old code failed/); f = fixture(); f.publish({ TESTFAIL: 'x' }, 'red'); r = f.run(['--test']); assert.equal(r.code, 1, r.all); assert.match(r.err, /test suite .*failed/); assert.equal(f.head(), f.old); assert.equal(f.calls().some((x) => x.startsWith('systemctl')), false, 'the live service was not even stopped'); }); test('rollback that does not recover: exit 3 with the snapshot named', opts, () => { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.writeFileSync(path.join(f.stub, 'restart_fail'), ''); const r = f.run(); assert.equal(r.code, 3, r.all); assert.match(r.out, /ROLLBACK DID NOT RECOVER/); assert.match(r.out, /upstream-20261011T041500Z\.db/); assert.equal(f.head(), f.old); }); test('a SIGTERM after the merge puts the old rev back and restarts (the next run must not say "up to date" with the old process running)', opts, () => { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.writeFileSync(path.join(f.stub, 'kill_once'), ''); // the first systemctl restart sends TERM to the script const r = f.run(); assert.equal(r.code, 1, r.all); assert.match(r.out, /FAILED: interrupted by a signal/); assert.equal(f.head(), f.old); assert.equal(fs.existsSync(path.join(f.root, 'lock')), false, 'lock released'); assert.equal(f.calls().filter((x) => x.startsWith('systemctl start')).length, 2); }); test('PORT comes from the env file for the health check; the script is plain POSIX sh syntax', opts, () => { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.writeFileSync(path.join(f.root, 'none.env'), 'SIGNUP_CODE=x\nPORT="9090"\n'); assert.equal(f.run().code, 0); assert.ok(f.calls().some((x) => x.startsWith('curl') && x.includes('http://127.0.0.1:9090/healthz'))); assert.equal(spawnSync('sh', ['-n', SCRIPT]).status, 0); assert.equal(/^\s*(local|source|declare|typeset)\s|\[\[(?!:)|\[ [^\]]*==|<\(|\$\{[A-Za-z_]+(\/|,,|\^\^)/m.test(fs.readFileSync(SCRIPT, 'utf8').replace(/^\s*#.*$/gm, '')), false, 'no bashisms the usual way'); }); // --------------------------------------------------------------------------------------------------------------------------- // review fixes: tools up front, credentials check, mixed-code window, tests as the service user in an isolated copy const SYS_TOOLS = ['git', 'sh', 'sed', 'tr', 'head', 'tail', 'id', 'rm', 'rmdir', 'mkdir', 'env', 'tar', 'mktemp', 'cat', 'basename', 'dirname', 'ls', 'chmod', 'date', 'uname', 'grep', 'awk', 'mv', 'cp', 'ln', 'readlink', 'sort', 'cut', 'wc', 'xargs', 'true', 'false', 'test', 'expr', 'printf', 'ssh']; // a PATH holding the stubs plus a few real tools, and NOT curl (or whatever is left out) function restrictedPath(f, leaveOut) { const sys = path.join(f.root, 'sys'); fs.mkdirSync(sys, { recursive: true }); for (const t of SYS_TOOLS) { if (leaveOut.includes(t)) continue; const r = spawnSync('sh', ['-c', 'command -v ' + t], { encoding: 'utf8' }); const real = r.stdout.trim(); if (real && real.startsWith('/') && !fs.existsSync(path.join(sys, t))) fs.symlinkSync(real, path.join(sys, t)); } return path.join(f.root, 'bin') + path.delimiter + sys; } test('a missing tool (curl, npm, systemctl, the admin wrapper) is refused up front: exit 2, nothing stopped, no misleading rollback', opts, () => { for (const tool of ['curl', 'npm', 'systemctl', 'upstream-admin']) { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.rmSync(path.join(f.root, 'bin', tool)); const r = f.run([], { PATH: restrictedPath(f, []), UPSTREAM_ADMIN: tool === 'upstream-admin' ? 'no-such-admin' : path.join(f.root, 'bin', 'upstream-admin') }); assert.equal(r.code, 2, tool + ': ' + r.all); assert.match(r.err, new RegExp('missing on the PATH:.* ' + (tool === 'upstream-admin' ? 'no-such-admin' : tool)), tool); assert.ok(!/ROLLBACK DID NOT RECOVER|rolling back/.test(r.all), tool + ': no rollback story'); assert.equal(f.head(), f.old); assert.deepEqual(f.calls(), [], tool + ': nothing ran'); } }); test('the credential check only refuses user:password@ / token@ in an http(s) authority, not ssh:// logins or an @ in a path', opts, () => { const refused = ['https://jvanvik:glpat-secret@git.example.com/x.git', 'https://glpat-secret@git.example.com/x.git', 'http://u:p@127.0.0.1:1/x.git']; for (const url of refused) { const f = fixture(); git(f.app, 'remote', 'set-url', 'origin', url); const r = f.run(); assert.equal(r.code, 2, url); assert.match(r.err, /credentials/, url); assert.ok(!r.all.includes('secret'), 'never echoed'); } // allowed: the script gets past the check and only then fails on the (unreachable) network, never with "credentials" for (const url of ['ssh://git@127.0.0.1:1/jvanvik/Upstream.git', 'git@127.0.0.1:jvanvik/Upstream.git', 'https://127.0.0.1:1/a@b/c.git', 'https://127.0.0.1:1/jvanvik/Upstream.git']) { const f = fixture(); git(f.app, 'remote', 'set-url', 'origin', url); const r = f.run([], { GIT_SSH_COMMAND: 'false' }); assert.ok(!/credentials/.test(r.all), url + ': ' + r.all); assert.match(r.err, /git fetch origin main failed/, url); assert.equal(r.code, 1); } }); test('a SIGTERM while the service is stopped but before the merge starts it again on the untouched tree', opts, () => { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); // the admin wrapper (runs between stop and merge) sends TERM to the script fs.writeFileSync(path.join(f.root, 'bin', 'upstream-admin'), '#!/bin/sh\necho "upstream-admin $*" >> "$STUB_DIR/calls.log"\nkill -TERM $PPID\nsleep 0\nexit 1\n', { mode: 0o755 }); const r = f.run(); assert.equal(r.code, 1, r.all); assert.equal(f.head(), f.old, 'tree untouched'); const sys = f.calls().filter((x) => x.startsWith('systemctl')).map((x) => x.split(' ')[1]); assert.equal(sys.join(), 'stop,start', 'stopped, then started again: never left down'); assert.equal(fs.existsSync(path.join(f.root, 'lock')), false); }); test('a stop that fails refuses before anything is changed', opts, () => { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); fs.writeFileSync(path.join(f.stub, 'stop_fail'), ''); const r = f.run(); assert.equal(r.code, 1, r.all); assert.match(r.err, /systemctl stop upstream failed/); assert.equal(f.head(), f.old); assert.equal(f.calls().some((x) => x.startsWith('upstream-admin') || x.startsWith('npm')), false); }); test('--test as root runs npm ci / npm test as the service user (runuser) and never as root', opts, () => { const f = fixture(); f.publish({ 'server.js': '// v2\n' }); // pretend to be root: id -u answers 0, and runuser is a stub that records who it switches to and then runs the command fs.writeFileSync(path.join(f.root, 'bin', 'id'), '#!/bin/sh\n[ "$1" = "-u" ] && { echo 0; exit 0; }\nexec /usr/bin/id "$@"\n', { mode: 0o755 }); fs.writeFileSync(path.join(f.root, 'bin', 'runuser'), '#!/bin/sh\necho "runuser $*" >> "$STUB_DIR/calls.log"\n[ "$1" = "-u" ] || exit 9\nshift 2\n[ "$1" = "--" ] && shift\nexec "$@"\n', { mode: 0o755 }); fs.writeFileSync(path.join(f.root, 'bin', 'chown'), '#!/bin/sh\necho "chown $*" >> "$STUB_DIR/calls.log"\n', { mode: 0o755 }); const r = f.run(['--test'], { UPSTREAM_ALLOW_NONROOT: '0', UPSTREAM_SERVICE_USER: 'svcuser' }); assert.equal(r.code, 0, r.all); const c = f.calls(); assert.match(c[idx(c, 'runuser')], /^runuser -u svcuser -- env /); assert.ok(idx(c, 'chown') >= 0 && idx(c, 'chown') < idx(c, 'runuser'), 'the copy is handed to the service user first'); assert.ok(idx(c, 'runuser') < idx(c, 'systemctl stop'), 'before the service is touched'); assert.equal(c.filter((x) => x.startsWith('runuser')).length, 1); }); test('a red --test in the isolated copy leaves the live tree, the service and the snapshot dir alone', opts, () => { const f = fixture(); f.publish({ TESTFAIL: 'x' }, 'red'); const r = f.run(['--test']); assert.equal(r.code, 1, r.all); assert.equal(f.head(), f.old); assert.equal(fs.existsSync(path.join(f.app, 'TESTFAIL')), false); assert.equal(fs.existsSync(f.backups), false, 'no snapshot was needed'); });