#!/bin/sh # Update the running game from git, safely. Runs ON THE SERVER as root (or via sudo); see docs/DEPLOY.md section 8. # # sudo sh /opt/upstream/deploy/update.sh # fetch origin/main, back up, stop, fast-forward, start, health check # sudo sh /opt/upstream/deploy/update.sh --dry-run # fetch and show what would change, touch nothing else # sudo sh /opt/upstream/deploy/update.sh --test # also run `npm test` on the new code (isolated copy, service user) before stopping # # What it does, in this order (it stops at the first failure; once the service is stopped it always puts it back): # 0. needs git, curl, npm, systemctl and the admin wrapper on the PATH (plus runuser and tar with --test): missing = exit 2, nothing touched # 1. refuses on a dirty tree (any change to tracked files, or untracked files that are not git-ignored), on a checkout # that is not on the branch, and on an origin URL whose http(s) authority carries credentials (user:password@ or token@; # ssh:// and git@host: URLs are fine, they carry no secret in the URL) # 2. git fetch origin (no credentials, no prompt), refuses a history that is not a fast-forward # 3. nothing new: says so and exits 0 (no stop, no backup, no restart) # 4. only with --test: `git archive` of the NEW rev into a temp directory, `npm ci --omit=dev --ignore-scripts` and `npm test` there AS THE # SERVICE USER (never as root, never in /opt/upstream); the live service keeps running meanwhile. Red = exit 1, nothing was touched # 5. systemctl stop: from here the tree and the process never disagree (the old process lazily require()s modules, so it must not # keep serving while the tree changes under it). Downtime = steps 6-9 # 6. verified database snapshot taken from the stopped (final) database (upstream-admin backup + verify-backup); no snapshot, no # update: the service is started again, exit 2, nothing changed # 7. git merge --ff-only # 8. npm ci --omit=dev --ignore-scripts, only if package.json / package-lock.json changed (or node_modules is missing) # 9. systemctl start, then wait for /healthz (HEALTH_TIMEOUT seconds); on failure: systemctl stop, git reset --hard , # npm ci again if the lock file had changed, systemctl start, wait again # 10. prints old -> new revs and the commits that went out # # Exit codes: 0 updated (or already current), 1 failed and the old version runs again (or --test was red and nothing was touched), # 2 refused or usage error (nothing was changed), 3 failed and the old version does NOT answer either (act now: # journalctl -u upstream, restore the snapshot). # # An interrupt (Ctrl-C, SIGTERM) after the stop counts as a failure: before the merge the (untouched) old tree is simply started again, after # it the old rev is put back first. SIGKILL or a power cut mid-update cannot be caught: check `git status`/`git log -1` and the unit, then rerun or # `git reset --hard ` and `systemctl start upstream`. # # Not handled on purpose: a migration that already ran cannot be un-run by git. When the update contains new migrations the script says so and # prints the snapshot to restore (docs/DEPLOY.md section 7) if the rolled-back code does not start. Migration 005_site_garrison.sql in particular # adds site_stationed rows (units garrisoned in landing sites) that the old code does not know: it neither lists them nor charges their power # nor lets them go home. Rolling back across it is only clean while nobody has garrisoned yet; otherwise restore the snapshot (and lose what # players did since). # # Settings (environment, all optional): # UPSTREAM_DIR /opt/upstream the git checkout # UPSTREAM_UNIT upstream systemd unit name # UPSTREAM_ENV_FILE /etc/upstream/upstream.env read for PORT (HEALTH_URL default) # UPSTREAM_ADMIN upstream-admin the admin wrapper (runs as the service user) # UPSTREAM_SERVICE_USER upstream the user --test runs npm ci / npm test as (when run as root) # BACKUP_DIR /var/lib/upstream/backups where the pre-update snapshot goes # BACKUP_KEEP 14 snapshots kept by the prune that backup does # UPSTREAM_REMOTE origin git remote # UPSTREAM_BRANCH main git branch (the checkout must be on it) # HEALTH_URL http://127.0.0.1:$PORT/healthz (PORT from the env file, else 8080) # HEALTH_TIMEOUT 30 seconds to wait for /healthz after a restart # LOCK_DIR /run/upstream-update.lock stops two updates at once # UPSTREAM_ALLOW_NONROOT=1 skip the root check (tests only) set -eu umask 022 # files from git/npm must stay readable by the service user whatever root's umask is APP_DIR=${UPSTREAM_DIR:-/opt/upstream} UNIT=${UPSTREAM_UNIT:-upstream} SVC_USER=${UPSTREAM_SERVICE_USER:-upstream} ENV_FILE=${UPSTREAM_ENV_FILE:-/etc/upstream/upstream.env} ADMIN=${UPSTREAM_ADMIN:-upstream-admin} BACKUP_DIR=${BACKUP_DIR:-/var/lib/upstream/backups} BACKUP_KEEP=${BACKUP_KEEP:-14} REMOTE=${UPSTREAM_REMOTE:-origin} BRANCH=${UPSTREAM_BRANCH:-main} HEALTH_TIMEOUT=${HEALTH_TIMEOUT:-30} LOCK_DIR=${LOCK_DIR:-/run/upstream-update.lock} RUN_TESTS=0 DRY_RUN=0 say() { printf '%s\n' "update: $*"; } die() { printf '%s\n' "update: $*" >&2; exit "${2:-2}"; } while [ $# -gt 0 ]; do case $1 in --test) RUN_TESTS=1 ;; --dry-run|-n) DRY_RUN=1 ;; --timeout) [ $# -ge 2 ] || die "--timeout needs seconds"; HEALTH_TIMEOUT=$2; shift ;; -h|--help) sed -n '2,/^set -eu/p' "$0" | sed -e '/^set -eu/d' -e 's/^# \{0,1\}//'; exit 0 ;; *) die "unknown argument: $1 (try --help)" ;; esac shift done case $HEALTH_TIMEOUT in ''|*[!0-9]*) die "HEALTH_TIMEOUT must be a whole number of seconds" ;; esac # ---- 0. tools: a missing one must stop us BEFORE anything is stopped (without curl the health check can never pass, and the # script would roll back a perfectly good release and then claim the rollback failed) ----------------------------------- missing= for tool in git curl npm systemctl "$ADMIN"; do command -v "$tool" >/dev/null 2>&1 || missing="$missing $tool" done if [ "$RUN_TESTS" = 1 ]; then for tool in tar mktemp; do command -v "$tool" >/dev/null 2>&1 || missing="$missing $tool"; done if [ "${UPSTREAM_ALLOW_NONROOT:-0}" != 1 ]; then command -v runuser >/dev/null 2>&1 || missing="$missing runuser"; fi fi [ -z "$missing" ] || die "missing on the PATH:$missing. Install them (docs/DEPLOY.md section 3; the admin wrapper is deploy/upstream-admin) and rerun: nothing was touched" if [ -z "${HEALTH_URL:-}" ]; then PORT= if [ -r "$ENV_FILE" ]; then PORT=$(sed -n 's/^[[:space:]]*PORT=//p' "$ENV_FILE" | tail -n 1 | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'\$/\1/") fi case $PORT in ''|*[!0-9]*) PORT=8080 ;; esac HEALTH_URL=http://127.0.0.1:$PORT/healthz fi if [ "${UPSTREAM_ALLOW_NONROOT:-0}" != 1 ] && [ "$(id -u)" != 0 ]; then die "run as root (sudo): the checkout is root-owned and the unit is stopped and started" fi # ---- 1. preconditions --------------------------------------------------------------------------------------------- cd "$APP_DIR" 2>/dev/null || die "no such directory: $APP_DIR" git rev-parse --is-inside-work-tree >/dev/null 2>&1 || die "$APP_DIR is not a git checkout" url=$(git remote get-url "$REMOTE" 2>/dev/null) || die "git remote '$REMOTE' is not configured" # Only a secret IN the URL is refused: user:password@ or token@ in the authority of an http(s) URL. ssh://git@host/... and git@host:path # carry a login name, not a secret, and an '@' later in the path (https://example.org/a@b/c.git) is just a path. case $url in http://*|https://*) authority=${url#*://}; authority=${authority%%/*} case $authority in *@*) die "remote '$REMOTE' has credentials in its URL; use a plain https:// URL (no user, no token) or an ssh remote, then rerun" ;; esac ;; esac cur_branch=$(git symbolic-ref --quiet --short HEAD 2>/dev/null || true) [ "$cur_branch" = "$BRANCH" ] || die "checkout is on '${cur_branch:-a detached HEAD}', not '$BRANCH': fix it by hand (git checkout $BRANCH) before updating" dirty=$(git status --porcelain) if [ -n "$dirty" ]; then printf '%s\n' "$dirty" | head -n 10 >&2 die "working tree is dirty (above, first 10 lines): refusing to update. Commit, stash or remove the changes; they would be lost on rollback" fi if ! mkdir "$LOCK_DIR" 2>/dev/null; then die "another update is running (lock $LOCK_DIR); remove it only if you are sure none is"; fi # State for the traps. STOPPED=1: we stopped the unit and have not started it again yet. MERGED=1: the tree is the new rev. STOPPED=0 MERGED=0 TMP_DIR= on_exit() { rc=$? trap - EXIT if [ "$STOPPED" = 1 ]; then # an unexpected failure with the service down: never leave it down printf '%s\n' "update: ABORTED with $UNIT stopped (exit $rc): starting it. Check git log -1, git status and journalctl -u $UNIT" >&2 systemctl start "$UNIT" >/dev/null 2>&1 || printf '%s\n' "update: could not start $UNIT: start it by hand" >&2 fi [ -z "$TMP_DIR" ] || rm -rf "$TMP_DIR" rmdir "$LOCK_DIR" 2>/dev/null || true exit "$rc" } trap on_exit EXIT # Ctrl-C / SIGTERM / hangup: before the stop nothing has changed, so just leave. After it the service must come back: with the old tree # (not merged yet) or after putting the old rev back (merged). on_signal() { if [ "$MERGED" = 1 ]; then rollback "interrupted by a signal"; fi if [ "$STOPPED" = 1 ]; then say "interrupted by a signal before the merge: the tree is untouched, starting $UNIT again" STOPPED=0 systemctl start "$UNIT" || say "could not start $UNIT: start it by hand" exit 1 fi exit 130 } trap on_signal INT TERM HUP # ---- 2/3. fetch, decide --------------------------------------------------------------------------------------------- old=$(git rev-parse HEAD) GIT_TERMINAL_PROMPT=0 git fetch --quiet "$REMOTE" "$BRANCH" || die "git fetch $REMOTE $BRANCH failed (network? URL: $url)" 1 new=$(git rev-parse FETCH_HEAD) short() { git rev-parse --short=9 "$1"; } if [ "$old" = "$new" ]; then say "already up to date at $(short "$old")" exit 0 fi if ! git merge-base --is-ancestor "$old" "$new"; then die "$REMOTE/$BRANCH ($(short "$new")) is not a fast-forward of the checkout ($(short "$old")): history was rewritten or the checkout has local commits. Not touching anything." fi lock_changed=0 git diff --quiet "$old" "$new" -- package.json package-lock.json || lock_changed=1 [ -d node_modules ] || lock_changed=1 migrations=$(git diff --name-only "$old" "$new" -- server/migrations | sed 's|^server/migrations/||' | tr '\n' ' ') say "$(short "$old") -> $(short "$new"): $(git rev-list --count "$old..$new") commit(s)" git log --oneline --no-decorate "$old..$new" | head -n 20 | sed 's/^/update: /' [ "$lock_changed" = 0 ] || say "dependencies changed: npm ci --omit=dev --ignore-scripts will run" [ -z "$migrations" ] || say "NEW MIGRATIONS in this update: $migrations(they run at the first start and cannot be undone by a rollback)" case " $migrations" in *" 005_site_garrison.sql "*) say " 005_site_garrison.sql adds garrisons in landing sites (site_stationed): the previous code cannot see those units, so once anybody has garrisoned a site a code-only rollback strands them; the snapshot is the way back" ;; esac if [ "$DRY_RUN" = 1 ]; then say "dry run: nothing changed (the fetched commits are in .git, the tree and the service are untouched)" exit 0 fi # ---- helpers -------------------------------------------------------------------------------------------------------- wait_healthy() { # seconds -> 0 as soon as /healthz answers 200 _n=0 while :; do if curl -fsS --max-time 3 -o /dev/null "$HEALTH_URL" 2>/dev/null; then return 0; fi [ "$_n" -lt "$1" ] || return 1 _n=$((_n + 1)) sleep 1 done } install_deps() { # --ignore-scripts: no package install script (ours or a dependency's) runs as root; ws needs none say "npm ci --omit=dev --ignore-scripts" npm ci --omit=dev --ignore-scripts --no-audit --no-fund } as_service_user() { if [ "$(id -u)" = 0 ]; then runuser -u "$SVC_USER" -- "$@"; else "$@"; fi } rollback() { # $1 = why. Puts the old code back, starts it, checks. Never returns. MERGED=0 say "FAILED: $1" say "rolling back to $(short "$old")" systemctl stop "$UNIT" || say "systemctl stop $UNIT failed (the new code may still be running)" STOPPED=1 git reset --hard --quiet "$old" rb=0 if [ "$lock_changed" = 1 ]; then install_deps || { say "npm ci for the old code failed"; rb=1; }; fi if [ "$rb" = 0 ] && systemctl start "$UNIT" && wait_healthy "$HEALTH_TIMEOUT"; then STOPPED=0 say "rolled back: $(short "$old") answers /healthz again" if [ -n "$migrations" ]; then say "this update had migrations ($migrations): the old code runs on the NEWER schema. If it misbehaves, stop the unit and restore the snapshot $snap (docs/DEPLOY.md section 7)" case " $migrations" in *" 005_site_garrison.sql "*) say "005_site_garrison.sql: units that players garrisoned in landing sites while the new code ran are rows in site_stationed that the old code neither shows, nor powers, nor sends home. If any site was garrisoned in that window, restore the snapshot instead (you lose what happened since it was taken)" ;; esac fi exit 1 fi STOPPED=0 # reported; do not auto-start from the exit trap on top of a failed recovery say "ROLLBACK DID NOT RECOVER the service. Look: journalctl -u $UNIT -n 50" say "last resort: systemctl stop $UNIT, restore $snap (docs/DEPLOY.md section 7), start $UNIT" exit 3 } # ---- 4. optional tests, on an isolated copy of the NEW rev, as the service user, while the old version keeps serving ----------- if [ "$RUN_TESTS" = 1 ]; then TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/upstream-update-test.XXXXXX") || die "mktemp failed, nothing was touched" git archive "$new" | tar -x -C "$TMP_DIR" || die "could not extract $(short "$new") into $TMP_DIR, nothing was touched" 1 [ "$(id -u)" != 0 ] || chown -R "$SVC_USER" "$TMP_DIR" say "npm ci + npm test on $(short "$new") in $TMP_DIR as ${SVC_USER} (the live service is not touched)" if ! as_service_user env -u DB_PATH -u PORT HOME="$TMP_DIR" npm_config_cache="$TMP_DIR/.npm-cache" \ sh -c 'cd "$1" && npm ci --omit=dev --ignore-scripts --no-audit --no-fund && npm test --silent' sh "$TMP_DIR"; then die "the test suite (or npm ci) failed on the new rev: nothing was stopped, changed or restarted" 1 fi rm -rf "$TMP_DIR"; TMP_DIR= fi # ---- 5. stop: from here on the tree and the process never disagree ------------------------------------------------------ say "stopping $UNIT" systemctl stop "$UNIT" || die "systemctl stop $UNIT failed: tree untouched, the service state is unknown (systemctl status $UNIT)" 1 STOPPED=1 # ---- 6. verified backup, from the final (stopped) database --------------------------------------------------------- say "taking a verified snapshot" snap= if bk_out=$("$ADMIN" backup "$BACKUP_DIR" --keep "$BACKUP_KEEP"); then printf '%s\n' "$bk_out" | sed 's/^/update: /' snap=$(printf '%s\n' "$bk_out" | sed -n 's/^backup written: \(.*\) ([0-9]* bytes, verified)$/\1/p' | head -n 1) fi if [ -z "$snap" ] || ! "$ADMIN" verify-backup "$snap" >/dev/null; then STOPPED=0 say "no verified snapshot (backup failed, its output was not readable, or it does not verify): update NOT started, nothing was changed; starting $UNIT again" systemctl start "$UNIT" && wait_healthy "$HEALTH_TIMEOUT" || { say "$UNIT does not answer after the restart: journalctl -u $UNIT -n 50"; exit 3; } exit 2 fi say "snapshot ok: $snap" # ---- 7/8. new code, deps ------------------------------------------------------------------------------------------ if ! git merge --ff-only --quiet "$new"; then STOPPED=0 say "fast-forward failed; tree unchanged, starting $UNIT again" systemctl start "$UNIT" && wait_healthy "$HEALTH_TIMEOUT" || { say "$UNIT does not answer after the restart: journalctl -u $UNIT -n 50"; exit 3; } exit 1 fi MERGED=1 say "checked out $(short "$new")" if [ "$lock_changed" = 1 ]; then install_deps || rollback "npm ci failed"; fi # ---- 9. start and look ------------------------------------------------------------------------------------------- say "starting $UNIT" systemctl start "$UNIT" || rollback "systemctl start $UNIT failed" wait_healthy "$HEALTH_TIMEOUT" || rollback "no 200 from $HEALTH_URL within ${HEALTH_TIMEOUT}s" MERGED=0 STOPPED=0 # ---- 10. done ----------------------------------------------------------------------------------------------------- say "OK: $(short "$old") -> $(short "$new"), $UNIT answers $HEALTH_URL" say "snapshot from before the update: $snap" exit 0