Faster tests, split everywhere: one sharded CI for pushes and releases

- The harness builds the DOM without the page's 2 MB of script text and
  runs the scripts straight in the window: a page load in tests takes
  about half the time (jsdom's parser was slow on the script text).
- That also took the game's source out of the DOM, which showed three
  tests that matched a phrase in the page source rather than the screen;
  they now check the panel, the routing window and the dictionary.
- CI plans its split once and runs the tests on 6 runners; every run on
  main records each file's time and caches it, so the parts stay even
  without anyone refreshing tests/durations.json.
- auto-release.yml and release.yml call the same sharded tests instead
  of running all 170 files on one runner each (11 and 15 min); a release
  is tested once, before it is tagged.
- Audit: five tests that duplicated a gameplay rule or checked nothing
  are gone; the two slowest checks sample a third of the cities, and a
  year of play to reach one due date became a moved due date.
This commit is contained in:
Frederic Jung 2026-11-07 07:24:09 +01:00
parent 7a6eb29013
commit 0342d90657
No known key found for this signature in database
17 changed files with 244 additions and 109 deletions

View file

@ -20,18 +20,19 @@ concurrency:
cancel-in-progress: false
jobs:
release:
check:
runs-on: ubuntu-latest
env:
GH_TOKEN: ${{ github.token }}
outputs:
skip: ${{ steps.v.outputs.skip }}
ver: ${{ steps.v.outputs.ver }}
tag: ${{ steps.v.outputs.tag }}
pre: ${{ steps.v.outputs.pre }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 24
cache: npm
- name: Work out the version and whether it is new
id: v
@ -57,34 +58,40 @@ jobs:
case "$ver" in *-*) echo "pre=true" >> "$GITHUB_OUTPUT";; *) echo "pre=false" >> "$GITHUB_OUTPUT";; esac
echo "skip=false" >> "$GITHUB_OUTPUT"
- name: Test before releasing
if: steps.v.outputs.skip == 'false'
run: npm ci && npm test
# The same tests as every push, split over several runners (ci.yml), before anything is tagged.
tests:
needs: check
if: needs.check.outputs.skip == 'false'
uses: ./.github/workflows/ci.yml
release:
needs: [check, tests]
if: needs.check.outputs.skip == 'false'
runs-on: ubuntu-latest
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ needs.check.outputs.tag }}
VER: ${{ needs.check.outputs.ver }}
PRE: ${{ needs.check.outputs.pre }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Publish the release
if: steps.v.outputs.skip == 'false'
env:
TAG: ${{ steps.v.outputs.tag }}
VER: ${{ steps.v.outputs.ver }}
PRE: ${{ steps.v.outputs.pre }}
run: |
set -euo pipefail
awk -v v="$VER" 'index($0,"## ["v"]")==1{f=1;next} /^## \[/{f=0} f' CHANGELOG.md | sed -e '/./,$!d' > notes.md
pre=(); [ "$PRE" = "true" ] && pre=(--prerelease)
gh release create "$TAG" --repo "$GITHUB_REPOSITORY" \
--target "$GITHUB_SHA" --title "$VER" --notes-file notes.md "${pre[@]}"
- name: Deploy it
# Pre-releases are for testing and never go live.
if: steps.v.outputs.skip == 'false' && steps.v.outputs.pre == 'false'
env:
TAG: ${{ steps.v.outputs.tag }}
run: gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f tag="$TAG"
# Pre-releases are for testing and never go live. The tag was just tested above, so the deploy does not
# test it again (tested=true).
if: env.PRE == 'false'
run: gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f tag="$TAG" -f tested=true
- name: Publish the Docker image
# Same pattern as the deploy above: the image is built and pushed from the
# release tag by docker.yml. Pre-releases get one too: an image tag is
# pulled on purpose, so a test image costs nothing.
if: steps.v.outputs.skip == 'false'
env:
TAG: ${{ steps.v.outputs.tag }}
run: gh workflow run docker.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f tag="$TAG"

View file

@ -1,44 +1,89 @@
name: CI
# Every push and PR, and the releases too (auto-release.yml and release.yml call this workflow), so the tests run
# one way, split over several runners, everywhere.
on:
push:
branches: [main]
pull_request:
workflow_call:
inputs:
ref:
description: The commit or tag to test (a release tag); the caller's commit when empty
type: string
default: ''
permissions:
contents: read
# A newer push to the same pull request makes the older run pointless: cancel it instead of queueing behind it.
# Runs on main always finish.
# Runs on main always finish. A release's call gets a group of its own, so it never waits behind main's push run.
concurrency:
group: ci-${{ github.ref }}
group: ci-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}${{ inputs.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
# Runners for the tests. Standard GitHub-hosted runners (4 cores each) are the only free ones for a public repo,
# and a run may use up to 20 at once, so more parts is how the tests get faster. Each part costs ~30 s of setup.
PARTS: 6
jobs:
# The tests in four parts of about equal time, each on its own runner, each running as many files at once as
# it has cores (scripts/test.js; the split comes from tests/durations.json).
# Splits the test files into parts of about equal time, once, so every part works from the same plan. The times
# come from the last run on main (the cache, see "timings" below), else from the committed tests/durations.json.
plan:
runs-on: ubuntu-latest
outputs:
parts: ${{ steps.plan.outputs.parts }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ inputs.ref }}
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: tests/durations.json
key: test-durations-${{ github.run_id }}
restore-keys: test-durations-
- id: plan
run: echo "parts=$(node scripts/test.js --plan "$PARTS")" >> "$GITHUB_OUTPUT"
part:
name: test ${{ matrix.part }}/4
name: part ${{ strategy.job-index }}
needs: plan
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
part: [1, 2, 3, 4]
files: ${{ fromJSON(needs.plan.outputs.parts) }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ inputs.ref }}
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 24
cache: npm
- run: npm ci
- run: npm test -- --shard ${{ matrix.part }}/4
- name: Tests
env:
FILES: ${{ matrix.files }}
# Word splitting on purpose: FILES is a space-separated list of tests/*.test.js paths.
run: npm test -- --timings "$RUNNER_TEMP/timings.json" $FILES
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
name: timings-${{ strategy.job-index }}
path: ${{ runner.temp }}/timings.json
retention-days: 1
if-no-files-found: ignore
build:
name: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ inputs.ref }}
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 24
@ -47,21 +92,12 @@ jobs:
- name: Build (not deployed — only releases deploy)
run: npm run build
# The one check to look at, named as before: green only when every part and the build are.
test:
needs: [part, build]
if: always()
runs-on: ubuntu-latest
steps:
- name: Every part and the build passed
run: |
echo "parts: ${{ needs.part.result }}, build: ${{ needs.build.result }}"
test "${{ needs.part.result }}" = success && test "${{ needs.build.result }}" = success
translations:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ inputs.ref }}
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 24
@ -69,3 +105,38 @@ jobs:
- run: npm ci
- name: Translation coverage
run: node scripts/i18n-check.js --summary "$GITHUB_STEP_SUMMARY"
# The one check to look at: green only when the plan, every part, the build and the translations are.
test:
needs: [plan, part, build, translations]
if: always()
runs-on: ubuntu-latest
steps:
- name: Everything passed
run: |
echo "plan: ${{ needs.plan.result }}, parts: ${{ needs.part.result }}, build: ${{ needs.build.result }}, translations: ${{ needs.translations.result }}"
test "${{ needs.plan.result }}" = success && test "${{ needs.part.result }}" = success
test "${{ needs.build.result }}" = success && test "${{ needs.translations.result }}" = success
# After a run on main: each file's time from this run goes into the cache, so the next run (main or a PR) splits
# its parts by today's tests. Nobody has to remember to refresh tests/durations.json.
timings:
needs: part
if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.part.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: tests/durations.json
key: test-durations-${{ github.run_id }}
restore-keys: test-durations-
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: timings-*
path: ${{ runner.temp }}/timings
- run: node scripts/test-timings.js --merge "$RUNNER_TEMP/timings"
- uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: tests/durations.json
key: test-durations-${{ github.run_id }}

View file

@ -23,6 +23,10 @@ on:
description: Only the weekly puzzles changed (weekly-puzzle.yml); the release itself was tested when it shipped
type: boolean
default: false
tested:
description: The tag's tests already passed (auto-release.yml tests it before tagging)
type: boolean
default: false
# Least privilege: only the deploy job can write to Pages.
permissions:
@ -33,9 +37,18 @@ concurrency:
cancel-in-progress: false
jobs:
# The release's tests, split over several runners like every push (ci.yml). Skipped when auto-release.yml has just
# run them on this tag, and for a weekly redeploy (the build step below tests the weekly files against the tag).
tests:
if: ${{ (github.event_name == 'workflow_dispatch' || !github.event.release.prerelease) && !inputs.weekly && !inputs.tested }}
uses: ./.github/workflows/ci.yml
with:
ref: ${{ github.event.release.tag_name || inputs.tag }}
build:
needs: tests
# Pre-releases are for testing and never go live.
if: ${{ github.event_name == 'workflow_dispatch' || !github.event.release.prerelease }}
if: ${{ always() && (github.event_name == 'workflow_dispatch' || !github.event.release.prerelease) && (needs.tests.result == 'success' || needs.tests.result == 'skipped') }}
runs-on: ubuntu-latest
env:
TAG: ${{ github.event.release.tag_name || inputs.tag }}
@ -93,10 +106,9 @@ jobs:
node scripts/weekly-puzzle.js --check
if [ -f scripts/weekly-scenario.js ]; then node scripts/weekly-scenario.js --check; fi
if [ -f scripts/weekly-architect.js ]; then node scripts/weekly-architect.js --check; fi
- run: npm test
if: ${{ !inputs.weekly }}
# The weekly files come from main, not the tag: whatever tested the tag, they are tested against it here.
- name: Test the weekly puzzles against this release
if: ${{ inputs.weekly && hashFiles('scripts/weekly-puzzle.js') != '' }}
if: ${{ hashFiles('scripts/weekly-puzzle.js') != '' }}
run: npm test -- tests/weekly-puzzle.test.js tests/puzzles.test.js $(ls tests/weekly-scenario.test.js tests/weekly-architect.test.js 2>/dev/null)
- name: Build site
run: VERSION="${TAG#v}" REQUIRE_NOTES=1 npm run build