Faster tests, split everywhere: one sharded CI for pushes and releases
- The harness builds the DOM without the page's 2 MB of script text and runs the scripts straight in the window: a page load in tests takes about half the time (jsdom's parser was slow on the script text). - That also took the game's source out of the DOM, which showed three tests that matched a phrase in the page source rather than the screen; they now check the panel, the routing window and the dictionary. - CI plans its split once and runs the tests on 6 runners; every run on main records each file's time and caches it, so the parts stay even without anyone refreshing tests/durations.json. - auto-release.yml and release.yml call the same sharded tests instead of running all 170 files on one runner each (11 and 15 min); a release is tested once, before it is tagged. - Audit: five tests that duplicated a gameplay rule or checked nothing are gone; the two slowest checks sample a third of the cities, and a year of play to reach one due date became a moved due date.
This commit is contained in:
parent
7a6eb29013
commit
0342d90657
17 changed files with 244 additions and 109 deletions
49
.github/workflows/auto-release.yml
vendored
49
.github/workflows/auto-release.yml
vendored
|
|
@ -20,18 +20,19 @@ concurrency:
|
|||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
outputs:
|
||||
skip: ${{ steps.v.outputs.skip }}
|
||||
ver: ${{ steps.v.outputs.ver }}
|
||||
tag: ${{ steps.v.outputs.tag }}
|
||||
pre: ${{ steps.v.outputs.pre }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
|
||||
- name: Work out the version and whether it is new
|
||||
id: v
|
||||
|
|
@ -57,34 +58,40 @@ jobs:
|
|||
case "$ver" in *-*) echo "pre=true" >> "$GITHUB_OUTPUT";; *) echo "pre=false" >> "$GITHUB_OUTPUT";; esac
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Test before releasing
|
||||
if: steps.v.outputs.skip == 'false'
|
||||
run: npm ci && npm test
|
||||
# The same tests as every push, split over several runners (ci.yml), before anything is tagged.
|
||||
tests:
|
||||
needs: check
|
||||
if: needs.check.outputs.skip == 'false'
|
||||
uses: ./.github/workflows/ci.yml
|
||||
|
||||
release:
|
||||
needs: [check, tests]
|
||||
if: needs.check.outputs.skip == 'false'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.check.outputs.tag }}
|
||||
VER: ${{ needs.check.outputs.ver }}
|
||||
PRE: ${{ needs.check.outputs.pre }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Publish the release
|
||||
if: steps.v.outputs.skip == 'false'
|
||||
env:
|
||||
TAG: ${{ steps.v.outputs.tag }}
|
||||
VER: ${{ steps.v.outputs.ver }}
|
||||
PRE: ${{ steps.v.outputs.pre }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
awk -v v="$VER" 'index($0,"## ["v"]")==1{f=1;next} /^## \[/{f=0} f' CHANGELOG.md | sed -e '/./,$!d' > notes.md
|
||||
pre=(); [ "$PRE" = "true" ] && pre=(--prerelease)
|
||||
gh release create "$TAG" --repo "$GITHUB_REPOSITORY" \
|
||||
--target "$GITHUB_SHA" --title "$VER" --notes-file notes.md "${pre[@]}"
|
||||
|
||||
- name: Deploy it
|
||||
# Pre-releases are for testing and never go live.
|
||||
if: steps.v.outputs.skip == 'false' && steps.v.outputs.pre == 'false'
|
||||
env:
|
||||
TAG: ${{ steps.v.outputs.tag }}
|
||||
run: gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f tag="$TAG"
|
||||
# Pre-releases are for testing and never go live. The tag was just tested above, so the deploy does not
|
||||
# test it again (tested=true).
|
||||
if: env.PRE == 'false'
|
||||
run: gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f tag="$TAG" -f tested=true
|
||||
|
||||
- name: Publish the Docker image
|
||||
# Same pattern as the deploy above: the image is built and pushed from the
|
||||
# release tag by docker.yml. Pre-releases get one too: an image tag is
|
||||
# pulled on purpose, so a test image costs nothing.
|
||||
if: steps.v.outputs.skip == 'false'
|
||||
env:
|
||||
TAG: ${{ steps.v.outputs.tag }}
|
||||
run: gh workflow run docker.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f tag="$TAG"
|
||||
|
|
|
|||
107
.github/workflows/ci.yml
vendored
107
.github/workflows/ci.yml
vendored
|
|
@ -1,44 +1,89 @@
|
|||
name: CI
|
||||
|
||||
# Every push and PR, and the releases too (auto-release.yml and release.yml call this workflow), so the tests run
|
||||
# one way, split over several runners, everywhere.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: The commit or tag to test (a release tag); the caller's commit when empty
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# A newer push to the same pull request makes the older run pointless: cancel it instead of queueing behind it.
|
||||
# Runs on main always finish.
|
||||
# Runs on main always finish. A release's call gets a group of its own, so it never waits behind main's push run.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
group: ci-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}${{ inputs.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
# Runners for the tests. Standard GitHub-hosted runners (4 cores each) are the only free ones for a public repo,
|
||||
# and a run may use up to 20 at once, so more parts is how the tests get faster. Each part costs ~30 s of setup.
|
||||
PARTS: 6
|
||||
|
||||
jobs:
|
||||
# The tests in four parts of about equal time, each on its own runner, each running as many files at once as
|
||||
# it has cores (scripts/test.js; the split comes from tests/durations.json).
|
||||
# Splits the test files into parts of about equal time, once, so every part works from the same plan. The times
|
||||
# come from the last run on main (the cache, see "timings" below), else from the committed tests/durations.json.
|
||||
plan:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
parts: ${{ steps.plan.outputs.parts }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: tests/durations.json
|
||||
key: test-durations-${{ github.run_id }}
|
||||
restore-keys: test-durations-
|
||||
- id: plan
|
||||
run: echo "parts=$(node scripts/test.js --plan "$PARTS")" >> "$GITHUB_OUTPUT"
|
||||
|
||||
part:
|
||||
name: test ${{ matrix.part }}/4
|
||||
name: part ${{ strategy.job-index }}
|
||||
needs: plan
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
part: [1, 2, 3, 4]
|
||||
files: ${{ fromJSON(needs.plan.outputs.parts) }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm test -- --shard ${{ matrix.part }}/4
|
||||
- name: Tests
|
||||
env:
|
||||
FILES: ${{ matrix.files }}
|
||||
# Word splitting on purpose: FILES is a space-separated list of tests/*.test.js paths.
|
||||
run: npm test -- --timings "$RUNNER_TEMP/timings.json" $FILES
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: timings-${{ strategy.job-index }}
|
||||
path: ${{ runner.temp }}/timings.json
|
||||
retention-days: 1
|
||||
if-no-files-found: ignore
|
||||
|
||||
build:
|
||||
name: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: 24
|
||||
|
|
@ -47,21 +92,12 @@ jobs:
|
|||
- name: Build (not deployed — only releases deploy)
|
||||
run: npm run build
|
||||
|
||||
# The one check to look at, named as before: green only when every part and the build are.
|
||||
test:
|
||||
needs: [part, build]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Every part and the build passed
|
||||
run: |
|
||||
echo "parts: ${{ needs.part.result }}, build: ${{ needs.build.result }}"
|
||||
test "${{ needs.part.result }}" = success && test "${{ needs.build.result }}" = success
|
||||
|
||||
translations:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: 24
|
||||
|
|
@ -69,3 +105,38 @@ jobs:
|
|||
- run: npm ci
|
||||
- name: Translation coverage
|
||||
run: node scripts/i18n-check.js --summary "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# The one check to look at: green only when the plan, every part, the build and the translations are.
|
||||
test:
|
||||
needs: [plan, part, build, translations]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Everything passed
|
||||
run: |
|
||||
echo "plan: ${{ needs.plan.result }}, parts: ${{ needs.part.result }}, build: ${{ needs.build.result }}, translations: ${{ needs.translations.result }}"
|
||||
test "${{ needs.plan.result }}" = success && test "${{ needs.part.result }}" = success
|
||||
test "${{ needs.build.result }}" = success && test "${{ needs.translations.result }}" = success
|
||||
|
||||
# After a run on main: each file's time from this run goes into the cache, so the next run (main or a PR) splits
|
||||
# its parts by today's tests. Nobody has to remember to refresh tests/durations.json.
|
||||
timings:
|
||||
needs: part
|
||||
if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.part.result != 'skipped'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: tests/durations.json
|
||||
key: test-durations-${{ github.run_id }}
|
||||
restore-keys: test-durations-
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: timings-*
|
||||
path: ${{ runner.temp }}/timings
|
||||
- run: node scripts/test-timings.js --merge "$RUNNER_TEMP/timings"
|
||||
- uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: tests/durations.json
|
||||
key: test-durations-${{ github.run_id }}
|
||||
|
|
|
|||
20
.github/workflows/release.yml
vendored
20
.github/workflows/release.yml
vendored
|
|
@ -23,6 +23,10 @@ on:
|
|||
description: Only the weekly puzzles changed (weekly-puzzle.yml); the release itself was tested when it shipped
|
||||
type: boolean
|
||||
default: false
|
||||
tested:
|
||||
description: The tag's tests already passed (auto-release.yml tests it before tagging)
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# Least privilege: only the deploy job can write to Pages.
|
||||
permissions:
|
||||
|
|
@ -33,9 +37,18 @@ concurrency:
|
|||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# The release's tests, split over several runners like every push (ci.yml). Skipped when auto-release.yml has just
|
||||
# run them on this tag, and for a weekly redeploy (the build step below tests the weekly files against the tag).
|
||||
tests:
|
||||
if: ${{ (github.event_name == 'workflow_dispatch' || !github.event.release.prerelease) && !inputs.weekly && !inputs.tested }}
|
||||
uses: ./.github/workflows/ci.yml
|
||||
with:
|
||||
ref: ${{ github.event.release.tag_name || inputs.tag }}
|
||||
|
||||
build:
|
||||
needs: tests
|
||||
# Pre-releases are for testing and never go live.
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || !github.event.release.prerelease }}
|
||||
if: ${{ always() && (github.event_name == 'workflow_dispatch' || !github.event.release.prerelease) && (needs.tests.result == 'success' || needs.tests.result == 'skipped') }}
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
TAG: ${{ github.event.release.tag_name || inputs.tag }}
|
||||
|
|
@ -93,10 +106,9 @@ jobs:
|
|||
node scripts/weekly-puzzle.js --check
|
||||
if [ -f scripts/weekly-scenario.js ]; then node scripts/weekly-scenario.js --check; fi
|
||||
if [ -f scripts/weekly-architect.js ]; then node scripts/weekly-architect.js --check; fi
|
||||
- run: npm test
|
||||
if: ${{ !inputs.weekly }}
|
||||
# The weekly files come from main, not the tag: whatever tested the tag, they are tested against it here.
|
||||
- name: Test the weekly puzzles against this release
|
||||
if: ${{ inputs.weekly && hashFiles('scripts/weekly-puzzle.js') != '' }}
|
||||
if: ${{ hashFiles('scripts/weekly-puzzle.js') != '' }}
|
||||
run: npm test -- tests/weekly-puzzle.test.js tests/puzzles.test.js $(ls tests/weekly-scenario.test.js tests/weekly-architect.test.js 2>/dev/null)
|
||||
- name: Build site
|
||||
run: VERSION="${TAG#v}" REQUIRE_NOTES=1 npm run build
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue