ci: reduce unit test flakiness and shard re-run cost (#3844)

A unit-test shard recently failed on a timing race rather than a real
regression - a run-engine waitpoint test sleeps 1250ms waiting on a
1000ms timeout that's processed by a ~1000ms worker poll, so on a
CPU-starved shard the margin evaporates and the whole matrix goes red.
Because `fail-fast` defaults on, that one flake cancels the sibling
shards, and the only recovery is re-running the entire matrix "just to
be sure" - which is itself slow.

This is the low-risk first pass at that pain:

- `fail-fast: false` on the webapp and internal shard matrices, so one
flaky shard no longer cancels its siblings. "Re-run failed jobs" now
re-runs just the failed shard instead of the whole matrix.
- CI-scoped `retry: process.env.CI ? 2 : 0` on the timing-sensitive
packages (`run-engine`, `redis-worker`, `schedule-engine`). Flakes
self-heal in CI; local runs stay at `retry: 0` so they still surface in
dev. A stopgap until the timing tests are made deterministic.
- `fetch-depth: 1` on the unit-test checkouts - they don't use git
history, so the full clone was wasted setup time across ~20 jobs.
- Reconcile the pre-pull image tags with what testcontainers actually
pulls (`redis:7-alpine` -> `redis:7.2`, `ryuk:0.11.0` -> `ryuk:0.14.0`)
and add `minio/minio:latest` to the webapp pre-pull. Otherwise those
images pull unauthenticated at test time and risk Docker Hub rate-limit
flakes (worst on fork PRs, where the authenticated pre-pull is skipped
entirely).

Deeper follow-ups - bigger runners, turbo remote cache, runtime-weighted
sharding, and the real root-cause fix (container reuse / template-DB
isolation + deterministic timing tests) - are tracked under TRI-10484.

Author: nicktrn

.github/workflows/unit-tests-internal.yml

@@ -16,6 +16,8 @@ jobs:
     name: "🧪 Unit Tests: Internal"
     runs-on: ubuntu-latest
     strategy:
+      # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard
+      fail-fast: false
       matrix:
         shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
         shardTotal: [8]
@@ -53,7 +55,7 @@ jobs:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: ⎔ Setup pnpm
@@ -84,8 +86,8 @@ jobs:
           echo "Pre-pulling Docker images with authenticated session..."
           docker pull postgres:14
           docker pull clickhouse/clickhouse-server:25.4-alpine
-          docker pull redis:7-alpine
-          docker pull testcontainers/ryuk:0.11.0
+          docker pull redis:7.2
+          docker pull testcontainers/ryuk:0.14.0
           docker pull electricsql/electric:1.2.4
           echo "Image pre-pull complete"
 
@@ -123,7 +125,7 @@ jobs:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: ⎔ Setup pnpm

.github/workflows/unit-tests-packages.yml

@@ -53,7 +53,7 @@ jobs:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: ⎔ Setup pnpm
@@ -84,8 +84,8 @@ jobs:
           echo "Pre-pulling Docker images with authenticated session..."
           docker pull postgres:14
           docker pull clickhouse/clickhouse-server:25.4-alpine
-          docker pull redis:7-alpine
-          docker pull testcontainers/ryuk:0.11.0
+          docker pull redis:7.2
+          docker pull testcontainers/ryuk:0.14.0
           docker pull electricsql/electric:1.2.4
           echo "Image pre-pull complete"
 
@@ -123,7 +123,7 @@ jobs:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: ⎔ Setup pnpm

.github/workflows/unit-tests-webapp.yml

@@ -16,6 +16,8 @@ jobs:
     name: "🧪 Unit Tests: Webapp"
     runs-on: ubuntu-latest
     strategy:
+      # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard
+      fail-fast: false
       matrix:
         shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
         shardTotal: [8]
@@ -53,7 +55,7 @@ jobs:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: ⎔ Setup pnpm
@@ -84,9 +86,10 @@ jobs:
           echo "Pre-pulling Docker images with authenticated session..."
           docker pull postgres:14
           docker pull clickhouse/clickhouse-server:25.4-alpine
-          docker pull redis:7-alpine
-          docker pull testcontainers/ryuk:0.11.0
+          docker pull redis:7.2
+          docker pull testcontainers/ryuk:0.14.0
           docker pull electricsql/electric:1.2.4
+          docker pull minio/minio:latest
           echo "Image pre-pull complete"
 
       - name: 📥 Download deps
@@ -131,7 +134,7 @@ jobs:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: ⎔ Setup pnpm

internal-packages/run-engine/vitest.config.ts

@@ -4,6 +4,8 @@ export default defineConfig({
   test: {
     include: ["**/*.test.ts"],
     globals: true,
+    // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
+    retry: process.env.CI ? 2 : 0,
     isolate: true,
     fileParallelism: false,
     testTimeout: 120_000,

internal-packages/schedule-engine/vitest.config.ts

@@ -3,6 +3,8 @@ import { defineConfig } from "vitest/config";
 export default defineConfig({
   test: {
     globals: true,
+    // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
+    retry: process.env.CI ? 2 : 0,
     environment: "node",
     setupFiles: ["./test/setup.ts"],
     testTimeout: 30000,

packages/redis-worker/vitest.config.ts

@@ -4,6 +4,8 @@ export default defineConfig({
   test: {
     include: ["**/*.test.ts"],
     globals: true,
+    // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
+    retry: process.env.CI ? 2 : 0,
     fileParallelism: false,
   },
 });