diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 162100079..1c9f6eb88 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -3,13 +3,13 @@ name: iOS on: pull_request: paths-ignore: - - "docs/**" - - "website/**" - - "README.md" - - ".github/actions/build-docs/action.yml" - - ".github/workflows/deploy.yml" - - ".github/workflows/pr-preview.yml" - - ".github/workflows/pr-preview-cleanup.yml" + - 'docs/**' + - 'website/**' + - 'README.md' + - '.github/actions/build-docs/action.yml' + - '.github/workflows/deploy.yml' + - '.github/workflows/pr-preview.yml' + - '.github/workflows/pr-preview-cleanup.yml' push: branches: - main @@ -27,7 +27,7 @@ jobs: runs-on: macos-26 timeout-minutes: 80 env: - IOS_RUNTIME_VERSION: "26.2" + IOS_RUNTIME_VERSION: '26.2' AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived steps: - name: Checkout @@ -46,7 +46,7 @@ jobs: build-command: sh ./scripts/build-xcuitest-apple.sh xcuitest-platform: ios xcuitest-destination: generic/platform=iOS Simulator - build-on-miss: "false" + build-on-miss: 'false' - name: Boot iOS test simulator uses: ./.github/actions/boot-ios-test-simulator @@ -61,7 +61,7 @@ jobs: - name: Run iOS simulator smoke replay run: | - node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --retries 2 --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml - name: Run iOS physical device smoke replay if: env.IOS_UDID != '' @@ -69,7 +69,7 @@ jobs: IOS_UDID: ${{ vars.IOS_UDID }} run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --report-junit test/artifacts/replays-ios-device-smoke.junit.xml + node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml - name: Upload iOS artifacts if: always() diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index ee410f5e7..1bdaa6c71 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -2,7 +2,7 @@ name: Replay Nightly on: schedule: - - cron: "0 3 * * *" + - cron: '0 3 * * *' workflow_dispatch: permissions: @@ -50,7 +50,7 @@ jobs: runs-on: macos-26 timeout-minutes: 80 env: - IOS_RUNTIME_VERSION: "26.2" + IOS_RUNTIME_VERSION: '26.2' AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived steps: - name: Checkout @@ -69,7 +69,7 @@ jobs: build-command: sh ./scripts/build-xcuitest-apple.sh xcuitest-platform: ios xcuitest-destination: generic/platform=iOS Simulator - build-on-miss: "false" + build-on-miss: 'false' - name: Boot iOS test simulator uses: ./.github/actions/boot-ios-test-simulator @@ -83,13 +83,13 @@ jobs: node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 300000 --json - name: Run iOS simulator replay suite - run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --retries 2 --report-junit test/artifacts/replays-ios-simulator.junit.xml + run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator --report-junit test/artifacts/replays-ios-simulator.junit.xml - name: Run iOS physical device replay suite if: env.IOS_UDID != '' env: IOS_UDID: ${{ vars.IOS_UDID }} - run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device --udid "$IOS_UDID" --retries 2 --report-junit test/artifacts/replays-ios-device.junit.xml + run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device --report-junit test/artifacts/replays-ios-device.junit.xml - name: Upload iOS artifacts if: always() diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift index 1216272f7..99e54ec98 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift @@ -116,12 +116,29 @@ extension RunnerTests { completion((jsonResponse(status: 200, response: executeUptime()), false)) return } + NSLog( + "AGENT_DEVICE_RUNNER_COMMAND_ACCEPTED command=%@ commandId=%@", + command.command.rawValue, + command.commandId ?? "" + ) commandJournal.accept(command: command) commandExecutionQueue.async { do { let response = try self.executeAccepted(command: command) + NSLog( + "AGENT_DEVICE_RUNNER_COMMAND_COMPLETED command=%@ commandId=%@ ok=%d", + command.command.rawValue, + command.commandId ?? "", + response.ok ? 1 : 0 + ) completion((self.jsonResponse(status: 200, response: response), command.command == .shutdown)) } catch { + NSLog( + "AGENT_DEVICE_RUNNER_COMMAND_FAILED command=%@ commandId=%@ error=%@", + command.command.rawValue, + command.commandId ?? "", + String(describing: error) + ) completion(( self.jsonResponse( status: 500, diff --git a/src/daemon/handlers/__tests__/session.test.ts b/src/daemon/handlers/__tests__/session.test.ts index 2f320f3ed..ea36f57f7 100644 --- a/src/daemon/handlers/__tests__/session.test.ts +++ b/src/daemon/handlers/__tests__/session.test.ts @@ -2795,7 +2795,7 @@ test('open --relaunch on iOS stops runner before close/open', async () => { expect(calls).toEqual(['stop-runner', 'close:com.example.app', 'open:com.example.app']); }); -test('open --relaunch on iOS simulator keeps runner while closing app', async () => { +test('open --relaunch on iOS simulator stops runner before close/open', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-simulator-session'; sessionStore.set(sessionName, { @@ -2841,13 +2841,14 @@ test('open --relaunch on iOS simulator keeps runner while closing app', async () expect(response).toBeTruthy(); expect(response?.ok).toBe(true); - expect(calls).toEqual(['close:com.example.app', 'open:com.example.app']); + expect(calls).toEqual(['stop-runner', 'close:com.example.app', 'open:com.example.app']); }); -test('open --relaunch includes timing and waits for iOS runner prewarm', async () => { +test('open --relaunch includes timing and waits for iOS runner prewarm after opening app', async () => { vi.useFakeTimers({ now: 1_000 }); const sessionStore = makeSessionStore(); const sessionName = 'ios-timing-session'; + const events: string[] = []; sessionStore.set(sessionName, { ...makeSession(sessionName, { platform: 'ios', @@ -2861,8 +2862,22 @@ test('open --relaunch includes timing and waits for iOS runner prewarm', async ( }); mockPrewarmIosRunnerSession.mockImplementation( - () => new Promise((resolve) => setTimeout(resolve, 250)), + () => + new Promise((resolve) => { + events.push('prewarm-start'); + setTimeout(() => { + events.push('prewarm-finish'); + resolve(); + }, 250); + }), ); + mockStopIosRunner.mockImplementation(async () => { + events.push('stop-runner'); + }); + mockDispatch.mockImplementation(async (_device, command) => { + events.push(`dispatch:${command}`); + return {}; + }); const responsePromise = handleSessionCommands({ req: { @@ -2882,6 +2897,13 @@ test('open --relaunch includes timing and waits for iOS runner prewarm', async ( const response = await responsePromise; expect(response?.ok).toBe(true); + expect(events).toEqual([ + 'stop-runner', + 'dispatch:close', + 'dispatch:open', + 'prewarm-start', + 'prewarm-finish', + ]); expect((response as any).data?.timing).toMatchObject({ runnerPrewarmKind: 'session', runnerPrewarmScheduled: true, diff --git a/src/daemon/handlers/session-open.ts b/src/daemon/handlers/session-open.ts index 83c367a1c..6e0ed9703 100644 --- a/src/daemon/handlers/session-open.ts +++ b/src/daemon/handlers/session-open.ts @@ -18,7 +18,6 @@ import { import { IOS_SIMULATOR_POST_CLOSE_SETTLE_MS, IOS_SIMULATOR_POST_OPEN_SETTLE_MS, - isIosSimulator, refreshSessionDeviceIfNeeded, settleIosSimulator, } from './session-device-utils.ts'; @@ -67,7 +66,7 @@ async function relaunchCloseApp(params: { context: Parameters[4]; }): Promise { const { device, closeTarget, outFlag, context } = params; - if (device.platform !== 'android' && !isIosSimulator(device)) { + if (device.platform !== 'android') { await stopIosRunnerSession(device.id); } await dispatchCommand(device, 'close', [closeTarget], outFlag, context); @@ -190,17 +189,18 @@ async function completeOpenCommand(params: { traceLogPath, requestId: req.meta?.requestId, }; + const shouldPrewarmRunnerBeforeOpen = req.flags?.maestro?.prewarmRunnerBeforeOpen === true; let runnerPrewarm: Promise | undefined; if (shouldPrewarmIosRunner && sessionAppBundleId) { timing.runnerPrewarmKind = 'session'; timing.runnerPrewarmScheduled = true; - runnerPrewarm = prewarmIosRunnerSession(device, runnerPrewarmOptions); - } - if (runnerPrewarm && req.flags?.maestro?.prewarmRunnerBeforeOpen === true) { - const runnerPrewarmStartedAtMs = Date.now(); - await runnerPrewarm; - timing.runnerPrewarmWaited = true; - timing.runnerPrewarmDurationMs = Math.max(0, Date.now() - runnerPrewarmStartedAtMs); + if (shouldPrewarmRunnerBeforeOpen) { + runnerPrewarm = prewarmIosRunnerSession(device, runnerPrewarmOptions); + const runnerPrewarmStartedAtMs = Date.now(); + await runnerPrewarm; + timing.runnerPrewarmWaited = true; + timing.runnerPrewarmDurationMs = Math.max(0, Date.now() - runnerPrewarmStartedAtMs); + } } const openStartedAtMs = Date.now(); await dispatchCommand(device, 'open', openPositionals, req.flags?.out, { @@ -218,6 +218,9 @@ async function completeOpenCommand(params: { openPositionals, }); timing.launchUrlDurationMs = Math.max(0, Date.now() - launchUrlStartedAtMs); + if (shouldPrewarmIosRunner && sessionAppBundleId && !runnerPrewarm) { + runnerPrewarm = prewarmIosRunnerSession(device, runnerPrewarmOptions); + } if (shouldRelaunch && runnerPrewarm && timing.runnerPrewarmWaited !== true) { const runnerPrewarmStartedAtMs = Date.now(); await runnerPrewarm; diff --git a/src/platforms/ios/__tests__/runner-command-retry.test.ts b/src/platforms/ios/__tests__/runner-command-retry.test.ts index 5c085c4bc..f51904bb8 100644 --- a/src/platforms/ios/__tests__/runner-command-retry.test.ts +++ b/src/platforms/ios/__tests__/runner-command-retry.test.ts @@ -49,7 +49,11 @@ vi.mock('../runner-xctestrun.ts', async () => { }; }); -import { prepareIosRunner, runIosRunnerCommand } from '../runner-client.ts'; +import { + prepareIosRunner, + prewarmIosRunnerSession, + runIosRunnerCommand, +} from '../runner-client.ts'; import type { RunnerXctestrunArtifact } from '../runner-xctestrun.ts'; beforeEach(() => { @@ -170,6 +174,28 @@ test('prepareIosRunner retries a fresh launch session when the health check cann ); }); +test('prewarmIosRunnerSession proves cached runner health with uptime', async () => { + const session = makeRunnerSession({ port: 8100 }); + mockEnsureRunnerSession.mockResolvedValueOnce(session); + mockExecuteRunnerCommandWithSession.mockResolvedValueOnce({ uptimeMs: 42 }); + + const prewarm = prewarmIosRunnerSession(IOS_SIMULATOR, { + buildTimeoutMs: 300_000, + requestId: 'prewarm-request', + }); + + await prewarm; + + assert.equal(mockEnsureRunnerSession.mock.calls.length, 1); + assert.equal(mockEnsureRunnerSession.mock.calls[0]?.[1]?.buildTimeoutMs, 300_000); + assert.equal(mockEnsureRunnerSession.mock.calls[0]?.[1]?.requestId, 'prewarm-request'); + assert.equal(mockEnsureRunnerSession.mock.calls[0]?.[1]?.healthTimeoutMs, 45_000); + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls.length, 1); + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[0]?.[1], session); + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[0]?.[2].command, 'uptime'); + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[0]?.[4], 45_000); +}); + test('prepareIosRunner does not force a rebuild when the relaunched fresh session still cannot connect', async () => { const missArtifact = makeRunnerArtifact({ xctestrunPath: '/tmp/miss.xctestrun', diff --git a/src/platforms/ios/__tests__/runner-session.test.ts b/src/platforms/ios/__tests__/runner-session.test.ts index a67721187..8165b00f3 100644 --- a/src/platforms/ios/__tests__/runner-session.test.ts +++ b/src/platforms/ios/__tests__/runner-session.test.ts @@ -102,6 +102,7 @@ import { executeRunnerCommandWithSession, getRunnerSessionSnapshot, invalidateRunnerSession, + stopIosRunnerSession, stopRunnerSession, validateRunnerDevice, } from '../runner-session.ts'; @@ -593,6 +594,26 @@ test('runner session starts xcodebuild through provider seams and reuses an aliv await stopRunnerSession(session); }); +test('runner session startup kills stale device-scoped xcodebuild before launching a new runner', async () => { + const device = { ...IOS_SIMULATOR, id: 'runner-session-startup-stale-sim' }; + + await ensureRunnerSession(device, {}); + + const pkillCalls = mockRunAppleToolCommand.mock.calls.filter((call) => call[0] === 'pkill'); + assert.equal(pkillCalls.length, 2); + assert.deepEqual(pkillCalls[0]?.[1]?.slice(0, 2), ['-TERM', '-f']); + assert.deepEqual(pkillCalls[1]?.[1]?.slice(0, 2), ['-KILL', '-f']); + assert.match( + String(pkillCalls[0]?.[1]?.[2] ?? ''), + /xcodebuild\.\*test-without-building\.\*AgentDeviceRunner\\\.env\\\.session-runner-session-startup-stale-sim-/, + ); + const staleCleanupCallOrder = mockRunAppleToolCommand.mock.invocationCallOrder[0]; + const runnerLaunchCallOrder = mockRunCmdBackground.mock.invocationCallOrder[0]; + assert.ok(staleCleanupCallOrder !== undefined); + assert.ok(runnerLaunchCallOrder !== undefined); + assert.ok(staleCleanupCallOrder < runnerLaunchCallOrder); +}); + test('runner session restarts alive runner when expected xctestrun artifact changes', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-stale-artifact-sim' }; @@ -695,6 +716,25 @@ test('runner session stop sends shutdown, cleans temporary runner files, and rel assert.equal(getRunnerSessionSnapshot(device.id), null); }); +test('runner session stop kills stale device-scoped xcodebuild runner processes without in-memory session', async () => { + const deviceId = '11C70358-8331-4872-A0CA-F15B6859B6FC'; + + await stopIosRunnerSession(deviceId); + + const pkillCalls = mockRunAppleToolCommand.mock.calls.filter((call) => call[0] === 'pkill'); + assert.equal(pkillCalls.length, 2); + assert.deepEqual(pkillCalls[0]?.[1]?.slice(0, 2), ['-TERM', '-f']); + assert.deepEqual(pkillCalls[1]?.[1]?.slice(0, 2), ['-KILL', '-f']); + assert.match( + String(pkillCalls[0]?.[1]?.[2] ?? ''), + /xcodebuild\.\*test-without-building\.\*AgentDeviceRunner\\\.env\\\.session-11C70358-8331-4872-A0CA-F15B6859B6FC-/, + ); + assert.deepEqual(pkillCalls[0]?.[2], { + allowFailure: true, + timeoutMs: 2_000, + }); +}); + test('runner session invalidation skips graceful shutdown and removes stale session', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-invalidate-sim' }; const session = await ensureRunnerSession(device, {}); diff --git a/src/platforms/ios/runner-client.ts b/src/platforms/ios/runner-client.ts index 50c566c3f..35ef20923 100644 --- a/src/platforms/ios/runner-client.ts +++ b/src/platforms/ios/runner-client.ts @@ -1,11 +1,7 @@ import { withRetry } from '../../utils/retry.ts'; import type { DeviceInfo } from '../../utils/device.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; -import { - type RunnerSessionOptions, - ensureRunnerSession, - validateRunnerDevice, -} from './runner-session.ts'; +import { type RunnerSessionOptions, validateRunnerDevice } from './runner-session.ts'; import { assertRunnerRequestActive, isReadOnlyRunnerCommand, @@ -25,6 +21,7 @@ import { type PrepareIosRunnerOptions, type PrepareIosRunnerResult, } from './runner-lifecycle.ts'; +import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts'; export { isRetryableRunnerError, resolveRunnerEarlyExitHint, @@ -128,7 +125,10 @@ function resolveAppleRunnerRuntime( const LOCAL_APPLE_RUNNER_RUNTIME = createLocalAppleRunnerProvider(executeRunnerCommand, { prepare: prepareLocalIosRunner, prewarm: async (device, options) => { - await ensureRunnerSession(device, options); + await prepareLocalIosRunner(device, { + ...options, + healthTimeoutMs: RUNNER_COMMAND_TIMEOUT_MS, + }); }, }); diff --git a/src/platforms/ios/runner-session.ts b/src/platforms/ios/runner-session.ts index 68332f383..11737dfd7 100644 --- a/src/platforms/ios/runner-session.ts +++ b/src/platforms/ios/runner-session.ts @@ -54,6 +54,7 @@ const RUNNER_INVALIDATE_WAIT_TIMEOUT_MS = 1_000; const RUNNER_READY_PREFLIGHT_TIMEOUT_MS = 1_000; const RUNNER_SHUTDOWN_TIMEOUT_MS = 15_000; const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; +const RUNNER_STALE_XCODEBUILD_KILL_TIMEOUT_MS = 2_000; type RunnerReadinessPreflightDecision = | { @@ -81,6 +82,9 @@ export async function ensureRunnerSession( } const startupTimings: Record = {}; + await measureRunnerStartupStep(startupTimings, 'cleanup_stale_xcodebuild', async () => { + await killStaleRunnerXcodebuildProcesses(device.id); + }); await measureRunnerStartupStep(startupTimings, 'ensure_booted', async () => { await ensureBootedIfNeeded(device); }); @@ -369,6 +373,7 @@ async function stopRunnerSessionInternal( export async function stopIosRunnerSession(deviceId: string): Promise { await withRunnerSessionLock(deviceId, async () => { await stopRunnerSessionInternal(deviceId); + await killStaleRunnerXcodebuildProcesses(deviceId); }); } @@ -463,6 +468,32 @@ async function killRunnerProcessTree( } catch {} } +async function killStaleRunnerXcodebuildProcesses(deviceId: string): Promise { + const pattern = `xcodebuild.*test-without-building.*AgentDeviceRunner\\.env\\.session-${escapeRegex(deviceId)}-`; + for (const signal of ['TERM', 'KILL'] as const) { + try { + await runAppleToolCommand('pkill', [`-${signal}`, '-f', pattern], { + allowFailure: true, + timeoutMs: RUNNER_STALE_XCODEBUILD_KILL_TIMEOUT_MS, + }); + } catch (error) { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_stale_xcodebuild_kill_failed', + data: { + deviceId, + signal, + error: error instanceof Error ? error.message : String(error), + }, + }); + } + } +} + +function escapeRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + function ensureBootedIfNeeded(device: DeviceInfo): Promise { if (device.kind !== 'simulator') { return Promise.resolve();