diff --git a/src/__tests__/cli-perf.test.ts b/src/__tests__/cli-perf.test.ts index b6537b82a..362bf2c3a 100644 --- a/src/__tests__/cli-perf.test.ts +++ b/src/__tests__/cli-perf.test.ts @@ -47,6 +47,114 @@ test('perf prints compact platform-independent frame health summary by default', assert.doesNotMatch(result.stdout, /android|Pixel|memory|cpu|gfxinfo/i); }); +test('perf metrics forwards explicit metrics area to daemon', async () => { + const result = await runCliCapture(['perf', 'metrics', '--json'], async () => ({ + ok: true, + data: { + metrics: { + fps: { + available: false, + reason: 'No frame data.', + }, + }, + }, + })); + + assert.equal(result.code, null); + assert.equal(result.calls[0]?.command, 'perf'); + assert.deepEqual(result.calls[0]?.positionals, ['metrics']); +}); + +test('perf frames forwards frames area and prints focused frame summary', async () => { + const result = await runCliCapture(['perf', 'frames'], async () => ({ + ok: true, + data: { + metrics: { + fps: { + available: true, + droppedFramePercent: 3.1, + droppedFrameCount: 12, + totalFrameCount: 390, + sampleWindowMs: 12_000, + worstWindows: [], + }, + }, + }, + })); + + assert.equal(result.code, null); + assert.equal(result.calls[0]?.command, 'perf'); + assert.deepEqual(result.calls[0]?.positionals, ['frames']); + assert.equal(result.stdout, 'Frame health: dropped 3.1% (12/390 frames) window 12s\n'); +}); + +test('perf frames sample forwards explicit sample action to daemon', async () => { + const result = await runCliCapture(['perf', 'frames', 'sample', '--json'], async () => ({ + ok: true, + data: { + metrics: { + fps: { + available: false, + reason: 'No frame data.', + }, + }, + }, + })); + + assert.equal(result.code, null); + assert.equal(result.calls[0]?.command, 'perf'); + assert.deepEqual(result.calls[0]?.positionals, ['frames', 'sample']); +}); + +test('perf sample defaults to metrics sample', async () => { + const result = await runCliCapture(['perf', 'sample', '--json'], async () => ({ + ok: true, + data: { + metrics: { + fps: { + available: false, + reason: 'No frame data.', + }, + }, + }, + })); + + assert.equal(result.code, null); + assert.equal(result.calls[0]?.command, 'perf'); + assert.deepEqual(result.calls[0]?.positionals, ['metrics', 'sample']); +}); + +test('perf area and action positionals are case-insensitive', async () => { + const result = await runCliCapture(['perf', 'FRAMES', 'SAMPLE', '--json'], async () => ({ + ok: true, + data: { + metrics: { + fps: { + available: false, + reason: 'No frame data.', + }, + }, + }, + })); + + assert.equal(result.code, null); + assert.equal(result.calls[0]?.command, 'perf'); + assert.deepEqual(result.calls[0]?.positionals, ['frames', 'sample']); +}); + +test('perf rejects unknown CLI area before daemon dispatch', async () => { + const result = await runCliCapture(['perf', 'cpu', '--json'], async () => ({ + ok: true, + data: {}, + })); + + assert.equal(result.code, 1); + assert.equal(result.calls.length, 0); + const payload = JSON.parse(result.stdout); + assert.equal(payload.error.code, 'INVALID_ARGS'); + assert.match(payload.error.message, /perf area must be metrics or frames/i); +}); + test('perf prints unavailable frame health reason by default', async () => { const result = await runCliCapture(['perf'], async () => ({ ok: true, @@ -54,7 +162,8 @@ test('perf prints unavailable frame health reason by default', async () => { metrics: { fps: { available: false, - reason: 'Dropped-frame sampling is currently available only on Android.', + reason: + 'Dropped-frame sampling is currently available only on Android app sessions and connected iOS device app sessions.', }, }, }, @@ -63,7 +172,7 @@ test('perf prints unavailable frame health reason by default', async () => { assert.equal(result.code, null); assert.equal( result.stdout, - 'Frame health: unavailable - Dropped-frame sampling is currently available only on Android.\n', + 'Frame health: unavailable - Dropped-frame sampling is currently available only on Android app sessions and connected iOS device app sessions.\n', ); }); @@ -74,7 +183,8 @@ test('perf prints compact CPU and memory summary when frame health is unavailabl metrics: { fps: { available: false, - reason: 'Dropped-frame sampling is currently available only on Android.', + reason: + 'Dropped-frame sampling is currently available only on Android app sessions and connected iOS device app sessions.', }, memory: { available: true, diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index cd312ae31..8735e2818 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -119,6 +119,32 @@ test('apps.open forwards explicit runtime hints through the daemon request', asy }); }); +test('observability.perf projects structured frame area to daemon positionals', async () => { + const setup = createTransport(async (req) => { + if (req.command === 'perf') { + return { + ok: true, + data: { + metrics: { + fps: { + available: false, + reason: 'No frame data.', + }, + }, + }, + }; + } + throw new Error(`Unexpected command: ${req.command}`); + }); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.observability.perf({ area: 'frames', action: 'sample' }); + + assert.equal(setup.calls.length, 1); + assert.equal(setup.calls[0]?.command, 'perf'); + assert.deepEqual(setup.calls[0]?.positionals, ['frames', 'sample']); +}); + test('structured command input accepts target as deviceTarget alias when no UI target exists', async () => { const setup = createTransport(async (req) => { if (req.command === 'open') { diff --git a/src/client-types.ts b/src/client-types.ts index ba56096b3..993143384 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -24,6 +24,7 @@ import type { import type { MetroBridgeScope } from './client-companion-tunnel-contract.ts'; import type { AppsFilter } from './commands/app-inventory-contract.ts'; import type { ScreenshotRequestFlags } from './commands/capture-screenshot-options.ts'; +import type { PerfAction, PerfArea } from './commands/perf-command-contract.ts'; import type { DaemonBatchStep } from './core/batch.ts'; import type { AlertInfo } from './alert-contract.ts'; @@ -728,7 +729,10 @@ export type BatchRunOptions = AgentDeviceRequestOverrides & { out?: string; }; -export type PerfOptions = ClientCommandBaseOptions; +export type PerfOptions = ClientCommandBaseOptions & { + area?: PerfArea; + action?: PerfAction; +}; export type LogsOptions = AgentDeviceRequestOverrides & { action?: 'path' | 'start' | 'stop' | 'doctor' | 'mark' | 'clear'; diff --git a/src/commands/cli-grammar/observability.ts b/src/commands/cli-grammar/observability.ts index 4773ef819..aaa91e790 100644 --- a/src/commands/cli-grammar/observability.ts +++ b/src/commands/cli-grammar/observability.ts @@ -1,6 +1,19 @@ import { PUBLIC_COMMANDS } from '../../command-catalog.ts'; -import type { LogsOptions, NetworkOptions, RecordOptions } from '../../client-types.ts'; +import type { + LogsOptions, + NetworkOptions, + PerfOptions, + RecordOptions, +} from '../../client-types.ts'; import { AppError } from '../../utils/errors.ts'; +import { + isPerfAction, + isPerfArea, + PERF_ACTION_ERROR_MESSAGE, + PERF_AREA_ERROR_MESSAGE, + type PerfAction, + type PerfArea, +} from '../perf-command-contract.ts'; import { commonInputFromFlags, direct, @@ -12,7 +25,10 @@ import { import type { CliReader, DaemonWriter } from './types.ts'; export const observabilityCliReaders = { - perf: (_positionals, flags) => commonInputFromFlags(flags), + perf: (positionals, flags) => ({ + ...commonInputFromFlags(flags), + ...readPerfPositionals(positionals), + }), logs: (positionals, flags) => ({ ...commonInputFromFlags(flags), action: readLogsAction(positionals[0]), @@ -41,7 +57,7 @@ export const observabilityCliReaders = { } satisfies Record; export const observabilityDaemonWriters = { - perf: direct(PUBLIC_COMMANDS.perf), + perf: direct(PUBLIC_COMMANDS.perf, (input) => perfPositionals(input as PerfOptions)), logs: direct(PUBLIC_COMMANDS.logs, (input) => logsPositionals(input as LogsOptions)), network: (input) => request(PUBLIC_COMMANDS.network, networkPositionals(input as NetworkOptions), { @@ -52,6 +68,22 @@ export const observabilityDaemonWriters = { trace: direct(PUBLIC_COMMANDS.trace, (input) => recordingPositionals(input as RecordOptions)), } satisfies Record; +function perfPositionals(input: PerfOptions): string[] { + const area = input.area ?? (input.action ? 'metrics' : undefined); + return [...optionalString(area), ...optionalString(input.action)]; +} + +function readPerfPositionals(positionals: string[]): Pick { + if (positionals[0] !== undefined && positionals[1] === undefined) { + const action = readPerfAction(positionals[0], { allowUndefined: true }); + if (action) return { action }; + } + return { + area: readPerfArea(positionals[0]), + action: readPerfAction(positionals[1]), + }; +} + function logsPositionals(input: { action?: string; message?: string }): string[] { return [input.action ?? 'path', ...optionalString(input.message)]; } @@ -69,6 +101,24 @@ function readStartStop(value: string | undefined, command: string): 'start' | 's throw new AppError('INVALID_ARGS', `${command} requires start|stop`); } +function readPerfArea(value: string | undefined): PerfArea | undefined { + if (value === undefined) return undefined; + const normalized = value.toLowerCase(); + if (isPerfArea(normalized)) return normalized; + throw new AppError('INVALID_ARGS', PERF_AREA_ERROR_MESSAGE); +} + +function readPerfAction( + value: string | undefined, + options: { allowUndefined?: boolean } = {}, +): PerfAction | undefined { + if (value === undefined) return undefined; + const normalized = value.toLowerCase(); + if (isPerfAction(normalized)) return normalized; + if (options.allowUndefined) return undefined; + throw new AppError('INVALID_ARGS', PERF_ACTION_ERROR_MESSAGE); +} + function readLogsAction( value: string | undefined, ): 'path' | 'start' | 'stop' | 'doctor' | 'mark' | 'clear' | undefined { diff --git a/src/commands/client-command-metadata.ts b/src/commands/client-command-metadata.ts index c22c270f3..35d32fe0a 100644 --- a/src/commands/client-command-metadata.ts +++ b/src/commands/client-command-metadata.ts @@ -18,6 +18,7 @@ import { type CommandFieldMap, } from './command-input.ts'; import { defineFieldCommandMetadata } from './field-command-contract.ts'; +import { PERF_ACTION_VALUES, PERF_AREA_VALUES } from './perf-command-contract.ts'; const SURFACE_VALUES = ['app', 'frontmost-app', 'desktop', 'menubar'] as const; const WAIT_KIND_VALUES = ['duration', 'text', 'ref', 'selector'] as const; @@ -177,7 +178,10 @@ export const clientCommandMetadata = [ artifactsDir: stringField(), reportJunit: stringField(), }), - defineClientCommandMetadata('perf', {}), + defineClientCommandMetadata('perf', { + area: enumField(PERF_AREA_VALUES), + action: enumField(PERF_ACTION_VALUES), + }), defineClientCommandMetadata('logs', { action: enumField(LOG_ACTION_VALUES), message: stringField(), diff --git a/src/commands/command-descriptions.ts b/src/commands/command-descriptions.ts index e87efa167..aec34b68d 100644 --- a/src/commands/command-descriptions.ts +++ b/src/commands/command-descriptions.ts @@ -26,7 +26,7 @@ const COMMAND_DESCRIPTIONS = { 'react-native': 'Run supported React Native app automation helpers.', replay: 'Replay a recorded session.', test: 'Run one or more replay scripts.', - perf: 'Show session performance metrics.', + perf: 'Show session performance metrics and frame health.', logs: 'Manage session app logs.', network: 'Show recent HTTP traffic.', record: 'Start or stop screen recording.', diff --git a/src/commands/perf-command-contract.ts b/src/commands/perf-command-contract.ts new file mode 100644 index 000000000..e332aa07e --- /dev/null +++ b/src/commands/perf-command-contract.ts @@ -0,0 +1,16 @@ +export const PERF_AREA_VALUES = ['metrics', 'frames'] as const; +export const PERF_ACTION_VALUES = ['sample'] as const; + +export type PerfArea = (typeof PERF_AREA_VALUES)[number]; +export type PerfAction = (typeof PERF_ACTION_VALUES)[number]; + +export const PERF_AREA_ERROR_MESSAGE = 'perf area must be metrics or frames'; +export const PERF_ACTION_ERROR_MESSAGE = 'perf action must be sample'; + +export function isPerfArea(value: string): value is PerfArea { + return (PERF_AREA_VALUES as readonly string[]).includes(value); +} + +export function isPerfAction(value: string): value is PerfAction { + return (PERF_ACTION_VALUES as readonly string[]).includes(value); +} diff --git a/src/daemon/handlers/session-observability.ts b/src/daemon/handlers/session-observability.ts index 93d4070c0..4f4c2782a 100644 --- a/src/daemon/handlers/session-observability.ts +++ b/src/daemon/handlers/session-observability.ts @@ -1,4 +1,10 @@ import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { + isPerfAction, + isPerfArea, + PERF_ACTION_ERROR_MESSAGE, + PERF_AREA_ERROR_MESSAGE, +} from '../../commands/perf-command-contract.ts'; import { normalizeError } from '../../utils/errors.ts'; import type { AndroidAdbExecutor } from '../../platforms/android/adb-executor.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; @@ -12,7 +18,7 @@ import { startAppLog, stopAppLog, } from '../app-log.ts'; -import { buildPerfResponseData } from './session-perf.ts'; +import { buildPerfFramesResponseData, buildPerfResponseData } from './session-perf.ts'; import { errorResponse, type DaemonFailureResponse } from './response.ts'; const LOG_ACTIONS = ['path', 'start', 'stop', 'doctor', 'mark', 'clear'] as const; @@ -22,6 +28,7 @@ const NETWORK_ACTIONS_MESSAGE = `network requires ${NETWORK_ACTIONS.join(' or ') const NETWORK_INCLUDE_MODES = ['summary', 'headers', 'body', 'all'] as const; const NETWORK_INCLUDE_MESSAGE = `network include mode must be one of: ${NETWORK_INCLUDE_MODES.join(', ')}`; +type LogsAction = (typeof LOG_ACTIONS)[number]; type NetworkIncludeMode = (typeof NETWORK_INCLUDE_MODES)[number]; type ObservabilityParams = { @@ -30,6 +37,27 @@ type ObservabilityParams = { sessionStore: SessionStore; androidAdbExecutor?: AndroidAdbExecutor; }; +type LogsHandlerParams = ObservabilityParams & { + session: SessionState; + restart: boolean; +}; + +const LOG_ACTION_HANDLERS: Record< + LogsAction, + (params: LogsHandlerParams) => Promise | DaemonResponse +> = { + path: ({ session, sessionName, sessionStore }) => + handleLogsPath(session, sessionName, sessionStore), + doctor: ({ session, sessionName, sessionStore }) => + handleLogsDoctor(session, sessionName, sessionStore), + mark: ({ req, sessionName, sessionStore }) => handleLogsMark(req, sessionName, sessionStore), + clear: ({ session, sessionName, sessionStore, restart }) => + handleLogsClear(session, sessionName, sessionStore, restart), + start: ({ session, sessionName, sessionStore }) => + handleLogsStart(session, sessionName, sessionStore), + stop: ({ session, sessionName, sessionStore }) => + handleLogsStop(session, sessionName, sessionStore), +}; function resolveSessionLogBackendLabel( session: SessionState, @@ -69,16 +97,28 @@ export async function handleSessionObservabilityCommands( // --------------------------------------------------------------------------- async function handlePerfCommand(params: ObservabilityParams): Promise { - const { sessionName, sessionStore, androidAdbExecutor } = params; + const { req, sessionName, sessionStore, androidAdbExecutor } = params; const session = sessionStore.get(sessionName); if (!session) { return errorResponse('SESSION_NOT_FOUND', 'perf requires an active session. Run open first.'); } + const area = (req.positionals?.[0] ?? 'metrics').toLowerCase(); + const action = (req.positionals?.[1] ?? 'sample').toLowerCase(); + if (!isPerfArea(area)) { + return errorResponse('INVALID_ARGS', PERF_AREA_ERROR_MESSAGE); + } + if (!isPerfAction(action)) { + return errorResponse('INVALID_ARGS', PERF_ACTION_ERROR_MESSAGE); + } + try { return { ok: true, - data: await buildPerfResponseData(session, { androidAdb: androidAdbExecutor }), + data: + area === 'frames' + ? await buildPerfFramesResponseData(session, { androidAdb: androidAdbExecutor }) + : await buildPerfResponseData(session, { androidAdb: androidAdbExecutor }), }; } catch (error) { return { ok: false, error: normalizeError(error) }; @@ -99,35 +139,27 @@ async function handleLogsCommand(params: ObservabilityParams): Promise first', ); } - const logPath = sessionStore.resolveAppLogPath(sessionName); - if (!restart) { - return { ok: true, data: clearAppLogFiles(logPath) }; - } - if (session.appLog) { await stopAppLog(session.appLog); } const cleared = clearAppLogFiles(logPath); const appLogPidPath = sessionStore.resolveAppLogPidPath(sessionName); try { - const appLogStream = await startAppLog( - session.device, - session.appBundleId as string, - logPath, - appLogPidPath, - ); + const appLogStream = await startAppLog(session.device, appBundleId, logPath, appLogPidPath); sessionStore.set(sessionName, { ...session, appLog: { @@ -300,35 +327,16 @@ async function handleLogsStop( // --------------------------------------------------------------------------- async function handleNetworkCommand(params: ObservabilityParams): Promise { - const { req, sessionName, sessionStore } = params; - const session = sessionStore.get(sessionName); - if (!session) { - return errorResponse('SESSION_NOT_FOUND', 'network requires an active session'); - } - if (!isCommandSupportedOnDevice('network', session.device)) { - return errorResponse('UNSUPPORTED_OPERATION', 'network is not supported on this device'); - } - - const action = (req.positionals?.[0] ?? 'dump').toLowerCase(); - if (!NETWORK_ACTIONS.includes(action as (typeof NETWORK_ACTIONS)[number])) { - return errorResponse('INVALID_ARGS', NETWORK_ACTIONS_MESSAGE); - } - - const maxEntries = req.positionals?.[1] ? Number.parseInt(req.positionals[1], 10) : 25; - if (!Number.isInteger(maxEntries) || maxEntries < 1 || maxEntries > 200) { - return errorResponse('INVALID_ARGS', 'network dump limit must be an integer in range 1..200'); - } - - const includeValidation = resolveNetworkIncludeMode(req); - if (!includeValidation.ok) return includeValidation; - const { include } = includeValidation; + const request = resolveNetworkCommandRequest(params); + if (!request.ok) return request; + const { include, maxEntries, session } = request; const capture = await readSessionNetworkCapture({ device: session.device, appBundleId: session.appBundleId, appLogState: session.appLog?.getState(), appLogStartedAt: session.appLog?.startedAt, - appLogPath: sessionStore.resolveAppLogPath(sessionName), + appLogPath: params.sessionStore.resolveAppLogPath(params.sessionName), maxEntries, include, maxPayloadChars: 2048, @@ -347,6 +355,35 @@ async function handleNetworkCommand(params: ObservabilityParams): Promise 200) { + return errorResponse('INVALID_ARGS', 'network dump limit must be an integer in range 1..200'); + } + + const includeValidation = resolveNetworkIncludeMode(req); + if (!includeValidation.ok) return includeValidation; + return { ok: true, session, maxEntries, include: includeValidation.include }; +} + function resolveNetworkIncludeMode( req: DaemonRequest, ): { ok: true; include: NetworkIncludeMode } | DaemonFailureResponse { diff --git a/src/daemon/handlers/session-perf.ts b/src/daemon/handlers/session-perf.ts index 526f2d7c5..eb19d8861 100644 --- a/src/daemon/handlers/session-perf.ts +++ b/src/daemon/handlers/session-perf.ts @@ -13,6 +13,7 @@ import { sampleAndroidMemoryPerf, } from '../../platforms/android/perf.ts'; import { + buildAppleFrameSamplingMetadata, buildAppleSamplingMetadata, sampleAppleFramePerf, sampleApplePerfMetrics, @@ -37,6 +38,10 @@ type PerfResponseData = { metrics: Record; sampling: Record; }; +type PerfFramesResponseData = Omit & { + metrics: { fps: unknown }; + sampling: { fps: unknown }; +}; type BuildPerfResponseOptions = { androidAdb?: AndroidAdbExecutor; }; @@ -79,7 +84,7 @@ function readStartupPerfSamples(actions: SessionAction[]): StartupPerfSample[] { export async function buildPerfResponseData( session: SessionState, options: BuildPerfResponseOptions = {}, -): Promise> { +): Promise { const response = buildBasePerfResponse(session); if (!supportsPlatformPerfMetrics(session)) { @@ -92,11 +97,30 @@ export async function buildPerfResponseData( } if (session.device.platform === 'android') { - await applyAndroidPerfMetrics(response, session, options); + await applyAndroidPerfMetrics(response, session, session.appBundleId, options); + return response; + } + + await applyApplePerfMetrics(response, session, session.appBundleId); + return response; +} + +export async function buildPerfFramesResponseData( + session: SessionState, + options: BuildPerfResponseOptions = {}, +): Promise { + const response = buildBasePerfFramesResponse(session); + + if (!supportsPlatformPerfMetrics(session)) { + return response; + } + + if (!session.appBundleId) { + response.metrics.fps = { available: false, reason: buildMissingAppPerfReason(session) }; return response; } - await applyApplePerfMetrics(response, session); + await applyFramePerfMetric(response, session, session.appBundleId, options); return response; } @@ -139,15 +163,35 @@ function buildBasePerfResponse(session: SessionState): PerfResponseData { function buildDefaultUnavailableMetrics(): Record { return { - fps: { - available: false, - reason: 'Dropped-frame sampling is currently available only on Android.', - }, + fps: buildDefaultUnavailableFrameMetric(), memory: { available: false, reason: PERF_UNAVAILABLE_REASON }, cpu: { available: false, reason: PERF_UNAVAILABLE_REASON }, }; } +function buildDefaultUnavailableFrameMetric(): Record { + return { + available: false, + reason: + 'Dropped-frame sampling is currently available only on Android app sessions and connected iOS device app sessions.', + }; +} + +function buildBasePerfFramesResponse(session: SessionState): PerfFramesResponseData { + return { + session: session.name, + platform: session.device.platform, + device: session.device.name, + deviceId: session.device.id, + metrics: { + fps: buildDefaultUnavailableFrameMetric(), + }, + sampling: { + fps: buildFrameSamplingMetadata(session), + }, + }; +} + function applyMissingAppPerfMetrics(response: PerfResponseData, session: SessionState): void { const reason = buildMissingAppPerfReason(session); response.metrics.fps = { available: false, reason }; @@ -158,22 +202,31 @@ function applyMissingAppPerfMetrics(response: PerfResponseData, session: Session async function applyAndroidPerfMetrics( response: PerfResponseData, session: SessionState, + appBundleId: string, options: BuildPerfResponseOptions, ): Promise { - const results = await sampleAndroidPerfResults(session, options); - response.metrics.memory = buildMetricResult(results.memory); - response.metrics.cpu = buildMetricResult(results.cpu); - response.metrics.fps = enrichFrameMetricWithSessionContext( - buildMetricResult(results.fps), - session, - ); + const results = await sampleAndroidPerfResults(session, appBundleId, options); + applySampledPerfMetrics(response, session, results); } async function applyApplePerfMetrics( response: PerfResponseData, session: SessionState, + appBundleId: string, ): Promise { - const results = await sampleApplePerfResultsForSession(session); + const results = await sampleApplePerfResultsForSession(session, appBundleId); + applySampledPerfMetrics(response, session, results); +} + +function applySampledPerfMetrics( + response: PerfResponseData, + session: SessionState, + results: { + memory: SettledMetricResult; + cpu: SettledMetricResult; + fps: SettledMetricResult; + }, +): void { response.metrics.memory = buildMetricResult(results.memory); response.metrics.cpu = buildMetricResult(results.cpu); response.metrics.fps = enrichFrameMetricWithSessionContext( @@ -182,6 +235,23 @@ async function applyApplePerfMetrics( ); } +async function applyFramePerfMetric( + response: PerfFramesResponseData, + session: SessionState, + appBundleId: string, + options: BuildPerfResponseOptions, +): Promise { + const result = + session.device.platform === 'android' + ? await settleMetric( + sampleAndroidFramePerf(session.device, appBundleId, { + adb: options.androidAdb, + }), + ) + : await settleMetric(sampleAppleFramePerf(session.device, appBundleId)); + response.metrics.fps = enrichFrameMetricWithSessionContext(buildMetricResult(result), session); +} + function supportsPlatformPerfMetrics(session: SessionState): boolean { return ( session.device.platform === 'android' || @@ -224,15 +294,30 @@ function buildPlatformSamplingMetadata(session: SessionState): Record { + if (session.device.platform === 'android') { + return { + method: ANDROID_FRAME_SAMPLE_METHOD, + description: ANDROID_FRAME_SAMPLE_DESCRIPTION, + unit: 'percent', + primaryField: 'droppedFramePercent', + window: 'since previous Android gfxinfo reset or app process start', + resetsAfterRead: true, + relatedActionsLimit: RELATED_PERF_ACTION_LIMIT, + }; + } + return buildAppleFrameSamplingMetadata(session.device); +} + async function sampleAndroidPerfResults( session: SessionState, + appBundleId: string, options: BuildPerfResponseOptions, ): Promise<{ memory: SettledMetricResult; cpu: SettledMetricResult; fps: SettledMetricResult; }> { - const appBundleId = session.appBundleId as string; const androidPerfOptions = { adb: options.androidAdb }; const [memory, cpu, fps] = await Promise.allSettled([ sampleAndroidMemoryPerf(session.device, appBundleId, androidPerfOptions), @@ -242,12 +327,14 @@ async function sampleAndroidPerfResults( return { memory, cpu, fps }; } -async function sampleApplePerfResultsForSession(session: SessionState): Promise<{ +async function sampleApplePerfResultsForSession( + session: SessionState, + appBundleId: string, +): Promise<{ memory: SettledMetricResult; cpu: SettledMetricResult; fps: SettledMetricResult; }> { - const appBundleId = session.appBundleId as string; const fps = await settleMetric(sampleAppleFramePerf(session.device, appBundleId)); const processSample = await settleMetric(sampleApplePerfMetrics(session.device, appBundleId)); if (processSample.status === 'fulfilled') { diff --git a/src/platforms/ios/perf.ts b/src/platforms/ios/perf.ts index 8de312dba..d6f289964 100644 --- a/src/platforms/ios/perf.ts +++ b/src/platforms/ios/perf.ts @@ -158,24 +158,27 @@ export async function sampleAppleFramePerf( }); } +export function buildAppleFrameSamplingMetadata(device: DeviceInfo): Record { + return device.platform === 'ios' && device.kind === 'device' + ? { + method: APPLE_FRAME_SAMPLE_METHOD, + description: APPLE_FRAME_SAMPLE_DESCRIPTION, + unit: 'percent', + primaryField: 'droppedFramePercent', + window: `short ${IOS_DEVICE_FRAME_TRACE_DURATION} xctrace Animation Hitches record of the active app process`, + resetsAfterRead: false, + } + : { + method: APPLE_FRAME_SAMPLE_METHOD, + description: + 'Unavailable on iOS simulators and macOS because local Apple tooling does not expose reliable app frame hitches for these targets.', + unit: 'percent', + primaryField: 'droppedFramePercent', + }; +} + export function buildAppleSamplingMetadata(device: DeviceInfo): Record { - const fps = - device.platform === 'ios' && device.kind === 'device' - ? { - method: APPLE_FRAME_SAMPLE_METHOD, - description: APPLE_FRAME_SAMPLE_DESCRIPTION, - unit: 'percent', - primaryField: 'droppedFramePercent', - window: `short ${IOS_DEVICE_FRAME_TRACE_DURATION} xctrace Animation Hitches record of the active app process`, - resetsAfterRead: false, - } - : { - method: APPLE_FRAME_SAMPLE_METHOD, - description: - 'Unavailable on iOS simulators and macOS because local Apple tooling does not expose reliable app frame hitches for these targets.', - unit: 'percent', - primaryField: 'droppedFramePercent', - }; + const fps = buildAppleFrameSamplingMetadata(device); if (device.platform === 'ios' && device.kind === 'device') { return { fps, diff --git a/src/utils/__tests__/perf-args.test.ts b/src/utils/__tests__/perf-args.test.ts new file mode 100644 index 000000000..fbbbce493 --- /dev/null +++ b/src/utils/__tests__/perf-args.test.ts @@ -0,0 +1,19 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { parseArgs, usageForCommand } from '../args.ts'; + +test('parseArgs accepts perf area subcommands', () => { + const metrics = parseArgs(['perf', 'metrics'], { strictFlags: true }); + assert.equal(metrics.command, 'perf'); + assert.deepEqual(metrics.positionals, ['metrics']); + + const frames = parseArgs(['perf', 'frames'], { strictFlags: true }); + assert.equal(frames.command, 'perf'); + assert.deepEqual(frames.positionals, ['frames']); +}); + +test('usageForCommand advertises perf area subcommands for metrics alias', () => { + const help = usageForCommand('metrics'); + assert.equal(help === null, false); + assert.match(help ?? '', /agent-device perf \[metrics\|frames\]/); +}); diff --git a/src/utils/cli-command-overrides.ts b/src/utils/cli-command-overrides.ts index 9eff15bbd..77e9ec888 100644 --- a/src/utils/cli-command-overrides.ts +++ b/src/utils/cli-command-overrides.ts @@ -138,6 +138,14 @@ const CLI_COMMAND_OVERRIDES = { appstate: { helpDescription: 'Show foreground app/activity', }, + perf: { + usageOverride: 'perf [metrics|frames] [sample]', + listUsageOverride: 'perf [metrics|frames]', + helpDescription: + 'Show session performance metrics or focused frame/jank health. Bare perf and metrics are aliases for perf metrics.', + summary: 'Show session performance and frame health', + positionalArgs: ['area?', 'action?'], + }, metro: { usageOverride: 'metro prepare (--public-base-url | --proxy-base-url ) [--project-root ] [--port ] [--kind auto|react-native|expo]\n agent-device metro reload [--metro-host ] [--metro-port ] [--bundle-url ]', diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index 027b9dc69..f405edfb4 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -209,7 +209,7 @@ Validation and evidence: If task says snapshot, use snapshot. If it asks visual evidence, use screenshot. Icon/tappable visual proof: screenshot --overlay-refs. Flag is --overlay-refs. If snapshot returns a sparse/AX-unavailable state, refs are not reliable. Use plain screenshot, not screenshot --overlay-refs, navigate with coordinates if needed, then retry snapshot -i after reaching another screen; the AX failure may be screen-specific. - Startup/frame health/CPU/memory: perf --json or metrics. Replay maintenance: replay -u ./flow.ad. + Startup/CPU/memory/frame first pass: perf metrics --json (bare perf and metrics are aliases). Focused frame/jank health: perf frames --json. Replay maintenance: replay -u ./flow.ad. Recording: record start/stop. By default, stop burns touch overlays into the video; use record start --hide-touches for the fastest raw recording. Android adb screenrecord has a 180s platform limit, so longer Android recordings are returned as multiple MP4 chunks. For gesture-heavy iOS simulator proof videos, prefer --hide-touches because overlay timing depends on a stable runner session while gestures are executing. Tracing: trace start ./trace.log, trace stop ./trace.log. Paths are positional. Stable known flow: batch ./steps.json, not workflow batch. Inline batch JSON example: @@ -367,7 +367,7 @@ Example: agent-device react-devtools profile report @c5 agent-device network dump --include headers -Use snapshot, screenshot, logs, network, and perf for device/app runtime evidence. Use react-devtools only when component internals or React rendering behavior matters.`, +Use snapshot, screenshot, logs, network, and perf metrics for device/app runtime evidence. Use react-devtools only when component internals or React rendering behavior matters.`, }, 'react-native': { summary: 'React Native app automation hazards and routing', @@ -406,12 +406,12 @@ Overlays and busy RN UIs: React DevTools routing: Keep the agent-device react-devtools prefix on every React DevTools command. Use help react-devtools for status/wait, component trees, props/state/hooks, profile windows, slow renders, rerenders, and remote bridge rules. - If React DevTools cannot connect, report status and continue with logs, network, perf, screenshot, and trace evidence instead of blocking the whole flow. + If React DevTools cannot connect, report status and continue with logs, network, perf metrics, screenshot, and trace evidence instead of blocking the whole flow. Slow-flow investigation: Keep one session, open the app, and snapshot -i. Use help react-devtools for the narrow React profile window. - Use help debugging for logs clear --restart, logs mark, network dump --include headers, perf --json, traces, and runtime failure evidence. + Use help debugging for logs clear --restart, logs mark, network dump --include headers, perf metrics --json, traces, and runtime failure evidence. For 15-20s async work, use wait with the exact expected text or selector instead of repeated snapshots. Report React render offenders separately from network/backend waits and device frame/CPU/memory findings.`, }, @@ -567,7 +567,7 @@ Rules: Re-snapshot after each mutation. Keep commands in the report reproducible; use selectors or refs from fresh snapshots, not guessed coordinates. Prefer refs for exploration and selectors for deterministic replay. - Use logs, network, screenshot --overlay-refs, trace, perf, or react-devtools only when they add evidence to a specific issue. + Use logs, network, screenshot --overlay-refs, trace, perf metrics, perf frames, or react-devtools only when they add evidence to a specific issue. Never delete screenshots, videos, traces, or report artifacts during a session. Escalate to help debugging or help react-devtools when runtime symptoms require those tools.`, }, diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index d2d0e376d..97cc772e0 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -816,6 +816,36 @@ async function runAndroidAppControlAndObservabilityWorkflow( JSON.stringify(metrics.fps), ); + const explicitMetrics = await client.observability.perf({ area: 'metrics', ...selection }); + assert.deepEqual(Object.keys(explicitMetrics.metrics as Record).sort(), [ + 'cpu', + 'fps', + 'memory', + 'startup', + ]); + + const frameCallStart = world.adbCalls.length; + const frames = await client.observability.perf({ + area: 'frames', + action: 'sample', + ...selection, + }); + const frameMetrics = frames.metrics as Record; + assert.deepEqual(Object.keys(frameMetrics), ['fps']); + assert.equal(frameMetrics.fps?.available, true, JSON.stringify(frames)); + assert.equal(frameMetrics.fps?.droppedFramePercent, 25); + assert.deepEqual(Object.keys(frames.sampling as Record), ['fps']); + assert.deepEqual(world.adbCalls.slice(frameCallStart), [ + ['shell', 'dumpsys', 'gfxinfo', 'com.example.demo', 'framestats'], + ['shell', 'dumpsys', 'gfxinfo', 'com.example.demo', 'reset'], + ]); + + const invalidPerfAction = await world.daemon.callCommand('perf', ['metrics', 'poll'], { + platform: 'android', + serial: PROVIDER_SCENARIO_ANDROID.id, + }); + assertRpcError(invalidPerfAction, 'INVALID_ARGS', /perf action must be sample/i); + const logsStop = await client.observability.logs({ action: 'stop', ...selection }); assert.equal(logsStop.stopped, true); diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index a7ca67e6f..837bbc360 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -1310,9 +1310,25 @@ const SKILL_GUIDANCE_CASES: Case[] = [ 'Need session startup, memory, and CPU data as JSON', ], task: 'Plan the commands to open the app first if needed, then collect session performance metrics as JSON.', - outputs: [plannedCommand('open'), plannedCommandAlternatives(['perf', 'metrics']), /--json/i], + outputs: [ + plannedCommand('open'), + plannedCommandAlternatives(['perf metrics', 'metrics']), + /--json/i, + ], forbiddenOutputs: [plannedCommand('network')], }), + makeCase({ + id: 'perf-session-frames', + contract: [ + 'App name: Agent Device Tester', + 'Platform: Android emulator', + 'The app is already open', + 'Need focused frame and jank health as JSON', + ], + task: 'Plan the command to collect focused frame and jank health as JSON without collecting React component render profiling.', + outputs: [plannedCommand('perf frames'), /--json/i], + forbiddenOutputs: [plannedCommand('react-devtools'), plannedCommand('network')], + }), makeCase({ id: 'react-devtools-profile-search', contract: [ diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 79e3c20c1..24b256562 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -256,7 +256,7 @@ Additional CLI-backed methods are exposed on their domain groups with typed opti - `client.recording.record()` and `client.recording.trace()` - `client.settings.update()` -`client.observability.perf()` returns daemon-shaped JSON so local and remote transports expose the same metrics payload. On Android and supported Apple targets, `data.metrics.fps.droppedFramePercent` is the primary frame-smoothness value. Android derives it from the current `adb shell dumpsys gfxinfo framestats` window; connected iOS devices derive it from `xcrun xctrace` Animation Hitches for the active app process. Frame samples include `windowStartedAt`, `windowEndedAt`, and `worstWindows` so agents can correlate dropped-frame clusters with logs, network entries, and their own session actions. A successful Android read resets Android frame stats; `open ` resets the Android frame window too, so agents can call `perf`, perform a transition or gesture, then call `perf` again to inspect that focused window. iOS simulator and macOS app sessions report frame health as unavailable rather than inventing FPS or dropped-frame values. +`client.observability.perf()` returns daemon-shaped JSON so local and remote transports expose the same metrics payload. Pass `{ area: 'metrics' }` for the broad startup/CPU/memory/frame first pass, or `{ area: 'frames' }` for a focused frame/jank-health payload. On Android and supported Apple targets, `data.metrics.fps.droppedFramePercent` is the primary frame-smoothness value. Android derives it from the current `adb shell dumpsys gfxinfo framestats` window; connected iOS devices derive it from `xcrun xctrace` Animation Hitches for the active app process. Frame samples include `windowStartedAt`, `windowEndedAt`, and `worstWindows` so agents can correlate dropped-frame clusters with logs, network entries, and their own session actions. A successful Android read resets Android frame stats; `open ` resets the Android frame window too, so agents can call `perf({ area: 'frames' })`, perform a transition or gesture, then call it again to inspect that focused window. iOS simulator and macOS app sessions report frame health as unavailable rather than inventing FPS or dropped-frame values. `client.recording.record({ action: 'start', path, quality: 5 })` starts a smaller 50% resolution video; omit `quality` to keep native/current resolution. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 9a0ee76be..cb7683171 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -577,9 +577,12 @@ agent-device keyboard dismiss ```bash agent-device perf --json agent-device metrics --json +agent-device perf metrics --json +agent-device perf frames --json ``` -- `perf` (alias: `metrics`) returns a session-scoped metrics JSON blob. +- `perf metrics` returns a session-scoped metrics JSON blob. Bare `perf` and `metrics` remain aliases for `perf metrics`. +- `perf frames` returns a focused frame/jank-health JSON blob from the same frame sampling source used by `perf metrics`. - Without `--json`, `perf` prints a compact summary: frame health when reliable frame data is available, otherwise CPU/memory when those samples are available. - `startup` is sampled from `open-command-roundtrip`: elapsed wall-clock time around each `open` command dispatch for the active session app target. - Android app sessions with an active package also sample: @@ -594,10 +597,10 @@ agent-device metrics --json - `startup`: iOS simulator, iOS physical device, Android emulator/device - `memory` and `cpu`: Android emulator/device, macOS app sessions, iOS simulators with an active app session (`open ` first), and iOS physical devices with an active app session - `fps`: Android emulator/device app sessions and connected iOS device app sessions. iOS simulator and macOS frame health is reported unavailable because Apple tooling does not expose trustworthy app hitch data there. -- If no startup sample exists yet for the session, run `open ` first and retry `perf`. +- If no startup sample exists yet for the session, run `open ` first and retry `perf metrics`. - Android URL/deep-link opens infer the foreground package after launch when possible, including Expo Go/dev-client shells. If the session still has no app package/bundle ID, package-bound metrics remain unavailable until you `open `. -- Android frame health is reset after each successful `perf` read and after `open `, so run `perf`, perform the interaction, then run `perf` again for a focused window. -- On physical iOS devices, `perf` records short `xcrun xctrace` Activity Monitor and Animation Hitches samples. Keep the device unlocked, connected, and the app active in the foreground while sampling. +- Android frame health is reset after each successful `perf metrics` or `perf frames` read and after `open `, so run `perf frames`, perform the interaction, then run `perf frames` again for a focused window. +- On physical iOS devices, `perf metrics` and `perf frames` record short `xcrun xctrace` samples. Keep the device unlocked, connected, and the app active in the foreground while sampling. - Interpretation note: this startup metric is command round-trip timing and does not represent true first frame / first interactive app instrumentation. - CPU data is a lightweight process snapshot, so an idle app may legitimately read as `0`. @@ -623,7 +626,7 @@ agent-device react-devtools profile report @c5 - Use it when a React Native workflow needs component hierarchy, props, state, hooks, render causes, slow components, or re-render counts. - For profiling, keep the window narrow and make one bounded first-pass survey: use the `profile stop` summary, run `profile slow --limit 5` and `profile rerenders --limit 5` once, add `profile timeline --limit 20` only when commit timing matters, then drill into a specific `@c` ref with `profile report`. - Do not repeatedly raise broad `profile slow` limits such as `--limit 50`, `--limit 200`, or `--limit 500` unless you have a specific target that needs more rows. -- Keep using `snapshot`, `press`, `fill`, `logs`, `network`, and `perf` for device/app runtime evidence. Use `react-devtools` for React internals. +- Keep using `snapshot`, `press`, `fill`, `logs`, `network`, `perf metrics`, and `perf frames` for device/app runtime evidence. Use `react-devtools` for React internals. - For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, start with `agent-device help react-native`. - On Android, use `alert get`, `alert wait `, `alert accept`, and `alert dismiss` for runtime permission prompts and native alerts. On iOS, use the same alert commands for XCTest alerts, app-owned modal popups with native blocking markers, and blocking system dialogs. Do not use `settings permission` to answer a dialog already on screen; reserve it for setup or resetting permission state before a flow. - React Native development builds can connect to the DevTools daemon on port 8097. For Android emulators or physical devices, run `adb reverse tcp:8097 tcp:8097` if the app cannot reach the host. diff --git a/website/docs/docs/debugging-profiling.md b/website/docs/docs/debugging-profiling.md index 8fc804d1f..ac54971cb 100644 --- a/website/docs/docs/debugging-profiling.md +++ b/website/docs/docs/debugging-profiling.md @@ -10,7 +10,7 @@ Use `agent-device` when the task moves past UI automation and you need runtime e - Session app logs for targeted debugging windows - Network inspection from recent HTTP(s) entries in app logs via `network dump` -- Performance snapshots with `perf` / `metrics` +- Performance snapshots with `perf metrics` / `perf frames` - Screenshots, recordings, and replayable repro flows ## React Native component internals @@ -46,11 +46,11 @@ agent-device logs clear --restart agent-device logs mark "before repro" agent-device press 'id="submit"' agent-device network dump 25 --include headers -agent-device perf --json +agent-device perf metrics --json agent-device logs path ``` -Use this flow when you need a clean repro window with logs, recent network activity, and a quick perf sample from the active app session. +Use this flow when you need a clean repro window with logs, recent network activity, and a quick metrics sample from the active app session. On iOS simulators, `logs` scope by bundle id and the resolved app executable. For launch-time stdout/stderr, capture the direct app launch console instead of starting raw `simctl` streams: @@ -94,12 +94,15 @@ agent-device network dump 25 --include all ```bash agent-device perf --json agent-device metrics --json +agent-device perf metrics --json +agent-device perf frames --json ``` -- `perf` returns session-scoped startup and, where supported, CPU, memory, and Android frame-health samples. +- `perf metrics` returns session-scoped startup and, where supported, CPU, memory, and frame-health samples. Bare `perf` and `metrics` remain aliases. +- `perf frames` returns a focused frame/jank-health payload. - Startup is measured around the `open` command; it is not first-frame instrumentation. - CPU, memory, and Android frame-health availability depend on platform and whether the active session is bound to an app/package. -- On Android, use `metrics.fps.droppedFramePercent` for the health check and `metrics.fps.worstWindows` to line up jank clusters with logs, network activity, or recent actions. +- On Android and supported Apple targets, use `metrics.fps.droppedFramePercent` for the health check and `metrics.fps.worstWindows` to line up jank clusters with logs, network activity, or recent actions. ## Where to go deeper