diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 27ea2061716a5..979f15c1f34c5 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -105,7 +105,7 @@ export class FullConfigInternal { globalTimeout: takeFirst(configCLIOverrides.debug ? 0 : undefined, configCLIOverrides.globalTimeout, userConfig.globalTimeout, 0), grep: takeFirst(userConfig.grep, defaultGrep), grepInvert: takeFirst(userConfig.grepInvert, null), - maxFailures: takeFirst(configCLIOverrides.debug ? 1 : undefined, configCLIOverrides.maxFailures, userConfig.maxFailures, 0), + maxFailures: takeFirst(configCLIOverrides.debug === 'inspector' ? 1 : undefined, configCLIOverrides.maxFailures, userConfig.maxFailures, 0), metadata: metadata ?? userConfig.metadata, preserveOutput: takeFirst(userConfig.preserveOutput, 'always'), projects: [], diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index c6aa1b0a71f26..e8457c610d8be 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -23,7 +23,7 @@ import type { ReporterDescription, TestInfoError, TestStatus } from '../../types import type { SerializedCompilationCache } from '../transform/compilationCache'; export type ConfigCLIOverrides = { - debug?: boolean; + debug?: 'cli' | 'inspector'; failOnFlakyTests?: boolean; forbidOnly?: boolean; fullyParallel?: boolean; diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index f430dafdc5ee2..247dcc563a072 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -22,7 +22,7 @@ import { setBoxedStackPrefixes, createGuid, currentZone, debugMode, jsonStringif import { currentTestInfo } from './common/globals'; import { rootTestType } from './common/testType'; -import { createCustomMessageHandler } from './mcp/test/browserBackend'; +import { createCustomMessageHandler, handleOnTestFunctionEnd } from './mcp/test/browserBackend'; import type { Fixtures, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, ScreenshotMode, TestInfo, TestType, VideoMode } from '../types/test'; import type { ContextReuseMode } from './common/config'; @@ -424,19 +424,24 @@ const playwrightFixtures: Fixtures = ({ await use(reuse); }, { scope: 'worker', title: 'context', box: true }], - context: async ({ browser, _reuseContext, _contextFactory }, use, testInfo) => { + context: async ({ browser, _reuseContext, _contextFactory }, use, info) => { + const testInfo = info as TestInfoImpl; const browserImpl = browser as BrowserImpl; attachConnectedHeaderIfNeeded(testInfo, browserImpl); if (!_reuseContext) { const { context, close } = await _contextFactory(); - (testInfo as TestInfoImpl)._onCustomMessageCallback = createCustomMessageHandler(testInfo, context); + testInfo._onCustomMessageCallback = createCustomMessageHandler(testInfo, context); + if (testInfo._configInternal.configCLIOverrides.debug === 'cli') + testInfo._onDidFinishTestFunctionCallbacks.add(() => handleOnTestFunctionEnd(testInfo, context)); await use(context); await close(); return; } const context = await browserImpl._wrapApiCall(() => browserImpl._newContextForReuse(), { internal: true }); - (testInfo as TestInfoImpl)._onCustomMessageCallback = createCustomMessageHandler(testInfo, context); + testInfo._onCustomMessageCallback = createCustomMessageHandler(testInfo, context); + if (testInfo._configInternal.configCLIOverrides.debug === 'cli') + testInfo._onDidFinishTestFunctionCallbacks.add(() => handleOnTestFunctionEnd(testInfo, context)); await use(context); const closeReason = testInfo.status === 'timedOut' ? 'Test timeout of ' + testInfo.timeout + 'ms exceeded.' : 'Test ended.'; await browserImpl._wrapApiCall(() => browserImpl._disconnectFromReusedContext(closeReason), { internal: true }); @@ -700,7 +705,7 @@ class ArtifactsRecorder { async willStartTest(testInfo: TestInfoImpl) { this._testInfo = testInfo; - testInfo._onDidFinishTestFunctionCallback = () => this.didFinishTestFunction(); + testInfo._onDidFinishTestFunctionCallbacks.add(() => this.didFinishTestFunction()); this._screenshotRecorder.fixOrdinal(); diff --git a/packages/playwright/src/mcp/terminal/commands.ts b/packages/playwright/src/mcp/terminal/commands.ts index bf3ebf5096828..36ab6109b32cb 100644 --- a/packages/playwright/src/mcp/terminal/commands.ts +++ b/packages/playwright/src/mcp/terminal/commands.ts @@ -804,6 +804,18 @@ const sessionList = declareCommand({ toolParams: () => ({}), }); +const sessionAttach = declareCommand({ + name: 'attach', + description: 'Attach an external browser session', + category: 'browsers', + hidden: true, + args: z.object({ + socket: z.string().describe('Socket path of the external browser session.'), + }), + toolName: '', + toolParams: ({ socket }) => ({ socket }), +}); + const sessionCloseAll = declareCommand({ name: 'close-all', description: 'Close all browser sessions', @@ -962,6 +974,7 @@ const commandsArray: AnyCommandSchema[] = [ // session category sessionList, + sessionAttach, sessionCloseAll, killAll, diff --git a/packages/playwright/src/mcp/terminal/program.ts b/packages/playwright/src/mcp/terminal/program.ts index 405e2f33ee387..e39211a5d8e66 100644 --- a/packages/playwright/src/mcp/terminal/program.ts +++ b/packages/playwright/src/mcp/terminal/program.ts @@ -163,19 +163,45 @@ export async function program() { console.log(result.text); return; } - - case 'close': + case 'close': { const closeEntry = registry.entry(clientInfo, sessionName); const session = closeEntry ? new Session(clientInfo, closeEntry.config) : undefined; + if (session?.isAttached()) { + await session.deleteSessionConfig(); + return; + } if (!session || !await session.canConnect()) { console.log(`Browser '${sessionName}' is not open.`); return; } await session.stop(); return; - case 'install': + } + case 'attach': { + if (sessionName === 'default') { + console.log(`Cannot attach 'default' session.`); + return; + } + const sessionConfig: SessionConfig = { + name: sessionName, + version: clientInfo.version, + socketPath: args._[1], + timestamp: Date.now(), + cli: { attached: true }, + workspaceDir: clientInfo.workspaceDir, + }; + const session = new Session(clientInfo, sessionConfig); + if (!await session.canConnect()) { + console.log(`Cannot connect to '${sessionConfig.socketPath}'.`); + return; + } + await session.writeSessionConfig(); + return; + } + case 'install': { await install(args); return; + } case 'show': { const daemonScript = path.join(__dirname, 'devtoolsApp.js'); const child = spawn(process.execPath, [daemonScript], { @@ -289,7 +315,7 @@ function defaultConfigFile(): string { return path.resolve('.playwright', 'cli.config.json'); } -function sessionConfigFromArgs(clientInfo: ClientInfo, sessionName: string, args: MinimistArgs): SessionConfig { +export function sessionConfigFromArgs(clientInfo: ClientInfo, sessionName: string, args: MinimistArgs): SessionConfig { let config = args.config ? path.resolve(args.config) : undefined; try { if (!config && fs.existsSync(defaultConfigFile())) @@ -417,6 +443,8 @@ async function renderSessionStatus(session: Session) { const config = session.config; const canConnect = await session.canConnect(); text.push(`- ${session.name}:`); + if (session.isAttached()) + text.push(` - attached to external browser`); text.push(` - status: ${canConnect ? 'open' : 'closed'}`); if (canConnect && !session.isCompatible()) text.push(` - version: v${config.version} [incompatible please re-open]`); diff --git a/packages/playwright/src/mcp/terminal/registry.ts b/packages/playwright/src/mcp/terminal/registry.ts index e03eae6474d0b..894570724ae63 100644 --- a/packages/playwright/src/mcp/terminal/registry.ts +++ b/packages/playwright/src/mcp/terminal/registry.ts @@ -40,6 +40,7 @@ export type SessionConfig = { persistent?: boolean; profile?: string; config?: string; + attached?: boolean; }; userDataDirPrefix?: string; workspaceDir?: string; diff --git a/packages/playwright/src/mcp/terminal/session.ts b/packages/playwright/src/mcp/terminal/session.ts index 60daf0ebed8ae..54c4d5ed1cdcd 100644 --- a/packages/playwright/src/mcp/terminal/session.ts +++ b/packages/playwright/src/mcp/terminal/session.ts @@ -47,8 +47,12 @@ export class Session { this.name = options.name; } + isAttached() { + return !!this.config.cli.attached; + } + isCompatible(): boolean { - return this._clientInfo.version === this.config.version; + return this.isAttached() || this._clientInfo.version === this.config.version; } checkCompatible() { @@ -74,6 +78,12 @@ to restart the browser session.`); } async stop(quiet: boolean = false): Promise { + if (this.isAttached()) { + if (!quiet) + console.log(`Cannot close attached browser '${this.name}'.`); + return; + } + if (!await this.canConnect()) { if (!quiet) console.log(`Browser '${this.name}' is not open.`); @@ -199,14 +209,8 @@ to restart the browser session.`); } private async _startDaemon(): Promise { - await fs.promises.mkdir(this._clientInfo.daemonProfilesDir, { recursive: true }); const cliPath = path.join(__dirname, '../../../cli.js'); - - const sessionConfigFile = this._sessionFile('.session'); - this.config.version = this._clientInfo.version; - this.config.timestamp = Date.now(); - await fs.promises.writeFile(sessionConfigFile, JSON.stringify(this.config, null, 2)); - + const sessionConfigFile = await this.writeSessionConfig(); const errLog = this._sessionFile('.err'); const err = fs.openSync(errLog, 'w'); @@ -296,6 +300,14 @@ to restart the browser session.`); throw error; } + async writeSessionConfig() { + const sessionConfigFile = this._sessionFile('.session'); + this.config.version = this._clientInfo.version; + await fs.promises.mkdir(path.dirname(sessionConfigFile), { recursive: true }); + await fs.promises.writeFile(sessionConfigFile, JSON.stringify(this.config, null, 2)); + return sessionConfigFile; + } + async deleteSessionConfig() { await fs.promises.rm(this._sessionFile('.session')).catch(() => {}); } diff --git a/packages/playwright/src/mcp/test/DEPS.list b/packages/playwright/src/mcp/test/DEPS.list index 4d49a78fb2bcd..fe028776b2e1f 100644 --- a/packages/playwright/src/mcp/test/DEPS.list +++ b/packages/playwright/src/mcp/test/DEPS.list @@ -17,3 +17,7 @@ [backend.ts] ../browser/tools.ts + +[browserBackend.ts] +../terminal/daemon.ts +../terminal/program.ts diff --git a/packages/playwright/src/mcp/test/browserBackend.ts b/packages/playwright/src/mcp/test/browserBackend.ts index 7bed39f6bbfa2..d0b0f95509b91 100644 --- a/packages/playwright/src/mcp/test/browserBackend.ts +++ b/packages/playwright/src/mcp/test/browserBackend.ts @@ -14,12 +14,18 @@ * limitations under the License. */ +import path from 'path'; +import { createGuid } from 'playwright-core/lib/utils'; + import * as mcp from '../sdk/exports'; import { defaultConfig } from '../browser/config'; import { BrowserServerBackend } from '../browser/browserServerBackend'; import { Tab } from '../browser/tab'; import { stripAnsiEscapes } from '../../util'; import { identityBrowserContextFactory } from '../browser/browserContextFactory'; +import { startMcpDaemonServer } from '../terminal/daemon'; +import { sessionConfigFromArgs } from '../terminal/program'; +import { createClientInfo } from '../terminal/registry'; import type * as playwright from '../../../index'; import type { Page } from '../../../../playwright-core/src/client/page'; @@ -115,3 +121,34 @@ async function generatePausedMessage(testInfo: TestInfo, context: playwright.Bro return lines.join('\n'); } + +export async function handleOnTestFunctionEnd(testInfo: TestInfo, context: playwright.BrowserContext) { + const sessionConfig = sessionConfigFromArgs(createClientInfo(), createGuid().slice(0, 8), { _: [] }); + const socketPath = await startMcpDaemonServer({ + ...defaultConfig, + outputMode: 'file', + snapshot: { mode: 'full', output: 'file' }, + outputDir: path.resolve(process.cwd(), '.playwright-cli'), + sessionConfig, + }, identityBrowserContextFactory(context)); + + const lines = ['']; + if (testInfo.errors.length) { + lines.push(`### Paused on test error`); + for (const error of testInfo.errors) + lines.push(stripAnsiEscapes(error.message || '')); + } else { + lines.push(`### Paused at the end of the test`); + } + lines.push( + `### Debugging Instructions`, + `- Use "playwright-cli --session attach '${socketPath}'" to add a session.`, + `- Use "playwright-cli --session " to explore the page and fix the problem.`, + `- Stop this test run when finished. Restart if needed.`, + ); + lines.push(''); + + /* eslint-disable-next-line no-console */ + console.log(lines.join('\n')); + await new Promise(() => {}); +} diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index da4ad0f6483e3..85fadc0d2c23b 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -289,6 +289,7 @@ async function mergeReports(reportDir: string | undefined, opts: { [key: string] function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides { const overrides: ConfigCLIOverrides = { + debug: options.debug, failOnFlakyTests: options.failOnFlakyTests ? true : undefined, forbidOnly: options.forbidOnly ? true : undefined, fullyParallel: options.fullyParallel ? true : undefined, @@ -309,6 +310,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid runAgents: options.runAgents, workers: options.workers, pause: process.env.PWPAUSE ? true : undefined, + use: {}, }; if (options.browser) { @@ -324,16 +326,25 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid }); } - if (options.headed || options.debug || overrides.pause) - overrides.use = { headless: false }; - if (!options.ui && options.debug) { - overrides.debug = true; + if (options.headed) + overrides.use.headless = false; + if (options.trace) + overrides.use.trace = options.trace; + + if (overrides.debug === 'inspector') { + overrides.use.headless = false; process.env.PWDEBUG = '1'; } - if (!options.ui && options.trace) { - overrides.use = overrides.use || {}; - overrides.use.trace = options.trace; + if (overrides.debug === 'cli') { + overrides.timeout = 0; + overrides.use.actionTimeout = 5000; } + + if (options.ui || options.uiHost || options.uiPort) { + delete overrides.use.trace; + overrides.debug = undefined; + } + if (overrides.tsconfig && !fs.existsSync(overrides.tsconfig)) throw new Error(`--tsconfig "${options.tsconfig}" does not exist`); @@ -403,7 +414,7 @@ const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries const testOptions: [string, { description: string, choices?: string[], preset?: string }][] = [ /* deprecated */ ['--browser ', { description: `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")` }], ['-c, --config ', { description: `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"` }], - ['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options` }], + ['--debug [mode]', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options`, choices: ['cli', 'inspector'], preset: 'inspector' }], ['--fail-on-flaky-tests', { description: `Fail if any test is flagged as flaky (default: false)` }], ['--forbid-only', { description: `Fail if test.only is called (default: false)` }], ['--fully-parallel', { description: `Run all tests in parallel (default: false)` }], diff --git a/packages/playwright/src/skill/SKILL.md b/packages/playwright/src/skill/SKILL.md index 29182e7630425..f0dbb95e1442f 100644 --- a/packages/playwright/src/skill/SKILL.md +++ b/packages/playwright/src/skill/SKILL.md @@ -1,7 +1,7 @@ --- name: playwright-cli -description: Automates browser interactions for web testing, form filling, screenshots, and data extraction. Use when the user needs to navigate websites, interact with web pages, fill forms, take screenshots, test web applications, or extract information from web pages. -allowed-tools: Bash(playwright-cli:*) +description: Automate browser interactions, test web pages and work with Playwright tests. +allowed-tools: Bash(playwright-cli:*) Bash(npx:*) Bash(npm:*) --- # Browser Automation with playwright-cli @@ -250,6 +250,7 @@ playwright-cli close ## Specific tasks +* **Running and Debugging Playwright tests** [references/playwright-tests.md](references/playwright-tests.md) * **Request mocking** [references/request-mocking.md](references/request-mocking.md) * **Running Playwright code** [references/running-code.md](references/running-code.md) * **Browser session management** [references/session-management.md](references/session-management.md) diff --git a/packages/playwright/src/skill/references/playwright-tests.md b/packages/playwright/src/skill/references/playwright-tests.md new file mode 100644 index 0000000000000..5e2bf1489cfa2 --- /dev/null +++ b/packages/playwright/src/skill/references/playwright-tests.md @@ -0,0 +1,50 @@ +# Running Playwright Tests + +To run Playwright tests, use the `npx playwright test` command, or a package manager script. To avoid opening the interactive html report, use `PLAYWRIGHT_HTML_OPEN=never` environment variable. + +```bash +# Run all tests +PLAYWRIGHT_HTML_OPEN=never npx playwright test + +# Run all tests through a custom npm script +PLAYWRIGHT_HTML_OPEN=never npm run special-test-command +``` + +# Debugging Playwright Tests + +To debug a failing test, run it with Playwright as usual, but append `--debug=cli` option. This command will pause the test at the point of failure, and print the "socket path" and instructions. + +IMPORTANT: run the command in the background and check the output until instructions are available. + +Once instructions are printed, attach a test session to `playwright-cli` and use it to explore the page. + +```bash +# Choose a name (e.g. test1) and attach +playwright-cli --session=test1 attach '' + +# Explore the page and interact if needed +playwright-cli --session=test1 snapshot +playwright-cli --session=test1 click e14 +``` + +Keep the test running in the background while you explore and look for a fix. After fixing the test, stop the background test run. + +Every action you perform with `playwright-cli` generates corresponding Playwright TypeScript code. +This code appears in the output and can be copied directly into the test. Most of the time, a specific locator or an expectation should be updated. + +## Example Workflow + +```bash +# Run tests in background: +npx playwright test --grep "failing test title" --debug=cli +# ... and wait for the debugging instructions + +# Attach test session +playwright-cli --session=test1 attach '/path/to/socket/file' +# Take a snapshot to explore the page +playwright-cli --session=test1 snapshot +# Find the right button to click, and perform the action to verify it works as expected +playwright-cli --session=test1 click e17 + +# Update locator in the test file, based on "Ran Playwright code" snippets +``` diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index db8a4d6e2ec5e..74719dcd52cea 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -100,7 +100,7 @@ export class TestInfoImpl implements TestInfo { readonly _configInternal: FullConfigInternal; private readonly _steps: TestStepInternal[] = []; private readonly _stepMap = new Map(); - _onDidFinishTestFunctionCallback?: () => Promise; + _onDidFinishTestFunctionCallbacks = new Set<() => Promise>(); _onCustomMessageCallback?: (data: any) => Promise; _hasNonRetriableError = false; _hasUnhandledError = false; @@ -204,7 +204,7 @@ export class TestInfoImpl implements TestInfo { this.expectedStatus = test?.expectedStatus ?? 'skipped'; this._timeoutManager = new TimeoutManager(this.project.timeout); - if (configInternal.configCLIOverrides.debug) + if (configInternal.configCLIOverrides.debug === 'inspector') this._setDebugMode(); this.outputDir = (() => { @@ -478,7 +478,8 @@ export class TestInfoImpl implements TestInfo { this._interruptedPromise, ]); } - await this._onDidFinishTestFunctionCallback?.(); + for (const cb of this._onDidFinishTestFunctionCallbacks) + await cb(); } // ------------ TestInfo methods ------------ diff --git a/tests/mcp/cli-test.spec.ts b/tests/mcp/cli-test.spec.ts new file mode 100644 index 0000000000000..a61829599abe0 --- /dev/null +++ b/tests/mcp/cli-test.spec.ts @@ -0,0 +1,58 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import { test, expect } from './cli-fixtures'; +import { writeFiles } from './fixtures'; + +const testEntrypoint = path.join(__dirname, '../../packages/playwright-test/cli.js'); + +test.only('debug test and attach', async ({ cli, childProcess }) => { + await writeFiles({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('example test', async ({ page }) => { + await page.setContent(''); + await expect(page.getByRole('button', { name: 'Missing' })).toBeVisible({ timeout: 1000 }); + }); + `, + }); + + const testProcess = childProcess({ + command: [testEntrypoint, 'test', '--debug=cli'], + cwd: test.info().outputDir, + }); + + await testProcess.waitForOutput('playwright-cli --session attach'); + const match = testProcess.output.match(/attach '([^']+)'/); + const socketPath = match[1]; + + const attachResult = await cli('--session=test', 'attach', socketPath); + expect(attachResult.exitCode).toBe(0); + + const snapshotResult = await cli('--session=test', 'snapshot'); + expect(snapshotResult.exitCode).toBe(0); + expect(snapshotResult.snapshot).toContain('button "Submit"'); + + const closeResult = await cli('--session=test', 'close'); + expect(closeResult.exitCode).toBe(0); + + const listResult = await cli('list'); + expect(listResult.exitCode).toBe(0); + expect(listResult.output).toContain('(no browsers)'); + + await testProcess.kill('SIGINT'); +});