From 5e9e04728e077388476f85bd2c4b4dd84e1b2db4 Mon Sep 17 00:00:00 2001 From: Phyllis Wu Date: Thu, 19 Mar 2026 10:42:51 -0400 Subject: [PATCH 1/4] QA to E2E: app basic flow (no extensions) --- packages/e2e/setup/auth.ts | 2 + packages/e2e/setup/cli.ts | 2 + packages/e2e/tests/app-basic.spec.ts | 122 +++++++++++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 packages/e2e/tests/app-basic.spec.ts diff --git a/packages/e2e/setup/auth.ts b/packages/e2e/setup/auth.ts index 5a37752bef3..04d77661b70 100644 --- a/packages/e2e/setup/auth.ts +++ b/packages/e2e/setup/auth.ts @@ -22,6 +22,8 @@ export const authFixture = cliFixture.extend<{}, {authLogin: void}>({ return } + process.stdout.write('[e2e] Authenticating automatically — no action required.\n') + // Clear any existing session await execa('node', [executables.cli, 'auth', 'logout'], { env: env.processEnv, diff --git a/packages/e2e/setup/cli.ts b/packages/e2e/setup/cli.ts index 1f39f38ca68..a5cb5216fcb 100644 --- a/packages/e2e/setup/cli.ts +++ b/packages/e2e/setup/cli.ts @@ -52,6 +52,7 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({ const execaOpts: ExecaOptions = { cwd: opts.cwd, env: {...env.processEnv, ...opts.env}, + extendEnv: false, timeout, reject: false, } @@ -75,6 +76,7 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({ const execaOpts: ExecaOptions = { cwd: opts.cwd, env: {...env.processEnv, ...opts.env}, + extendEnv: false, timeout, reject: false, } diff --git a/packages/e2e/tests/app-basic.spec.ts b/packages/e2e/tests/app-basic.spec.ts new file mode 100644 index 00000000000..6ee01df80e4 --- /dev/null +++ b/packages/e2e/tests/app-basic.spec.ts @@ -0,0 +1,122 @@ +import {appScaffoldFixture as test} from '../setup/app.js' +import {requireEnv} from '../setup/env.js' +import {expect} from '@playwright/test' +import {writeFileSync} from 'node:fs' +import {join} from 'node:path' + +test.describe('App basic flow (no extensions)', () => { + test('init, dev, execute, quit, clean, deploy, versions, config link, deploy to secondary', async ({ + appScaffold, + cli, + env, + }) => { + // Full flow: init + dev (3 min) + deploy + config link + secondary deploy — needs 10 min + test.setTimeout(10 * 60 * 1000) + + requireEnv(env, 'clientId', 'storeFqdn', 'secondaryClientId') + + // Step 1: Create a React Router app + const initResult = await appScaffold.init({ + template: 'reactRouter', + flavor: 'javascript', + packageManager: 'npm', + }) + expect(initResult.exitCode, `app init failed:\n${initResult.stdout}\n${initResult.stderr}`).toBe(0) + + // Step 2: Start dev server via PTY + // Unset CI so keyboard shortcuts are enabled in the Dev UI + const dev = await cli.spawn(['app', 'dev', '--path', appScaffold.appDir], {env: {CI: ''}}) + try { + await dev.waitForOutput('Ready, watching for changes in your app', 3 * 60 * 1000) + + // Step 3: Run a GraphQL query while the dev server is running + const executeResult = await cli.exec( + ['app', 'execute', '--query', 'query { shop { name } }', '--path', appScaffold.appDir], + {timeout: 60 * 1000}, + ) + const executeOutput = executeResult.stdout + executeResult.stderr + expect(executeResult.exitCode, `app execute failed:\n${executeOutput}`).toBe(0) + expect(executeOutput).toContain('shop') + + // Step 4: Press q to quit the dev server + dev.sendKey('q') + const devExitCode = await dev.waitForExit(30_000) + expect(devExitCode).toBe(0) + } finally { + // Step 5: Always clean up the dev preview, even if the test fails + dev.kill() + const cleanResult = await cli.exec(['app', 'dev', 'clean', '--path', appScaffold.appDir]) + const cleanOutput = cleanResult.stdout + cleanResult.stderr + expect(cleanResult.exitCode, `dev clean failed:\n${cleanOutput}`).toBe(0) + expect(cleanOutput).toContain('Dev preview stopped') + } + + // Step 6: Deploy the primary app + const versionTag = `e2e-v-${Date.now()}` + const deployResult = await cli.exec( + [ + 'app', + 'deploy', + '--path', + appScaffold.appDir, + '--force', + '--version', + versionTag, + '--message', + 'E2E basic flow deployment', + ], + {timeout: 5 * 60 * 1000}, + ) + const deployOutput = deployResult.stdout + deployResult.stderr + expect(deployResult.exitCode, `deploy failed:\n${deployOutput}`).toBe(0) + + // Step 7: List versions and verify our tag appears + const listResult = await cli.exec(['app', 'versions', 'list', '--path', appScaffold.appDir, '--json'], { + timeout: 60 * 1000, + }) + const listOutput = listResult.stdout + listResult.stderr + expect(listResult.exitCode, `versions list failed:\n${listOutput}`).toBe(0) + expect(listOutput).toContain(versionTag) + + // Step 8: Config link to the secondary app + // Pre-create a minimal TOML stub so getTomls() finds the secondary client ID and skips + // the interactive "Configuration file name" prompt entirely. This avoids PTY timing races + // where the Enter key arrives before ink has fully initialized the text prompt, which + // causes renderTextPrompt to return '' → filenameFromName('') = 'shopify.app.toml' → + // that file already exists → overwrite confirmation prompt hangs. + // (--config and --client-id are mutually exclusive flags, so we can't pass both directly.) + writeFileSync( + join(appScaffold.appDir, 'shopify.app.secondary.toml'), + `client_id = "${env.secondaryClientId}"\n`, + ) + + const configLink = await cli.spawn( + ['app', 'config', 'link', '--path', appScaffold.appDir, '--client-id', env.secondaryClientId], + {env: {CI: '', SHOPIFY_FLAG_CLIENT_ID: undefined}}, + ) + await configLink.waitForOutput('is now linked to', 2 * 60 * 1000) + const configLinkExitCode = await configLink.waitForExit(30_000) + expect(configLinkExitCode, `config link failed:\n${configLink.getOutput()}`).toBe(0) + + // Step 9: Deploy to the secondary app using the linked config file + const secondaryVersionTag = `e2e-secondary-v-${Date.now()}` + const secondaryDeployResult = await cli.exec( + [ + 'app', + 'deploy', + '--path', + appScaffold.appDir, + '--config', + 'secondary', + '--force', + '--version', + secondaryVersionTag, + '--message', + 'E2E secondary app deployment', + ], + {timeout: 5 * 60 * 1000, env: {SHOPIFY_FLAG_CLIENT_ID: undefined}}, + ) + const secondaryDeployOutput = secondaryDeployResult.stdout + secondaryDeployResult.stderr + expect(secondaryDeployResult.exitCode, `secondary deploy failed:\n${secondaryDeployOutput}`).toBe(0) + }) +}) From 09b47d5eed31f56c86a5be76a0271e53da57dbb4 Mon Sep 17 00:00:00 2001 From: Phyllis Wu Date: Thu, 19 Mar 2026 12:05:30 -0400 Subject: [PATCH 2/4] fix lint issues --- packages/e2e/package.json | 1 + packages/e2e/setup/auth.ts | 1 - packages/e2e/tests/app-basic.spec.ts | 8 ++++---- pnpm-lock.yaml | 14 ++++---------- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/e2e/package.json b/packages/e2e/package.json index 8b9d5de63e8..eb14548088a 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -30,6 +30,7 @@ }, "devDependencies": { "@playwright/test": "^1.50.0", + "@shopify/cli-kit": "3.92.0", "@types/node": "18.19.70", "execa": "^7.2.0", "node-pty": "^1.0.0", diff --git a/packages/e2e/setup/auth.ts b/packages/e2e/setup/auth.ts index 04d77661b70..93f224c1844 100644 --- a/packages/e2e/setup/auth.ts +++ b/packages/e2e/setup/auth.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-restricted-imports */ import {cliFixture} from './cli.js' import {executables} from './env.js' import {stripAnsi} from '../helpers/strip-ansi.js' diff --git a/packages/e2e/tests/app-basic.spec.ts b/packages/e2e/tests/app-basic.spec.ts index 6ee01df80e4..c51eb383acc 100644 --- a/packages/e2e/tests/app-basic.spec.ts +++ b/packages/e2e/tests/app-basic.spec.ts @@ -1,8 +1,8 @@ import {appScaffoldFixture as test} from '../setup/app.js' import {requireEnv} from '../setup/env.js' import {expect} from '@playwright/test' -import {writeFileSync} from 'node:fs' -import {join} from 'node:path' +import {joinPath} from '@shopify/cli-kit/node/path' +import * as fs from 'fs' test.describe('App basic flow (no extensions)', () => { test('init, dev, execute, quit, clean, deploy, versions, config link, deploy to secondary', async ({ @@ -85,8 +85,8 @@ test.describe('App basic flow (no extensions)', () => { // causes renderTextPrompt to return '' → filenameFromName('') = 'shopify.app.toml' → // that file already exists → overwrite confirmation prompt hangs. // (--config and --client-id are mutually exclusive flags, so we can't pass both directly.) - writeFileSync( - join(appScaffold.appDir, 'shopify.app.secondary.toml'), + fs.writeFileSync( + joinPath(appScaffold.appDir, 'shopify.app.secondary.toml'), `client_id = "${env.secondaryClientId}"\n`, ) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c611fe81206..b4ed830452e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -569,6 +569,9 @@ importers: '@playwright/test': specifier: ^1.50.0 version: 1.58.2 + '@shopify/cli-kit': + specifier: 3.92.0 + version: link:../cli-kit '@types/node': specifier: 18.19.70 version: 18.19.70 @@ -13842,15 +13845,6 @@ snapshots: msw: 2.12.10(@types/node@22.19.11)(typescript@5.9.3) vite: 6.4.1(@types/node@18.19.70)(jiti@2.6.1)(sass@1.97.3)(yaml@2.8.2) - '@vitest/mocker@3.2.4(msw@2.12.10(@types/node@22.19.11)(typescript@5.9.3))(vite@6.4.1(@types/node@22.19.11)(jiti@2.6.1)(sass@1.97.3)(yaml@2.8.2))': - dependencies: - '@vitest/spy': 3.2.4 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - msw: 2.12.10(@types/node@22.19.11)(typescript@5.9.3) - vite: 6.4.1(@types/node@22.19.11)(jiti@2.6.1)(sass@1.97.3)(yaml@2.8.2) - '@vitest/pretty-format@3.2.4': dependencies: tinyrainbow: 2.0.0 @@ -19245,7 +19239,7 @@ snapshots: dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(msw@2.12.10(@types/node@22.19.11)(typescript@5.9.3))(vite@6.4.1(@types/node@22.19.11)(jiti@2.6.1)(sass@1.97.3)(yaml@2.8.2)) + '@vitest/mocker': 3.2.4(msw@2.12.10(@types/node@22.19.11)(typescript@5.9.3))(vite@6.4.1(@types/node@18.19.70)(jiti@2.6.1)(sass@1.97.3)(yaml@2.8.2)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 From 9a1893fa2080b8b1fd0791fecdd84bf483382eb5 Mon Sep 17 00:00:00 2001 From: Phyllis Wu Date: Thu, 19 Mar 2026 21:09:40 -0400 Subject: [PATCH 3/4] optimize error reporting ux --- packages/e2e/setup/app.ts | 4 +++ packages/e2e/tests/app-basic.spec.ts | 38 ++++++++++++++++------------ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/packages/e2e/setup/app.ts b/packages/e2e/setup/app.ts index dbd3da4b995..7a2097aa16d 100644 --- a/packages/e2e/setup/app.ts +++ b/packages/e2e/setup/app.ts @@ -78,6 +78,10 @@ export const appScaffoldFixture = authFixture.extend<{appScaffold: AppScaffold}> timeout: 5 * 60 * 1000, }) + if (result.exitCode !== 0) { + return result + } + const allOutput = `${result.stdout}\n${result.stderr}` const match = allOutput.match(/([\w-]+) is ready for you to build!/) diff --git a/packages/e2e/tests/app-basic.spec.ts b/packages/e2e/tests/app-basic.spec.ts index c51eb383acc..e360a847ca2 100644 --- a/packages/e2e/tests/app-basic.spec.ts +++ b/packages/e2e/tests/app-basic.spec.ts @@ -21,13 +21,15 @@ test.describe('App basic flow (no extensions)', () => { flavor: 'javascript', packageManager: 'npm', }) - expect(initResult.exitCode, `app init failed:\n${initResult.stdout}\n${initResult.stderr}`).toBe(0) + expect(initResult.exitCode, '‼️ Step 1 - app init failed').toBe(0) // Step 2: Start dev server via PTY // Unset CI so keyboard shortcuts are enabled in the Dev UI const dev = await cli.spawn(['app', 'dev', '--path', appScaffold.appDir], {env: {CI: ''}}) try { - await dev.waitForOutput('Ready, watching for changes in your app', 3 * 60 * 1000) + await dev.waitForOutput('Ready, watching for changes in your app', 3 * 60 * 1000).catch((err: Error) => { + throw new Error(`‼️ Step 2 - app dev failed\n${err.message}`) + }) // Step 3: Run a GraphQL query while the dev server is running const executeResult = await cli.exec( @@ -35,20 +37,24 @@ test.describe('App basic flow (no extensions)', () => { {timeout: 60 * 1000}, ) const executeOutput = executeResult.stdout + executeResult.stderr - expect(executeResult.exitCode, `app execute failed:\n${executeOutput}`).toBe(0) - expect(executeOutput).toContain('shop') + expect(executeResult.exitCode, '‼️ Step 3 - app execute failed').toBe(0) + expect(executeOutput, '‼️ Step 3 - app execute: response missing "shop" field').toContain('shop') // Step 4: Press q to quit the dev server dev.sendKey('q') - const devExitCode = await dev.waitForExit(30_000) - expect(devExitCode).toBe(0) + const devExitCode = await dev.waitForExit(30_000).catch((err: Error) => { + throw new Error(`‼️ Step 4 - app dev did not exit after pressing q\n${err.message}`) + }) + expect(devExitCode, '‼️ Step 4 - app dev quit failed').toBe(0) } finally { // Step 5: Always clean up the dev preview, even if the test fails dev.kill() const cleanResult = await cli.exec(['app', 'dev', 'clean', '--path', appScaffold.appDir]) const cleanOutput = cleanResult.stdout + cleanResult.stderr - expect(cleanResult.exitCode, `dev clean failed:\n${cleanOutput}`).toBe(0) - expect(cleanOutput).toContain('Dev preview stopped') + expect(cleanResult.exitCode, '‼️ Step 5 - app dev clean failed').toBe(0) + expect(cleanOutput, '‼️ Step 5 - app dev clean: missing "Dev preview stopped" in output').toContain( + 'Dev preview stopped', + ) } // Step 6: Deploy the primary app @@ -67,16 +73,15 @@ test.describe('App basic flow (no extensions)', () => { ], {timeout: 5 * 60 * 1000}, ) - const deployOutput = deployResult.stdout + deployResult.stderr - expect(deployResult.exitCode, `deploy failed:\n${deployOutput}`).toBe(0) + expect(deployResult.exitCode, '‼️ Step 6 - app deploy failed').toBe(0) // Step 7: List versions and verify our tag appears const listResult = await cli.exec(['app', 'versions', 'list', '--path', appScaffold.appDir, '--json'], { timeout: 60 * 1000, }) const listOutput = listResult.stdout + listResult.stderr - expect(listResult.exitCode, `versions list failed:\n${listOutput}`).toBe(0) - expect(listOutput).toContain(versionTag) + expect(listResult.exitCode, '‼️ Step 7 - app versions list failed').toBe(0) + expect(listOutput, `‼️ Step 7 - app versions list: missing version tag "${versionTag}"`).toContain(versionTag) // Step 8: Config link to the secondary app // Pre-create a minimal TOML stub so getTomls() finds the secondary client ID and skips @@ -94,9 +99,11 @@ test.describe('App basic flow (no extensions)', () => { ['app', 'config', 'link', '--path', appScaffold.appDir, '--client-id', env.secondaryClientId], {env: {CI: '', SHOPIFY_FLAG_CLIENT_ID: undefined}}, ) - await configLink.waitForOutput('is now linked to', 2 * 60 * 1000) + await configLink.waitForOutput('is now linked to', 2 * 60 * 1000).catch((err: Error) => { + throw new Error(`‼️ Step 8 - app config link failed\n${err.message}`) + }) const configLinkExitCode = await configLink.waitForExit(30_000) - expect(configLinkExitCode, `config link failed:\n${configLink.getOutput()}`).toBe(0) + expect(configLinkExitCode, '‼️ Step 8 - app config link failed').toBe(0) // Step 9: Deploy to the secondary app using the linked config file const secondaryVersionTag = `e2e-secondary-v-${Date.now()}` @@ -116,7 +123,6 @@ test.describe('App basic flow (no extensions)', () => { ], {timeout: 5 * 60 * 1000, env: {SHOPIFY_FLAG_CLIENT_ID: undefined}}, ) - const secondaryDeployOutput = secondaryDeployResult.stdout + secondaryDeployResult.stderr - expect(secondaryDeployResult.exitCode, `secondary deploy failed:\n${secondaryDeployOutput}`).toBe(0) + expect(secondaryDeployResult.exitCode, '‼️ Step 9 - app deploy (secondary) failed').toBe(0) }) }) From eaad57bbb1f30039399bad2406e355d99b5ab32a Mon Sep 17 00:00:00 2001 From: Phyllis Wu Date: Fri, 20 Mar 2026 15:09:25 -0400 Subject: [PATCH 4/4] polish --- packages/e2e/package.json | 3 +-- packages/e2e/setup/app.ts | 4 ---- packages/e2e/setup/cli.ts | 16 ++++++++++++++-- packages/e2e/tests/app-basic.spec.ts | 11 ++++++----- pnpm-lock.yaml | 3 --- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/e2e/package.json b/packages/e2e/package.json index eb14548088a..fa6c33eb21d 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -30,8 +30,7 @@ }, "devDependencies": { "@playwright/test": "^1.50.0", - "@shopify/cli-kit": "3.92.0", - "@types/node": "18.19.70", +"@types/node": "18.19.70", "execa": "^7.2.0", "node-pty": "^1.0.0", "strip-ansi": "^7.1.0", diff --git a/packages/e2e/setup/app.ts b/packages/e2e/setup/app.ts index 7a2097aa16d..dbd3da4b995 100644 --- a/packages/e2e/setup/app.ts +++ b/packages/e2e/setup/app.ts @@ -78,10 +78,6 @@ export const appScaffoldFixture = authFixture.extend<{appScaffold: AppScaffold}> timeout: 5 * 60 * 1000, }) - if (result.exitCode !== 0) { - return result - } - const allOutput = `${result.stdout}\n${result.stderr}` const match = allOutput.match(/([\w-]+) is ready for you to build!/) diff --git a/packages/e2e/setup/cli.ts b/packages/e2e/setup/cli.ts index a5cb5216fcb..f3e178abaea 100644 --- a/packages/e2e/setup/cli.ts +++ b/packages/e2e/setup/cli.ts @@ -49,9 +49,15 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({ async exec(args, opts = {}) { // 3 min default const timeout = opts.timeout ?? 3 * 60 * 1000 + const execEnv: {[key: string]: string} = {} + for (const [key, value] of Object.entries({...env.processEnv, ...opts.env})) { + if (value !== undefined) { + execEnv[key] = value + } + } const execaOpts: ExecaOptions = { cwd: opts.cwd, - env: {...env.processEnv, ...opts.env}, + env: execEnv, extendEnv: false, timeout, reject: false, @@ -73,9 +79,15 @@ export const cliFixture = envFixture.extend<{cli: CLIProcess}>({ async execCreateApp(args, opts = {}) { // 5 min default for scaffolding const timeout = opts.timeout ?? 5 * 60 * 1000 + const execEnv: {[key: string]: string} = {} + for (const [key, value] of Object.entries({...env.processEnv, ...opts.env})) { + if (value !== undefined) { + execEnv[key] = value + } + } const execaOpts: ExecaOptions = { cwd: opts.cwd, - env: {...env.processEnv, ...opts.env}, + env: execEnv, extendEnv: false, timeout, reject: false, diff --git a/packages/e2e/tests/app-basic.spec.ts b/packages/e2e/tests/app-basic.spec.ts index e360a847ca2..0ca04eb118d 100644 --- a/packages/e2e/tests/app-basic.spec.ts +++ b/packages/e2e/tests/app-basic.spec.ts @@ -1,8 +1,9 @@ +/* eslint-disable no-restricted-imports */ import {appScaffoldFixture as test} from '../setup/app.js' import {requireEnv} from '../setup/env.js' import {expect} from '@playwright/test' -import {joinPath} from '@shopify/cli-kit/node/path' import * as fs from 'fs' +import * as path from 'path' test.describe('App basic flow (no extensions)', () => { test('init, dev, execute, quit, clean, deploy, versions, config link, deploy to secondary', async ({ @@ -38,7 +39,7 @@ test.describe('App basic flow (no extensions)', () => { ) const executeOutput = executeResult.stdout + executeResult.stderr expect(executeResult.exitCode, '‼️ Step 3 - app execute failed').toBe(0) - expect(executeOutput, '‼️ Step 3 - app execute: response missing "shop" field').toContain('shop') + expect(executeOutput, '‼️ Step 3 - app execute: response missing "shop" field').toContain('"shop"') // Step 4: Press q to quit the dev server dev.sendKey('q') @@ -58,7 +59,7 @@ test.describe('App basic flow (no extensions)', () => { } // Step 6: Deploy the primary app - const versionTag = `e2e-v-${Date.now()}` + const versionTag = `QA-E2E-1st-${Date.now()}` const deployResult = await cli.exec( [ 'app', @@ -91,7 +92,7 @@ test.describe('App basic flow (no extensions)', () => { // that file already exists → overwrite confirmation prompt hangs. // (--config and --client-id are mutually exclusive flags, so we can't pass both directly.) fs.writeFileSync( - joinPath(appScaffold.appDir, 'shopify.app.secondary.toml'), + path.join(appScaffold.appDir, 'shopify.app.secondary.toml'), `client_id = "${env.secondaryClientId}"\n`, ) @@ -106,7 +107,7 @@ test.describe('App basic flow (no extensions)', () => { expect(configLinkExitCode, '‼️ Step 8 - app config link failed').toBe(0) // Step 9: Deploy to the secondary app using the linked config file - const secondaryVersionTag = `e2e-secondary-v-${Date.now()}` + const secondaryVersionTag = `QA-E2E-2nd-${Date.now()}` const secondaryDeployResult = await cli.exec( [ 'app', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b4ed830452e..83a7cf0eef1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -569,9 +569,6 @@ importers: '@playwright/test': specifier: ^1.50.0 version: 1.58.2 - '@shopify/cli-kit': - specifier: 3.92.0 - version: link:../cli-kit '@types/node': specifier: 18.19.70 version: 18.19.70