Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ There are two modes for uploading results using the commands:
- `--run-name` - Optional name template for creating new test run. It supports `{env:VAR}`, `{YYYY}`, `{YY}`, `{MM}`, `{MMM}`, `{DD}`, `{HH}`, `{hh}`, `{mm}`, `{ss}`, `{AMPM}` placeholders (default: `Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}`)
- `--create-tcases` - Automatically create test cases in QA Sphere for results that don't have valid test case markers. A mapping file (`qasphere-automapping-YYYYMMDD-HHmmss.txt`) is generated showing the sequence numbers assigned to each new test case (default: `false`)
- `--attachments` - Try to detect and upload any attachments with the test result
- `--force` - Ignore API request errors, invalid test cases, or attachments
- `--force` - Ignore API request errors, invalid or duplicate test case mappings, or attachments
- `--ignore-unmatched` - Suppress individual unmatched test messages, show summary only
- `--skip-report-stdout` - Control when to skip stdout blocks from test report (choices: `on-success`, `never`; default: `never`)
- `--skip-report-stderr` - Control when to skip stderr blocks from test report (choices: `on-success`, `never`; default: `never`)
Expand Down Expand Up @@ -297,7 +297,7 @@ Ensure the required environment variables are defined before running these comma

## Test Report Requirements

The QAS CLI maps test results from your reports (JUnit XML, Playwright JSON, or Allure) to corresponding test cases in QA Sphere. If a test result lacks a valid marker/reference, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to skip unmatched results.
The QAS CLI maps test results from your reports (JUnit XML, Playwright JSON, or Allure) to corresponding test cases in QA Sphere. If a test result lacks a valid marker/reference, or multiple results resolve to the same run test case, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to bypass the mapping issue.

### JUnit XML

Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "qas-cli",
"version": "0.6.0",
"version": "0.7.0",
"description": "QAS CLI is a command line tool for submitting your automation test results to QA Sphere at https://qasphere.com/",
"type": "module",
"main": "./build/bin/qasphere.js",
Expand Down
3 changes: 2 additions & 1 deletion src/commands/resultUpload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ export class ResultUploadCommandModule implements CommandModule<unknown, ResultU
type: 'boolean',
},
force: {
describe: 'Ignore API request errors, invalid test cases or attachments',
describe:
'Ignore API request errors, invalid or duplicate test case mappings, or attachments',
type: 'boolean',
},
'ignore-unmatched': {
Expand Down
39 changes: 39 additions & 0 deletions src/tests/fixtures/playwright-json/describe-marker-collision.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"suites": [
{
"title": "describe-collision.spec.ts",
"specs": [],
"suites": [
{
"title": "Test Case Folders (TEST-002, TEST-003)",
"specs": [
{
"title": "should create a new root folder (TEST-004)",
"tags": [],
"tests": [
{
"annotations": [],
"expectedStatus": "passed",
"projectName": "chromium",
"results": [
{
"status": "passed",
"errors": [],
"stdout": [],
"stderr": [],
"retry": 0,
"duration": 1000,
"attachments": []
}
],
"status": "expected"
}
]
}
],
"suites": []
}
]
}
]
}
39 changes: 39 additions & 0 deletions src/tests/fixtures/playwright-json/describe-marker-unmatched.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"suites": [
{
"title": "describe-marker-unmatched.spec.ts",
"specs": [],
"suites": [
{
"title": "Dashboard Projects (TEST-034, TEST-035, TEST-365)",
"specs": [
{
"title": "viewer should not see project management actions",
"tags": [],
"tests": [
{
"annotations": [],
"expectedStatus": "passed",
"projectName": "chromium",
"results": [
{
"status": "passed",
"errors": [],
"stdout": [],
"stderr": [],
"retry": 0,
"duration": 1000,
"attachments": []
}
],
"status": "expected"
}
]
}
],
"suites": []
}
]
}
]
}
120 changes: 120 additions & 0 deletions src/tests/playwright-json-parsing.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -391,8 +391,14 @@ describe('Playwright JSON parsing', () => {
// Fixture has 1 test with 3 annotations (2x 10427 deduped to 1, plus 10428) + 1 test with no annotations = 3 results
expect(testcases).toHaveLength(3)
expect(testcases[0].name).toBe('TEST-10427: Login flow covers multiple cases')
expect(testcases[0].marker).toEqual({ projectCode: 'TEST', seq: 10427 })
expect(testcases[0].markerResolution).toBe('resolved')
expect(testcases[1].name).toBe('TEST-10428: Login flow covers multiple cases')
expect(testcases[1].marker).toEqual({ projectCode: 'TEST', seq: 10428 })
expect(testcases[1].markerResolution).toBe('resolved')
expect(testcases[2].name).toBe('Navigation bar items TEST-006')
expect(testcases[2].marker).toEqual({ projectCode: 'TEST', seq: 6 })
expect(testcases[2].markerResolution).toBe('resolved')

// The two fan-out entries share the same status, duration, folder
for (const tc of testcases.slice(0, 2)) {
Expand Down Expand Up @@ -454,6 +460,8 @@ describe('Playwright JSON parsing', () => {

expect(testcases).toHaveLength(1)
expect(testcases[0].name).toBe('PRJ-100: Simple test')
expect(testcases[0].marker).toEqual({ projectCode: 'PRJ', seq: 100 })
expect(testcases[0].markerResolution).toBe('resolved')
})

test('Should fan out by annotations even when name has a marker', async () => {
Expand Down Expand Up @@ -509,6 +517,118 @@ describe('Playwright JSON parsing', () => {
expect(testcases).toHaveLength(2)
expect(testcases[0].name).toBe('PRJ-100: PRJ-999: Test with marker in name')
expect(testcases[1].name).toBe('PRJ-200: PRJ-999: Test with marker in name')
expect(testcases[0].marker).toEqual({ projectCode: 'PRJ', seq: 100 })
expect(testcases[1].marker).toEqual({ projectCode: 'PRJ', seq: 200 })
expect(testcases[0].markerResolution).toBe('resolved')
expect(testcases[1].markerResolution).toBe('resolved')
})

test('Should extract fallback marker from spec title, not describe titles', async () => {
const jsonContent = JSON.stringify({
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The newly created JSON files is the same as this. Import it instead.

suites: [
{
title: 'describe-collision.spec.ts',
specs: [],
suites: [
{
title: 'Test Case Folders (TEST-002, TEST-003)',
specs: [
{
title: 'should create a new root folder (TEST-004)',
tags: [],
tests: [
{
annotations: [],
expectedStatus: 'passed',
projectName: 'chromium',
results: [
{
status: 'passed',
errors: [],
stdout: [],
stderr: [],
retry: 0,
duration: 1000,
attachments: [],
},
],
status: 'expected',
},
],
},
],
suites: [],
},
],
},
],
})

const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', {
skipStdout: 'never',
skipStderr: 'never',
})

expect(testcases).toHaveLength(1)
expect(testcases[0].name).toBe(
'Test Case Folders (TEST-002, TEST-003) › should create a new root folder (TEST-004)'
)
expect(testcases[0].marker).toEqual({ projectCode: 'TEST', seq: 4 })
expect(testcases[0].markerResolution).toBe('resolved')
})

test('Should keep marker unresolved-none when only describe title contains markers', async () => {
const jsonContent = JSON.stringify({
suites: [
{
title: 'describe-only-markers.spec.ts',
specs: [],
suites: [
{
title: 'Dashboard Projects (TEST-034, TEST-035, TEST-365)',
specs: [
{
title: 'viewer should not see project management actions',
tags: [],
tests: [
{
annotations: [],
expectedStatus: 'passed',
projectName: 'chromium',
results: [
{
status: 'passed',
errors: [],
stdout: [],
stderr: [],
retry: 0,
duration: 1000,
attachments: [],
},
],
status: 'expected',
},
],
},
],
suites: [],
},
],
},
],
})

const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', {
skipStdout: 'never',
skipStderr: 'never',
})

expect(testcases).toHaveLength(1)
expect(testcases[0].name).toBe(
'Dashboard Projects (TEST-034, TEST-035, TEST-365) › viewer should not see project management actions'
)
expect(testcases[0].marker).toBeNull()
expect(testcases[0].markerResolution).toBe('resolved-none')
})

test('Should map test status correctly', async () => {
Expand Down
Loading
Loading