From 3150e871856ce3bf59e7926f51a4e2f820c19ecc Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Wed, 10 Jun 2026 11:26:33 +0200 Subject: [PATCH 1/2] test: harden flaky test e2e coverage --- .../src/services/api/api.check.ts | 1 + packages/cli/e2e/__tests__/test.spec.ts | 10 +++++-- .../constructs/__tests__/api-check.spec.ts | 28 +++++++++++++++++++ .../test-skip-ssl/checkly.config.js | 11 ++++++++ .../test-cases/test-skip-ssl/test.check.js | 10 +++++++ 5 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js diff --git a/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts b/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts index a348ed45e..dc938fafe 100644 --- a/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts +++ b/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts @@ -24,6 +24,7 @@ const apiCheck = new ApiCheck('homepage-api-check-1', { const skipSslApiCheck = new ApiCheck('ssl-api-check-1', { name: 'Skip SSL Check', + activated: false, alertChannels: [slackChannel, webhookChannel], degradedResponseTime: 10000, maxResponseTime: 20000, diff --git a/packages/cli/e2e/__tests__/test.spec.ts b/packages/cli/e2e/__tests__/test.spec.ts index b4c0f5db7..508ac5677 100644 --- a/packages/cli/e2e/__tests__/test.spec.ts +++ b/packages/cli/e2e/__tests__/test.spec.ts @@ -51,7 +51,13 @@ describe('test', { timeout: 45000 }, () => { it('Test project should run successfully', async () => { const secretEnv = uuid.v4() - const result = await runTest(fixt, ['-e', `SECRET_ENV=${secretEnv}`, '--verbose']) + const result = await runTest(fixt, [ + '-e', + `SECRET_ENV=${secretEnv}`, + '--verbose', + '--grep', + '^(?!Skip SSL Check$).*', + ]) expect(result.stdout).not.toContain('File extension type example') expect(result.stdout).toContain(secretEnv) }, 130_000) @@ -98,7 +104,7 @@ describe('test', { timeout: 45000 }, () => { } catch { // No-op } - const result = await runTest(fixt, ['--record', '--reporter', 'github'], { + const result = await runTest(fixt, ['secret.check.ts', '--record', '--reporter', 'github'], { env: { CHECKLY_REPORTER_GITHUB_OUTPUT: reportFilename, }, diff --git a/packages/cli/src/constructs/__tests__/api-check.spec.ts b/packages/cli/src/constructs/__tests__/api-check.spec.ts index 145eab535..b3588af34 100644 --- a/packages/cli/src/constructs/__tests__/api-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/api-check.spec.ts @@ -272,6 +272,34 @@ describe('ApiCheck', () => { })) }, DEFAULT_TEST_TIMEOUT) + it('should synthesize skipSSL requests', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-skip-ssl/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'check', + type: 'check', + member: true, + payload: expect.objectContaining({ + request: expect.objectContaining({ + skipSSL: true, + }), + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + describe('retryStrategy', () => { it('should synthesize `onlyOn`', async () => { const output = await parseProject( diff --git a/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js new file mode 100644 index 000000000..19a81a24e --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Check Fixture', + logicalId: 'check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js new file mode 100644 index 000000000..9e701b2db --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js @@ -0,0 +1,10 @@ +import { ApiCheck } from 'checkly/constructs' + +new ApiCheck('check', { + name: 'Skip SSL Check', + request: { + method: 'GET', + url: 'https://self-signed.example.test', + skipSSL: true, + }, +}) From bf24678c2f56b627bd34285c8b7c1b024cc671f4 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Wed, 10 Jun 2026 11:26:33 +0200 Subject: [PATCH 2/2] test: harden flaky test e2e coverage --- .../src/services/api/api.check.ts | 1 + packages/cli/e2e/__tests__/test.spec.ts | 12 ++++++-- .../constructs/__tests__/api-check.spec.ts | 28 +++++++++++++++++++ .../test-skip-ssl/checkly.config.js | 11 ++++++++ .../test-cases/test-skip-ssl/test.check.js | 11 ++++++++ 5 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js diff --git a/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts b/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts index a348ed45e..dc938fafe 100644 --- a/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts +++ b/packages/cli/e2e/__tests__/fixtures/test-project/src/services/api/api.check.ts @@ -24,6 +24,7 @@ const apiCheck = new ApiCheck('homepage-api-check-1', { const skipSslApiCheck = new ApiCheck('ssl-api-check-1', { name: 'Skip SSL Check', + activated: false, alertChannels: [slackChannel, webhookChannel], degradedResponseTime: 10000, maxResponseTime: 20000, diff --git a/packages/cli/e2e/__tests__/test.spec.ts b/packages/cli/e2e/__tests__/test.spec.ts index b4c0f5db7..9be0e069a 100644 --- a/packages/cli/e2e/__tests__/test.spec.ts +++ b/packages/cli/e2e/__tests__/test.spec.ts @@ -51,7 +51,13 @@ describe('test', { timeout: 45000 }, () => { it('Test project should run successfully', async () => { const secretEnv = uuid.v4() - const result = await runTest(fixt, ['-e', `SECRET_ENV=${secretEnv}`, '--verbose']) + const result = await runTest(fixt, [ + '-e', + `SECRET_ENV=${secretEnv}`, + '--verbose', + '--grep', + '^(?!Skip SSL Check$).*', + ]) expect(result.stdout).not.toContain('File extension type example') expect(result.stdout).toContain(secretEnv) }, 130_000) @@ -98,7 +104,9 @@ describe('test', { timeout: 45000 }, () => { } catch { // No-op } - const result = await runTest(fixt, ['--record', '--reporter', 'github'], { + // Keep this reporter test scoped to one stable check. The GitHub assertions below do not + // need the whole fixture project, and unrelated remote checks can make this fail first. + const result = await runTest(fixt, ['secret.check.ts', '--record', '--reporter', 'github'], { env: { CHECKLY_REPORTER_GITHUB_OUTPUT: reportFilename, }, diff --git a/packages/cli/src/constructs/__tests__/api-check.spec.ts b/packages/cli/src/constructs/__tests__/api-check.spec.ts index 145eab535..b3588af34 100644 --- a/packages/cli/src/constructs/__tests__/api-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/api-check.spec.ts @@ -272,6 +272,34 @@ describe('ApiCheck', () => { })) }, DEFAULT_TEST_TIMEOUT) + it('should synthesize skipSSL requests', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-skip-ssl/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'check', + type: 'check', + member: true, + payload: expect.objectContaining({ + request: expect.objectContaining({ + skipSSL: true, + }), + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + describe('retryStrategy', () => { it('should synthesize `onlyOn`', async () => { const output = await parseProject( diff --git a/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js new file mode 100644 index 000000000..19a81a24e --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Check Fixture', + logicalId: 'check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js new file mode 100644 index 000000000..612b5d540 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/api-check/test-cases/test-skip-ssl/test.check.js @@ -0,0 +1,11 @@ +import { ApiCheck } from 'checkly/constructs' + +new ApiCheck('check', { + name: 'Skip SSL Check', + request: { + method: 'GET', + // This fixture is parsed and synthesized only; it must never perform a network request. + url: 'https://self-signed.example.test', + skipSSL: true, + }, +})