From b742fa9c6ddd608579a4bb68f9c0301d4bf0e662 Mon Sep 17 00:00:00 2001 From: Jacek Date: Tue, 12 May 2026 08:48:18 -0500 Subject: [PATCH] chore(e2e): drain test-user backlog in cleanup script The cleanup script used `getUserList` with `orderBy: '-created_at'` and a hard limit of 150 per query. On heavily-used dev instances, more than 150 test users would accumulate matching `clerkcookie` and the oldest never came back from the paginated head, so backlog grew indefinitely. On the shared `with-email-codes` (neutral-cub-53) instance this stranded a dozen test users going back to 2025-11. A second bug: only the current `+clerk_test@clerkcookie.com` email pattern matched the query, missing the older `test+clerk_test_@example.com` pattern (used through ~early 2026) entirely. Fix: - Paginate through every matching user (`+created_at`, page size 500) so the oldest entries actually get reached. - Run two email queries (`clerkcookie` + `clerk_test`) alongside the phone query, then gate deletion behind a strict test-domain allowlist so developer accounts like `fredrik+debugging-clerk_test@clerk.dev` that the broader query also matches are preserved. - Log the friendly env key name alongside the FAPI subdomain so it's obvious which entry corresponds to which test config. --- .changeset/cleanup-paginate-backlog.md | 4 ++ integration/cleanup/cleanup.setup.ts | 77 +++++++++++++++++++------- 2 files changed, 60 insertions(+), 21 deletions(-) create mode 100644 .changeset/cleanup-paginate-backlog.md diff --git a/.changeset/cleanup-paginate-backlog.md b/.changeset/cleanup-paginate-backlog.md new file mode 100644 index 00000000000..9d4fe932af2 --- /dev/null +++ b/.changeset/cleanup-paginate-backlog.md @@ -0,0 +1,4 @@ +--- +--- + +Fix the E2E cleanup script so it actually drains test-user backlogs: paginate `getUserList` (oldest-first, no 150 cap), broaden the search query to catch the legacy `+clerk_test_@example.com` email pattern alongside the current `@clerkcookie.com` pattern, and gate deletion behind a strict test-domain whitelist so Clerk team accounts caught by the broader query are preserved. diff --git a/integration/cleanup/cleanup.setup.ts b/integration/cleanup/cleanup.setup.ts index cc2ca3a3a42..96d297c9e1c 100644 --- a/integration/cleanup/cleanup.setup.ts +++ b/integration/cleanup/cleanup.setup.ts @@ -7,8 +7,8 @@ import { test as setup } from '@playwright/test'; import { appConfigs } from '../presets/'; setup('cleanup instances ', async () => { - const entries = Array.from(appConfigs.secrets.instanceKeys.values()) - .map(({ pk, sk }) => { + const entries = Array.from(appConfigs.secrets.instanceKeys.entries()) + .map(([keyName, { pk, sk }]) => { const secretKey = sk; if (!secretKey) { return null; @@ -16,6 +16,7 @@ setup('cleanup instances ', async () => { const parsedPk = parsePublishableKey(pk); const apiUrl = isStaging(parsedPk.frontendApi) ? 'https://api.clerkstage.dev' : 'https://api.clerk.com'; return { + keyName, secretKey, apiUrl, instanceName: parsedPk.instanceId || parsedPk.frontendApi.split('.')[0] || 'unknown', @@ -24,6 +25,7 @@ setup('cleanup instances ', async () => { .filter(Boolean); const cleanupSummary: Array<{ + keyName: string; instanceName: string; usersDeleted: number; orgsDeleted: number; @@ -35,6 +37,7 @@ setup('cleanup instances ', async () => { for (const entry of entries) { const instanceSummary = { + keyName: entry.keyName, instanceName: entry.instanceName, usersDeleted: 0, orgsDeleted: 0, @@ -45,27 +48,24 @@ setup('cleanup instances ', async () => { try { const clerkClient = createClerkClient({ secretKey: entry.secretKey, apiUrl: entry.apiUrl }); - // Get users with error handling + // Fetch test users with broad queries, then filter strictly by test-domain + // email or test-phone marker. The previous `clerkcookie` query missed users + // from the older `test+clerk_test_@example.com` pattern, letting a + // backlog accumulate indefinitely. Broadening the fetch to `clerk_test` + // also matches some real team accounts (e.g. fredrik+debugging-clerk_test + // @clerk.dev), so deletion is gated by a strict domain/phone whitelist. let users: any[] = []; try { - const { data: usersWithEmail } = await clerkClient.users.getUserList({ - orderBy: '-created_at', - query: 'clerkcookie', - limit: 150, - }); - - const { data: usersWithPhoneNumber } = await clerkClient.users.getUserList({ - orderBy: '-created_at', - query: '55501', - limit: 150, - }); + const usersWithClerkCookie = await fetchAllUsers(clerkClient, { query: 'clerkcookie' }); + const usersWithClerkTest = await fetchAllUsers(clerkClient, { query: 'clerk_test' }); + const usersWithPhoneNumber = await fetchAllUsers(clerkClient, { query: '55501' }); // Deduplicate users by ID const allUsersMap = new Map(); - [...usersWithEmail, ...usersWithPhoneNumber].forEach(user => { + [...usersWithClerkCookie, ...usersWithClerkTest, ...usersWithPhoneNumber].forEach(user => { allUsersMap.set(user.id, user); }); - users = Array.from(allUsersMap.values()); + users = Array.from(allUsersMap.values()).filter(isTestUser); } catch (error) { instanceSummary.errors.push(`Failed to get users: ${error.message}`); console.error(`Error getting users for ${entry.instanceName}:`, error); @@ -146,10 +146,10 @@ setup('cleanup instances ', async () => { const maskedKey = entry.secretKey.replace(/(sk_(test|live)_)(.+)(...)/, '$1***$4'); if (instanceSummary.usersDeleted > 0 || instanceSummary.orgsDeleted > 0) { console.log( - `✅ ${entry.instanceName} (${maskedKey}): ${instanceSummary.usersDeleted} users, ${instanceSummary.orgsDeleted} orgs deleted`, + `✅ ${entry.keyName} / ${entry.instanceName} (${maskedKey}): ${instanceSummary.usersDeleted} users, ${instanceSummary.orgsDeleted} orgs deleted`, ); } else { - console.log(`✅ ${entry.instanceName} (${maskedKey}): clean`); + console.log(`✅ ${entry.keyName} / ${entry.instanceName} (${maskedKey}): clean`); } if (instanceSummary.errors.length > 0) { @@ -158,10 +158,10 @@ setup('cleanup instances ', async () => { } catch (error) { const maskedKey = entry.secretKey.replace(/(sk_(test|live)_)(.+)(...)/, '$1***$4'); if (isClerkAPIResponseError(error) && (error.status === 401 || error.status === 403)) { - console.log(`🔒 ${entry.instanceName} (${maskedKey}): Unauthorized access`); + console.log(`🔒 ${entry.keyName} / ${entry.instanceName} (${maskedKey}): Unauthorized access`); instanceSummary.status = 'unauthorized'; } else { - console.log(`❌ ${entry.instanceName} (${maskedKey}): ${error.message}`); + console.log(`❌ ${entry.keyName} / ${entry.instanceName} (${maskedKey}): ${error.message}`); instanceSummary.errors.push(error.message); instanceSummary.status = 'error'; } @@ -186,7 +186,7 @@ setup('cleanup instances ', async () => { if (instancesWithErrors.length > 0) { console.log('\n=== DETAILED ERROR REPORT ==='); instancesWithErrors.forEach(instance => { - console.log(`\n${instance.instanceName}:`); + console.log(`\n${instance.keyName} / ${instance.instanceName}:`); instance.errors.forEach(error => console.log(` - ${error}`)); }); } @@ -208,3 +208,38 @@ function batchElements(objects: T[], batchSize = 5): T[][] { } return batches; } + +const PAGE_SIZE = 500; +const MAX_PAGES = 50; + +const TEST_EMAIL_DOMAINS = new Set(['clerkcookie.com', 'example.com', 'mailsac.com']); +const TEST_PHONE_PATTERN = /55501\d{2}$/; + +function isTestUser(user: any): boolean { + const emails: string[] = (user.emailAddresses ?? []).map((e: any) => e.emailAddress ?? ''); + for (const email of emails) { + const domain = email.split('@')[1]?.toLowerCase(); + if (domain && TEST_EMAIL_DOMAINS.has(domain)) return true; + } + const phones: string[] = (user.phoneNumbers ?? []).map((p: any) => p.phoneNumber ?? ''); + if (phones.some(p => TEST_PHONE_PATTERN.test(p))) return true; + return false; +} + +async function fetchAllUsers( + clerkClient: ReturnType, + filter: { query: string }, +): Promise { + const collected: any[] = []; + for (let page = 0; page < MAX_PAGES; page++) { + const { data } = await clerkClient.users.getUserList({ + orderBy: '+created_at', + query: filter.query, + limit: PAGE_SIZE, + offset: page * PAGE_SIZE, + }); + collected.push(...data); + if (data.length < PAGE_SIZE) break; + } + return collected; +}