fix(tables): per-batch delete-job commits, real trigger.dev retries, post-index ANALYZE guard (#4997)

* fix(tables): per-batch delete-job commits, real trigger.dev retries, post-index ANALYZE guard

* fix(tables): resume job progress across retries, rethrow root cause for clean failure messages

Author: TheodoreSpeaks

apps/sim/app/api/table/[tableId]/delete-async/route.ts

@@ -8,7 +8,7 @@ import { isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
 import { runDetached } from '@/lib/core/utils/background'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
-import { runTableDelete } from '@/lib/table/delete-runner'
+import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner'
 import { markTableJobRunning, releaseJobClaim } from '@/lib/table/service'
 import type { TableDeleteJobPayload } from '@/lib/table/types'
 import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils'
@@ -110,6 +110,10 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro
         filter,
         excludeRowIds,
         cutoff,
+      }).catch(async (error) => {
+        // No retry machinery on the detached path — fail the job immediately.
+        await markTableDeleteFailed(tableId, jobId, error)
+        throw error
       })
     )
   }

apps/sim/background/table-delete.ts

@@ -1,5 +1,9 @@
 import { task } from '@trigger.dev/sdk'
-import { runTableDelete, type TableDeletePayload } from '@/lib/table/delete-runner'
+import {
+  markTableDeleteFailed,
+  runTableDelete,
+  type TableDeletePayload,
+} from '@/lib/table/delete-runner'
 
 /**
  * `TableDeletePayload` with the cutoff as an ISO string — task payloads cross a JSON boundary, so
@@ -10,10 +14,12 @@ export interface TableDeleteTaskPayload extends Omit<TableDeletePayload, 'cutoff
 }
 
 /**
- * Trigger.dev wrapper around `runTableDelete`. Retry-safe: the worker keysets by id with a
- * `created_at <= cutoff` floor and pages are committed independently, so a retried attempt simply
- * re-walks and deletes whatever remains. The `table_jobs` ownership gate stops a retried run that
- * lost the job (canceled / janitor-failed) within one page.
+ * Trigger.dev wrapper around `runTableDelete`. Errors propagate out of `run` so the retry policy
+ * actually fires; the job is marked failed only in `onFailure`, after the final attempt. Retry-
+ * safe: the worker keysets by id with a `created_at <= cutoff` floor and batches are committed
+ * independently, so a retried attempt simply re-walks and deletes whatever remains. The
+ * `table_jobs` ownership gate stops a retried run that lost the job (canceled / janitor-failed)
+ * within one page.
  */
 export const tableDeleteTask = task({
   id: 'table-delete',
@@ -26,4 +32,7 @@ export const tableDeleteTask = task({
   run: async (payload: TableDeleteTaskPayload) => {
     await runTableDelete({ ...payload, cutoff: new Date(payload.cutoff) })
   },
+  onFailure: async ({ payload, error }) => {
+    await markTableDeleteFailed(payload.tableId, payload.jobId, error)
+  },
 })

apps/sim/lib/table/constants.ts

@@ -26,9 +26,9 @@ export const TABLE_LIMITS = {
   MAX_BULK_OPERATION_SIZE: 1000,
   /** Maximum rows a single clipboard copy/cut serializes; beyond this the user is steered to Export. */
   MAX_COPY_ROWS: 50000,
-  /** Rows selected + deleted per page in the async background delete-job loop. Each page is one
-   *  transaction (chunked into DELETE_BATCH_SIZE statements inside it); the page is also the
-   *  cancel/ownership-check granularity. */
+  /** Rows selected + deleted per page in the async background delete-job loop. Each
+   *  DELETE_BATCH_SIZE chunk inside the page commits in its own transaction; the page is the
+   *  keyset-select and cancel/ownership-check granularity. */
   DELETE_PAGE_SIZE: 10000,
   /** Row count above which an export runs as a background job instead of a synchronous stream.
    *  Matches the default per-table row cap, so non-enterprise tables keep instant downloads. */

apps/sim/lib/table/delete-runner.test.ts

@@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const {
   mockGetTableById,
+  mockGetJobProgress,
   mockSelectRowIdPage,
   mockDeletePageByIds,
   mockUpdateJobProgress,
@@ -14,6 +15,7 @@ const {
   mockBuildFilterClause,
 } = vi.hoisted(() => ({
   mockGetTableById: vi.fn(),
+  mockGetJobProgress: vi.fn(),
   mockSelectRowIdPage: vi.fn(),
   mockDeletePageByIds: vi.fn(),
   mockUpdateJobProgress: vi.fn(),
@@ -25,6 +27,7 @@ const {
 
 vi.mock('@/lib/table/service', () => ({
   getTableById: mockGetTableById,
+  getJobProgress: mockGetJobProgress,
   selectRowIdPage: mockSelectRowIdPage,
   deletePageByIds: mockDeletePageByIds,
   updateJobProgress: mockUpdateJobProgress,
@@ -38,7 +41,7 @@ vi.mock('@/lib/table/constants', () => ({
   USER_TABLE_ROWS_SQL_NAME: 'user_table_rows',
 }))
 
-import { runTableDelete } from '@/lib/table/delete-runner'
+import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner'
 
 const table = { id: 'tbl_1', workspaceId: 'ws_1', schema: { columns: [] } }
 const cutoff = new Date('2026-06-05T00:00:00Z')
@@ -51,6 +54,7 @@ describe('runTableDelete', () => {
   beforeEach(() => {
     vi.clearAllMocks()
     mockGetTableById.mockResolvedValue(table)
+    mockGetJobProgress.mockResolvedValue(0)
     mockUpdateJobProgress.mockResolvedValue(true)
     mockMarkJobReady.mockResolvedValue(true)
     mockMarkJobFailed.mockResolvedValue(undefined)
@@ -103,17 +107,57 @@ describe('runTableDelete', () => {
     )
   })
 
-  it('marks the job failed and emits a failed event on error', async () => {
+  it('rethrows unexpected errors without failing the job (caller retries decide)', async () => {
     mockSelectRowIdPage.mockRejectedValue(new Error('boom'))
 
+    await expect(runTableDelete(basePayload())).rejects.toThrow('boom')
+
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+    expect(mockAppendTableEvent).not.toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'failed' })
+    )
+  })
+
+  it('returns quietly when superseded mid-run without failing the job', async () => {
+    mockSelectRowIdPage.mockResolvedValue(['a', 'b'])
+    mockUpdateJobProgress.mockResolvedValueOnce(true).mockResolvedValueOnce(false)
+
+    await expect(runTableDelete(basePayload())).resolves.toBeUndefined()
+
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+  })
+
+  it('rethrows the root cause so the clean message survives serialization', async () => {
+    const cause = new Error('canceling statement due to statement timeout')
+    mockSelectRowIdPage.mockRejectedValue(new Error('Failed query: delete ...', { cause }))
+
+    await expect(runTableDelete(basePayload())).rejects.toThrow(
+      'canceling statement due to statement timeout'
+    )
+  })
+
+  it('resumes cumulative progress on retry instead of resetting to zero', async () => {
+    mockGetJobProgress.mockResolvedValue(7)
+    mockSelectRowIdPage.mockResolvedValueOnce(['a', 'b']).mockResolvedValueOnce([])
+
     await runTableDelete(basePayload())
 
-    expect(mockMarkJobFailed).toHaveBeenCalledWith('tbl_1', 'job_1', 'boom')
+    expect(mockUpdateJobProgress).toHaveBeenNthCalledWith(1, 'tbl_1', 7, 'job_1')
     expect(mockAppendTableEvent).toHaveBeenCalledWith(
-      expect.objectContaining({ kind: 'job', type: 'delete', status: 'failed', error: 'boom' })
+      expect.objectContaining({ status: 'ready', progress: 9 })
     )
   })
 
+  it('stops at the seed read when the job is no longer owned', async () => {
+    mockGetJobProgress.mockResolvedValue(null)
+
+    await expect(runTableDelete(basePayload())).resolves.toBeUndefined()
+
+    expect(mockSelectRowIdPage).not.toHaveBeenCalled()
+    expect(mockDeletePageByIds).not.toHaveBeenCalled()
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+  })
+
   it('passes the cutoff and filter clause through to the page query', async () => {
     mockSelectRowIdPage.mockResolvedValueOnce([])
 
@@ -129,3 +173,42 @@ describe('runTableDelete', () => {
     )
   })
 })
+
+describe('markTableDeleteFailed', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockMarkJobFailed.mockResolvedValue(undefined)
+  })
+
+  it('marks the job failed and emits the failed event', async () => {
+    await markTableDeleteFailed('tbl_1', 'job_1', new Error('boom'))
+
+    expect(mockMarkJobFailed).toHaveBeenCalledWith('tbl_1', 'job_1', 'boom')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: 'job', type: 'delete', status: 'failed', error: 'boom' })
+    )
+  })
+
+  it('prefers the error cause over a verbose wrapper message', async () => {
+    const cause = new Error('canceling statement due to statement timeout')
+    const wrapper = new Error(`Failed query: delete from x where id in (${'$1,'.repeat(5000)})`, {
+      cause,
+    })
+
+    await markTableDeleteFailed('tbl_1', 'job_1', wrapper)
+
+    expect(mockMarkJobFailed).toHaveBeenCalledWith(
+      'tbl_1',
+      'job_1',
+      'canceling statement due to statement timeout'
+    )
+  })
+
+  it('truncates oversized messages', async () => {
+    await markTableDeleteFailed('tbl_1', 'job_1', new Error('x'.repeat(2000)))
+
+    const [, , message] = mockMarkJobFailed.mock.calls[0]
+    expect(message).toHaveLength(503)
+    expect(message.endsWith('...')).toBe(true)
+  })
+})

apps/sim/lib/table/delete-runner.ts

@@ -1,11 +1,13 @@
 import { createLogger } from '@sim/logger'
-import { getErrorMessage } from '@sim/utils/errors'
+import { getErrorMessage, toError } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
+import { truncate } from '@sim/utils/string'
 import type { Filter } from '@/lib/table'
 import { TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants'
 import { appendTableEvent } from '@/lib/table/events'
 import {
   deletePageByIds,
+  getJobProgress,
   getTableById,
   markJobFailed,
   markJobReady,
@@ -38,12 +40,17 @@ export interface TableDeletePayload {
 }
 
 /**
- * Background worker for large filtered row deletes. Runs detached on the web container (see the
- * delete-async kickoff route). Deletes in keyset-paginated pages — `created_at <= cutoff` spares
- * rows inserted while the job runs, and `excludeRowIds` spares specific rows (the
- * "select all then deselect a few" case). Ownership-gated per page so a cancel/supersede stops
- * it within one page; committed pages are never rolled back. Progress and the terminal state are
- * surfaced via the table-events SSE stream.
+ * Background worker for large filtered row deletes (trigger.dev task, or detached on the web
+ * container when trigger.dev is disabled — see the delete-async kickoff route). Deletes in
+ * keyset-paginated pages — `created_at <= cutoff` spares rows inserted while the job runs, and
+ * `excludeRowIds` spares specific rows (the "select all then deselect a few" case).
+ * Ownership-gated per page so a cancel/supersede stops it within one page; committed batches are
+ * never rolled back. Progress and the terminal state are surfaced via the table-events SSE
+ * stream.
+ *
+ * Unexpected errors are rethrown so the caller's retry machinery sees them — the caller marks
+ * the job failed via `markTableDeleteFailed` once it gives up. A superseded run (cancel, or a
+ * newer job took the table) returns quietly.
  */
 export async function runTableDelete(payload: TableDeletePayload): Promise<void> {
   const { jobId, tableId, workspaceId, filter, excludeRowIds, cutoff } = payload
@@ -58,8 +65,14 @@ export async function runTableDelete(payload: TableDeletePayload): Promise<void>
       : undefined
     const excluded = new Set(excludeRowIds ?? [])
 
-    let processed = 0
-    let lastReported = 0
+    // Resume the persisted count: a retried attempt's earlier batches are already committed,
+    // so starting at zero would overwrite cumulative progress with this attempt's smaller
+    // number. Doubles as the initial ownership gate.
+    const resumed = await getJobProgress(tableId, jobId)
+    if (resumed === null) throw new JobSupersededError()
+
+    let processed = resumed
+    let lastReported = resumed
     let afterId: string | undefined
 
     while (true) {
@@ -128,19 +141,37 @@ export async function runTableDelete(payload: TableDeletePayload): Promise<void>
   } catch (err) {
     if (err instanceof JobSupersededError) {
       logger.info(`[${requestId}] Delete superseded by a newer run; stopping`, { tableId, jobId })
-    } else {
-      const message = getErrorMessage(err, 'Delete failed')
-      logger.error(`[${requestId}] Delete failed for table ${tableId}:`, err)
-      // Scoped to jobId — a no-op if a newer job has taken over.
-      await markJobFailed(tableId, jobId, message).catch(() => {})
-      void appendTableEvent({
-        kind: 'job',
-        type: 'delete',
-        tableId,
-        jobId,
-        status: 'failed',
-        error: message,
-      })
+      return
     }
+    // Rethrow the root cause, not the wrapper: drizzle query errors embed the full SQL + params
+    // list (tens of KB for a batch delete) in `message`, and `cause` does not survive
+    // trigger.dev's serialization between the failed `run` and `onFailure` — the clean message
+    // must already be the thrown error's own `message`.
+    const cause = toError(err).cause
+    const error = cause ? toError(cause) : toError(err)
+    logger.error(`[${requestId}] Delete failed for table ${tableId}:`, error)
+    throw error
   }
 }
+
+/**
+ * Marks the delete job failed and emits the failed SSE event. Called once the caller gives up on
+ * the run: the trigger.dev task's `onFailure` (after retries are exhausted) or the detached
+ * web-container fallback (no retries). Scoped to jobId — a no-op if a newer job has taken over.
+ */
+export async function markTableDeleteFailed(
+  tableId: string,
+  jobId: string,
+  error: unknown
+): Promise<void> {
+  const message = truncate(getErrorMessage(toError(error).cause ?? error, 'Delete failed'), 500)
+  await markJobFailed(tableId, jobId, message).catch(() => {})
+  void appendTableEvent({
+    kind: 'job',
+    type: 'delete',
+    tableId,
+    jobId,
+    status: 'failed',
+    error: message,
+  })
+}