fix(export): stream large database dumps with keyset pagination (#59)

src/export/dump.test.ts

@@ -1,7 +1,6 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest'
 import { dumpDatabaseRoute } from './dump'
 import { executeOperation } from '.'
-import { createResponse } from '../utils'
 import type { DataSource } from '../types'
 import type { StarbaseDBConfiguration } from '../handler'
 
@@ -19,15 +18,18 @@ vi.mock('../utils', () => ({
     ),
 }))
 
+// Internal alias dump.ts attaches to keyset-paginated rows. Mock row pages
+// include it so the cursor logic is exercised the same way as in production.
+const ROWID = '__starbasedb_export_rowid__'
+
 let mockDataSource: DataSource
 let mockConfig: StarbaseDBConfiguration
 
 beforeEach(() => {
     vi.clearAllMocks()
 
     mockDataSource = {
-        source: 'external',
-        external: { dialect: 'sqlite' },
+        source: 'internal',
         rpc: { executeQuery: vi.fn() },
     } as any
 
@@ -39,22 +41,22 @@ beforeEach(() => {
 })
 
 describe('Database Dump Module', () => {
-    it('should return a database dump when tables exist', async () => {
+    it('streams a dump with header, schema and quoted INSERTs', async () => {
         vi.mocked(executeOperation)
             .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }])
             .mockResolvedValueOnce([
-                { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
+                { sql: 'CREATE TABLE users (id INTEGER, name TEXT)' },
             ])
             .mockResolvedValueOnce([
-                { id: 1, name: 'Alice' },
-                { id: 2, name: 'Bob' },
+                { id: 1, name: 'Alice', [ROWID]: 1 },
+                { id: 2, name: 'Bob', [ROWID]: 2 },
             ])
             .mockResolvedValueOnce([
-                { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
+                { sql: 'CREATE TABLE orders (id INTEGER, total REAL)' },
             ])
             .mockResolvedValueOnce([
-                { id: 1, total: 99.99 },
-                { id: 2, total: 49.5 },
+                { id: 1, total: 99.99, [ROWID]: 1 },
+                { id: 2, total: 49.5, [ROWID]: 2 },
             ])
 
         const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
@@ -67,79 +69,140 @@ describe('Database Dump Module', () => {
             'attachment; filename="database_dump.sql"'
         )
 
-        const dumpText = await response.text()
-        expect(dumpText).toContain(
-            'CREATE TABLE users (id INTEGER, name TEXT);'
+        const dump = await response.text()
+        expect(dump.startsWith('SQLite format 3\0')).toBe(true)
+        expect(dump).toContain('CREATE TABLE users (id INTEGER, name TEXT);')
+        expect(dump).toContain(`INSERT INTO "users" VALUES (1, 'Alice');`)
+        expect(dump).toContain(`INSERT INTO "users" VALUES (2, 'Bob');`)
+        expect(dump).toContain('CREATE TABLE orders (id INTEGER, total REAL);')
+        expect(dump).toContain('INSERT INTO "orders" VALUES (1, 99.99);')
+        expect(dump).toContain('INSERT INTO "orders" VALUES (2, 49.5);')
+        // the internal rowid cursor must never leak into the dump
+        expect(dump).not.toContain(ROWID)
+    })
+
+    it('paginates large tables with keyset pagination (no OFFSET)', async () => {
+        const pageSize = 2
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([{ name: 'items' }])
+            .mockResolvedValueOnce([{ sql: 'CREATE TABLE items (id INTEGER)' }])
+            // a full page -> another fetch must follow
+            .mockResolvedValueOnce([
+                { id: 10, [ROWID]: 10 },
+                { id: 20, [ROWID]: 20 },
+            ])
+            // a partial page -> last page
+            .mockResolvedValueOnce([{ id: 30, [ROWID]: 30 }])
+
+        const response = await dumpDatabaseRoute(
+            mockDataSource,
+            mockConfig,
+            pageSize
         )
-        expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');")
-        expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');")
-        expect(dumpText).toContain(
-            'CREATE TABLE orders (id INTEGER, total REAL);'
+        const dump = await response.text()
+
+        expect(dump).toContain('INSERT INTO "items" VALUES (10);')
+        expect(dump).toContain('INSERT INTO "items" VALUES (20);')
+        expect(dump).toContain('INSERT INTO "items" VALUES (30);')
+
+        // 1 (tables) + 1 (schema) + 2 (row pages) = 4 calls
+        expect(executeOperation).toHaveBeenCalledTimes(4)
+
+        // the second row page must continue via a keyset cursor, not OFFSET
+        const secondPage = vi.mocked(executeOperation).mock.calls[3][0][0]
+        expect(secondPage.sql).toContain('_rowid_ > ?')
+        expect(secondPage.sql).not.toContain('OFFSET')
+        expect(secondPage.params).toEqual([20, pageSize])
+    })
+
+    it('encodes NULL, numbers, booleans, blobs and escapes quotes', async () => {
+        vi.mocked(executeOperation)
+            .mockResolvedValueOnce([{ name: 't' }])
+            .mockResolvedValueOnce([{ sql: 'CREATE TABLE t (a, b, c, d, e)' }])
+            .mockResolvedValueOnce([
+                {
+                    a: null,
+                    b: 42,
+                    c: true,
+                    d: "O'Brien",
+                    e: new Uint8Array([0xde, 0xad]).buffer,
+                    [ROWID]: 1,
+                },
+            ])
+
+        const dump = await (
+            await dumpDatabaseRoute(mockDataSource, mockConfig)
+        ).text()
+
+        expect(dump).toContain(
+            `INSERT INTO "t" VALUES (NULL, 42, 1, 'O''Brien', X'dead');`
         )
-        expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);')
-        expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);')
     })
 
-    it('should handle empty databases (no tables)', async () => {
+    it('handles an empty database (header only)', async () => {
         vi.mocked(executeOperation).mockResolvedValueOnce([])
 
-        const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+        const dump = await (
+            await dumpDatabaseRoute(mockDataSource, mockConfig)
+        ).text()
 
-        expect(response).toBeInstanceOf(Response)
-        expect(response.headers.get('Content-Type')).toBe(
-            'application/x-sqlite3'
-        )
-        const dumpText = await response.text()
-        expect(dumpText).toBe('SQLite format 3\0')
+        expect(dump).toBe('SQLite format 3\0')
     })
 
-    it('should handle databases with tables but no data', async () => {
+    it('handles a table with no rows', async () => {
         vi.mocked(executeOperation)
             .mockResolvedValueOnce([{ name: 'users' }])
-            .mockResolvedValueOnce([
-                { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
-            ])
+            .mockResolvedValueOnce([{ sql: 'CREATE TABLE users (id INTEGER)' }])
             .mockResolvedValueOnce([])
 
-        const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+        const dump = await (
+            await dumpDatabaseRoute(mockDataSource, mockConfig)
+        ).text()
 
-        expect(response).toBeInstanceOf(Response)
-        const dumpText = await response.text()
-        expect(dumpText).toContain(
-            'CREATE TABLE users (id INTEGER, name TEXT);'
-        )
-        expect(dumpText).not.toContain('INSERT INTO users VALUES')
+        expect(dump).toContain('CREATE TABLE users (id INTEGER);')
+        expect(dump).not.toContain('INSERT INTO')
     })
 
-    it('should escape single quotes properly in string values', async () => {
+    it('falls back to OFFSET pagination for WITHOUT ROWID tables', async () => {
+        const pageSize = 2
         vi.mocked(executeOperation)
-            .mockResolvedValueOnce([{ name: 'users' }])
+            .mockResolvedValueOnce([{ name: 'kv' }])
             .mockResolvedValueOnce([
-                { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' },
+                {
+                    sql: 'CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT) WITHOUT ROWID',
+                },
             ])
-            .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }])
-
-        const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+            .mockResolvedValueOnce([
+                { k: 'a', v: '1' },
+                { k: 'b', v: '2' },
+            ])
+            .mockResolvedValueOnce([{ k: 'c', v: '3' }])
 
-        expect(response).toBeInstanceOf(Response)
-        const dumpText = await response.text()
-        expect(dumpText).toContain(
-            "INSERT INTO users VALUES (1, 'Alice''s adventure');"
+        const response = await dumpDatabaseRoute(
+            mockDataSource,
+            mockConfig,
+            pageSize
         )
+        const dump = await response.text()
+
+        expect(dump).toContain(`INSERT INTO "kv" VALUES ('a', '1');`)
+        expect(dump).toContain(`INSERT INTO "kv" VALUES ('c', '3');`)
+
+        const firstPage = vi.mocked(executeOperation).mock.calls[2][0][0]
+        expect(firstPage.sql).toContain('OFFSET')
+        expect(firstPage.sql).not.toContain('_rowid_')
     })
 
-    it('should return a 500 response when an error occurs', async () => {
-        const consoleErrorMock = vi
-            .spyOn(console, 'error')
-            .mockImplementation(() => {})
+    it('returns 500 when the database cannot be read', async () => {
+        vi.spyOn(console, 'error').mockImplementation(() => {})
         vi.mocked(executeOperation).mockRejectedValue(
             new Error('Database Error')
         )
 
         const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
 
         expect(response.status).toBe(500)
-        const jsonResponse: { error: string } = await response.json()
-        expect(jsonResponse.error).toBe('Failed to create database dump')
+        const body = (await response.json()) as { error: string }
+        expect(body.error).toBe('Failed to create database dump')
     })
 })