Skip to content

Commit d7cdec5

Browse files
feat(stdlib): Sqlite schema introspection + bulk I/O + error inspection — db-theory #1c (6 externs) (#525)
## Summary Closes out the SQLite stdlib triad (#522 foundation + #524 prepared statements + this PR). Adds the three remaining practical surfaces: | layer | externs | |---|---| | **Schema introspection** | `db_schema_tables(d)`, `db_schema_columns(d, table)`, `db_table_exists(d, table)` | | **Bulk CSV I/O** | `db_import_csv(d, table, path, has_header)`, `db_export_csv(d, sql, params, path)` | | **Error inspection** | `db_last_error(d)` (typed `DbError` carrier lands in #1d) | ## Codegen + adapter 6 `b "name"` registrations + 6 `__as_db*` JS helpers, each delegating to one new method on `globalThis.__as_sqlite` (`schemaTables`, `schemaColumns`, `tableExists`, `importCsv`, `exportCsv`, `lastError`). Real adapters back these with one-line wrappers (PRAGMA table_info, prepare().iterate(), fs writes). ## Smoke harness (`tests/codegen-deno/sqlite_introspect_bulk.{affine,harness.mjs}`) 8 smoke functions × 9 assertions — schema_tables ordering, column descriptor shape (incl. PK flag), table_exists true/false, CSV import skipping header, CSV export including header, last_error empty/fault-injected. Mock extends the #1a/#1b adapter with the 6 new methods plus a virtual filesystem map for hermetic CSV testing. ## Verification - `dune build`: clean - `dune runtest`: **363 / 363** green - `tools/run_codegen_deno_tests.sh`: **33 / 33** harnesses pass (was 32) ## Stacked on #524 #524 (db-theory #1b — prepared statements) **MERGED 13:15Z**. This branch forks from its head and rebases cleanly. ## db-theory #1d (next PR, separate) - `extern type DbError` opaque carrier - `Result<T, DbError>`-shaped variants of the convenience surface - Design-blocked on the wider error-model question (do we add `Result` returns to all extern fns or keep the JSON-string sentinel pattern?) ## Test plan - [x] dune build clean - [x] dune runtest 363/363 green - [x] 33/33 codegen-deno harnesses pass - [x] 9 introspection/bulk/error smoke assertions pass 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 0f88028 commit d7cdec5

4 files changed

Lines changed: 379 additions & 1 deletion

File tree

lib/codegen_deno.ml

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,25 @@ const __as_dbColumnText = (s, idx) => {
541541
};
542542
const __as_dbReset = (s) => { globalThis.__as_sqlite.reset(s); return 0; };
543543
const __as_dbFinalize = (s) => { globalThis.__as_sqlite.finalize(s); return 0; };
544+
// ---- Sqlite schema introspection + bulk I/O + error inspection (db-theory #1c) ----
545+
// Five more adapter methods (`schemaTables`, `schemaColumns`,
546+
// `tableExists`, `importCsv`, `exportCsv`, `lastError`); each
547+
// real-world adapter (jsr:@db/sqlite, better-sqlite3) backs them with
548+
// a one-liner over `PRAGMA table_info` / a `Database.prepare()`
549+
// iterator / a `fs.writeFileSync(..., csv)` call.
550+
const __as_dbSchemaTables = (h) => String(globalThis.__as_sqlite.schemaTables(h));
551+
const __as_dbSchemaColumns = (h, table) => String(globalThis.__as_sqlite.schemaColumns(h, table));
552+
const __as_dbTableExists = (h, table) => Boolean(globalThis.__as_sqlite.tableExists(h, table));
553+
const __as_dbImportCsv = (h, table, path, hasHeader) =>
554+
Number(globalThis.__as_sqlite.importCsv(h, table, path, Boolean(hasHeader))) | 0;
555+
const __as_dbExportCsv = (h, sql, paramsJson, path) => {
556+
const params = paramsJson === "" || paramsJson === "[]" ? [] : JSON.parse(paramsJson);
557+
return Number(globalThis.__as_sqlite.exportCsv(h, sql, params, path)) | 0;
558+
};
559+
const __as_dbLastError = (h) => {
560+
const v = globalThis.__as_sqlite.lastError(h);
561+
return v == null ? "" : String(v);
562+
};
544563
const __as_httpFetch = async (url, method, headers, bodyOpt) => {
545564
const init = { method, headers: __as_httpHeadersToObject(headers) };
546565
if (bodyOpt && bodyOpt.tag === "Some") init.body = bodyOpt.value;
@@ -840,7 +859,14 @@ let () =
840859
b "db_column_int" (fun a -> Printf.sprintf "__as_dbColumnInt(%s, %s)" (arg 0 a) (arg 1 a));
841860
b "db_column_text" (fun a -> Printf.sprintf "__as_dbColumnText(%s, %s)" (arg 0 a) (arg 1 a));
842861
b "db_reset" (fun a -> Printf.sprintf "__as_dbReset(%s)" (arg 0 a));
843-
b "db_finalize" (fun a -> Printf.sprintf "__as_dbFinalize(%s)" (arg 0 a))
862+
b "db_finalize" (fun a -> Printf.sprintf "__as_dbFinalize(%s)" (arg 0 a));
863+
(* ---- Sqlite schema introspection + bulk I/O + error inspection (db-theory #1c) ---- *)
864+
b "db_schema_tables" (fun a -> Printf.sprintf "__as_dbSchemaTables(%s)" (arg 0 a));
865+
b "db_schema_columns" (fun a -> Printf.sprintf "__as_dbSchemaColumns(%s, %s)" (arg 0 a) (arg 1 a));
866+
b "db_table_exists" (fun a -> Printf.sprintf "__as_dbTableExists(%s, %s)" (arg 0 a) (arg 1 a));
867+
b "db_import_csv" (fun a -> Printf.sprintf "__as_dbImportCsv(%s, %s, %s, %s)" (arg 0 a) (arg 1 a) (arg 2 a) (arg 3 a));
868+
b "db_export_csv" (fun a -> Printf.sprintf "__as_dbExportCsv(%s, %s, %s, %s)" (arg 0 a) (arg 1 a) (arg 2 a) (arg 3 a));
869+
b "db_last_error" (fun a -> Printf.sprintf "__as_dbLastError(%s)" (arg 0 a))
844870

845871
(* ============================================================================
846872
Identifier sanitisation (JS reserved words -> trailing underscore)

stdlib/Sqlite.affine

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,59 @@ pub extern fn db_column_int(s: Stmt, idx: Int) -> Int;
6868
pub extern fn db_column_text(s: Stmt, idx: Int) -> String;
6969
pub extern fn db_reset(s: Stmt) -> Int;
7070
pub extern fn db_finalize(s: Stmt) -> Int;
71+
72+
// ── Schema introspection (db-theory #1c) ───────────────────────────
73+
//
74+
// Read the database's `sqlite_master`-equivalent metadata. Both
75+
// returns are JSON-encoded strings for consistency with the
76+
// convenience surface — caller parses with `json::parse`. A typed
77+
// `Schema` carrier with stronger guarantees (column types as a typed
78+
// enum, NOT NULL / PRIMARY KEY flags exposed) is the natural follow-on
79+
// once the wider db-theory programme lands a typed `Schema<T>` carrier
80+
// (db-theory #8 — declarative schema as type).
81+
82+
/// JSON array of table names — `'["users","posts","comments"]'`.
83+
/// Excludes sqlite_*-prefixed internal tables.
84+
pub extern fn db_schema_tables(d: Db) -> String;
85+
86+
/// JSON array of column descriptors for `table` — each entry has
87+
/// shape `{"name": "id", "type": "INTEGER", "notnull": true, "pk": true}`.
88+
/// Returns `"[]"` if the table does not exist.
89+
pub extern fn db_schema_columns(d: Db, table: String) -> String;
90+
91+
/// `true` iff `table` exists in the current database (excludes views,
92+
/// indexes, and the sqlite_* internal namespace). Convenience over
93+
/// parsing `db_schema_tables`.
94+
pub extern fn db_table_exists(d: Db, table: String) -> Bool;
95+
96+
// ── Bulk I/O (db-theory #1c) ───────────────────────────────────────
97+
//
98+
// CSV is the lingua franca for spreadsheet / data-engineering
99+
// pipelines; both directions stay narrow (no JSON-lines / Parquet
100+
// / Avro yet — those are separate bindings if and when needed).
101+
//
102+
// `db_import_csv` assumes the destination table already exists and the
103+
// CSV column order matches the table column order. `has_header=true`
104+
// skips the first row. Returns the number of rows inserted.
105+
//
106+
// `db_export_csv` runs `sql` with `params_json` substituted (same shape
107+
// as `db_query`), writes one row per result-row to `csv_path`, prepending
108+
// a header row of column names. Returns the number of rows written.
109+
110+
pub extern fn db_import_csv(d: Db, table: String, csv_path: String, has_header: Bool) -> Int;
111+
pub extern fn db_export_csv(d: Db, sql: String, params_json: String, csv_path: String) -> Int;
112+
113+
// ── Error inspection (db-theory #1c) ───────────────────────────────
114+
//
115+
// Most extern calls signal failure by raising in the host adapter
116+
// (caught by the AffineScript-side `try`/`catch` desugar). When a call
117+
// returns a sentinel instead of raising (e.g. `db_query_one` returns
118+
// `"null"` for "no row"), use `db_last_error` to disambiguate "no row"
119+
// from "error". Returns `""` if no error has been recorded on this
120+
// connection.
121+
//
122+
// A typed `DbError` carrier — landing in db-theory #1d alongside the
123+
// `Result<T, DbError>` shape — will eventually obsolete this string
124+
// surface, but it stays for the prototyping path.
125+
126+
pub extern fn db_last_error(d: Db) -> String;
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// SPDX-License-Identifier: MPL-2.0
2+
// db-theory #1c — Sqlite schema introspection + bulk I/O + error
3+
// inspection codegen smoke.
4+
//
5+
// Six smoke functions exercise:
6+
// - db_schema_tables (list created tables, JSON array)
7+
// - db_schema_columns (list columns of a table, JSON array)
8+
// - db_table_exists (boolean reflection)
9+
// - db_import_csv (mocked path import, returns row count)
10+
// - db_export_csv (mocked path export, returns row count)
11+
// - db_last_error (string error sentinel after a forced fault)
12+
13+
use Sqlite::{db_open, db_close, db_execute, db_schema_tables, db_schema_columns, db_table_exists, db_import_csv, db_export_csv, db_last_error};
14+
15+
pub fn smoke_schema_tables(path: String) -> String {
16+
let d = db_open(path);
17+
let _id1 = db_execute(d, "CREATE TABLE users(id INTEGER, name TEXT)");
18+
let _id2 = db_execute(d, "CREATE TABLE posts(id INTEGER, body TEXT)");
19+
let json = db_schema_tables(d);
20+
let _id3 = db_close(d);
21+
json
22+
}
23+
24+
pub fn smoke_schema_columns(path: String) -> String {
25+
let d = db_open(path);
26+
let _id1 = db_execute(d, "CREATE TABLE t(id INTEGER, name TEXT, age INTEGER)");
27+
let json = db_schema_columns(d, "t");
28+
let _id2 = db_close(d);
29+
json
30+
}
31+
32+
pub fn smoke_table_exists_true(path: String) -> Bool {
33+
let d = db_open(path);
34+
let _id1 = db_execute(d, "CREATE TABLE present(id INTEGER)");
35+
let yes = db_table_exists(d, "present");
36+
let _id2 = db_close(d);
37+
yes
38+
}
39+
40+
pub fn smoke_table_exists_false(path: String) -> Bool {
41+
let d = db_open(path);
42+
let no = db_table_exists(d, "absent");
43+
let _id1 = db_close(d);
44+
no
45+
}
46+
47+
pub fn smoke_import_csv(path: String, table: String, csv_path: String) -> Int {
48+
let d = db_open(path);
49+
let _id1 = db_execute(d, "CREATE TABLE rows(a INTEGER, b TEXT)");
50+
let n = db_import_csv(d, table, csv_path, true);
51+
let _id2 = db_close(d);
52+
n
53+
}
54+
55+
pub fn smoke_export_csv(path: String, sql: String, params_json: String, csv_path: String) -> Int {
56+
let d = db_open(path);
57+
let _id1 = db_execute(d, "CREATE TABLE rows(a INTEGER, b TEXT)");
58+
let _id2 = db_execute(d, "INSERT INTO rows VALUES (1, 'one'), (2, 'two'), (3, 'three')");
59+
let n = db_export_csv(d, sql, params_json, csv_path);
60+
let _id3 = db_close(d);
61+
n
62+
}
63+
64+
pub fn smoke_last_error_empty(path: String) -> String {
65+
let d = db_open(path);
66+
let e = db_last_error(d);
67+
let _id1 = db_close(d);
68+
e
69+
}
70+
71+
pub fn smoke_last_error_set(path: String) -> String {
72+
let d = db_open(path);
73+
// Mock-side: an execute with the magic string `RAISE` records an
74+
// error retrievable via db_last_error without throwing through to
75+
// the AffineScript side (the smoke validates the read-back path).
76+
let _id1 = db_execute(d, "RAISE 'simulated failure'");
77+
let e = db_last_error(d);
78+
let _id2 = db_close(d);
79+
e
80+
}
Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
// SPDX-License-Identifier: MPL-2.0
2+
// db-theory #1c — Node ESM harness for the Sqlite schema-introspection +
3+
// bulk-I/O + error-inspection codegen smoke.
4+
//
5+
// Extends the #1a/#1b mock adapter with `schemaTables`, `schemaColumns`,
6+
// `tableExists`, `importCsv`, `exportCsv`, `lastError`. The csv methods
7+
// here use a virtual filesystem map keyed by path so the smoke runs
8+
// hermetically (no real disk I/O); production adapters back the same
9+
// methods with `Deno.readTextFileSync` / `fs.writeFileSync`.
10+
11+
import assert from "node:assert/strict";
12+
13+
let nextDbHandle = 1;
14+
const dbs = new Map();
15+
const vfs = new Map(); // path -> string contents
16+
const errors = new Map(); // db handle -> last error message
17+
18+
const parseRowLiteral = (s) => {
19+
const out = [];
20+
let buf = "";
21+
let inStr = false;
22+
for (const ch of s) {
23+
if (ch === "'") { inStr = !inStr; buf += ch; }
24+
else if (ch === "," && !inStr) { out.push(buf.trim()); buf = ""; }
25+
else buf += ch;
26+
}
27+
if (buf.trim()) out.push(buf.trim());
28+
return out.map((t) => (
29+
t.startsWith("'") && t.endsWith("'") ? t.slice(1, -1) :
30+
/^-?\d+$/.test(t) ? Number(t) :
31+
t
32+
));
33+
};
34+
35+
globalThis.__as_sqlite = {
36+
open(path) {
37+
const h = nextDbHandle++;
38+
dbs.set(h, { path, tables: new Map(), schema: new Map() });
39+
errors.set(h, "");
40+
return h;
41+
},
42+
close(h) { dbs.delete(h); errors.delete(h); },
43+
44+
execute(h, sql) {
45+
const db = dbs.get(h);
46+
if (!db) throw new Error("invalid db handle " + h);
47+
48+
// Fault-injection convention for the smoke: a literal SQL string
49+
// starting with `RAISE` records a last-error and returns without
50+
// throwing, so the smoke can validate the read-back path.
51+
if (/^\s*RAISE\s+'(.+)'\s*$/i.test(sql)) {
52+
const m = sql.match(/^\s*RAISE\s+'(.+)'\s*$/i);
53+
errors.set(h, m[1]);
54+
return;
55+
}
56+
57+
const create = sql.match(/CREATE TABLE (\w+)\s*\(([^)]+)\)/i);
58+
if (create) {
59+
const cols = create[2].split(",").map((c) => {
60+
const parts = c.trim().split(/\s+/);
61+
return { name: parts[0], type: parts[1] || "" };
62+
});
63+
db.tables.set(create[1], []);
64+
db.schema.set(create[1], cols);
65+
errors.set(h, "");
66+
return;
67+
}
68+
69+
const insert = sql.match(/INSERT INTO (\w+)\s+VALUES\s+(.+)/i);
70+
if (insert) {
71+
const tableName = insert[1];
72+
const valuesPart = insert[2].replace(/;$/, "").trim();
73+
const tupleRe = /\(([^)]+)\)/g;
74+
let m;
75+
while ((m = tupleRe.exec(valuesPart))) {
76+
const cols = parseRowLiteral(m[1]);
77+
const tbl = db.tables.get(tableName);
78+
if (!tbl) { errors.set(h, "no such table " + tableName); throw new Error(errors.get(h)); }
79+
tbl.push(cols);
80+
}
81+
errors.set(h, "");
82+
}
83+
},
84+
85+
// ── Schema introspection ──────────────────────────────────────────
86+
87+
schemaTables(h) {
88+
const db = dbs.get(h);
89+
if (!db) return "[]";
90+
const names = [...db.tables.keys()].filter((n) => !n.startsWith("sqlite_"));
91+
return JSON.stringify(names);
92+
},
93+
94+
schemaColumns(h, table) {
95+
const db = dbs.get(h);
96+
if (!db) return "[]";
97+
const cols = db.schema.get(table);
98+
if (!cols) return "[]";
99+
return JSON.stringify(cols.map((c, i) => ({
100+
name: c.name,
101+
type: c.type,
102+
notnull: false,
103+
pk: i === 0 && /id/i.test(c.name),
104+
})));
105+
},
106+
107+
tableExists(h, table) {
108+
const db = dbs.get(h);
109+
if (!db) return false;
110+
return db.tables.has(table) && !table.startsWith("sqlite_");
111+
},
112+
113+
// ── Bulk I/O ──────────────────────────────────────────────────────
114+
115+
importCsv(h, table, csvPath, hasHeader) {
116+
const db = dbs.get(h);
117+
if (!db) { errors.set(h, "invalid handle"); return 0; }
118+
const text = vfs.get(csvPath);
119+
if (text == null) { errors.set(h, "no such csv " + csvPath); return 0; }
120+
const tbl = db.tables.get(table);
121+
if (!tbl) { errors.set(h, "no such table " + table); return 0; }
122+
const cols = db.schema.get(table) || [];
123+
const lines = text.split(/\r?\n/).filter((l) => l.length > 0);
124+
const data = hasHeader ? lines.slice(1) : lines;
125+
let inserted = 0;
126+
for (const line of data) {
127+
const fields = line.split(",").map((f) => {
128+
const t = f.trim();
129+
return /^-?\d+$/.test(t) ? Number(t) : t;
130+
});
131+
// Pad/truncate to schema width.
132+
while (fields.length < cols.length) fields.push(null);
133+
tbl.push(fields.slice(0, cols.length));
134+
inserted++;
135+
}
136+
errors.set(h, "");
137+
return inserted;
138+
},
139+
140+
exportCsv(h, sql, params, csvPath) {
141+
const db = dbs.get(h);
142+
if (!db) { errors.set(h, "invalid handle"); return 0; }
143+
const tblMatch = sql.match(/FROM\s+(\w+)/i);
144+
if (!tblMatch) { errors.set(h, "no FROM in sql"); return 0; }
145+
const table = tblMatch[1];
146+
const rows = db.tables.get(table) || [];
147+
const schema = db.schema.get(table) || [];
148+
const header = schema.map((c) => c.name).join(",");
149+
const body = rows.map((r) => r.map((v) => v == null ? "" : String(v)).join(",")).join("\n");
150+
vfs.set(csvPath, header + "\n" + body + "\n");
151+
errors.set(h, "");
152+
return rows.length;
153+
},
154+
155+
lastError(h) { return errors.get(h) ?? ""; },
156+
157+
// Convenience surface methods present from the #1a/#1b mock — not
158+
// exercised in this smoke but kept for future stack-on harnesses.
159+
query() { return []; },
160+
queryOne() { return null; },
161+
queryInt() { return 0; },
162+
prepare() { throw new Error("prepare not used in this smoke"); },
163+
bindInt() {}, bindText() {}, bindNull() {},
164+
step() { return false; },
165+
columnCount() { return 0; },
166+
columnInt() { return 0; },
167+
columnText() { return ""; },
168+
reset() {}, finalize() {},
169+
};
170+
171+
const mod = await import("./sqlite_introspect_bulk.deno.js");
172+
173+
// ── schema_tables ────────────────────────────────────────────────────
174+
const tablesJson = mod.smoke_schema_tables(":memory:");
175+
const tables = JSON.parse(tablesJson);
176+
assert.deepEqual(
177+
tables.sort(),
178+
["posts", "users"],
179+
"smoke_schema_tables: returns the two created tables (sqlite_* excluded)",
180+
);
181+
182+
// ── schema_columns ──────────────────────────────────────────────────
183+
const colsJson = mod.smoke_schema_columns(":memory:");
184+
const cols = JSON.parse(colsJson);
185+
assert.equal(cols.length, 3, "smoke_schema_columns: 3 columns parsed");
186+
assert.equal(cols[0].name, "id", "first column is id");
187+
assert.equal(cols[0].pk, true, "id column flagged as PK");
188+
assert.equal(cols[1].name, "name", "second column is name");
189+
assert.equal(cols[2].name, "age", "third column is age");
190+
191+
// ── table_exists ────────────────────────────────────────────────────
192+
assert.equal(mod.smoke_table_exists_true(":memory:"), true, "present table: exists=true");
193+
assert.equal(mod.smoke_table_exists_false(":memory:"), false, "absent table: exists=false");
194+
195+
// ── import_csv ──────────────────────────────────────────────────────
196+
vfs.set("/tmp/in.csv", "a,b\n1,alpha\n2,beta\n3,gamma\n");
197+
const inserted = mod.smoke_import_csv(":memory:", "rows", "/tmp/in.csv");
198+
assert.equal(inserted, 3, "smoke_import_csv: 3 data rows imported (header skipped)");
199+
200+
// ── export_csv ──────────────────────────────────────────────────────
201+
const exported = mod.smoke_export_csv(":memory:", "SELECT a, b FROM rows", "[]", "/tmp/out.csv");
202+
assert.equal(exported, 3, "smoke_export_csv: 3 rows exported");
203+
const out = vfs.get("/tmp/out.csv");
204+
assert.ok(out.startsWith("a,b\n"), "exported CSV starts with column header");
205+
assert.ok(out.includes("1,one"), "exported CSV contains first row");
206+
assert.ok(out.includes("3,three"), "exported CSV contains third row");
207+
208+
// ── last_error ──────────────────────────────────────────────────────
209+
assert.equal(mod.smoke_last_error_empty(":memory:"), "", "smoke_last_error_empty: returns ''");
210+
assert.equal(
211+
mod.smoke_last_error_set(":memory:"),
212+
"simulated failure",
213+
"smoke_last_error_set: fault-injected message round-trips",
214+
);
215+
216+
console.log("sqlite_introspect_bulk OK");

0 commit comments

Comments
 (0)