Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test-database.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
- name: Setup node
uses: actions/setup-node@v4
with:
node-version: "20"
node-version: "22"
cache: "pnpm"
- name: Install Dependencies
run: pnpm install --frozen-lockfile
Expand Down
27 changes: 13 additions & 14 deletions apps/website/app/api/supabase/agent-identifier/route.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { NextResponse, NextRequest } from "next/server";

import { createClient } from "~/utils/supabase/server";
import { getOrCreateEntity, ItemValidator } from "~/utils/supabase/dbUtils";
import { getOrCreateEntity } from "~/utils/supabase/dbUtils";
import { asPostgrestFailure } from "@repo/database/lib/contextFunctions";
import {
createApiResponse,
Expand All @@ -11,25 +11,26 @@ import {
import { type TablesInsert, Constants } from "@repo/database/dbTypes";

type AgentIdentifierDataInput = TablesInsert<"AgentIdentifier">;
// eslint-disable-next-line @typescript-eslint/naming-convention
const { AgentIdentifierType } = Constants.public.Enums;

const agentIdentifierValidator: ItemValidator<AgentIdentifierDataInput> = (
agent_identifier: any,
) => {
// ItemValidator<"AgentIdentifier">
const agentIdentifierValidator = (
// eslint-disable-next-line @typescript-eslint/naming-convention
agent_identifier: AgentIdentifierDataInput,
): string | null => {
if (!agent_identifier || typeof agent_identifier !== "object")
return "Invalid request body: expected a JSON object.";
// eslint-disable-next-line @typescript-eslint/naming-convention
const { identifier_type, account_id, value, trusted } = agent_identifier;

if (!AgentIdentifierType.includes(identifier_type))
return "Invalid identifier_type";
if (!value || typeof value !== "string" || value.trim() === "")
return "Missing or invalid value";
if (!account_id || Number.isNaN(Number.parseInt(account_id, 10)))
if (!account_id || typeof account_id !== "number" || Number.isNaN(account_id))
return "Missing or invalid account_id";
if (
trusted !== undefined &&
!["true", "false", true, false].includes(trusted)
)
if (trusted !== undefined && typeof trusted !== "boolean")
return "if included, trusted should be a boolean";

const keys = ["identifier_type", "account_id", "value", "trusted"];
Expand All @@ -42,18 +43,16 @@ export const POST = async (request: NextRequest): Promise<NextResponse> => {
const supabasePromise = createClient();

try {
const body = await request.json();
const body = (await request.json()) as AgentIdentifierDataInput;
const error = agentIdentifierValidator(body);
if (error !== null)
return createApiResponse(request, asPostgrestFailure(error, "invalid"));

body.account_id = Number.parseInt(body.account_id, 10);
body.trusted = body.trusted === true || body.trusted === "true" || false;
const supabase = await supabasePromise;
const result = await getOrCreateEntity<"AgentIdentifier">({
const result = await getOrCreateEntity({
supabase,
tableName: "AgentIdentifier",
insertData: body as AgentIdentifierDataInput,
insertData: body,
uniqueOn: ["value", "identifier_type", "account_id"],
});

Expand Down
28 changes: 13 additions & 15 deletions apps/website/app/api/supabase/content-embedding/batch/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import {
KNOWN_EMBEDDING_TABLES,
} from "~/utils/supabase/dbUtils";
import {
ApiInputEmbeddingItem,
ApiOutputEmbeddingRecord,
type ContentEmbeddingVecTablesInsert,
type ContentEmbeddingVecTables,
embeddingInputProcessing,
embeddingOutputProcessing,
} from "~/utils/supabase/validators";
Expand All @@ -23,19 +23,19 @@ const DEFAULT_MODEL = "openai_text_embedding_3_small_1536";

const batchInsertEmbeddingsProcess = async (
supabase: Awaited<ReturnType<typeof createClient>>,
embeddingItems: ApiInputEmbeddingItem[],
): Promise<PostgrestResponse<ApiOutputEmbeddingRecord>> => {
embeddingItems: ContentEmbeddingVecTablesInsert[],
): Promise<PostgrestResponse<ContentEmbeddingVecTables>> => {
// groupBy is node21 only, we are using 20. Group by model, by hand.
// Note: This means that later index values may be totally wrong.
// Note2: The key is a ModelName, but I cannot use an enum as a key.
const byModel: { [key: string]: ApiInputEmbeddingItem[] } = {};
const byModel: { [key: string]: ContentEmbeddingVecTablesInsert[] } = {};
try {
embeddingItems.reduce((acc, item) => {
const model = item?.model || DEFAULT_MODEL;
if (acc[model] === undefined) {
acc[model] = [];
}
acc[model]!.push(item);
acc[model].push(item);
return acc;
}, byModel);
} catch (error) {
Expand All @@ -45,7 +45,7 @@ const batchInsertEmbeddingsProcess = async (
throw error;
}

const globalResults: ApiOutputEmbeddingRecord[] = [];
const globalResults: ContentEmbeddingVecTables[] = [];
const partialErrors: string[] = [];
let created = false,
count = 0,
Expand All @@ -55,15 +55,11 @@ const batchInsertEmbeddingsProcess = async (
if (embeddingItemsSet === undefined) continue;
const tableData = KNOWN_EMBEDDING_TABLES[modelName];
if (tableData === undefined) continue;
const results = await processAndInsertBatch<
// any ContentEmbedding table for type checking purposes only
"ContentEmbedding_openai_text_embedding_3_small_1536",
ApiInputEmbeddingItem,
ApiOutputEmbeddingRecord
>({
const { tableName } = tableData;
const results = await processAndInsertBatch({
supabase,
items: embeddingItemsSet,
tableName: tableData.tableName,
tableName,
inputProcessor: embeddingInputProcessing,
outputProcessor: embeddingOutputProcessing,
});
Expand All @@ -81,6 +77,7 @@ const batchInsertEmbeddingsProcess = async (
return {
data: globalResults,
error: null,
success: true,
status: has_400 ? 400 : 500,
count,
statusText: partialErrors.join("; "),
Expand All @@ -89,6 +86,7 @@ const batchInsertEmbeddingsProcess = async (
return {
data: globalResults,
error: null,
success: true,
status: created ? 201 : 200,
count,
statusText: created ? "created" : "success",
Expand All @@ -106,7 +104,7 @@ export const POST = async (request: NextRequest): Promise<NextResponse> => {
const supabase = await createClient();

try {
const body: ApiInputEmbeddingItem[] = await request.json();
const body = (await request.json()) as ContentEmbeddingVecTablesInsert[];
if (!Array.isArray(body)) {
return createApiResponse(
request,
Expand Down
27 changes: 13 additions & 14 deletions apps/website/app/api/supabase/content-embedding/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import {
KNOWN_EMBEDDING_TABLES,
} from "~/utils/supabase/dbUtils";
import {
ApiInputEmbeddingItem,
ApiOutputEmbeddingRecord,
ContentEmbeddingVecTablesInsert,
ContentEmbeddingVecTables,
embeddingInputProcessing,
embeddingOutputProcessing,
} from "~/utils/supabase/validators";
Expand All @@ -23,9 +23,11 @@ const DEFAULT_MODEL = "openai_text_embedding_3_small_1536";

const processAndCreateEmbedding = async (
supabasePromise: ReturnType<typeof createClient>,
data: ApiInputEmbeddingItem,
): Promise<PostgrestSingleResponse<ApiOutputEmbeddingRecord>> => {
const { valid, error, processedItem } = embeddingInputProcessing(data);
data: ContentEmbeddingVecTablesInsert,
): Promise<PostgrestSingleResponse<ContentEmbeddingVecTables>> => {
const processed = embeddingInputProcessing(data);
if (!processed) return asPostgrestFailure("Could not process input", "valid");
const { valid, error, processedItem } = processed;
if (
!valid ||
processedItem === undefined ||
Expand All @@ -41,14 +43,11 @@ const processAndCreateEmbedding = async (
const { tableName } = tableData;
// Using getOrCreateEntity, forcing create path by providing non-matching criteria
// This standardizes return type and error handling (e.g., FK violations from dbUtils)
const result =
await getOrCreateEntity<"ContentEmbedding_openai_text_embedding_3_small_1536">(
{
supabase,
tableName,
insertData: processedItem,
},
);
const result = await getOrCreateEntity({
supabase,
tableName,
insertData: processedItem,
});

if (result.error) {
return result;
Expand Down Expand Up @@ -81,7 +80,7 @@ export const POST = async (request: NextRequest): Promise<NextResponse> => {
const supabasePromise = createClient();

try {
const body: ApiInputEmbeddingItem = await request.json();
const body = (await request.json()) as ContentEmbeddingVecTablesInsert;
const result = await processAndCreateEmbedding(supabasePromise, body);
return createApiResponse(request, result);
} catch (e: unknown) {
Expand Down
2 changes: 1 addition & 1 deletion apps/website/app/api/supabase/content/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const processAndUpsertContentEntry = async (
// If no solid matchCriteria for a "get", getOrCreateEntity will likely proceed to "create".
// If there are unique constraints other than (space_id, source_local_id), it will handle race conditions.

const result = await getOrCreateEntity<"Content">({
const result = await getOrCreateEntity({
supabase,
tableName: "Content",
insertData: data,
Expand Down
10 changes: 6 additions & 4 deletions apps/website/app/api/supabase/document/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { NextResponse, NextRequest } from "next/server";
import type { PostgrestSingleResponse } from "@supabase/supabase-js";

import { createClient } from "~/utils/supabase/server";
import { getOrCreateEntity, ItemValidator } from "~/utils/supabase/dbUtils";
import { getOrCreateEntity } from "~/utils/supabase/dbUtils";
import { asPostgrestFailure } from "@repo/database/lib/contextFunctions";
import {
createApiResponse,
Expand All @@ -14,9 +14,11 @@ import type { Tables, TablesInsert } from "@repo/database/dbTypes";
type DocumentDataInput = TablesInsert<"Document">;
type DocumentRecord = Tables<"Document">;

const validateDocument: ItemValidator<DocumentDataInput> = (data) => {
// ItemValidator<"Document">
const validateDocument = (data: DocumentDataInput): string | null => {
if (!data || typeof data !== "object")
return "Invalid request body: expected a JSON object.";
// eslint-disable-next-line @typescript-eslint/naming-convention
const { space_id, author_id, source_local_id } = data;

if (!author_id) return "Missing required author_id field.";
Expand All @@ -32,7 +34,7 @@ const createDocument = async (
): Promise<PostgrestSingleResponse<DocumentRecord>> => {
const supabase = await supabasePromise;

const result = await getOrCreateEntity<"Document">({
const result = await getOrCreateEntity({
supabase,
tableName: "Document",
insertData: data,
Expand All @@ -49,7 +51,7 @@ export const POST = async (request: NextRequest): Promise<NextResponse> => {
const supabasePromise = createClient();

try {
const body: DocumentDataInput = await request.json();
const body = (await request.json()) as DocumentDataInput;
const error = validateDocument(body);
if (error !== null)
return createApiResponse(request, asPostgrestFailure(error, "invalid"));
Expand Down
16 changes: 9 additions & 7 deletions apps/website/app/api/supabase/platform-account/route.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { NextResponse, NextRequest } from "next/server";

import { createClient } from "~/utils/supabase/server";
import { getOrCreateEntity, ItemValidator } from "~/utils/supabase/dbUtils";
import { getOrCreateEntity } from "~/utils/supabase/dbUtils";
import { asPostgrestFailure } from "@repo/database/lib/contextFunctions";
import {
createApiResponse,
Expand All @@ -10,15 +10,16 @@ import {
} from "~/utils/supabase/apiUtils";
import { type TablesInsert, Constants } from "@repo/database/dbTypes";

// eslint-disable-next-line @typescript-eslint/naming-convention
const { AgentType, Platform } = Constants.public.Enums;

type PlatformAccountDataInput = TablesInsert<"PlatformAccount">;

const accountValidator: ItemValidator<PlatformAccountDataInput> = (
account: any,
) => {
// ItemValidator<"PlatformAccount">
const accountValidator = (account: PlatformAccountDataInput): string | null => {
if (!account || typeof account !== "object")
return "Invalid request body: expected a JSON object.";
/* eslint-disable @typescript-eslint/naming-convention */
const {
name,
platform,
Expand All @@ -29,6 +30,7 @@ const accountValidator: ItemValidator<PlatformAccountDataInput> = (
metadata,
dg_account,
} = account;
/* eslint-enable @typescript-eslint/naming-convention */

if (!name || typeof name !== "string" || name.trim() === "")
return "Missing or invalid name";
Expand Down Expand Up @@ -81,16 +83,16 @@ export const POST = async (request: NextRequest): Promise<NextResponse> => {
const supabasePromise = createClient();

try {
const body = await request.json();
const body = (await request.json()) as PlatformAccountDataInput;
const error = accountValidator(body);
if (error !== null)
return createApiResponse(request, asPostgrestFailure(error, "invalid"));

const supabase = await supabasePromise;
const result = await getOrCreateEntity<"PlatformAccount">({
const result = await getOrCreateEntity({
supabase,
tableName: "PlatformAccount",
insertData: body as PlatformAccountDataInput,
insertData: body,
uniqueOn: ["account_local_id", "platform"],
});

Expand Down
Loading
Loading