diff --git a/backend/src/api/v1/chat/completions.ts b/backend/src/api/v1/chat/completions.ts new file mode 100644 index 0000000000..1974862a68 --- /dev/null +++ b/backend/src/api/v1/chat/completions.ts @@ -0,0 +1,24 @@ +import { handleOpenrouterStream } from '../../../llm-apis/openrouter' +import { extractAuthTokenFromHeader } from '../../../util/auth-helpers' +import { getUserIdFromAuthToken } from '../../../websockets/auth' + +import type { Request, Response } from 'express' + +export async function completionsStreamHandler(req: Request, res: Response) { + console.log('asdf', { req: { headers: req.headers, body: req.body } }) + const token = extractAuthTokenFromHeader(req) + if (!token) { + res.status(401).json({ message: 'Unauthorized' }) + return + } + const userId = await getUserIdFromAuthToken(token) + if (!userId) { + res.status(401).json({ message: 'Invalid Codebuff API key' }) + return + } + + if (req.body.stream) { + return await handleOpenrouterStream({ req, res, userId }) + } + res.status(500).json({ message: 'Not implemented. Use stream=true.' }) +} diff --git a/backend/src/index.ts b/backend/src/index.ts index bda39f9607..e441cf1b84 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -13,6 +13,7 @@ import { import { validateAgentNameHandler } from './api/agents' import { isRepoCoveredHandler } from './api/org' import usageHandler from './api/usage' +import { completionsStreamHandler } from './api/v1/chat/completions' import { checkAdmin } from './util/check-auth' import { logger } from './util/logger' import { @@ -59,6 +60,9 @@ app.post( relabelForUserHandler, ) +// Openai compatible completions API +app.post('/api/v1/chat/completions', completionsStreamHandler) + app.use( ( err: Error, diff --git a/backend/src/llm-apis/openrouter.ts b/backend/src/llm-apis/openrouter.ts index 4e0d296aa0..777c4d78dd 100644 --- a/backend/src/llm-apis/openrouter.ts +++ b/backend/src/llm-apis/openrouter.ts @@ -2,8 +2,11 @@ import { models } from '@codebuff/common/old-constants' import { isExplicitlyDefinedModel } from '@codebuff/common/util/model-utils' import { env } from '@codebuff/internal/env' import { createOpenRouter } from '@codebuff/internal/openrouter-ai-sdk' +import { cloneDeep } from 'lodash' +import z from 'zod/v4' import type { Model } from '@codebuff/common/old-constants' +import type { Request, Response } from 'express' // Provider routing documentation: https://openrouter.ai/docs/features/provider-routing const providerOrder = { @@ -45,3 +48,178 @@ export function openRouterLanguageModel(model: Model) { logprobs: true, }) } + +const openrouterUsageSchema = z + .object({ + prompt_tokens: z.number(), + prompt_tokens_details: z + .object({ + cached_tokens: z.number(), + }) + .nullish(), + completion_tokens: z.number(), + completion_tokens_details: z + .object({ + reasoning_tokens: z.number(), + }) + .nullish(), + total_tokens: z.number(), + cost: z.number().optional(), + cost_details: z + .object({ + upstream_inference_cost: z.number().nullish(), + }) + .nullish(), + }) + .nullish() + +export async function handleOpenrouterStream({ + req, + res, + userId, +}: { + req: Request + res: Response + userId: string +}) { + res.writeHead(200, { + // Mandatory SSE headers + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + // (optional) allow local browser demos + 'Access-Control-Allow-Origin': '*', + }) + + res.write(`: connected ${new Date().toISOString()}\n`) + const heartbeat = setInterval(() => { + res.write(`: heartbeat ${new Date().toISOString()}\n\n`) + }, 30000) + res.on('close', () => { + clearInterval(heartbeat) + }) + + const body = cloneDeep(req.body) + if (body.usage === undefined) { + body.usage = {} + } + body.usage.include = true + const response = await fetch( + 'https://openrouter.ai/api/v1/chat/completions', + { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.OPEN_ROUTER_API_KEY}`, + 'HTTP-Referer': 'https://codebuff.com', + 'X-Title': 'Codebuff', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }, + ) + + const reader = response.body?.getReader() + if (!reader) { + res.status(500).json({ message: 'Failed to get response reader' }) + return + } + + const decoder = new TextDecoder() + let buffer = '' + try { + while (true) { + const { done, value } = await reader.read() + console.log('asdf', { + done, + value: decoder.decode(value, { stream: true }), + }) + if (done) { + break + } + + buffer += decoder.decode(value, { stream: true }) + let lineEnd = buffer.indexOf('\n') + while (lineEnd !== -1) { + const line = buffer.slice(0, lineEnd + 1) + buffer = buffer.slice(lineEnd + 1) + // if (line.startsWith('data: ')) { + // const data = line.trim().slice('data: '.length) + // await processData(data, userId) + // } + res.write(line) + lineEnd = buffer.indexOf('\n') + } + } + } finally { + reader.cancel() + } + res.end() +} + +/* + +async function processData(data: string, userId: string) { + if (data === '[DONE]') { + return + } + + let obj + try { + obj = JSON.parse(data) + } catch (error) { + trackEvent( + AnalyticsEvent.OPENROUTER_MALFORMED_JSON_RESPONSE_CHUNK, + userId, + { + data, + }, + ) + return + } + + if (typeof obj !== 'object') { + return + } + if (typeof obj.usage !== 'object') { + return + } + const parseResult = openrouterUsageSchema.safeParse(obj.usage) + if (!parseResult.success) { + trackEvent( + AnalyticsEvent.OPENROUTER_MALFORMED_JSON_RESPONSE_CHUNK, + userId, + { + message: `Usage does not match schema:\n${parseResult.error.message}`, + data, + }, + ) + return + } + + const directCost = parseResult?.data?.cost ?? 0 + const upstreamCost = parseResult?.data?.cost_details?.upstream_inference_cost + + saveMessage({ + messageId: obj.id, + userId, + clientSessionId: generateCompactId('direct-'), + fingerprintId: generateCompactId('direct-'), + userInputId: generateCompactId('direct-'), + model, + request, + request: Message[] + response: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens?: number + cacheReadInputTokens?: number + finishedAt: Date + latencyMs: number + usesUserApiKey?: boolean + chargeUser?: boolean + costOverrideDollars?: number + agentId?: string + }) +} + +*/ diff --git a/backend/src/util/auth-helpers.ts b/backend/src/util/auth-helpers.ts index 084e0da97d..9926de776b 100644 --- a/backend/src/util/auth-helpers.ts +++ b/backend/src/util/auth-helpers.ts @@ -1,10 +1,20 @@ import type { Request } from 'express' /** - * Extract auth token from x-codebuff-api-key header + * Extract auth token from x-codebuff-api-key header or authorization header */ export function extractAuthTokenFromHeader(req: Request): string | undefined { - const token = req.headers['x-codebuff-api-key'] as string | undefined - // Trim any whitespace that might be present - return token?.trim() -} \ No newline at end of file + const token = req.headers['x-codebuff-api-key'] + if (typeof token === 'string' && token) { + return token + } + + const authorization = req.headers['authorization'] + if (!authorization) { + return undefined + } + if (!authorization.startsWith('Bearer ')) { + return undefined + } + return authorization.slice('Bearer '.length) +} diff --git a/common/src/constants/analytics-events.ts b/common/src/constants/analytics-events.ts index 7e427320f8..079044d701 100644 --- a/common/src/constants/analytics-events.ts +++ b/common/src/constants/analytics-events.ts @@ -33,6 +33,7 @@ export enum AnalyticsEvent { TOOL_USE = 'backend.tool_use', UNKNOWN_TOOL_CALL = 'backend.unknown_tool_call', USER_INPUT = 'backend.user_input', + OPENROUTER_MALFORMED_JSON_RESPONSE_CHUNK = 'backend.openrouter_malformed_json_response_chunk', // Web SIGNUP = 'web.signup', diff --git a/npm-app/package.json b/npm-app/package.json index 0d509a9583..f0f1b825e7 100644 --- a/npm-app/package.json +++ b/npm-app/package.json @@ -34,6 +34,7 @@ "bun": ">=1.2.11" }, "dependencies": { + "@ai-sdk/openai-compatible": "^1.0.19", "@codebuff/code-map": "workspace:*", "@codebuff/common": "workspace:*", "@types/diff": "8.0.0", diff --git a/npm-app/src/asdf.ts b/npm-app/src/asdf.ts new file mode 100644 index 0000000000..4946290704 --- /dev/null +++ b/npm-app/src/asdf.ts @@ -0,0 +1,35 @@ +import { createOpenAICompatible } from '@ai-sdk/openai-compatible' +import { streamText } from 'ai' + +import { backendUrl } from './config' + +const codebuffBackendProvider = createOpenAICompatible({ + name: 'codebuff', + apiKey: '12345', + baseURL: backendUrl + '/api/v1', +}) + +const response = streamText({ + model: codebuffBackendProvider('anthropic/claude-sonnet-4.5'), + messages: [ + { + role: 'user', + content: + 'This is a bunch of text just to fill out some space. Ignore this.'.repeat( + 1000, + ), + }, + { + role: 'user', + content: 'Hello', + providerOptions: { + codebuff: { + cacheControl: { type: 'ephemeral' }, + }, + }, + }, + ], +}) +for await (const chunk of response.fullStream) { + console.log('asdf', { chunk }) +}