Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions src/main/backend/__tests__/tokenUsage.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { describe, expect, it } from 'vitest'
import { normalizeUsage } from '../tokenUsage'

describe('normalizeUsage', () => {
it('normalizes OpenAI-compatible snake_case usage', () => {
expect(normalizeUsage({
prompt_tokens: 100,
completion_tokens: 25,
total_tokens: 125,
})).toEqual({
promptTokens: 100,
completionTokens: 25,
totalTokens: 125,
})
})

it('normalizes input/output token usage fields', () => {
expect(normalizeUsage({
input_tokens: 120,
output_tokens: 30,
})).toEqual({
promptTokens: 120,
completionTokens: 30,
totalTokens: 150,
})
})

it('normalizes camelCase usage fields', () => {
expect(normalizeUsage({
promptTokens: 80,
completionTokens: 20,
totalTokens: 100,
})).toEqual({
promptTokens: 80,
completionTokens: 20,
totalTokens: 100,
})
})

it('calculates total when prompt and completion tokens are present', () => {
expect(normalizeUsage({
prompt_tokens: 60,
completion_tokens: 15,
})).toEqual({
promptTokens: 60,
completionTokens: 15,
totalTokens: 75,
})
})

it('keeps zero token counts', () => {
expect(normalizeUsage({
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
})).toEqual({
promptTokens: 0,
completionTokens: 0,
totalTokens: 0,
})
})

it('returns undefined for missing or invalid usage', () => {
expect(normalizeUsage(null)).toBeUndefined()
expect(normalizeUsage('usage')).toBeUndefined()
expect(normalizeUsage([])).toBeUndefined()
expect(normalizeUsage({})).toBeUndefined()
expect(normalizeUsage({ prompt_tokens: '100' })).toBeUndefined()
})
})
48 changes: 42 additions & 6 deletions src/main/backend/deepseekClient.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,16 @@
import { AppError, ErrorCodes } from './errors'
import { getActiveSettings } from './settingsStore'
import { normalizeUsage, TokenUsage } from './tokenUsage'

interface ProviderConfig {
baseURL: string
includeStreamUsage?: boolean
}

export interface ChatCompletionResult {
content: string
usage?: TokenUsage
rawUsage?: unknown
}

interface JiekouModelConfig {
Expand All @@ -29,12 +37,12 @@ interface MimoModelConfig {
}

const PROVIDER_CONFIGS: Record<string, ProviderConfig> = {
deepseek: { baseURL: 'https://api.deepseek.com/v1' },
deepseek: { baseURL: 'https://api.deepseek.com/v1', includeStreamUsage: true },
jiekou: { baseURL: 'https://api.jiekou.ai/openai' },
minimax: { baseURL: 'https://api.minimaxi.com/v1' },
glm: { baseURL: 'https://open.bigmodel.cn/api/paas/v4' },
mimo: { baseURL: 'https://api.xiaomimimo.com/v1' },
kimi: { baseURL: 'https://api.moonshot.cn/v1' },
kimi: { baseURL: 'https://api.moonshot.cn/v1', includeStreamUsage: true },
}

const JIEKOU_MODEL_CONFIGS: Record<string, JiekouModelConfig> = {
Expand Down Expand Up @@ -100,8 +108,9 @@ export function loadConfig() {

export async function callDeepSeek(
messages: { role: 'system' | 'user' | 'assistant'; content: string }[],
onUpdate?: (chunk: string) => void
): Promise<string> {
onUpdate?: (chunk: string) => void,
signal?: AbortSignal
): Promise<ChatCompletionResult> {
const config = loadConfig()
const providerCfg = PROVIDER_CONFIGS[config.provider]

Expand All @@ -110,7 +119,17 @@ export async function callDeepSeek(
}

const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), 120_000)
let timedOut = false
const abortFromSignal = () => controller.abort()
if (signal?.aborted) {
controller.abort()
} else {
signal?.addEventListener('abort', abortFromSignal, { once: true })
}
const timeout = setTimeout(() => {
timedOut = true
controller.abort()
}, 120_000)

try {
const modelCfg = config.provider === 'jiekou'
Expand Down Expand Up @@ -141,12 +160,17 @@ export async function callDeepSeek(
presence_penalty?: number
stop?: null
thinking?: { type: string }
stream_options?: { include_usage: boolean }
} = {
model: config.model,
messages,
stream: true,
}

if (providerCfg.includeStreamUsage) {
requestBody.stream_options = { include_usage: true }
}

if (modelCfg && 'tokenParam' in modelCfg && modelCfg.tokenParam && modelCfg.maxTokens) {
requestBody[modelCfg.tokenParam] = modelCfg.maxTokens
} else if (modelCfg?.maxTokens) {
Expand Down Expand Up @@ -210,6 +234,8 @@ export async function callDeepSeek(
}

let result = ''
let usage: TokenUsage | undefined
let rawUsage: unknown
const decoder = new TextDecoder()
let buffer = ''

Expand All @@ -229,6 +255,11 @@ export async function callDeepSeek(

try {
const json = JSON.parse(payload)
if (json.usage) {
rawUsage = json.usage
usage = normalizeUsage(json.usage) ?? usage
}

const content = json.choices?.[0]?.delta?.content || ''
if (content) {
result += content
Expand All @@ -244,11 +275,16 @@ export async function callDeepSeek(
throw new AppError(ErrorCodes.API_RESPONSE_INVALID, 'API returned empty response')
}

return result
signal?.removeEventListener('abort', abortFromSignal)
return { content: result, usage, rawUsage }
} catch (err) {
clearTimeout(timeout)
signal?.removeEventListener('abort', abortFromSignal)
if (err instanceof AppError) throw err
if ((err as Error).name === 'AbortError') {
if (!timedOut && signal?.aborted) {
throw new AppError(ErrorCodes.ANALYSIS_CANCELLED, 'Analysis was cancelled by the user.')
}
throw new AppError(ErrorCodes.API_TIMEOUT, 'Request timed out after 120 seconds. The paper may be too long.')
}
throw new AppError(
Expand Down
13 changes: 11 additions & 2 deletions src/main/backend/errors.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { TokenUsage } from './tokenUsage'

export const ErrorCodes = {
PDF_NOT_FOUND: 'PDF_NOT_FOUND',
PDF_INVALID: 'PDF_INVALID',
Expand All @@ -10,6 +12,7 @@ export const ErrorCodes = {
API_NETWORK_ERROR: 'API_NETWORK_ERROR',
API_RESPONSE_INVALID: 'API_RESPONSE_INVALID',
ANALYSIS_FAILED: 'ANALYSIS_FAILED',
ANALYSIS_CANCELLED: 'ANALYSIS_CANCELLED',
NO_CODE_CACHE: 'NO_CODE_CACHE',
EXPORT_FAILED: 'EXPORT_FAILED',
} as const
Expand All @@ -34,12 +37,18 @@ export type AnalysisProgress =
export type AnalysisResult =
| {
ok: true
summary: string
hasCoreCode: boolean
result: {
summary: string
hasCoreCode: boolean
}
usage?: TokenUsage
rawUsage?: unknown
}
| {
ok: false
error: { code: string; message: string; detail?: string }
usage?: TokenUsage
rawUsage?: unknown
}

export type ExportResult =
Expand Down
32 changes: 27 additions & 5 deletions src/main/backend/paperAnalyzer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { buildCombinedAnalysisPrompt } from './promptBuilder'
import { cacheCodeBundle, clearCache, CodeBlueprint, GeneratedFile } from './codeCache'
import { AnalysisProgress, AnalysisResult, AppError, ErrorCodes } from './errors'
import { getActiveSettings } from './settingsStore'
import { TokenUsage } from './tokenUsage'
import { BLUEPRINT_START, BLUEPRINT_END, extractJsonObject, parseCodeBlueprint, validateFilesAgainstBlueprint, validateGeneratedFilePath } from './codeBlueprint'

const SUMMARY_START = '<P2CC_SUMMARY>'
Expand Down Expand Up @@ -89,19 +90,29 @@ export function parseCodeDecision(raw: string): { needed: boolean; reason?: stri
export async function analyzePaper(
pdfPath: string,
onProgress: (progress: AnalysisProgress) => void,
onSummaryChunk?: (chunk: string) => void
onSummaryChunk?: (chunk: string) => void,
signal?: AbortSignal
): Promise<AnalysisResult> {
let language = 'zh-CN'
try {
language = getActiveSettings().language || 'zh-CN'
} catch {}

const msg = (zh: string, en: string) => language === 'en-US' ? en : zh
let usage: TokenUsage | undefined
let rawUsage: unknown
const throwIfCancelled = () => {
if (signal?.aborted) {
throw new AppError(ErrorCodes.ANALYSIS_CANCELLED, msg('分析已取消', 'Analysis was cancelled'))
}
}

try {
throwIfCancelled()
clearCache()
onProgress({ stage: 'parsing', message: msg('读取 PDF 文件...', 'Reading PDF file...') })
const { text, pageCount } = await parsePDF(pdfPath)
throwIfCancelled()
onProgress({ stage: 'parsing', message: msg(`PDF 解析完成 (${pageCount} 页)`, `PDF parsed (${pageCount} pages)`) })

onProgress({ stage: 'summarizing', message: msg('正在分析论文结构并生成总结...', 'Analyzing paper structure and generating summary...') })
Expand All @@ -113,7 +124,7 @@ export async function analyzePaper(
let blueprintProgressSent = false
let bundleProgressSent = false

await callDeepSeek(
const llmResult = await callDeepSeek(
[
{ role: 'system', content: analysisPrompt.system },
{ role: 'user', content: analysisPrompt.user },
Expand Down Expand Up @@ -154,8 +165,11 @@ export async function analyzePaper(
bundleProgressSent = true
onProgress({ stage: 'generating_code', message: msg('正在按蓝图生成核心代码文件...', 'Generating core code files from blueprint...') })
}
}
},
signal
)
usage = llmResult.usage
rawUsage = llmResult.rawUsage

const cleanedSummary = extractTaggedContent(rawOutput, SUMMARY_START, SUMMARY_END, '模型返回缺少论文总结区块')
const decisionContent = extractTaggedContent(rawOutput, DECISION_START, DECISION_END, '模型返回缺少代码生成决策')
Expand Down Expand Up @@ -190,19 +204,27 @@ export async function analyzePaper(

return {
ok: true,
summary: cleanedSummary.trim(),
hasCoreCode,
result: {
summary: cleanedSummary.trim(),
hasCoreCode,
},
usage,
rawUsage,
}
} catch (err) {
if (err instanceof AppError) {
return {
ok: false,
error: { code: err.code, message: err.message, detail: err.detail },
usage,
rawUsage,
}
}
return {
ok: false,
error: { code: ErrorCodes.ANALYSIS_FAILED, message: msg('分析过程中发生未知错误', 'An unknown error occurred during analysis'), detail: (err as Error).message },
usage,
rawUsage,
}
}
}
42 changes: 42 additions & 0 deletions src/main/backend/tokenUsage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
export interface TokenUsage {
promptTokens?: number
completionTokens?: number
totalTokens?: number
}

function readTokenCount(raw: Record<string, unknown>, keys: string[]): number | undefined {
for (const key of keys) {
const value = raw[key]
if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
return value
}
}

return undefined
}

export function normalizeUsage(rawUsage: unknown): TokenUsage | undefined {
if (rawUsage === null || typeof rawUsage !== 'object' || Array.isArray(rawUsage)) {
return undefined
}

const raw = rawUsage as Record<string, unknown>
const promptTokens = readTokenCount(raw, ['prompt_tokens', 'input_tokens', 'promptTokens', 'inputTokens'])
const completionTokens = readTokenCount(raw, ['completion_tokens', 'output_tokens', 'completionTokens', 'outputTokens'])
const providedTotalTokens = readTokenCount(raw, ['total_tokens', 'totalTokens'])
const totalTokens = providedTotalTokens ?? (
promptTokens !== undefined && completionTokens !== undefined
? promptTokens + completionTokens
: undefined
)

if (promptTokens === undefined && completionTokens === undefined && totalTokens === undefined) {
return undefined
}

return {
promptTokens,
completionTokens,
totalTokens,
}
}
18 changes: 17 additions & 1 deletion src/main/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import * as path from 'path'
import { getActiveSettings, saveSettingsPatch, SettingsPatch } from './backend/settingsStore'

let mainWindow: BrowserWindow | null = null
let currentAnalysisController: AbortController | null = null

function createWindow() {
mainWindow = new BrowserWindow({
Expand Down Expand Up @@ -54,6 +55,9 @@ ipcMain.handle('get-settings', async () => {

ipcMain.handle('analyze-paper', async (_event, pdfPath: string) => {
if (!mainWindow) return { ok: false, error: { code: 'NO_WINDOW', message: 'Window not available' } }
currentAnalysisController?.abort()
const controller = new AbortController()
currentAnalysisController = controller

const sendProgress = (progress: { stage: string; message: string }) => {
mainWindow?.webContents.send('analysis-progress', progress)
Expand All @@ -64,7 +68,19 @@ ipcMain.handle('analyze-paper', async (_event, pdfPath: string) => {
}

const { analyzePaper } = await import('./backend/paperAnalyzer')
return analyzePaper(pdfPath, sendProgress, sendSummaryChunk)
try {
return await analyzePaper(pdfPath, sendProgress, sendSummaryChunk, controller.signal)
} finally {
if (currentAnalysisController === controller) {
currentAnalysisController = null
}
}
})

ipcMain.handle('cancel-analysis', async () => {
currentAnalysisController?.abort()
currentAnalysisController = null
return true
})

ipcMain.handle('download-core-code', async () => {
Expand Down
Loading