From e867f1c651e5fc4dab637ccc144623d3e7020bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Thu, 4 Dec 2025 08:32:52 +0100 Subject: [PATCH 1/5] feat: Enhance research capabilities with insights extraction and query classification - Updated Go module to use Go 1.24.0 and upgraded golang.org/x/sys dependency to v0.38.0. - Introduced structured insights extraction from search results in the SubResearcherAgent, capturing detailed findings and confidence scores. - Enhanced SupervisorAgent to accumulate insights from sub-researchers and include them in the session results. - Implemented a QueryClassifier to classify user intents (research, question, expand) based on natural language input, improving interaction with the REPL. - Added new handlers for question answering from existing research and expanded session management features. - Enhanced documentation to reflect new functionalities and usage instructions for insights and query classification. --- go-research/go.mod | 4 +- go-research/go.sum | 4 +- go-research/internal/agents/sub_researcher.go | 103 ++ go-research/internal/agents/supervisor.go | 8 + .../internal/architectures/interface.go | 8 +- .../architectures/think_deep/think_deep.go | 66 +- go-research/internal/config/config.go | 6 +- go-research/internal/e2e/e2e_test.go | 4 +- go-research/internal/obsidian/writer.go | 205 ++++ .../internal/orchestrator/orchestrator.go | 1 + .../internal/orchestrator/think_deep.go | 73 ++ go-research/internal/repl/classifier.go | 149 +++ go-research/internal/repl/classifier_test.go | 260 ++++++ go-research/internal/repl/dag_display.go | 65 +- go-research/internal/repl/dag_display_test.go | 8 +- .../internal/repl/handlers/architectures.go | 42 +- go-research/internal/repl/handlers/expand.go | 108 ++- .../internal/repl/handlers/handlers.go | 1 + .../internal/repl/handlers/question.go | 132 +++ .../internal/repl/handlers/question_test.go | 316 +++++++ go-research/internal/repl/handlers/session.go | 2 +- go-research/internal/repl/renderer.go | 12 +- go-research/internal/repl/repl.go | 7 + go-research/internal/repl/router.go | 43 +- go-research/internal/session/cost_test.go | 1 + go-research/internal/session/session.go | 5 +- go-research/internal/think_deep/injection.go | 58 ++ go-research/internal/think_deep/state.go | 57 ++ go-research/internal/think_deep/state_test.go | 269 ++++++ .../plans/interactive-cli-agentic-research.md | 878 ++++++++++++++++++ ...-12-03_interactive-cli-agentic-research.md | 595 ++++++++++++ .../2025-12-03_think-deep-data-tools.md | 483 ++++++++++ 32 files changed, 3886 insertions(+), 87 deletions(-) create mode 100644 go-research/internal/repl/classifier.go create mode 100644 go-research/internal/repl/classifier_test.go create mode 100644 go-research/internal/repl/handlers/question.go create mode 100644 go-research/internal/repl/handlers/question_test.go create mode 100644 go-research/internal/think_deep/injection.go create mode 100644 go-research/internal/think_deep/state_test.go create mode 100644 go-research/thoughts/shared/plans/interactive-cli-agentic-research.md create mode 100644 go-research/thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md create mode 100644 go-research/thoughts/shared/research/2025-12-03_think-deep-data-tools.md diff --git a/go-research/go.mod b/go-research/go.mod index 9cd5c5a..c111982 100644 --- a/go-research/go.mod +++ b/go-research/go.mod @@ -1,6 +1,6 @@ module go-research -go 1.22 +go 1.24.0 require ( github.com/chzyer/readline v1.5.1 @@ -14,5 +14,5 @@ require ( require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.38.0 // indirect ) diff --git a/go-research/go.sum b/go-research/go.sum index f7ca6a1..1d48a7d 100644 --- a/go-research/go.sum +++ b/go-research/go.sum @@ -20,8 +20,8 @@ golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go-research/internal/agents/sub_researcher.go b/go-research/internal/agents/sub_researcher.go index 83fbb38..5847dee 100644 --- a/go-research/internal/agents/sub_researcher.go +++ b/go-research/internal/agents/sub_researcher.go @@ -75,6 +75,9 @@ type SubResearcherResult struct { // VisitedURLs contains all URLs visited during this research session VisitedURLs []string + // Insights contains structured insights extracted from search results + Insights []think_deep.SubInsight + // Cost tracks token usage for this sub-researcher run Cost session.CostBreakdown } @@ -192,6 +195,9 @@ func (r *SubResearcherAgent) researchWithIteration(ctx context.Context, topic st // Count unique sources and extract visited URLs sourceCount, visitedURLs := countUniqueSourcesAndURLs(state.RawNotes) + // Extract structured insights from search results + insights := extractInsightsFromSearchResults(topic, state.RawNotes, researcherNum, diffusionIteration) + r.emitProgress(researcherNum, diffusionIteration, "complete", sourceCount, topic) return &SubResearcherResult{ @@ -199,6 +205,7 @@ func (r *SubResearcherAgent) researchWithIteration(ctx context.Context, topic st RawNotes: state.RawNotes, SourcesFound: sourceCount, VisitedURLs: visitedURLs, + Insights: insights, Cost: totalCost, }, nil } @@ -299,3 +306,99 @@ func truncateForLog(s string, maxLen int) string { } return s[:maxLen] + "..." } + +// extractInsightsFromSearchResults extracts structured insights from search results. +// It parses the raw search results and creates SubInsight structures for each +// distinct finding with source attribution. +func extractInsightsFromSearchResults(topic string, rawNotes []string, researcherNum int, iteration int) []think_deep.SubInsight { + var insights []think_deep.SubInsight + insightNum := 0 + + // URL extraction regex + urlRegex := regexp.MustCompile(`URL:\s*(https?://[^\s]+)`) + // Title extraction regex (looks for Title: or ## patterns) + titleRegex := regexp.MustCompile(`(?:Title:|##)\s*(.+?)(?:\n|$)`) + // Snippet/content extraction regex + snippetRegex := regexp.MustCompile(`(?:Snippet:|Content:)\s*(.+?)(?:\n\n|$)`) + + for _, note := range rawNotes { + // Extract URL from this note + urlMatches := urlRegex.FindStringSubmatch(note) + sourceURL := "" + if len(urlMatches) > 1 { + sourceURL = strings.TrimSpace(urlMatches[1]) + } + + // Extract title + titleMatches := titleRegex.FindStringSubmatch(note) + title := "" + if len(titleMatches) > 1 { + title = strings.TrimSpace(titleMatches[1]) + } + + // Extract snippet/content for the finding + snippetMatches := snippetRegex.FindStringSubmatch(note) + finding := "" + if len(snippetMatches) > 1 { + finding = strings.TrimSpace(snippetMatches[1]) + } + + // If we couldn't extract structured content, use truncated raw note + if finding == "" && len(note) > 0 { + finding = truncateForLog(note, 500) + } + + // Only create insight if we have meaningful content + if finding == "" { + continue + } + + insightNum++ + insight := think_deep.SubInsight{ + ID: fmt.Sprintf("insight-%03d", insightNum), + Topic: topic, + Title: title, + Finding: finding, + Implication: "", // Will be filled by synthesis later + SourceURL: sourceURL, + SourceContent: truncateForLog(note, 1000), + Confidence: calculateConfidence(sourceURL, finding), + Iteration: iteration, + ResearcherNum: researcherNum, + Timestamp: time.Now(), + } + + insights = append(insights, insight) + } + + return insights +} + +// calculateConfidence estimates confidence score based on source quality indicators. +func calculateConfidence(sourceURL, finding string) float64 { + confidence := 0.5 // Base confidence + + // Higher confidence for well-known domains + trustedDomains := []string{ + "wikipedia.org", "github.com", "arxiv.org", "nature.com", + "science.org", "ieee.org", "acm.org", "gov", "edu", + } + for _, domain := range trustedDomains { + if strings.Contains(sourceURL, domain) { + confidence += 0.2 + break + } + } + + // Higher confidence for longer, more detailed findings + if len(finding) > 200 { + confidence += 0.1 + } + + // Cap at 1.0 + if confidence > 1.0 { + confidence = 1.0 + } + + return confidence +} diff --git a/go-research/internal/agents/supervisor.go b/go-research/internal/agents/supervisor.go index 3de0ea2..ef4c5cf 100644 --- a/go-research/internal/agents/supervisor.go +++ b/go-research/internal/agents/supervisor.go @@ -79,6 +79,9 @@ type SupervisorResult struct { // IterationsUsed is the number of diffusion iterations completed IterationsUsed int + // SubInsights contains all structured insights captured during diffusion + SubInsights []think_deep.SubInsight + // Cost tracks token usage for the supervisor Cost session.CostBreakdown } @@ -195,6 +198,7 @@ func (s *SupervisorAgent) Coordinate( RawNotes: state.RawNotes, DraftReport: state.DraftReport, IterationsUsed: state.Iterations, + SubInsights: state.GetSubInsights(), Cost: totalCost, }, nil } @@ -294,6 +298,8 @@ func (s *SupervisorAgent) executeConductResearch( } // Track visited URLs for deduplication state.AddVisitedURLs(result.VisitedURLs) + // Accumulate insights from sub-researcher + state.AddSubInsights(result.Insights) // Track costs totalCost.Add(result.Cost) @@ -410,6 +416,8 @@ func (s *SupervisorAgent) executeParallelResearch( } // Track visited URLs for deduplication state.AddVisitedURLs(res.subResult.VisitedURLs) + // Accumulate insights from sub-researcher + state.AddSubInsights(res.subResult.Insights) totalCost.Add(res.subResult.Cost) } diff --git a/go-research/internal/architectures/interface.go b/go-research/internal/architectures/interface.go index 28343ba..05919bd 100644 --- a/go-research/internal/architectures/interface.go +++ b/go-research/internal/architectures/interface.go @@ -9,6 +9,7 @@ import ( "go-research/internal/agents" "go-research/internal/session" + "go-research/internal/think_deep" ) // Architecture defines the interface that all research architectures must implement. @@ -40,9 +41,10 @@ type Result struct { Summary string // Collected data - Facts []agents.Fact - Sources []string - Workers []WorkerResult + Facts []agents.Fact + Sources []string + Workers []WorkerResult + SubInsights []think_deep.SubInsight // Structured insights from ThinkDeep // Metrics for benchmarking Metrics Metrics diff --git a/go-research/internal/architectures/think_deep/think_deep.go b/go-research/internal/architectures/think_deep/think_deep.go index 78ae2bf..0bec984 100644 --- a/go-research/internal/architectures/think_deep/think_deep.go +++ b/go-research/internal/architectures/think_deep/think_deep.go @@ -50,33 +50,37 @@ import ( "go-research/internal/events" "go-research/internal/llm" "go-research/internal/orchestrator" + "go-research/internal/think_deep" "go-research/internal/tools" ) // Architecture implements the ThinkDepth Self-Balancing Test-Time Diffusion // research pattern using iterative refinement. type Architecture struct { - bus *events.Bus - config *config.Config - client llm.ChatClient - tools tools.ToolExecutor + bus *events.Bus + config *config.Config + client llm.ChatClient + tools tools.ToolExecutor + injectionContext *think_deep.InjectionContext } // Config holds configuration for the ThinkDeep architecture. type Config struct { - AppConfig *config.Config - Bus *events.Bus - Client llm.ChatClient // Optional: inject for testing - Tools tools.ToolExecutor // Optional: inject for testing + AppConfig *config.Config + Bus *events.Bus + Client llm.ChatClient // Optional: inject for testing + Tools tools.ToolExecutor // Optional: inject for testing + InjectionContext *think_deep.InjectionContext // Optional: prior context for expansion } // New creates a new ThinkDeep architecture instance. func New(cfg Config) *Architecture { return &Architecture{ - bus: cfg.Bus, - config: cfg.AppConfig, - client: cfg.Client, - tools: cfg.Tools, + bus: cfg.Bus, + config: cfg.AppConfig, + client: cfg.Client, + tools: cfg.Tools, + injectionContext: cfg.InjectionContext, } } @@ -95,6 +99,12 @@ func (a *Architecture) SupportsResume() bool { return false } +// SetInjectionContext sets the context for expansion workflows. +// This allows building upon existing research findings. +func (a *Architecture) SetInjectionContext(ctx *think_deep.InjectionContext) { + a.injectionContext = ctx +} + // Research executes the ThinkDeep research workflow: // 1. Brief generation - Transform query to detailed research brief // 2. Initial draft - Generate draft from model's knowledge @@ -111,6 +121,9 @@ func (a *Architecture) Research(ctx context.Context, sessionID string, query str if a.tools != nil { opts = append(opts, orchestrator.WithThinkDeepTools(a.tools)) } + if a.injectionContext != nil { + opts = append(opts, orchestrator.WithInjectionContext(a.injectionContext)) + } // Create the ThinkDeep orchestrator orch := orchestrator.NewThinkDeepOrchestrator(a.bus, a.config, opts...) @@ -141,11 +154,12 @@ func (a *Architecture) Resume(ctx context.Context, sessionID string) (*architect // convertResult transforms ThinkDeepResult into the standard architectures.Result format. func (a *Architecture) convertResult(sessionID string, tdr *orchestrator.ThinkDeepResult, startTime time.Time) *architectures.Result { result := &architectures.Result{ - SessionID: sessionID, - Query: tdr.Query, - Report: tdr.FinalReport, - Summary: tdr.ResearchBrief, - Status: "complete", + SessionID: sessionID, + Query: tdr.Query, + Report: tdr.FinalReport, + Summary: tdr.ResearchBrief, + SubInsights: tdr.SubInsights, + Status: "complete", Metrics: architectures.Metrics{ Duration: time.Since(startTime), Cost: tdr.Cost, @@ -153,10 +167,20 @@ func (a *Architecture) convertResult(sessionID string, tdr *orchestrator.ThinkDe }, } - // Extract sources from notes (compressed research findings) - // Each note may contain inline citations that we could parse - // For now, we track the iteration count as a proxy for research depth - result.Metrics.FactCount = len(tdr.Notes) + // Extract sources from SubInsights + sourceSet := make(map[string]bool) + for _, insight := range tdr.SubInsights { + if insight.SourceURL != "" { + sourceSet[insight.SourceURL] = true + } + } + for url := range sourceSet { + result.Sources = append(result.Sources, url) + } + + // Use insight count for quality metrics + result.Metrics.FactCount = len(tdr.SubInsights) + result.Metrics.SourceCount = len(result.Sources) return result } diff --git a/go-research/internal/config/config.go b/go-research/internal/config/config.go index 1a43a07..32f2774 100644 --- a/go-research/internal/config/config.go +++ b/go-research/internal/config/config.go @@ -30,7 +30,8 @@ type Config struct { MaxWorkers int // Model - Model string + Model string + ClassifierModel string // Fast model for query classification (e.g., claude-3-haiku) // Verbose mode Verbose bool @@ -59,7 +60,8 @@ func Load() *Config { MaxTokens: 50000, MaxWorkers: 5, - Model: "alibaba/tongyi-deepresearch-30b-a3b", + Model: "alibaba/tongyi-deepresearch-30b-a3b", + ClassifierModel: getEnvOrDefault("CLASSIFIER_MODEL", "anthropic/claude-3-haiku"), Verbose: os.Getenv("RESEARCH_VERBOSE") == "true", } diff --git a/go-research/internal/e2e/e2e_test.go b/go-research/internal/e2e/e2e_test.go index aa708ba..3e847dc 100644 --- a/go-research/internal/e2e/e2e_test.go +++ b/go-research/internal/e2e/e2e_test.go @@ -184,13 +184,13 @@ func TestRouterNaturalLanguageGoesToStormWithoutSession(t *testing.T) { ctx := &repl.Context{Store: store, Bus: bus, Config: cfg, Renderer: repl.NewRenderer(&bytes.Buffer{})} router := repl.NewRouter(ctx, handlers.RegisterAll()) - // Without a session, natural language should go to storm + // Without a session, natural language should go to think_deep (default research) handler, args, err := router.Route("What is the ReAct pattern?") if err != nil { t.Errorf("Unexpected error: %v", err) } if handler == nil { - t.Error("Expected storm handler for natural language without session") + t.Error("Expected think_deep handler for natural language without session") } if len(args) != 1 || args[0] != "What is the ReAct pattern?" { t.Errorf("Expected full text as single arg, got: %v", args) diff --git a/go-research/internal/obsidian/writer.go b/go-research/internal/obsidian/writer.go index 29d4ecf..28df950 100644 --- a/go-research/internal/obsidian/writer.go +++ b/go-research/internal/obsidian/writer.go @@ -9,6 +9,7 @@ import ( "time" "go-research/internal/session" + "go-research/internal/think_deep" "gopkg.in/yaml.v3" ) @@ -133,6 +134,127 @@ func (w *Writer) GetReportPath(sess *session.Session) string { return filepath.Join(w.vaultPath, sess.ID, "reports", fmt.Sprintf("report_v%d.md", sess.Version)) } +// WriteInsight writes a single insight to the insights directory. +func (w *Writer) WriteInsight(sessionDir string, insight think_deep.SubInsight, index int) error { + filename := filepath.Join(sessionDir, "insights", fmt.Sprintf("insight_%03d.md", index)) + + frontmatter := map[string]interface{}{ + "insight_id": insight.ID, + "topic": insight.Topic, + "confidence": fmt.Sprintf("%.2f", insight.Confidence), + "source_url": insight.SourceURL, + "iteration": insight.Iteration, + "researcher": insight.ResearcherNum, + "timestamp": insight.Timestamp.Format(time.RFC3339), + } + + fm, err := yaml.Marshal(frontmatter) + if err != nil { + return fmt.Errorf("marshal frontmatter: %w", err) + } + + var content bytes.Buffer + content.WriteString("---\n") + content.Write(fm) + content.WriteString("---\n\n") + + // Title + if insight.Title != "" { + content.WriteString(fmt.Sprintf("# %s\n\n", insight.Title)) + } else { + content.WriteString(fmt.Sprintf("# Insight %d: %s\n\n", index, truncateString(insight.Topic, 50))) + } + + // Finding + content.WriteString("## Finding\n\n") + content.WriteString(insight.Finding) + content.WriteString("\n\n") + + // Implication (if present) + if insight.Implication != "" { + content.WriteString("## Implication\n\n") + content.WriteString(insight.Implication) + content.WriteString("\n\n") + } + + // Source + content.WriteString("## Source\n\n") + if insight.SourceURL != "" { + content.WriteString(fmt.Sprintf("- [%s](%s)\n", insight.SourceURL, insight.SourceURL)) + } + if insight.SourceContent != "" { + content.WriteString("\n### Source Excerpt\n\n") + content.WriteString("> ") + content.WriteString(truncateString(insight.SourceContent, 500)) + content.WriteString("\n") + } + + return os.WriteFile(filename, content.Bytes(), 0644) +} + +// WriteInsights writes all insights for a session. +func (w *Writer) WriteInsights(sessionDir string, insights []think_deep.SubInsight) error { + for i, insight := range insights { + if err := w.WriteInsight(sessionDir, insight, i+1); err != nil { + return fmt.Errorf("write insight %d: %w", i+1, err) + } + } + return nil +} + +// WriteWithInsights writes a session with its sub-insights to the Obsidian vault. +func (w *Writer) WriteWithInsights(sess *session.Session, subInsights []think_deep.SubInsight) error { + // Create session directory structure + sessionDir := filepath.Join(w.vaultPath, sess.ID) + dirs := []string{ + sessionDir, + filepath.Join(sessionDir, "workers"), + filepath.Join(sessionDir, "insights"), + filepath.Join(sessionDir, "sources"), + filepath.Join(sessionDir, "reports"), + } + + for _, dir := range dirs { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("create dir %s: %w", dir, err) + } + } + + // Write worker files + for i, worker := range sess.Workers { + if err := w.writeWorker(sessionDir, i+1, worker); err != nil { + return fmt.Errorf("write worker %d: %w", i+1, err) + } + } + + // Write insight files + if len(subInsights) > 0 { + if err := w.WriteInsights(sessionDir, subInsights); err != nil { + return fmt.Errorf("write insights: %w", err) + } + } + + // Write report + if err := w.writeReport(sessionDir, sess); err != nil { + return fmt.Errorf("write report: %w", err) + } + + // Write session MOC with insights + if err := w.writeSessionMOCWithInsights(sessionDir, sess, subInsights); err != nil { + return fmt.Errorf("write session MOC: %w", err) + } + + return nil +} + +// truncateString truncates a string to maxLen characters +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "..." +} + func (w *Writer) writeSessionMOC(dir string, sess *session.Session) error { filename := filepath.Join(dir, "session.md") @@ -201,3 +323,86 @@ const sessionMOCTemplate = `--- - **Sources**: {{len .Sources}} - **Cost**: ${{printf "%.4f" .Cost.TotalCost}} ` + +// writeSessionMOCWithInsights writes a session MOC that includes links to insights +func (w *Writer) writeSessionMOCWithInsights(dir string, sess *session.Session, insights []think_deep.SubInsight) error { + filename := filepath.Join(dir, "session.md") + + frontmatter := map[string]interface{}{ + "session_id": sess.ID, + "version": sess.Version, + "query": sess.Query, + "complexity_score": sess.ComplexityScore, + "status": sess.Status, + "created_at": sess.CreatedAt.Format(time.RFC3339), + "updated_at": sess.UpdatedAt.Format(time.RFC3339), + "cost": sess.Cost.TotalCost, + "insights_count": len(insights), + } + + fm, err := yaml.Marshal(frontmatter) + if err != nil { + return fmt.Errorf("marshal frontmatter: %w", err) + } + + tmpl := template.Must(template.New("moc").Funcs(template.FuncMap{ + "inc": func(i int) int { return i + 1 }, + }).Parse(sessionMOCWithInsightsTemplate)) + + data := struct { + Frontmatter string + *session.Session + SubInsights []think_deep.SubInsight + }{string(fm), sess, insights} + + var content bytes.Buffer + if err := tmpl.Execute(&content, data); err != nil { + return fmt.Errorf("execute template: %w", err) + } + + return os.WriteFile(filename, content.Bytes(), 0644) +} + +const sessionMOCWithInsightsTemplate = `--- +{{.Frontmatter}}--- + +# Research Session: {{.Query}} + +## Workers + +{{range $i, $w := .Workers}} +- [[workers/worker_{{inc $i}}.md|Worker {{inc $i}}]]: {{$w.Objective}} +{{end}} + +## Reports + +- [[reports/report_v{{.Version}}.md|Report v{{.Version}}]] + +## Insights + +{{if .SubInsights}} +{{range $i, $ins := .SubInsights}} +- [[insights/insight_{{printf "%03d" (inc $i)}}.md|{{if $ins.Title}}{{$ins.Title}}{{else}}Insight {{inc $i}}{{end}}]] ({{$ins.Topic}}) +{{end}} +{{else}} +*No insights captured* +{{end}} + +## Sources + +{{if .Sources}} +{{range .Sources}} +- {{.}} +{{end}} +{{else}} +*No sources collected* +{{end}} + +## Stats + +- **Complexity**: {{printf "%.2f" .ComplexityScore}} +- **Workers**: {{len .Workers}} +- **Insights**: {{len .SubInsights}} +- **Sources**: {{len .Sources}} +- **Cost**: ${{printf "%.4f" .Cost.TotalCost}} +` diff --git a/go-research/internal/orchestrator/orchestrator.go b/go-research/internal/orchestrator/orchestrator.go index 2ad2442..f083773 100644 --- a/go-research/internal/orchestrator/orchestrator.go +++ b/go-research/internal/orchestrator/orchestrator.go @@ -125,6 +125,7 @@ func (o *Orchestrator) Research(ctx context.Context, query string) (*Result, err Data: events.PlanCreatedData{ WorkerCount: len(tasks), Complexity: complexity, + Topic: query, }, }) diff --git a/go-research/internal/orchestrator/think_deep.go b/go-research/internal/orchestrator/think_deep.go index 66502ee..f7c486e 100644 --- a/go-research/internal/orchestrator/think_deep.go +++ b/go-research/internal/orchestrator/think_deep.go @@ -32,6 +32,9 @@ type ThinkDeepOrchestrator struct { tools tools.ToolExecutor supervisor *agents.SupervisorAgent model string + + // Injection context for expansion workflows + injectionContext *think_deep.InjectionContext } // ThinkDeepConfig holds configuration for the ThinkDeep orchestrator. @@ -82,6 +85,14 @@ func WithThinkDeepTools(toolExec tools.ToolExecutor) ThinkDeepOption { } } +// WithInjectionContext provides prior context for expansion workflows. +// This enables the orchestrator to build upon existing research findings. +func WithInjectionContext(ctx *think_deep.InjectionContext) ThinkDeepOption { + return func(o *ThinkDeepOrchestrator) { + o.injectionContext = ctx + } +} + // NewThinkDeepOrchestrator creates a new ThinkDeep-style research orchestrator. func NewThinkDeepOrchestrator(bus *events.Bus, cfg *config.Config, opts ...ThinkDeepOption) *ThinkDeepOrchestrator { client := llm.NewClient(cfg) @@ -127,6 +138,9 @@ type ThinkDeepResult struct { // FinalReport is the fully optimized final output FinalReport string + // SubInsights contains all structured insights captured during diffusion + SubInsights []think_deep.SubInsight + // Cost tracks total token usage Cost session.CostBreakdown @@ -160,6 +174,11 @@ func (o *ThinkDeepOrchestrator) Research(ctx context.Context, query string) (*Th } totalCost.Add(briefCost) + // Apply injection context to enhance brief for expansion workflows + if o.injectionContext != nil { + researchBrief = o.enhanceBriefForExpansion(researchBrief) + } + o.emitPhaseProgress("brief", "Research brief generated") // Check for cancellation @@ -234,6 +253,7 @@ func (o *ThinkDeepOrchestrator) Research(ctx context.Context, query string) (*Th Notes: supervisorResult.Notes, DraftReport: supervisorResult.DraftReport, FinalReport: finalReport, + SubInsights: supervisorResult.SubInsights, Cost: totalCost, Duration: time.Since(startTime), }, nil @@ -416,6 +436,59 @@ func (o *ThinkDeepOrchestrator) emitPhaseProgress(phase, message string) { }) } +// enhanceBriefForExpansion modifies the research brief to focus on expansion. +// It adds context about existing research to guide sub-researchers toward new insights. +func (o *ThinkDeepOrchestrator) enhanceBriefForExpansion(brief string) string { + if o.injectionContext == nil { + return brief + } + + var enhanced strings.Builder + enhanced.WriteString(brief) + enhanced.WriteString("\n\n## Expansion Context\n\n") + + if o.injectionContext.ExpansionTopic != "" { + enhanced.WriteString(fmt.Sprintf("This is a follow-up research expanding on: %s\n\n", o.injectionContext.ExpansionTopic)) + } + + if len(o.injectionContext.PreviousFindings) > 0 { + enhanced.WriteString("### Known Findings (do not re-research these):\n") + limit := len(o.injectionContext.PreviousFindings) + if limit > 10 { + limit = 10 + } + for _, finding := range o.injectionContext.PreviousFindings[:limit] { + enhanced.WriteString(fmt.Sprintf("- %s\n", finding)) + } + enhanced.WriteString("\n") + } + + if len(o.injectionContext.KnownGaps) > 0 { + enhanced.WriteString("### Known Gaps (prioritize these):\n") + for _, gap := range o.injectionContext.KnownGaps { + enhanced.WriteString(fmt.Sprintf("- %s\n", gap)) + } + enhanced.WriteString("\n") + } + + if len(o.injectionContext.VisitedURLs) > 0 { + enhanced.WriteString(fmt.Sprintf("### Previously Visited Sources (%d URLs - avoid revisiting):\n", len(o.injectionContext.VisitedURLs))) + limit := len(o.injectionContext.VisitedURLs) + if limit > 5 { + limit = 5 + } + for _, url := range o.injectionContext.VisitedURLs[:limit] { + enhanced.WriteString(fmt.Sprintf("- %s\n", url)) + } + if len(o.injectionContext.VisitedURLs) > 5 { + enhanced.WriteString(fmt.Sprintf("- ... and %d more\n", len(o.injectionContext.VisitedURLs)-5)) + } + enhanced.WriteString("\n") + } + + return enhanced.String() +} + // deduplicateFindings removes notes with entirely redundant URLs. // A note is kept if it contains at least one URL not seen in previous notes, // or if it contains no URLs (general content that should be preserved). diff --git a/go-research/internal/repl/classifier.go b/go-research/internal/repl/classifier.go new file mode 100644 index 0000000..41375a0 --- /dev/null +++ b/go-research/internal/repl/classifier.go @@ -0,0 +1,149 @@ +package repl + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "go-research/internal/llm" +) + +// IntentType represents the classified type of user intent. +type IntentType string + +const ( + IntentResearch IntentType = "research" // New research query requiring web search + IntentQuestion IntentType = "question" // QA about existing reports + IntentExpand IntentType = "expand" // Expand on specific topic from existing research +) + +// QueryIntent represents the classified intent of user input. +type QueryIntent struct { + Type IntentType // Research, Question, Expand + Confidence float64 // 0-1 confidence score + Topic string // For Expand: specific topic to expand on +} + +// QueryClassifier classifies user queries using LLM. +type QueryClassifier struct { + client llm.ChatClient + model string +} + +// NewQueryClassifier creates a classifier with the specified model. +// Model is developer-configured, not user choice. +func NewQueryClassifier(client llm.ChatClient, model string) *QueryClassifier { + return &QueryClassifier{client: client, model: model} +} + +// Classify determines the intent of a user query given session context. +func (c *QueryClassifier) Classify(ctx context.Context, query string, hasSession bool, sessionSummary string) (*QueryIntent, error) { + // Store original model + originalModel := c.client.GetModel() + + // Switch to classifier model + c.client.SetModel(c.model) + defer c.client.SetModel(originalModel) + + prompt := c.buildClassificationPrompt(query, hasSession, sessionSummary) + + resp, err := c.client.Chat(ctx, []llm.Message{ + {Role: "user", Content: prompt}, + }) + if err != nil { + return nil, fmt.Errorf("classification request: %w", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("empty response from classifier") + } + + return c.parseResponse(resp.Choices[0].Message.Content) +} + +// buildClassificationPrompt creates the prompt for intent classification. +func (c *QueryClassifier) buildClassificationPrompt(query string, hasSession bool, sessionSummary string) string { + var contextInfo string + var biasGuidance string + + if hasSession && sessionSummary != "" { + contextInfo = fmt.Sprintf("The user has an active research session about: %s", sessionSummary) + biasGuidance = ` +IMPORTANT BIAS: Since there is existing research, STRONGLY prefer QUESTION over EXPAND or RESEARCH: +- If the query can possibly be answered from the existing research, classify as QUESTION +- Only use EXPAND if the user explicitly asks to "expand", "go deeper", "research more", or "find more about" +- Only use RESEARCH if the query is clearly about a completely different topic unrelated to the session +- When in doubt, choose QUESTION - the system will indicate if it can't answer from existing content` + } else if hasSession { + contextInfo = "The user has an active research session." + biasGuidance = ` +IMPORTANT BIAS: Since there is existing research, prefer QUESTION when the query relates to the session topic.` + } else { + contextInfo = "The user has no active research session." + biasGuidance = "" + } + + return fmt.Sprintf(`Classify the following user query into one of three categories: + +1. QUESTION - A question or query that can be answered from existing research reports (e.g., "What did you find?", "Summarize the findings", "What about X?", "How does Y work?", any follow-up question) +2. EXPAND - An explicit request to conduct MORE research on a specific topic (e.g., "Expand on X", "Research more about Y", "Go deeper on Z", "Find more information about...") +3. RESEARCH - A completely NEW research topic unrelated to any existing session (e.g., "Research a totally different subject") + +Context: %s +%s + +User query: "%s" + +IMPORTANT: Respond ONLY with valid JSON in this exact format: +{"intent": "QUESTION", "confidence": 0.9, "topic": ""} + +Rules: +- "intent" must be exactly "QUESTION", "EXPAND", or "RESEARCH" +- "confidence" must be a number between 0.0 and 1.0 +- "topic" should be the specific expansion topic for EXPAND intents, empty string otherwise +- No other text before or after the JSON +- Default to QUESTION when there's an active session and the query relates to it`, contextInfo, biasGuidance, query) +} + +// classificationResponse represents the expected JSON response from the LLM. +type classificationResponse struct { + Intent string `json:"intent"` + Confidence float64 `json:"confidence"` + Topic string `json:"topic"` +} + +// parseResponse extracts the intent from the LLM response. +func (c *QueryClassifier) parseResponse(content string) (*QueryIntent, error) { + content = strings.TrimSpace(content) + + // Try to extract JSON from the response (handle potential markdown code blocks) + jsonStart := strings.Index(content, "{") + jsonEnd := strings.LastIndex(content, "}") + if jsonStart >= 0 && jsonEnd > jsonStart { + content = content[jsonStart : jsonEnd+1] + } + + var resp classificationResponse + if err := json.Unmarshal([]byte(content), &resp); err != nil { + return nil, fmt.Errorf("parse classification response: %w (content: %s)", err, content) + } + + intent := &QueryIntent{ + Confidence: resp.Confidence, + Topic: resp.Topic, + } + + switch strings.ToUpper(resp.Intent) { + case "RESEARCH": + intent.Type = IntentResearch + case "QUESTION": + intent.Type = IntentQuestion + case "EXPAND": + intent.Type = IntentExpand + default: + return nil, fmt.Errorf("unknown intent type: %s", resp.Intent) + } + + return intent, nil +} diff --git a/go-research/internal/repl/classifier_test.go b/go-research/internal/repl/classifier_test.go new file mode 100644 index 0000000..b7cf66d --- /dev/null +++ b/go-research/internal/repl/classifier_test.go @@ -0,0 +1,260 @@ +package repl + +import ( + "context" + "testing" + + "go-research/internal/llm" +) + +// mockChatClient implements llm.ChatClient for testing +type mockChatClient struct { + response string + model string +} + +func (m *mockChatClient) Chat(ctx context.Context, messages []llm.Message) (*llm.ChatResponse, error) { + resp := &llm.ChatResponse{} + // ChatResponse.Choices is an anonymous struct slice, construct it properly + resp.Choices = append(resp.Choices, struct { + Message llm.Message `json:"message"` + }{Message: llm.Message{Content: m.response}}) + return resp, nil +} + +func (m *mockChatClient) StreamChat(ctx context.Context, messages []llm.Message, handler func(chunk string) error) error { + return handler(m.response) +} + +func (m *mockChatClient) GetModel() string { + return m.model +} + +func (m *mockChatClient) SetModel(model string) { + m.model = model +} + +func TestNewQueryClassifier(t *testing.T) { + client := &mockChatClient{model: "test-model"} + classifier := NewQueryClassifier(client, "haiku") + + if classifier == nil { + t.Fatal("NewQueryClassifier returned nil") + } + if classifier.model != "haiku" { + t.Errorf("model = %q, want %q", classifier.model, "haiku") + } +} + +func TestQueryClassifier_ParseResponse(t *testing.T) { + client := &mockChatClient{model: "test-model"} + classifier := NewQueryClassifier(client, "haiku") + + tests := []struct { + name string + response string + wantType IntentType + wantConf float64 + wantTopic string + wantErr bool + }{ + { + name: "research intent", + response: `{"intent": "RESEARCH", "confidence": 0.9, "topic": ""}`, + wantType: IntentResearch, + wantConf: 0.9, + wantTopic: "", + wantErr: false, + }, + { + name: "question intent", + response: `{"intent": "QUESTION", "confidence": 0.85, "topic": ""}`, + wantType: IntentQuestion, + wantConf: 0.85, + wantTopic: "", + wantErr: false, + }, + { + name: "expand intent with topic", + response: `{"intent": "EXPAND", "confidence": 0.95, "topic": "tax policies"}`, + wantType: IntentExpand, + wantConf: 0.95, + wantTopic: "tax policies", + wantErr: false, + }, + { + name: "lowercase intent", + response: `{"intent": "research", "confidence": 0.8, "topic": ""}`, + wantType: IntentResearch, + wantConf: 0.8, + wantTopic: "", + wantErr: false, + }, + { + name: "JSON in markdown code block", + response: "```json\n{\"intent\": \"QUESTION\", \"confidence\": 0.9, \"topic\": \"\"}\n```", + wantType: IntentQuestion, + wantConf: 0.9, + wantTopic: "", + wantErr: false, + }, + { + name: "invalid JSON", + response: "This is not JSON", + wantErr: true, + }, + { + name: "unknown intent type", + response: `{"intent": "UNKNOWN", "confidence": 0.9, "topic": ""}`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + intent, err := classifier.parseResponse(tt.response) + + if tt.wantErr { + if err == nil { + t.Error("expected error, got nil") + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if intent.Type != tt.wantType { + t.Errorf("Type = %q, want %q", intent.Type, tt.wantType) + } + if intent.Confidence != tt.wantConf { + t.Errorf("Confidence = %f, want %f", intent.Confidence, tt.wantConf) + } + if intent.Topic != tt.wantTopic { + t.Errorf("Topic = %q, want %q", intent.Topic, tt.wantTopic) + } + }) + } +} + +func TestQueryClassifier_BuildClassificationPrompt(t *testing.T) { + client := &mockChatClient{model: "test-model"} + classifier := NewQueryClassifier(client, "haiku") + + tests := []struct { + name string + query string + hasSession bool + sessionSummary string + wantContains []string + }{ + { + name: "with session and summary", + query: "What did you find about taxes?", + hasSession: true, + sessionSummary: "Swedish political parties", + wantContains: []string{ + "What did you find about taxes?", + "active research session about: Swedish political parties", + "QUESTION", + "EXPAND", + "RESEARCH", + "STRONGLY prefer QUESTION", + }, + }, + { + name: "with session no summary", + query: "Tell me more", + hasSession: true, + wantContains: []string{"active research session", "prefer QUESTION"}, + }, + { + name: "no session", + query: "Research AI trends", + hasSession: false, + wantContains: []string{"no active research session"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prompt := classifier.buildClassificationPrompt(tt.query, tt.hasSession, tt.sessionSummary) + + for _, want := range tt.wantContains { + if !contains(prompt, want) { + t.Errorf("prompt should contain %q", want) + } + } + }) + } +} + +func TestQueryClassifier_Classify(t *testing.T) { + tests := []struct { + name string + response string + wantType IntentType + }{ + { + name: "classify as research", + response: `{"intent": "RESEARCH", "confidence": 0.9, "topic": ""}`, + wantType: IntentResearch, + }, + { + name: "classify as question", + response: `{"intent": "QUESTION", "confidence": 0.85, "topic": ""}`, + wantType: IntentQuestion, + }, + { + name: "classify as expand", + response: `{"intent": "EXPAND", "confidence": 0.95, "topic": "details"}`, + wantType: IntentExpand, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := &mockChatClient{ + model: "test-model", + response: tt.response, + } + classifier := NewQueryClassifier(client, "haiku") + + intent, err := classifier.Classify(context.Background(), "test query", true, "test summary") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if intent.Type != tt.wantType { + t.Errorf("Type = %q, want %q", intent.Type, tt.wantType) + } + }) + } +} + +func TestIntentType_Constants(t *testing.T) { + // Verify intent type constants have expected values + if IntentResearch != "research" { + t.Errorf("IntentResearch = %q, want %q", IntentResearch, "research") + } + if IntentQuestion != "question" { + t.Errorf("IntentQuestion = %q, want %q", IntentQuestion, "question") + } + if IntentExpand != "expand" { + t.Errorf("IntentExpand = %q, want %q", IntentExpand, "expand") + } +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsHelper(s, substr)) +} + +func containsHelper(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/go-research/internal/repl/dag_display.go b/go-research/internal/repl/dag_display.go index 3a444c3..46a2363 100644 --- a/go-research/internal/repl/dag_display.go +++ b/go-research/internal/repl/dag_display.go @@ -32,23 +32,24 @@ func (d *DAGDisplay) Render(data events.PlanCreatedData) { func (d *DAGDisplay) renderHeader(topic string) { headerColor := color.New(color.FgHiCyan, color.Bold) - boxColor := color.New(color.FgCyan) topicColor := color.New(color.FgWhite) + dimColor := color.New(color.Faint) fmt.Fprintln(d.w) - boxColor.Fprintln(d.w, "╭──────────────────────────────────────────────────────────────────────────────╮") - headerColor.Fprintf(d.w, "│%s│\n", centerText("🔬 STORM RESEARCH PLAN", 78)) - boxColor.Fprintln(d.w, "│ │") - - // Truncate topic to fit - maxTopicLen := 68 - displayTopic := topic + headerColor.Fprintln(d.w, "🔬 STORM Research") + dimColor.Fprintln(d.w, strings.Repeat("─", 40)) + + // Display topic (truncate if needed) + maxTopicLen := 70 + displayTopic := strings.TrimSpace(topic) + if displayTopic == "" { + displayTopic = "(no topic)" + } if len(displayTopic) > maxTopicLen { displayTopic = displayTopic[:maxTopicLen-3] + "..." } - topicLine := fmt.Sprintf(" Topic: %s", displayTopic) - topicColor.Fprintf(d.w, "│%-78s│\n", topicLine) - boxColor.Fprintln(d.w, "╰──────────────────────────────────────────────────────────────────────────────╯") + dimColor.Fprint(d.w, "Topic: ") + topicColor.Fprintln(d.w, displayTopic) fmt.Fprintln(d.w) } @@ -324,46 +325,46 @@ func (d *DAGDisplay) renderPerspectives(perspectives []events.PerspectiveData) { return } - boxColor := color.New(color.FgCyan) headerColor := color.New(color.FgHiCyan, color.Bold) nameColor := color.New(color.FgHiYellow) - focusColor := color.New(color.FgWhite) + focusColor := color.New(color.FgWhite, color.Faint) dimColor := color.New(color.Faint) - boxColor.Fprintln(d.w, "╭──────────────────────────────────────────────────────────────────────────────╮") - headerColor.Fprintf(d.w, "│ %-76s│\n", "PERSPECTIVES (WikiWriter↔TopicExpert conversations):") - boxColor.Fprintln(d.w, "│ │") + // Simple header + headerColor.Fprintln(d.w, " Perspectives:") + fmt.Fprintln(d.w) for i, p := range perspectives { - // Truncate name and focus to fit + // Truncate name smartly (keep it readable) name := p.Name - if len(name) > 20 { - name = name[:17] + "..." + maxNameLen := 28 + if len(name) > maxNameLen { + name = name[:maxNameLen-3] + "..." } + // Truncate focus to fit on line focus := p.Focus - maxFocusLen := 48 + maxFocusLen := 60 if len(focus) > maxFocusLen { focus = focus[:maxFocusLen-3] + "..." } - // Format: " 1. Name ─ Focus..." - numStr := fmt.Sprintf("%d.", i+1) - dimColor.Fprint(d.w, "│ ") - dimColor.Fprintf(d.w, "%-3s", numStr) - nameColor.Fprintf(d.w, "%-20s", name) - dimColor.Fprint(d.w, " ─ ") - focusColor.Fprintf(d.w, "%-50s", focus) - dimColor.Fprintln(d.w, "│") + // Format: " 1. Name" + // " Focus description..." + dimColor.Fprintf(d.w, " %d. ", i+1) + nameColor.Fprintln(d.w, name) + fmt.Fprint(d.w, " ") + focusColor.Fprintln(d.w, focus) + if i < len(perspectives)-1 { + fmt.Fprintln(d.w) // Space between perspectives + } } - - boxColor.Fprintln(d.w, "╰──────────────────────────────────────────────────────────────────────────────╯") + fmt.Fprintln(d.w) } func (d *DAGDisplay) renderFooter() { dimColor := color.New(color.Faint) - fmt.Fprintln(d.w) - dimColor.Fprintln(d.w, centerText("Starting STORM conversations...", 80)) + dimColor.Fprintln(d.w, strings.Repeat("─", 40)) fmt.Fprintln(d.w) } diff --git a/go-research/internal/repl/dag_display_test.go b/go-research/internal/repl/dag_display_test.go index 3522ee6..a613408 100644 --- a/go-research/internal/repl/dag_display_test.go +++ b/go-research/internal/repl/dag_display_test.go @@ -46,8 +46,8 @@ func TestDAGDisplay_Render_ThreePerspectives(t *testing.T) { if len(output) == 0 { t.Error("expected non-empty output") } - if !bytes.Contains(buf.Bytes(), []byte("STORM RESEARCH PLAN")) { - t.Error("expected header to contain 'STORM RESEARCH PLAN'") + if !bytes.Contains(buf.Bytes(), []byte("STORM Research")) { + t.Error("expected header to contain 'STORM Research'") } if !bytes.Contains(buf.Bytes(), []byte("Conv 1")) { t.Error("expected output to contain 'Conv 1'") @@ -61,8 +61,8 @@ func TestDAGDisplay_Render_ThreePerspectives(t *testing.T) { if !bytes.Contains(buf.Bytes(), []byte("Academic Researcher")) { t.Error("expected output to contain 'Academic Researcher'") } - if !bytes.Contains(buf.Bytes(), []byte("PERSPECTIVES")) { - t.Error("expected output to contain 'PERSPECTIVES'") + if !bytes.Contains(buf.Bytes(), []byte("Perspectives:")) { + t.Error("expected output to contain 'Perspectives:'") } // Check for STORM phases if !bytes.Contains(buf.Bytes(), []byte("1. DISCOVER")) { diff --git a/go-research/internal/repl/handlers/architectures.go b/go-research/internal/repl/handlers/architectures.go index 0483695..53dabc6 100644 --- a/go-research/internal/repl/handlers/architectures.go +++ b/go-research/internal/repl/handlers/architectures.go @@ -62,7 +62,14 @@ func (h *ArchitectureCommandHandler) Execute(ctx *repl.Context, args []string) e } query := strings.Join(args, " ") - sess := session.New(query, session.ModeStorm) + + // Set mode based on architecture name + mode := session.ModeStorm + if h.definition.Name == "think_deep" { + mode = session.ModeThinkDeep + } + + sess := session.New(query, mode) ctx.Session = sess ctx.Renderer.ResearchStarting(sess.ID, fmt.Sprintf("%s architecture", h.definition.Name)) @@ -88,6 +95,21 @@ func (h *ArchitectureCommandHandler) Execute(ctx *repl.Context, args []string) e sess.Cost = result.Metrics.Cost sess.Status = session.StatusComplete + // Convert SubInsights to session Insights for persistence and QA access + if len(result.SubInsights) > 0 { + sess.Insights = make([]session.Insight, 0, len(result.SubInsights)) + for _, subIns := range result.SubInsights { + sess.Insights = append(sess.Insights, session.Insight{ + Title: subIns.Title, + Finding: subIns.Finding, + Implication: subIns.Implication, + Confidence: subIns.Confidence, + Sources: []string{subIns.SourceURL}, + WorkerID: fmt.Sprintf("sub-researcher-%d", subIns.ResearcherNum), + }) + } + } + if err := ctx.Store.Save(sess); err != nil { ctx.Renderer.Error(fmt.Errorf("save session: %w", err)) } @@ -100,11 +122,21 @@ func (h *ArchitectureCommandHandler) Execute(ctx *repl.Context, args []string) e var obsidianLink string var reportPath string - if err := ctx.Obsidian.Write(sess); err != nil { - ctx.Renderer.Error(fmt.Errorf("save to obsidian: %w", err)) + // Use WriteWithInsights if SubInsights are available, otherwise use standard Write + if len(result.SubInsights) > 0 { + if err := ctx.Obsidian.WriteWithInsights(sess, result.SubInsights); err != nil { + ctx.Renderer.Error(fmt.Errorf("save to obsidian: %w", err)) + } else { + reportPath = ctx.Obsidian.GetReportPath(sess) + obsidianLink = fmt.Sprintf("obsidian://open?path=%s", url.QueryEscape(reportPath)) + } } else { - reportPath = ctx.Obsidian.GetReportPath(sess) - obsidianLink = fmt.Sprintf("obsidian://open?path=%s", url.QueryEscape(reportPath)) + if err := ctx.Obsidian.Write(sess); err != nil { + ctx.Renderer.Error(fmt.Errorf("save to obsidian: %w", err)) + } else { + reportPath = ctx.Obsidian.GetReportPath(sess) + obsidianLink = fmt.Sprintf("obsidian://open?path=%s", url.QueryEscape(reportPath)) + } } ctx.Renderer.ResearchComplete( diff --git a/go-research/internal/repl/handlers/expand.go b/go-research/internal/repl/handlers/expand.go index 611b4f9..c869027 100644 --- a/go-research/internal/repl/handlers/expand.go +++ b/go-research/internal/repl/handlers/expand.go @@ -8,10 +8,12 @@ import ( "time" "go-research/internal/agent" + think_deep_arch "go-research/internal/architectures/think_deep" "go-research/internal/events" "go-research/internal/orchestrator" "go-research/internal/repl" "go-research/internal/session" + "go-research/internal/think_deep" ) // ExpandHandler handles /expand and natural language follow-ups @@ -48,9 +50,13 @@ Follow-up question: %s`, ctx.Session.Query, continuationCtx, followUp) // Run based on original mode var err error - if newSess.Mode == session.ModeFast { + switch newSess.Mode { + case session.ModeFast: err = h.runFast(ctx, expandedQuery, newSess) - } else { + case session.ModeThinkDeep: + err = h.runThinkDeep(ctx, expandedQuery, newSess) + default: + // Default to STORM for storm mode and legacy sessions err = h.runDeep(ctx, expandedQuery, newSess) } @@ -158,3 +164,101 @@ func (h *ExpandHandler) runDeep(ctx *repl.Context, query string, sess *session.S return nil } + +// runThinkDeep runs expansion using ThinkDeep architecture with injection context. +func (h *ExpandHandler) runThinkDeep(ctx *repl.Context, query string, sess *session.Session) error { + // Build injection context from parent session chain + injection := h.buildInjectionContext(ctx, query) + + // Start visualization + viz := repl.NewVisualizer(ctx) + viz.Start() + defer viz.Stop() + + // Create ThinkDeep architecture with injection context + arch := think_deep_arch.New(think_deep_arch.Config{ + AppConfig: ctx.Config, + Bus: ctx.Bus, + InjectionContext: injection, + }) + + // Use the cancelable context from REPL with timeout + runCtx, cancel := context.WithTimeout(ctx.RunContext, 30*time.Minute) + defer cancel() + + sess.Status = session.StatusRunning + result, err := arch.Research(runCtx, sess.ID, query) + + // Stop visualization + viz.Stop() + + if err != nil { + sess.Status = session.StatusFailed + return fmt.Errorf("research failed: %w", err) + } + + sess.Report = result.Report + sess.Sources = result.Sources + sess.Cost = result.Metrics.Cost + sess.Status = session.StatusExpanded + + // Convert SubInsights to session Insights for persistence and QA access + if len(result.SubInsights) > 0 { + sess.Insights = make([]session.Insight, 0, len(result.SubInsights)) + for _, subIns := range result.SubInsights { + sess.Insights = append(sess.Insights, session.Insight{ + Title: subIns.Title, + Finding: subIns.Finding, + Implication: subIns.Implication, + Confidence: subIns.Confidence, + Sources: []string{subIns.SourceURL}, + WorkerID: fmt.Sprintf("sub-researcher-%d", subIns.ResearcherNum), + }) + } + } + + ctx.Bus.Publish(events.Event{ + Type: events.EventResearchComplete, + Timestamp: time.Now(), + Data: sess, + }) + + return nil +} + +// buildInjectionContext creates an injection context from the session chain. +func (h *ExpandHandler) buildInjectionContext(ctx *repl.Context, expansionTopic string) *think_deep.InjectionContext { + injection := think_deep.NewInjectionContext() + injection.SetExpansionTopic(expansionTopic) + + // Walk session chain and accumulate context + current := ctx.Session + for current != nil { + // Accumulate findings from insights + for _, ins := range current.Insights { + injection.AddFinding(ins.Finding) + } + + // Accumulate sources as visited URLs + for _, src := range current.Sources { + injection.AddVisitedURL(src) + } + + // Keep existing report for context + if injection.ExistingReport == "" && current.Report != "" { + injection.SetExistingReport(current.Report) + } + + // Walk to parent + if current.ParentID == nil { + break + } + parent, err := ctx.Store.Load(*current.ParentID) + if err != nil { + break + } + current = parent + } + + return injection +} diff --git a/go-research/internal/repl/handlers/handlers.go b/go-research/internal/repl/handlers/handlers.go index 0bb143c..361723b 100644 --- a/go-research/internal/repl/handlers/handlers.go +++ b/go-research/internal/repl/handlers/handlers.go @@ -28,6 +28,7 @@ func RegisterAll() map[string]repl.Handler { add("fast", &FastHandler{}, "/fast ", "Quick single-worker research", repl.CommandCategoryAgents) add("expand", &ExpandHandler{}, "/expand ", "Expand on current research", repl.CommandCategoryWorkflow) + add("question", &QuestionHandler{}, "/question ", "Ask about existing research", repl.CommandCategoryWorkflow) add("sessions", &SessionsHandler{}, "/sessions", "List all sessions", repl.CommandCategorySessions) add("load", &LoadHandler{}, "/load ", "Load a previous session", repl.CommandCategorySessions) add("new", &NewHandler{}, "/new", "Clear session and start fresh", repl.CommandCategorySessions) diff --git a/go-research/internal/repl/handlers/question.go b/go-research/internal/repl/handlers/question.go new file mode 100644 index 0000000..f52a757 --- /dev/null +++ b/go-research/internal/repl/handlers/question.go @@ -0,0 +1,132 @@ +// Package handlers provides REPL command handlers. +package handlers + +import ( + "context" + "fmt" + "strings" + "time" + + "go-research/internal/llm" + "go-research/internal/repl" + "go-research/internal/session" +) + +// QuestionHandler answers questions from existing session content without performing new research. +type QuestionHandler struct{} + +// Execute runs the question-answering workflow against the current session. +func (h *QuestionHandler) Execute(ctx *repl.Context, args []string) error { + if ctx.Session == nil { + return fmt.Errorf("no active session. Start research first with /fast, /storm, or /think_deep") + } + + if len(args) == 0 { + return fmt.Errorf("usage: /question ") + } + + question := strings.Join(args, " ") + + // Build QA context from session chain + qaContext := h.buildQAContext(ctx) + + ctx.Renderer.Info("Searching existing research for answer...") + + // Call LLM to answer + answer, err := h.answerFromContext(ctx, question, qaContext) + if err != nil { + return fmt.Errorf("answer question: %w", err) + } + + // Render answer + ctx.Renderer.Answer(answer) + + return nil +} + +// buildQAContext builds context from all sessions in the chain. +func (h *QuestionHandler) buildQAContext(ctx *repl.Context) string { + var sessions []*session.Session + + // Walk up the session chain via ParentID + current := ctx.Session + for current != nil { + sessions = append([]*session.Session{current}, sessions...) // Prepend for chronological order + if current.ParentID == nil { + break + } + parent, err := ctx.Store.Load(*current.ParentID) + if err != nil { + break + } + current = parent + } + + // Build context string with all reports and insights + var contextBuilder strings.Builder + for _, sess := range sessions { + contextBuilder.WriteString(fmt.Sprintf("## Session: %s\n", sess.Query)) + contextBuilder.WriteString(fmt.Sprintf("Report:\n%s\n\n", truncateContext(sess.Report, 3000))) + if len(sess.Insights) > 0 { + contextBuilder.WriteString("Key insights:\n") + for _, ins := range sess.Insights { + contextBuilder.WriteString(fmt.Sprintf("- %s: %s\n", ins.Title, ins.Finding)) + } + contextBuilder.WriteString("\n") + } + if len(sess.Sources) > 0 { + contextBuilder.WriteString("Sources:\n") + limit := len(sess.Sources) + if limit > 10 { + limit = 10 + } + for _, src := range sess.Sources[:limit] { + contextBuilder.WriteString(fmt.Sprintf("- %s\n", src)) + } + contextBuilder.WriteString("\n") + } + contextBuilder.WriteString("---\n\n") + } + return contextBuilder.String() +} + +// answerFromContext uses LLM to answer the question from existing research context. +func (h *QuestionHandler) answerFromContext(ctx *repl.Context, question, qaContext string) (string, error) { + prompt := fmt.Sprintf(`You are a research assistant. Answer the following question using ONLY the provided research context. If the answer is not in the context, say "I don't have enough information to answer that based on the current research." + + +%s + + +Question: %s + +Answer concisely and cite which session/report the information came from when possible.`, qaContext, question) + + // Create LLM client + client := llm.NewClient(ctx.Config) + + // Use the cancelable context from REPL with timeout + runCtx, cancel := context.WithTimeout(ctx.RunContext, 2*time.Minute) + defer cancel() + + resp, err := client.Chat(runCtx, []llm.Message{ + {Role: "user", Content: prompt}, + }) + if err != nil { + return "", err + } + + if len(resp.Choices) == 0 { + return "", fmt.Errorf("empty response from LLM") + } + + return resp.Choices[0].Message.Content, nil +} + +// truncateContext truncates text to a maximum length for context building. +func truncateContext(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "...[truncated]" +} diff --git a/go-research/internal/repl/handlers/question_test.go b/go-research/internal/repl/handlers/question_test.go new file mode 100644 index 0000000..d7e4863 --- /dev/null +++ b/go-research/internal/repl/handlers/question_test.go @@ -0,0 +1,316 @@ +package handlers + +import ( + "bytes" + "os" + "testing" + "time" + + "go-research/internal/config" + "go-research/internal/repl" + "go-research/internal/session" +) + +func TestQuestionHandler_Execute_NoSession(t *testing.T) { + handler := &QuestionHandler{} + + ctx := &repl.Context{ + Session: nil, // No active session + } + + err := handler.Execute(ctx, []string{"What did you find?"}) + + if err == nil { + t.Fatal("expected error when no session") + } + if err.Error() != "no active session. Start research first with /fast, /storm, or /think_deep" { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestQuestionHandler_Execute_NoArgs(t *testing.T) { + handler := &QuestionHandler{} + + sess := session.New("test query", session.ModeFast) + ctx := &repl.Context{ + Session: sess, + } + + err := handler.Execute(ctx, []string{}) + + if err == nil { + t.Fatal("expected error when no args") + } + if err.Error() != "usage: /question " { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestQuestionHandler_BuildQAContext_SingleSession(t *testing.T) { + handler := &QuestionHandler{} + + sess := &session.Session{ + ID: "test-session-1", + Query: "Swedish politics", + Report: "This is a detailed report about Swedish politics.", + Sources: []string{"https://example.com/1", "https://example.com/2"}, + Insights: []session.Insight{ + {Title: "Tax Policy", Finding: "Sweden has high taxes"}, + {Title: "Healthcare", Finding: "Universal healthcare system"}, + }, + } + + ctx := &repl.Context{ + Session: sess, + } + + qaContext := handler.buildQAContext(ctx) + + // Verify session query is included + if !containsString(qaContext, "Swedish politics") { + t.Error("QA context should contain session query") + } + + // Verify report is included + if !containsString(qaContext, "detailed report about Swedish politics") { + t.Error("QA context should contain report content") + } + + // Verify insights are included + if !containsString(qaContext, "Tax Policy") { + t.Error("QA context should contain insight title") + } + if !containsString(qaContext, "Sweden has high taxes") { + t.Error("QA context should contain insight finding") + } + + // Verify sources are included + if !containsString(qaContext, "https://example.com/1") { + t.Error("QA context should contain sources") + } +} + +func TestQuestionHandler_BuildQAContext_SessionChain(t *testing.T) { + handler := &QuestionHandler{} + + // Create a temporary directory for the store + tmpDir, err := os.MkdirTemp("", "question-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + store, err := session.NewStore(tmpDir) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + // Create parent session + parentID := "parent-session" + parentSess := &session.Session{ + ID: parentID, + Query: "Initial research", + Report: "Parent report content", + } + if err := store.Save(parentSess); err != nil { + t.Fatalf("failed to save parent session: %v", err) + } + + childSess := &session.Session{ + ID: "child-session", + ParentID: &parentID, + Query: "Follow-up research", + Report: "Child report content", + } + + ctx := &repl.Context{ + Session: childSess, + Store: store, + } + + qaContext := handler.buildQAContext(ctx) + + // Verify both sessions are included + if !containsString(qaContext, "Initial research") { + t.Error("QA context should contain parent session query") + } + if !containsString(qaContext, "Follow-up research") { + t.Error("QA context should contain child session query") + } + if !containsString(qaContext, "Parent report content") { + t.Error("QA context should contain parent report") + } + if !containsString(qaContext, "Child report content") { + t.Error("QA context should contain child report") + } +} + +func TestQuestionHandler_BuildQAContext_TruncatesLongReport(t *testing.T) { + handler := &QuestionHandler{} + + // Create a very long report + longReport := make([]byte, 5000) + for i := range longReport { + longReport[i] = 'x' + } + + sess := &session.Session{ + ID: "test-session", + Query: "Test query", + Report: string(longReport), + } + + ctx := &repl.Context{ + Session: sess, + } + + qaContext := handler.buildQAContext(ctx) + + // Should be truncated to 3000 chars + truncation marker + if !containsString(qaContext, "...[truncated]") { + t.Error("Long report should be truncated") + } +} + +func TestQuestionHandler_BuildQAContext_LimitsSourceCount(t *testing.T) { + handler := &QuestionHandler{} + + // Create session with many sources + sources := make([]string, 20) + for i := range sources { + sources[i] = "https://example.com/" + string(rune('a'+i)) + } + + sess := &session.Session{ + ID: "test-session", + Query: "Test query", + Report: "Test report", + Sources: sources, + } + + ctx := &repl.Context{ + Session: sess, + } + + qaContext := handler.buildQAContext(ctx) + + // Should limit to 10 sources + // Count occurrences of "https://example.com/" + count := 0 + for i := 0; i < len(qaContext)-len("https://example.com/"); i++ { + if qaContext[i:i+len("https://example.com/")] == "https://example.com/" { + count++ + } + } + + if count > 10 { + t.Errorf("Should limit sources to 10, got %d", count) + } +} + +func TestTruncateContext(t *testing.T) { + tests := []struct { + name string + input string + maxLen int + want string + }{ + { + name: "short string unchanged", + input: "hello", + maxLen: 10, + want: "hello", + }, + { + name: "exact length unchanged", + input: "hello", + maxLen: 5, + want: "hello", + }, + { + name: "long string truncated", + input: "hello world", + maxLen: 5, + want: "hello...[truncated]", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := truncateContext(tt.input, tt.maxLen) + if got != tt.want { + t.Errorf("truncateContext(%q, %d) = %q, want %q", tt.input, tt.maxLen, got, tt.want) + } + }) + } +} + +func containsString(s, substr string) bool { + return bytes.Contains([]byte(s), []byte(substr)) +} + +// TestQuestionHandler integration test would require mocking LLM client +// which is tested separately via the full REPL tests + +func TestQuestionHandler_NewInstance(t *testing.T) { + handler := &QuestionHandler{} + if handler == nil { + t.Fatal("QuestionHandler should be instantiable") + } +} + +// Verify QuestionHandler implements Handler interface +func TestQuestionHandler_ImplementsHandler(t *testing.T) { + var _ repl.Handler = (*QuestionHandler)(nil) +} + +// Test renderer output +func TestRenderer_Answer(t *testing.T) { + var buf bytes.Buffer + renderer := repl.NewRenderer(&buf) + + renderer.Answer("This is a test answer from research.") + + output := buf.String() + if !containsString(output, "Answer:") { + t.Error("Output should contain 'Answer:' header") + } + if !containsString(output, "This is a test answer from research.") { + t.Error("Output should contain the answer text") + } +} + +// Test that config has ClassifierModel +func TestConfig_HasClassifierModel(t *testing.T) { + cfg := config.Load() + + if cfg.ClassifierModel == "" { + t.Error("Config should have ClassifierModel set") + } +} + +// Benchmark QA context building +func BenchmarkBuildQAContext(b *testing.B) { + handler := &QuestionHandler{} + + sess := &session.Session{ + ID: "test-session", + Query: "Test query", + Report: "This is a detailed report with lots of content.", + Sources: []string{"https://example.com/1", "https://example.com/2"}, + Insights: []session.Insight{ + {Title: "Insight 1", Finding: "Finding 1"}, + {Title: "Insight 2", Finding: "Finding 2"}, + }, + CreatedAt: time.Now(), + } + + ctx := &repl.Context{ + Session: sess, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + handler.buildQAContext(ctx) + } +} diff --git a/go-research/internal/repl/handlers/session.go b/go-research/internal/repl/handlers/session.go index 8a0872d..063f266 100644 --- a/go-research/internal/repl/handlers/session.go +++ b/go-research/internal/repl/handlers/session.go @@ -71,7 +71,7 @@ func (h *NewHandler) Execute(ctx *repl.Context, args []string) error { ctx.Session = sess ctx.Renderer.Info(fmt.Sprintf("✓ New session created: %s", sess.ID)) - ctx.Renderer.Info(" Ready. Type a question or use /storm to start.") + ctx.Renderer.Info(" Ready. Type a question to start research.") return nil } diff --git a/go-research/internal/repl/renderer.go b/go-research/internal/repl/renderer.go index 1fa23e9..9088a0e 100644 --- a/go-research/internal/repl/renderer.go +++ b/go-research/internal/repl/renderer.go @@ -188,7 +188,7 @@ func (r *Renderer) SessionRestored(sessionID string, query string, workerCount i // SessionCleared shows that the session was cleared func (r *Renderer) SessionCleared(oldSessionID string) { green.Fprintf(r.w, "✓ Session cleared: %s\n", oldSessionID) - cyan.Fprintln(r.w, " Ready for new research. Type a question or use /storm ") + cyan.Fprintln(r.w, " Ready for new research. Type a question to start.") } func truncate(s string, n int) string { @@ -463,3 +463,13 @@ type WorkerDisplay struct { Status string SourceCount int } + +// Answer displays an answer from the QA handler +func (r *Renderer) Answer(answer string) { + fmt.Fprintln(r.w) + bold.Fprintln(r.w, "Answer:") + fmt.Fprintln(r.w, strings.Repeat("─", 60)) + fmt.Fprintln(r.w, answer) + fmt.Fprintln(r.w, strings.Repeat("─", 60)) + fmt.Fprintln(r.w) +} diff --git a/go-research/internal/repl/repl.go b/go-research/internal/repl/repl.go index 8b7dcec..24412e1 100644 --- a/go-research/internal/repl/repl.go +++ b/go-research/internal/repl/repl.go @@ -11,6 +11,7 @@ import ( "github.com/chzyer/readline" "go-research/internal/config" "go-research/internal/events" + "go-research/internal/llm" "go-research/internal/obsidian" "go-research/internal/session" ) @@ -60,6 +61,12 @@ func New(store *session.Store, bus *events.Bus, cfg *config.Config, handlers map CommandDocs: docs, } + // Initialize classifier if configured + if cfg.ClassifierModel != "" { + client := llm.NewClient(cfg) + r.ctx.Classifier = NewQueryClassifier(client, cfg.ClassifierModel) + } + r.router = NewRouter(r.ctx, handlers) return r, nil diff --git a/go-research/internal/repl/router.go b/go-research/internal/repl/router.go index d7cc2ec..9c79015 100644 --- a/go-research/internal/repl/router.go +++ b/go-research/internal/repl/router.go @@ -26,6 +26,7 @@ type Context struct { CommandDocs []CommandDoc RunContext context.Context // Cancelable context for current operation Cancel context.CancelFunc // Cancel function to abort current operation + Classifier *QueryClassifier // LLM-based query classifier for intent routing } // Router routes input to appropriate handlers @@ -56,19 +57,45 @@ func (r *Router) Route(input string) (Handler, []string, error) { return r.routeCommand(parsed) } - // Natural language: if session exists, expand; otherwise start storm research - if r.ctx.Session != nil { + // Check if session has actual research content (not just a blank session) + hasResearchContent := r.ctx.Session != nil && r.ctx.Session.Report != "" + + // Natural language with session that has content - classify intent using LLM + if hasResearchContent && r.ctx.Classifier != nil { + sessionSummary := r.ctx.Session.Query + intent, err := r.ctx.Classifier.Classify(r.ctx.RunContext, parsed.RawText, true, sessionSummary) + if err == nil { + switch intent.Type { + case IntentQuestion: + if handler, ok := r.handlers["question"]; ok { + return handler, []string{parsed.RawText}, nil + } + case IntentExpand: + if handler, ok := r.handlers["expand"]; ok { + args := []string{parsed.RawText} + if intent.Topic != "" { + args = []string{intent.Topic} + } + return handler, args, nil + } + case IntentResearch: + // Use think_deep for new research (better quality than storm) + if handler, ok := r.handlers["think_deep"]; ok { + return handler, []string{parsed.RawText}, nil + } + } + } + // On classification error, fall through to default expand behavior for sessions with content handler, ok := r.handlers["expand"] - if !ok { - return nil, nil, fmt.Errorf("expand handler not registered") + if ok { + return handler, []string{parsed.RawText}, nil } - return handler, []string{parsed.RawText}, nil } - // No session - use storm as default research handler - handler, ok := r.handlers["storm"] + // No session OR empty session - use think_deep as default research handler + handler, ok := r.handlers["think_deep"] if !ok { - return nil, nil, fmt.Errorf("no default research handler registered (storm)") + return nil, nil, fmt.Errorf("no default research handler registered (think_deep)") } return handler, []string{parsed.RawText}, nil } diff --git a/go-research/internal/session/cost_test.go b/go-research/internal/session/cost_test.go index 28dba3f..6923b06 100644 --- a/go-research/internal/session/cost_test.go +++ b/go-research/internal/session/cost_test.go @@ -13,3 +13,4 @@ func TestNewCostBreakdown(t *testing.T) { t.Fatalf("expected total cost to be > 0, got %f", cost.TotalCost) } } + diff --git a/go-research/internal/session/session.go b/go-research/internal/session/session.go index 962a106..76770fd 100644 --- a/go-research/internal/session/session.go +++ b/go-research/internal/session/session.go @@ -13,8 +13,9 @@ import ( type Mode string const ( - ModeFast Mode = "fast" - ModeStorm Mode = "storm" + ModeFast Mode = "fast" + ModeStorm Mode = "storm" + ModeThinkDeep Mode = "think_deep" ) // SessionStatus represents the current state of a session diff --git a/go-research/internal/think_deep/injection.go b/go-research/internal/think_deep/injection.go new file mode 100644 index 0000000..1db9513 --- /dev/null +++ b/go-research/internal/think_deep/injection.go @@ -0,0 +1,58 @@ +package think_deep + +// InjectionContext provides prior knowledge for expansion workflows. +// This context is injected into the supervisor state before research begins. +type InjectionContext struct { + // Prior findings to preserve and build upon + PreviousFindings []string + ValidatedFacts []string + + // URLs already visited (for deduplication) + VisitedURLs []string + + // Existing report structure + ExistingReport string + ExistingOutline []string + + // Expansion focus + ExpansionTopic string + RelatedTopics []string + KnownGaps []string +} + +// NewInjectionContext creates an empty injection context. +func NewInjectionContext() *InjectionContext { + return &InjectionContext{ + PreviousFindings: make([]string, 0), + ValidatedFacts: make([]string, 0), + VisitedURLs: make([]string, 0), + ExistingOutline: make([]string, 0), + RelatedTopics: make([]string, 0), + KnownGaps: make([]string, 0), + } +} + +// AddFinding adds a previous finding to the context. +func (ic *InjectionContext) AddFinding(finding string) { + ic.PreviousFindings = append(ic.PreviousFindings, finding) +} + +// AddVisitedURL adds a visited URL to the context. +func (ic *InjectionContext) AddVisitedURL(url string) { + ic.VisitedURLs = append(ic.VisitedURLs, url) +} + +// AddKnownGap adds a known gap to prioritize. +func (ic *InjectionContext) AddKnownGap(gap string) { + ic.KnownGaps = append(ic.KnownGaps, gap) +} + +// SetExpansionTopic sets the topic to expand on. +func (ic *InjectionContext) SetExpansionTopic(topic string) { + ic.ExpansionTopic = topic +} + +// SetExistingReport sets the existing report to build upon. +func (ic *InjectionContext) SetExistingReport(report string) { + ic.ExistingReport = report +} diff --git a/go-research/internal/think_deep/state.go b/go-research/internal/think_deep/state.go index db117a0..4a88e87 100644 --- a/go-research/internal/think_deep/state.go +++ b/go-research/internal/think_deep/state.go @@ -8,10 +8,48 @@ package think_deep import ( "regexp" "strings" + "time" "go-research/internal/llm" ) +// SubInsight represents a single research finding extracted from search results. +// Granularity: one insight per search result with extracted findings. +type SubInsight struct { + // ID is a unique identifier (e.g., "insight-001") + ID string + + // Topic is the research topic from supervisor delegation + Topic string + + // Title is a short title summarizing the insight + Title string + + // Finding is the factual finding extracted + Finding string + + // Implication is what this means for the research question + Implication string + + // SourceURL is the URL this insight was extracted from + SourceURL string + + // SourceContent is the relevant excerpt from source (truncated) + SourceContent string + + // Confidence is a 0-1 confidence score + Confidence float64 + + // Iteration is the diffusion iteration when captured + Iteration int + + // ResearcherNum is which sub-researcher found this + ResearcherNum int + + // Timestamp is when the insight was captured + Timestamp time.Time +} + // SupervisorState manages the lead researcher's coordination of the diffusion // process. The supervisor orchestrates sub-researchers and maintains the // evolving draft report. @@ -42,6 +80,9 @@ type SupervisorState struct { // VisitedURLs tracks all URLs that have been searched/fetched to enable deduplication. // This prevents redundant fetching of the same source across sub-researchers. VisitedURLs map[string]bool + + // SubInsights contains all insights captured during diffusion from sub-researchers + SubInsights []SubInsight } // ResearcherState manages individual sub-researcher state during focused @@ -80,6 +121,7 @@ func NewSupervisorState(researchBrief string) *SupervisorState { DraftReport: "", Iterations: 0, VisitedURLs: make(map[string]bool), + SubInsights: make([]SubInsight, 0), } } @@ -155,6 +197,21 @@ func (s *SupervisorState) GetVisitedURLs() []string { return urls } +// AddSubInsight adds a sub-insight captured from sub-researcher results. +func (s *SupervisorState) AddSubInsight(insight SubInsight) { + s.SubInsights = append(s.SubInsights, insight) +} + +// AddSubInsights adds multiple sub-insights from sub-researcher results. +func (s *SupervisorState) AddSubInsights(insights []SubInsight) { + s.SubInsights = append(s.SubInsights, insights...) +} + +// GetSubInsights returns all captured sub-insights. +func (s *SupervisorState) GetSubInsights() []SubInsight { + return s.SubInsights +} + // AddRawNote adds a raw search result to the researcher state. func (r *ResearcherState) AddRawNote(note string) { r.RawNotes = append(r.RawNotes, note) diff --git a/go-research/internal/think_deep/state_test.go b/go-research/internal/think_deep/state_test.go new file mode 100644 index 0000000..9bceb82 --- /dev/null +++ b/go-research/internal/think_deep/state_test.go @@ -0,0 +1,269 @@ +package think_deep + +import ( + "testing" + "time" +) + +func TestNewSupervisorState(t *testing.T) { + brief := "Test research brief" + state := NewSupervisorState(brief) + + if state.ResearchBrief != brief { + t.Errorf("ResearchBrief = %q, want %q", state.ResearchBrief, brief) + } + if len(state.Messages) != 0 { + t.Errorf("Messages should be empty, got %d", len(state.Messages)) + } + if len(state.Notes) != 0 { + t.Errorf("Notes should be empty, got %d", len(state.Notes)) + } + if len(state.SubInsights) != 0 { + t.Errorf("SubInsights should be empty, got %d", len(state.SubInsights)) + } + if state.VisitedURLs == nil { + t.Error("VisitedURLs should be initialized") + } +} + +func TestSupervisorState_AddSubInsight(t *testing.T) { + state := NewSupervisorState("test brief") + + insight := SubInsight{ + ID: "insight-001", + Topic: "Test topic", + Title: "Test title", + Finding: "Test finding", + Implication: "Test implication", + SourceURL: "https://example.com", + SourceContent: "Test content", + Confidence: 0.8, + Iteration: 1, + ResearcherNum: 1, + Timestamp: time.Now(), + } + + state.AddSubInsight(insight) + + if len(state.SubInsights) != 1 { + t.Fatalf("SubInsights length = %d, want 1", len(state.SubInsights)) + } + if state.SubInsights[0].ID != "insight-001" { + t.Errorf("Insight ID = %q, want %q", state.SubInsights[0].ID, "insight-001") + } +} + +func TestSupervisorState_AddSubInsights(t *testing.T) { + state := NewSupervisorState("test brief") + + insights := []SubInsight{ + {ID: "insight-001", Finding: "Finding 1"}, + {ID: "insight-002", Finding: "Finding 2"}, + {ID: "insight-003", Finding: "Finding 3"}, + } + + state.AddSubInsights(insights) + + if len(state.SubInsights) != 3 { + t.Fatalf("SubInsights length = %d, want 3", len(state.SubInsights)) + } + + // Add more insights + state.AddSubInsights([]SubInsight{{ID: "insight-004", Finding: "Finding 4"}}) + + if len(state.SubInsights) != 4 { + t.Fatalf("SubInsights length = %d, want 4", len(state.SubInsights)) + } +} + +func TestSupervisorState_GetSubInsights(t *testing.T) { + state := NewSupervisorState("test brief") + + insights := []SubInsight{ + {ID: "insight-001"}, + {ID: "insight-002"}, + } + state.AddSubInsights(insights) + + got := state.GetSubInsights() + + if len(got) != 2 { + t.Fatalf("GetSubInsights length = %d, want 2", len(got)) + } + if got[0].ID != "insight-001" || got[1].ID != "insight-002" { + t.Error("GetSubInsights returned wrong insights") + } +} + +func TestSupervisorState_VisitedURLs(t *testing.T) { + state := NewSupervisorState("test brief") + + // Test AddVisitedURL + state.AddVisitedURL("https://example.com/1") + state.AddVisitedURL("https://example.com/2") + + if !state.IsURLVisited("https://example.com/1") { + t.Error("URL should be marked as visited") + } + if state.IsURLVisited("https://example.com/3") { + t.Error("URL should not be marked as visited") + } + + // Test AddVisitedURLs + state.AddVisitedURLs([]string{"https://example.com/3", "https://example.com/4"}) + + if !state.IsURLVisited("https://example.com/3") { + t.Error("URL should be marked as visited after AddVisitedURLs") + } + + // Test GetVisitedURLs + urls := state.GetVisitedURLs() + if len(urls) != 4 { + t.Errorf("GetVisitedURLs length = %d, want 4", len(urls)) + } +} + +func TestSupervisorState_Notes(t *testing.T) { + state := NewSupervisorState("test brief") + + state.AddNote("Note 1") + state.AddNote("Note 2") + state.AddRawNote("Raw note 1") + + if len(state.Notes) != 2 { + t.Errorf("Notes length = %d, want 2", len(state.Notes)) + } + if len(state.RawNotes) != 1 { + t.Errorf("RawNotes length = %d, want 1", len(state.RawNotes)) + } +} + +func TestSupervisorState_DraftReport(t *testing.T) { + state := NewSupervisorState("test brief") + + state.UpdateDraft("Initial draft") + if state.DraftReport != "Initial draft" { + t.Errorf("DraftReport = %q, want %q", state.DraftReport, "Initial draft") + } + + state.UpdateDraft("Updated draft") + if state.DraftReport != "Updated draft" { + t.Errorf("DraftReport = %q, want %q", state.DraftReport, "Updated draft") + } +} + +func TestSupervisorState_Iterations(t *testing.T) { + state := NewSupervisorState("test brief") + + if state.Iterations != 0 { + t.Errorf("Iterations = %d, want 0", state.Iterations) + } + + state.IncrementIteration() + state.IncrementIteration() + + if state.Iterations != 2 { + t.Errorf("Iterations = %d, want 2", state.Iterations) + } +} + +func TestNewResearcherState(t *testing.T) { + topic := "Test research topic" + state := NewResearcherState(topic) + + if state.ResearchTopic != topic { + t.Errorf("ResearchTopic = %q, want %q", state.ResearchTopic, topic) + } + if len(state.Messages) != 0 { + t.Errorf("Messages should be empty, got %d", len(state.Messages)) + } + if state.Iteration != 0 { + t.Errorf("Iteration = %d, want 0", state.Iteration) + } +} + +func TestResearcherState_VisitedURLs(t *testing.T) { + state := NewResearcherState("test topic") + + state.AddVisitedURL("https://example.com/1") + state.AddVisitedURL("https://example.com/2") + + urls := state.GetVisitedURLs() + if len(urls) != 2 { + t.Errorf("GetVisitedURLs length = %d, want 2", len(urls)) + } +} + +func TestExtractURLs(t *testing.T) { + tests := []struct { + name string + content string + want int + }{ + { + name: "single URL", + content: "Check out https://example.com for more info", + want: 1, + }, + { + name: "multiple URLs", + content: "See https://example.com and http://test.org for details", + want: 2, + }, + { + name: "duplicate URLs", + content: "https://example.com is great. Visit https://example.com again.", + want: 1, // Should deduplicate + }, + { + name: "URLs with trailing punctuation", + content: "Check https://example.com. Also see https://test.org!", + want: 2, + }, + { + name: "no URLs", + content: "This text has no URLs", + want: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + urls := ExtractURLs(tt.content) + if len(urls) != tt.want { + t.Errorf("ExtractURLs() returned %d URLs, want %d", len(urls), tt.want) + } + }) + } +} + +func TestSubInsight_Fields(t *testing.T) { + now := time.Now() + insight := SubInsight{ + ID: "insight-001", + Topic: "AI Research", + Title: "Key Finding", + Finding: "AI improves efficiency", + Implication: "Companies should adopt AI", + SourceURL: "https://example.com/article", + SourceContent: "Full article content here", + Confidence: 0.95, + Iteration: 2, + ResearcherNum: 3, + Timestamp: now, + } + + // Verify all fields are accessible and correct + if insight.ID != "insight-001" { + t.Errorf("ID = %q, want %q", insight.ID, "insight-001") + } + if insight.Confidence != 0.95 { + t.Errorf("Confidence = %f, want %f", insight.Confidence, 0.95) + } + if insight.ResearcherNum != 3 { + t.Errorf("ResearcherNum = %d, want %d", insight.ResearcherNum, 3) + } + if !insight.Timestamp.Equal(now) { + t.Errorf("Timestamp mismatch") + } +} diff --git a/go-research/thoughts/shared/plans/interactive-cli-agentic-research.md b/go-research/thoughts/shared/plans/interactive-cli-agentic-research.md new file mode 100644 index 0000000..6dff613 --- /dev/null +++ b/go-research/thoughts/shared/plans/interactive-cli-agentic-research.md @@ -0,0 +1,878 @@ +# Interactive CLI Agentic Research Experience - Implementation Plan + +## Overview + +Transform the go-research CLI into a Claude Code-style interactive experience where users can: +1. Ask questions about existing research (answered from chat history + reports) +2. Expand on specific topics with context injection +3. Have queries intelligently routed to the appropriate handler via LLM classification + +This plan implements the architecture from `thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md`. + +## Current State Analysis + +### Existing Infrastructure +- **Router** (`internal/repl/router.go:46-74`): Routes natural language → expand (if session) or storm (if no session) +- **ExpandHandler** (`internal/repl/handlers/expand.go:21-88`): Creates versioned sessions with continuation context +- **Session Context** (`internal/session/context.go:9-45`): Builds text summary from prior session (report, insights, sources) +- **SupervisorState** (`internal/think_deep/state.go:20-45`): Tracks Notes, RawNotes, VisitedURLs during diffusion +- **Obsidian Writer** (`internal/obsidian/writer.go:27-62`): Creates directories for insights/ but never populates them + +### Key Gaps +1. No sub-insight capture during ThinkDeep research +2. Insights directory created but empty - no insight files written +3. No question-answering from existing content +4. No intelligent query classification - only session-exists check +5. No context injection into ThinkDeep orchestrator for expansion + +## Desired End State + +After implementation: +1. ThinkDeep captures insights per search result and saves them to Obsidian +2. Users can ask questions and get answers from chat history + all reports in session chain +3. Natural language queries are classified by LLM into research/question/expand intents +4. Expansion injects prior context (findings, visited URLs, existing report) into supervisor state + +### Verification +- Run ThinkDeep research → insights/ directory populated with files +- Ask question about report → get answer without new research +- Type ambiguous query → LLM classifies intent correctly +- Expand on topic → new research uses prior context (fewer redundant searches) + +## What We're NOT Doing + +- **Explorer/lookup agent**: Questions only use existing content; separate agent for knowledge base search is out of scope +- **STORM injection points**: Focus on ThinkDeep only; STORM expansion deferred +- **Heuristic classification**: LLM-only classification as specified +- **User-configurable classifier model**: Model is developer-configured, not user choice +- **Research-notes directory**: Sub-researcher raw output not persisted (only compressed insights) + +## Implementation Approach + +Build bottom-up: capture insights first, then write them, then enable Q&A on them, then route intelligently, finally enable context injection for expansion. + +--- + +## Phase 1: Sub-Insight Capture in ThinkDeep + +### Overview + +Capture structured insights from each sub-researcher's search results during the ThinkDeep diffusion loop. Insights are extracted per search result and accumulated in supervisor state. + +### Changes Required + +#### 1. Add SubInsight Type + +**File**: `internal/think_deep/state.go` +**Changes**: Add SubInsight struct and tracking to SupervisorState + +```go +// SubInsight represents a single research finding extracted from search results. +// Granularity: one insight per search result with extracted findings. +type SubInsight struct { + ID string // Unique identifier (e.g., "insight-001") + Topic string // Research topic from supervisor delegation + Title string // Short title summarizing the insight + Finding string // The factual finding extracted + Implication string // What this means for the research question + SourceURL string // URL this insight was extracted from + SourceContent string // Relevant excerpt from source (truncated) + Confidence float64 // 0-1 confidence score + Iteration int // Diffusion iteration when captured + ResearcherNum int // Which sub-researcher found this + Timestamp time.Time // When captured +} + +// Add to SupervisorState struct: +// SubInsights []SubInsight // All insights captured during diffusion +``` + +Add methods: +```go +func (s *SupervisorState) AddSubInsight(insight SubInsight) +func (s *SupervisorState) GetSubInsights() []SubInsight +``` + +#### 2. Extract Insights from Search Results + +**File**: `internal/agents/sub_researcher.go` +**Changes**: After search tool execution, extract insights from results + +Add function to extract insights from search result content: +```go +// extractInsightsFromSearch parses search results and extracts structured insights. +// Called after each search tool execution with the raw search results. +func extractInsightsFromSearch(topic string, searchResult string, researcherNum int, iteration int) []SubInsight +``` + +This function should: +1. Parse the search result content +2. Extract key findings (facts, data points, claims) +3. Create SubInsight for each distinct finding +4. Assign confidence based on source quality indicators + +#### 3. Accumulate Insights in Supervisor + +**File**: `internal/agents/supervisor.go` +**Changes**: Collect insights from sub-researcher results + +In `executeParallelResearch()` around line 395-417, after processing each sub-researcher result: +```go +// After accumulating notes and raw notes, also accumulate insights +for _, insight := range result.Insights { + state.AddSubInsight(insight) +} +``` + +#### 4. Update SubResearcherResult + +**File**: `internal/agents/sub_researcher.go` +**Changes**: Add Insights field to result struct + +```go +type SubResearcherResult struct { + CompressedResearch string + RawNotes []string + SourcesFound int + VisitedURLs []string + Insights []think_deep.SubInsight // NEW: Insights extracted from searches + Cost session.CostBreakdown +} +``` + +#### 5. Return Insights from ThinkDeep Orchestrator + +**File**: `internal/orchestrator/think_deep.go` +**Changes**: Include insights in ThinkDeepResult + +```go +// In ThinkDeepResult struct, add: +SubInsights []think_deep.SubInsight +``` + +Pass through from supervisor result to orchestrator result. + +### Success Criteria + +#### Automated Verification +- [x] Build succeeds: `go build ./...` +- [x] Tests pass: `go test ./internal/think_deep/... ./internal/agents/...` +- [x] No linting errors: `golangci-lint run ./internal/think_deep/... ./internal/agents/...` + +#### Manual Verification +- [ ] Run ThinkDeep research and verify `ThinkDeepResult.SubInsights` is populated +- [ ] Each insight has valid Topic, Finding, SourceURL fields +- [ ] Insight count correlates with number of search results processed + +--- + +## Phase 2: Enhanced Obsidian Writer + +### Overview + +Write captured sub-insights to the Obsidian vault's insights/ directory using the existing template. Link insights from the session MOC. + +### Changes Required + +#### 1. Add WriteInsights Method + +**File**: `internal/obsidian/writer.go` +**Changes**: New method to write insight files + +```go +// WriteInsight writes a single insight to the insights directory. +func (w *Writer) WriteInsight(sessionDir string, insight think_deep.SubInsight, index int) error { + filename := filepath.Join(sessionDir, "insights", fmt.Sprintf("insight_%03d.md", index)) + + // Use frontmatter: insight_id, session_id, topic, confidence, source_url, timestamp + // Body: ## Finding, ## Implication, ## Source +} + +// WriteInsights writes all insights for a session. +func (w *Writer) WriteInsights(sessionDir string, insights []think_deep.SubInsight) error { + for i, insight := range insights { + if err := w.WriteInsight(sessionDir, insight, i+1); err != nil { + return err + } + } + return nil +} +``` + +#### 2. Update Write Method to Include Insights + +**File**: `internal/obsidian/writer.go` +**Changes**: Call WriteInsights in main Write method + +In `Write()` method after writing workers (around line 49): +```go +// Write insight files +if len(sess.Insights) > 0 { + // Convert session.Insight to think_deep.SubInsight or use directly + // For now, we need to pass SubInsights separately +} +``` + +**Alternative approach**: Add a new method `WriteWithInsights(sess, subInsights)` that the handler calls when insights are available. + +#### 3. Update Session MOC Template + +**File**: `internal/obsidian/writer.go` +**Changes**: Add Insights section to sessionMOCTemplate + +Add after the Sources section: +```go +## Insights + +{{if .SubInsights}} +{{range $i, $ins := .SubInsights}} +- [[insights/insight_{{printf "%03d" (inc $i)}}.md|{{$ins.Title}}]] ({{$ins.Topic}}) +{{end}} +{{else}} +*No insights captured* +{{end}} +``` + +#### 4. Wire Up in Handlers + +**File**: `internal/repl/handlers/expand.go` and similar handlers +**Changes**: Pass insights to Obsidian writer + +After research completion, call the extended write method: +```go +if err := ctx.Obsidian.WriteWithInsights(newSess, result.SubInsights); err != nil { + ctx.Renderer.Error(fmt.Errorf("save to obsidian: %w", err)) +} +``` + +### Success Criteria + +#### Automated Verification +- [x] Build succeeds: `go build ./...` +- [x] Tests pass: `go test ./internal/obsidian/...` +- [x] Existing Obsidian tests still pass (no regression) + +#### Manual Verification +- [ ] Run ThinkDeep research → insights/ directory contains insight_001.md, insight_002.md, etc. +- [ ] Each insight file has valid YAML frontmatter +- [ ] Session MOC links to all insight files +- [ ] Clicking insight links in Obsidian opens correct files + +--- + +## Phase 3: Question Handler + +### Overview + +Create a handler that answers questions using only the chat history and all reports in the session chain (no new research). Uses LLM to synthesize answer from existing content. + +### Changes Required + +#### 1. Create QuestionHandler + +**File**: `internal/repl/handlers/question.go` (NEW) +**Changes**: New handler for question-answering + +```go +package handlers + +// QuestionHandler answers questions from existing session content. +type QuestionHandler struct{} + +func (h *QuestionHandler) Execute(ctx *repl.Context, args []string) error { + if ctx.Session == nil { + return fmt.Errorf("no active session. Start research first") + } + + question := strings.Join(args, " ") + + // Build QA context from session chain + qaContext := h.buildQAContext(ctx) + + // Call LLM to answer + answer, err := h.answerFromContext(ctx, question, qaContext) + if err != nil { + return err + } + + // Render answer + ctx.Renderer.Answer(answer) + + return nil +} +``` + +#### 2. Build QA Context from Session Chain + +**File**: `internal/repl/handlers/question.go` +**Changes**: Traverse session chain and build context + +```go +// buildQAContext builds context from all sessions in the chain. +func (h *QuestionHandler) buildQAContext(ctx *repl.Context) string { + var sessions []*session.Session + + // Walk up the session chain via ParentID + current := ctx.Session + for current != nil { + sessions = append([]*session.Session{current}, sessions...) // Prepend for chronological order + if current.ParentID == nil { + break + } + parent, err := ctx.Store.Load(*current.ParentID) + if err != nil { + break + } + current = parent + } + + // Build context string with all reports and chat history + var ctx strings.Builder + for _, sess := range sessions { + ctx.WriteString(fmt.Sprintf("## Session: %s\n", sess.Query)) + ctx.WriteString(fmt.Sprintf("Report:\n%s\n\n", sess.Report)) + if len(sess.Insights) > 0 { + ctx.WriteString("Key insights:\n") + for _, ins := range sess.Insights { + ctx.WriteString(fmt.Sprintf("- %s: %s\n", ins.Title, ins.Finding)) + } + } + } + return ctx.String() +} +``` + +#### 3. Answer from Context via LLM + +**File**: `internal/repl/handlers/question.go` +**Changes**: LLM call to answer question + +```go +func (h *QuestionHandler) answerFromContext(ctx *repl.Context, question, qaContext string) (string, error) { + prompt := fmt.Sprintf(`You are a research assistant. Answer the following question using ONLY the provided research context. If the answer is not in the context, say "I don't have enough information to answer that based on the current research." + + +%s + + +Question: %s + +Answer concisely and cite which session/report the information came from.`, qaContext, question) + + // Use configured LLM client + resp, err := ctx.LLM.Chat(ctx.RunContext, []llm.Message{ + {Role: "user", Content: prompt}, + }) + // ... handle response +} +``` + +#### 4. Add Renderer Method for Answers + +**File**: `internal/repl/renderer.go` +**Changes**: Add Answer rendering method + +```go +func (r *Renderer) Answer(answer string) { + fmt.Println() + fmt.Println(answer) + fmt.Println() +} +``` + +#### 5. Register Handler + +**File**: `internal/repl/handlers/handlers.go` +**Changes**: Register question handler + +```go +add("question", &QuestionHandler{}, "/question ", "Ask a question about existing research", "Research") +``` + +### Success Criteria + +#### Automated Verification +- [ ] Build succeeds: `go build ./...` +- [ ] Tests pass: `go test ./internal/repl/handlers/...` +- [ ] No linting errors: `golangci-lint run ./internal/repl/handlers/...` + +#### Manual Verification +- [ ] Start research on a topic +- [ ] Run `/question What did the report say about X?` +- [ ] Get answer synthesized from report content +- [ ] Answer includes citation of which session it came from +- [ ] Asking about something not in report gets "I don't have enough information" response + +--- + +## Phase 4: Smart Chat Router with LLM Classification + +### Overview + +Replace the simple session-exists check with LLM-based query classification. The classifier determines if input is a research query, question about existing content, or expansion request. + +### Changes Required + +#### 1. Create QueryClassifier + +**File**: `internal/repl/classifier.go` (NEW) +**Changes**: LLM-based query classification + +```go +package repl + +// QueryIntent represents the classified intent of user input. +type QueryIntent struct { + Type IntentType // Research, Question, Expand + Confidence float64 // 0-1 confidence score + Topic string // For Expand: specific topic to expand on +} + +type IntentType string + +const ( + IntentResearch IntentType = "research" // New research query + IntentQuestion IntentType = "question" // QA about existing reports + IntentExpand IntentType = "expand" // Expand on specific topic +) + +// QueryClassifier classifies user queries using LLM. +type QueryClassifier struct { + client llm.ChatClient + model string // Configurable model for classification +} + +// NewQueryClassifier creates a classifier with the specified model. +// Model is developer-configured, not user choice. +func NewQueryClassifier(client llm.ChatClient, model string) *QueryClassifier { + return &QueryClassifier{client: client, model: model} +} + +// Classify determines the intent of a user query given session context. +func (c *QueryClassifier) Classify(ctx context.Context, query string, hasSession bool, sessionSummary string) (*QueryIntent, error) { + // Build classification prompt + prompt := c.buildClassificationPrompt(query, hasSession, sessionSummary) + + // Call LLM + resp, err := c.client.Chat(ctx, []llm.Message{ + {Role: "user", Content: prompt}, + }) + // ... parse structured response (JSON or tagged output) +} +``` + +#### 2. Classification Prompt + +**File**: `internal/repl/classifier.go` +**Changes**: Add classification prompt + +```go +func (c *QueryClassifier) buildClassificationPrompt(query, hasSession bool, sessionSummary string) string { + var contextInfo string + if hasSession { + contextInfo = fmt.Sprintf(`The user has an active research session about: %s`, sessionSummary) + } else { + contextInfo = "The user has no active research session." + } + + return fmt.Sprintf(`Classify the following user query into one of three categories: + +1. RESEARCH - A new research question requiring web search (e.g., "What are the best practices for X?", "Compare A vs B") +2. QUESTION - A question about existing research that can be answered from current reports (e.g., "What did you find about X?", "Summarize the findings on Y") +3. EXPAND - A request to expand or go deeper on a specific topic from existing research (e.g., "Tell me more about X", "Expand on the section about Y") + +Context: %s + +User query: "%s" + +Respond in JSON format: +{"intent": "RESEARCH|QUESTION|EXPAND", "confidence": 0.0-1.0, "topic": "specific topic if EXPAND"}`, contextInfo, query) +} +``` + +#### 3. Add Classifier to Context + +**File**: `internal/repl/router.go` +**Changes**: Add Classifier field to Context + +```go +type Context struct { + // ... existing fields ... + Classifier *QueryClassifier // LLM-based query classifier +} +``` + +#### 4. Update Router to Use Classifier + +**File**: `internal/repl/router.go` +**Changes**: Replace simple session check with classifier + +In `Route()` method, replace lines 59-73: +```go +// Natural language with session - classify intent +if r.ctx.Session != nil && r.ctx.Classifier != nil { + sessionSummary := r.ctx.Session.Query // Or more detailed summary + intent, err := r.ctx.Classifier.Classify(r.ctx.RunContext, parsed.RawText, true, sessionSummary) + if err != nil { + // Fallback to expand on classification error + handler, ok := r.handlers["expand"] + if ok { + return handler, []string{parsed.RawText}, nil + } + } + + switch intent.Type { + case IntentQuestion: + handler, ok := r.handlers["question"] + if ok { + return handler, []string{parsed.RawText}, nil + } + case IntentExpand: + handler, ok := r.handlers["expand"] + if ok { + args := []string{parsed.RawText} + if intent.Topic != "" { + args = []string{intent.Topic} + } + return handler, args, nil + } + case IntentResearch: + // Use storm for new research even with session + handler, ok := r.handlers["storm"] + if ok { + return handler, []string{parsed.RawText}, nil + } + } +} + +// Fallback to existing behavior +if r.ctx.Session != nil { + handler, ok := r.handlers["expand"] + // ... +} +``` + +#### 5. Initialize Classifier in Main + +**File**: `cmd/research/main.go` +**Changes**: Create and inject classifier + +```go +// Create classifier with configured model (developer choice, not user) +classifierModel := cfg.ClassifierModel // e.g., "claude-3-haiku-20240307" for fast/cheap classification +classifierClient := llm.NewClient(classifierModel, cfg.APIKey) +classifier := repl.NewQueryClassifier(classifierClient, classifierModel) + +// Add to context +ctx.Classifier = classifier +``` + +#### 6. Add ClassifierModel to Config + +**File**: `internal/config/config.go` +**Changes**: Add classifier model configuration + +```go +type Config struct { + // ... existing fields ... + ClassifierModel string // Model for query classification (e.g., "claude-3-haiku-20240307") +} +``` + +Default to a fast, cheap model like Haiku for classification. + +### Success Criteria + +#### Automated Verification +- [ ] Build succeeds: `go build ./...` +- [ ] Tests pass: `go test ./internal/repl/...` +- [ ] No linting errors: `golangci-lint run ./internal/repl/...` + +#### Manual Verification +- [ ] Start research on "Swedish political parties" +- [ ] Type "What did you find about Moderaterna?" → routes to question handler +- [ ] Type "Tell me more about tax policies" → routes to expand handler +- [ ] Type "What are the immigration policies in Germany?" → routes to research (new topic) +- [ ] Classification happens quickly (Haiku latency ~200-500ms) + +--- + +## Phase 5: Expand Knowledge Pipeline with Context Injection + +### Overview + +When expanding research, inject prior context (findings, visited URLs, existing report structure) into the ThinkDeep supervisor before the diffusion loop starts. This enables more efficient research that builds on existing knowledge. + +### Changes Required + +#### 1. Create InjectionContext Type + +**File**: `internal/think_deep/injection.go` (NEW) +**Changes**: Define context injection structure + +```go +package think_deep + +// InjectionContext provides prior knowledge for expansion workflows. +// This context is injected into the supervisor state before research begins. +type InjectionContext struct { + // Prior findings to preserve and build upon + PreviousFindings []string + ValidatedFacts []string + + // URLs already visited (for deduplication) + VisitedURLs []string + + // Existing report structure + ExistingReport string + ExistingOutline []string + + // Expansion focus + ExpansionTopic string + RelatedTopics []string + KnownGaps []string +} +``` + +#### 2. Add Functional Options to ThinkDeep + +**File**: `internal/orchestrator/think_deep.go` +**Changes**: Add option functions for injection + +```go +// ThinkDeepOption configures the ThinkDeep orchestrator. +type ThinkDeepOption func(*ThinkDeep) + +// WithInjectionContext provides prior context for expansion workflows. +func WithInjectionContext(ctx *think_deep.InjectionContext) ThinkDeepOption { + return func(td *ThinkDeep) { + td.injectionContext = ctx + } +} + +// WithVisitedURLs pre-populates visited URLs for deduplication. +func WithVisitedURLs(urls []string) ThinkDeepOption { + return func(td *ThinkDeep) { + td.preVisitedURLs = urls + } +} + +// WithExistingFindings provides prior research to build upon. +func WithExistingFindings(findings []string) ThinkDeepOption { + return func(td *ThinkDeep) { + td.existingFindings = findings + } +} + +// WithExpansionFocus narrows research to specific topic. +func WithExpansionFocus(topic string) ThinkDeepOption { + return func(td *ThinkDeep) { + td.expansionFocus = topic + } +} +``` + +#### 3. Apply Injection in Research Method + +**File**: `internal/orchestrator/think_deep.go` +**Changes**: Use injection context when initializing supervisor + +In `Research()` method, after creating supervisor state (around line 189-207): +```go +// Apply injection context if provided +if td.injectionContext != nil { + // Pre-populate visited URLs for deduplication + for _, url := range td.injectionContext.VisitedURLs { + state.AddVisitedURL(url) + } + + // Add existing findings as notes + for _, finding := range td.injectionContext.PreviousFindings { + state.AddNote(finding) + } + + // Modify research brief to focus on expansion + if td.injectionContext.ExpansionTopic != "" { + researchBrief = td.enhanceBriefForExpansion(researchBrief, td.injectionContext) + } +} +``` + +#### 4. Build Injection Context in ExpandHandler + +**File**: `internal/repl/handlers/expand.go` +**Changes**: Build and pass injection context + +```go +func (h *ExpandHandler) runDeep(ctx *repl.Context, query string, sess *session.Session) error { + // Build injection context from parent session + injection := h.buildInjectionContext(ctx) + + // Create orchestrator with injection options + orch := orchestrator.NewThinkDeep(ctx.Bus, ctx.Config, + orchestrator.WithInjectionContext(injection), + orchestrator.WithVisitedURLs(injection.VisitedURLs), + orchestrator.WithExpansionFocus(sess.Query), + ) + + // ... rest of method +} + +func (h *ExpandHandler) buildInjectionContext(ctx *repl.Context) *think_deep.InjectionContext { + // Walk session chain and accumulate context + var allFindings []string + var allURLs []string + var existingReport string + + current := ctx.Session + for current != nil { + // Accumulate findings from insights + for _, ins := range current.Insights { + allFindings = append(allFindings, ins.Finding) + } + // Accumulate sources as visited URLs + allURLs = append(allURLs, current.Sources...) + // Keep most recent report + if existingReport == "" { + existingReport = current.Report + } + + // Walk to parent + if current.ParentID == nil { + break + } + parent, err := ctx.Store.Load(*current.ParentID) + if err != nil { + break + } + current = parent + } + + return &think_deep.InjectionContext{ + PreviousFindings: allFindings, + VisitedURLs: allURLs, + ExistingReport: existingReport, + } +} +``` + +#### 5. Enhance Brief for Expansion + +**File**: `internal/orchestrator/think_deep.go` +**Changes**: Modify research brief to focus expansion + +```go +func (td *ThinkDeep) enhanceBriefForExpansion(brief string, injection *think_deep.InjectionContext) string { + var enhanced strings.Builder + + enhanced.WriteString(brief) + enhanced.WriteString("\n\n## Expansion Context\n\n") + enhanced.WriteString(fmt.Sprintf("This is a follow-up research expanding on: %s\n\n", injection.ExpansionTopic)) + + if len(injection.PreviousFindings) > 0 { + enhanced.WriteString("### Known Findings (do not re-research these):\n") + for _, finding := range injection.PreviousFindings[:min(10, len(injection.PreviousFindings))] { + enhanced.WriteString(fmt.Sprintf("- %s\n", finding)) + } + } + + if len(injection.KnownGaps) > 0 { + enhanced.WriteString("\n### Known Gaps (prioritize these):\n") + for _, gap := range injection.KnownGaps { + enhanced.WriteString(fmt.Sprintf("- %s\n", gap)) + } + } + + return enhanced.String() +} +``` + +### Success Criteria + +#### Automated Verification +- [ ] Build succeeds: `go build ./...` +- [ ] Tests pass: `go test ./internal/orchestrator/... ./internal/think_deep/...` +- [ ] No linting errors: `golangci-lint run ./internal/orchestrator/... ./internal/think_deep/...` + +#### Manual Verification +- [ ] Start research on "Swedish political parties economic policies" +- [ ] Note the number of sources and iterations +- [ ] Expand on "corporate tax proposals" +- [ ] Verify expansion: + - [ ] Uses fewer iterations (prior context reduces search space) + - [ ] Doesn't re-fetch URLs from parent session + - [ ] Builds on existing findings rather than starting fresh +- [ ] Final report includes both original and expanded content + +--- + +## Testing Strategy + +### Unit Tests + +| Phase | Test File | Key Tests | +|-------|-----------|-----------| +| 1 | `internal/think_deep/state_test.go` | SubInsight struct, Add/Get methods | +| 1 | `internal/agents/sub_researcher_test.go` | Insight extraction from search results | +| 2 | `internal/obsidian/writer_test.go` | WriteInsight, WriteInsights methods | +| 3 | `internal/repl/handlers/question_test.go` | QA context building, answer generation | +| 4 | `internal/repl/classifier_test.go` | Classification accuracy for each intent | +| 5 | `internal/think_deep/injection_test.go` | InjectionContext struct, builder functions | + +### Integration Tests + +| Phase | Test File | Key Tests | +|-------|-----------|-----------| +| 1-2 | `internal/e2e/think_deep_insights_test.go` | Full ThinkDeep run captures and persists insights | +| 3-4 | `internal/e2e/interactive_flow_test.go` | Research → Question → Expand flow | +| 5 | `internal/e2e/expansion_context_test.go` | Expansion uses prior context correctly | + +### Manual Testing Steps + +1. **Phase 1-2**: Run `/deep What are electric vehicle battery technologies?` + - Check `insights/` directory in Obsidian vault + - Verify insight files have valid content and frontmatter + +2. **Phase 3**: After research, run `/question What did you find about solid-state batteries?` + - Verify answer comes from report, not new research + - Verify "I don't have enough information" for unrelated questions + +3. **Phase 4**: With active session, type natural queries: + - "What about lithium mining?" → should classify as question + - "Expand on manufacturing challenges" → should classify as expand + - "Compare Tesla vs BYD battery strategies" → should classify as research + +4. **Phase 5**: Run `/expand solid-state battery commercialization` + - Verify fewer iterations than fresh research + - Verify no duplicate URL fetches + - Verify final report integrates prior findings + +--- + +## Performance Considerations + +- **Classification latency**: Using Haiku keeps classification under 500ms +- **Insight extraction**: Parse search results synchronously during sub-researcher execution (no additional LLM call) +- **Context building**: Walk session chain once, cache if needed +- **Injection overhead**: Minimal - just pre-populating maps and arrays + +--- + +## Migration Notes + +No migration needed - this is additive functionality: +- Existing sessions work unchanged +- New fields (SubInsights) default to empty +- Classification is opt-in via Classifier presence in context +- Injection only happens when options provided + +--- + +## References + +- Original research: `thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md` +- ThinkDeep state: `internal/think_deep/state.go:20-45` +- Obsidian writer: `internal/obsidian/writer.go:27-62` +- Router: `internal/repl/router.go:46-74` +- Expand handler: `internal/repl/handlers/expand.go:21-88` +- Supervisor agent: `internal/agents/supervisor.go:93-200` diff --git a/go-research/thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md b/go-research/thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md new file mode 100644 index 0000000..86ce925 --- /dev/null +++ b/go-research/thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md @@ -0,0 +1,595 @@ +--- +date: 2025-12-03T10:30:00+01:00 +researcher: Claude +git_commit: 6a32cb5cc41e10a32999f565d10ca639bbecc06c +branch: main +repository: addcommitpush.io/go-research +topic: "Interactive CLI Agentic Research Experience" +tags: [research, cli, interactive, obsidian, think_deep, storm, agents] +status: complete +last_updated: 2025-12-03 +last_updated_by: Claude +--- + +# Research: Interactive CLI Agentic Research Experience + +**Date**: 2025-12-03T10:30:00+01:00 +**Researcher**: Claude +**Git Commit**: 6a32cb5cc41e10a32999f565d10ca639bbecc06c +**Branch**: main +**Repository**: addcommitpush.io/go-research + +## Research Question + +Design an interactive CLI experience for deep research where: +1. Users can invoke different research modes (fast, storm, think_deep) +2. Sessions maintain context about written reports (outputs only, not full agent context) +3. Smart mode selection based on query complexity +4. Users can ask questions about reports and expand on specific topics +5. "Expand knowledge" workflow injects context at the right stage in each agent +6. All sub-insights from think_deep saved to Obsidian for full traceability + +## Summary + +The current go-research CLI has strong foundations for an interactive agentic experience. The architecture supports: +- Session management with versioning and parent tracking +- Event-driven visualization of research progress +- Existing `/expand` handler for follow-up queries +- Obsidian integration for persistence (though sub-insights not yet saved) + +The proposed "Claude Code-style" interactive experience requires: +1. A **Chat Router** that intelligently routes queries to appropriate agents +2. **Session Context Manager** that maintains report summaries without full agent state +3. **Expand Knowledge Pipeline** with injection points into each agent architecture +4. **Enhanced Obsidian Writer** that saves all sub-insights with provenance + +## Detailed Findings + +### 1. Current Architecture Analysis + +#### Entry Points (`cmd/research/main.go:51-52`) +```go +vaultWriter := obsidian.NewWriter(cfg.VaultPath) +store.SetVaultWriter(vaultWriter) +``` + +The CLI initializes with: +- Session store (filesystem-based JSON) +- Event bus for real-time visualization +- Obsidian vault writer for human-readable output +- REPL with command router + +#### Router Intelligence (`internal/repl/router.go:59-74`) +```go +// Natural language: if session exists, expand; otherwise start storm research +if r.ctx.Session != nil { + handler, ok := r.handlers["expand"] + return handler, []string{parsed.RawText}, nil +} +// No session - use storm as default research handler +handler, ok := r.handlers["storm"] +return handler, []string{parsed.RawText}, nil +``` + +**Current behavior**: +- Commands (`/fast`, `/deep`, `/expand`) → explicit routing +- Natural language WITH session → `/expand` handler +- Natural language WITHOUT session → `/storm` handler + +**Gap**: No smart mode selection or chat/QA detection. + +#### Expand Handler (`internal/repl/handlers/expand.go:32-55`) +```go +// Build continuation context from previous session +continuationCtx := session.BuildContinuationContext(ctx.Session) + +// Create expansion query with context +expandedQuery := fmt.Sprintf(`Based on previous research about "%s": +%s +Follow-up question: %s`, ctx.Session.Query, continuationCtx, followUp) + +// Create new version of session +newSess := ctx.Session.NewVersion() +``` + +**Current behavior**: +- Builds context from previous session (report + sources) +- Creates versioned session with parent link +- Runs research with injected context +- Saves to both JSON and Obsidian + +**Gap**: Uses same research mode as parent; doesn't inject at agent-specific points. + +### 2. Agent Architecture Injection Points + +#### ThinkDeep Injection Points (`internal/orchestrator/think_deep.go`) + +| Stage | Line | Injection Opportunity | +|-------|------|----------------------| +| Research Brief | 264 | Inject domain knowledge, previous findings | +| Initial Draft | 284 | Inject existing report as baseline | +| Supervisor Context | `supervisor.go:209` | Add `` section | +| Sub-Researcher | `sub_researcher.go:102` | Inject visited URLs, known facts | +| Final Report | 312 | Add style guidelines, structure template | + +**Key insight**: ThinkDeep's `SupervisorState` already tracks: +- `Notes []string` - compressed findings +- `RawNotes []string` - raw search results +- `VisitedURLs map[string]bool` - deduplication + +These can be pre-populated for "expand" workflows. + +#### STORM Injection Points (`internal/orchestrator/deep_storm.go`) + +| Stage | Line | Injection Opportunity | +|-------|------|----------------------| +| Perspective Discovery | 124 | Inject known perspectives, skip survey | +| Conversation Simulation | 159 | Inject previous conversations as context | +| Cross-Validation | 192 | Inject validated facts from previous run | +| Synthesis | 230 | Inject previous outline, sections | + +**Key insight**: STORM produces rich intermediate artifacts: +- `[]Perspective` - expert viewpoints +- `map[string]*ConversationResult` - full dialogue transcripts +- `*AnalysisResult` - validated facts, contradictions, gaps +- Draft/refined outlines + +### 3. Proposed Architecture + +#### 3.1 Chat Router with Smart Mode Selection + +```go +// internal/repl/chat_router.go + +type ChatRouter struct { + classifier QueryClassifier + session *session.Session + handlers map[string]Handler +} + +type QueryClassifier interface { + Classify(query string, sessionContext *SessionContext) QueryIntent +} + +type QueryIntent struct { + Type IntentType // Research, Question, Expand, Command + Mode Mode // Fast, Storm, ThinkDeep (for Research) + Complexity float64 // 0-1 scale + TopicFocus string // Specific topic for expansion + Confidence float64 +} + +type IntentType string +const ( + IntentResearch IntentType = "research" // New research query + IntentQuestion IntentType = "question" // QA about existing report + IntentExpand IntentType = "expand" // Expand on specific topic + IntentCommand IntentType = "command" // System command +) +``` + +**Classification Logic**: +1. No session → `IntentResearch` +2. Question about report content → `IntentQuestion` +3. "Expand on X", "Tell me more about Y" → `IntentExpand` +4. Complex new topic → `IntentResearch` (with `ModeThinkDeep`) +5. Simple fact query → `IntentResearch` (with `ModeFast`) + +#### 3.2 Session Context Manager + +```go +// internal/session/context_manager.go + +type SessionContext struct { + // Summarized outputs (not full agent state) + Query string + ReportSummary string // Compressed version of report + KeyFindings []Finding // Top 10 extracted findings + TopicsCovered []string // Main topics from report + SourcesSummary []SourceMeta // URLs with relevance scores + + // For expansion workflows + UnexploredGaps []string // Topics mentioned but not covered + VisitedURLs []string // For deduplication + + // Lineage tracking + SessionChain []string // Parent session IDs + Version int +} + +type Finding struct { + Topic string + Content string + Sources []string + Confidence float64 +} + +type SourceMeta struct { + URL string + Relevance float64 + CitedCount int +} +``` + +**Key design**: Context is extracted from reports, not stored agent state. This keeps memory bounded while preserving useful information. + +#### 3.3 Question Answering Handler + +```go +// internal/repl/handlers/question.go + +type QuestionHandler struct{} + +func (h *QuestionHandler) Execute(ctx *repl.Context, args []string) error { + question := strings.Join(args, " ") + + // Build QA context from session + qaContext := buildQAContext(ctx.Session) + + // Use LLM to answer from report content + answer, err := answerFromReport(ctx, question, qaContext) + if err != nil { + return err + } + + // Check if answer is sufficient or needs expansion + if answer.NeedsExpansion { + ctx.Renderer.SuggestExpansion(answer.SuggestedTopic) + } + + ctx.Renderer.Answer(answer) + return nil +} + +func buildQAContext(sess *session.Session) string { + return fmt.Sprintf(` +%s + + + +%s + + + +%s +`, sess.Report, formatSources(sess.Sources), formatInsights(sess.Insights)) +} +``` + +#### 3.4 Expand Knowledge Pipeline + +```go +// internal/repl/handlers/expand_knowledge.go + +type ExpandKnowledgeHandler struct{} + +func (h *ExpandKnowledgeHandler) Execute(ctx *repl.Context, args []string) error { + expandTopic := strings.Join(args, " ") + + // Determine best architecture for expansion + arch := selectArchitecture(ctx.Session, expandTopic) + + // Build injection context + injection := buildInjectionContext(ctx.Session, expandTopic) + + switch arch { + case "think_deep": + return h.expandWithThinkDeep(ctx, expandTopic, injection) + case "storm": + return h.expandWithStorm(ctx, expandTopic, injection) + case "fast": + return h.expandWithFast(ctx, expandTopic, injection) + } + return nil +} + +type InjectionContext struct { + // Existing knowledge to preserve + PreviousFindings []string + ValidatedFacts []ValidatedFact + VisitedURLs []string + + // Focus guidance + ExpansionTopic string + RelatedTopics []string + KnownGaps []string + + // Report continuity + ExistingReport string + ExistingOutline []string + ExistingSections map[string]string +} +``` + +**ThinkDeep Expansion Flow**: +```go +func (h *ExpandKnowledgeHandler) expandWithThinkDeep( + ctx *repl.Context, + topic string, + injection InjectionContext, +) error { + // 1. Create orchestrator with expansion options + orch := orchestrator.NewThinkDeep(ctx.Bus, ctx.Config, + orchestrator.WithExistingBrief(injection.ExistingReport), + orchestrator.WithPreviousNotes(injection.PreviousFindings), + orchestrator.WithVisitedURLs(injection.VisitedURLs), + orchestrator.WithExpansionFocus(topic), + ) + + // 2. Run research (supervisor will see existing context) + result, err := orch.Research(ctx.RunContext, expandQuery) + + // 3. Merge results with existing session + mergedSession := mergeExpansion(ctx.Session, result, topic) + + // 4. Save with full sub-insights + if err := ctx.Obsidian.WriteWithInsights(mergedSession, result.SubInsights); err != nil { + ctx.Renderer.Error(err) + } + + return nil +} +``` + +### 4. Enhanced Obsidian Integration + +#### 4.1 Sub-Insight Storage + +Current gap: `internal/obsidian/writer.go` creates `insights/` directory but never populates it. + +**Proposed structure**: +``` +/ +└── / + ├── session.md # MOC + ├── workers/ + │ └── worker_N.md + ├── insights/ # NEW: Populated with sub-insights + │ ├── insight_001.md # Individual insights + │ ├── insight_002.md + │ └── ... + ├── research-notes/ # NEW: Raw sub-researcher output + │ ├── iteration_01/ + │ │ ├── topic_1.md # Research topic + findings + │ │ ├── topic_2.md + │ │ └── sources.md # URLs with context + │ └── iteration_02/ + │ └── ... + ├── sources/ # NEW: Source metadata + │ └── source_index.md # All sources with quality scores + └── reports/ + └── report_vN.md +``` + +#### 4.2 Insight Template + +```go +// internal/obsidian/templates.go + +const insightTemplate = `--- +insight_id: {{.ID}} +session_id: {{.SessionID}} +iteration: {{.Iteration}} +topic: {{.Topic}} +confidence: {{printf "%.2f" .Confidence}} +created_at: {{.CreatedAt}} +sources: {{len .Sources}} +--- + +# {{.Title}} + +## Finding + +{{.Finding}} + +## Implication + +{{.Implication}} + +## Sources + +{{range .Sources}} +- {{.}} +{{end}} + +## Related Insights + +{{range .RelatedInsights}} +- [[insights/{{.}}.md|{{.}}]] +{{end}} +` +``` + +#### 4.3 ThinkDeep Sub-Insight Capture + +```go +// internal/agents/supervisor.go + +// Modify executeParallelResearch to capture sub-insights +func (s *SupervisorAgent) executeParallelResearch( + ctx context.Context, + calls []ToolCall, +) ([]SubResearcherResult, error) { + // ... existing parallel execution ... + + for result := range resultCh { + // Extract insights from each sub-researcher + insights := extractInsights(result.CompressedResearch, result.Topic) + + // Add to supervisor state for tracking + for _, insight := range insights { + s.state.AddInsight(SubInsight{ + Topic: result.Topic, + Finding: insight.Finding, + Sources: result.VisitedURLs, + Iteration: s.state.Iterations, + ResearcherNum: result.ResearcherNum, + }) + } + } +} + +// New state field in SupervisorState +type SupervisorState struct { + // ... existing fields ... + SubInsights []SubInsight // NEW: Track all sub-insights +} + +type SubInsight struct { + Topic string + Finding string + Sources []string + Iteration int + ResearcherNum int + Confidence float64 +} +``` + +### 5. User Flow Examples + +#### Example 1: Initial Research + Question + Expansion + +``` +research> What are the economic policies of Swedish political parties? + +[ThinkDeep runs - 15 iterations, 45 sub-researchers] +[Saves: session.md, workers/, insights/, reports/report_v1.md] + +Research complete! Report saved to Obsidian. +Cost: $0.45 | Sources: 127 | Insights: 34 + +research> What does Moderaterna specifically propose for corporate taxes? + +[QA handler - answers from report content] + +Based on the report: Moderaterna proposes reducing corporate tax from +20.6% to 18%, with additional deductions for R&D investments... + +Sources: [3], [17], [23] + +💡 This topic could be expanded. Say "expand corporate tax policies" + for deeper research. + +research> expand corporate tax policies + +[ExpandKnowledge handler - ThinkDeep with injection] +- Existing findings injected into supervisor context +- 127 URLs marked as visited +- Focus narrowed to "corporate tax policies" + +[ThinkDeep runs - 8 iterations (fewer due to existing knowledge)] +[Merges with session v1 → creates session v2] +[Updates: report_v2.md, new insights/] + +Expansion complete! Report updated. +Cost: $0.18 | New sources: 42 | New insights: 12 +Total sources: 169 | Total insights: 46 +``` + +#### Example 2: Direct Agent Invocation + +``` +research> /think_deep What is the ReAct agent pattern? + +[Direct invocation - full ThinkDeep workflow] + +research> /storm Comparison of agent architectures + +[Direct invocation - full STORM workflow] + +research> /fast Who invented transformers? + +[Direct invocation - fast single-agent] +``` + +### 6. Implementation Roadmap + +#### Phase 1: Sub-Insight Capture (think_deep) +- [ ] Add `SubInsights []SubInsight` to `SupervisorState` +- [ ] Capture insights in `executeParallelResearch` +- [ ] Return insights in `ThinkDeepResult` +- [ ] Update Obsidian writer to save insights + +#### Phase 2: Session Context Manager +- [ ] Create `SessionContext` struct +- [ ] Implement `ExtractContext(session) SessionContext` +- [ ] Add context to session store + +#### Phase 3: Question Handler +- [ ] Create `QuestionHandler` +- [ ] Implement `buildQAContext` +- [ ] Add expansion suggestion logic + +#### Phase 4: Smart Router +- [ ] Create `QueryClassifier` interface +- [ ] Implement LLM-based classifier +- [ ] Update `Router` to use classifier + +#### Phase 5: Expand Knowledge Pipeline +- [ ] Create `InjectionContext` struct +- [ ] Implement ThinkDeep injection options +- [ ] Implement STORM injection options +- [ ] Create merge logic for expanded sessions + +#### Phase 6: Enhanced Obsidian +- [ ] Implement research-notes structure +- [ ] Add source index with quality scores +- [ ] Create bi-directional links between insights + +## Code References + +- `internal/repl/router.go:46-74` - Current routing logic +- `internal/repl/handlers/expand.go:21-88` - Existing expand handler +- `internal/session/session.go:41-57` - Session struct definition +- `internal/think_deep/state.go:20-45` - SupervisorState (injection target) +- `internal/agents/supervisor.go:307-420` - Sub-researcher execution +- `internal/obsidian/writer.go:26-62` - Vault writer (needs enhancement) +- `internal/orchestrator/think_deep.go:262-278` - Brief generation (injection point) +- `internal/orchestrator/think_deep.go:302-326` - Final report (injection point) + +## Architecture Insights + +### Key Design Decisions + +1. **Output-based Context**: Store summarized outputs, not full agent state. This bounds memory while preserving usefulness. + +2. **Agent-Specific Injection**: Each architecture has different injection points. ThinkDeep injects into supervisor state; STORM injects into perspective discovery and conversation context. + +3. **Version Chaining**: Sessions form a linked list via `ParentID`. This enables traceable expansion history. + +4. **Event-Driven Progress**: Existing event bus supports real-time UI feedback. New handlers should emit progress events. + +5. **Non-Blocking Obsidian**: Vault writes are best-effort. This prevents I/O issues from blocking research. + +### Patterns to Follow + +1. **Handler Pattern**: All commands implement `Handler.Execute(ctx, args)`. New chat handlers should follow this. + +2. **Functional Options**: ThinkDeep uses `ThinkDeepOption` for configuration. Injection context should use same pattern. + +3. **Graceful Degradation**: If classification fails, fall back to `/storm`. If expansion fails, continue with new research. + +## Open Questions + +1. **Classification Model**: Use LLM for query classification or simpler heuristics? + - LLM: More accurate, but adds latency and cost + - Heuristics: Fast, but may misclassify + +2. **Context Window Management**: How much previous context to inject? + - Too much: Expensive, may confuse agent + - Too little: Loses valuable knowledge + +3. **Merge Strategy**: When expanding, how to merge new findings? + - Append: Simple, but may duplicate + - Intelligent merge: Complex, but cleaner output + +4. **Insight Granularity**: What constitutes a single "insight"? + - Per search result? + - Per sub-researcher? + - Per topic? + +## Related Research + +- [Stanford STORM Paper](https://arxiv.org/abs/2402.14207) - Multi-perspective research +- [ThinkDepth.ai](https://www.thinkdepth.ai) - Diffusion-based research +- [ReAct Pattern](https://arxiv.org/abs/2210.03629) - Think-Act-Observe loop diff --git a/go-research/thoughts/shared/research/2025-12-03_think-deep-data-tools.md b/go-research/thoughts/shared/research/2025-12-03_think-deep-data-tools.md new file mode 100644 index 0000000..421a865 --- /dev/null +++ b/go-research/thoughts/shared/research/2025-12-03_think-deep-data-tools.md @@ -0,0 +1,483 @@ +--- +date: 2025-12-03T14:30:00+01:00 +researcher: Claude +git_commit: 6a32cb5cc41e10a32999f565d10ca639bbecc06c +branch: main +repository: addcommitpush.io/go-research +topic: "Data Analysis and File Reading Tools for ThinkDeep Agent" +tags: [research, think_deep, tools, data_analysis, eda, pdf, csv, pickle] +status: complete +last_updated: 2025-12-03 +last_updated_by: Claude +--- + +# Research: Data Analysis and File Reading Tools for ThinkDeep Agent + +**Date**: 2025-12-03T14:30:00+01:00 +**Researcher**: Claude +**Git Commit**: 6a32cb5cc41e10a32999f565d10ca639bbecc06c +**Branch**: main +**Repository**: addcommitpush.io/go-research + +## Research Question + +How to add reusable data analysis tools (CSV/pickle EDA) and file reading tools (PDF, DOCX, PPTX) to the ThinkDeep agent that follow the existing architecture patterns? + +## Summary + +The ThinkDeep architecture supports extensible tools through a clean `Tool` interface pattern. Adding data analysis and file reading capabilities requires: + +1. **New Tool implementations** following the `internal/tools/Tool` interface +2. **Specialized sub-agent** for complex data analysis (following the `SubResearcherAgent` pattern) +3. **Prompt modifications** to expose new tools to the research agents +4. **Registry updates** to include new tools where needed + +The design should mirror the existing `search` → `ContentSummarizer` pattern where simple tools can optionally leverage LLM processing for deeper insights. + +## Detailed Findings + +### 1. Current Tool Architecture + +**Tool Interface** (`internal/tools/registry.go:9-13`): +```go +type Tool interface { + Name() string + Description() string + Execute(ctx context.Context, args map[string]interface{}) (string, error) +} +``` + +**ToolExecutor Interface** (`internal/tools/registry.go:15-19`): +```go +type ToolExecutor interface { + Execute(ctx context.Context, name string, args map[string]interface{}) (string, error) + ToolNames() []string +} +``` + +**Key Pattern - Search with Optional Summarization** (`internal/tools/search.go`): +- `SearchTool` performs basic web search +- Optional `ContentSummarizer` enhances results with LLM-generated summaries +- This pattern is directly applicable to data analysis tools + +### 2. Sub-Researcher Agent Pattern + +The `SubResearcherAgent` (`internal/agents/sub_researcher.go:23-29`) demonstrates how to create a focused agent that: +- Has access to specific tools (search, think) +- Executes an iterative loop with hard limits +- Compresses findings for the supervisor +- Emits progress events via the event bus + +This pattern can be adapted for a `DataAnalysisAgent` sub-researcher. + +### 3. Proposed Tool Implementations + +#### 3.1 Data Analysis Tools + +**CSVAnalysisTool** - For CSV/tabular data: +```go +// internal/tools/csv_analysis.go + +type CSVAnalysisTool struct { + client llm.ChatClient // Optional: for LLM-enhanced analysis +} + +func (t *CSVAnalysisTool) Name() string { return "analyze_csv" } + +func (t *CSVAnalysisTool) Description() string { + return `Analyze CSV data file. Performs EDA including: shape, dtypes, summary stats, +missing values, correlation analysis. Args: {"path": "/path/to/file.csv", "goal": "specific analysis objective"}` +} + +func (t *CSVAnalysisTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path := args["path"].(string) + goal := args["goal"].(string) // Optional: guides what to look for + + // 1. Read CSV using encoding/csv or go-gota/gota + // 2. Generate statistical summary + // 3. If client is set, use LLM to interpret findings relative to goal + // 4. Return formatted analysis +} +``` + +**PickleAnalysisTool** - For Python pickle files: +```go +// internal/tools/pickle_analysis.go + +type PickleAnalysisTool struct { + pythonPath string // Path to Python interpreter + client llm.ChatClient +} + +func (t *PickleAnalysisTool) Name() string { return "analyze_pickle" } + +func (t *PickleAnalysisTool) Description() string { + return `Analyze Python pickle file. Loads pickle and extracts structure/data. +Args: {"path": "/path/to/file.pkl", "goal": "analysis objective"}` +} + +func (t *PickleAnalysisTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + // Use Python subprocess to safely load and inspect pickle + // Security: Run in sandboxed environment, only inspect structure +} +``` + +**GoalDirectedEDATool** - High-level EDA orchestrator: +```go +// internal/tools/eda.go + +type GoalDirectedEDATool struct { + client llm.ChatClient +} + +func (t *GoalDirectedEDATool) Name() string { return "exploratory_analysis" } + +func (t *GoalDirectedEDATool) Description() string { + return `Conduct goal-directed exploratory data analysis. Analyzes data structure, +distributions, correlations, and anomalies relative to a research objective. +Args: {"path": "/path/to/data", "goal": "research question or hypothesis to explore"}` +} +``` + +#### 3.2 Document Reading Tools + +**PDFReadTool** - For PDF documents: +```go +// internal/tools/pdf.go + +import "github.com/pdfcpu/pdfcpu/pkg/api" + +type PDFReadTool struct { + client llm.ChatClient // For summarization +} + +func (t *PDFReadTool) Name() string { return "read_pdf" } + +func (t *PDFReadTool) Description() string { + return `Extract text from PDF file. Args: {"path": "/path/to/doc.pdf"}` +} + +func (t *PDFReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path := args["path"].(string) + // Extract text using pdfcpu or unidoc + // Return extracted text (truncated if too long) +} +``` + +**DOCXReadTool** - For Word documents: +```go +// internal/tools/docx.go + +import "github.com/unidoc/unioffice/document" + +type DOCXReadTool struct{} + +func (t *DOCXReadTool) Name() string { return "read_docx" } + +func (t *DOCXReadTool) Description() string { + return `Extract text from DOCX file. Args: {"path": "/path/to/doc.docx"}` +} +``` + +**PPTXReadTool** - For PowerPoint: +```go +// internal/tools/pptx.go + +import "github.com/unidoc/unioffice/presentation" + +type PPTXReadTool struct{} + +func (t *PPTXReadTool) Name() string { return "read_pptx" } + +func (t *PPTXReadTool) Description() string { + return `Extract text from PPTX file. Args: {"path": "/path/to/deck.pptx"}` +} +``` + +**GenericDocumentReadTool** - Auto-detect format: +```go +// internal/tools/document.go + +type DocumentReadTool struct { + pdfTool *PDFReadTool + docxTool *DOCXReadTool + pptxTool *PPTXReadTool + client llm.ChatClient +} + +func (t *DocumentReadTool) Name() string { return "read_document" } + +func (t *DocumentReadTool) Description() string { + return `Read and extract text from document (PDF, DOCX, PPTX - auto-detected). +Args: {"path": "/path/to/doc", "summarize": true|false}` +} + +func (t *DocumentReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path := args["path"].(string) + summarize := args["summarize"].(bool) + + // Detect format from extension + // Delegate to appropriate reader + // Optionally summarize with LLM +} +``` + +### 4. Data Analysis Sub-Agent Design + +Following the `SubResearcherAgent` pattern, create a specialized data analysis agent: + +```go +// internal/agents/data_analyst.go + +type DataAnalystAgent struct { + client llm.ChatClient + tools tools.ToolExecutor + bus *events.Bus + maxIterations int + model string +} + +type DataAnalystConfig struct { + MaxIterations int // Default: 5 +} + +type DataAnalystResult struct { + Analysis string + KeyInsights []string + Methodology string + Visualizations []string // Paths to generated charts + Cost session.CostBreakdown +} + +func (a *DataAnalystAgent) Analyze(ctx context.Context, dataPath string, goal string) (*DataAnalystResult, error) { + // 1. Load and inspect data structure + // 2. Plan analysis approach based on goal + // 3. Execute EDA tools iteratively + // 4. Synthesize insights + // 5. Generate visualizations (optional) +} +``` + +**Data Analyst Prompt** (new file: `internal/think_deep/data_prompts.go`): +```go +func DataAnalystPrompt(date, dataDescription string) string { + return fmt.Sprintf(`You are a data analyst conducting exploratory data analysis. Today is %s. + + +%s + + + +1. **analyze_csv**: Analyze CSV files - get shape, statistics, correlations + Usage: {"path": "/path/to/file.csv", "goal": "what to look for"} + +2. **analyze_pickle**: Analyze Python pickle files + Usage: {"path": "/path/to/file.pkl"} + +3. **read_document**: Read PDF/DOCX/PPTX files + Usage: {"path": "/path/to/doc.pdf"} + +4. **think**: Reflect on findings and plan next steps + Usage: {"reflection": "your analysis..."} + + + +1. Start by understanding the data structure +2. Formulate hypotheses based on the research goal +3. Test hypotheses through targeted analysis +4. Document key insights with supporting evidence +5. Stop when you have actionable insights +`, date, dataDescription) +} +``` + +### 5. Integration with ThinkDeep Supervisor + +**Option A: Direct Tool Access** (simpler) +Add data tools directly to the sub-researcher tool registry: + +```go +// internal/think_deep/tools.go (modification) + +func SubResearcherToolRegistry(braveAPIKey string, client ...llm.ChatClient) *tools.Registry { + registry := tools.NewEmptyRegistry() + + // Existing tools... + searchTool := tools.NewSearchTool(braveAPIKey) + registry.Register(searchTool) + registry.Register(tools.NewFetchTool()) + registry.Register(&ThinkTool{}) + + // NEW: Add data analysis tools + registry.Register(tools.NewCSVAnalysisTool()) + registry.Register(tools.NewDocumentReadTool()) + + // With optional LLM enhancement + if len(client) > 0 && client[0] != nil { + summarizer := tools.NewContentSummarizer(client[0]) + searchTool.SetSummarizer(summarizer) + + // Also enhance data tools + registry.Register(tools.NewGoalDirectedEDATool(client[0])) + } + + return registry +} +``` + +**Option B: Dedicated Sub-Agent** (more powerful) +Add `conduct_data_analysis` as a supervisor tool that delegates to `DataAnalystAgent`: + +```go +// internal/think_deep/tools.go (addition) + +type ConductDataAnalysisTool struct { + callback func(ctx context.Context, dataPath, goal string) (string, error) +} + +func (t *ConductDataAnalysisTool) Name() string { return "conduct_data_analysis" } + +func (t *ConductDataAnalysisTool) Description() string { + return `Delegate data analysis to a specialized sub-agent. +Args: {"data_path": "/path/to/data", "goal": "analysis objective"}` +} +``` + +Supervisor prompt update (`internal/think_deep/prompts.go`): +```go +// Add to LeadResearcherPrompt: + +5. **conduct_data_analysis**: Delegate data analysis to specialized analyst + Usage: {"data_path": "/path/to/data.csv", "goal": "Find correlations with sales"} + +6. **read_document**: Extract text from documents (PDF, DOCX, PPTX) + Usage: {"path": "/path/to/report.pdf"} +``` + +### 6. Go Library Recommendations + +For implementing these tools in Go: + +**CSV Processing:** +- `encoding/csv` (stdlib) - Basic CSV reading +- `github.com/go-gota/gota` - DataFrame operations, statistics + +**Pickle Files:** +- Python subprocess approach (safest) +- `github.com/nlpodyssey/spago` has some pickle support + +**PDF Extraction:** +- `github.com/pdfcpu/pdfcpu` - Pure Go, good extraction +- `github.com/unidoc/unipdf` - Commercial, more features + +**Office Documents:** +- `github.com/unidoc/unioffice` - DOCX, XLSX, PPTX +- `github.com/nguyenthenguyen/docx` - Simpler DOCX-only + +**Statistics:** +- `gonum.org/v1/gonum/stat` - Statistical functions +- `github.com/montanaflynn/stats` - Descriptive statistics + +### 7. Implementation Roadmap + +**Phase 1: Document Reading Tools** +- [ ] Implement `PDFReadTool` with pdfcpu +- [ ] Implement `DOCXReadTool` with unioffice +- [ ] Implement `PPTXReadTool` with unioffice +- [ ] Create `DocumentReadTool` wrapper with auto-detection +- [ ] Add to SubResearcherToolRegistry + +**Phase 2: Basic Data Analysis Tools** +- [ ] Implement `CSVAnalysisTool` with gota +- [ ] Add basic statistics: shape, dtypes, missing values, summary stats +- [ ] Add correlation analysis +- [ ] Create LLM-enhanced interpretation mode + +**Phase 3: Goal-Directed EDA** +- [ ] Create `GoalDirectedEDATool` with LLM planning +- [ ] Implement iterative analysis loop +- [ ] Add hypothesis testing support + +**Phase 4: Data Analyst Sub-Agent** (optional) +- [ ] Create `DataAnalystAgent` following SubResearcherAgent pattern +- [ ] Implement `conduct_data_analysis` supervisor tool +- [ ] Update supervisor prompt with new capability + +**Phase 5: Pickle Support** (optional) +- [ ] Implement Python subprocess bridge for pickle inspection +- [ ] Add sandbox/security measures +- [ ] Create `PickleAnalysisTool` + +## Code References + +- `internal/tools/registry.go:9-13` - Tool interface definition +- `internal/tools/search.go:17-30` - SearchTool with optional summarizer pattern +- `internal/tools/summarizer.go:14-27` - ContentSummarizer pattern for LLM enhancement +- `internal/agents/sub_researcher.go:23-62` - SubResearcherAgent architecture +- `internal/think_deep/tools.go:246-267` - SubResearcherToolRegistry (where to add new tools) +- `internal/think_deep/prompts.go:16-89` - LeadResearcherPrompt (needs tool docs) +- `internal/think_deep/prompts.go:91-145` - ResearchAgentPrompt (needs tool docs) + +## Architecture Insights + +### Key Patterns to Follow + +1. **Tool Interface Compliance**: All tools must implement `Name()`, `Description()`, `Execute()` + +2. **Optional LLM Enhancement**: Like `SearchTool.SetSummarizer()`, data tools should work standalone but optionally leverage LLM for deeper analysis + +3. **Event Bus Integration**: For long-running analysis, emit progress events: + ```go + bus.Publish(events.Event{ + Type: events.EventDataAnalysisProgress, + Timestamp: time.Now(), + Data: DataAnalysisProgressData{...}, + }) + ``` + +4. **Hard Limits**: Follow the 5-iteration max pattern from sub-researcher + +5. **Compression Pattern**: After analysis, compress findings for supervisor (like `compressResearch`) + +### Security Considerations + +- **Pickle files**: Never use Go native pickle loading; always subprocess to Python with timeout and resource limits +- **File paths**: Validate paths are within allowed directories +- **Resource limits**: Set memory caps for large dataset analysis + +## Open Questions + +1. **Visualization Support**: Should the data analyst generate charts? Would require image handling in reports. + +2. **Large File Handling**: How to handle CSVs larger than memory? Consider chunked processing or sampling. + +3. **Python Interop**: For pickle and advanced analysis, is Python subprocess acceptable or should we use CGO bindings? + +4. **Tool Discovery**: Should tools be dynamically discovered based on data type, or explicitly called by agents? + +5. **Caching**: Should analysis results be cached for repeated queries on same data? + +## Example Usage Flow + +``` +User: "Analyze the sales data in /data/sales_2024.csv and find what factors + most influence customer retention" + +1. ThinkDeep Supervisor receives query +2. Supervisor calls: conduct_data_analysis(path="/data/sales_2024.csv", + goal="factors influencing customer retention") +3. DataAnalystAgent spawns: + - Calls analyze_csv to get shape, types, summary + - Uses think to plan analysis approach + - Calls analyze_csv with specific correlation goals + - Identifies key factors through iteration + - Compresses findings +4. Supervisor receives compressed analysis +5. Supervisor may call conduct_research for external validation +6. Final report incorporates data-driven insights +``` + +## Related Research + +- Previous research on ThinkDeep architecture: `thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md` +- ThinkDepth.ai original paper approach (using GPT-5 with tool calling) From ada100d673fda8070c43dd035eac776959b0abef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Thu, 4 Dec 2025 10:09:58 +0100 Subject: [PATCH 2/5] feat: Introduce document reading and data analysis tools for enhanced research capabilities - Updated Go module to version 1.24.1 and added new dependencies for PDF, DOCX, and CSV analysis tools. - Implemented `read_document`, `read_pdf`, and `read_docx` tools for extracting text from various document formats. - Developed `analyze_csv` tool for exploratory data analysis on CSV files, including shape, column types, and summary statistics. - Enhanced the `SubResearcherToolRegistry` to include new tools, improving the research agent's functionality. - Updated prompts to document new tools and their usage, ensuring clarity for users. - Added comprehensive unit tests for new tools to validate functionality and reliability. --- go-research/go.mod | 5 +- go-research/go.sum | 6 + go-research/internal/think_deep/prompts.go | 23 +- go-research/internal/think_deep/tools.go | 8 +- .../think_deep/tools_integration_test.go | 97 ++ go-research/internal/tools/csv_analysis.go | 286 ++++ .../internal/tools/csv_analysis_test.go | 220 +++ go-research/internal/tools/document.go | 50 + go-research/internal/tools/document_test.go | 91 ++ go-research/internal/tools/docx.go | 74 + go-research/internal/tools/docx_test.go | 59 + go-research/internal/tools/pdf.go | 82 + go-research/internal/tools/pdf_test.go | 60 + .../data-analysis-tools-implementation.md | 1430 +++++++++++++++++ 14 files changed, 2485 insertions(+), 6 deletions(-) create mode 100644 go-research/internal/think_deep/tools_integration_test.go create mode 100644 go-research/internal/tools/csv_analysis.go create mode 100644 go-research/internal/tools/csv_analysis_test.go create mode 100644 go-research/internal/tools/document.go create mode 100644 go-research/internal/tools/document_test.go create mode 100644 go-research/internal/tools/docx.go create mode 100644 go-research/internal/tools/docx_test.go create mode 100644 go-research/internal/tools/pdf.go create mode 100644 go-research/internal/tools/pdf_test.go create mode 100644 go-research/thoughts/shared/plans/data-analysis-tools-implementation.md diff --git a/go-research/go.mod b/go-research/go.mod index c111982..590bff4 100644 --- a/go-research/go.mod +++ b/go-research/go.mod @@ -1,12 +1,15 @@ module go-research -go 1.24.0 +go 1.24.1 require ( github.com/chzyer/readline v1.5.1 github.com/fatih/color v1.16.0 github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 + github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 + github.com/montanaflynn/stats v0.7.1 + github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db golang.org/x/net v0.30.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go-research/go.sum b/go-research/go.sum index 1d48a7d..21ccbf6 100644 --- a/go-research/go.sum +++ b/go-research/go.sum @@ -10,11 +10,17 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8= +github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db h1:v0cW/tTMrJQyZr7r6t+t9+NhH2OBAjydHisVYxuyObc= +github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db/go.mod h1:BZyH8oba3hE/BTt2FfBDGPOHhXiKs9RFmUvvXRdzrhM= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/go-research/internal/think_deep/prompts.go b/go-research/internal/think_deep/prompts.go index c483040..90deb16 100644 --- a/go-research/internal/think_deep/prompts.go +++ b/go-research/internal/think_deep/prompts.go @@ -80,6 +80,10 @@ After each conduct_research tool call, use think to analyze the results: - *Example*: Compare OpenAI vs. Anthropic vs. DeepMind approaches to AI safety → Use 3 sub-agents - Delegate clear, distinct, non-overlapping subtopics +**Data Analysis Tasks** require specialized handling: +- *Example*: Analyze the sales data in /data/sales.csv → Delegate to sub-agent with clear analysis goal +- When delegating data analysis, specify the file path AND the analysis objective clearly + **Important Reminders:** - Each conduct_research call spawns a dedicated research agent for that specific topic - A separate agent will write the final report - you just need to gather information @@ -101,14 +105,24 @@ You can use any of the tools provided to you to find resources that can help ans -You have access to two main tools: +You have access to the following tools: + 1. **search**: For conducting web searches to gather information Usage: {"query": "your search query"} -2. **think**: For reflection and strategic planning during research +2. **fetch**: For fetching content from a specific URL + Usage: {"url": "https://example.com"} + +3. **read_document**: For reading PDF or DOCX documents + Usage: {"path": "/path/to/document.pdf"} + +4. **analyze_csv**: For analyzing CSV data files (EDA, statistics, column analysis) + Usage: {"path": "/path/to/data.csv", "goal": "what to look for"} + +5. **think**: For reflection and strategic planning during research Usage: {"reflection": "Your analysis of results..."} -**CRITICAL: Use think after each search to reflect on results and plan next steps** +**CRITICAL: Use think after each search or analysis to reflect on results and plan next steps** @@ -118,7 +132,8 @@ Think like a human researcher with limited time. Follow these steps: 2. **Start with broader searches** - Use broad, comprehensive queries first 3. **After each search, pause and assess** - Do I have enough to answer? What's still missing? 4. **Execute narrower searches as you gather information** - Fill in the gaps -5. **Stop when you can answer confidently** - Don't keep searching for perfection +5. **For data analysis tasks** - Use analyze_csv for CSV files, read_document for PDFs/DOCX +6. **Stop when you can answer confidently** - Don't keep searching for perfection diff --git a/go-research/internal/think_deep/tools.go b/go-research/internal/think_deep/tools.go index 4a5fa1e..d5ddbff 100644 --- a/go-research/internal/think_deep/tools.go +++ b/go-research/internal/think_deep/tools.go @@ -244,7 +244,7 @@ func SupervisorToolRegistry( } // SubResearcherToolRegistry creates a tool registry for sub-researcher agents. -// This includes search (with optional summarization), fetch, and think tools. +// This includes search (with optional summarization), fetch, document reading, CSV analysis, and think tools. // If client is provided, search results will include LLM-generated summaries of page content. func SubResearcherToolRegistry(braveAPIKey string, client ...llm.ChatClient) *tools.Registry { registry := tools.NewEmptyRegistry() @@ -260,6 +260,12 @@ func SubResearcherToolRegistry(braveAPIKey string, client ...llm.ChatClient) *to // Add fetch tool (for direct URL fetching if needed) registry.Register(tools.NewFetchTool()) + // Add document reading tools + registry.Register(tools.NewDocumentReadTool()) + + // Add CSV analysis tool + registry.Register(tools.NewCSVAnalysisTool()) + // Add think tool for reflection registry.Register(&ThinkTool{}) diff --git a/go-research/internal/think_deep/tools_integration_test.go b/go-research/internal/think_deep/tools_integration_test.go new file mode 100644 index 0000000..80d11f1 --- /dev/null +++ b/go-research/internal/think_deep/tools_integration_test.go @@ -0,0 +1,97 @@ +package think_deep + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestSubResearcherToolRegistry_HasAllTools(t *testing.T) { + // Use empty API key for test (search won't actually be called) + registry := SubResearcherToolRegistry("") + + expectedTools := []string{ + "search", + "fetch", + "read_document", + "analyze_csv", + "think", + } + + registeredTools := registry.ToolNames() + + for _, expected := range expectedTools { + found := false + for _, registered := range registeredTools { + if registered == expected { + found = true + break + } + } + if !found { + t.Errorf("missing expected tool: %s", expected) + } + } +} + +func TestSubResearcherToolRegistry_ExecuteDocumentTool(t *testing.T) { + registry := SubResearcherToolRegistry("") + + // Test that read_document tool exists and can be executed (will fail on missing file) + _, err := registry.Execute(context.Background(), "read_document", map[string]interface{}{ + "path": "/nonexistent/file.pdf", + }) + + // We expect a "file not found" error, which proves the tool was found and executed + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +func TestSubResearcherToolRegistry_ExecuteCSVTool(t *testing.T) { + registry := SubResearcherToolRegistry("") + + // Create temp CSV + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "test.csv") + csvContent := "a,b\n1,2\n3,4\n" + if err := os.WriteFile(csvPath, []byte(csvContent), 0644); err != nil { + t.Fatalf("failed to create test CSV: %v", err) + } + + result, err := registry.Execute(context.Background(), "analyze_csv", map[string]interface{}{ + "path": csvPath, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if result == "" { + t.Error("expected non-empty result") + } +} + +func TestSubResearcherToolRegistry_ExecuteThinkTool(t *testing.T) { + registry := SubResearcherToolRegistry("") + + result, err := registry.Execute(context.Background(), "think", map[string]interface{}{ + "reflection": "Testing think tool integration", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if result == "" { + t.Error("expected non-empty result") + } +} + +func TestSubResearcherToolRegistry_UnknownTool(t *testing.T) { + registry := SubResearcherToolRegistry("") + + _, err := registry.Execute(context.Background(), "nonexistent_tool", map[string]interface{}{}) + if err == nil { + t.Error("expected error for unknown tool") + } +} diff --git a/go-research/internal/tools/csv_analysis.go b/go-research/internal/tools/csv_analysis.go new file mode 100644 index 0000000..6943b3c --- /dev/null +++ b/go-research/internal/tools/csv_analysis.go @@ -0,0 +1,286 @@ +package tools + +import ( + "context" + "encoding/csv" + "fmt" + "os" + "sort" + "strconv" + "strings" + + "github.com/montanaflynn/stats" +) + +// CSVAnalysisTool performs exploratory data analysis on CSV files. +type CSVAnalysisTool struct { + maxRows int // Maximum rows to analyze (0 = all) +} + +// NewCSVAnalysisTool creates a new CSV analysis tool. +func NewCSVAnalysisTool() *CSVAnalysisTool { + return &CSVAnalysisTool{ + maxRows: 10000, // Default: analyze first 10k rows + } +} + +func (t *CSVAnalysisTool) Name() string { + return "analyze_csv" +} + +func (t *CSVAnalysisTool) Description() string { + return `Analyze a CSV data file. Performs EDA including: shape, column types, summary statistics, missing values. +Args: {"path": "/path/to/file.csv", "goal": "optional analysis objective"}` +} + +func (t *CSVAnalysisTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("analyze_csv requires a 'path' argument") + } + + goal, _ := args["goal"].(string) + + // Validate file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + + // Open CSV file + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open CSV: %w", err) + } + defer f.Close() + + // Parse CSV + reader := csv.NewReader(f) + records, err := reader.ReadAll() + if err != nil { + return "", fmt.Errorf("parse CSV: %w", err) + } + + if len(records) == 0 { + return "Empty CSV file.", nil + } + + // Extract headers and data + headers := records[0] + data := records[1:] + + // Limit rows if needed + if t.maxRows > 0 && len(data) > t.maxRows { + data = data[:t.maxRows] + } + + // Build analysis report + var report strings.Builder + + // Header + report.WriteString(fmt.Sprintf("# CSV Analysis: %s\n\n", path)) + if goal != "" { + report.WriteString(fmt.Sprintf("**Analysis Goal**: %s\n\n", goal)) + } + + // Shape + report.WriteString("## Shape\n") + report.WriteString(fmt.Sprintf("- Rows: %d (showing first %d)\n", len(records)-1, len(data))) + report.WriteString(fmt.Sprintf("- Columns: %d\n\n", len(headers))) + + // Columns overview + report.WriteString("## Columns\n") + report.WriteString("| Column | Type | Non-Null | Missing | Sample Values |\n") + report.WriteString("|--------|------|----------|---------|---------------|\n") + + columnData := make([][]string, len(headers)) + for i := range headers { + columnData[i] = make([]string, len(data)) + for j, row := range data { + if i < len(row) { + columnData[i][j] = row[i] + } + } + } + + for i, header := range headers { + colType, nonNull, missing := analyzeColumn(columnData[i]) + samples := getSampleValues(columnData[i], 3) + report.WriteString(fmt.Sprintf("| %s | %s | %d | %d | %s |\n", + header, colType, nonNull, missing, samples)) + } + report.WriteString("\n") + + // Summary statistics for numeric columns + report.WriteString("## Summary Statistics (Numeric Columns)\n") + hasNumeric := false + for i, header := range headers { + if isNumericColumn(columnData[i]) { + hasNumeric = true + numericStats := computeNumericStats(columnData[i]) + report.WriteString(fmt.Sprintf("### %s\n", header)) + report.WriteString(fmt.Sprintf("- Count: %d\n", numericStats.count)) + report.WriteString(fmt.Sprintf("- Mean: %.4f\n", numericStats.mean)) + report.WriteString(fmt.Sprintf("- Std: %.4f\n", numericStats.std)) + report.WriteString(fmt.Sprintf("- Min: %.4f\n", numericStats.min)) + report.WriteString(fmt.Sprintf("- 25%%: %.4f\n", numericStats.q25)) + report.WriteString(fmt.Sprintf("- 50%% (Median): %.4f\n", numericStats.median)) + report.WriteString(fmt.Sprintf("- 75%%: %.4f\n", numericStats.q75)) + report.WriteString(fmt.Sprintf("- Max: %.4f\n\n", numericStats.max)) + } + } + if !hasNumeric { + report.WriteString("No numeric columns detected.\n\n") + } + + // Categorical column summaries + report.WriteString("## Categorical Column Value Counts\n") + hasCategorical := false + for i, header := range headers { + if !isNumericColumn(columnData[i]) { + hasCategorical = true + valueCounts := getValueCounts(columnData[i], 10) + report.WriteString(fmt.Sprintf("### %s\n", header)) + for _, vc := range valueCounts { + report.WriteString(fmt.Sprintf("- %s: %d\n", vc.value, vc.count)) + } + report.WriteString("\n") + } + } + if !hasCategorical { + report.WriteString("No categorical columns detected.\n\n") + } + + return report.String(), nil +} + +// Column analysis helpers + +func analyzeColumn(col []string) (colType string, nonNull, missing int) { + for _, val := range col { + if val == "" { + missing++ + } else { + nonNull++ + } + } + + if isNumericColumn(col) { + colType = "numeric" + } else { + colType = "string" + } + + return +} + +func isNumericColumn(col []string) bool { + numericCount := 0 + totalNonEmpty := 0 + + for _, val := range col { + if val == "" { + continue + } + totalNonEmpty++ + if _, err := strconv.ParseFloat(val, 64); err == nil { + numericCount++ + } + } + + if totalNonEmpty == 0 { + return false + } + + // Consider numeric if >80% of non-empty values are numeric + return float64(numericCount)/float64(totalNonEmpty) > 0.8 +} + +func getSampleValues(col []string, n int) string { + seen := make(map[string]bool) + var samples []string + + for _, val := range col { + if val != "" && !seen[val] { + seen[val] = true + samples = append(samples, val) + if len(samples) >= n { + break + } + } + } + + return strings.Join(samples, ", ") +} + +type numericStats struct { + count int + mean float64 + std float64 + min float64 + q25 float64 + median float64 + q75 float64 + max float64 +} + +func computeNumericStats(col []string) numericStats { + var values []float64 + for _, val := range col { + if f, err := strconv.ParseFloat(val, 64); err == nil { + values = append(values, f) + } + } + + if len(values) == 0 { + return numericStats{} + } + + mean, _ := stats.Mean(values) + std, _ := stats.StandardDeviation(values) + min, _ := stats.Min(values) + max, _ := stats.Max(values) + median, _ := stats.Median(values) + q25, _ := stats.Percentile(values, 25) + q75, _ := stats.Percentile(values, 75) + + return numericStats{ + count: len(values), + mean: mean, + std: std, + min: min, + q25: q25, + median: median, + q75: q75, + max: max, + } +} + +type valueCount struct { + value string + count int +} + +func getValueCounts(col []string, limit int) []valueCount { + counts := make(map[string]int) + for _, val := range col { + if val != "" { + counts[val]++ + } + } + + var result []valueCount + for v, c := range counts { + result = append(result, valueCount{value: v, count: c}) + } + + // Sort by count descending + sort.Slice(result, func(i, j int) bool { + return result[i].count > result[j].count + }) + + if len(result) > limit { + result = result[:limit] + } + + return result +} diff --git a/go-research/internal/tools/csv_analysis_test.go b/go-research/internal/tools/csv_analysis_test.go new file mode 100644 index 0000000..6d24e22 --- /dev/null +++ b/go-research/internal/tools/csv_analysis_test.go @@ -0,0 +1,220 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCSVAnalysisTool_Name(t *testing.T) { + tool := NewCSVAnalysisTool() + if tool.Name() != "analyze_csv" { + t.Errorf("expected name 'analyze_csv', got '%s'", tool.Name()) + } +} + +func TestCSVAnalysisTool_Description(t *testing.T) { + tool := NewCSVAnalysisTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestCSVAnalysisTool_Execute_MissingPath(t *testing.T) { + tool := NewCSVAnalysisTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestCSVAnalysisTool_Execute_FileNotFound(t *testing.T) { + tool := NewCSVAnalysisTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.csv", + }) + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +func TestCSVAnalysisTool_Execute_SimpleCSV(t *testing.T) { + // Create temp CSV file + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "test.csv") + + csvContent := `name,age,score +Alice,30,85.5 +Bob,25,92.3 +Charlie,35,78.9 +Diana,28,88.0 +` + if err := os.WriteFile(csvPath, []byte(csvContent), 0644); err != nil { + t.Fatalf("failed to create test CSV: %v", err) + } + + tool := NewCSVAnalysisTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": csvPath, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify report contains expected sections + if !strings.Contains(result, "## Shape") { + t.Error("report missing Shape section") + } + if !strings.Contains(result, "## Columns") { + t.Error("report missing Columns section") + } + if !strings.Contains(result, "## Summary Statistics") { + t.Error("report missing Summary Statistics section") + } + if !strings.Contains(result, "Rows: 4") { + t.Error("incorrect row count") + } + if !strings.Contains(result, "Columns: 3") { + t.Error("incorrect column count") + } +} + +func TestCSVAnalysisTool_Execute_WithGoal(t *testing.T) { + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "test.csv") + + csvContent := `x,y +1,2 +3,4 +` + if err := os.WriteFile(csvPath, []byte(csvContent), 0644); err != nil { + t.Fatalf("failed to create test CSV: %v", err) + } + + tool := NewCSVAnalysisTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": csvPath, + "goal": "Find correlation between x and y", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !strings.Contains(result, "Find correlation between x and y") { + t.Error("goal not included in report") + } +} + +func TestCSVAnalysisTool_Execute_EmptyCSV(t *testing.T) { + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "empty.csv") + + if err := os.WriteFile(csvPath, []byte(""), 0644); err != nil { + t.Fatalf("failed to create empty CSV: %v", err) + } + + tool := NewCSVAnalysisTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": csvPath, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !strings.Contains(result, "Empty CSV file") { + t.Errorf("expected 'Empty CSV file' message, got: %s", result) + } +} + +func TestCSVAnalysisTool_Execute_MixedTypes(t *testing.T) { + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "mixed.csv") + + csvContent := `category,value,notes +A,100,First +B,200,Second +A,150,Third +C,50,Fourth +` + if err := os.WriteFile(csvPath, []byte(csvContent), 0644); err != nil { + t.Fatalf("failed to create test CSV: %v", err) + } + + tool := NewCSVAnalysisTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": csvPath, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Should have numeric stats for 'value' column + if !strings.Contains(result, "### value") { + t.Error("missing numeric stats for 'value' column") + } + + // Should have value counts for categorical columns + if !strings.Contains(result, "### category") { + t.Error("missing value counts for 'category' column") + } +} + +func TestIsNumericColumn(t *testing.T) { + tests := []struct { + col []string + expected bool + }{ + {[]string{"1", "2", "3"}, true}, + {[]string{"1.5", "2.7", "3.9"}, true}, + {[]string{"a", "b", "c"}, false}, + {[]string{"1", "2", ""}, true}, + {[]string{"1", "a", "b"}, false}, + {[]string{}, false}, + } + + for _, tt := range tests { + result := isNumericColumn(tt.col) + if result != tt.expected { + t.Errorf("isNumericColumn(%v) = %v, expected %v", tt.col, result, tt.expected) + } + } +} + +func TestGetValueCounts(t *testing.T) { + col := []string{"a", "b", "a", "c", "a", "b", ""} + + counts := getValueCounts(col, 10) + + if len(counts) != 3 { + t.Errorf("expected 3 value counts, got %d", len(counts)) + } + + // "a" should be first (most frequent) + if counts[0].value != "a" || counts[0].count != 3 { + t.Errorf("expected 'a' with count 3, got '%s' with count %d", counts[0].value, counts[0].count) + } +} + +func TestComputeNumericStats(t *testing.T) { + col := []string{"1", "2", "3", "4", "5"} + + s := computeNumericStats(col) + + if s.count != 5 { + t.Errorf("expected count 5, got %d", s.count) + } + if s.mean != 3.0 { + t.Errorf("expected mean 3.0, got %f", s.mean) + } + if s.min != 1.0 { + t.Errorf("expected min 1.0, got %f", s.min) + } + if s.max != 5.0 { + t.Errorf("expected max 5.0, got %f", s.max) + } + if s.median != 3.0 { + t.Errorf("expected median 3.0, got %f", s.median) + } +} diff --git a/go-research/internal/tools/document.go b/go-research/internal/tools/document.go new file mode 100644 index 0000000..9035a88 --- /dev/null +++ b/go-research/internal/tools/document.go @@ -0,0 +1,50 @@ +package tools + +import ( + "context" + "fmt" + "path/filepath" + "strings" +) + +// DocumentReadTool reads documents of various formats (PDF, DOCX). +// It auto-detects the format based on file extension. +type DocumentReadTool struct { + pdfTool *PDFReadTool + docxTool *DOCXReadTool +} + +// NewDocumentReadTool creates a new document reading tool. +func NewDocumentReadTool() *DocumentReadTool { + return &DocumentReadTool{ + pdfTool: NewPDFReadTool(), + docxTool: NewDOCXReadTool(), + } +} + +func (t *DocumentReadTool) Name() string { + return "read_document" +} + +func (t *DocumentReadTool) Description() string { + return `Read and extract text from a document file (PDF or DOCX - auto-detected from extension). +Args: {"path": "/path/to/document.pdf"}` +} + +func (t *DocumentReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("read_document requires a 'path' argument") + } + + ext := strings.ToLower(filepath.Ext(path)) + + switch ext { + case ".pdf": + return t.pdfTool.Execute(ctx, args) + case ".docx": + return t.docxTool.Execute(ctx, args) + default: + return "", fmt.Errorf("unsupported file format: %s (supported: .pdf, .docx)", ext) + } +} diff --git a/go-research/internal/tools/document_test.go b/go-research/internal/tools/document_test.go new file mode 100644 index 0000000..3565912 --- /dev/null +++ b/go-research/internal/tools/document_test.go @@ -0,0 +1,91 @@ +package tools + +import ( + "context" + "strings" + "testing" +) + +func TestDocumentReadTool_Name(t *testing.T) { + tool := NewDocumentReadTool() + if tool.Name() != "read_document" { + t.Errorf("expected name 'read_document', got '%s'", tool.Name()) + } +} + +func TestDocumentReadTool_Description(t *testing.T) { + tool := NewDocumentReadTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestDocumentReadTool_Execute_MissingPath(t *testing.T) { + tool := NewDocumentReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestDocumentReadTool_Execute_UnsupportedFormat(t *testing.T) { + tool := NewDocumentReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/path/to/file.txt", + }) + if err == nil { + t.Error("expected error for unsupported format") + } + if !strings.Contains(err.Error(), "unsupported file format") { + t.Errorf("expected 'unsupported file format' error, got: %v", err) + } +} + +func TestDocumentReadTool_Execute_DetectsPDF(t *testing.T) { + tool := NewDocumentReadTool() + // This will fail with "file not found" which proves PDF detection worked + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.pdf", + }) + if err == nil { + t.Error("expected error") + } + // PDF tool returns "file not found" which proves correct routing + if !strings.Contains(err.Error(), "file not found") { + t.Errorf("expected 'file not found' error (proving PDF routing), got: %v", err) + } +} + +func TestDocumentReadTool_Execute_DetectsDOCX(t *testing.T) { + tool := NewDocumentReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.docx", + }) + if err == nil { + t.Error("expected error") + } + // DOCX tool returns "file not found" which proves correct routing + if !strings.Contains(err.Error(), "file not found") { + t.Errorf("expected 'file not found' error (proving DOCX routing), got: %v", err) + } +} + +func TestDocumentReadTool_Execute_CaseInsensitiveExtension(t *testing.T) { + tool := NewDocumentReadTool() + + // Test uppercase PDF + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.PDF", + }) + if err == nil || !strings.Contains(err.Error(), "file not found") { + t.Error("expected PDF tool to handle .PDF extension") + } + + // Test mixed case DOCX + _, err = tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.DocX", + }) + if err == nil || !strings.Contains(err.Error(), "file not found") { + t.Error("expected DOCX tool to handle .DocX extension") + } +} diff --git a/go-research/internal/tools/docx.go b/go-research/internal/tools/docx.go new file mode 100644 index 0000000..6375f34 --- /dev/null +++ b/go-research/internal/tools/docx.go @@ -0,0 +1,74 @@ +package tools + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/nguyenthenguyen/docx" +) + +// DOCXReadTool extracts text content from DOCX files. +type DOCXReadTool struct{} + +// NewDOCXReadTool creates a new DOCX reading tool. +func NewDOCXReadTool() *DOCXReadTool { + return &DOCXReadTool{} +} + +func (t *DOCXReadTool) Name() string { + return "read_docx" +} + +func (t *DOCXReadTool) Description() string { + return `Extract text from a DOCX (Word) file. Args: {"path": "/path/to/file.docx"}` +} + +func (t *DOCXReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("read_docx requires a 'path' argument") + } + + // Validate file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + + // Open DOCX file + r, err := docx.ReadDocxFile(path) + if err != nil { + return "", fmt.Errorf("open DOCX: %w", err) + } + defer r.Close() + + // Extract text content + doc := r.Editable() + content := doc.GetContent() + + // Clean up whitespace + content = cleanDocxContent(content) + + // Truncate if too long + const maxLen = 100000 + if len(content) > maxLen { + content = content[:maxLen] + "\n...[truncated]" + } + + return content, nil +} + +// cleanDocxContent normalizes whitespace and formatting in extracted DOCX text. +func cleanDocxContent(s string) string { + // Replace multiple newlines with double newline (paragraph separator) + lines := strings.Split(s, "\n") + var cleaned []string + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + cleaned = append(cleaned, trimmed) + } + } + return strings.Join(cleaned, "\n\n") +} diff --git a/go-research/internal/tools/docx_test.go b/go-research/internal/tools/docx_test.go new file mode 100644 index 0000000..300bd81 --- /dev/null +++ b/go-research/internal/tools/docx_test.go @@ -0,0 +1,59 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestDOCXReadTool_Name(t *testing.T) { + tool := NewDOCXReadTool() + if tool.Name() != "read_docx" { + t.Errorf("expected name 'read_docx', got '%s'", tool.Name()) + } +} + +func TestDOCXReadTool_Description(t *testing.T) { + tool := NewDOCXReadTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestDOCXReadTool_Execute_MissingPath(t *testing.T) { + tool := NewDOCXReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestDOCXReadTool_Execute_FileNotFound(t *testing.T) { + tool := NewDOCXReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.docx", + }) + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +// Integration test with a real DOCX file +func TestDOCXReadTool_Execute_RealFile(t *testing.T) { + testDOCX := filepath.Join("testdata", "sample.docx") + if _, err := os.Stat(testDOCX); os.IsNotExist(err) { + t.Skip("no test DOCX file available") + } + + tool := NewDOCXReadTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": testDOCX, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == "" { + t.Error("expected non-empty result") + } +} diff --git a/go-research/internal/tools/pdf.go b/go-research/internal/tools/pdf.go new file mode 100644 index 0000000..4b150cd --- /dev/null +++ b/go-research/internal/tools/pdf.go @@ -0,0 +1,82 @@ +package tools + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/ledongthuc/pdf" +) + +// PDFReadTool extracts text content from PDF files. +type PDFReadTool struct { + maxPages int // Maximum pages to extract (0 = all) +} + +// NewPDFReadTool creates a new PDF reading tool. +func NewPDFReadTool() *PDFReadTool { + return &PDFReadTool{ + maxPages: 50, // Default: first 50 pages + } +} + +func (t *PDFReadTool) Name() string { + return "read_pdf" +} + +func (t *PDFReadTool) Description() string { + return `Extract text from a PDF file. Args: {"path": "/path/to/file.pdf"}` +} + +func (t *PDFReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("read_pdf requires a 'path' argument") + } + + // Validate file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + + // Open PDF file + f, r, err := pdf.Open(path) + if err != nil { + return "", fmt.Errorf("open PDF: %w", err) + } + defer f.Close() + + var text strings.Builder + numPages := r.NumPage() + maxPages := t.maxPages + if maxPages <= 0 || maxPages > numPages { + maxPages = numPages + } + + for i := 1; i <= maxPages; i++ { + p := r.Page(i) + if p.V.IsNull() { + continue + } + content, err := p.GetPlainText(nil) + if err != nil { + continue + } + text.WriteString(fmt.Sprintf("--- Page %d ---\n", i)) + text.WriteString(content) + text.WriteString("\n\n") + } + + if maxPages < numPages { + text.WriteString(fmt.Sprintf("\n...[truncated after %d of %d pages]\n", maxPages, numPages)) + } + + result := text.String() + const maxLen = 100000 + if len(result) > maxLen { + result = result[:maxLen] + "\n...[truncated]" + } + + return result, nil +} diff --git a/go-research/internal/tools/pdf_test.go b/go-research/internal/tools/pdf_test.go new file mode 100644 index 0000000..8cfc35f --- /dev/null +++ b/go-research/internal/tools/pdf_test.go @@ -0,0 +1,60 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestPDFReadTool_Name(t *testing.T) { + tool := NewPDFReadTool() + if tool.Name() != "read_pdf" { + t.Errorf("expected name 'read_pdf', got '%s'", tool.Name()) + } +} + +func TestPDFReadTool_Description(t *testing.T) { + tool := NewPDFReadTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestPDFReadTool_Execute_MissingPath(t *testing.T) { + tool := NewPDFReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestPDFReadTool_Execute_FileNotFound(t *testing.T) { + tool := NewPDFReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.pdf", + }) + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +// Integration test with a real PDF file +func TestPDFReadTool_Execute_RealFile(t *testing.T) { + // Skip if no test PDF available + testPDF := filepath.Join("testdata", "sample.pdf") + if _, err := os.Stat(testPDF); os.IsNotExist(err) { + t.Skip("no test PDF file available") + } + + tool := NewPDFReadTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": testPDF, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == "" { + t.Error("expected non-empty result") + } +} diff --git a/go-research/thoughts/shared/plans/data-analysis-tools-implementation.md b/go-research/thoughts/shared/plans/data-analysis-tools-implementation.md new file mode 100644 index 0000000..fa1d0d2 --- /dev/null +++ b/go-research/thoughts/shared/plans/data-analysis-tools-implementation.md @@ -0,0 +1,1430 @@ +# Data Analysis and File Reading Tools Implementation Plan + +## Overview + +This plan implements data analysis tools (CSV EDA) and document reading tools (PDF, DOCX, PPTX) for the ThinkDeep agent, following the existing architecture patterns from the research document `thoughts/shared/research/2025-12-03_think-deep-data-tools.md`. + +## Current State Analysis + +### Existing Tool Architecture + +The codebase has a clean tool architecture: + +- **Tool Interface** (`internal/tools/registry.go:9-13`): + ```go + type Tool interface { + Name() string + Description() string + Execute(ctx context.Context, args map[string]interface{}) (string, error) + } + ``` + +- **Registry Pattern** (`internal/tools/registry.go:21-83`): Tools are registered in registries and executed by name + +- **Optional LLM Enhancement Pattern** (`internal/tools/search.go:17-35`): SearchTool has optional `SetSummarizer()` for LLM-enhanced results + +- **ContentSummarizer Pattern** (`internal/tools/summarizer.go:14-92`): Fetches content and summarizes using LLM + +### Existing Tools + +1. `SearchTool` - Web search via Brave API with optional summarization +2. `FetchTool` - Direct URL content fetching +3. `ThinkTool` - Strategic reflection (no-op acknowledgment) +4. `ResearchCompleteTool` - Signal research completion +5. `RefineDraftTool` - Refine draft with LLM +6. `ConductResearchTool` - Delegate to sub-researchers + +### Tool Registry Builders + +- `SubResearcherToolRegistry()` (`internal/think_deep/tools.go:246-267`): Creates registry for sub-researchers with search, fetch, think tools + +### Current Dependencies + +From `go.mod`: +- Go 1.24.0 +- No existing PDF/document parsing libraries +- No CSV/data analysis libraries + +## Desired End State + +After this plan is complete: + +1. **Document Reading Tools** available to sub-researchers: + - `read_pdf` - Extract text from PDF files + - `read_docx` - Extract text from Word documents + - `read_document` - Auto-detect format and extract text (wrapper) + +2. **Data Analysis Tools** available to sub-researchers: + - `analyze_csv` - Perform EDA on CSV files (shape, types, stats, correlations) + +3. **Integration** with existing tool registries: + - Tools added to `SubResearcherToolRegistry()` + - Optional LLM enhancement for goal-directed analysis + +4. **Verification**: + - All tools implement the `Tool` interface + - Unit tests for each tool + - Build succeeds without errors + +## What We're NOT Doing + +- **Pickle file support** - Python interop adds complexity; defer to later phase +- **PPTX support** - unioffice library is commercial/complex; start with PDF and DOCX +- **DataAnalystAgent sub-agent** - Keep it simple; add tools directly to existing sub-researchers +- **Visualization generation** - No chart/image generation; focus on text-based analysis +- **Large file streaming** - Use reasonable size limits (10MB) and fail gracefully + +## Implementation Approach + +Follow the `SearchTool` + `ContentSummarizer` pattern: +1. Create standalone tools that work without LLM +2. Add optional LLM enhancement for deeper analysis +3. Register tools in `SubResearcherToolRegistry()` +4. Update prompts to document new tools + +--- + +## Phase 1: PDF Reading Tool + +### Overview + +Implement a PDF text extraction tool using the `pdfcpu` library (pure Go, no CGO required). + +### Changes Required + +#### 1. Add pdfcpu dependency + +**Command**: +```bash +go get github.com/pdfcpu/pdfcpu +``` + +This adds the PDF processing library to `go.mod`. + +#### 2. Create PDF Tool + +**File**: `internal/tools/pdf.go` + +```go +package tools + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/pdfcpu/pdfcpu/pkg/api" + "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" +) + +// PDFReadTool extracts text content from PDF files. +type PDFReadTool struct { + maxPages int // Maximum pages to extract (0 = all) +} + +// NewPDFReadTool creates a new PDF reading tool. +func NewPDFReadTool() *PDFReadTool { + return &PDFReadTool{ + maxPages: 50, // Default: first 50 pages + } +} + +func (t *PDFReadTool) Name() string { + return "read_pdf" +} + +func (t *PDFReadTool) Description() string { + return `Extract text from a PDF file. Args: {"path": "/path/to/file.pdf"}` +} + +func (t *PDFReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("read_pdf requires a 'path' argument") + } + + // Validate file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + + // Open PDF file + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open PDF: %w", err) + } + defer f.Close() + + // Create configuration + conf := model.NewDefaultConfiguration() + + // Extract text from PDF + var text strings.Builder + + // Use pdfcpu's ExtractContent or similar API + // Note: pdfcpu's text extraction API may need adjustment based on actual API + content, err := api.ExtractContentFile(path, nil, conf) + if err != nil { + return "", fmt.Errorf("extract PDF content: %w", err) + } + + for pageNum, pageContent := range content { + if t.maxPages > 0 && pageNum >= t.maxPages { + text.WriteString(fmt.Sprintf("\n...[truncated after %d pages]\n", t.maxPages)) + break + } + text.WriteString(fmt.Sprintf("--- Page %d ---\n", pageNum+1)) + text.WriteString(pageContent) + text.WriteString("\n\n") + } + + result := text.String() + + // Truncate if too long + const maxLen = 100000 + if len(result) > maxLen { + result = result[:maxLen] + "\n...[truncated]" + } + + return result, nil +} +``` + +**Note**: The actual pdfcpu API may differ. The implementation should be adjusted after testing with the real library. An alternative approach using `ledongthuc/pdf` (pure Go) may be simpler: + +```go +// Alternative using ledongthuc/pdf +import "github.com/ledongthuc/pdf" + +func (t *PDFReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("read_pdf requires a 'path' argument") + } + + f, r, err := pdf.Open(path) + if err != nil { + return "", fmt.Errorf("open PDF: %w", err) + } + defer f.Close() + + var text strings.Builder + numPages := r.NumPage() + maxPages := t.maxPages + if maxPages <= 0 || maxPages > numPages { + maxPages = numPages + } + + for i := 1; i <= maxPages; i++ { + p := r.Page(i) + if p.V.IsNull() { + continue + } + content, err := p.GetPlainText(nil) + if err != nil { + continue + } + text.WriteString(fmt.Sprintf("--- Page %d ---\n", i)) + text.WriteString(content) + text.WriteString("\n\n") + } + + if maxPages < numPages { + text.WriteString(fmt.Sprintf("\n...[truncated after %d of %d pages]\n", maxPages, numPages)) + } + + result := text.String() + const maxLen = 100000 + if len(result) > maxLen { + result = result[:maxLen] + "\n...[truncated]" + } + + return result, nil +} +``` + +#### 3. Create PDF Tool Tests + +**File**: `internal/tools/pdf_test.go` + +```go +package tools + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestPDFReadTool_Name(t *testing.T) { + tool := NewPDFReadTool() + if tool.Name() != "read_pdf" { + t.Errorf("expected name 'read_pdf', got '%s'", tool.Name()) + } +} + +func TestPDFReadTool_Description(t *testing.T) { + tool := NewPDFReadTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestPDFReadTool_Execute_MissingPath(t *testing.T) { + tool := NewPDFReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestPDFReadTool_Execute_FileNotFound(t *testing.T) { + tool := NewPDFReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.pdf", + }) + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +// Integration test with a real PDF file +func TestPDFReadTool_Execute_RealFile(t *testing.T) { + // Skip if no test PDF available + testPDF := filepath.Join("testdata", "sample.pdf") + if _, err := os.Stat(testPDF); os.IsNotExist(err) { + t.Skip("no test PDF file available") + } + + tool := NewPDFReadTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": testPDF, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == "" { + t.Error("expected non-empty result") + } +} +``` + +### Success Criteria + +#### Automated Verification: +- [x] Build succeeds: `cd go-research && go build ./...` +- [x] Tests pass: `cd go-research && go test ./internal/tools/...` +- [x] No linting errors: `cd go-research && golangci-lint run ./...` (if available) + +#### Manual Verification: +- [ ] Create a test PDF and verify text extraction works +- [ ] Large PDFs are truncated appropriately + +--- + +## Phase 2: DOCX Reading Tool + +### Overview + +Implement a DOCX text extraction tool. Since unioffice is commercial, use `baliance/gooxml` (Apache 2.0 license, predecessor to unioffice) or a simpler approach. + +### Changes Required + +#### 1. Add docx dependency + +**Command**: +```bash +go get github.com/nguyenthenguyen/docx +``` + +This is a lightweight, pure Go DOCX reader. + +#### 2. Create DOCX Tool + +**File**: `internal/tools/docx.go` + +```go +package tools + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/nguyenthenguyen/docx" +) + +// DOCXReadTool extracts text content from DOCX files. +type DOCXReadTool struct{} + +// NewDOCXReadTool creates a new DOCX reading tool. +func NewDOCXReadTool() *DOCXReadTool { + return &DOCXReadTool{} +} + +func (t *DOCXReadTool) Name() string { + return "read_docx" +} + +func (t *DOCXReadTool) Description() string { + return `Extract text from a DOCX (Word) file. Args: {"path": "/path/to/file.docx"}` +} + +func (t *DOCXReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("read_docx requires a 'path' argument") + } + + // Validate file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + + // Open DOCX file + r, err := docx.ReadDocxFile(path) + if err != nil { + return "", fmt.Errorf("open DOCX: %w", err) + } + defer r.Close() + + // Extract text content + doc := r.Editable() + content := doc.GetContent() + + // Clean up whitespace + content = cleanDocxContent(content) + + // Truncate if too long + const maxLen = 100000 + if len(content) > maxLen { + content = content[:maxLen] + "\n...[truncated]" + } + + return content, nil +} + +// cleanDocxContent normalizes whitespace and formatting in extracted DOCX text. +func cleanDocxContent(s string) string { + // Replace multiple newlines with double newline (paragraph separator) + lines := strings.Split(s, "\n") + var cleaned []string + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + cleaned = append(cleaned, trimmed) + } + } + return strings.Join(cleaned, "\n\n") +} +``` + +#### 3. Create DOCX Tool Tests + +**File**: `internal/tools/docx_test.go` + +```go +package tools + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestDOCXReadTool_Name(t *testing.T) { + tool := NewDOCXReadTool() + if tool.Name() != "read_docx" { + t.Errorf("expected name 'read_docx', got '%s'", tool.Name()) + } +} + +func TestDOCXReadTool_Description(t *testing.T) { + tool := NewDOCXReadTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestDOCXReadTool_Execute_MissingPath(t *testing.T) { + tool := NewDOCXReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestDOCXReadTool_Execute_FileNotFound(t *testing.T) { + tool := NewDOCXReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.docx", + }) + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +// Integration test with a real DOCX file +func TestDOCXReadTool_Execute_RealFile(t *testing.T) { + testDOCX := filepath.Join("testdata", "sample.docx") + if _, err := os.Stat(testDOCX); os.IsNotExist(err) { + t.Skip("no test DOCX file available") + } + + tool := NewDOCXReadTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": testDOCX, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == "" { + t.Error("expected non-empty result") + } +} +``` + +### Success Criteria + +#### Automated Verification: +- [x] Build succeeds: `cd go-research && go build ./...` +- [x] Tests pass: `cd go-research && go test ./internal/tools/...` + +#### Manual Verification: +- [ ] Create a test DOCX and verify text extraction works +- [ ] Complex DOCX with tables/formatting extracts readable text + +--- + +## Phase 3: Generic Document Reader Tool + +### Overview + +Create a unified document reading tool that auto-detects format from file extension. + +### Changes Required + +#### 1. Create Document Tool + +**File**: `internal/tools/document.go` + +```go +package tools + +import ( + "context" + "fmt" + "path/filepath" + "strings" +) + +// DocumentReadTool reads documents of various formats (PDF, DOCX). +// It auto-detects the format based on file extension. +type DocumentReadTool struct { + pdfTool *PDFReadTool + docxTool *DOCXReadTool +} + +// NewDocumentReadTool creates a new document reading tool. +func NewDocumentReadTool() *DocumentReadTool { + return &DocumentReadTool{ + pdfTool: NewPDFReadTool(), + docxTool: NewDOCXReadTool(), + } +} + +func (t *DocumentReadTool) Name() string { + return "read_document" +} + +func (t *DocumentReadTool) Description() string { + return `Read and extract text from a document file (PDF or DOCX - auto-detected from extension). +Args: {"path": "/path/to/document.pdf"}` +} + +func (t *DocumentReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("read_document requires a 'path' argument") + } + + ext := strings.ToLower(filepath.Ext(path)) + + switch ext { + case ".pdf": + return t.pdfTool.Execute(ctx, args) + case ".docx": + return t.docxTool.Execute(ctx, args) + default: + return "", fmt.Errorf("unsupported file format: %s (supported: .pdf, .docx)", ext) + } +} +``` + +#### 2. Create Document Tool Tests + +**File**: `internal/tools/document_test.go` + +```go +package tools + +import ( + "context" + "testing" +) + +func TestDocumentReadTool_Name(t *testing.T) { + tool := NewDocumentReadTool() + if tool.Name() != "read_document" { + t.Errorf("expected name 'read_document', got '%s'", tool.Name()) + } +} + +func TestDocumentReadTool_Description(t *testing.T) { + tool := NewDocumentReadTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestDocumentReadTool_Execute_MissingPath(t *testing.T) { + tool := NewDocumentReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestDocumentReadTool_Execute_UnsupportedFormat(t *testing.T) { + tool := NewDocumentReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/path/to/file.txt", + }) + if err == nil { + t.Error("expected error for unsupported format") + } +} + +func TestDocumentReadTool_Execute_DetectsPDF(t *testing.T) { + tool := NewDocumentReadTool() + // This will fail with "file not found" which proves PDF detection worked + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.pdf", + }) + if err == nil || err.Error() != "file not found: /nonexistent/file.pdf" { + // PDF tool was called (correct detection) + } +} + +func TestDocumentReadTool_Execute_DetectsDOCX(t *testing.T) { + tool := NewDocumentReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.docx", + }) + if err == nil || err.Error() != "file not found: /nonexistent/file.docx" { + // DOCX tool was called (correct detection) + } +} +``` + +### Success Criteria + +#### Automated Verification: +- [x] Build succeeds: `cd go-research && go build ./...` +- [x] Tests pass: `cd go-research && go test ./internal/tools/...` + +#### Manual Verification: +- [x] Tool correctly routes PDF files to PDF reader +- [x] Tool correctly routes DOCX files to DOCX reader +- [x] Unsupported formats return clear error message + +--- + +## Phase 4: CSV Analysis Tool + +### Overview + +Implement a CSV analysis tool that performs exploratory data analysis (EDA) including shape, data types, summary statistics, and correlations. + +### Changes Required + +#### 1. Add statistics dependency + +**Command**: +```bash +go get gonum.org/v1/gonum/stat +go get github.com/montanaflynn/stats +``` + +These provide statistical functions for analysis. + +#### 2. Create CSV Analysis Tool + +**File**: `internal/tools/csv_analysis.go` + +```go +package tools + +import ( + "context" + "encoding/csv" + "fmt" + "os" + "sort" + "strconv" + "strings" + + "github.com/montanaflynn/stats" +) + +// CSVAnalysisTool performs exploratory data analysis on CSV files. +type CSVAnalysisTool struct { + maxRows int // Maximum rows to analyze (0 = all) +} + +// NewCSVAnalysisTool creates a new CSV analysis tool. +func NewCSVAnalysisTool() *CSVAnalysisTool { + return &CSVAnalysisTool{ + maxRows: 10000, // Default: analyze first 10k rows + } +} + +func (t *CSVAnalysisTool) Name() string { + return "analyze_csv" +} + +func (t *CSVAnalysisTool) Description() string { + return `Analyze a CSV data file. Performs EDA including: shape, column types, summary statistics, missing values. +Args: {"path": "/path/to/file.csv", "goal": "optional analysis objective"}` +} + +func (t *CSVAnalysisTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || path == "" { + return "", fmt.Errorf("analyze_csv requires a 'path' argument") + } + + goal, _ := args["goal"].(string) + + // Validate file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + + // Open CSV file + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open CSV: %w", err) + } + defer f.Close() + + // Parse CSV + reader := csv.NewReader(f) + records, err := reader.ReadAll() + if err != nil { + return "", fmt.Errorf("parse CSV: %w", err) + } + + if len(records) == 0 { + return "Empty CSV file.", nil + } + + // Extract headers and data + headers := records[0] + data := records[1:] + + // Limit rows if needed + if t.maxRows > 0 && len(data) > t.maxRows { + data = data[:t.maxRows] + } + + // Build analysis report + var report strings.Builder + + // Header + report.WriteString(fmt.Sprintf("# CSV Analysis: %s\n\n", path)) + if goal != "" { + report.WriteString(fmt.Sprintf("**Analysis Goal**: %s\n\n", goal)) + } + + // Shape + report.WriteString(fmt.Sprintf("## Shape\n")) + report.WriteString(fmt.Sprintf("- Rows: %d (showing first %d)\n", len(records)-1, len(data))) + report.WriteString(fmt.Sprintf("- Columns: %d\n\n", len(headers))) + + // Columns overview + report.WriteString("## Columns\n") + report.WriteString("| Column | Type | Non-Null | Missing | Sample Values |\n") + report.WriteString("|--------|------|----------|---------|---------------|\n") + + columnData := make([][]string, len(headers)) + for i := range headers { + columnData[i] = make([]string, len(data)) + for j, row := range data { + if i < len(row) { + columnData[i][j] = row[i] + } + } + } + + for i, header := range headers { + colType, nonNull, missing := analyzeColumn(columnData[i]) + samples := getSampleValues(columnData[i], 3) + report.WriteString(fmt.Sprintf("| %s | %s | %d | %d | %s |\n", + header, colType, nonNull, missing, samples)) + } + report.WriteString("\n") + + // Summary statistics for numeric columns + report.WriteString("## Summary Statistics (Numeric Columns)\n") + hasNumeric := false + for i, header := range headers { + if isNumericColumn(columnData[i]) { + hasNumeric = true + numericStats := computeNumericStats(columnData[i]) + report.WriteString(fmt.Sprintf("### %s\n", header)) + report.WriteString(fmt.Sprintf("- Count: %d\n", numericStats.count)) + report.WriteString(fmt.Sprintf("- Mean: %.4f\n", numericStats.mean)) + report.WriteString(fmt.Sprintf("- Std: %.4f\n", numericStats.std)) + report.WriteString(fmt.Sprintf("- Min: %.4f\n", numericStats.min)) + report.WriteString(fmt.Sprintf("- 25%%: %.4f\n", numericStats.q25)) + report.WriteString(fmt.Sprintf("- 50%% (Median): %.4f\n", numericStats.median)) + report.WriteString(fmt.Sprintf("- 75%%: %.4f\n", numericStats.q75)) + report.WriteString(fmt.Sprintf("- Max: %.4f\n\n", numericStats.max)) + } + } + if !hasNumeric { + report.WriteString("No numeric columns detected.\n\n") + } + + // Categorical column summaries + report.WriteString("## Categorical Column Value Counts\n") + hasCategorical := false + for i, header := range headers { + if !isNumericColumn(columnData[i]) { + hasCategorical = true + valueCounts := getValueCounts(columnData[i], 10) + report.WriteString(fmt.Sprintf("### %s\n", header)) + for _, vc := range valueCounts { + report.WriteString(fmt.Sprintf("- %s: %d\n", vc.value, vc.count)) + } + report.WriteString("\n") + } + } + if !hasCategorical { + report.WriteString("No categorical columns detected.\n\n") + } + + return report.String(), nil +} + +// Column analysis helpers + +func analyzeColumn(col []string) (colType string, nonNull, missing int) { + for _, val := range col { + if val == "" { + missing++ + } else { + nonNull++ + } + } + + if isNumericColumn(col) { + colType = "numeric" + } else { + colType = "string" + } + + return +} + +func isNumericColumn(col []string) bool { + numericCount := 0 + totalNonEmpty := 0 + + for _, val := range col { + if val == "" { + continue + } + totalNonEmpty++ + if _, err := strconv.ParseFloat(val, 64); err == nil { + numericCount++ + } + } + + if totalNonEmpty == 0 { + return false + } + + // Consider numeric if >80% of non-empty values are numeric + return float64(numericCount)/float64(totalNonEmpty) > 0.8 +} + +func getSampleValues(col []string, n int) string { + seen := make(map[string]bool) + var samples []string + + for _, val := range col { + if val != "" && !seen[val] { + seen[val] = true + samples = append(samples, val) + if len(samples) >= n { + break + } + } + } + + return strings.Join(samples, ", ") +} + +type numericStats struct { + count int + mean float64 + std float64 + min float64 + q25 float64 + median float64 + q75 float64 + max float64 +} + +func computeNumericStats(col []string) numericStats { + var values []float64 + for _, val := range col { + if f, err := strconv.ParseFloat(val, 64); err == nil { + values = append(values, f) + } + } + + if len(values) == 0 { + return numericStats{} + } + + mean, _ := stats.Mean(values) + std, _ := stats.StandardDeviation(values) + min, _ := stats.Min(values) + max, _ := stats.Max(values) + median, _ := stats.Median(values) + q25, _ := stats.Percentile(values, 25) + q75, _ := stats.Percentile(values, 75) + + return numericStats{ + count: len(values), + mean: mean, + std: std, + min: min, + q25: q25, + median: median, + q75: q75, + max: max, + } +} + +type valueCount struct { + value string + count int +} + +func getValueCounts(col []string, limit int) []valueCount { + counts := make(map[string]int) + for _, val := range col { + if val != "" { + counts[val]++ + } + } + + var result []valueCount + for v, c := range counts { + result = append(result, valueCount{value: v, count: c}) + } + + // Sort by count descending + sort.Slice(result, func(i, j int) bool { + return result[i].count > result[j].count + }) + + if len(result) > limit { + result = result[:limit] + } + + return result +} +``` + +#### 3. Create CSV Tool Tests + +**File**: `internal/tools/csv_analysis_test.go` + +```go +package tools + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCSVAnalysisTool_Name(t *testing.T) { + tool := NewCSVAnalysisTool() + if tool.Name() != "analyze_csv" { + t.Errorf("expected name 'analyze_csv', got '%s'", tool.Name()) + } +} + +func TestCSVAnalysisTool_Description(t *testing.T) { + tool := NewCSVAnalysisTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestCSVAnalysisTool_Execute_MissingPath(t *testing.T) { + tool := NewCSVAnalysisTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestCSVAnalysisTool_Execute_FileNotFound(t *testing.T) { + tool := NewCSVAnalysisTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.csv", + }) + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +func TestCSVAnalysisTool_Execute_SimpleCSV(t *testing.T) { + // Create temp CSV file + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "test.csv") + + csvContent := `name,age,score +Alice,30,85.5 +Bob,25,92.3 +Charlie,35,78.9 +Diana,28,88.0 +` + if err := os.WriteFile(csvPath, []byte(csvContent), 0644); err != nil { + t.Fatalf("failed to create test CSV: %v", err) + } + + tool := NewCSVAnalysisTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": csvPath, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify report contains expected sections + if !strings.Contains(result, "## Shape") { + t.Error("report missing Shape section") + } + if !strings.Contains(result, "## Columns") { + t.Error("report missing Columns section") + } + if !strings.Contains(result, "## Summary Statistics") { + t.Error("report missing Summary Statistics section") + } + if !strings.Contains(result, "Rows: 4") { + t.Error("incorrect row count") + } + if !strings.Contains(result, "Columns: 3") { + t.Error("incorrect column count") + } +} + +func TestCSVAnalysisTool_Execute_WithGoal(t *testing.T) { + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "test.csv") + + csvContent := `x,y +1,2 +3,4 +` + if err := os.WriteFile(csvPath, []byte(csvContent), 0644); err != nil { + t.Fatalf("failed to create test CSV: %v", err) + } + + tool := NewCSVAnalysisTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": csvPath, + "goal": "Find correlation between x and y", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !strings.Contains(result, "Find correlation between x and y") { + t.Error("goal not included in report") + } +} + +func TestIsNumericColumn(t *testing.T) { + tests := []struct { + col []string + expected bool + }{ + {[]string{"1", "2", "3"}, true}, + {[]string{"1.5", "2.7", "3.9"}, true}, + {[]string{"a", "b", "c"}, false}, + {[]string{"1", "2", ""}, true}, + {[]string{"1", "a", "b"}, false}, + {[]string{}, false}, + } + + for _, tt := range tests { + result := isNumericColumn(tt.col) + if result != tt.expected { + t.Errorf("isNumericColumn(%v) = %v, expected %v", tt.col, result, tt.expected) + } + } +} +``` + +### Success Criteria + +#### Automated Verification: +- [x] Build succeeds: `cd go-research && go build ./...` +- [x] Tests pass: `cd go-research && go test ./internal/tools/...` + +#### Manual Verification: +- [x] Tool correctly identifies numeric vs string columns +- [x] Summary statistics are accurate +- [ ] Large CSV files are handled without memory issues + +--- + +## Phase 5: Tool Registry Integration + +### Overview + +Register the new tools in `SubResearcherToolRegistry()` and update the prompts to document them. + +### Changes Required + +#### 1. Update SubResearcherToolRegistry + +**File**: `internal/think_deep/tools.go` + +**Changes**: Add document and CSV tools to the sub-researcher registry + +```go +// SubResearcherToolRegistry creates a tool registry for sub-researcher agents. +// This includes search (with optional summarization), fetch, document reading, CSV analysis, and think tools. +// If client is provided, search results will include LLM-generated summaries of page content. +func SubResearcherToolRegistry(braveAPIKey string, client ...llm.ChatClient) *tools.Registry { + registry := tools.NewEmptyRegistry() + + // Create search tool with optional summarization + searchTool := tools.NewSearchTool(braveAPIKey) + if len(client) > 0 && client[0] != nil { + summarizer := tools.NewContentSummarizer(client[0]) + searchTool.SetSummarizer(summarizer) + } + registry.Register(searchTool) + + // Add fetch tool (for direct URL fetching if needed) + registry.Register(tools.NewFetchTool()) + + // Add document reading tools + registry.Register(tools.NewDocumentReadTool()) + + // Add CSV analysis tool + registry.Register(tools.NewCSVAnalysisTool()) + + // Add think tool for reflection + registry.Register(&ThinkTool{}) + + return registry +} +``` + +#### 2. Update ResearchAgentPrompt + +**File**: `internal/think_deep/prompts.go` + +**Changes**: Update the `ResearchAgentPrompt` function to document new tools + +```go +// ResearchAgentPrompt returns the prompt for sub-researcher agents. +// Sub-researchers perform focused searches with strict iteration limits. +// +// Original: research_agent_prompt +func ResearchAgentPrompt(date string) string { + return fmt.Sprintf(`You are a research assistant conducting research on the user's input topic. For context, today's date is %s. + + +Your job is to use tools to gather information about the user's input topic. +You can use any of the tools provided to you to find resources that can help answer the research question. You can call these tools in series or in parallel, your research is conducted in a tool-calling loop. + + + +You have access to the following tools: + +1. **search**: For conducting web searches to gather information + Usage: {"query": "your search query"} + +2. **fetch**: For fetching content from a specific URL + Usage: {"url": "https://example.com"} + +3. **read_document**: For reading PDF or DOCX documents + Usage: {"path": "/path/to/document.pdf"} + +4. **analyze_csv**: For analyzing CSV data files (EDA, statistics, column analysis) + Usage: {"path": "/path/to/data.csv", "goal": "what to look for"} + +5. **think**: For reflection and strategic planning during research + Usage: {"reflection": "Your analysis of results..."} + +**CRITICAL: Use think after each search or analysis to reflect on results and plan next steps** + + + +Think like a human researcher with limited time. Follow these steps: + +1. **Read the question carefully** - What specific information does the user need? +2. **Start with broader searches** - Use broad, comprehensive queries first +3. **After each search, pause and assess** - Do I have enough to answer? What's still missing? +4. **Execute narrower searches as you gather information** - Fill in the gaps +5. **For data analysis tasks** - Use analyze_csv for CSV files, read_document for PDFs/DOCX +6. **Stop when you can answer confidently** - Don't keep searching for perfection + + + +**Tool Call Budgets** (Prevent excessive searching): +- **Simple queries**: Use 2-3 search tool calls maximum +- **Complex queries**: Use up to 5 search tool calls maximum +- **Always stop**: After 5 search tool calls if you cannot find the right sources + +**Stop Immediately When**: +- You can answer the user's question comprehensively +- You have 3+ relevant examples/sources for the question +- Your last 2 searches returned similar information + + + +After each search tool call, use think to analyze the results: +- What key information did I find? +- What's missing? +- Do I have enough to answer the question comprehensively? +- Should I search more or provide my answer? + + +When you have gathered sufficient information, provide your findings without using any tool tags.`, date) +} +``` + +#### 3. Update LeadResearcherPrompt (Optional) + +**File**: `internal/think_deep/prompts.go` + +**Changes**: Add brief mention of data analysis capabilities in supervisor prompt + +Add to the `` section: + +```go +**Data Analysis Tasks** require specialized handling: +- *Example*: Analyze the sales data in /data/sales.csv → Delegate to sub-agent with clear analysis goal +- When delegating data analysis, specify the file path AND the analysis objective clearly +``` + +### Success Criteria + +#### Automated Verification: +- [x] Build succeeds: `cd go-research && go build ./...` +- [x] Tests pass: `cd go-research && go test ./...` +- [x] Type check passes: `cd go-research && go vet ./...` + +#### Manual Verification: +- [x] New tools appear in sub-researcher tool registry +- [x] Updated prompts include documentation for new tools +- [ ] Sub-researcher can call document and CSV tools during research + +--- + +## Phase 6: End-to-End Testing + +### Overview + +Create integration tests that verify the new tools work correctly within the ThinkDeep research flow. + +### Changes Required + +#### 1. Create Test Data Directory + +**Directory**: `internal/tools/testdata/` + +Create test files: +- `testdata/sample.pdf` - Simple PDF with text content +- `testdata/sample.docx` - Simple DOCX with text content +- `testdata/sample.csv` - CSV with numeric and string columns + +#### 2. Create Integration Test + +**File**: `internal/think_deep/tools_integration_test.go` + +```go +package think_deep + +import ( + "context" + "os" + "path/filepath" + "testing" + + "go-research/internal/tools" +) + +func TestSubResearcherToolRegistry_HasAllTools(t *testing.T) { + // Use empty API key for test (search won't actually be called) + registry := SubResearcherToolRegistry("") + + expectedTools := []string{ + "search", + "fetch", + "read_document", + "analyze_csv", + "think", + } + + registeredTools := registry.ToolNames() + + for _, expected := range expectedTools { + found := false + for _, registered := range registeredTools { + if registered == expected { + found = true + break + } + } + if !found { + t.Errorf("missing expected tool: %s", expected) + } + } +} + +func TestSubResearcherToolRegistry_ExecuteDocumentTool(t *testing.T) { + registry := SubResearcherToolRegistry("") + + // Test that read_document tool exists and can be executed (will fail on missing file) + _, err := registry.Execute(context.Background(), "read_document", map[string]interface{}{ + "path": "/nonexistent/file.pdf", + }) + + // We expect a "file not found" error, which proves the tool was found and executed + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +func TestSubResearcherToolRegistry_ExecuteCSVTool(t *testing.T) { + registry := SubResearcherToolRegistry("") + + // Create temp CSV + tmpDir := t.TempDir() + csvPath := filepath.Join(tmpDir, "test.csv") + csvContent := "a,b\n1,2\n3,4\n" + if err := os.WriteFile(csvPath, []byte(csvContent), 0644); err != nil { + t.Fatalf("failed to create test CSV: %v", err) + } + + result, err := registry.Execute(context.Background(), "analyze_csv", map[string]interface{}{ + "path": csvPath, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if result == "" { + t.Error("expected non-empty result") + } +} +``` + +### Success Criteria + +#### Automated Verification: +- [x] Build succeeds: `cd go-research && go build ./...` +- [x] All tests pass: `cd go-research && go test ./...` +- [x] Tool registry contains all expected tools + +#### Manual Verification: +- [ ] End-to-end test with real PDF/DOCX/CSV files +- [ ] Sub-researcher can use document tools during research session +- [ ] Performance is acceptable with moderately sized files (< 10MB) + +--- + +## Testing Strategy + +### Unit Tests + +Each tool has dedicated unit tests covering: +- Name and description methods +- Missing/invalid arguments +- File not found scenarios +- Basic functionality with test data + +### Integration Tests + +- Tool registry contains all expected tools +- Tools can be executed through the registry +- Tools work within the sub-researcher context + +### Manual Testing Steps + +1. **PDF Reading**: + - Create a simple PDF with text + - Execute: `read_document` with PDF path + - Verify extracted text is readable + +2. **DOCX Reading**: + - Create a simple DOCX with text + - Execute: `read_document` with DOCX path + - Verify extracted text is readable + +3. **CSV Analysis**: + - Create CSV with mixed column types + - Execute: `analyze_csv` with CSV path + - Verify shape, types, and statistics are correct + +4. **End-to-End Research**: + - Start a research session that requires reading a local document + - Verify sub-researcher can access and use document tools + - Verify findings are incorporated into research output + +--- + +## Performance Considerations + +- **File Size Limits**: All tools have max length limits (100KB for documents, 10K rows for CSV) to prevent memory issues +- **Truncation**: Large outputs are truncated with clear indicators +- **Lazy Loading**: Documents are only read when the tool is called, not at registry creation +- **No Caching**: Each tool call reads the file fresh (caching could be added later if needed) + +--- + +## Migration Notes + +No migration required - these are additive changes that don't affect existing functionality. + +--- + +## References + +- Original research document: `thoughts/shared/research/2025-12-03_think-deep-data-tools.md` +- Tool interface: `internal/tools/registry.go:9-13` +- SearchTool with summarizer pattern: `internal/tools/search.go:17-35` +- SubResearcherToolRegistry: `internal/think_deep/tools.go:246-267` +- ResearchAgentPrompt: `internal/think_deep/prompts.go:95-145` +- LeadResearcherPrompt: `internal/think_deep/prompts.go:16-89` From ccb1746d0ba9bf7d38f594950d9e805d29ca77fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Thu, 4 Dec 2025 16:39:49 +0100 Subject: [PATCH 3/5] feat: Add XLSX document reading and enhance data analysis tools - Introduced XLSX reading capabilities with the new `read_xlsx` tool for extracting data from Excel files. - Updated the `DocumentReadTool` to support XLSX format alongside PDF and DOCX. - Enhanced the `analyze_csv` tool for exploratory data analysis, ensuring compatibility with new document formats. - Implemented integration tests for XLSX reading and CSV analysis, validating functionality with real datasets. - Updated documentation to reflect new tools and usage instructions, ensuring clarity for users. --- CLAUDE.md | 6 + go-research/go.mod | 8 + go-research/go.sum | 25 +++ go-research/internal/agents/sub_researcher.go | 16 +- .../agents/sub_researcher_integration_test.go | 172 ++++++++++++++++++ go-research/internal/e2e/e2e_test.go | 97 ++++++++++ go-research/internal/think_deep/prompts.go | 3 +- go-research/internal/tools/document.go | 10 +- go-research/internal/tools/document_test.go | 22 +++ go-research/internal/tools/xlsx.go | 147 +++++++++++++++ go-research/internal/tools/xlsx_test.go | 132 ++++++++++++++ .../data-analysis-tools-implementation.md | 0 .../plans/interactive-cli-agentic-research.md | 0 ...-12-03_interactive-cli-agentic-research.md | 0 .../2025-12-03_think-deep-data-tools.md | 0 15 files changed, 633 insertions(+), 5 deletions(-) create mode 100644 go-research/internal/agents/sub_researcher_integration_test.go create mode 100644 go-research/internal/tools/xlsx.go create mode 100644 go-research/internal/tools/xlsx_test.go rename {go-research/thoughts => thoughts}/shared/plans/data-analysis-tools-implementation.md (100%) rename {go-research/thoughts => thoughts}/shared/plans/interactive-cli-agentic-research.md (100%) rename {go-research/thoughts => thoughts}/shared/research/2025-12-03_interactive-cli-agentic-research.md (100%) rename {go-research/thoughts => thoughts}/shared/research/2025-12-03_think-deep-data-tools.md (100%) diff --git a/CLAUDE.md b/CLAUDE.md index 8b1ef55..f5ec408 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -267,3 +267,9 @@ export default async function PostPage({ params }) { - Guidance on concise `CLAUDE.md` files — see community write-ups like the Apidog "Claude.md" overview for keeping this file focused and high-signal Keep this document up to date as the source of truth for how this blog is structured and extended. + + + +# Thoughts: + +All thoughs (even in sub-projects) should be in thoughts/shared/ (root), not in the sub projects. \ No newline at end of file diff --git a/go-research/go.mod b/go-research/go.mod index 590bff4..cca11f2 100644 --- a/go-research/go.mod +++ b/go-research/go.mod @@ -10,6 +10,7 @@ require ( github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 github.com/montanaflynn/stats v0.7.1 github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db + github.com/xuri/excelize/v2 v2.8.1 golang.org/x/net v0.30.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -17,5 +18,12 @@ require ( require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect + github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect + golang.org/x/crypto v0.28.0 // indirect golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.19.0 // indirect ) diff --git a/go-research/go.sum b/go-research/go.sum index 21ccbf6..258449c 100644 --- a/go-research/go.sum +++ b/go-research/go.sum @@ -4,6 +4,8 @@ github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -17,10 +19,31 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db h1:v0cW/tTMrJQyZr7r6t+t9+NhH2OBAjydHisVYxuyObc= github.com/nguyenthenguyen/docx v0.0.0-20230621112118-9c8e795a11db/go.mod h1:BZyH8oba3hE/BTt2FfBDGPOHhXiKs9RFmUvvXRdzrhM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= +github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0= +github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.8.1 h1:pZLMEwK8ep+CLIUWpWmvW8IWE/yxqG0I1xcN6cVMGuQ= +github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluUhG7c+CEE= +github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4= +github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -28,6 +51,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go-research/internal/agents/sub_researcher.go b/go-research/internal/agents/sub_researcher.go index 5847dee..9502f3f 100644 --- a/go-research/internal/agents/sub_researcher.go +++ b/go-research/internal/agents/sub_researcher.go @@ -171,7 +171,21 @@ func (r *SubResearcherAgent) researchWithIteration(ctx context.Context, topic st result = fmt.Sprintf("Reflection recorded: %s", truncateForLog(reflection, 100)) default: - result = fmt.Sprintf("Unknown tool: %s", tc.Tool) + // Execute any other tool from the registry (read_document, analyze_csv, fetch, etc.) + r.emitProgress(researcherNum, diffusionIteration, fmt.Sprintf("using %s", tc.Tool), len(state.RawNotes), topic) + toolResult, toolErr := r.tools.Execute(ctx, tc.Tool, tc.Args) + if toolErr != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + result = fmt.Sprintf("Tool error (%s): %v", tc.Tool, toolErr) + } else { + result = toolResult + // Add to raw notes if it's a data-gathering tool (not think) + if tc.Tool != "think" { + state.AddRawNote(toolResult) + } + } } // Add tool result to conversation diff --git a/go-research/internal/agents/sub_researcher_integration_test.go b/go-research/internal/agents/sub_researcher_integration_test.go new file mode 100644 index 0000000..fd714c8 --- /dev/null +++ b/go-research/internal/agents/sub_researcher_integration_test.go @@ -0,0 +1,172 @@ +package agents + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "go-research/internal/config" + "go-research/internal/events" + "go-research/internal/llm" + "go-research/internal/think_deep" + "go-research/internal/tools" + + "github.com/joho/godotenv" +) + +type loggedToolCall struct { + name string + args map[string]interface{} +} + +type loggingToolExecutor struct { + exec tools.ToolExecutor + calls []loggedToolCall +} + +func (l *loggingToolExecutor) Execute(ctx context.Context, name string, args map[string]interface{}) (string, error) { + argCopy := make(map[string]interface{}, len(args)) + for k, v := range args { + argCopy[k] = v + } + l.calls = append(l.calls, loggedToolCall{name: name, args: argCopy}) + return l.exec.Execute(ctx, name, args) +} + +func (l *loggingToolExecutor) ToolNames() []string { + return l.exec.ToolNames() +} + +// To run this integration test (requires real OPENROUTER_API_KEY + BRAVE_API_KEY): +// +// go test -run TestSubResearcher_UsesDocumentReaderOnAMDataset -timeout 30m ./internal/agents -v +func TestSubResearcher_UsesDocumentReaderOnAMDataset(t *testing.T) { + // Load .env from repo root explicitly + root := repoRoot(t) + envPath := filepath.Join(root, ".env") + if err := godotenv.Load(envPath); err != nil { + t.Logf("Note: .env file not found at %s (will use environment variables): %v", envPath, err) + } + + cfg := config.Load() + cfg.OpenRouterAPIKey = strings.TrimSpace(cfg.OpenRouterAPIKey) + cfg.BraveAPIKey = strings.TrimSpace(cfg.BraveAPIKey) + if cfg.OpenRouterAPIKey == "" { + t.Fatalf("OPENROUTER_API_KEY must be set (can be placed in %s)", envPath) + } + if cfg.BraveAPIKey == "" { + t.Fatalf("BRAVE_API_KEY must be set (can be placed in %s)", envPath) + } + + dataset := datasetPath(t) + + client := llm.NewClient(cfg) + registry := think_deep.SubResearcherToolRegistry(cfg.BraveAPIKey, client) + loggingRegistry := &loggingToolExecutor{exec: registry} + + bus := events.NewBus(32) + defer bus.Close() + + agent := NewSubResearcherAgent(client, loggingRegistry, bus, DefaultSubResearcherConfig()) + + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + defer cancel() + + query := fmt.Sprintf( + "MANDATORY TASK: You MUST call the read_document tool immediately. "+ + "This is NOT optional - you cannot answer without calling the tool.\n\n"+ + "File location: %s\n\n"+ + "EXACT TOOL CALL REQUIRED:\n"+ + "{\"path\": \"%s\"}\n\n"+ + "After calling read_document, extract from the Excel file:\n"+ + "- Unemployment numbers for 'inrikes födda men' (domestic-born males) ages 15-24\n"+ + "- Values for years 2005 and 2024\n"+ + "- Report the exact numbers you see in the spreadsheet\n\n"+ + "CRITICAL: You MUST make the tool call above. Do NOT skip it. Do NOT answer from memory.", + dataset, dataset, + ) + + result, err := agent.Research(ctx, query, 1) + if err != nil { + t.Fatalf("sub researcher failed: %v", err) + } + + if result == nil || result.CompressedResearch == "" { + t.Fatalf("expected non-empty compressed research result") + } + + // Log all tool calls and raw notes for debugging + t.Logf("Tool calls made: %d", len(loggingRegistry.calls)) + for i, call := range loggingRegistry.calls { + t.Logf(" Call %d: %s with args: %+v", i+1, call.name, call.args) + } + t.Logf("Raw notes count: %d", len(result.RawNotes)) + for i, note := range result.RawNotes { + preview := note + if len(preview) > 200 { + preview = preview[:200] + "..." + } + t.Logf(" Raw note %d: %s", i+1, preview) + } + + // Fail immediately if no tool calls were made - agent didn't follow instructions + if len(loggingRegistry.calls) == 0 { + t.Fatalf("CRITICAL: Agent made ZERO tool calls. This means it didn't follow instructions to use read_document. Compressed research: %s", result.CompressedResearch) + } + + var readDocUsed bool + var readDocPath string + for _, call := range loggingRegistry.calls { + if call.name == "read_document" { + if path, ok := call.args["path"].(string); ok { + readDocPath = path + if filepath.Clean(path) == filepath.Clean(dataset) { + readDocUsed = true + break + } + } + } + } + + if !readDocUsed { + t.Logf("Compressed research output:\n%s", result.CompressedResearch) + if readDocPath != "" { + t.Fatalf("read_document was called but with wrong path. Expected: %s, Got: %s. All calls: %#v", dataset, readDocPath, loggingRegistry.calls) + } + t.Fatalf("expected read_document tool to be invoked with dataset %s, but it was never called. All calls: %#v", dataset, loggingRegistry.calls) + } + + if !strings.Contains(result.CompressedResearch, "52.7") { + t.Fatalf("expected compressed research to include concrete numbers from dataset (e.g., 52.7). got:\n%s", result.CompressedResearch) + } +} + +func datasetPath(t *testing.T) string { + t.Helper() + root := repoRoot(t) + return filepath.Join(root, "internal", "data", "AM0401U1_20251204-100847.xlsx") +} + +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd failed: %v", err) + } + + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("could not locate go.mod starting from %s", dir) + } + dir = parent + } +} diff --git a/go-research/internal/e2e/e2e_test.go b/go-research/internal/e2e/e2e_test.go index 3e847dc..06171fa 100644 --- a/go-research/internal/e2e/e2e_test.go +++ b/go-research/internal/e2e/e2e_test.go @@ -19,6 +19,7 @@ import ( "go-research/internal/repl" "go-research/internal/repl/handlers" "go-research/internal/session" + "go-research/internal/tools" ) // ============================================================================= @@ -533,6 +534,102 @@ func TestFastResearchMultipleTools(t *testing.T) { } } +// To run this Excel-focused scenario manually (non-interactive): +// +// go test -run TestFastResearchWorkflow_AnalyzesExcelDataset -timeout 20m ./internal/e2e -v +func TestFastResearchWorkflow_AnalyzesExcelDataset(t *testing.T) { + cfg := testConfig() + defer os.RemoveAll(filepath.Dir(cfg.StateFile)) + + excelPath := datasetPath(t) + if _, err := os.Stat(excelPath); os.IsNotExist(err) { + t.Skipf("sample Excel dataset not found at %s", excelPath) + } + + bus := events.NewBus(100) + defer bus.Close() + + mockLLM := NewMockLLMClient( + fmt.Sprintf(`I should review Statistics Sweden's labor data to answer the immigration and employment question. +{"path": "%s"}`, excelPath), + ` +Sweden's AM0401U1 dataset shows that unemployed inrikesfödda men (15-24) dropped from 52.7k in 2005 to around 57.3k in 2024 after a pandemic spike. +It also highlights 71.8k unemployed in 25-54 age group during 2005, underscoring generational challenges. +These figures indicate that newcomers still face barriers in integration programs, so policymakers plan targeted employment guarantees. +`, + ) + + registry := tools.NewEmptyRegistry() + registry.Register(tools.NewDocumentReadTool()) + + agentCfg := agent.DefaultConfig(cfg) + agentCfg.Client = mockLLM + agentCfg.Tools = registry + + reactAgent := agent.NewReact(agentCfg, bus) + + ctx := context.Background() + result, err := reactAgent.Research(ctx, "How is Sweden handling immigration and employment integration using the Stats Sweden AM0401U1 dataset?") + if err != nil { + t.Fatalf("Research failed: %v", err) + } + + if result.Status != session.WorkerComplete { + t.Fatalf("Expected worker to complete, got %v", result.Status) + } + + if len(result.ToolCalls) != 1 { + t.Fatalf("Expected exactly one tool call, got %d", len(result.ToolCalls)) + } + + call := result.ToolCalls[0] + if call.Tool != "read_document" { + t.Fatalf("Expected read_document tool call, got %s", call.Tool) + } + + toolPreview := call.Result + if len(toolPreview) > 200 { + toolPreview = toolPreview[:200] + } + + if !strings.Contains(call.Result, "Arbetslösa 15-74 år") { + t.Fatalf("Expected tool output to include dataset heading, got: %s", toolPreview) + } + + if !strings.Contains(call.Result, "52.7") || !strings.Contains(call.Result, "71.8") { + t.Fatalf("Expected tool output to include numeric data (52.7, 71.8), got: %s", toolPreview) + } + + if !strings.Contains(result.FinalOutput, "52.7") { + t.Fatalf("Expected final answer to mention extracted figures, got: %s", result.FinalOutput) + } +} + +func datasetPath(t *testing.T) string { + t.Helper() + root := repoRoot(t) + return filepath.Join(root, "internal", "data", "AM0401U1_20251204-100847.xlsx") +} + +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd failed: %v", err) + } + + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("go.mod not found from %s", dir) + } + dir = parent + } +} + // ============================================================================= // Mock Agent for Worker Pool // ============================================================================= diff --git a/go-research/internal/think_deep/prompts.go b/go-research/internal/think_deep/prompts.go index 90deb16..c8a6162 100644 --- a/go-research/internal/think_deep/prompts.go +++ b/go-research/internal/think_deep/prompts.go @@ -113,8 +113,9 @@ You have access to the following tools: 2. **fetch**: For fetching content from a specific URL Usage: {"url": "https://example.com"} -3. **read_document**: For reading PDF or DOCX documents +3. **read_document**: For reading documents (PDF, DOCX, XLSX) Usage: {"path": "/path/to/document.pdf"} + Supports: PDF files, DOCX (Word) files, and XLSX (Excel) files 4. **analyze_csv**: For analyzing CSV data files (EDA, statistics, column analysis) Usage: {"path": "/path/to/data.csv", "goal": "what to look for"} diff --git a/go-research/internal/tools/document.go b/go-research/internal/tools/document.go index 9035a88..bb079b9 100644 --- a/go-research/internal/tools/document.go +++ b/go-research/internal/tools/document.go @@ -7,11 +7,12 @@ import ( "strings" ) -// DocumentReadTool reads documents of various formats (PDF, DOCX). +// DocumentReadTool reads documents of various formats (PDF, DOCX, XLSX). // It auto-detects the format based on file extension. type DocumentReadTool struct { pdfTool *PDFReadTool docxTool *DOCXReadTool + xlsxTool *XLSXReadTool } // NewDocumentReadTool creates a new document reading tool. @@ -19,6 +20,7 @@ func NewDocumentReadTool() *DocumentReadTool { return &DocumentReadTool{ pdfTool: NewPDFReadTool(), docxTool: NewDOCXReadTool(), + xlsxTool: NewXLSXReadTool(), } } @@ -27,7 +29,7 @@ func (t *DocumentReadTool) Name() string { } func (t *DocumentReadTool) Description() string { - return `Read and extract text from a document file (PDF or DOCX - auto-detected from extension). + return `Read and extract text from a document file (PDF, DOCX, XLSX - auto-detected from extension). Args: {"path": "/path/to/document.pdf"}` } @@ -44,7 +46,9 @@ func (t *DocumentReadTool) Execute(ctx context.Context, args map[string]interfac return t.pdfTool.Execute(ctx, args) case ".docx": return t.docxTool.Execute(ctx, args) + case ".xlsx": + return t.xlsxTool.Execute(ctx, args) default: - return "", fmt.Errorf("unsupported file format: %s (supported: .pdf, .docx)", ext) + return "", fmt.Errorf("unsupported file format: %s (supported: .pdf, .docx, .xlsx)", ext) } } diff --git a/go-research/internal/tools/document_test.go b/go-research/internal/tools/document_test.go index 3565912..1d6bea1 100644 --- a/go-research/internal/tools/document_test.go +++ b/go-research/internal/tools/document_test.go @@ -70,6 +70,20 @@ func TestDocumentReadTool_Execute_DetectsDOCX(t *testing.T) { } } +func TestDocumentReadTool_Execute_DetectsXLSX(t *testing.T) { + tool := NewDocumentReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.xlsx", + }) + if err == nil { + t.Error("expected error") + } + // XLSX tool returns "file not found" which proves correct routing + if !strings.Contains(err.Error(), "file not found") { + t.Errorf("expected 'file not found' error (proving XLSX routing), got: %v", err) + } +} + func TestDocumentReadTool_Execute_CaseInsensitiveExtension(t *testing.T) { tool := NewDocumentReadTool() @@ -88,4 +102,12 @@ func TestDocumentReadTool_Execute_CaseInsensitiveExtension(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "file not found") { t.Error("expected DOCX tool to handle .DocX extension") } + + // Test uppercase XLSX + _, err = tool.Execute(context.Background(), map[string]interface{}{ + "path": "/nonexistent/file.XLSX", + }) + if err == nil || !strings.Contains(err.Error(), "file not found") { + t.Error("expected XLSX tool to handle .XLSX extension") + } } diff --git a/go-research/internal/tools/xlsx.go b/go-research/internal/tools/xlsx.go new file mode 100644 index 0000000..923f0d5 --- /dev/null +++ b/go-research/internal/tools/xlsx.go @@ -0,0 +1,147 @@ +package tools + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/xuri/excelize/v2" +) + +// XLSXReadTool extracts a textual preview from Excel workbooks. +// It focuses on the first N sheets/rows/columns to keep responses concise. +type XLSXReadTool struct { + maxSheets int + maxRowsPerSheet int + maxColsPerRow int +} + +// NewXLSXReadTool creates a new XLSX reading tool with sane limits. +func NewXLSXReadTool() *XLSXReadTool { + return &XLSXReadTool{ + maxSheets: 3, + maxRowsPerSheet: 20, + maxColsPerRow: 12, + } +} + +func (t *XLSXReadTool) Name() string { + return "read_xlsx" +} + +func (t *XLSXReadTool) Description() string { + return `Extract a textual summary from an Excel (.xlsx) workbook. Args: {"path": "/path/to/file.xlsx"}` +} + +func (t *XLSXReadTool) Execute(ctx context.Context, args map[string]interface{}) (string, error) { + path, ok := args["path"].(string) + if !ok || strings.TrimSpace(path) == "" { + return "", fmt.Errorf("read_xlsx requires a 'path' argument") + } + + if _, err := os.Stat(path); os.IsNotExist(err) { + return "", fmt.Errorf("file not found: %s", path) + } + + f, err := excelize.OpenFile(path) + if err != nil { + return "", fmt.Errorf("open XLSX: %w", err) + } + defer func() { + _ = f.Close() + }() + + sheets := f.GetSheetList() + if len(sheets) == 0 { + return fmt.Sprintf("Workbook %s contains no sheets.", filepath.Base(path)), nil + } + + var b strings.Builder + b.WriteString(fmt.Sprintf("Workbook: %s\n", filepath.Base(path))) + b.WriteString(fmt.Sprintf("Total sheets: %d\n\n", len(sheets))) + + maxSheets := t.maxSheets + if maxSheets <= 0 || maxSheets > len(sheets) { + maxSheets = len(sheets) + } + + for i := 0; i < maxSheets; i++ { + select { + case <-ctx.Done(): + return "", ctx.Err() + default: + } + + sheetName := sheets[i] + b.WriteString(fmt.Sprintf("=== Sheet %d: %s ===\n", i+1, sheetName)) + + rows, err := f.GetRows(sheetName) + if err != nil { + b.WriteString(fmt.Sprintf("error reading sheet: %v\n\n", err)) + continue + } + + if len(rows) == 0 { + b.WriteString("(sheet is empty)\n\n") + continue + } + + maxRows := t.maxRowsPerSheet + if maxRows <= 0 || maxRows > len(rows) { + maxRows = len(rows) + } + + for rowIdx := 0; rowIdx < maxRows; rowIdx++ { + row := rows[rowIdx] + formatted := formatXLSXRow(row, t.maxColsPerRow) + b.WriteString(fmt.Sprintf("Row %d: %s\n", rowIdx+1, formatted)) + } + + if maxRows < len(rows) { + b.WriteString(fmt.Sprintf("...%d more rows not shown\n", len(rows)-maxRows)) + } + + b.WriteString("\n") + } + + if maxSheets < len(sheets) { + b.WriteString(fmt.Sprintf("...%d additional sheets not shown\n", len(sheets)-maxSheets)) + } + + result := b.String() + const maxLen = 100000 + if len(result) > maxLen { + result = result[:maxLen] + "\n...[truncated]" + } + + return result, nil +} + +func formatXLSXRow(row []string, maxCols int) string { + if len(row) == 0 { + return "[empty row]" + } + + maxColumns := len(row) + if maxCols > 0 && maxCols < maxColumns { + maxColumns = maxCols + } + + values := make([]string, 0, maxColumns) + for i := 0; i < maxColumns; i++ { + cell := strings.TrimSpace(row[i]) + if cell == "" { + cell = " " + } + values = append(values, cell) + } + + line := strings.Join(values, " | ") + if maxCols > 0 && len(row) > maxCols { + line += " | ..." + } + + return line +} diff --git a/go-research/internal/tools/xlsx_test.go b/go-research/internal/tools/xlsx_test.go new file mode 100644 index 0000000..231ed98 --- /dev/null +++ b/go-research/internal/tools/xlsx_test.go @@ -0,0 +1,132 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xuri/excelize/v2" +) + +func TestXLSXReadTool_Name(t *testing.T) { + tool := NewXLSXReadTool() + if tool.Name() != "read_xlsx" { + t.Errorf("expected name 'read_xlsx', got '%s'", tool.Name()) + } +} + +func TestXLSXReadTool_Description(t *testing.T) { + tool := NewXLSXReadTool() + if tool.Description() == "" { + t.Error("description should not be empty") + } +} + +func TestXLSXReadTool_Execute_MissingPath(t *testing.T) { + tool := NewXLSXReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{}) + if err == nil { + t.Error("expected error for missing path") + } +} + +func TestXLSXReadTool_Execute_FileNotFound(t *testing.T) { + tool := NewXLSXReadTool() + _, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": "/does/not/exist.xlsx", + }) + if err == nil { + t.Error("expected error for nonexistent file") + } + if !strings.Contains(err.Error(), "file not found") { + t.Errorf("expected 'file not found' error, got: %v", err) + } +} + +func TestXLSXReadTool_Execute_RealFile(t *testing.T) { + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "sample.xlsx") + + f := excelize.NewFile() + defer func() { + _ = f.Close() + }() + + f.SetCellValue("Sheet1", "A1", "Hello") + f.SetCellValue("Sheet1", "B2", 123) + f.NewSheet("Summary") + f.SetCellValue("Summary", "A1", "Total") + f.SetCellValue("Summary", "B1", 42) + + if err := f.SaveAs(tmpFile); err != nil { + t.Fatalf("failed saving test workbook: %v", err) + } + + tool := NewXLSXReadTool() + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": tmpFile, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(result, "Hello") || !strings.Contains(result, "Summary") { + t.Errorf("expected output to include workbook content, got:\n%s", result) + } +} + +func TestXLSXReadTool_Execute_AMDataset(t *testing.T) { + tool := NewXLSXReadTool() + excelPath := datasetPath(t) + + if _, err := os.Stat(excelPath); os.IsNotExist(err) { + t.Skipf("sample dataset not found at %s", excelPath) + } + + result, err := tool.Execute(context.Background(), map[string]interface{}{ + "path": excelPath, + }) + if err != nil { + t.Fatalf("unexpected error reading dataset: %v", err) + } + + preview := result + if len(preview) > 200 { + preview = preview[:200] + } + + if !strings.Contains(result, "Arbetslösa 15-74 år") { + t.Fatalf("expected dataset heading in output, got: %s", preview) + } + + if !strings.Contains(result, "52.7") || !strings.Contains(result, "71.8") { + t.Fatalf("expected dataset numeric values (52.7, 71.8), got: %s", preview) + } +} + +func datasetPath(t *testing.T) string { + t.Helper() + root := repoRoot(t) + return filepath.Join(root, "internal", "data", "AM0401U1_20251204-100847.xlsx") +} + +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd failed: %v", err) + } + + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("go.mod not found from %s", dir) + } + dir = parent + } +} diff --git a/go-research/thoughts/shared/plans/data-analysis-tools-implementation.md b/thoughts/shared/plans/data-analysis-tools-implementation.md similarity index 100% rename from go-research/thoughts/shared/plans/data-analysis-tools-implementation.md rename to thoughts/shared/plans/data-analysis-tools-implementation.md diff --git a/go-research/thoughts/shared/plans/interactive-cli-agentic-research.md b/thoughts/shared/plans/interactive-cli-agentic-research.md similarity index 100% rename from go-research/thoughts/shared/plans/interactive-cli-agentic-research.md rename to thoughts/shared/plans/interactive-cli-agentic-research.md diff --git a/go-research/thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md b/thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md similarity index 100% rename from go-research/thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md rename to thoughts/shared/research/2025-12-03_interactive-cli-agentic-research.md diff --git a/go-research/thoughts/shared/research/2025-12-03_think-deep-data-tools.md b/thoughts/shared/research/2025-12-03_think-deep-data-tools.md similarity index 100% rename from go-research/thoughts/shared/research/2025-12-03_think-deep-data-tools.md rename to thoughts/shared/research/2025-12-03_think-deep-data-tools.md From 64e4bd4b11aecf93b618c713580469f35e26db26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 5 Dec 2025 09:43:00 +0100 Subject: [PATCH 4/5] feat: Enhance insights extraction and cancellation handling in research process - Improved the `extractInsightsFromSearchResults` function to support detailed source extraction, including individual source blocks, titles, summaries, and URLs. - Introduced a new `CancelReason` type and related constants to track cancellation reasons during research operations. - Updated the REPL context to handle cancellation with specific reasons, enhancing user feedback and logging. - Enhanced the `WriteInsight` function to include additional metadata such as data points, analysis chain, and source links for better traceability. - Implemented comprehensive tests to validate new functionalities and ensure reliability in insights extraction and cancellation handling. --- go-research/internal/agents/sub_researcher.go | 301 +++++++++++++++--- go-research/internal/events/types.go | 28 ++ go-research/internal/obsidian/writer.go | 289 ++++++++++++++++- .../internal/repl/handlers/architectures.go | 4 + go-research/internal/repl/handlers/expand.go | 12 + go-research/internal/repl/handlers/start.go | 4 + go-research/internal/repl/repl.go | 65 +++- go-research/internal/repl/router.go | 47 ++- go-research/internal/session/cost_test.go | 1 + go-research/internal/think_deep/state.go | 95 +++++- go-research/internal/tools/xlsx.go | 4 + 11 files changed, 769 insertions(+), 81 deletions(-) diff --git a/go-research/internal/agents/sub_researcher.go b/go-research/internal/agents/sub_researcher.go index 9502f3f..8120dd2 100644 --- a/go-research/internal/agents/sub_researcher.go +++ b/go-research/internal/agents/sub_researcher.go @@ -324,68 +324,283 @@ func truncateForLog(s string, maxLen int) string { // extractInsightsFromSearchResults extracts structured insights from search results. // It parses the raw search results and creates SubInsight structures for each // distinct finding with source attribution. +// Enhanced to extract individual sources from search result blocks and capture +// tool usage, query context, and full source references. func extractInsightsFromSearchResults(topic string, rawNotes []string, researcherNum int, iteration int) []think_deep.SubInsight { var insights []think_deep.SubInsight insightNum := 0 - // URL extraction regex - urlRegex := regexp.MustCompile(`URL:\s*(https?://[^\s]+)`) - // Title extraction regex (looks for Title: or ## patterns) - titleRegex := regexp.MustCompile(`(?:Title:|##)\s*(.+?)(?:\n|$)`) - // Snippet/content extraction regex - snippetRegex := regexp.MustCompile(`(?:Snippet:|Content:)\s*(.+?)(?:\n\n|$)`) - for _, note := range rawNotes { - // Extract URL from this note - urlMatches := urlRegex.FindStringSubmatch(note) - sourceURL := "" - if len(urlMatches) > 1 { - sourceURL = strings.TrimSpace(urlMatches[1]) + // Try to split by SOURCE markers (search results typically have multiple sources) + sourceBlocks := splitIntoSourceBlocks(note) + + if len(sourceBlocks) == 0 { + // No source blocks found, treat entire note as one insight + sourceBlocks = []sourceBlock{{content: note}} } - // Extract title - titleMatches := titleRegex.FindStringSubmatch(note) - title := "" - if len(titleMatches) > 1 { - title = strings.TrimSpace(titleMatches[1]) + for _, block := range sourceBlocks { + insightNum++ + insight := createInsightFromBlock(block, topic, insightNum, researcherNum, iteration, note) + if insight != nil { + insights = append(insights, *insight) + } + } + } + + return insights +} + +// sourceBlock represents a parsed source from search results +type sourceBlock struct { + url string + title string + summary string + content string +} + +// splitIntoSourceBlocks parses search result text into individual source blocks. +// Search results typically have format: +// --- SOURCE N: Title --- +// URL: https://... +// SUMMARY: ... +func splitIntoSourceBlocks(note string) []sourceBlock { + var blocks []sourceBlock + + // Pattern to match source blocks: "--- SOURCE N: Title ---" or "SOURCE N:" + sourcePattern := regexp.MustCompile(`(?i)(?:---\s*)?SOURCE\s*\d+:?\s*([^-\n]*?)(?:\s*---)?`) + urlPattern := regexp.MustCompile(`URL:\s*(https?://[^\s\n]+)`) + summaryPattern := regexp.MustCompile(`(?i)SUMMARY:\s*(.+?)(?:\n\n|\n---|\z)`) + titlePattern := regexp.MustCompile(`(?:Title:|##)\s*(.+?)(?:\n|$)`) + + // Split by SOURCE markers + parts := sourcePattern.Split(note, -1) + titles := sourcePattern.FindAllStringSubmatch(note, -1) + + for i, part := range parts { + if strings.TrimSpace(part) == "" { + continue } - // Extract snippet/content for the finding - snippetMatches := snippetRegex.FindStringSubmatch(note) - finding := "" - if len(snippetMatches) > 1 { - finding = strings.TrimSpace(snippetMatches[1]) + block := sourceBlock{content: part} + + // Extract title from the marker + if i > 0 && i-1 < len(titles) && len(titles[i-1]) > 1 { + block.title = strings.TrimSpace(titles[i-1][1]) } - // If we couldn't extract structured content, use truncated raw note - if finding == "" && len(note) > 0 { - finding = truncateForLog(note, 500) + // Extract URL + if urlMatch := urlPattern.FindStringSubmatch(part); len(urlMatch) > 1 { + block.url = strings.TrimSpace(urlMatch[1]) } - // Only create insight if we have meaningful content - if finding == "" { - continue + // Extract summary + if summaryMatch := summaryPattern.FindStringSubmatch(part); len(summaryMatch) > 1 { + block.summary = strings.TrimSpace(summaryMatch[1]) } - insightNum++ - insight := think_deep.SubInsight{ - ID: fmt.Sprintf("insight-%03d", insightNum), - Topic: topic, - Title: title, - Finding: finding, - Implication: "", // Will be filled by synthesis later - SourceURL: sourceURL, - SourceContent: truncateForLog(note, 1000), - Confidence: calculateConfidence(sourceURL, finding), - Iteration: iteration, - ResearcherNum: researcherNum, - Timestamp: time.Now(), + // If no title from marker, try to extract from content + if block.title == "" { + if titleMatch := titlePattern.FindStringSubmatch(part); len(titleMatch) > 1 { + block.title = strings.TrimSpace(titleMatch[1]) + } } - insights = append(insights, insight) + // Only add if we have meaningful content + if block.url != "" || block.summary != "" || len(strings.TrimSpace(part)) > 50 { + blocks = append(blocks, block) + } } - return insights + // If no blocks extracted, try to extract individual URLs and their surrounding content + if len(blocks) == 0 { + urlMatches := urlPattern.FindAllStringSubmatchIndex(note, -1) + for i, match := range urlMatches { + if len(match) < 4 { + continue + } + + url := note[match[2]:match[3]] + // Get content around the URL (100 chars before, 500 after) + start := match[0] - 100 + if start < 0 { + start = 0 + } + end := match[3] + 500 + if end > len(note) { + end = len(note) + } + // Don't overlap with next URL + if i+1 < len(urlMatches) && end > urlMatches[i+1][0] { + end = urlMatches[i+1][0] + } + + content := note[start:end] + blocks = append(blocks, sourceBlock{ + url: url, + content: content, + }) + } + } + + return blocks +} + +// createInsightFromBlock creates a SubInsight from a parsed source block +func createInsightFromBlock(block sourceBlock, topic string, insightNum int, researcherNum int, iteration int, fullNote string) *think_deep.SubInsight { + // Determine the finding text + finding := block.summary + if finding == "" { + finding = truncateForLog(block.content, 500) + } + + // Skip if no meaningful content + if finding == "" || len(strings.TrimSpace(finding)) < 20 { + return nil + } + + // Extract tool and query from the note if present + toolUsed, queryUsed := extractToolContext(fullNote) + + // Determine source type based on tool used or content + sourceType := think_deep.SourceTypeWeb + if toolUsed == "read_document" || toolUsed == "read_xlsx" || toolUsed == "analyze_csv" { + sourceType = think_deep.SourceTypeDocument + } else if toolUsed == "fetch" { + sourceType = think_deep.SourceTypeWeb + } else if strings.Contains(fullNote, "Read document:") || strings.Contains(fullNote, "Workbook:") { + sourceType = think_deep.SourceTypeDocument + } + + // Create source reference with full content + var sources []think_deep.SourceReference + if block.url != "" { + sources = append(sources, think_deep.SourceReference{ + URL: block.url, + Type: sourceType, + Title: block.title, + RelevantExcerpt: block.summary, + RawContent: block.content, + FetchedAt: time.Now(), + }) + } else if sourceType == think_deep.SourceTypeDocument && queryUsed != "" { + // For document sources, create a file-based source reference + sources = append(sources, think_deep.SourceReference{ + FilePath: queryUsed, + Type: sourceType, + Title: block.title, + RelevantExcerpt: block.summary, + RawContent: block.content, + FetchedAt: time.Now(), + }) + } + + // Generate title if not present + title := block.title + if title == "" && block.url != "" { + // Use domain as title + title = extractDomain(block.url) + } + if title == "" && sourceType == think_deep.SourceTypeDocument && queryUsed != "" { + // Use filename as title for documents + parts := strings.Split(queryUsed, "/") + if len(parts) > 0 { + title = parts[len(parts)-1] + } + } + if title == "" { + title = truncateForLog(finding, 60) + } + + // Create analysis chain showing derivation + analysisChain := []string{ + fmt.Sprintf("Research topic: %s", truncateForLog(topic, 100)), + } + if toolUsed != "" { + analysisChain = append(analysisChain, fmt.Sprintf("Tool used: %s", toolUsed)) + } + if queryUsed != "" { + if sourceType == think_deep.SourceTypeDocument { + analysisChain = append(analysisChain, fmt.Sprintf("Document analyzed: %s", queryUsed)) + } else { + analysisChain = append(analysisChain, fmt.Sprintf("Query: %s", queryUsed)) + } + } + if block.url != "" { + analysisChain = append(analysisChain, fmt.Sprintf("Source retrieved: %s", block.url)) + } + analysisChain = append(analysisChain, "Finding extracted from source content") + + // For document sources, use file path as source URL for tracking + sourceURL := block.url + if sourceURL == "" && sourceType == think_deep.SourceTypeDocument && queryUsed != "" { + sourceURL = "file://" + queryUsed + } + + return &think_deep.SubInsight{ + ID: fmt.Sprintf("insight-%03d", insightNum), + Topic: topic, + Title: title, + Finding: finding, + Implication: "", // Will be filled by synthesis later + SourceURL: sourceURL, + SourceContent: block.content, + Sources: sources, + AnalysisChain: analysisChain, + ToolUsed: toolUsed, + QueryUsed: queryUsed, + Confidence: calculateConfidence(sourceURL, finding), + Iteration: iteration, + ResearcherNum: researcherNum, + Timestamp: time.Now(), + } +} + +// extractToolContext extracts tool name and query from the note context +func extractToolContext(note string) (tool string, query string) { + // Try to detect search queries + searchPattern := regexp.MustCompile(`(?i)Search results for:\s*(.+?)(?:\n|$)`) + if match := searchPattern.FindStringSubmatch(note); len(match) > 1 { + return "search", strings.TrimSpace(match[1]) + } + + // Try to detect fetch operations + fetchPattern := regexp.MustCompile(`(?i)Fetched from:\s*(https?://[^\s]+)`) + if match := fetchPattern.FindStringSubmatch(note); len(match) > 1 { + return "fetch", strings.TrimSpace(match[1]) + } + + // Try to detect document reads (matches "Read document: /path/to/file") + docPattern := regexp.MustCompile(`(?i)Read document:\s*(.+?)(?:\n|$)`) + if match := docPattern.FindStringSubmatch(note); len(match) > 1 { + return "read_document", strings.TrimSpace(match[1]) + } + + // Try to detect XLSX reads + xlsxPattern := regexp.MustCompile(`(?i)Workbook:\s*(.+?)(?:\n|$)`) + if match := xlsxPattern.FindStringSubmatch(note); len(match) > 1 { + return "read_xlsx", strings.TrimSpace(match[1]) + } + + // Try to detect CSV analysis + csvPattern := regexp.MustCompile(`(?i)CSV Analysis:\s*(.+?)(?:\n|$)`) + if match := csvPattern.FindStringSubmatch(note); len(match) > 1 { + return "analyze_csv", strings.TrimSpace(match[1]) + } + + return "", "" +} + +// extractDomain extracts the domain from a URL for use as a title +func extractDomain(urlStr string) string { + // Simple extraction - just get host + if strings.HasPrefix(urlStr, "http") { + parts := strings.Split(urlStr, "/") + if len(parts) >= 3 { + return parts[2] + } + } + return "" } // calculateConfidence estimates confidence score based on source quality indicators. diff --git a/go-research/internal/events/types.go b/go-research/internal/events/types.go index f672b9d..e50cf57 100644 --- a/go-research/internal/events/types.go +++ b/go-research/internal/events/types.go @@ -73,6 +73,25 @@ const ( EventDiffusionComplete // Diffusion phase complete EventFinalReportStarted // Full optimization phase started EventFinalReportComplete // Final report generated + + // Cancellation event + EventResearchCancelled // Research was cancelled with a reason +) + +// CancelReason indicates why research was cancelled +type CancelReason string + +const ( + // CancelReasonUserInterrupt means the user pressed Ctrl+C + CancelReasonUserInterrupt CancelReason = "user_interrupt" + // CancelReasonTimeout means the research exceeded the time limit + CancelReasonTimeout CancelReason = "timeout" + // CancelReasonParentCancelled means the parent context was cancelled + CancelReasonParentCancelled CancelReason = "parent_cancelled" + // CancelReasonShutdown means the system is shutting down + CancelReasonShutdown CancelReason = "shutdown" + // CancelReasonUnknown means the cancellation reason could not be determined + CancelReasonUnknown CancelReason = "unknown" ) // ResearchStartedData contains data for research start events @@ -229,3 +248,12 @@ type DraftRefinedData struct { NewSources int // New sources incorporated Progress float64 // Estimated progress (0.0-1.0) } + +// ResearchCancelledData captures why research was cancelled +type ResearchCancelledData struct { + SessionID string // Session that was cancelled + Query string // Original query + Reason CancelReason // Why it was cancelled + Phase string // Phase when cancelled (brief, draft, diffuse, finalize) + Message string // Human-readable cancellation message +} diff --git a/go-research/internal/obsidian/writer.go b/go-research/internal/obsidian/writer.go index 28df950..780e8c3 100644 --- a/go-research/internal/obsidian/writer.go +++ b/go-research/internal/obsidian/writer.go @@ -2,9 +2,13 @@ package obsidian import ( "bytes" + "crypto/sha256" "fmt" + "net/url" "os" "path/filepath" + "regexp" + "strings" "text/template" "time" @@ -134,19 +138,171 @@ func (w *Writer) GetReportPath(sess *session.Session) string { return filepath.Join(w.vaultPath, sess.ID, "reports", fmt.Sprintf("report_v%d.md", sess.Version)) } +// WriteSource writes a source file to the sources directory. +// Returns the generated filename for linking from insights. +func (w *Writer) WriteSource(sessionDir string, sourceRef think_deep.SourceReference, index int) (string, error) { + // Generate safe filename from URL or file path + var safeName string + if sourceRef.URL != "" { + safeName = sanitizeFilename(sourceRef.URL) + } else if sourceRef.FilePath != "" { + safeName = sanitizeFilename(filepath.Base(sourceRef.FilePath)) + } else { + safeName = fmt.Sprintf("source_%03d", index) + } + + filename := filepath.Join(sessionDir, "sources", fmt.Sprintf("%s.md", safeName)) + + frontmatter := map[string]interface{}{ + "source_type": string(sourceRef.Type), + "fetched_at": sourceRef.FetchedAt.Format(time.RFC3339), + } + if sourceRef.URL != "" { + frontmatter["url"] = sourceRef.URL + } + if sourceRef.FilePath != "" { + frontmatter["file_path"] = sourceRef.FilePath + } + if sourceRef.Title != "" { + frontmatter["title"] = sourceRef.Title + } + if sourceRef.ContentHash != "" { + frontmatter["content_hash"] = sourceRef.ContentHash + } + + fm, err := yaml.Marshal(frontmatter) + if err != nil { + return "", fmt.Errorf("marshal frontmatter: %w", err) + } + + var content bytes.Buffer + content.WriteString("---\n") + content.Write(fm) + content.WriteString("---\n\n") + + // Title + title := sourceRef.Title + if title == "" && sourceRef.URL != "" { + title = sourceRef.URL + } else if title == "" { + title = fmt.Sprintf("Source %d", index) + } + content.WriteString(fmt.Sprintf("# %s\n\n", title)) + + // Source info + content.WriteString("## Source Information\n\n") + content.WriteString(fmt.Sprintf("- **Type**: %s\n", sourceRef.Type)) + if sourceRef.URL != "" { + content.WriteString(fmt.Sprintf("- **URL**: [%s](%s)\n", sourceRef.URL, sourceRef.URL)) + } + if sourceRef.FilePath != "" { + content.WriteString(fmt.Sprintf("- **File**: `%s`\n", sourceRef.FilePath)) + } + content.WriteString(fmt.Sprintf("- **Fetched**: %s\n", sourceRef.FetchedAt.Format(time.RFC3339))) + content.WriteString("\n") + + // Relevant excerpt (highlighted) + if sourceRef.RelevantExcerpt != "" { + content.WriteString("## Relevant Excerpt\n\n") + content.WriteString("> ") + content.WriteString(strings.ReplaceAll(sourceRef.RelevantExcerpt, "\n", "\n> ")) + content.WriteString("\n\n") + } + + // Full raw content + if sourceRef.RawContent != "" { + content.WriteString("## Full Content\n\n") + content.WriteString("```\n") + // Truncate very long content but keep it substantial + rawContent := sourceRef.RawContent + if len(rawContent) > 50000 { + rawContent = rawContent[:50000] + "\n\n... [truncated, total length: " + fmt.Sprintf("%d", len(sourceRef.RawContent)) + " chars]" + } + content.WriteString(rawContent) + content.WriteString("\n```\n") + } + + if err := os.WriteFile(filename, content.Bytes(), 0644); err != nil { + return "", err + } + + return safeName, nil +} + +// WriteSources writes all sources from insights to the sources directory. +// Returns a map of source URL/path to filename for linking. +func (w *Writer) WriteSources(sessionDir string, insights []think_deep.SubInsight) (map[string]string, error) { + sourceMap := make(map[string]string) + sourceIndex := 0 + + // Collect all unique sources from insights + for _, insight := range insights { + // Handle embedded sources in the insight + for _, src := range insight.Sources { + key := src.URL + if key == "" { + key = src.FilePath + } + if key == "" || sourceMap[key] != "" { + continue // Skip empty or already written + } + + sourceIndex++ + filename, err := w.WriteSource(sessionDir, src, sourceIndex) + if err != nil { + // Log but don't fail - continue with other sources + continue + } + sourceMap[key] = filename + } + + // Also create source from legacy SourceURL/SourceContent if not already covered + if insight.SourceURL != "" && sourceMap[insight.SourceURL] == "" { + sourceIndex++ + src := think_deep.SourceReference{ + URL: insight.SourceURL, + Type: think_deep.SourceTypeWeb, + RelevantExcerpt: insight.SourceContent, + FetchedAt: insight.Timestamp, + } + filename, err := w.WriteSource(sessionDir, src, sourceIndex) + if err == nil { + sourceMap[insight.SourceURL] = filename + } + } + } + + return sourceMap, nil +} + // WriteInsight writes a single insight to the insights directory. -func (w *Writer) WriteInsight(sessionDir string, insight think_deep.SubInsight, index int) error { +// Enhanced to include data points, analysis chain, and source links. +func (w *Writer) WriteInsight(sessionDir string, insight think_deep.SubInsight, index int, sourceMap map[string]string) error { filename := filepath.Join(sessionDir, "insights", fmt.Sprintf("insight_%03d.md", index)) frontmatter := map[string]interface{}{ "insight_id": insight.ID, "topic": insight.Topic, "confidence": fmt.Sprintf("%.2f", insight.Confidence), - "source_url": insight.SourceURL, "iteration": insight.Iteration, "researcher": insight.ResearcherNum, "timestamp": insight.Timestamp.Format(time.RFC3339), } + if insight.SourceURL != "" { + frontmatter["source_url"] = insight.SourceURL + } + if insight.ToolUsed != "" { + frontmatter["tool_used"] = insight.ToolUsed + } + if len(insight.Sources) > 0 { + frontmatter["source_count"] = len(insight.Sources) + } + if len(insight.DataPoints) > 0 { + frontmatter["data_point_count"] = len(insight.DataPoints) + } + if len(insight.RelatedInsightIDs) > 0 { + frontmatter["related_insights"] = insight.RelatedInsightIDs + } fm, err := yaml.Marshal(frontmatter) if err != nil { @@ -177,31 +333,142 @@ func (w *Writer) WriteInsight(sessionDir string, insight think_deep.SubInsight, content.WriteString("\n\n") } - // Source - content.WriteString("## Source\n\n") - if insight.SourceURL != "" { - content.WriteString(fmt.Sprintf("- [%s](%s)\n", insight.SourceURL, insight.SourceURL)) + // Data Points (if present) + if len(insight.DataPoints) > 0 { + content.WriteString("## Supporting Data\n\n") + content.WriteString("| Data Point | Value | Context |\n") + content.WriteString("|------------|-------|--------|\n") + for _, dp := range insight.DataPoints { + context := truncateString(dp.Context, 100) + content.WriteString(fmt.Sprintf("| %s | %s | %s |\n", dp.Label, dp.Value, context)) + } + content.WriteString("\n") } - if insight.SourceContent != "" { - content.WriteString("\n### Source Excerpt\n\n") - content.WriteString("> ") - content.WriteString(truncateString(insight.SourceContent, 500)) + + // Analysis Chain (if present) + if len(insight.AnalysisChain) > 0 { + content.WriteString("## Analysis Chain\n\n") + content.WriteString("How this insight was derived:\n\n") + for i, step := range insight.AnalysisChain { + content.WriteString(fmt.Sprintf("%d. %s\n", i+1, step)) + } content.WriteString("\n") } + // Query/Tool used + if insight.QueryUsed != "" { + content.WriteString("## Research Method\n\n") + if insight.ToolUsed != "" { + content.WriteString(fmt.Sprintf("- **Tool**: `%s`\n", insight.ToolUsed)) + } + content.WriteString(fmt.Sprintf("- **Query/Args**: %s\n\n", insight.QueryUsed)) + } + + // Sources with links + content.WriteString("## Sources\n\n") + if len(insight.Sources) > 0 { + for _, src := range insight.Sources { + key := src.URL + if key == "" { + key = src.FilePath + } + linkedFile := sourceMap[key] + if linkedFile != "" { + content.WriteString(fmt.Sprintf("- [[sources/%s|%s]] (%s)\n", linkedFile, src.Title, src.Type)) + } else if src.URL != "" { + content.WriteString(fmt.Sprintf("- [%s](%s) (%s)\n", src.URL, src.URL, src.Type)) + } else if src.FilePath != "" { + content.WriteString(fmt.Sprintf("- `%s` (%s)\n", src.FilePath, src.Type)) + } + } + content.WriteString("\n") + } else if insight.SourceURL != "" { + // Legacy source handling + linkedFile := sourceMap[insight.SourceURL] + if linkedFile != "" { + content.WriteString(fmt.Sprintf("- [[sources/%s|%s]]\n", linkedFile, insight.SourceURL)) + } else { + content.WriteString(fmt.Sprintf("- [%s](%s)\n", insight.SourceURL, insight.SourceURL)) + } + content.WriteString("\n") + } else { + content.WriteString("*No sources linked*\n\n") + } + + // Source excerpt (legacy, for backwards compatibility) + if insight.SourceContent != "" && len(insight.Sources) == 0 { + content.WriteString("### Source Excerpt\n\n") + content.WriteString("> ") + content.WriteString(strings.ReplaceAll(truncateString(insight.SourceContent, 1000), "\n", "\n> ")) + content.WriteString("\n\n") + } + + // Related insights + if len(insight.RelatedInsightIDs) > 0 { + content.WriteString("## Related Insights\n\n") + for _, relID := range insight.RelatedInsightIDs { + content.WriteString(fmt.Sprintf("- [[insights/%s]]\n", relID)) + } + } + return os.WriteFile(filename, content.Bytes(), 0644) } // WriteInsights writes all insights for a session. +// It first writes all sources, then writes insights with source links. func (w *Writer) WriteInsights(sessionDir string, insights []think_deep.SubInsight) error { + // First, write all sources and get the mapping + sourceMap, err := w.WriteSources(sessionDir, insights) + if err != nil { + // Continue even if some sources failed to write + sourceMap = make(map[string]string) + } + + // Then write insights with source links for i, insight := range insights { - if err := w.WriteInsight(sessionDir, insight, i+1); err != nil { + if err := w.WriteInsight(sessionDir, insight, i+1, sourceMap); err != nil { return fmt.Errorf("write insight %d: %w", i+1, err) } } return nil } +// sanitizeFilename creates a safe filename from a URL or path +func sanitizeFilename(input string) string { + // Try to parse as URL and extract host + path + if u, err := url.Parse(input); err == nil && u.Host != "" { + input = u.Host + u.Path + } + + // Remove protocol prefixes + input = strings.TrimPrefix(input, "https://") + input = strings.TrimPrefix(input, "http://") + + // Replace unsafe characters + re := regexp.MustCompile(`[^a-zA-Z0-9_-]`) + result := re.ReplaceAllString(input, "_") + + // Collapse multiple underscores + re = regexp.MustCompile(`_+`) + result = re.ReplaceAllString(result, "_") + + // Trim underscores from ends + result = strings.Trim(result, "_") + + // Limit length + if len(result) > 100 { + // Use hash suffix for uniqueness + hash := sha256.Sum256([]byte(input)) + result = result[:80] + "_" + fmt.Sprintf("%x", hash[:8]) + } + + if result == "" { + result = "source" + } + + return result +} + // WriteWithInsights writes a session with its sub-insights to the Obsidian vault. func (w *Writer) WriteWithInsights(sess *session.Session, subInsights []think_deep.SubInsight) error { // Create session directory structure diff --git a/go-research/internal/repl/handlers/architectures.go b/go-research/internal/repl/handlers/architectures.go index 53dabc6..d98a805 100644 --- a/go-research/internal/repl/handlers/architectures.go +++ b/go-research/internal/repl/handlers/architectures.go @@ -86,6 +86,10 @@ func (h *ArchitectureCommandHandler) Execute(ctx *repl.Context, args []string) e viz.Stop() if err != nil { + // Check if it was a timeout and set the cancel reason appropriately + if runCtx.Err() == context.DeadlineExceeded { + ctx.CancelReason = events.CancelReasonTimeout + } sess.Status = session.StatusFailed return fmt.Errorf("%s research failed: %w", h.definition.Name, err) } diff --git a/go-research/internal/repl/handlers/expand.go b/go-research/internal/repl/handlers/expand.go index c869027..bbe83f3 100644 --- a/go-research/internal/repl/handlers/expand.go +++ b/go-research/internal/repl/handlers/expand.go @@ -106,6 +106,10 @@ func (h *ExpandHandler) runFast(ctx *repl.Context, query string, sess *session.S sess.Status = session.StatusRunning workerCtx, err := reactAgent.Research(runCtx, query) if err != nil { + // Check if it was a timeout and set the cancel reason appropriately + if runCtx.Err() == context.DeadlineExceeded { + ctx.CancelReason = events.CancelReasonTimeout + } sess.Status = session.StatusFailed return fmt.Errorf("research failed: %w", err) } @@ -144,6 +148,10 @@ func (h *ExpandHandler) runDeep(ctx *repl.Context, query string, sess *session.S viz.Stop() if err != nil { + // Check if it was a timeout and set the cancel reason appropriately + if runCtx.Err() == context.DeadlineExceeded { + ctx.CancelReason = events.CancelReasonTimeout + } sess.Status = session.StatusFailed return fmt.Errorf("research failed: %w", err) } @@ -193,6 +201,10 @@ func (h *ExpandHandler) runThinkDeep(ctx *repl.Context, query string, sess *sess viz.Stop() if err != nil { + // Check if it was a timeout and set the cancel reason appropriately + if runCtx.Err() == context.DeadlineExceeded { + ctx.CancelReason = events.CancelReasonTimeout + } sess.Status = session.StatusFailed return fmt.Errorf("research failed: %w", err) } diff --git a/go-research/internal/repl/handlers/start.go b/go-research/internal/repl/handlers/start.go index 77ff1a9..d5e0f2a 100644 --- a/go-research/internal/repl/handlers/start.go +++ b/go-research/internal/repl/handlers/start.go @@ -62,6 +62,10 @@ func (h *FastHandler) Execute(ctx *repl.Context, args []string) error { sess.Status = session.StatusRunning workerCtx, err := reactAgent.Research(runCtx, query) if err != nil { + // Check if it was a timeout and set the cancel reason appropriately + if runCtx.Err() == context.DeadlineExceeded { + ctx.CancelReason = events.CancelReasonTimeout + } sess.Status = session.StatusFailed return fmt.Errorf("research failed: %w", err) } diff --git a/go-research/internal/repl/repl.go b/go-research/internal/repl/repl.go index 24412e1..00e5c59 100644 --- a/go-research/internal/repl/repl.go +++ b/go-research/internal/repl/repl.go @@ -7,6 +7,7 @@ import ( "os/signal" "sync" "syscall" + "time" "github.com/chzyer/readline" "go-research/internal/config" @@ -124,8 +125,8 @@ func (r *REPL) Run(ctx context.Context) error { for range sigCh { r.mu.Lock() if r.running && r.ctx.Cancel != nil { - r.renderer.Info("\n⚠ Cancelling research...") - r.ctx.Cancel() + r.renderer.Info("\n⚠ Cancelling research (user interrupt)...") + r.ctx.CancelWithReason(events.CancelReasonUserInterrupt) } r.mu.Unlock() } @@ -144,8 +145,8 @@ func (r *REPL) Run(ctx context.Context) error { // Ctrl+C pressed at prompt - check if running r.mu.Lock() if r.running && r.ctx.Cancel != nil { - r.renderer.Info("\n⚠ Cancelling research...") - r.ctx.Cancel() + r.renderer.Info("\n⚠ Cancelling research (user interrupt)...") + r.ctx.CancelWithReason(events.CancelReasonUserInterrupt) } r.mu.Unlock() continue @@ -169,6 +170,7 @@ func (r *REPL) Run(ctx context.Context) error { runCtx, cancel := context.WithCancel(ctx) r.ctx.RunContext = runCtx r.ctx.Cancel = cancel + r.ctx.CancelReason = "" // Reset cancel reason for new execution r.mu.Lock() r.running = true @@ -178,7 +180,9 @@ func (r *REPL) Run(ctx context.Context) error { r.mu.Lock() r.running = false + cancelReason := r.ctx.CancelReason r.ctx.Cancel = nil + r.ctx.CancelReason = "" r.mu.Unlock() cancel() // Clean up context @@ -190,7 +194,21 @@ func (r *REPL) Run(ctx context.Context) error { } // Check for cancellation if execErr == context.Canceled || runCtx.Err() == context.Canceled { - r.renderer.Info("Research cancelled.") + // Determine the cancellation reason + reason := cancelReason + if reason == "" { + if runCtx.Err() == context.DeadlineExceeded { + reason = events.CancelReasonTimeout + } else { + reason = events.CancelReasonUnknown + } + } + + // Emit cancellation event with reason + r.emitCancellation(reason) + + // Log the cancellation with reason + r.renderer.Info(fmt.Sprintf("Research cancelled: %s", r.formatCancelReason(reason))) continue } r.renderer.Error(execErr) @@ -199,6 +217,43 @@ func (r *REPL) Run(ctx context.Context) error { } } +// emitCancellation publishes a cancellation event with the given reason. +func (r *REPL) emitCancellation(reason events.CancelReason) { + var sessionID, query string + if r.ctx.Session != nil { + sessionID = r.ctx.Session.ID + query = r.ctx.Session.Query + } + + r.bus.Publish(events.Event{ + Type: events.EventResearchCancelled, + Timestamp: time.Now(), + Data: events.ResearchCancelledData{ + SessionID: sessionID, + Query: query, + Reason: reason, + Phase: "unknown", // Could be enhanced to track current phase + Message: r.formatCancelReason(reason), + }, + }) +} + +// formatCancelReason returns a human-readable description of the cancellation reason. +func (r *REPL) formatCancelReason(reason events.CancelReason) string { + switch reason { + case events.CancelReasonUserInterrupt: + return "user pressed Ctrl+C" + case events.CancelReasonTimeout: + return "operation timed out" + case events.CancelReasonParentCancelled: + return "parent operation was cancelled" + case events.CancelReasonShutdown: + return "system is shutting down" + default: + return "unknown reason (this is a bug - please report)" + } +} + // shutdown saves current session and cleans up func (r *REPL) shutdown() { r.renderer.Info("\nShutting down...") diff --git a/go-research/internal/repl/router.go b/go-research/internal/repl/router.go index 9c79015..f2efe2a 100644 --- a/go-research/internal/repl/router.go +++ b/go-research/internal/repl/router.go @@ -17,16 +17,43 @@ type Handler interface { // Context provides shared state to handlers type Context struct { - Session *session.Session - Store *session.Store - Bus *events.Bus - Renderer *Renderer - Config *config.Config - Obsidian *obsidian.Writer - CommandDocs []CommandDoc - RunContext context.Context // Cancelable context for current operation - Cancel context.CancelFunc // Cancel function to abort current operation - Classifier *QueryClassifier // LLM-based query classifier for intent routing + Session *session.Session + Store *session.Store + Bus *events.Bus + Renderer *Renderer + Config *config.Config + Obsidian *obsidian.Writer + CommandDocs []CommandDoc + RunContext context.Context // Cancelable context for current operation + Cancel context.CancelFunc // Cancel function to abort current operation + CancelReason events.CancelReason // Reason for cancellation (set before calling Cancel) + Classifier *QueryClassifier // LLM-based query classifier for intent routing +} + +// CancelWithReason cancels the current operation with a specific reason. +// This must be called instead of Cancel() directly to track cancellation reasons. +func (c *Context) CancelWithReason(reason events.CancelReason) { + if c.Cancel == nil { + return + } + c.CancelReason = reason + c.Cancel() +} + +// GetCancelReason returns the reason for cancellation, attempting to infer it +// if not explicitly set. +func (c *Context) GetCancelReason() events.CancelReason { + if c.CancelReason != "" { + return c.CancelReason + } + if c.RunContext == nil { + return events.CancelReasonUnknown + } + // Try to infer from context error + if c.RunContext.Err() == context.DeadlineExceeded { + return events.CancelReasonTimeout + } + return events.CancelReasonUnknown } // Router routes input to appropriate handlers diff --git a/go-research/internal/session/cost_test.go b/go-research/internal/session/cost_test.go index 6923b06..2e0190d 100644 --- a/go-research/internal/session/cost_test.go +++ b/go-research/internal/session/cost_test.go @@ -14,3 +14,4 @@ func TestNewCostBreakdown(t *testing.T) { } } + diff --git a/go-research/internal/think_deep/state.go b/go-research/internal/think_deep/state.go index 4a88e87..1b416ec 100644 --- a/go-research/internal/think_deep/state.go +++ b/go-research/internal/think_deep/state.go @@ -13,41 +13,112 @@ import ( "go-research/internal/llm" ) +// SourceType indicates the type of source that provided the insight +type SourceType string + +const ( + SourceTypeWeb SourceType = "web" + SourceTypeDocument SourceType = "document" + SourceTypeAPI SourceType = "api" + SourceTypeFile SourceType = "file" +) + +// SourceReference contains detailed information about a source for traceability +type SourceReference struct { + // URL is the source URL (for web sources) + URL string `json:"url,omitempty"` + + // FilePath is the local file path (for document sources) + FilePath string `json:"file_path,omitempty"` + + // Type indicates what kind of source this is + Type SourceType `json:"type"` + + // Title is the source title or filename + Title string `json:"title,omitempty"` + + // RawContent is the full raw content fetched from the source + RawContent string `json:"raw_content,omitempty"` + + // RelevantExcerpt is the specific excerpt relevant to the insight + RelevantExcerpt string `json:"relevant_excerpt,omitempty"` + + // FetchedAt is when the source was fetched + FetchedAt time.Time `json:"fetched_at"` + + // ContentHash is a hash of the raw content for deduplication + ContentHash string `json:"content_hash,omitempty"` +} + +// DataPoint represents a specific piece of data extracted from a source +type DataPoint struct { + // Label describes what this data point represents + Label string `json:"label"` + + // Value is the actual data value + Value string `json:"value"` + + // Context provides surrounding context for the data point + Context string `json:"context,omitempty"` + + // SourceRef links to the source this was extracted from + SourceRef string `json:"source_ref,omitempty"` +} + // SubInsight represents a single research finding extracted from search results. // Granularity: one insight per search result with extracted findings. +// Enhanced to support full traceability and rich data capture. type SubInsight struct { // ID is a unique identifier (e.g., "insight-001") - ID string + ID string `json:"id"` // Topic is the research topic from supervisor delegation - Topic string + Topic string `json:"topic"` // Title is a short title summarizing the insight - Title string + Title string `json:"title"` // Finding is the factual finding extracted - Finding string + Finding string `json:"finding"` // Implication is what this means for the research question - Implication string + Implication string `json:"implication,omitempty"` - // SourceURL is the URL this insight was extracted from - SourceURL string + // SourceURL is the URL this insight was extracted from (primary source) + SourceURL string `json:"source_url,omitempty"` // SourceContent is the relevant excerpt from source (truncated) - SourceContent string + SourceContent string `json:"source_content,omitempty"` + + // Sources contains all source references with full traceability + Sources []SourceReference `json:"sources,omitempty"` + + // DataPoints contains specific data extracted supporting the finding + DataPoints []DataPoint `json:"data_points,omitempty"` + + // AnalysisChain describes how this insight was derived + AnalysisChain []string `json:"analysis_chain,omitempty"` + + // RelatedInsightIDs links to other insights that corroborate this finding + RelatedInsightIDs []string `json:"related_insight_ids,omitempty"` // Confidence is a 0-1 confidence score - Confidence float64 + Confidence float64 `json:"confidence"` // Iteration is the diffusion iteration when captured - Iteration int + Iteration int `json:"iteration"` // ResearcherNum is which sub-researcher found this - ResearcherNum int + ResearcherNum int `json:"researcher_num"` // Timestamp is when the insight was captured - Timestamp time.Time + Timestamp time.Time `json:"timestamp"` + + // ToolUsed indicates which tool produced this insight (search, fetch, read_document, etc.) + ToolUsed string `json:"tool_used,omitempty"` + + // QueryUsed is the search query or tool arguments that produced this insight + QueryUsed string `json:"query_used,omitempty"` } // SupervisorState manages the lead researcher's coordination of the diffusion diff --git a/go-research/internal/tools/xlsx.go b/go-research/internal/tools/xlsx.go index 923f0d5..6d17abc 100644 --- a/go-research/internal/tools/xlsx.go +++ b/go-research/internal/tools/xlsx.go @@ -59,6 +59,9 @@ func (t *XLSXReadTool) Execute(ctx context.Context, args map[string]interface{}) } var b strings.Builder + // Header with source traceability information + b.WriteString(fmt.Sprintf("Read document: %s\n", path)) + b.WriteString(fmt.Sprintf("Document type: Excel Workbook (XLSX)\n")) b.WriteString(fmt.Sprintf("Workbook: %s\n", filepath.Base(path))) b.WriteString(fmt.Sprintf("Total sheets: %d\n\n", len(sheets))) @@ -145,3 +148,4 @@ func formatXLSXRow(row []string, maxCols int) string { return line } + From 2b050bcb7b384ecfedd51ed38973e62cd4b8c451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 5 Dec 2025 13:12:46 +0100 Subject: [PATCH 5/5] feat: Implement context handling in REPL with new ContextHandler - Added a new ContextHandler to manage the `/context` command, providing users with session context statistics and content. - Implemented summary and verbose output modes for context display, enhancing user experience and information accessibility. - Developed comprehensive tests for the ContextHandler to ensure reliability and correctness in context management. - Updated REPL command registration to include the new context command, improving the command set available to users. - Enhanced session management by integrating context snapshot functionality, allowing for better tracking of session details. --- .../architectures/think_deep/README.md | 1 + go-research/internal/repl/handlers/context.go | 154 ++++++++ .../internal/repl/handlers/context_test.go | 332 ++++++++++++++++++ .../internal/repl/handlers/handlers.go | 1 + go-research/internal/repl/repl.go | 23 +- go-research/internal/repl/router.go | 2 + go-research/internal/session/context_view.go | 197 +++++++++++ .../internal/session/context_view_test.go | 247 +++++++++++++ go-research/internal/session/cost_test.go | 1 + go-research/internal/tools/xlsx.go | 1 + 10 files changed, 944 insertions(+), 15 deletions(-) create mode 100644 go-research/internal/repl/handlers/context.go create mode 100644 go-research/internal/repl/handlers/context_test.go create mode 100644 go-research/internal/session/context_view.go create mode 100644 go-research/internal/session/context_view_test.go diff --git a/go-research/internal/architectures/think_deep/README.md b/go-research/internal/architectures/think_deep/README.md index 86668f6..8e21dd1 100644 --- a/go-research/internal/architectures/think_deep/README.md +++ b/go-research/internal/architectures/think_deep/README.md @@ -392,3 +392,4 @@ For more technical details, see: - **[ThinkDepth.ai GitHub](https://github.com/thinkdepthai/Deep_Research)** - Reference Python implementation - **[DeepResearch Bench](https://github.com/Ayanami0730/deep_research_bench)** - Evaluation benchmark - **[Self-Balancing Agentic AI](https://paichunlin.substack.com/p/self-balancing-agentic-ai-test-time)** - Algorithm deep-dive +- **[Google: Deep-Researcher with Test-Time Diffusion](https://research.google/blog/deep-researcher-with-test-time-diffusion/)** - Google Research blog post \ No newline at end of file diff --git a/go-research/internal/repl/handlers/context.go b/go-research/internal/repl/handlers/context.go new file mode 100644 index 0000000..a657542 --- /dev/null +++ b/go-research/internal/repl/handlers/context.go @@ -0,0 +1,154 @@ +package handlers + +import ( + "fmt" + "strings" + + "go-research/internal/repl" + "go-research/internal/session" +) + +// ContextHandler handles /context command +type ContextHandler struct{} + +// Execute runs the context command +func (h *ContextHandler) Execute(ctx *repl.Context, args []string) error { + if ctx.Session == nil { + return fmt.Errorf("no active session. Start research first with /fast, /storm, or /think_deep") + } + + // Parse flags + verbose := false + for _, arg := range args { + if arg == "-v" || arg == "--verbose" { + verbose = true + } + } + + // Build context snapshot + maxTokens := ctx.Config.MaxTokens + if maxTokens == 0 { + maxTokens = 50000 // Default fallback + } + + snapshot, err := session.BuildContextSnapshot(ctx.Session, ctx.Store, maxTokens) + if err != nil { + return fmt.Errorf("build context snapshot: %w", err) + } + + // Render based on mode + if verbose { + h.renderVerbose(ctx, snapshot) + } else { + h.renderSummary(ctx, snapshot) + } + + return nil +} + +// renderSummary renders a concise overview of context statistics +func (h *ContextHandler) renderSummary(ctx *repl.Context, snapshot *session.ContextSnapshot) { + renderer := ctx.Renderer + + renderer.Info("Context Usage") + fmt.Fprintf(renderer.StreamWriter(), "\n") + + // Session info + fmt.Fprintf(renderer.StreamWriter(), "Session: %s\n", snapshot.SessionID) + fmt.Fprintf(renderer.StreamWriter(), "Mode: %s | Status: %s\n", snapshot.Mode, snapshot.Status) + if snapshot.Query != "" { + fmt.Fprintf(renderer.StreamWriter(), "Query: %s\n", truncateString(snapshot.Query, 60)) + } + fmt.Fprintf(renderer.StreamWriter(), "\n") + + // Token usage + tokenPercent := 0.0 + if snapshot.MaxTokens > 0 { + tokenPercent = float64(snapshot.EstimatedTokens) / float64(snapshot.MaxTokens) * 100 + } + fmt.Fprintf(renderer.StreamWriter(), "Tokens: %d/%d (%.1f%%)\n", + snapshot.EstimatedTokens, snapshot.MaxTokens, tokenPercent) + + // Visual token representation (10x10 grid) + gridSize := 100 + filled := int(float64(gridSize) * tokenPercent / 100) + if filled > gridSize { + filled = gridSize + } + fmt.Fprintf(renderer.StreamWriter(), "Visual: ") + for i := 0; i < gridSize; i++ { + if i < filled { + fmt.Fprintf(renderer.StreamWriter(), "█") + } else { + fmt.Fprintf(renderer.StreamWriter(), "░") + } + if (i+1)%10 == 0 && i < gridSize-1 { + fmt.Fprintf(renderer.StreamWriter(), "\n ") + } + } + fmt.Fprintf(renderer.StreamWriter(), "\n\n") + + // Breakdown by category + fmt.Fprintf(renderer.StreamWriter(), "Breakdown:\n") + fmt.Fprintf(renderer.StreamWriter(), " Report: %d chars (%d tokens)\n", + snapshot.ReportLength, snapshot.ReportLength/4) + fmt.Fprintf(renderer.StreamWriter(), " Sources: %d\n", snapshot.SourcesCount) + fmt.Fprintf(renderer.StreamWriter(), " Insights: %d\n", snapshot.InsightsCount) + fmt.Fprintf(renderer.StreamWriter(), " Workers: %d\n", snapshot.WorkersCount) + fmt.Fprintf(renderer.StreamWriter(), " Iterations: %d\n", snapshot.IterationsCount) + fmt.Fprintf(renderer.StreamWriter(), " Tool calls: %d\n", snapshot.ToolCallsCount) + + // Think-deep specific stats + if snapshot.HasThinkDeepContext { + fmt.Fprintf(renderer.StreamWriter(), "\nThink-Deep Context:\n") + fmt.Fprintf(renderer.StreamWriter(), " Previous findings: %d\n", snapshot.ThinkDeepFindings) + fmt.Fprintf(renderer.StreamWriter(), " Visited URLs: %d\n", snapshot.ThinkDeepVisitedURLs) + if snapshot.ThinkDeepHasReport { + fmt.Fprintf(renderer.StreamWriter(), " Has existing report: yes\n") + } + } + + // Cost + if snapshot.Cost.TotalCost > 0 { + fmt.Fprintf(renderer.StreamWriter(), "\nCost: $%.4f\n", snapshot.Cost.TotalCost) + fmt.Fprintf(renderer.StreamWriter(), " Input tokens: %d ($%.4f)\n", + snapshot.Cost.InputTokens, snapshot.Cost.InputCost) + fmt.Fprintf(renderer.StreamWriter(), " Output tokens: %d ($%.4f)\n", + snapshot.Cost.OutputTokens, snapshot.Cost.OutputCost) + } + + fmt.Fprintf(renderer.StreamWriter(), "\n") + fmt.Fprintf(renderer.StreamWriter(), "Use /context -v to see full raw context\n") +} + +// renderVerbose renders the full raw context string +func (h *ContextHandler) renderVerbose(ctx *repl.Context, snapshot *session.ContextSnapshot) { + renderer := ctx.Renderer + + renderer.Info("Full Context (Verbose Mode)") + fmt.Fprintf(renderer.StreamWriter(), "\n") + fmt.Fprintf(renderer.StreamWriter(), "Session: %s\n", snapshot.SessionID) + fmt.Fprintf(renderer.StreamWriter(), "Estimated tokens: %d\n", snapshot.EstimatedTokens) + fmt.Fprintf(renderer.StreamWriter(), "Context length: %d chars\n\n", len(snapshot.RawContext)) + fmt.Fprint(renderer.StreamWriter(), strings.Repeat("=", 80)) + fmt.Fprint(renderer.StreamWriter(), "\n\n") + + // Print raw context + if snapshot.RawContext == "" { + fmt.Fprint(renderer.StreamWriter(), "(No context available)\n") + } else { + fmt.Fprintf(renderer.StreamWriter(), "%s\n", snapshot.RawContext) + } + + fmt.Fprint(renderer.StreamWriter(), "\n") + fmt.Fprint(renderer.StreamWriter(), strings.Repeat("=", 80)) + fmt.Fprint(renderer.StreamWriter(), "\n") +} + +// truncateString truncates a string to max length +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "..." +} diff --git a/go-research/internal/repl/handlers/context_test.go b/go-research/internal/repl/handlers/context_test.go new file mode 100644 index 0000000..2f620c7 --- /dev/null +++ b/go-research/internal/repl/handlers/context_test.go @@ -0,0 +1,332 @@ +package handlers + +import ( + "bytes" + "os" + "strings" + "testing" + "time" + + "go-research/internal/config" + "go-research/internal/events" + "go-research/internal/repl" + "go-research/internal/session" +) + +func TestContextHandler_Execute_NoSession(t *testing.T) { + handler := &ContextHandler{} + + ctx := &repl.Context{ + Session: nil, // No active session + Config: config.Load(), + } + + err := handler.Execute(ctx, []string{}) + + if err == nil { + t.Fatal("expected error when no session") + } + if !strings.Contains(err.Error(), "no active session") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestContextHandler_Execute_SummaryMode(t *testing.T) { + handler := &ContextHandler{} + + sess := &session.Session{ + ID: "test-session", + Query: "Test query", + Mode: session.ModeFast, + Status: session.StatusComplete, + Report: "This is a test report.", + Sources: []string{"https://example.com"}, + Insights: []session.Insight{ + {Title: "Test Insight", Finding: "Test Finding"}, + }, + Workers: []session.WorkerContext{ + { + ID: "worker-1", + Iterations: []session.ReActIteration{ + {Number: 1}, + {Number: 2}, + }, + ToolCalls: []session.ToolCall{ + {Tool: "search"}, + }, + }, + }, + Cost: session.CostBreakdown{ + TotalCost: 0.01, + }, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + tmpDir, err := os.MkdirTemp("", "context-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + store, err := session.NewStore(tmpDir) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + var buf bytes.Buffer + renderer := repl.NewRenderer(&buf) + + ctx := &repl.Context{ + Session: sess, + Store: store, + Renderer: renderer, + Config: config.Load(), + Bus: events.NewBus(100), + } + + err = handler.Execute(ctx, []string{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + output := buf.String() + + // Verify summary output contains key information + if !strings.Contains(output, "Context Usage") { + t.Error("output should contain 'Context Usage' header") + } + if !strings.Contains(output, "test-session") { + t.Error("output should contain session ID") + } + if !strings.Contains(output, "Test query") { + t.Error("output should contain query") + } + if !strings.Contains(output, "Tokens:") { + t.Error("output should contain token information") + } + if !strings.Contains(output, "Workers:") { + t.Error("output should contain workers count") + } + if !strings.Contains(output, "Sources:") { + t.Error("output should contain sources count") + } + if !strings.Contains(output, "Insights:") { + t.Error("output should contain insights count") + } + if !strings.Contains(output, "Use /context -v") { + t.Error("output should suggest verbose mode") + } +} + +func TestContextHandler_Execute_VerboseMode(t *testing.T) { + handler := &ContextHandler{} + + sess := &session.Session{ + ID: "test-session", + Query: "Test query", + Mode: session.ModeFast, + Status: session.StatusComplete, + Report: "This is a test report with content.", + Sources: []string{"https://example.com"}, + Insights: []session.Insight{ + {Title: "Test Insight", Finding: "Test Finding"}, + }, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + tmpDir, err := os.MkdirTemp("", "context-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + store, err := session.NewStore(tmpDir) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + var buf bytes.Buffer + renderer := repl.NewRenderer(&buf) + + ctx := &repl.Context{ + Session: sess, + Store: store, + Renderer: renderer, + Config: config.Load(), + Bus: events.NewBus(100), + } + + err = handler.Execute(ctx, []string{"-v"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + output := buf.String() + + // Verify verbose output contains raw context + if !strings.Contains(output, "Full Context (Verbose Mode)") { + t.Error("output should contain verbose header") + } + if !strings.Contains(output, "test-session") { + t.Error("output should contain session ID") + } + if !strings.Contains(output, "Estimated tokens:") { + t.Error("output should contain token estimate") + } + if !strings.Contains(output, "Test query") { + t.Error("output should contain query in raw context") + } + if !strings.Contains(output, "This is a test report") { + t.Error("output should contain report in raw context") + } +} + +func TestContextHandler_Execute_VerboseModeWithFlag(t *testing.T) { + handler := &ContextHandler{} + + sess := &session.Session{ + ID: "test-session", + Query: "Test query", + Mode: session.ModeFast, + Status: session.StatusComplete, + Report: "Test report", + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + tmpDir, err := os.MkdirTemp("", "context-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + store, err := session.NewStore(tmpDir) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + var buf bytes.Buffer + renderer := repl.NewRenderer(&buf) + + ctx := &repl.Context{ + Session: sess, + Store: store, + Renderer: renderer, + Config: config.Load(), + Bus: events.NewBus(100), + } + + // Test with --verbose flag + err = handler.Execute(ctx, []string{"--verbose"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + output := buf.String() + + // Should be in verbose mode + if !strings.Contains(output, "Full Context (Verbose Mode)") { + t.Error("output should be in verbose mode with --verbose flag") + } +} + +func TestContextHandler_ThinkDeepMode(t *testing.T) { + handler := &ContextHandler{} + + sess := &session.Session{ + ID: "think-deep-session", + Query: "Think deep query", + Mode: session.ModeThinkDeep, + Status: session.StatusComplete, + Report: "Think deep report", + Sources: []string{"https://thinkdeep.com"}, + Insights: []session.Insight{ + {Title: "TD Insight", Finding: "TD Finding"}, + }, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + tmpDir, err := os.MkdirTemp("", "context-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + store, err := session.NewStore(tmpDir) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + var buf bytes.Buffer + renderer := repl.NewRenderer(&buf) + + ctx := &repl.Context{ + Session: sess, + Store: store, + Renderer: renderer, + Config: config.Load(), + Bus: events.NewBus(100), + } + + err = handler.Execute(ctx, []string{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + output := buf.String() + + // Should show think-deep specific context + if !strings.Contains(output, "Think-Deep Context") { + t.Error("output should contain think-deep context section") + } + if !strings.Contains(output, "Previous findings:") { + t.Error("output should show previous findings count") + } + if !strings.Contains(output, "Visited URLs:") { + t.Error("output should show visited URLs count") + } +} + +func TestContextHandler_ImplementsHandler(t *testing.T) { + var _ repl.Handler = (*ContextHandler)(nil) +} + +func TestTruncateString(t *testing.T) { + tests := []struct { + name string + input string + maxLen int + want string + }{ + { + name: "short string unchanged", + input: "hello", + maxLen: 10, + want: "hello", + }, + { + name: "exact length unchanged", + input: "hello", + maxLen: 5, + want: "hello", + }, + { + name: "long string truncated", + input: "hello world", + maxLen: 5, + want: "hello...", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := truncateString(tt.input, tt.maxLen) + if got != tt.want { + t.Errorf("truncateString(%q, %d) = %q, want %q", tt.input, tt.maxLen, got, tt.want) + } + }) + } +} diff --git a/go-research/internal/repl/handlers/handlers.go b/go-research/internal/repl/handlers/handlers.go index 361723b..ef64cde 100644 --- a/go-research/internal/repl/handlers/handlers.go +++ b/go-research/internal/repl/handlers/handlers.go @@ -33,6 +33,7 @@ func RegisterAll() map[string]repl.Handler { add("load", &LoadHandler{}, "/load ", "Load a previous session", repl.CommandCategorySessions) add("new", &NewHandler{}, "/new", "Clear session and start fresh", repl.CommandCategorySessions) add("workers", &WorkersHandler{}, "/workers", "Show worker status", repl.CommandCategoryWorkflow) + add("context", &ContextHandler{}, "/context [-v]", "Show context window statistics and content", repl.CommandCategoryWorkflow) add("rerun", &RerunHandler{}, "/rerun ", "Rerun a previous query", repl.CommandCategorySessions) add("recompile", &RecompileHandler{}, "/recompile", "Hot reload the agent", repl.CommandCategorySettings) add("verbose", &VerboseHandler{}, "/verbose on|off", "Toggle verbose mode", repl.CommandCategorySettings) diff --git a/go-research/internal/repl/repl.go b/go-research/internal/repl/repl.go index 00e5c59..6d4e40a 100644 --- a/go-research/internal/repl/repl.go +++ b/go-research/internal/repl/repl.go @@ -9,12 +9,13 @@ import ( "syscall" "time" - "github.com/chzyer/readline" "go-research/internal/config" "go-research/internal/events" "go-research/internal/llm" "go-research/internal/obsidian" "go-research/internal/session" + + "github.com/chzyer/readline" ) // REPL is the interactive research shell @@ -84,18 +85,6 @@ func (r *REPL) Run(ctx context.Context) error { r.renderer.Welcome(r.commandDocs) - // Try to restore last session - if sess, err := r.store.LoadLast(); err == nil && sess != nil { - r.ctx.Session = sess - r.renderer.SessionRestored( - sess.ID, - sess.Query, - len(sess.Workers), - len(sess.Sources), - sess.Cost.TotalCost, - ) - } - // Subscribe to events for rendering eventCh := r.bus.Subscribe( events.EventWorkerStarted, @@ -184,6 +173,10 @@ func (r *REPL) Run(ctx context.Context) error { r.ctx.Cancel = nil r.ctx.CancelReason = "" r.mu.Unlock() + + // Capture context error BEFORE calling cancel(), otherwise runCtx.Err() + // will always return context.Canceled after cancel() is called + contextErr := runCtx.Err() cancel() // Clean up context if execErr != nil { @@ -193,11 +186,11 @@ func (r *REPL) Run(ctx context.Context) error { return nil } // Check for cancellation - if execErr == context.Canceled || runCtx.Err() == context.Canceled { + if execErr == context.Canceled || contextErr == context.Canceled { // Determine the cancellation reason reason := cancelReason if reason == "" { - if runCtx.Err() == context.DeadlineExceeded { + if contextErr == context.DeadlineExceeded { reason = events.CancelReasonTimeout } else { reason = events.CancelReasonUnknown diff --git a/go-research/internal/repl/router.go b/go-research/internal/repl/router.go index f2efe2a..dfa6636 100644 --- a/go-research/internal/repl/router.go +++ b/go-research/internal/repl/router.go @@ -145,6 +145,8 @@ func (r *Router) routeCommand(parsed ParsedInput) (Handler, []string, error) { cmd = "new" case "w", "workers": cmd = "workers" + case "ctx", "context": + cmd = "context" case "r", "rerun": cmd = "rerun" case "rc", "recompile": diff --git a/go-research/internal/session/context_view.go b/go-research/internal/session/context_view.go new file mode 100644 index 0000000..c101580 --- /dev/null +++ b/go-research/internal/session/context_view.go @@ -0,0 +1,197 @@ +package session + +import ( + "fmt" + "strings" + + "go-research/internal/think_deep" +) + +// ContextSnapshot contains statistics and raw context for a session +type ContextSnapshot struct { + // Session metadata + SessionID string + Mode Mode + Status SessionStatus + Query string + + // Statistics + ReportLength int + SourcesCount int + InsightsCount int + WorkersCount int + IterationsCount int + ToolCallsCount int + Cost CostBreakdown + + // Token estimates + EstimatedTokens int + MaxTokens int + + // Think-deep specific + HasThinkDeepContext bool + ThinkDeepFindings int + ThinkDeepVisitedURLs int + ThinkDeepHasReport bool + + // Raw context string + RawContext string +} + +// BuildContextSnapshot creates a snapshot of the current session's context +func BuildContextSnapshot(sess *Session, store *Store, maxTokens int) (*ContextSnapshot, error) { + if sess == nil { + return nil, fmt.Errorf("no active session") + } + + snapshot := &ContextSnapshot{ + SessionID: sess.ID, + Mode: sess.Mode, + Status: sess.Status, + Query: sess.Query, + ReportLength: len(sess.Report), + SourcesCount: len(sess.Sources), + InsightsCount: len(sess.Insights), + WorkersCount: len(sess.Workers), + Cost: sess.Cost, + MaxTokens: maxTokens, + } + + // Count iterations and tool calls across all workers + for _, worker := range sess.Workers { + snapshot.IterationsCount += len(worker.Iterations) + snapshot.ToolCallsCount += len(worker.ToolCalls) + } + + // Build raw context based on mode + if sess.Mode == ModeThinkDeep { + // For think_deep, build injection context like expand handler does + injection := buildInjectionContextForSnapshot(sess, store) + snapshot.HasThinkDeepContext = true + snapshot.ThinkDeepFindings = len(injection.PreviousFindings) + snapshot.ThinkDeepVisitedURLs = len(injection.VisitedURLs) + snapshot.ThinkDeepHasReport = injection.ExistingReport != "" + snapshot.RawContext = serializeInjectionContext(injection) + } else { + // For fast/storm, use continuation context + snapshot.RawContext = BuildContinuationContext(sess) + } + + // Estimate tokens (rough approximation: chars/4) + snapshot.EstimatedTokens = estimateTokens(snapshot.RawContext) + + return snapshot, nil +} + +// buildInjectionContextForSnapshot builds injection context from session chain +// Similar to ExpandHandler.buildInjectionContext but doesn't need expansion topic +func buildInjectionContextForSnapshot(sess *Session, store *Store) *think_deep.InjectionContext { + injection := think_deep.NewInjectionContext() + + // Walk session chain and accumulate context + current := sess + for current != nil { + // Accumulate findings from insights + for _, ins := range current.Insights { + injection.AddFinding(ins.Finding) + } + + // Accumulate sources as visited URLs + for _, src := range current.Sources { + injection.AddVisitedURL(src) + } + + // Keep existing report for context + if injection.ExistingReport == "" && current.Report != "" { + injection.SetExistingReport(current.Report) + } + + // Walk to parent + if current.ParentID == nil { + break + } + parent, err := store.Load(*current.ParentID) + if err != nil { + break + } + current = parent + } + + return injection +} + +// serializeInjectionContext converts injection context to readable string +func serializeInjectionContext(injection *think_deep.InjectionContext) string { + var sb strings.Builder + + if injection.ExpansionTopic != "" { + sb.WriteString(fmt.Sprintf("Expansion topic: %s\n\n", injection.ExpansionTopic)) + } + + if injection.ExistingReport != "" { + sb.WriteString("Existing report:\n") + report := injection.ExistingReport + if len(report) > 2000 { + report = report[:2000] + "...[truncated]" + } + sb.WriteString(report) + sb.WriteString("\n\n") + } + + if len(injection.PreviousFindings) > 0 { + sb.WriteString("Previous findings:\n") + for _, finding := range injection.PreviousFindings { + sb.WriteString(fmt.Sprintf("- %s\n", finding)) + } + sb.WriteString("\n") + } + + if len(injection.ValidatedFacts) > 0 { + sb.WriteString("Validated facts:\n") + for _, fact := range injection.ValidatedFacts { + sb.WriteString(fmt.Sprintf("- %s\n", fact)) + } + sb.WriteString("\n") + } + + if len(injection.VisitedURLs) > 0 { + sb.WriteString("Visited URLs:\n") + limit := len(injection.VisitedURLs) + if limit > 20 { + limit = 20 + } + for _, url := range injection.VisitedURLs[:limit] { + sb.WriteString(fmt.Sprintf("- %s\n", url)) + } + if len(injection.VisitedURLs) > 20 { + sb.WriteString(fmt.Sprintf("... and %d more\n", len(injection.VisitedURLs)-20)) + } + sb.WriteString("\n") + } + + if len(injection.KnownGaps) > 0 { + sb.WriteString("Known gaps:\n") + for _, gap := range injection.KnownGaps { + sb.WriteString(fmt.Sprintf("- %s\n", gap)) + } + sb.WriteString("\n") + } + + if len(injection.RelatedTopics) > 0 { + sb.WriteString("Related topics:\n") + for _, topic := range injection.RelatedTopics { + sb.WriteString(fmt.Sprintf("- %s\n", topic)) + } + sb.WriteString("\n") + } + + return sb.String() +} + +// estimateTokens provides a rough token count estimate (chars/4 approximation) +func estimateTokens(s string) int { + if len(s) == 0 { + return 0 + } + return len(s) / 4 +} diff --git a/go-research/internal/session/context_view_test.go b/go-research/internal/session/context_view_test.go new file mode 100644 index 0000000..9bdb537 --- /dev/null +++ b/go-research/internal/session/context_view_test.go @@ -0,0 +1,247 @@ +package session + +import ( + "testing" + "time" +) + +func TestBuildContextSnapshot_NoSession(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + _, err = BuildContextSnapshot(nil, store, 50000) + if err == nil { + t.Error("expected error for nil session") + } + if err.Error() != "no active session" { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestBuildContextSnapshot_FastMode(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + sess := &Session{ + ID: "test-session", + Query: "Test query", + Mode: ModeFast, + Status: StatusComplete, + Report: "This is a test report with some content.", + Sources: []string{"https://example.com", "https://test.com"}, + Insights: []Insight{ + {Title: "Insight 1", Finding: "Finding 1"}, + {Title: "Insight 2", Finding: "Finding 2"}, + }, + Workers: []WorkerContext{ + { + ID: "worker-1", + Objective: "Test objective", + Iterations: []ReActIteration{ + {Number: 1, Thought: "Thought 1"}, + {Number: 2, Thought: "Thought 2"}, + }, + ToolCalls: []ToolCall{ + {Tool: "search", Args: map[string]interface{}{"query": "test"}}, + }, + }, + }, + Cost: CostBreakdown{ + InputTokens: 1000, + OutputTokens: 500, + TotalCost: 0.05, + }, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + snapshot, err := BuildContextSnapshot(sess, store, 50000) + if err != nil { + t.Fatalf("failed to build snapshot: %v", err) + } + + // Verify basic fields + if snapshot.SessionID != "test-session" { + t.Errorf("expected session ID 'test-session', got '%s'", snapshot.SessionID) + } + if snapshot.Mode != ModeFast { + t.Errorf("expected mode Fast, got %s", snapshot.Mode) + } + if snapshot.Query != "Test query" { + t.Errorf("expected query 'Test query', got '%s'", snapshot.Query) + } + + // Verify counts + if snapshot.ReportLength != len(sess.Report) { + t.Errorf("expected report length %d, got %d", len(sess.Report), snapshot.ReportLength) + } + if snapshot.SourcesCount != 2 { + t.Errorf("expected 2 sources, got %d", snapshot.SourcesCount) + } + if snapshot.InsightsCount != 2 { + t.Errorf("expected 2 insights, got %d", snapshot.InsightsCount) + } + if snapshot.WorkersCount != 1 { + t.Errorf("expected 1 worker, got %d", snapshot.WorkersCount) + } + if snapshot.IterationsCount != 2 { + t.Errorf("expected 2 iterations, got %d", snapshot.IterationsCount) + } + if snapshot.ToolCallsCount != 1 { + t.Errorf("expected 1 tool call, got %d", snapshot.ToolCallsCount) + } + + // Verify cost + if snapshot.Cost.TotalCost != 0.05 { + t.Errorf("expected cost 0.05, got %f", snapshot.Cost.TotalCost) + } + + // Verify raw context is built (should use BuildContinuationContext for fast mode) + if snapshot.RawContext == "" { + t.Error("expected non-empty raw context") + } + if !contains(snapshot.RawContext, "Test query") { + t.Error("raw context should contain query") + } + if !contains(snapshot.RawContext, "This is a test report") { + t.Error("raw context should contain report") + } + + // Verify token estimate + if snapshot.EstimatedTokens == 0 { + t.Error("expected non-zero token estimate") + } + + // Fast mode should not have think-deep context + if snapshot.HasThinkDeepContext { + t.Error("fast mode should not have think-deep context") + } +} + +func TestBuildContextSnapshot_ThinkDeepMode(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + sess := &Session{ + ID: "think-deep-session", + Query: "Think deep query", + Mode: ModeThinkDeep, + Status: StatusComplete, + Report: "Think deep report", + Sources: []string{"https://thinkdeep.com"}, + Insights: []Insight{ + {Title: "TD Insight", Finding: "TD Finding"}, + }, + Workers: []WorkerContext{}, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + snapshot, err := BuildContextSnapshot(sess, store, 50000) + if err != nil { + t.Fatalf("failed to build snapshot: %v", err) + } + + // Verify think-deep context is detected + if !snapshot.HasThinkDeepContext { + t.Error("think-deep mode should have think-deep context") + } + + // Verify think-deep stats + if snapshot.ThinkDeepFindings != 1 { + t.Errorf("expected 1 finding, got %d", snapshot.ThinkDeepFindings) + } + if snapshot.ThinkDeepVisitedURLs != 1 { + t.Errorf("expected 1 visited URL, got %d", snapshot.ThinkDeepVisitedURLs) + } + if !snapshot.ThinkDeepHasReport { + t.Error("expected existing report to be present") + } + + // Verify raw context contains think-deep content + if snapshot.RawContext == "" { + t.Error("expected non-empty raw context") + } + if !contains(snapshot.RawContext, "Think deep report") { + t.Error("raw context should contain report") + } +} + +func TestBuildContextSnapshot_WithParentChain(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + // Create parent session + parent := &Session{ + ID: "parent-session", + Query: "Parent query", + Mode: ModeFast, + Status: StatusComplete, + Report: "Parent report", + Sources: []string{"https://parent.com"}, + Insights: []Insight{ + {Title: "Parent Insight", Finding: "Parent Finding"}, + }, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + // Save parent + if err := store.Save(parent); err != nil { + t.Fatalf("failed to save parent: %v", err) + } + + // Create child session + parentID := parent.ID + child := &Session{ + ID: "child-session", + Query: "Child query", + Mode: ModeThinkDeep, + Status: StatusExpanded, + ParentID: &parentID, + Report: "Child report", + Sources: []string{"https://child.com"}, + Insights: []Insight{ + {Title: "Child Insight", Finding: "Child Finding"}, + }, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + snapshot, err := BuildContextSnapshot(child, store, 50000) + if err != nil { + t.Fatalf("failed to build snapshot: %v", err) + } + + // For think-deep mode with parent, should accumulate findings + if snapshot.ThinkDeepFindings < 2 { + t.Errorf("expected at least 2 findings (from parent + child), got %d", snapshot.ThinkDeepFindings) + } + if snapshot.ThinkDeepVisitedURLs < 2 { + t.Errorf("expected at least 2 visited URLs (from parent + child), got %d", snapshot.ThinkDeepVisitedURLs) + } +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(substr) == 0 || + (len(s) > len(substr) && (s[:len(substr)] == substr || + s[len(s)-len(substr):] == substr || + containsMiddle(s, substr)))) +} + +func containsMiddle(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/go-research/internal/session/cost_test.go b/go-research/internal/session/cost_test.go index 2e0190d..4aa6726 100644 --- a/go-research/internal/session/cost_test.go +++ b/go-research/internal/session/cost_test.go @@ -15,3 +15,4 @@ func TestNewCostBreakdown(t *testing.T) { } + diff --git a/go-research/internal/tools/xlsx.go b/go-research/internal/tools/xlsx.go index 6d17abc..87a2be0 100644 --- a/go-research/internal/tools/xlsx.go +++ b/go-research/internal/tools/xlsx.go @@ -149,3 +149,4 @@ func formatXLSXRow(row []string, maxCols int) string { return line } +