example_test.go

package tok_test

import (
	"fmt"

	"github.com/GrayCodeAI/tok"
)

func ExampleCompress() {
	text := `This is a very long piece of text that contains redundant information.
The purpose of this text is to demonstrate how tok can compress it.
We repeat the same ideas multiple times to inflate the token count.
The key insight is that tok uses entropy-based filtering to remove low-information tokens.
This allows us to reduce the token count while preserving the core meaning.`

	compressed, stats := tok.Compress(text)

	fmt.Printf("Original tokens: %d\n", stats.OriginalTokens)
	fmt.Printf("Final tokens: %d\n", stats.FinalTokens)
	fmt.Printf("Tokens saved: %d\n", stats.TokensSaved)
	fmt.Printf("Reduction: %.1f%%\n", stats.ReductionPercent)
	fmt.Printf("Compressed text length: %d\n", len(compressed))
}

func ExampleEstimateTokens() {
	text := "Hello, how are you today?"

	// Fast heuristic estimate
	count := tok.EstimateTokens(text)
	fmt.Printf("Estimated tokens: %d\n", count)
}

func ExampleNewCompressor() {
	// Create a reusable compressor for multiple calls
	c := tok.NewCompressor(
		tok.WithBudget(1000),
	)

	text1 := "First document to compress..."
	text2 := "Second document to compress..."

	_, stats1 := c.Compress(text1)
	_, stats2 := c.Compress(text2)

	fmt.Printf("Doc 1 saved: %d tokens\n", stats1.TokensSaved)
	fmt.Printf("Doc 2 saved: %d tokens\n", stats2.TokensSaved)
}

func ExampleCompress_withBudget() {
	text := `Long document with many details that we want to fit within a token budget.
We can use the budget option to limit the output to a specific number of tokens.
This is useful when working with context windows that have strict limits.`

	compressed, stats := tok.Compress(text, tok.WithBudget(50))

	fmt.Printf("Final tokens: %d (budget: 50)\n", stats.FinalTokens)
	fmt.Printf("Compressed text: %s\n", compressed[:50]+"...")
}

func ExampleCompress_withQuery() {
	text := `The application uses a REST API with JSON payloads.
Database migrations are handled by the migration package.
The frontend is built with React and TypeScript.
Authentication uses JWT tokens with refresh token rotation.
The CI/CD pipeline runs on GitHub Actions.`

	// Query-aware compression preserves content relevant to the query
	_, stats := tok.Compress(text, tok.WithQuery("authentication"))

	fmt.Printf("Tokens saved: %d\n", stats.TokensSaved)
}