From 0908ade5d870657f2b9ec49c36959fb96028e2ca Mon Sep 17 00:00:00 2001 From: Pablo Ontiveros Date: Fri, 10 Jul 2026 09:00:26 +0000 Subject: [PATCH] refactor(ingest): preserve cancellable Markdown contracts during API retirement --- .coverage-floors.toml | 1 - AGENTS.md | 28 +- internal/corrections/corrections.go | 45 +-- internal/corrections/corrections_test.go | 12 - internal/plans/plans.go | 94 ----- internal/plans/plans_test.go | 184 ---------- internal/sources/sources.go | 89 +---- internal/sources/sources_test.go | 180 +--------- internal/sync/noise.go | 47 --- internal/sync/noise_test.go | 37 -- internal/sync/sync.go | 229 +----------- internal/sync/sync_test.go | 421 +---------------------- 12 files changed, 38 insertions(+), 1329 deletions(-) delete mode 100644 internal/plans/plans.go delete mode 100644 internal/plans/plans_test.go delete mode 100644 internal/sync/noise.go diff --git a/.coverage-floors.toml b/.coverage-floors.toml index e4ea50dd..4ea363c3 100644 --- a/.coverage-floors.toml +++ b/.coverage-floors.toml @@ -6,7 +6,6 @@ packages = [ "internal/hybrid", "internal/input_config", "internal/models", - "internal/plans", "internal/projects", "internal/reader", "internal/readers", diff --git a/AGENTS.md b/AGENTS.md index 0adcd32b..51262cf6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,10 +8,10 @@ Backscroll is a Go CLI tool that indexes Claude Code, Pi, OMP, OpenCode, Codex C **Status**: Go port complete — `main` branch is the active Go implementation. The Rust implementation is frozen in the `v0` branch. - Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra). + Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra). github.com/pablontiv/backscroll/internal/autoupdate — Private context-aware release staging and safe replacement -Stack: cobra, go-toml/v2, goldmark, modernc.org/sqlite (pure Go, no CGO), stdlib testing. +Stack: cobra, go-toml/v2, modernc.org/sqlite (pure Go, no CGO), stdlib testing. ## Build & Test Commands @@ -64,17 +64,16 @@ internal/ ├── input_config/ — input manifest loading, discovery, and legacy session-dirs compatibility ├── presets/ — embedded input preset installation with explicit overwrite control ├── models/ — domain types: SessionRecord, MessageContent, ParsedFile, SearchResult, Stats -├── sync/ — WalkDir, SHA-256 dedup, JSONL parsing, noise filtering, content-type classification +├── sync/ — cancellable JSONL iteration, noise-type filtering, content cleaning, exit-code extraction ├── tagging/ — heuristic auto-tagging (debugging, refactoring, feature, testing, docs, config) -├── plans/ — Markdown plan parser (split by ## headers, goldmark) -├── sources/ — external source parsers (ke, decision, memory, rule, spec, backlog) + SourceRegistry +├── sources/ — cancellable whole-document and sectioned Markdown parsers shared by manifest readers ├── projects/ — project identity registry: LoadGlobalRegistry() (ProjectRegistry, error), Identify(), LoadLocalHint() ├── readers/ — SessionReader interface, Registry, ClaudeReader (text+tool_use+tool_result), PiReader (text+toolCall+custom results), OMPReader (native v3 text+thinking+toolCall+toolResult; parser-backed identities and origin), OpenCodeReader (text+tool state.input+state.output), CodexReader (rollout response items; docs/input-contract.md), MarkdownDocumentReader (`markdown_document`), MarkdownSectionsReader (`markdown_sections`); toolfmt serializer ├── recovery/ — stranded database recovery orchestration, verified active+stranded union, durable backup, atomic replacement, and post-install sync ├── startuplock/ — canonical database lock coordination via persistent OS advisory sidecar ├── skillinstall/ — safe Backscroll skill installation, receipts, rollback, and Python v1 restore compatibility ├── templates/ — F2 Drain-inspired miner: Miner, ProcessLine, ExtractErrorLines, deterministic signature via SHA256 -├── corrections/ — F3 correction detection: bilingual lexicon, interrupt flags, denial heuristics, rephrase-similarity; detector registry + implementations +├── corrections/ — F3 correction detection: bilingual lexicon, interrupt flags, denial heuristics, rephrase-similarity; prose-filtered cancellable orchestration ├── categories/ — F4 category map loader (rule engine, versioning, tool→category mapping) with embedded default preset ├── hybrid/ — Reciprocal Rank Fusion helpers for merged lexical retrieval ├── sequences/ — F4 PrefixSpan mining (deterministic discovery of frequent tool-call sequences per session) @@ -103,11 +102,10 @@ validated invocation → startup command class → canonical startup lock └─ remediation owner: retain mutation-grade lease → skip ordinary compatible-open and pre-handler sync → handler Ordinary startup sync inputs ────────────────────────────────┐ -JSONL files → fs.WalkDir → SHA-256 dedup → ParseSessions() ─┤ -Markdown plans → DiscoverPlanFiles() → ParsePlan() ──────────┤ -Declarative Markdown inputs → markdown readers ──────────────┤ +Native sessions → manifest discovery → native readers ───────┤ +Markdown documents/plans → manifest discovery → Markdown readers ▼ - SyncFiles() → perennial SQLite FTS5 + SHA-256 dedup → SyncFiles() → perennial SQLite FTS5 │ search/context/list/patterns/status/validate → database-backed query/output recover apply → post-install sync under retained remediation lease @@ -182,7 +180,6 @@ External knowledge sources are configured with active `*.inputs.toml` manifests - `github.com/spf13/cobra` — CLI argument parsing with subcommands - `modernc.org/sqlite` — Pure Go SQLite with FTS5, WAL mode (no CGO) - `github.com/pelletier/go-toml/v2` — TOML config parsing -- `github.com/yuin/goldmark` — Markdown parsing for plan indexing - `github.com/pablontiv/picokit` — Output formatting (text/robot/JSON), file hashing, path security (release updating is owned by private `internal/autoupdate`) - `crypto/sha256` (stdlib) — SHA-256 hashing for incremental sync deduplication - `io/fs` + `path/filepath` (stdlib) — Recursive directory traversal @@ -254,12 +251,11 @@ github.com/pablontiv/backscroll/internal/directsearch — Shared direct-search github.com/pablontiv/backscroll/internal/input_config — Input manifest loading, discovery, and legacy session-dirs compatibility github.com/pablontiv/backscroll/internal/presets — Embedded input preset installation with explicit overwrite control github.com/pablontiv/backscroll/internal/models — Domain types and SearchEngine interface -github.com/pablontiv/backscroll/internal/sync — Session parsing and noise filtering -github.com/pablontiv/backscroll/internal/plans — Markdown plan parsing +github.com/pablontiv/backscroll/internal/sync — Cancellable JSONL iteration, noise filtering, content cleaning, exit-code extraction github.com/pablontiv/backscroll/internal/tagging — Heuristic session auto-tagging -github.com/pablontiv/backscroll/internal/sources — External source parsers + SourceRegistry +github.com/pablontiv/backscroll/internal/sources — Cancellable whole-document and sectioned Markdown parsers github.com/pablontiv/backscroll/internal/templates — F2 Drain-inspired miner: Miner, ProcessLine, ExtractErrorLines -github.com/pablontiv/backscroll/internal/corrections — F3 correction-signal detectors: lexicon (es+en), interrupt, denial, rephrase-similarity; registry pattern for deterministic, pluggable detectors +github.com/pablontiv/backscroll/internal/corrections — F3 correction-signal detectors: lexicon (es+en), interrupt, denial, rephrase-similarity; prose-filtered cancellable orchestration github.com/pablontiv/backscroll/internal/categories — F4 category map loader (rule engine, versioning, tool→category mapping) with embedded default preset github.com/pablontiv/backscroll/internal/hybrid — Reciprocal Rank Fusion helpers for merged lexical retrieval github.com/pablontiv/backscroll/internal/sequences — F4 PrefixSpan mining (deterministic pattern discovery per session) @@ -268,7 +264,7 @@ github.com/pablontiv/backscroll/internal/storage — Database schema, migr github.com/pablontiv/backscroll/internal/projects — Project identity registry github.com/pablontiv/backscroll/internal/readers — SessionReader interface, Registry, ClaudeReader (text+tool_use+tool_result), PiReader (text+toolCall+custom results), OMPReader (native v3 text+thinking+toolCall+toolResult; parser-backed identities and origin), OpenCodeReader (text+tool state.input+state.output), CodexReader (rollout response items; docs/input-contract.md), MarkdownDocumentReader (`markdown_document`), MarkdownSectionsReader (`markdown_sections`); toolfmt serializer github.com/pablontiv/backscroll/internal/recovery — Stranded database recovery orchestration, durable backup, atomic replacement, and post-install sync - Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra). + Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra). github.com/pablontiv/backscroll/internal/autoupdate — Private context-aware release staging and safe replacement github.com/pablontiv/backscroll/internal/startuplock — Canonical database lock coordination via persistent OS advisory sidecar github.com/pablontiv/backscroll/internal/skillinstall — Safe Backscroll skill installation, receipts, rollback, and Python v1 restore compatibility diff --git a/internal/corrections/corrections.go b/internal/corrections/corrections.go index f38750d3..d4f93e87 100644 --- a/internal/corrections/corrections.go +++ b/internal/corrections/corrections.go @@ -14,41 +14,8 @@ type Detection struct { Confidence float64 } -// Detector is a function that analyzes a message at a given ordinal position -// within a session and returns a Detection if a correction signal is found. -// Returns nil if no signal is detected. -type Detector func(msgs []models.Message, idx int) *Detection - -// Registry maps detector names to their functions (for orderly, repeatable -// execution and testing). Detectors are registered at init() time. -var detectorRegistry map[string]Detector - -// Register adds a detector to the registry. Called at init() time. -func Register(name string, fn Detector) { - detectorRegistry[name] = fn -} - -// RunDetectors executes all registered detectors over a session's messages. -// Returns a map of message ordinal -> slice of Detections (detectors that fired). -// The slice is sorted by detector name for determinism. -func RunDetectors(msgs []models.Message) map[int][]Detection { - if len(msgs) == 0 { - return nil - } - result := make(map[int][]Detection) - for i := range msgs { - for name, fn := range detectorRegistry { - if detection := fn(msgs, i); detection != nil { - detection.DetectorName = name - result[i] = append(result[i], *detection) - } - } - } - return result -} - -// RunDetectorsFiltered executes detectors with prose-only filtering without -// cancellation. It is retained for compatibility with existing callers. +// RunDetectorsFiltered executes prose-filtered detection for callers without +// a cancellation context, including stored-text backfill. func RunDetectorsFiltered(msgs []models.Message) map[int][]Detection { result, _ := RunDetectorsFilteredContext(context.Background(), msgs) return result @@ -345,11 +312,3 @@ func cRephraseDetector(msgs []models.Message, idx int) *Detection { } return nil } - -func init() { - detectorRegistry = make(map[string]Detector) - Register("lexicon", cCorrectionLexiconDetector) - Register("interrupt", cInterruptDetector) - Register("denial", cDenialDetector) - Register("rephrase", cRephraseDetector) -} diff --git a/internal/corrections/corrections_test.go b/internal/corrections/corrections_test.go index b1175e05..ce66a04d 100644 --- a/internal/corrections/corrections_test.go +++ b/internal/corrections/corrections_test.go @@ -15,18 +15,6 @@ func testMsg(role, content string) models.Message { } } -func TestDetectorRegistry(t *testing.T) { - // Test that registered detectors can be run - result := RunDetectors([]models.Message{{Role: "user", Content: "test"}}) - if result == nil { - result = make(map[int][]Detection) - } - // For a normal non-correction message, no detectors should fire - if len(result) != 0 { - t.Errorf("expected no detections for normal message, got %d", len(result)) - } -} - func TestCorrectionLexiconDetector(t *testing.T) { tests := []struct { name string diff --git a/internal/plans/plans.go b/internal/plans/plans.go deleted file mode 100644 index 2f53aa7d..00000000 --- a/internal/plans/plans.go +++ /dev/null @@ -1,94 +0,0 @@ -package plans - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -// PlanSection represents a section of a markdown plan file. -type PlanSection struct { - Title string - Content string // full section content including title - Source string // always "plan" -} - -// ParsePlan parses a markdown file and returns one PlanSection per ## header. -// If no ## headers are found, the entire file is returned as a single section. -func ParsePlan(path string) ([]PlanSection, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read plan file: %w", err) - } - - content := string(data) - lines := strings.Split(content, "\n") - - var sections []PlanSection - var currentSection *PlanSection - var currentContent strings.Builder - - for _, line := range lines { - // Check if this line starts a new section with ## - if strings.HasPrefix(line, "## ") { - // Save previous section if any - if currentSection != nil { - currentSection.Content = strings.TrimSpace(currentContent.String()) - sections = append(sections, *currentSection) - currentContent.Reset() - } - - // Start new section - title := strings.TrimPrefix(line, "## ") - title = strings.TrimSpace(title) - currentSection = &PlanSection{ - Title: title, - Content: "", // will be populated - Source: "plan", - } - currentContent.WriteString(line) - currentContent.WriteString("\n") - } else if currentSection != nil { - // Add to current section - currentContent.WriteString(line) - currentContent.WriteString("\n") - } - } - - // Save last section if any - if currentSection != nil { - currentSection.Content = strings.TrimSpace(currentContent.String()) - sections = append(sections, *currentSection) - } - - // If no sections were found (no ## headers), treat entire file as one section - if len(sections) == 0 { - sections = []PlanSection{ - { - Title: "Untitled", - Content: strings.TrimSpace(content), - Source: "plan", - }, - } - } - - return sections, nil -} - -// DiscoverPlanFiles returns all .md files in the given directory. -func DiscoverPlanFiles(dir string) ([]string, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return nil, fmt.Errorf("failed to read directory: %w", err) - } - - var files []string - for _, entry := range entries { - if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".md") { - files = append(files, filepath.Join(dir, entry.Name())) - } - } - - return files, nil -} diff --git a/internal/plans/plans_test.go b/internal/plans/plans_test.go deleted file mode 100644 index ae773300..00000000 --- a/internal/plans/plans_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package plans - -import ( - "os" - "path/filepath" - "testing" -) - -func TestParsePlan(t *testing.T) { - tests := []struct { - name string - content string - expectedCount int - expectedTitle string - }{ - { - name: "single section", - content: "## First Section\n\nContent here", - expectedCount: 1, - expectedTitle: "First Section", - }, - { - name: "multiple sections", - content: `## Overview -Content for overview - -## Details -More detailed content - -## Summary -Final summary`, - expectedCount: 3, - expectedTitle: "Overview", - }, - { - name: "no sections (no headers)", - content: "This is just plain content without any headers", - expectedCount: 1, - expectedTitle: "Untitled", - }, - { - name: "sections with blank lines", - content: `## First - -Content - -## Second - -More content`, - expectedCount: 2, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create temporary file - tmpfile, err := os.CreateTemp("", "plan_*.md") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer func() { _ = os.Remove(tmpfile.Name()) }() - - if _, err := tmpfile.WriteString(tt.content); err != nil { - t.Fatalf("failed to write to temp file: %v", err) - } - _ = tmpfile.Close() - - // Parse the file - sections, err := ParsePlan(tmpfile.Name()) - if err != nil { - t.Fatalf("ParsePlan failed: %v", err) - } - - if len(sections) != tt.expectedCount { - t.Errorf("expected %d sections, got %d", tt.expectedCount, len(sections)) - } - - if tt.expectedTitle != "" && len(sections) > 0 { - if sections[0].Title != tt.expectedTitle { - t.Errorf("expected first title %q, got %q", tt.expectedTitle, sections[0].Title) - } - } - - // Verify all sections have Source = "plan" - for _, section := range sections { - if section.Source != "plan" { - t.Errorf("expected Source=%q, got %q", "plan", section.Source) - } - if section.Content == "" { - t.Errorf("section %q has empty content", section.Title) - } - } - }) - } -} - -func TestDiscoverPlanFiles(t *testing.T) { - // Create temp directory - tmpdir, err := os.MkdirTemp("", "plans_*") - if err != nil { - t.Fatalf("failed to create temp dir: %v", err) - } - defer func() { _ = os.RemoveAll(tmpdir) }() - - // Create some test files - testFiles := []string{"plan1.md", "plan2.md", "notes.txt", "README.md"} - for _, f := range testFiles { - path := filepath.Join(tmpdir, f) - if err := os.WriteFile(path, []byte("test"), 0o644); err != nil { - t.Fatalf("failed to create test file: %v", err) - } - } - - // Discover files - files, err := DiscoverPlanFiles(tmpdir) - if err != nil { - t.Fatalf("DiscoverPlanFiles failed: %v", err) - } - - // Should find 3 .md files (plan1.md, plan2.md, README.md) - if len(files) != 3 { - t.Errorf("expected 3 .md files, got %d", len(files)) - } - - // Verify all returned files are .md - for _, f := range files { - if filepath.Ext(f) != ".md" { - t.Errorf("expected .md file, got %s", f) - } - } -} - -func TestDiscoverPlanFilesEmpty(t *testing.T) { - // Create temp directory with no files - tmpdir, err := os.MkdirTemp("", "plans_empty_*") - if err != nil { - t.Fatalf("failed to create temp dir: %v", err) - } - defer func() { _ = os.RemoveAll(tmpdir) }() - - // Discover files - files, err := DiscoverPlanFiles(tmpdir) - if err != nil { - t.Fatalf("DiscoverPlanFiles failed: %v", err) - } - - if len(files) != 0 { - t.Errorf("expected 0 files, got %d", len(files)) - } -} - -func TestDiscoverPlanFilesVerifyMd(t *testing.T) { - // Create temp directory - tmpdir, err := os.MkdirTemp("", "plans_verify_*") - if err != nil { - t.Fatalf("failed to create temp dir: %v", err) - } - defer func() { _ = os.RemoveAll(tmpdir) }() - - // Create test files - testFiles := []string{"plan1.md", "plan2.md", "notes.txt"} - for _, f := range testFiles { - path := filepath.Join(tmpdir, f) - if err := os.WriteFile(path, []byte("test"), 0o644); err != nil { - t.Fatalf("failed to create test file: %v", err) - } - } - - files, err := DiscoverPlanFiles(tmpdir) - if err != nil { - t.Fatalf("DiscoverPlanFiles failed: %v", err) - } - - // Should find 2 .md files - if len(files) != 2 { - t.Errorf("expected 2 .md files, got %d: %v", len(files), files) - } - - for _, f := range files { - if filepath.Ext(f) != ".md" { - t.Errorf("expected .md file, got %s", f) - } - } -} diff --git a/internal/sources/sources.go b/internal/sources/sources.go index 1cd1c512..64dd92a8 100644 --- a/internal/sources/sources.go +++ b/internal/sources/sources.go @@ -17,86 +17,10 @@ type SourceItem struct { Path string // path to the source file } -// SourceConfig contains paths for different source types. -type SourceConfig struct { - KE []string `toml:"ke"` - Decisions []string `toml:"decisions"` - Memories []string `toml:"memories"` - Rules []string `toml:"rules"` - Specs []string `toml:"specs"` - Backlog []string `toml:"backlog"` -} - -// ParseAll returns all source items from the configured paths. -func ParseAll(cfg SourceConfig) ([]SourceItem, error) { - var items []SourceItem - - // KE: whole-document parsing - for _, path := range cfg.KE { - item, err := ParseDocument(path, "ke") - if err != nil { - return nil, fmt.Errorf("failed to parse KE %s: %w", path, err) - } - items = append(items, item) - } - - // Decisions: sectioned (by ## headers) - for _, path := range cfg.Decisions { - sectioned, err := ParseSectioned(path, "decision") - if err != nil { - return nil, fmt.Errorf("failed to parse Decision %s: %w", path, err) - } - items = append(items, sectioned...) - } - - // Memories: whole-document parsing - for _, path := range cfg.Memories { - item, err := ParseDocument(path, "memory") - if err != nil { - return nil, fmt.Errorf("failed to parse Memory %s: %w", path, err) - } - items = append(items, item) - } - - // Rules: sectioned (by ## headers) - for _, path := range cfg.Rules { - sectioned, err := ParseSectioned(path, "rule") - if err != nil { - return nil, fmt.Errorf("failed to parse Rule %s: %w", path, err) - } - items = append(items, sectioned...) - } - - // Specs: sectioned (by ## headers) - for _, path := range cfg.Specs { - sectioned, err := ParseSectioned(path, "spec") - if err != nil { - return nil, fmt.Errorf("failed to parse Spec %s: %w", path, err) - } - items = append(items, sectioned...) - } - - // Backlog: sectioned (by ## headers) or whole-document - for _, path := range cfg.Backlog { - sectioned, err := ParseSectioned(path, "backlog") - if err != nil { - return nil, fmt.Errorf("failed to parse Backlog %s: %w", path, err) - } - items = append(items, sectioned...) - } - - return items, nil -} - const sourceReadBufferSize = 32 * 1024 -// ParseDocument parses a whole-document source (single item). -// Extracts frontmatter ID and returns the entire content as one item. -func ParseDocument(path string, sourceType string) (SourceItem, error) { - return ParseDocumentContext(context.Background(), path, sourceType) -} - -// ParseDocumentContext is ParseDocument with cancellation support. +// ParseDocumentContext parses a whole-document source with cancellation support. +// It extracts the frontmatter ID and returns the entire content as one item. func ParseDocumentContext(ctx context.Context, path string, sourceType string) (SourceItem, error) { content, id, err := prepareSource(ctx, path, sourceType) if err != nil { @@ -111,13 +35,8 @@ func ParseDocumentContext(ctx context.Context, path string, sourceType string) ( }, nil } -// ParseSectioned parses a markdown file split by ## headers (each section = item). -// If no ## headers are found, returns the entire content as a single item. -func ParseSectioned(path string, sourceType string) ([]SourceItem, error) { - return ParseSectionedContext(context.Background(), path, sourceType) -} - -// ParseSectionedContext is ParseSectioned with cancellation support. +// ParseSectionedContext splits a Markdown source by ## headers with cancellation support. +// If no ## headers are found, it returns the entire content as a single item. func ParseSectionedContext(ctx context.Context, path string, sourceType string) ([]SourceItem, error) { content, baseID, err := prepareSource(ctx, path, sourceType) if err != nil { diff --git a/internal/sources/sources_test.go b/internal/sources/sources_test.go index bfe99dda..1ea40e39 100644 --- a/internal/sources/sources_test.go +++ b/internal/sources/sources_test.go @@ -54,7 +54,7 @@ func TestSplitSectionsCancelsDuringTraversalWithoutPartialItems(t *testing.T) { } } -func TestParseDocument(t *testing.T) { +func TestParseDocumentContext(t *testing.T) { tests := []struct { name string content string @@ -114,9 +114,9 @@ Rules are plain markdown with instructions.`, _ = tmpfile.Close() // Parse the file - item, err := ParseDocument(tmpfile.Name(), tt.sourceType) + item, err := ParseDocumentContext(context.Background(), tmpfile.Name(), tt.sourceType) if err != nil { - t.Fatalf("ParseDocument failed: %v", err) + t.Fatalf("ParseDocumentContext failed: %v", err) } if item.ID != tt.expectedID { @@ -132,7 +132,7 @@ Rules are plain markdown with instructions.`, } } -func TestParseSectioned(t *testing.T) { +func TestParseSectionedContext(t *testing.T) { tests := []struct { name string content string @@ -209,9 +209,9 @@ Investigation needed for test purposes.`, _ = tmpfile.Close() // Parse the file - items, err := ParseSectioned(tmpfile.Name(), tt.sourceType) + items, err := ParseSectionedContext(context.Background(), tmpfile.Name(), tt.sourceType) if err != nil { - t.Fatalf("ParseSectioned failed: %v", err) + t.Fatalf("ParseSectionedContext failed: %v", err) } if len(items) != tt.expectedItemCount { @@ -312,95 +312,6 @@ func TestMarkdownPreparationContracts(t *testing.T) { } } -func TestParseAll(t *testing.T) { - // Create temp directory and files - tmpdir, err := os.MkdirTemp("", "sources_all_*") - if err != nil { - t.Fatalf("failed to create temp dir: %v", err) - } - defer func() { _ = os.RemoveAll(tmpdir) }() - - // Create test files - keContent := `--- -id: KE-0001 -estado: activo ---- - -# Kyverno admission webhook timeout - -## Symptom - -Pods fail to schedule with admission webhook timeout errors.` - - decisionContent := `--- -id: DEC-AUTH-001 -status: accepted -scope: technical ---- - -# Implement OAuth 2.0 for API Authentication - -## Rationale - -OAuth 2.0 provides industry-standard security guarantees.` - - memoryContent := `--- -name: feedback_example -description: Example feedback memory -type: feedback ---- - -Always verify state after deploy before declaring complete.` - - keFile := filepath.Join(tmpdir, "ke.md") - decisionFile := filepath.Join(tmpdir, "decision.md") - memoryFile := filepath.Join(tmpdir, "memory.md") - - if err := os.WriteFile(keFile, []byte(keContent), 0o644); err != nil { - t.Fatalf("failed to create KE file: %v", err) - } - if err := os.WriteFile(decisionFile, []byte(decisionContent), 0o644); err != nil { - t.Fatalf("failed to create Decision file: %v", err) - } - if err := os.WriteFile(memoryFile, []byte(memoryContent), 0o644); err != nil { - t.Fatalf("failed to create Memory file: %v", err) - } - - // Create config - cfg := SourceConfig{ - KE: []string{keFile}, - Decisions: []string{decisionFile}, - Memories: []string{memoryFile}, - } - - // Parse all - items, err := ParseAll(cfg) - if err != nil { - t.Fatalf("ParseAll failed: %v", err) - } - - // We expect: 1 KE + 1 Decision (with 1 section) + 1 Memory = 3 items - if len(items) != 3 { - t.Errorf("expected 3 items, got %d", len(items)) - } - - // Verify source types - sources := make(map[string]int) - for _, item := range items { - sources[item.Source]++ - } - - if sources["ke"] != 1 { - t.Errorf("expected 1 ke item, got %d", sources["ke"]) - } - if sources["decision"] != 1 { - t.Errorf("expected 1 decision item, got %d", sources["decision"]) - } - if sources["memory"] != 1 { - t.Errorf("expected 1 memory item, got %d", sources["memory"]) - } -} - func TestExtractID(t *testing.T) { tests := []struct { name string @@ -463,61 +374,6 @@ No frontmatter here.`, } } -func TestParseAllWithAllTypes(t *testing.T) { - tmpdir, err := os.MkdirTemp("", "sources_all_types_*") - if err != nil { - t.Fatalf("create tmpdir: %v", err) - } - defer func() { _ = os.RemoveAll(tmpdir) }() - - writeFile := func(name, content string) string { - path := filepath.Join(tmpdir, name) - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatalf("write %s: %v", name, err) - } - return path - } - - ruleFile := writeFile("rule.md", "---\nid: RULE-001\n---\n## Rule One\nDo this.\n## Rule Two\nDo that.") - specFile := writeFile("spec.md", "---\nid: SPEC-001\n---\n## Spec One\nSpec detail.") - backlogFile := writeFile("backlog.md", "---\nid: BACK-001\n---\n## Task One\nDescription.") - - cfg := SourceConfig{ - Rules: []string{ruleFile}, - Specs: []string{specFile}, - Backlog: []string{backlogFile}, - } - - items, err := ParseAll(cfg) - if err != nil { - t.Fatalf("ParseAll: %v", err) - } - - src := make(map[string]int) - for _, item := range items { - src[item.Source]++ - } - if src["rule"] == 0 { - t.Error("expected at least one rule item") - } - if src["spec"] == 0 { - t.Error("expected at least one spec item") - } - if src["backlog"] == 0 { - t.Error("expected at least one backlog item") - } -} - -func TestParseAllMissingFile(t *testing.T) { - cfg := SourceConfig{ - KE: []string{"/nonexistent/ke.md"}, - } - _, err := ParseAll(cfg) - if err == nil { - t.Error("expected error for missing file, got nil") - } -} - func TestFixtureFiles(t *testing.T) { // Test with actual fixture files if they exist fixtureDir := "/home/shared/harness/backscroll/tests/fixtures" @@ -525,9 +381,9 @@ func TestFixtureFiles(t *testing.T) { // Test KE fixture keFile := filepath.Join(fixtureDir, "ke-001.md") if _, err := os.Stat(keFile); err == nil { - item, err := ParseDocument(keFile, "ke") + item, err := ParseDocumentContext(context.Background(), keFile, "ke") if err != nil { - t.Errorf("ParseDocument(ke-001.md) failed: %v", err) + t.Errorf("ParseDocumentContext(ke-001.md) failed: %v", err) } else { if item.ID == "" { t.Errorf("KE fixture: empty ID") @@ -541,9 +397,9 @@ func TestFixtureFiles(t *testing.T) { // Test Decision fixture (sectioned) decisionFile := filepath.Join(fixtureDir, "decisions-test-001.md") if _, err := os.Stat(decisionFile); err == nil { - items, err := ParseSectioned(decisionFile, "decision") + items, err := ParseSectionedContext(context.Background(), decisionFile, "decision") if err != nil { - t.Errorf("ParseSectioned(decisions-test-001.md) failed: %v", err) + t.Errorf("ParseSectionedContext(decisions-test-001.md) failed: %v", err) } else { if len(items) == 0 { t.Errorf("Decision fixture: no items parsed") @@ -559,9 +415,9 @@ func TestFixtureFiles(t *testing.T) { // Test Memory fixture memoryFile := filepath.Join(fixtureDir, "memory-test.md") if _, err := os.Stat(memoryFile); err == nil { - item, err := ParseDocument(memoryFile, "memory") + item, err := ParseDocumentContext(context.Background(), memoryFile, "memory") if err != nil { - t.Errorf("ParseDocument(memory-test.md) failed: %v", err) + t.Errorf("ParseDocumentContext(memory-test.md) failed: %v", err) } else { if item.Source != "memory" { t.Errorf("Memory fixture: wrong source type: %s", item.Source) @@ -572,9 +428,9 @@ func TestFixtureFiles(t *testing.T) { // Test Spec fixture (sectioned) specFile := filepath.Join(fixtureDir, "spec-test.md") if _, err := os.Stat(specFile); err == nil { - items, err := ParseSectioned(specFile, "spec") + items, err := ParseSectionedContext(context.Background(), specFile, "spec") if err != nil { - t.Errorf("ParseSectioned(spec-test.md) failed: %v", err) + t.Errorf("ParseSectionedContext(spec-test.md) failed: %v", err) } else { if len(items) == 0 { t.Errorf("Spec fixture: no items parsed") @@ -590,9 +446,9 @@ func TestFixtureFiles(t *testing.T) { // Test Rule fixture ruleFile := filepath.Join(fixtureDir, "rule-test.md") if _, err := os.Stat(ruleFile); err == nil { - items, err := ParseSectioned(ruleFile, "rule") + items, err := ParseSectionedContext(context.Background(), ruleFile, "rule") if err != nil { - t.Errorf("ParseSectioned(rule-test.md) failed: %v", err) + t.Errorf("ParseSectionedContext(rule-test.md) failed: %v", err) } else { if len(items) == 0 { t.Errorf("Rule fixture: no items parsed") @@ -608,9 +464,9 @@ func TestFixtureFiles(t *testing.T) { // Test Backlog fixture backlogFile := filepath.Join(fixtureDir, "backlog-test.md") if _, err := os.Stat(backlogFile); err == nil { - items, err := ParseSectioned(backlogFile, "backlog") + items, err := ParseSectionedContext(context.Background(), backlogFile, "backlog") if err != nil { - t.Errorf("ParseSectioned(backlog-test.md) failed: %v", err) + t.Errorf("ParseSectionedContext(backlog-test.md) failed: %v", err) } else { if len(items) == 0 { t.Errorf("Backlog fixture: no items parsed") diff --git a/internal/sync/noise.go b/internal/sync/noise.go deleted file mode 100644 index 729f26b6..00000000 --- a/internal/sync/noise.go +++ /dev/null @@ -1,47 +0,0 @@ -package sync - -import ( - "regexp" - "sync" -) - -// Noise filter patterns compiled at init time (LazyLock equivalent) -var ( - noisePatternOnce sync.Once - noisePatterns *NoisePatterns -) - -// NoisePatterns holds compiled regex patterns for noise filtering -type NoisePatterns struct { - // Type patterns that should be filtered - FilteredTypes map[string]bool - - // Content patterns for additional filtering - SystemReminderPattern *regexp.Regexp - TaskNotificationPattern *regexp.Regexp - CaveatPattern *regexp.Regexp - CommandPattern *regexp.Regexp -} - -// GetNoisePatterns returns the lazily-initialized noise patterns -func GetNoisePatterns() *NoisePatterns { - noisePatternOnce.Do(func() { - noisePatterns = &NoisePatterns{ - FilteredTypes: map[string]bool{ - "system-reminder": true, - "task-notification": true, - "command": true, - "command-result": true, - "local-command-caveat": true, - "progress": true, - }, - // These are compiled but not actively used in the current implementation - // They're here for potential future use - SystemReminderPattern: regexp.MustCompile(`.*?`), - TaskNotificationPattern: regexp.MustCompile(`.*?`), - CaveatPattern: regexp.MustCompile(`Caveat:.*?(?:\n|$)`), - CommandPattern: regexp.MustCompile(`<(?:command|local-command-caveat)>.*?`), - } - }) - return noisePatterns -} diff --git a/internal/sync/noise_test.go b/internal/sync/noise_test.go index 20bd3da6..4b32b242 100644 --- a/internal/sync/noise_test.go +++ b/internal/sync/noise_test.go @@ -4,43 +4,6 @@ import ( "testing" ) -func TestGetNoisePatterns(t *testing.T) { - np := GetNoisePatterns() - if np == nil { - t.Fatal("GetNoisePatterns returned nil") - } - - // Check filtered types - for _, typ := range []string{"system-reminder", "task-notification", "command", "command-result", "local-command-caveat", "progress"} { - if !np.FilteredTypes[typ] { - t.Errorf("expected FilteredTypes[%q] = true", typ) - } - } - - // Patterns should be compiled - if np.SystemReminderPattern == nil { - t.Error("SystemReminderPattern is nil") - } - if np.TaskNotificationPattern == nil { - t.Error("TaskNotificationPattern is nil") - } - if np.CaveatPattern == nil { - t.Error("CaveatPattern is nil") - } - if np.CommandPattern == nil { - t.Error("CommandPattern is nil") - } -} - -func TestGetNoisePatternsLazy(t *testing.T) { - // Calling twice should return the same pointer (lazy init) - np1 := GetNoisePatterns() - np2 := GetNoisePatterns() - if np1 != np2 { - t.Error("GetNoisePatterns should return the same pointer (singleton)") - } -} - func TestIsNoiseType(t *testing.T) { tests := []struct { typ string diff --git a/internal/sync/sync.go b/internal/sync/sync.go index 19b61df8..465343c7 100644 --- a/internal/sync/sync.go +++ b/internal/sync/sync.go @@ -1,156 +1,6 @@ package sync -import ( - "encoding/json" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "time" - - "github.com/pablontiv/backscroll/internal/models" -) - -// ParseSessions parses a JSONL session file and returns Message records. -// It defensively handles both Claude and Pi formats. -func ParseSessions(path string) ([]models.Message, error) { - var messages []models.Message - err := IterateJSONLFile(path, func(_ int, line []byte) error { - var rawRec rawRecord - if err := json.Unmarshal(line, &rawRec); err != nil { - // Skip malformed lines - return nil - } - - // Skip noise records - if IsNoiseRecord(rawRec) { - return nil - } - - // Extract message if present - if rawRec.Message == nil { - return nil - } - - // Parse timestamp - ts, err := time.Parse(time.RFC3339, rawRec.Timestamp) - if err != nil { - ts = time.Now() - } - - // Extract content and content type - content, contentType := extractContent(rawRec.Message.Content) - if content == "" { - return nil - } - - messages = append(messages, models.Message{ - Role: rawRec.Message.Role, - Content: content, - ContentType: contentType, - Timestamp: ts, - }) - return nil - }) - if err != nil { - return nil, fmt.Errorf("scan session file: %w", err) - } - - return messages, nil -} - -// WalkSessionDirs traverses session directories and returns parseable JSONL paths. -// If includeAgents is false, subagent sessions are skipped. -func WalkSessionDirs(dirs []string, includeAgents bool) ([]string, error) { - var jsonlPaths []string - - for _, dir := range dirs { - err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - // Skip inaccessible directories - return nil - } - - // Skip subagent directories unless explicitly included - if !includeAgents && strings.Contains(path, string(filepath.Separator)+"subagents"+string(filepath.Separator)) { - if d.IsDir() { - return filepath.SkipDir - } - return nil - } - - // Collect .jsonl files - if !d.IsDir() && strings.HasSuffix(path, ".jsonl") { - jsonlPaths = append(jsonlPaths, path) - } - - return nil - }) - - if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("walk directory %s: %w", dir, err) - } - } - - return jsonlPaths, nil -} - -// IsNoiseRecord returns true if the record should be filtered out. -func IsNoiseRecord(r rawRecord) bool { - // Filter by type - if IsNoiseType(r.Type) { - return true - } - - // Filter metadata records - if r.IsMeta { - return true - } - - // Filter records with empty role - if r.Message == nil || r.Message.Role == "" { - return true - } - - // Filter records where content extraction yields nothing - content, _ := extractContent(r.Message.Content) - return content == "" -} - -// rawRecord covers both Claude and Pi formats defensively -type rawRecord struct { - Type string `json:"type"` - UUID string `json:"uuid"` - ID string `json:"id"` - SessionID string `json:"sessionId"` - ParentID string `json:"parentId"` - Timestamp string `json:"timestamp"` - CWD string `json:"cwd"` - IsAgent bool `json:"isAgent"` - IsMeta bool `json:"isMeta"` - Message *rawMessage `json:"message"` -} - -type rawMessage struct { - Role string `json:"role"` - Content json.RawMessage `json:"content"` -} - -type rawBlock struct { - Type string `json:"type"` - Text string `json:"text"` - Thinking string `json:"thinking"` - ToolCall json.RawMessage `json:"toolCall"` - ToolResult json.RawMessage `json:"toolResult"` - ToolCallID string `json:"toolCallId"` - ToolUseID string `json:"tool_use_id"` - ID string `json:"id"` - Name string `json:"name"` - Input json.RawMessage `json:"input"` - Arguments json.RawMessage `json:"arguments"` - Content json.RawMessage `json:"content"` -} +import "strings" // IsNoiseType returns true if the type should be filtered func IsNoiseType(typ string) bool { @@ -165,72 +15,6 @@ func IsNoiseType(typ string) bool { return noiseTypes[typ] } -// extractContent extracts text from either a string or array of blocks. -// Returns (content, contentType). -func extractContent(raw json.RawMessage) (string, string) { - if raw == nil { - return "", "text" - } - - // Try parsing as string first - var strContent string - if err := json.Unmarshal(raw, &strContent); err == nil { - content := CleanContent(strContent) - return content, classifyContentType(content, false) - } - - // Try parsing as array of blocks - var blocks []rawBlock - if err := json.Unmarshal(raw, &blocks); err == nil { - return extractFromBlocks(blocks) - } - - return "", "text" -} - -// extractFromBlocks extracts content and determines type from an array of blocks -func extractFromBlocks(blocks []rawBlock) (string, string) { - var textParts []string - hasToolCall := false - hasCode := false - - for _, block := range blocks { - switch block.Type { - case "text": - cleaned := CleanContent(block.Text) - if cleaned != "" { - textParts = append(textParts, cleaned) - if strings.Contains(cleaned, "```") { - hasCode = true - } - } - case "thinking": - // Ignore thinking blocks (hidden reasoning) - case "tool_use", "toolCall": - hasToolCall = true - case "tool_result", "toolResult": - // Ignore tool result blocks in message content - case "tool_use_id": - hasToolCall = true - } - } - - content := strings.TrimSpace(strings.Join(textParts, " ")) - if content == "" { - return "", "text" - } - - // Determine content type - contentType := "text" - if hasCode { - contentType = "code" - } else if hasToolCall { - contentType = "tool" - } - - return content, contentType -} - // CleanContent removes noise patterns from content func CleanContent(content string) string { if content == "" { @@ -284,14 +68,3 @@ func CleanContent(content string) string { return content } - -// classifyContentType determines content type from text -func classifyContentType(content string, hasCode bool) string { - if hasCode { - return "code" - } - if strings.Contains(content, "```") { - return "code" - } - return "text" -} diff --git a/internal/sync/sync_test.go b/internal/sync/sync_test.go index a1ecc3f5..de369d37 100644 --- a/internal/sync/sync_test.go +++ b/internal/sync/sync_test.go @@ -1,380 +1,6 @@ package sync -import ( - "os" - "strings" - "testing" -) - -// TestParseSessions_PiFormat tests parsing Pi format sessions -func TestParseSessions_PiFormat(t *testing.T) { - path := "../../tests/fixtures/pi-session.jsonl" - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Skipf("fixture file not found: %s", path) - } - - messages, err := ParseSessions(path) - if err != nil { - t.Fatalf("ParseSessions failed: %v", err) - } - - if len(messages) == 0 { - t.Fatal("expected messages, got none") - } - - // Check we have user and assistant roles - hasUser := false - hasAssistant := false - - for _, msg := range messages { - if msg.Role == "user" { - hasUser = true - } - if msg.Role == "assistant" { - hasAssistant = true - } - } - - if !hasUser { - t.Error("expected user role, got none") - } - if !hasAssistant { - t.Error("expected assistant role, got none") - } - - // Check timestamp parsing - for _, msg := range messages { - if msg.Timestamp.IsZero() { - t.Error("expected non-zero timestamp") - } - } -} - -// TestParseSessions_ClaudeFormat tests parsing Claude format sessions -func TestParseSessions_ClaudeFormat(t *testing.T) { - path := "../../tests/fixtures/claude-preset/projects/project-a/session-main.jsonl" - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Skipf("fixture file not found: %s", path) - } - - messages, err := ParseSessions(path) - if err != nil { - t.Fatalf("ParseSessions failed: %v", err) - } - - if len(messages) == 0 { - t.Fatal("expected messages, got none") - } - - // Verify noise filtering worked: - // - "progress" type should be filtered - // - isMeta records should be filtered - // - task-notification type should be filtered - // - System reminder content should be removed - - for _, msg := range messages { - if strings.Contains(msg.Content, "system-reminder") { - t.Error("system-reminder should be filtered from content") - } - if strings.Contains(msg.Content, "drop progress") { - t.Error("progress type message should be filtered") - } - if strings.Contains(msg.Content, "drop metadata") { - t.Error("metadata (isMeta) message should be filtered") - } - if strings.Contains(msg.Content, "remove whole notification") { - t.Error("task-notification should be filtered") - } - } - - // Check we have user and assistant roles - hasUser := false - hasAssistant := false - for _, msg := range messages { - if msg.Role == "user" { - hasUser = true - } - if msg.Role == "assistant" { - hasAssistant = true - } - } - - if !hasUser { - t.Error("expected user role") - } - if !hasAssistant { - t.Error("expected assistant role") - } -} - -// TestNoiseFiltering_ToolEvents tests that tool events are parsed but not tool result content -func TestNoiseFiltering_ToolEvents(t *testing.T) { - path := "../../tests/fixtures/claude-tool-events.jsonl" - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Skipf("fixture file not found: %s", path) - } - - messages, err := ParseSessions(path) - if err != nil { - t.Fatalf("ParseSessions failed: %v", err) - } - - if len(messages) == 0 { - t.Fatal("expected messages, got none") - } - - // Should have parsed messages despite tool events - hasNormalMessage := false - hasToolContent := false - - for _, msg := range messages { - if strings.Contains(msg.Content, "normal claude message") { - hasNormalMessage = true - } - if msg.ContentType == "tool" { - hasToolContent = true - } - } - - if !hasNormalMessage { - t.Error("expected to find normal message") - } - if !hasToolContent { - t.Error("expected to find tool content type") - } -} - -// TestWalkSessionDirs_Exclusion tests subagent filtering -func TestWalkSessionDirs_Exclusion(t *testing.T) { - dir := "../../tests/fixtures/claude-preset" - if _, err := os.Stat(dir); os.IsNotExist(err) { - t.Skipf("fixture directory not found: %s", dir) - } - - // With includeAgents=false, subagent files should be excluded - paths, err := WalkSessionDirs([]string{dir}, false) - if err != nil { - t.Fatalf("WalkSessionDirs failed: %v", err) - } - - for _, p := range paths { - if strings.Contains(p, "/subagents/") { - t.Errorf("subagent file should be excluded: %s", p) - } - } - - // With includeAgents=true, subagent files should be included - pathsWithAgents, err := WalkSessionDirs([]string{dir}, true) - if err != nil { - t.Fatalf("WalkSessionDirs with includeAgents failed: %v", err) - } - - hasSubagent := false - for _, p := range pathsWithAgents { - if strings.Contains(p, "/subagents/") { - hasSubagent = true - break - } - } - - if !hasSubagent { - t.Error("expected to find subagent files with includeAgents=true") - } -} - -// TestIsNoiseRecord_FilterTypes tests that noise types are filtered -func TestIsNoiseRecord_FilterTypes(t *testing.T) { - tests := []struct { - name string - record rawRecord - wantNoise bool - }{ - { - name: "system-reminder type", - record: rawRecord{ - Type: "system-reminder", - Message: &rawMessage{ - Role: "user", - Content: []byte(`"content"`), - }, - }, - wantNoise: true, - }, - { - name: "task-notification type", - record: rawRecord{ - Type: "task-notification", - Message: &rawMessage{ - Role: "assistant", - Content: []byte(`"content"`), - }, - }, - wantNoise: true, - }, - { - name: "progress type", - record: rawRecord{ - Type: "progress", - Message: &rawMessage{ - Role: "assistant", - Content: []byte(`"content"`), - }, - }, - wantNoise: true, - }, - { - name: "isMeta true", - record: rawRecord{ - Type: "user", - IsMeta: true, - Message: &rawMessage{ - Role: "user", - Content: []byte(`"content"`), - }, - }, - wantNoise: true, - }, - { - name: "empty role", - record: rawRecord{ - Type: "message", - Message: &rawMessage{ - Role: "", - Content: []byte(`"content"`), - }, - }, - wantNoise: true, - }, - { - name: "nil message", - record: rawRecord{ - Type: "message", - Message: nil, - }, - wantNoise: true, - }, - { - name: "valid user message", - record: rawRecord{ - Type: "user", - Message: &rawMessage{ - Role: "user", - Content: []byte(`"valid content"`), - }, - }, - wantNoise: false, - }, - { - name: "valid assistant message", - record: rawRecord{ - Type: "assistant", - Message: &rawMessage{ - Role: "assistant", - Content: []byte(`"valid content"`), - }, - }, - wantNoise: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := IsNoiseRecord(tt.record) - if got != tt.wantNoise { - t.Errorf("IsNoiseRecord() = %v, want %v", got, tt.wantNoise) - } - }) - } -} - -// TestExtractContent_StringFormat tests string content extraction -func TestExtractContent_StringFormat(t *testing.T) { - tests := []struct { - name string - input string - wantContent string - wantType string - }{ - { - name: "simple string", - input: `"hello world"`, - wantContent: "hello world", - wantType: "text", - }, - { - name: "string with code fence", - input: "\"```go\\nfunc main() {}\\n```\"", - wantContent: "```go func main() {} ```", - wantType: "code", - }, - { - name: "empty string", - input: `""`, - wantContent: "", - wantType: "text", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - content, contentType := extractContent([]byte(tt.input)) - - // For empty content, we expect empty string - if tt.wantContent == "" { - if content != "" { - t.Errorf("expected empty content, got %q", content) - } - return - } - - if !strings.Contains(content, strings.TrimSpace(tt.wantContent)) { - t.Errorf("content = %q, want to contain %q", content, tt.wantContent) - } - if contentType != tt.wantType { - t.Errorf("contentType = %q, want %q", contentType, tt.wantType) - } - }) - } -} - -// TestExtractContent_BlockFormat tests block array content extraction -func TestExtractContent_BlockFormat(t *testing.T) { - tests := []struct { - name string - blocks string - wantType string - }{ - { - name: "text block", - blocks: `[{"type":"text","text":"hello"}]`, - wantType: "text", - }, - { - name: "text with code fence", - blocks: "[{\"type\":\"text\",\"text\":\"code: ```go\\nfunc main() {}\\n```\"}]", - wantType: "code", - }, - { - name: "tool_use block", - blocks: `[{"type":"text","text":"calling tool"},{"type":"tool_use","id":"t1","name":"bash"}]`, - wantType: "tool", - }, - { - name: "thinking ignored", - blocks: `[{"type":"thinking","thinking":"hidden"},{"type":"text","text":"visible"}]`, - wantType: "text", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, contentType := extractContent([]byte(tt.blocks)) - if contentType != tt.wantType { - t.Errorf("contentType = %q, want %q", contentType, tt.wantType) - } - }) - } -} +import "testing" // TestCleanContent tests noise pattern removal func TestCleanContent(t *testing.T) { @@ -454,48 +80,3 @@ func TestCleanContent(t *testing.T) { }) } } - -// TestParseSessions_IgnoresSubagents tests that subagent sessions are excluded when reading -func TestParseSessions_IgnoresSubagents(t *testing.T) { - path := "../../tests/fixtures/claude-preset/projects/project-a/subagents/agent.jsonl" - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Skipf("fixture file not found: %s", path) - } - - messages, err := ParseSessions(path) - if err != nil { - t.Fatalf("ParseSessions failed: %v", err) - } - - // Subagent file should be parsed if passed directly - // The filtering happens at the directory walk level - if len(messages) == 0 { - t.Log("no messages from subagent file (expected)") - } -} - -// BenchmarkParseSessions benchmarks session parsing -func BenchmarkParseSessions(b *testing.B) { - path := "../../tests/fixtures/pi-session.jsonl" - if _, err := os.Stat(path); os.IsNotExist(err) { - b.Skipf("fixture file not found: %s", path) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = ParseSessions(path) - } -} - -// BenchmarkWalkSessionDirs benchmarks directory walking -func BenchmarkWalkSessionDirs(b *testing.B) { - dir := "../../tests/fixtures" - if _, err := os.Stat(dir); os.IsNotExist(err) { - b.Skipf("fixture directory not found: %s", dir) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = WalkSessionDirs([]string{dir}, false) - } -}