diff --git a/.coverage-floors.toml b/.coverage-floors.toml
index e4ea50dd..4ea363c3 100644
--- a/.coverage-floors.toml
+++ b/.coverage-floors.toml
@@ -6,7 +6,6 @@ packages = [
"internal/hybrid",
"internal/input_config",
"internal/models",
- "internal/plans",
"internal/projects",
"internal/reader",
"internal/readers",
diff --git a/AGENTS.md b/AGENTS.md
index 0adcd32b..51262cf6 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -8,10 +8,10 @@ Backscroll is a Go CLI tool that indexes Claude Code, Pi, OMP, OpenCode, Codex C
**Status**: Go port complete — `main` branch is the active Go implementation. The Rust implementation is frozen in the `v0` branch.
- Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra).
+ Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra).
github.com/pablontiv/backscroll/internal/autoupdate — Private context-aware release staging and safe replacement
-Stack: cobra, go-toml/v2, goldmark, modernc.org/sqlite (pure Go, no CGO), stdlib testing.
+Stack: cobra, go-toml/v2, modernc.org/sqlite (pure Go, no CGO), stdlib testing.
## Build & Test Commands
@@ -64,17 +64,16 @@ internal/
├── input_config/ — input manifest loading, discovery, and legacy session-dirs compatibility
├── presets/ — embedded input preset installation with explicit overwrite control
├── models/ — domain types: SessionRecord, MessageContent, ParsedFile, SearchResult, Stats
-├── sync/ — WalkDir, SHA-256 dedup, JSONL parsing, noise filtering, content-type classification
+├── sync/ — cancellable JSONL iteration, noise-type filtering, content cleaning, exit-code extraction
├── tagging/ — heuristic auto-tagging (debugging, refactoring, feature, testing, docs, config)
-├── plans/ — Markdown plan parser (split by ## headers, goldmark)
-├── sources/ — external source parsers (ke, decision, memory, rule, spec, backlog) + SourceRegistry
+├── sources/ — cancellable whole-document and sectioned Markdown parsers shared by manifest readers
├── projects/ — project identity registry: LoadGlobalRegistry() (ProjectRegistry, error), Identify(), LoadLocalHint()
├── readers/ — SessionReader interface, Registry, ClaudeReader (text+tool_use+tool_result), PiReader (text+toolCall+custom results), OMPReader (native v3 text+thinking+toolCall+toolResult; parser-backed identities and origin), OpenCodeReader (text+tool state.input+state.output), CodexReader (rollout response items; docs/input-contract.md), MarkdownDocumentReader (`markdown_document`), MarkdownSectionsReader (`markdown_sections`); toolfmt serializer
├── recovery/ — stranded database recovery orchestration, verified active+stranded union, durable backup, atomic replacement, and post-install sync
├── startuplock/ — canonical database lock coordination via persistent OS advisory sidecar
├── skillinstall/ — safe Backscroll skill installation, receipts, rollback, and Python v1 restore compatibility
├── templates/ — F2 Drain-inspired miner: Miner, ProcessLine, ExtractErrorLines, deterministic signature via SHA256
-├── corrections/ — F3 correction detection: bilingual lexicon, interrupt flags, denial heuristics, rephrase-similarity; detector registry + implementations
+├── corrections/ — F3 correction detection: bilingual lexicon, interrupt flags, denial heuristics, rephrase-similarity; prose-filtered cancellable orchestration
├── categories/ — F4 category map loader (rule engine, versioning, tool→category mapping) with embedded default preset
├── hybrid/ — Reciprocal Rank Fusion helpers for merged lexical retrieval
├── sequences/ — F4 PrefixSpan mining (deterministic discovery of frequent tool-call sequences per session)
@@ -103,11 +102,10 @@ validated invocation → startup command class → canonical startup lock
└─ remediation owner: retain mutation-grade lease → skip ordinary compatible-open and pre-handler sync → handler
Ordinary startup sync inputs ────────────────────────────────┐
-JSONL files → fs.WalkDir → SHA-256 dedup → ParseSessions() ─┤
-Markdown plans → DiscoverPlanFiles() → ParsePlan() ──────────┤
-Declarative Markdown inputs → markdown readers ──────────────┤
+Native sessions → manifest discovery → native readers ───────┤
+Markdown documents/plans → manifest discovery → Markdown readers
▼
- SyncFiles() → perennial SQLite FTS5
+ SHA-256 dedup → SyncFiles() → perennial SQLite FTS5
│
search/context/list/patterns/status/validate → database-backed query/output
recover apply → post-install sync under retained remediation lease
@@ -182,7 +180,6 @@ External knowledge sources are configured with active `*.inputs.toml` manifests
- `github.com/spf13/cobra` — CLI argument parsing with subcommands
- `modernc.org/sqlite` — Pure Go SQLite with FTS5, WAL mode (no CGO)
- `github.com/pelletier/go-toml/v2` — TOML config parsing
-- `github.com/yuin/goldmark` — Markdown parsing for plan indexing
- `github.com/pablontiv/picokit` — Output formatting (text/robot/JSON), file hashing, path security (release updating is owned by private `internal/autoupdate`)
- `crypto/sha256` (stdlib) — SHA-256 hashing for incremental sync deduplication
- `io/fs` + `path/filepath` (stdlib) — Recursive directory traversal
@@ -254,12 +251,11 @@ github.com/pablontiv/backscroll/internal/directsearch — Shared direct-search
github.com/pablontiv/backscroll/internal/input_config — Input manifest loading, discovery, and legacy session-dirs compatibility
github.com/pablontiv/backscroll/internal/presets — Embedded input preset installation with explicit overwrite control
github.com/pablontiv/backscroll/internal/models — Domain types and SearchEngine interface
-github.com/pablontiv/backscroll/internal/sync — Session parsing and noise filtering
-github.com/pablontiv/backscroll/internal/plans — Markdown plan parsing
+github.com/pablontiv/backscroll/internal/sync — Cancellable JSONL iteration, noise filtering, content cleaning, exit-code extraction
github.com/pablontiv/backscroll/internal/tagging — Heuristic session auto-tagging
-github.com/pablontiv/backscroll/internal/sources — External source parsers + SourceRegistry
+github.com/pablontiv/backscroll/internal/sources — Cancellable whole-document and sectioned Markdown parsers
github.com/pablontiv/backscroll/internal/templates — F2 Drain-inspired miner: Miner, ProcessLine, ExtractErrorLines
-github.com/pablontiv/backscroll/internal/corrections — F3 correction-signal detectors: lexicon (es+en), interrupt, denial, rephrase-similarity; registry pattern for deterministic, pluggable detectors
+github.com/pablontiv/backscroll/internal/corrections — F3 correction-signal detectors: lexicon (es+en), interrupt, denial, rephrase-similarity; prose-filtered cancellable orchestration
github.com/pablontiv/backscroll/internal/categories — F4 category map loader (rule engine, versioning, tool→category mapping) with embedded default preset
github.com/pablontiv/backscroll/internal/hybrid — Reciprocal Rank Fusion helpers for merged lexical retrieval
github.com/pablontiv/backscroll/internal/sequences — F4 PrefixSpan mining (deterministic pattern discovery per session)
@@ -268,7 +264,7 @@ github.com/pablontiv/backscroll/internal/storage — Database schema, migr
github.com/pablontiv/backscroll/internal/projects — Project identity registry
github.com/pablontiv/backscroll/internal/readers — SessionReader interface, Registry, ClaudeReader (text+tool_use+tool_result), PiReader (text+toolCall+custom results), OMPReader (native v3 text+thinking+toolCall+toolResult; parser-backed identities and origin), OpenCodeReader (text+tool state.input+state.output), CodexReader (rollout response items; docs/input-contract.md), MarkdownDocumentReader (`markdown_document`), MarkdownSectionsReader (`markdown_sections`); toolfmt serializer
github.com/pablontiv/backscroll/internal/recovery — Stranded database recovery orchestration, durable backup, atomic replacement, and post-install sync
- Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/plans`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra).
+ Implemented: `internal/autoupdate`, `internal/config`, `internal/input_config`, `internal/presets`, `internal/models`, `internal/readers`, `internal/sync`, `internal/tagging`, `internal/sources`, `internal/storage`, `internal/services`, `internal/projects`, `internal/templates`, `internal/corrections`, `internal/categories`, `internal/sequences`, `internal/hybrid`, `internal/compat`, `internal/recovery`, `internal/startuplock`, `internal/skillinstall`. CLI commands in `cmd/backscroll/` (12 v2 commands via cobra).
github.com/pablontiv/backscroll/internal/autoupdate — Private context-aware release staging and safe replacement
github.com/pablontiv/backscroll/internal/startuplock — Canonical database lock coordination via persistent OS advisory sidecar
github.com/pablontiv/backscroll/internal/skillinstall — Safe Backscroll skill installation, receipts, rollback, and Python v1 restore compatibility
diff --git a/internal/corrections/corrections.go b/internal/corrections/corrections.go
index f38750d3..d4f93e87 100644
--- a/internal/corrections/corrections.go
+++ b/internal/corrections/corrections.go
@@ -14,41 +14,8 @@ type Detection struct {
Confidence float64
}
-// Detector is a function that analyzes a message at a given ordinal position
-// within a session and returns a Detection if a correction signal is found.
-// Returns nil if no signal is detected.
-type Detector func(msgs []models.Message, idx int) *Detection
-
-// Registry maps detector names to their functions (for orderly, repeatable
-// execution and testing). Detectors are registered at init() time.
-var detectorRegistry map[string]Detector
-
-// Register adds a detector to the registry. Called at init() time.
-func Register(name string, fn Detector) {
- detectorRegistry[name] = fn
-}
-
-// RunDetectors executes all registered detectors over a session's messages.
-// Returns a map of message ordinal -> slice of Detections (detectors that fired).
-// The slice is sorted by detector name for determinism.
-func RunDetectors(msgs []models.Message) map[int][]Detection {
- if len(msgs) == 0 {
- return nil
- }
- result := make(map[int][]Detection)
- for i := range msgs {
- for name, fn := range detectorRegistry {
- if detection := fn(msgs, i); detection != nil {
- detection.DetectorName = name
- result[i] = append(result[i], *detection)
- }
- }
- }
- return result
-}
-
-// RunDetectorsFiltered executes detectors with prose-only filtering without
-// cancellation. It is retained for compatibility with existing callers.
+// RunDetectorsFiltered executes prose-filtered detection for callers without
+// a cancellation context, including stored-text backfill.
func RunDetectorsFiltered(msgs []models.Message) map[int][]Detection {
result, _ := RunDetectorsFilteredContext(context.Background(), msgs)
return result
@@ -345,11 +312,3 @@ func cRephraseDetector(msgs []models.Message, idx int) *Detection {
}
return nil
}
-
-func init() {
- detectorRegistry = make(map[string]Detector)
- Register("lexicon", cCorrectionLexiconDetector)
- Register("interrupt", cInterruptDetector)
- Register("denial", cDenialDetector)
- Register("rephrase", cRephraseDetector)
-}
diff --git a/internal/corrections/corrections_test.go b/internal/corrections/corrections_test.go
index b1175e05..ce66a04d 100644
--- a/internal/corrections/corrections_test.go
+++ b/internal/corrections/corrections_test.go
@@ -15,18 +15,6 @@ func testMsg(role, content string) models.Message {
}
}
-func TestDetectorRegistry(t *testing.T) {
- // Test that registered detectors can be run
- result := RunDetectors([]models.Message{{Role: "user", Content: "test"}})
- if result == nil {
- result = make(map[int][]Detection)
- }
- // For a normal non-correction message, no detectors should fire
- if len(result) != 0 {
- t.Errorf("expected no detections for normal message, got %d", len(result))
- }
-}
-
func TestCorrectionLexiconDetector(t *testing.T) {
tests := []struct {
name string
diff --git a/internal/plans/plans.go b/internal/plans/plans.go
deleted file mode 100644
index 2f53aa7d..00000000
--- a/internal/plans/plans.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package plans
-
-import (
- "fmt"
- "os"
- "path/filepath"
- "strings"
-)
-
-// PlanSection represents a section of a markdown plan file.
-type PlanSection struct {
- Title string
- Content string // full section content including title
- Source string // always "plan"
-}
-
-// ParsePlan parses a markdown file and returns one PlanSection per ## header.
-// If no ## headers are found, the entire file is returned as a single section.
-func ParsePlan(path string) ([]PlanSection, error) {
- data, err := os.ReadFile(path)
- if err != nil {
- return nil, fmt.Errorf("failed to read plan file: %w", err)
- }
-
- content := string(data)
- lines := strings.Split(content, "\n")
-
- var sections []PlanSection
- var currentSection *PlanSection
- var currentContent strings.Builder
-
- for _, line := range lines {
- // Check if this line starts a new section with ##
- if strings.HasPrefix(line, "## ") {
- // Save previous section if any
- if currentSection != nil {
- currentSection.Content = strings.TrimSpace(currentContent.String())
- sections = append(sections, *currentSection)
- currentContent.Reset()
- }
-
- // Start new section
- title := strings.TrimPrefix(line, "## ")
- title = strings.TrimSpace(title)
- currentSection = &PlanSection{
- Title: title,
- Content: "", // will be populated
- Source: "plan",
- }
- currentContent.WriteString(line)
- currentContent.WriteString("\n")
- } else if currentSection != nil {
- // Add to current section
- currentContent.WriteString(line)
- currentContent.WriteString("\n")
- }
- }
-
- // Save last section if any
- if currentSection != nil {
- currentSection.Content = strings.TrimSpace(currentContent.String())
- sections = append(sections, *currentSection)
- }
-
- // If no sections were found (no ## headers), treat entire file as one section
- if len(sections) == 0 {
- sections = []PlanSection{
- {
- Title: "Untitled",
- Content: strings.TrimSpace(content),
- Source: "plan",
- },
- }
- }
-
- return sections, nil
-}
-
-// DiscoverPlanFiles returns all .md files in the given directory.
-func DiscoverPlanFiles(dir string) ([]string, error) {
- entries, err := os.ReadDir(dir)
- if err != nil {
- return nil, fmt.Errorf("failed to read directory: %w", err)
- }
-
- var files []string
- for _, entry := range entries {
- if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".md") {
- files = append(files, filepath.Join(dir, entry.Name()))
- }
- }
-
- return files, nil
-}
diff --git a/internal/plans/plans_test.go b/internal/plans/plans_test.go
deleted file mode 100644
index ae773300..00000000
--- a/internal/plans/plans_test.go
+++ /dev/null
@@ -1,184 +0,0 @@
-package plans
-
-import (
- "os"
- "path/filepath"
- "testing"
-)
-
-func TestParsePlan(t *testing.T) {
- tests := []struct {
- name string
- content string
- expectedCount int
- expectedTitle string
- }{
- {
- name: "single section",
- content: "## First Section\n\nContent here",
- expectedCount: 1,
- expectedTitle: "First Section",
- },
- {
- name: "multiple sections",
- content: `## Overview
-Content for overview
-
-## Details
-More detailed content
-
-## Summary
-Final summary`,
- expectedCount: 3,
- expectedTitle: "Overview",
- },
- {
- name: "no sections (no headers)",
- content: "This is just plain content without any headers",
- expectedCount: 1,
- expectedTitle: "Untitled",
- },
- {
- name: "sections with blank lines",
- content: `## First
-
-Content
-
-## Second
-
-More content`,
- expectedCount: 2,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- // Create temporary file
- tmpfile, err := os.CreateTemp("", "plan_*.md")
- if err != nil {
- t.Fatalf("failed to create temp file: %v", err)
- }
- defer func() { _ = os.Remove(tmpfile.Name()) }()
-
- if _, err := tmpfile.WriteString(tt.content); err != nil {
- t.Fatalf("failed to write to temp file: %v", err)
- }
- _ = tmpfile.Close()
-
- // Parse the file
- sections, err := ParsePlan(tmpfile.Name())
- if err != nil {
- t.Fatalf("ParsePlan failed: %v", err)
- }
-
- if len(sections) != tt.expectedCount {
- t.Errorf("expected %d sections, got %d", tt.expectedCount, len(sections))
- }
-
- if tt.expectedTitle != "" && len(sections) > 0 {
- if sections[0].Title != tt.expectedTitle {
- t.Errorf("expected first title %q, got %q", tt.expectedTitle, sections[0].Title)
- }
- }
-
- // Verify all sections have Source = "plan"
- for _, section := range sections {
- if section.Source != "plan" {
- t.Errorf("expected Source=%q, got %q", "plan", section.Source)
- }
- if section.Content == "" {
- t.Errorf("section %q has empty content", section.Title)
- }
- }
- })
- }
-}
-
-func TestDiscoverPlanFiles(t *testing.T) {
- // Create temp directory
- tmpdir, err := os.MkdirTemp("", "plans_*")
- if err != nil {
- t.Fatalf("failed to create temp dir: %v", err)
- }
- defer func() { _ = os.RemoveAll(tmpdir) }()
-
- // Create some test files
- testFiles := []string{"plan1.md", "plan2.md", "notes.txt", "README.md"}
- for _, f := range testFiles {
- path := filepath.Join(tmpdir, f)
- if err := os.WriteFile(path, []byte("test"), 0o644); err != nil {
- t.Fatalf("failed to create test file: %v", err)
- }
- }
-
- // Discover files
- files, err := DiscoverPlanFiles(tmpdir)
- if err != nil {
- t.Fatalf("DiscoverPlanFiles failed: %v", err)
- }
-
- // Should find 3 .md files (plan1.md, plan2.md, README.md)
- if len(files) != 3 {
- t.Errorf("expected 3 .md files, got %d", len(files))
- }
-
- // Verify all returned files are .md
- for _, f := range files {
- if filepath.Ext(f) != ".md" {
- t.Errorf("expected .md file, got %s", f)
- }
- }
-}
-
-func TestDiscoverPlanFilesEmpty(t *testing.T) {
- // Create temp directory with no files
- tmpdir, err := os.MkdirTemp("", "plans_empty_*")
- if err != nil {
- t.Fatalf("failed to create temp dir: %v", err)
- }
- defer func() { _ = os.RemoveAll(tmpdir) }()
-
- // Discover files
- files, err := DiscoverPlanFiles(tmpdir)
- if err != nil {
- t.Fatalf("DiscoverPlanFiles failed: %v", err)
- }
-
- if len(files) != 0 {
- t.Errorf("expected 0 files, got %d", len(files))
- }
-}
-
-func TestDiscoverPlanFilesVerifyMd(t *testing.T) {
- // Create temp directory
- tmpdir, err := os.MkdirTemp("", "plans_verify_*")
- if err != nil {
- t.Fatalf("failed to create temp dir: %v", err)
- }
- defer func() { _ = os.RemoveAll(tmpdir) }()
-
- // Create test files
- testFiles := []string{"plan1.md", "plan2.md", "notes.txt"}
- for _, f := range testFiles {
- path := filepath.Join(tmpdir, f)
- if err := os.WriteFile(path, []byte("test"), 0o644); err != nil {
- t.Fatalf("failed to create test file: %v", err)
- }
- }
-
- files, err := DiscoverPlanFiles(tmpdir)
- if err != nil {
- t.Fatalf("DiscoverPlanFiles failed: %v", err)
- }
-
- // Should find 2 .md files
- if len(files) != 2 {
- t.Errorf("expected 2 .md files, got %d: %v", len(files), files)
- }
-
- for _, f := range files {
- if filepath.Ext(f) != ".md" {
- t.Errorf("expected .md file, got %s", f)
- }
- }
-}
diff --git a/internal/sources/sources.go b/internal/sources/sources.go
index 1cd1c512..64dd92a8 100644
--- a/internal/sources/sources.go
+++ b/internal/sources/sources.go
@@ -17,86 +17,10 @@ type SourceItem struct {
Path string // path to the source file
}
-// SourceConfig contains paths for different source types.
-type SourceConfig struct {
- KE []string `toml:"ke"`
- Decisions []string `toml:"decisions"`
- Memories []string `toml:"memories"`
- Rules []string `toml:"rules"`
- Specs []string `toml:"specs"`
- Backlog []string `toml:"backlog"`
-}
-
-// ParseAll returns all source items from the configured paths.
-func ParseAll(cfg SourceConfig) ([]SourceItem, error) {
- var items []SourceItem
-
- // KE: whole-document parsing
- for _, path := range cfg.KE {
- item, err := ParseDocument(path, "ke")
- if err != nil {
- return nil, fmt.Errorf("failed to parse KE %s: %w", path, err)
- }
- items = append(items, item)
- }
-
- // Decisions: sectioned (by ## headers)
- for _, path := range cfg.Decisions {
- sectioned, err := ParseSectioned(path, "decision")
- if err != nil {
- return nil, fmt.Errorf("failed to parse Decision %s: %w", path, err)
- }
- items = append(items, sectioned...)
- }
-
- // Memories: whole-document parsing
- for _, path := range cfg.Memories {
- item, err := ParseDocument(path, "memory")
- if err != nil {
- return nil, fmt.Errorf("failed to parse Memory %s: %w", path, err)
- }
- items = append(items, item)
- }
-
- // Rules: sectioned (by ## headers)
- for _, path := range cfg.Rules {
- sectioned, err := ParseSectioned(path, "rule")
- if err != nil {
- return nil, fmt.Errorf("failed to parse Rule %s: %w", path, err)
- }
- items = append(items, sectioned...)
- }
-
- // Specs: sectioned (by ## headers)
- for _, path := range cfg.Specs {
- sectioned, err := ParseSectioned(path, "spec")
- if err != nil {
- return nil, fmt.Errorf("failed to parse Spec %s: %w", path, err)
- }
- items = append(items, sectioned...)
- }
-
- // Backlog: sectioned (by ## headers) or whole-document
- for _, path := range cfg.Backlog {
- sectioned, err := ParseSectioned(path, "backlog")
- if err != nil {
- return nil, fmt.Errorf("failed to parse Backlog %s: %w", path, err)
- }
- items = append(items, sectioned...)
- }
-
- return items, nil
-}
-
const sourceReadBufferSize = 32 * 1024
-// ParseDocument parses a whole-document source (single item).
-// Extracts frontmatter ID and returns the entire content as one item.
-func ParseDocument(path string, sourceType string) (SourceItem, error) {
- return ParseDocumentContext(context.Background(), path, sourceType)
-}
-
-// ParseDocumentContext is ParseDocument with cancellation support.
+// ParseDocumentContext parses a whole-document source with cancellation support.
+// It extracts the frontmatter ID and returns the entire content as one item.
func ParseDocumentContext(ctx context.Context, path string, sourceType string) (SourceItem, error) {
content, id, err := prepareSource(ctx, path, sourceType)
if err != nil {
@@ -111,13 +35,8 @@ func ParseDocumentContext(ctx context.Context, path string, sourceType string) (
}, nil
}
-// ParseSectioned parses a markdown file split by ## headers (each section = item).
-// If no ## headers are found, returns the entire content as a single item.
-func ParseSectioned(path string, sourceType string) ([]SourceItem, error) {
- return ParseSectionedContext(context.Background(), path, sourceType)
-}
-
-// ParseSectionedContext is ParseSectioned with cancellation support.
+// ParseSectionedContext splits a Markdown source by ## headers with cancellation support.
+// If no ## headers are found, it returns the entire content as a single item.
func ParseSectionedContext(ctx context.Context, path string, sourceType string) ([]SourceItem, error) {
content, baseID, err := prepareSource(ctx, path, sourceType)
if err != nil {
diff --git a/internal/sources/sources_test.go b/internal/sources/sources_test.go
index bfe99dda..1ea40e39 100644
--- a/internal/sources/sources_test.go
+++ b/internal/sources/sources_test.go
@@ -54,7 +54,7 @@ func TestSplitSectionsCancelsDuringTraversalWithoutPartialItems(t *testing.T) {
}
}
-func TestParseDocument(t *testing.T) {
+func TestParseDocumentContext(t *testing.T) {
tests := []struct {
name string
content string
@@ -114,9 +114,9 @@ Rules are plain markdown with instructions.`,
_ = tmpfile.Close()
// Parse the file
- item, err := ParseDocument(tmpfile.Name(), tt.sourceType)
+ item, err := ParseDocumentContext(context.Background(), tmpfile.Name(), tt.sourceType)
if err != nil {
- t.Fatalf("ParseDocument failed: %v", err)
+ t.Fatalf("ParseDocumentContext failed: %v", err)
}
if item.ID != tt.expectedID {
@@ -132,7 +132,7 @@ Rules are plain markdown with instructions.`,
}
}
-func TestParseSectioned(t *testing.T) {
+func TestParseSectionedContext(t *testing.T) {
tests := []struct {
name string
content string
@@ -209,9 +209,9 @@ Investigation needed for test purposes.`,
_ = tmpfile.Close()
// Parse the file
- items, err := ParseSectioned(tmpfile.Name(), tt.sourceType)
+ items, err := ParseSectionedContext(context.Background(), tmpfile.Name(), tt.sourceType)
if err != nil {
- t.Fatalf("ParseSectioned failed: %v", err)
+ t.Fatalf("ParseSectionedContext failed: %v", err)
}
if len(items) != tt.expectedItemCount {
@@ -312,95 +312,6 @@ func TestMarkdownPreparationContracts(t *testing.T) {
}
}
-func TestParseAll(t *testing.T) {
- // Create temp directory and files
- tmpdir, err := os.MkdirTemp("", "sources_all_*")
- if err != nil {
- t.Fatalf("failed to create temp dir: %v", err)
- }
- defer func() { _ = os.RemoveAll(tmpdir) }()
-
- // Create test files
- keContent := `---
-id: KE-0001
-estado: activo
----
-
-# Kyverno admission webhook timeout
-
-## Symptom
-
-Pods fail to schedule with admission webhook timeout errors.`
-
- decisionContent := `---
-id: DEC-AUTH-001
-status: accepted
-scope: technical
----
-
-# Implement OAuth 2.0 for API Authentication
-
-## Rationale
-
-OAuth 2.0 provides industry-standard security guarantees.`
-
- memoryContent := `---
-name: feedback_example
-description: Example feedback memory
-type: feedback
----
-
-Always verify state after deploy before declaring complete.`
-
- keFile := filepath.Join(tmpdir, "ke.md")
- decisionFile := filepath.Join(tmpdir, "decision.md")
- memoryFile := filepath.Join(tmpdir, "memory.md")
-
- if err := os.WriteFile(keFile, []byte(keContent), 0o644); err != nil {
- t.Fatalf("failed to create KE file: %v", err)
- }
- if err := os.WriteFile(decisionFile, []byte(decisionContent), 0o644); err != nil {
- t.Fatalf("failed to create Decision file: %v", err)
- }
- if err := os.WriteFile(memoryFile, []byte(memoryContent), 0o644); err != nil {
- t.Fatalf("failed to create Memory file: %v", err)
- }
-
- // Create config
- cfg := SourceConfig{
- KE: []string{keFile},
- Decisions: []string{decisionFile},
- Memories: []string{memoryFile},
- }
-
- // Parse all
- items, err := ParseAll(cfg)
- if err != nil {
- t.Fatalf("ParseAll failed: %v", err)
- }
-
- // We expect: 1 KE + 1 Decision (with 1 section) + 1 Memory = 3 items
- if len(items) != 3 {
- t.Errorf("expected 3 items, got %d", len(items))
- }
-
- // Verify source types
- sources := make(map[string]int)
- for _, item := range items {
- sources[item.Source]++
- }
-
- if sources["ke"] != 1 {
- t.Errorf("expected 1 ke item, got %d", sources["ke"])
- }
- if sources["decision"] != 1 {
- t.Errorf("expected 1 decision item, got %d", sources["decision"])
- }
- if sources["memory"] != 1 {
- t.Errorf("expected 1 memory item, got %d", sources["memory"])
- }
-}
-
func TestExtractID(t *testing.T) {
tests := []struct {
name string
@@ -463,61 +374,6 @@ No frontmatter here.`,
}
}
-func TestParseAllWithAllTypes(t *testing.T) {
- tmpdir, err := os.MkdirTemp("", "sources_all_types_*")
- if err != nil {
- t.Fatalf("create tmpdir: %v", err)
- }
- defer func() { _ = os.RemoveAll(tmpdir) }()
-
- writeFile := func(name, content string) string {
- path := filepath.Join(tmpdir, name)
- if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
- t.Fatalf("write %s: %v", name, err)
- }
- return path
- }
-
- ruleFile := writeFile("rule.md", "---\nid: RULE-001\n---\n## Rule One\nDo this.\n## Rule Two\nDo that.")
- specFile := writeFile("spec.md", "---\nid: SPEC-001\n---\n## Spec One\nSpec detail.")
- backlogFile := writeFile("backlog.md", "---\nid: BACK-001\n---\n## Task One\nDescription.")
-
- cfg := SourceConfig{
- Rules: []string{ruleFile},
- Specs: []string{specFile},
- Backlog: []string{backlogFile},
- }
-
- items, err := ParseAll(cfg)
- if err != nil {
- t.Fatalf("ParseAll: %v", err)
- }
-
- src := make(map[string]int)
- for _, item := range items {
- src[item.Source]++
- }
- if src["rule"] == 0 {
- t.Error("expected at least one rule item")
- }
- if src["spec"] == 0 {
- t.Error("expected at least one spec item")
- }
- if src["backlog"] == 0 {
- t.Error("expected at least one backlog item")
- }
-}
-
-func TestParseAllMissingFile(t *testing.T) {
- cfg := SourceConfig{
- KE: []string{"/nonexistent/ke.md"},
- }
- _, err := ParseAll(cfg)
- if err == nil {
- t.Error("expected error for missing file, got nil")
- }
-}
-
func TestFixtureFiles(t *testing.T) {
// Test with actual fixture files if they exist
fixtureDir := "/home/shared/harness/backscroll/tests/fixtures"
@@ -525,9 +381,9 @@ func TestFixtureFiles(t *testing.T) {
// Test KE fixture
keFile := filepath.Join(fixtureDir, "ke-001.md")
if _, err := os.Stat(keFile); err == nil {
- item, err := ParseDocument(keFile, "ke")
+ item, err := ParseDocumentContext(context.Background(), keFile, "ke")
if err != nil {
- t.Errorf("ParseDocument(ke-001.md) failed: %v", err)
+ t.Errorf("ParseDocumentContext(ke-001.md) failed: %v", err)
} else {
if item.ID == "" {
t.Errorf("KE fixture: empty ID")
@@ -541,9 +397,9 @@ func TestFixtureFiles(t *testing.T) {
// Test Decision fixture (sectioned)
decisionFile := filepath.Join(fixtureDir, "decisions-test-001.md")
if _, err := os.Stat(decisionFile); err == nil {
- items, err := ParseSectioned(decisionFile, "decision")
+ items, err := ParseSectionedContext(context.Background(), decisionFile, "decision")
if err != nil {
- t.Errorf("ParseSectioned(decisions-test-001.md) failed: %v", err)
+ t.Errorf("ParseSectionedContext(decisions-test-001.md) failed: %v", err)
} else {
if len(items) == 0 {
t.Errorf("Decision fixture: no items parsed")
@@ -559,9 +415,9 @@ func TestFixtureFiles(t *testing.T) {
// Test Memory fixture
memoryFile := filepath.Join(fixtureDir, "memory-test.md")
if _, err := os.Stat(memoryFile); err == nil {
- item, err := ParseDocument(memoryFile, "memory")
+ item, err := ParseDocumentContext(context.Background(), memoryFile, "memory")
if err != nil {
- t.Errorf("ParseDocument(memory-test.md) failed: %v", err)
+ t.Errorf("ParseDocumentContext(memory-test.md) failed: %v", err)
} else {
if item.Source != "memory" {
t.Errorf("Memory fixture: wrong source type: %s", item.Source)
@@ -572,9 +428,9 @@ func TestFixtureFiles(t *testing.T) {
// Test Spec fixture (sectioned)
specFile := filepath.Join(fixtureDir, "spec-test.md")
if _, err := os.Stat(specFile); err == nil {
- items, err := ParseSectioned(specFile, "spec")
+ items, err := ParseSectionedContext(context.Background(), specFile, "spec")
if err != nil {
- t.Errorf("ParseSectioned(spec-test.md) failed: %v", err)
+ t.Errorf("ParseSectionedContext(spec-test.md) failed: %v", err)
} else {
if len(items) == 0 {
t.Errorf("Spec fixture: no items parsed")
@@ -590,9 +446,9 @@ func TestFixtureFiles(t *testing.T) {
// Test Rule fixture
ruleFile := filepath.Join(fixtureDir, "rule-test.md")
if _, err := os.Stat(ruleFile); err == nil {
- items, err := ParseSectioned(ruleFile, "rule")
+ items, err := ParseSectionedContext(context.Background(), ruleFile, "rule")
if err != nil {
- t.Errorf("ParseSectioned(rule-test.md) failed: %v", err)
+ t.Errorf("ParseSectionedContext(rule-test.md) failed: %v", err)
} else {
if len(items) == 0 {
t.Errorf("Rule fixture: no items parsed")
@@ -608,9 +464,9 @@ func TestFixtureFiles(t *testing.T) {
// Test Backlog fixture
backlogFile := filepath.Join(fixtureDir, "backlog-test.md")
if _, err := os.Stat(backlogFile); err == nil {
- items, err := ParseSectioned(backlogFile, "backlog")
+ items, err := ParseSectionedContext(context.Background(), backlogFile, "backlog")
if err != nil {
- t.Errorf("ParseSectioned(backlog-test.md) failed: %v", err)
+ t.Errorf("ParseSectionedContext(backlog-test.md) failed: %v", err)
} else {
if len(items) == 0 {
t.Errorf("Backlog fixture: no items parsed")
diff --git a/internal/sync/noise.go b/internal/sync/noise.go
deleted file mode 100644
index 729f26b6..00000000
--- a/internal/sync/noise.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package sync
-
-import (
- "regexp"
- "sync"
-)
-
-// Noise filter patterns compiled at init time (LazyLock equivalent)
-var (
- noisePatternOnce sync.Once
- noisePatterns *NoisePatterns
-)
-
-// NoisePatterns holds compiled regex patterns for noise filtering
-type NoisePatterns struct {
- // Type patterns that should be filtered
- FilteredTypes map[string]bool
-
- // Content patterns for additional filtering
- SystemReminderPattern *regexp.Regexp
- TaskNotificationPattern *regexp.Regexp
- CaveatPattern *regexp.Regexp
- CommandPattern *regexp.Regexp
-}
-
-// GetNoisePatterns returns the lazily-initialized noise patterns
-func GetNoisePatterns() *NoisePatterns {
- noisePatternOnce.Do(func() {
- noisePatterns = &NoisePatterns{
- FilteredTypes: map[string]bool{
- "system-reminder": true,
- "task-notification": true,
- "command": true,
- "command-result": true,
- "local-command-caveat": true,
- "progress": true,
- },
- // These are compiled but not actively used in the current implementation
- // They're here for potential future use
- SystemReminderPattern: regexp.MustCompile(`.*?`),
- TaskNotificationPattern: regexp.MustCompile(`.*?`),
- CaveatPattern: regexp.MustCompile(`Caveat:.*?(?:\n|$)`),
- CommandPattern: regexp.MustCompile(`<(?:command|local-command-caveat)>.*?`),
- }
- })
- return noisePatterns
-}
diff --git a/internal/sync/noise_test.go b/internal/sync/noise_test.go
index 20bd3da6..4b32b242 100644
--- a/internal/sync/noise_test.go
+++ b/internal/sync/noise_test.go
@@ -4,43 +4,6 @@ import (
"testing"
)
-func TestGetNoisePatterns(t *testing.T) {
- np := GetNoisePatterns()
- if np == nil {
- t.Fatal("GetNoisePatterns returned nil")
- }
-
- // Check filtered types
- for _, typ := range []string{"system-reminder", "task-notification", "command", "command-result", "local-command-caveat", "progress"} {
- if !np.FilteredTypes[typ] {
- t.Errorf("expected FilteredTypes[%q] = true", typ)
- }
- }
-
- // Patterns should be compiled
- if np.SystemReminderPattern == nil {
- t.Error("SystemReminderPattern is nil")
- }
- if np.TaskNotificationPattern == nil {
- t.Error("TaskNotificationPattern is nil")
- }
- if np.CaveatPattern == nil {
- t.Error("CaveatPattern is nil")
- }
- if np.CommandPattern == nil {
- t.Error("CommandPattern is nil")
- }
-}
-
-func TestGetNoisePatternsLazy(t *testing.T) {
- // Calling twice should return the same pointer (lazy init)
- np1 := GetNoisePatterns()
- np2 := GetNoisePatterns()
- if np1 != np2 {
- t.Error("GetNoisePatterns should return the same pointer (singleton)")
- }
-}
-
func TestIsNoiseType(t *testing.T) {
tests := []struct {
typ string
diff --git a/internal/sync/sync.go b/internal/sync/sync.go
index 19b61df8..465343c7 100644
--- a/internal/sync/sync.go
+++ b/internal/sync/sync.go
@@ -1,156 +1,6 @@
package sync
-import (
- "encoding/json"
- "fmt"
- "io/fs"
- "os"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/pablontiv/backscroll/internal/models"
-)
-
-// ParseSessions parses a JSONL session file and returns Message records.
-// It defensively handles both Claude and Pi formats.
-func ParseSessions(path string) ([]models.Message, error) {
- var messages []models.Message
- err := IterateJSONLFile(path, func(_ int, line []byte) error {
- var rawRec rawRecord
- if err := json.Unmarshal(line, &rawRec); err != nil {
- // Skip malformed lines
- return nil
- }
-
- // Skip noise records
- if IsNoiseRecord(rawRec) {
- return nil
- }
-
- // Extract message if present
- if rawRec.Message == nil {
- return nil
- }
-
- // Parse timestamp
- ts, err := time.Parse(time.RFC3339, rawRec.Timestamp)
- if err != nil {
- ts = time.Now()
- }
-
- // Extract content and content type
- content, contentType := extractContent(rawRec.Message.Content)
- if content == "" {
- return nil
- }
-
- messages = append(messages, models.Message{
- Role: rawRec.Message.Role,
- Content: content,
- ContentType: contentType,
- Timestamp: ts,
- })
- return nil
- })
- if err != nil {
- return nil, fmt.Errorf("scan session file: %w", err)
- }
-
- return messages, nil
-}
-
-// WalkSessionDirs traverses session directories and returns parseable JSONL paths.
-// If includeAgents is false, subagent sessions are skipped.
-func WalkSessionDirs(dirs []string, includeAgents bool) ([]string, error) {
- var jsonlPaths []string
-
- for _, dir := range dirs {
- err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
- if err != nil {
- // Skip inaccessible directories
- return nil
- }
-
- // Skip subagent directories unless explicitly included
- if !includeAgents && strings.Contains(path, string(filepath.Separator)+"subagents"+string(filepath.Separator)) {
- if d.IsDir() {
- return filepath.SkipDir
- }
- return nil
- }
-
- // Collect .jsonl files
- if !d.IsDir() && strings.HasSuffix(path, ".jsonl") {
- jsonlPaths = append(jsonlPaths, path)
- }
-
- return nil
- })
-
- if err != nil && !os.IsNotExist(err) {
- return nil, fmt.Errorf("walk directory %s: %w", dir, err)
- }
- }
-
- return jsonlPaths, nil
-}
-
-// IsNoiseRecord returns true if the record should be filtered out.
-func IsNoiseRecord(r rawRecord) bool {
- // Filter by type
- if IsNoiseType(r.Type) {
- return true
- }
-
- // Filter metadata records
- if r.IsMeta {
- return true
- }
-
- // Filter records with empty role
- if r.Message == nil || r.Message.Role == "" {
- return true
- }
-
- // Filter records where content extraction yields nothing
- content, _ := extractContent(r.Message.Content)
- return content == ""
-}
-
-// rawRecord covers both Claude and Pi formats defensively
-type rawRecord struct {
- Type string `json:"type"`
- UUID string `json:"uuid"`
- ID string `json:"id"`
- SessionID string `json:"sessionId"`
- ParentID string `json:"parentId"`
- Timestamp string `json:"timestamp"`
- CWD string `json:"cwd"`
- IsAgent bool `json:"isAgent"`
- IsMeta bool `json:"isMeta"`
- Message *rawMessage `json:"message"`
-}
-
-type rawMessage struct {
- Role string `json:"role"`
- Content json.RawMessage `json:"content"`
-}
-
-type rawBlock struct {
- Type string `json:"type"`
- Text string `json:"text"`
- Thinking string `json:"thinking"`
- ToolCall json.RawMessage `json:"toolCall"`
- ToolResult json.RawMessage `json:"toolResult"`
- ToolCallID string `json:"toolCallId"`
- ToolUseID string `json:"tool_use_id"`
- ID string `json:"id"`
- Name string `json:"name"`
- Input json.RawMessage `json:"input"`
- Arguments json.RawMessage `json:"arguments"`
- Content json.RawMessage `json:"content"`
-}
+import "strings"
// IsNoiseType returns true if the type should be filtered
func IsNoiseType(typ string) bool {
@@ -165,72 +15,6 @@ func IsNoiseType(typ string) bool {
return noiseTypes[typ]
}
-// extractContent extracts text from either a string or array of blocks.
-// Returns (content, contentType).
-func extractContent(raw json.RawMessage) (string, string) {
- if raw == nil {
- return "", "text"
- }
-
- // Try parsing as string first
- var strContent string
- if err := json.Unmarshal(raw, &strContent); err == nil {
- content := CleanContent(strContent)
- return content, classifyContentType(content, false)
- }
-
- // Try parsing as array of blocks
- var blocks []rawBlock
- if err := json.Unmarshal(raw, &blocks); err == nil {
- return extractFromBlocks(blocks)
- }
-
- return "", "text"
-}
-
-// extractFromBlocks extracts content and determines type from an array of blocks
-func extractFromBlocks(blocks []rawBlock) (string, string) {
- var textParts []string
- hasToolCall := false
- hasCode := false
-
- for _, block := range blocks {
- switch block.Type {
- case "text":
- cleaned := CleanContent(block.Text)
- if cleaned != "" {
- textParts = append(textParts, cleaned)
- if strings.Contains(cleaned, "```") {
- hasCode = true
- }
- }
- case "thinking":
- // Ignore thinking blocks (hidden reasoning)
- case "tool_use", "toolCall":
- hasToolCall = true
- case "tool_result", "toolResult":
- // Ignore tool result blocks in message content
- case "tool_use_id":
- hasToolCall = true
- }
- }
-
- content := strings.TrimSpace(strings.Join(textParts, " "))
- if content == "" {
- return "", "text"
- }
-
- // Determine content type
- contentType := "text"
- if hasCode {
- contentType = "code"
- } else if hasToolCall {
- contentType = "tool"
- }
-
- return content, contentType
-}
-
// CleanContent removes noise patterns from content
func CleanContent(content string) string {
if content == "" {
@@ -284,14 +68,3 @@ func CleanContent(content string) string {
return content
}
-
-// classifyContentType determines content type from text
-func classifyContentType(content string, hasCode bool) string {
- if hasCode {
- return "code"
- }
- if strings.Contains(content, "```") {
- return "code"
- }
- return "text"
-}
diff --git a/internal/sync/sync_test.go b/internal/sync/sync_test.go
index a1ecc3f5..de369d37 100644
--- a/internal/sync/sync_test.go
+++ b/internal/sync/sync_test.go
@@ -1,380 +1,6 @@
package sync
-import (
- "os"
- "strings"
- "testing"
-)
-
-// TestParseSessions_PiFormat tests parsing Pi format sessions
-func TestParseSessions_PiFormat(t *testing.T) {
- path := "../../tests/fixtures/pi-session.jsonl"
- if _, err := os.Stat(path); os.IsNotExist(err) {
- t.Skipf("fixture file not found: %s", path)
- }
-
- messages, err := ParseSessions(path)
- if err != nil {
- t.Fatalf("ParseSessions failed: %v", err)
- }
-
- if len(messages) == 0 {
- t.Fatal("expected messages, got none")
- }
-
- // Check we have user and assistant roles
- hasUser := false
- hasAssistant := false
-
- for _, msg := range messages {
- if msg.Role == "user" {
- hasUser = true
- }
- if msg.Role == "assistant" {
- hasAssistant = true
- }
- }
-
- if !hasUser {
- t.Error("expected user role, got none")
- }
- if !hasAssistant {
- t.Error("expected assistant role, got none")
- }
-
- // Check timestamp parsing
- for _, msg := range messages {
- if msg.Timestamp.IsZero() {
- t.Error("expected non-zero timestamp")
- }
- }
-}
-
-// TestParseSessions_ClaudeFormat tests parsing Claude format sessions
-func TestParseSessions_ClaudeFormat(t *testing.T) {
- path := "../../tests/fixtures/claude-preset/projects/project-a/session-main.jsonl"
- if _, err := os.Stat(path); os.IsNotExist(err) {
- t.Skipf("fixture file not found: %s", path)
- }
-
- messages, err := ParseSessions(path)
- if err != nil {
- t.Fatalf("ParseSessions failed: %v", err)
- }
-
- if len(messages) == 0 {
- t.Fatal("expected messages, got none")
- }
-
- // Verify noise filtering worked:
- // - "progress" type should be filtered
- // - isMeta records should be filtered
- // - task-notification type should be filtered
- // - System reminder content should be removed
-
- for _, msg := range messages {
- if strings.Contains(msg.Content, "system-reminder") {
- t.Error("system-reminder should be filtered from content")
- }
- if strings.Contains(msg.Content, "drop progress") {
- t.Error("progress type message should be filtered")
- }
- if strings.Contains(msg.Content, "drop metadata") {
- t.Error("metadata (isMeta) message should be filtered")
- }
- if strings.Contains(msg.Content, "remove whole notification") {
- t.Error("task-notification should be filtered")
- }
- }
-
- // Check we have user and assistant roles
- hasUser := false
- hasAssistant := false
- for _, msg := range messages {
- if msg.Role == "user" {
- hasUser = true
- }
- if msg.Role == "assistant" {
- hasAssistant = true
- }
- }
-
- if !hasUser {
- t.Error("expected user role")
- }
- if !hasAssistant {
- t.Error("expected assistant role")
- }
-}
-
-// TestNoiseFiltering_ToolEvents tests that tool events are parsed but not tool result content
-func TestNoiseFiltering_ToolEvents(t *testing.T) {
- path := "../../tests/fixtures/claude-tool-events.jsonl"
- if _, err := os.Stat(path); os.IsNotExist(err) {
- t.Skipf("fixture file not found: %s", path)
- }
-
- messages, err := ParseSessions(path)
- if err != nil {
- t.Fatalf("ParseSessions failed: %v", err)
- }
-
- if len(messages) == 0 {
- t.Fatal("expected messages, got none")
- }
-
- // Should have parsed messages despite tool events
- hasNormalMessage := false
- hasToolContent := false
-
- for _, msg := range messages {
- if strings.Contains(msg.Content, "normal claude message") {
- hasNormalMessage = true
- }
- if msg.ContentType == "tool" {
- hasToolContent = true
- }
- }
-
- if !hasNormalMessage {
- t.Error("expected to find normal message")
- }
- if !hasToolContent {
- t.Error("expected to find tool content type")
- }
-}
-
-// TestWalkSessionDirs_Exclusion tests subagent filtering
-func TestWalkSessionDirs_Exclusion(t *testing.T) {
- dir := "../../tests/fixtures/claude-preset"
- if _, err := os.Stat(dir); os.IsNotExist(err) {
- t.Skipf("fixture directory not found: %s", dir)
- }
-
- // With includeAgents=false, subagent files should be excluded
- paths, err := WalkSessionDirs([]string{dir}, false)
- if err != nil {
- t.Fatalf("WalkSessionDirs failed: %v", err)
- }
-
- for _, p := range paths {
- if strings.Contains(p, "/subagents/") {
- t.Errorf("subagent file should be excluded: %s", p)
- }
- }
-
- // With includeAgents=true, subagent files should be included
- pathsWithAgents, err := WalkSessionDirs([]string{dir}, true)
- if err != nil {
- t.Fatalf("WalkSessionDirs with includeAgents failed: %v", err)
- }
-
- hasSubagent := false
- for _, p := range pathsWithAgents {
- if strings.Contains(p, "/subagents/") {
- hasSubagent = true
- break
- }
- }
-
- if !hasSubagent {
- t.Error("expected to find subagent files with includeAgents=true")
- }
-}
-
-// TestIsNoiseRecord_FilterTypes tests that noise types are filtered
-func TestIsNoiseRecord_FilterTypes(t *testing.T) {
- tests := []struct {
- name string
- record rawRecord
- wantNoise bool
- }{
- {
- name: "system-reminder type",
- record: rawRecord{
- Type: "system-reminder",
- Message: &rawMessage{
- Role: "user",
- Content: []byte(`"content"`),
- },
- },
- wantNoise: true,
- },
- {
- name: "task-notification type",
- record: rawRecord{
- Type: "task-notification",
- Message: &rawMessage{
- Role: "assistant",
- Content: []byte(`"content"`),
- },
- },
- wantNoise: true,
- },
- {
- name: "progress type",
- record: rawRecord{
- Type: "progress",
- Message: &rawMessage{
- Role: "assistant",
- Content: []byte(`"content"`),
- },
- },
- wantNoise: true,
- },
- {
- name: "isMeta true",
- record: rawRecord{
- Type: "user",
- IsMeta: true,
- Message: &rawMessage{
- Role: "user",
- Content: []byte(`"content"`),
- },
- },
- wantNoise: true,
- },
- {
- name: "empty role",
- record: rawRecord{
- Type: "message",
- Message: &rawMessage{
- Role: "",
- Content: []byte(`"content"`),
- },
- },
- wantNoise: true,
- },
- {
- name: "nil message",
- record: rawRecord{
- Type: "message",
- Message: nil,
- },
- wantNoise: true,
- },
- {
- name: "valid user message",
- record: rawRecord{
- Type: "user",
- Message: &rawMessage{
- Role: "user",
- Content: []byte(`"valid content"`),
- },
- },
- wantNoise: false,
- },
- {
- name: "valid assistant message",
- record: rawRecord{
- Type: "assistant",
- Message: &rawMessage{
- Role: "assistant",
- Content: []byte(`"valid content"`),
- },
- },
- wantNoise: false,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := IsNoiseRecord(tt.record)
- if got != tt.wantNoise {
- t.Errorf("IsNoiseRecord() = %v, want %v", got, tt.wantNoise)
- }
- })
- }
-}
-
-// TestExtractContent_StringFormat tests string content extraction
-func TestExtractContent_StringFormat(t *testing.T) {
- tests := []struct {
- name string
- input string
- wantContent string
- wantType string
- }{
- {
- name: "simple string",
- input: `"hello world"`,
- wantContent: "hello world",
- wantType: "text",
- },
- {
- name: "string with code fence",
- input: "\"```go\\nfunc main() {}\\n```\"",
- wantContent: "```go func main() {} ```",
- wantType: "code",
- },
- {
- name: "empty string",
- input: `""`,
- wantContent: "",
- wantType: "text",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- content, contentType := extractContent([]byte(tt.input))
-
- // For empty content, we expect empty string
- if tt.wantContent == "" {
- if content != "" {
- t.Errorf("expected empty content, got %q", content)
- }
- return
- }
-
- if !strings.Contains(content, strings.TrimSpace(tt.wantContent)) {
- t.Errorf("content = %q, want to contain %q", content, tt.wantContent)
- }
- if contentType != tt.wantType {
- t.Errorf("contentType = %q, want %q", contentType, tt.wantType)
- }
- })
- }
-}
-
-// TestExtractContent_BlockFormat tests block array content extraction
-func TestExtractContent_BlockFormat(t *testing.T) {
- tests := []struct {
- name string
- blocks string
- wantType string
- }{
- {
- name: "text block",
- blocks: `[{"type":"text","text":"hello"}]`,
- wantType: "text",
- },
- {
- name: "text with code fence",
- blocks: "[{\"type\":\"text\",\"text\":\"code: ```go\\nfunc main() {}\\n```\"}]",
- wantType: "code",
- },
- {
- name: "tool_use block",
- blocks: `[{"type":"text","text":"calling tool"},{"type":"tool_use","id":"t1","name":"bash"}]`,
- wantType: "tool",
- },
- {
- name: "thinking ignored",
- blocks: `[{"type":"thinking","thinking":"hidden"},{"type":"text","text":"visible"}]`,
- wantType: "text",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- _, contentType := extractContent([]byte(tt.blocks))
- if contentType != tt.wantType {
- t.Errorf("contentType = %q, want %q", contentType, tt.wantType)
- }
- })
- }
-}
+import "testing"
// TestCleanContent tests noise pattern removal
func TestCleanContent(t *testing.T) {
@@ -454,48 +80,3 @@ func TestCleanContent(t *testing.T) {
})
}
}
-
-// TestParseSessions_IgnoresSubagents tests that subagent sessions are excluded when reading
-func TestParseSessions_IgnoresSubagents(t *testing.T) {
- path := "../../tests/fixtures/claude-preset/projects/project-a/subagents/agent.jsonl"
- if _, err := os.Stat(path); os.IsNotExist(err) {
- t.Skipf("fixture file not found: %s", path)
- }
-
- messages, err := ParseSessions(path)
- if err != nil {
- t.Fatalf("ParseSessions failed: %v", err)
- }
-
- // Subagent file should be parsed if passed directly
- // The filtering happens at the directory walk level
- if len(messages) == 0 {
- t.Log("no messages from subagent file (expected)")
- }
-}
-
-// BenchmarkParseSessions benchmarks session parsing
-func BenchmarkParseSessions(b *testing.B) {
- path := "../../tests/fixtures/pi-session.jsonl"
- if _, err := os.Stat(path); os.IsNotExist(err) {
- b.Skipf("fixture file not found: %s", path)
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = ParseSessions(path)
- }
-}
-
-// BenchmarkWalkSessionDirs benchmarks directory walking
-func BenchmarkWalkSessionDirs(b *testing.B) {
- dir := "../../tests/fixtures"
- if _, err := os.Stat(dir); os.IsNotExist(err) {
- b.Skipf("fixture directory not found: %s", dir)
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, _ = WalkSessionDirs([]string{dir}, false)
- }
-}