diff --git a/cmd/backscroll/normalization_e2e_test.go b/cmd/backscroll/normalization_e2e_test.go
new file mode 100644
index 0000000..d9dcf9a
--- /dev/null
+++ b/cmd/backscroll/normalization_e2e_test.go
@@ -0,0 +1,273 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/pablontiv/backscroll/internal/input_config"
+ "github.com/pablontiv/backscroll/internal/readers"
+ "github.com/pablontiv/backscroll/internal/storage"
+)
+
+func TestFencedProviderNormalizationCLI(t *testing.T) {
+ e := newQueryEchoE2E(t)
+ paths := make(map[string]string)
+ for _, provider := range []string{"claude", "pi", "codex", "omp", "opencode"} {
+ paths[provider] = writeNormalizationFixture(t, e, provider)
+ }
+ code := e.searchJSON("normalization167", "code", 20)
+ if len(code) != 5 {
+ t.Fatalf("fenced provider results = %+v, want five", code)
+ }
+ for _, result := range code {
+ if result.Content != "normalization167 beacon ```python value = 167 print(value) ``` finish marker" {
+ t.Fatalf("normalized fenced text = %+v", result)
+ }
+ }
+ plain := e.searchJSON("normalization167", "text", 20)
+ if len(plain) != 10 {
+ t.Fatalf("prose/provider-wrapper controls = %+v, want ten", plain)
+ }
+ assertRows := func(reasoning bool) {
+ t.Helper()
+ index, err := storage.OpenReadOnly(e.database)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer index.Close()
+ for provider, path := range paths {
+ var text, origin, evidence string
+ var uuid *string
+ if err := index.DB().QueryRow(`SELECT text,origin,uuid,project_evidence FROM search_items WHERE source_path=? AND text LIKE 'normalization167 beacon value%'`, path).Scan(&text, &origin, &uuid, &evidence); err != nil {
+ t.Fatal(err)
+ }
+ if text != "normalization167 beacon value = 167 print(value) finish marker" || origin != "human" || evidence != "native_cwd" || (uuid == nil) != (provider == "codex") {
+ t.Fatalf("%s prose/identity/origin/project = %q/%v/%q/%q", provider, text, uuid, origin, evidence)
+ }
+ var count int
+ if err := index.DB().QueryRow(`SELECT COUNT(*) FROM search_items WHERE source_path=? AND content_type='reasoning'`, path).Scan(&count); err != nil {
+ t.Fatal(err)
+ }
+ want := 0
+ if reasoning && provider != "claude" && provider != "opencode" {
+ want = 1
+ }
+ if count != want {
+ t.Fatalf("%s reasoning rows=%d, want %d", provider, count, want)
+ }
+ if err := index.DB().QueryRow(`SELECT COUNT(*) FROM search_items WHERE source_path=? AND content_type='tool' AND origin='automation' AND text=?`, path, "normalizationtool167 output\n kept tool spacing").Scan(&count); err != nil || count != 1 {
+ t.Fatalf("%s native tool output count=%d err=%v", provider, count, err)
+ }
+ if err := index.DB().QueryRow(`SELECT COUNT(*) FROM tool_events WHERE source_path=? AND tool_name IN ('bash','exec_command') AND command_head='printf'`, path).Scan(&count); err != nil || count != 1 {
+ t.Fatalf("%s native call count=%d err=%v", provider, count, err)
+ }
+ }
+ }
+ assertRows(false)
+ for provider := range paths {
+ manifest := filepath.Join(e.config, "backscroll", "inputs", "normalization-"+provider+".inputs.toml")
+ data, err := os.ReadFile(manifest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(manifest, append(data, []byte("index_reasoning = true\n")...), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ e.run("status", "--json")
+ assertRows(true)
+ t.Log("actual hermetic dev CLI: all five providers classify fences, normalize prose, preserve native identity/origin/project/tool output, and replay reasoning opt-in")
+}
+
+func writeNormalizationFixture(t *testing.T, e *queryEchoE2E, provider string) string {
+ t.Helper()
+ root := e.addReaderManifest("normalization-"+provider, provider)
+ texts := []string{
+ " normalization167 beacon\n```python\nvalue = 167\nprint(value)\n```\n finish\tmarker ",
+ " normalization167 beacon\nvalue = 167\nprint(value)\n finish\tmarker ",
+ "```hidden fence``` normalization167 wrapper\n visible",
+ }
+ if provider == "opencode" {
+ texts[2] = "normalization167 native visible\n second line"
+ }
+ const timestamp = "2026-01-01T00:00:00Z"
+ const cwd = "/projects/normalization167"
+ const output = "normalizationtool167 output\n kept tool spacing"
+ const reasoning = "private167 reasoning\n normalized"
+ if provider == "opencode" {
+ path := filepath.Join(root, "opencode.db")
+ db := openOpenCodeFixture(t, path)
+ defer db.Close()
+ if _, err := db.Exec(`CREATE TABLE session(id TEXT PRIMARY KEY,directory TEXT); INSERT INTO session VALUES('normalization167',?)`, cwd); err != nil {
+ t.Fatal(err)
+ }
+ for i, text := range texts {
+ id := fmt.Sprintf("normalization-%d", i)
+ if _, err := db.Exec(`INSERT INTO message VALUES(?,?,?,?,?)`, id, "normalization167", i, i, `{"role":"user"}`); err != nil {
+ t.Fatal(err)
+ }
+ parts := []map[string]any{{"type": "text", "text": text}}
+ if i == 2 {
+ parts = append(parts, map[string]any{"type": "text", "text": "ignored167", "ignored": true}, map[string]any{"type": "reasoning", "text": reasoning}, map[string]any{"type": "tool", "tool": "bash", "state": map[string]any{"input": map[string]any{"command": "printf normalizationtool167"}, "output": output}})
+ }
+ for j, part := range parts {
+ data, _ := json.Marshal(part)
+ if _, err := db.Exec(`INSERT INTO part VALUES(?,?,?,?,?,?)`, fmt.Sprintf("%s-%d", id, j), id, "normalization167", i, i, string(data)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ return path
+ }
+ var records []map[string]any
+ if provider == "pi" || provider == "omp" {
+ records = append(records, map[string]any{"type": "session", "version": 3, "id": "native167", "cwd": cwd, "timestamp": timestamp})
+ }
+ if provider == "codex" {
+ records = append(records, map[string]any{"type": "session_meta", "payload": map[string]any{"id": "native167", "cwd": cwd}})
+ }
+ for i, text := range texts {
+ content := any(text)
+ if i != 1 {
+ content = []map[string]any{{"type": "text", "text": text}}
+ }
+ rec := map[string]any{"type": "message", "id": fmt.Sprintf("native-%d", i), "uuid": fmt.Sprintf("native-%d", i), "cwd": cwd, "timestamp": timestamp, "message": map[string]any{"role": "user", "attribution": "user", "content": content}}
+ if provider == "claude" {
+ rec["type"] = "user"
+ }
+ if provider == "codex" {
+ // Leading injected wrappers are native structural removals, not a
+ // reason to classify the remaining ordinary request as fenced code.
+ if i == 2 {
+ text = "```native injected```\n" + text
+ }
+ rec = map[string]any{"type": "response_item", "timestamp": timestamp, "payload": map[string]any{"type": "message", "role": "user", "content": []map[string]any{{"type": "input_text", "text": text}}}}
+ }
+ records = append(records, rec)
+ }
+ toolType, toolIDKey, argsKey := "toolCall", "id", "arguments"
+ if provider == "claude" {
+ toolType, argsKey = "tool_use", "input"
+ }
+ thinking := map[string]any{"type": "thinking", "text": reasoning, "thinking": reasoning}
+ call := map[string]any{"type": toolType, toolIDKey: "call167", "name": "bash", argsKey: map[string]any{"command": "printf normalizationtool167"}}
+ records = append(records, map[string]any{"type": "message", "id": "call", "uuid": "call", "timestamp": timestamp, "message": map[string]any{"role": "assistant", "content": []map[string]any{call, thinking}}})
+ result := map[string]any{"role": "toolResult", "toolCallId": "call167", "toolName": "bash", "content": []map[string]any{{"type": "text", "text": output}}, "isError": false}
+ if provider == "claude" {
+ records[len(records)-1]["type"] = "assistant"
+ result = map[string]any{"role": "user", "content": []map[string]any{{"type": "tool_result", "tool_use_id": "call167", "content": output}}}
+ }
+ records = append(records, map[string]any{"type": "message", "id": "result", "uuid": "result", "timestamp": timestamp, "message": result})
+ if provider == "claude" {
+ records[len(records)-1]["type"] = "user"
+ }
+ if provider == "codex" {
+ records = records[:len(records)-2]
+ for _, item := range []map[string]any{
+ {"type": "function_call", "name": "exec_command", "call_id": "call167", "arguments": `{"cmd":"printf normalizationtool167"}`},
+ {"type": "function_call_output", "call_id": "call167", "output": output},
+ {"type": "reasoning", "summary": []map[string]any{{"type": "summary_text", "text": reasoning}}},
+ } {
+ records = append(records, map[string]any{"type": "response_item", "timestamp": timestamp, "payload": item})
+ }
+ }
+ var lines []string
+ for _, record := range records {
+ data, err := json.Marshal(record)
+ if err != nil {
+ t.Fatal(err)
+ }
+ lines = append(lines, string(data))
+ }
+ path := filepath.Join(root, "session.jsonl")
+ if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ return path
+}
+
+func TestOpenCodeNormalizationReplayPreservesIdentifiedPayloadCLI(t *testing.T) {
+ e := newQueryEchoE2E(t)
+ path := writeNormalizationFixture(t, e, "opencode")
+ e.run("status", "--json")
+ def := input_config.InputDefinition{Source: "session"}
+ old := readers.IngestionFingerprint(&semanticReplayReader{name: "opencode", epoch: 2}, def)
+ current := readers.IngestionFingerprint(&readers.OpenCodeReader{}, def)
+ index, err := storage.Open(e.database)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var uuid string
+ var id, ordinal int64
+ if err := index.DB().QueryRow(`SELECT id,ordinal,uuid FROM search_items WHERE content_type='code'`).Scan(&id, &ordinal, &uuid); err != nil {
+ t.Fatal(err)
+ }
+ // Recreate an epoch-2 retained payload and an available source with a
+ // previously unindexed message; source bytes and its native hash stay equal.
+ const retained = "normalization167 beacon\n```python\nvalue = 167\n```"
+ if _, err := index.DB().Exec(`UPDATE search_items SET text=?,content_type='text' WHERE uuid=?`, retained, uuid); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := index.DB().Exec(`DELETE FROM search_items WHERE text LIKE 'normalization167 beacon value%'`); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := index.DB().Exec(`UPDATE indexed_files SET ingestion_fingerprint=? WHERE path=?`, old, path); err != nil {
+ t.Fatal(err)
+ }
+ var hash string
+ if err := index.DB().QueryRow(`SELECT hash FROM indexed_files WHERE path=?`, path).Scan(&hash); err != nil {
+ t.Fatal(err)
+ }
+ index.Close()
+ e.run("status", "--json")
+ index, err = storage.OpenReadOnly(e.database)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var gotText, gotKind, gotFingerprint, gotHash string
+ var gotID, gotOrdinal int64
+ if err := index.DB().QueryRow(`SELECT id,ordinal,text,content_type FROM search_items WHERE uuid=?`, uuid).Scan(&gotID, &gotOrdinal, &gotText, &gotKind); err != nil {
+ t.Fatal(err)
+ }
+ if gotID != id || gotOrdinal != ordinal || gotText != retained || gotKind != "text" {
+ t.Fatalf("identified replay replaced retained payload: %d/%d/%q/%q", gotID, gotOrdinal, gotText, gotKind)
+ }
+ if err := index.DB().QueryRow(`SELECT hash,ingestion_fingerprint FROM indexed_files WHERE path=?`, path).Scan(&gotHash, &gotFingerprint); err != nil {
+ t.Fatal(err)
+ }
+ if gotHash != hash || gotFingerprint != current || old == current {
+ t.Fatalf("stable-byte replay hash/fingerprint=%q/%q", gotHash, gotFingerprint)
+ }
+ var restored string
+ if err := index.DB().QueryRow(`SELECT text FROM search_items WHERE text LIKE 'normalization167 beacon value%'`).Scan(&restored); err != nil || restored != "normalization167 beacon value = 167 print(value) finish marker" {
+ t.Fatalf("newly indexed replay payload=%q err=%v", restored, err)
+ }
+ index.Close()
+ if err := os.Remove(path); err != nil {
+ t.Fatal(err)
+ }
+ index, err = storage.Open(e.database)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := index.DB().Exec(`UPDATE indexed_files SET ingestion_fingerprint=? WHERE path=?`, old, path); err != nil {
+ t.Fatal(err)
+ }
+ index.Close()
+ e.run("status", "--json")
+ index, err = storage.OpenReadOnly(e.database)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer index.Close()
+ if err := index.DB().QueryRow(`SELECT ingestion_fingerprint FROM indexed_files WHERE path=?`, path).Scan(&gotFingerprint); err != nil || gotFingerprint != old {
+ t.Fatalf("expired fingerprint=%q err=%v", gotFingerprint, err)
+ }
+ if err := index.DB().QueryRow(`SELECT text FROM search_items WHERE uuid=?`, uuid).Scan(&gotText); err != nil || gotText != retained {
+ t.Fatalf("expired perennial payload=%q err=%v", gotText, err)
+ }
+}
diff --git a/cmd/backscroll/opencode_native_e2e_test.go b/cmd/backscroll/opencode_native_e2e_test.go
index 28bd8a6..90cc671 100644
--- a/cmd/backscroll/opencode_native_e2e_test.go
+++ b/cmd/backscroll/opencode_native_e2e_test.go
@@ -190,3 +190,50 @@ func TestOpenCodeNativeSessionsCLI(t *testing.T) {
}
t.Log("copied fixture: native sessions/projects, registry precedence, context/sequence/tag isolation, stable perennial identities, absent-cwd unknown proof verified through actual CLI")
}
+
+func TestOpenCodeFencedNormalizationCLI(t *testing.T) {
+ e := newQueryEchoE2E(t)
+ root := e.addReaderManifest("normalization", "opencode")
+ path := filepath.Join(root, "opencode.db")
+ source := openOpenCodeFixture(t, path)
+ if _, err := source.Exec(`CREATE TABLE session(id TEXT PRIMARY KEY,directory TEXT); INSERT INTO session VALUES('native167','/projects/normalization')`); err != nil {
+ t.Fatal(err)
+ }
+ for i, text := range []string{
+ " quartz167 beacon\n```python\nvalue = 167\nprint(value)\n```\n finish\tmarker ",
+ " quartz167 beacon\nvalue = 167\nprint(value)\n finish\tmarker ",
+ "quartz167 native visible\n second line",
+ } {
+ id := fmt.Sprintf("native167-%d", i)
+ if _, err := source.Exec(`INSERT INTO message VALUES(?,?,?,?,?)`, id, "native167", i, i, `{"role":"user"}`); err != nil {
+ t.Fatal(err)
+ }
+ part, _ := json.Marshal(map[string]any{"type": "text", "text": text})
+ if _, err := source.Exec(`INSERT INTO part VALUES(?,?,?,?,?,?)`, id+"-part", id, "native167", i, i, string(part)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := source.Close(); err != nil {
+ t.Fatal(err)
+ }
+ code := e.searchJSON("quartz167", "code", 20)
+ if len(code) != 1 || code[0].Content != "quartz167 beacon ```python value = 167 print(value) ``` finish marker" {
+ t.Fatalf("code-filtered normalized fenced fixture = %+v", code)
+ }
+ plain := e.searchJSON("quartz167", "text", 20)
+ if len(plain) != 2 {
+ t.Fatalf("text-filtered controls = %+v", plain)
+ }
+ index, err := storage.OpenReadOnly(e.database)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer index.Close()
+ var text, origin, session, evidence string
+ if err := index.DB().QueryRow(`SELECT text,origin,session_id,project_evidence FROM search_items WHERE text LIKE '%system-reminder%'`).Scan(&text, &origin, &session, &evidence); err != nil {
+ t.Fatal(err)
+ }
+ if text != "quartz167 native visible second line" || origin != "human" || session != "native167" || evidence != "native_cwd" {
+ t.Fatalf("native envelope/ownership = %q/%q/%q/%q", text, origin, session, evidence)
+ }
+}
diff --git a/docs/input-contract.md b/docs/input-contract.md
index 0c5eb05..71ce6f1 100644
--- a/docs/input-contract.md
+++ b/docs/input-contract.md
@@ -556,7 +556,7 @@ visible without relying on main-file size, modification time, or
records inside that same snapshot; a concurrent later commit is detected on the
next startup rather than attached to older parsed records. Equal-timestamp
additions, deletions, and part-only edits therefore invalidate the token, while
-an unchanged native snapshot hashes identically. OpenCode reader epoch 2 queues pre-native exports through the shared, bounded 200-path ingestion-fingerprint replay.
+an unchanged native snapshot hashes identically. OpenCode reader epoch 3 queues available older exports through the shared, bounded 200-path ingestion-fingerprint replay for native identity and normalized text.
### OpenCode native session identity
@@ -576,6 +576,15 @@ position, nor timestamp participates. Identified OpenCode records therefore use
perennial append/reconciliation: insertion and reordering preserve anchors;
mutation never replaces retained payload or proven session/project ownership.
+After native envelope selection, session prose uses collapsed whitespace across
+Claude, Pi, Codex, OMP, and OpenCode. Fenced code is classified before whitespace
+collapse. Each provider keeps its existing content removals and structural
+wrapper handling; OpenCode selects non-ignored native text parts and applies
+whitespace-only cleanup. Tool payload serialization and reasoning opt-in remain
+separate. Replay indexes newly available records with this policy; already
+identified retained payloads keep their original text/type and anchors, and
+unavailable perennial sources keep their rows and fingerprint.
+
Migration V20 adds nullable `search_items.session_id` and provider-neutral
`session_identity_tags`, because the prior schema had no native-session column
and path-only tags cannot distinguish sibling sessions. Historical NULL session
diff --git a/docs/sync.md b/docs/sync.md
index 17cc21b..2fa8246 100644
--- a/docs/sync.md
+++ b/docs/sync.md
@@ -292,7 +292,7 @@ The SQLite database is the perennial event store, not a disposable cache. When a
## Noise filtering
-Each dedicated reader owns provider-specific record inclusion, text cleanup, and tool extraction. Discovery exclusions in the manifest remove configured paths such as Claude subagent sessions; readers remove provider noise such as system reminders, task notifications, and local command metadata. Pi, OMP, and Codex reasoning is indexed only when its manifest sets `index_reasoning = true`.
+Each dedicated reader owns provider-specific record inclusion, text cleanup, and tool extraction. Discovery exclusions in the manifest remove configured paths such as Claude subagent sessions; readers retain their existing provider removals, including system reminders, task notifications, local command metadata, and Codex structural wrappers. After native decoding and those removals, prose whitespace is collapsed consistently and fenced code is classified before collapse. OpenCode applies whitespace-only cleanup to its selected native text parts; epoch 3 replays available sources through the shared 200-path queue while preserving identified retained payloads and unavailable history. Tool payloads keep their serialization policy. Pi, OMP, and Codex reasoning is indexed only when its manifest sets `index_reasoning = true`.
## Exit codes
diff --git a/internal/readers/claude_reader.go b/internal/readers/claude_reader.go
index 47e8993..908e909 100644
--- a/internal/readers/claude_reader.go
+++ b/internal/readers/claude_reader.go
@@ -181,11 +181,11 @@ func extractClaudeMessages(ctx context.Context, rec claudeRecord) ([]models.Mess
var s string
if err := json.Unmarshal(rec.Message.Content, &s); err == nil {
interrupted := strings.Contains(s, interruptMarker)
- text := sync.CleanContent(s)
+ text, kind := cleanText(s)
if text == "" {
return nil, nil
}
- return []models.Message{{Role: role, Origin: origin, Content: text, ContentType: classifyText(text), Timestamp: ts,
+ return []models.Message{{Role: role, Origin: origin, Content: text, ContentType: kind, Timestamp: ts,
UUID: rec.UUID, WasInterrupted: interrupted}}, nil
}
@@ -200,6 +200,7 @@ func extractClaudeMessages(ctx context.Context, rec claudeRecord) ([]models.Mess
var out []models.Message
var textParts []string
+ textKind := "text"
interrupted := false
for i, b := range blocks {
if err := ctx.Err(); err != nil {
@@ -210,8 +211,11 @@ func extractClaudeMessages(ctx context.Context, rec claudeRecord) ([]models.Mess
if strings.Contains(b.Text, interruptMarker) {
interrupted = true
}
- if c := sync.CleanContent(b.Text); c != "" {
+ if c, kind := cleanText(b.Text); c != "" {
textParts = append(textParts, c)
+ if kind == "code" {
+ textKind = kind
+ }
}
case "tool_use":
if t := SerializeToolInput(b.Name, b.Input); strings.TrimSpace(t) != "" {
@@ -251,7 +255,7 @@ func extractClaudeMessages(ctx context.Context, rec claudeRecord) ([]models.Mess
}
if len(textParts) > 0 {
text := strings.TrimSpace(strings.Join(textParts, " "))
- out = append([]models.Message{{Role: role, Origin: origin, Content: text, ContentType: classifyText(text), Timestamp: ts,
+ out = append([]models.Message{{Role: role, Origin: origin, Content: text, ContentType: textKind, Timestamp: ts,
UUID: rec.UUID, WasInterrupted: interrupted}}, out...)
}
return out, nil
@@ -279,10 +283,3 @@ func isDirectSearchInput(tool string, input json.RawMessage) bool {
}
return directsearch.IsDirectSearchCommand(obj.Command)
}
-
-func classifyText(text string) string {
- if strings.Contains(text, "```") {
- return "code"
- }
- return "text"
-}
diff --git a/internal/readers/codex_reader.go b/internal/readers/codex_reader.go
index 2427d8c..72e34ae 100644
--- a/internal/readers/codex_reader.go
+++ b/internal/readers/codex_reader.go
@@ -176,8 +176,7 @@ func codexMessage(ctx context.Context, item codexItem, ts time.Time, reasoning b
parts[i] = stripCodexInjectedWrappers(text)
}
}
- msg.Content = sync.CleanContent(strings.Join(parts, "\n"))
- msg.ContentType = classifyText(msg.Content)
+ msg.Content, msg.ContentType = cleanText(strings.Join(parts, "\n"))
case "function_call":
// Codex stores arguments as JSON encoded inside a JSON string.
var args map[string]json.RawMessage
diff --git a/internal/readers/omp_reader.go b/internal/readers/omp_reader.go
index 35d2e72..862a610 100644
--- a/internal/readers/omp_reader.go
+++ b/internal/readers/omp_reader.go
@@ -108,8 +108,8 @@ func (*OMPReader) Parse(ctx context.Context, path string, def input_config.Input
}
var text string
if m.Role != "toolResult" && json.Unmarshal(m.Content, &text) == nil {
- if cleaned := sync.CleanContent(text); cleaned != "" {
- emit(0, cleaned, classifyText(cleaned))
+ if cleaned, kind := cleanText(text); cleaned != "" {
+ emit(0, cleaned, kind)
}
return nil
}
@@ -155,8 +155,8 @@ func (*OMPReader) Parse(ctx context.Context, path string, def input_config.Input
}
switch b.Type {
case "text":
- if cleaned := sync.CleanContent(b.Text); cleaned != "" {
- emit(i, cleaned, classifyText(cleaned))
+ if cleaned, kind := cleanText(b.Text); cleaned != "" {
+ emit(i, cleaned, kind)
}
case "thinking":
if m.Role == "assistant" && def.Decode.IndexReasoning {
diff --git a/internal/readers/opencode_reader.go b/internal/readers/opencode_reader.go
index 2d704f9..e0765db 100644
--- a/internal/readers/opencode_reader.go
+++ b/internal/readers/opencode_reader.go
@@ -23,7 +23,7 @@ import (
type OpenCodeReader struct{}
func (r *OpenCodeReader) Name() string { return "opencode" }
-func (r *OpenCodeReader) IngestionEpoch() uint64 { return 2 }
+func (r *OpenCodeReader) IngestionEpoch() uint64 { return 3 }
// Discover returns paths to OpenCode database files matching the definition's discover config.
func (r *OpenCodeReader) Discover(ctx context.Context, def input_config.InputDefinition) ([]string, error) {
@@ -209,13 +209,14 @@ func parseOpenCodeSnapshot(ctx context.Context, tx *sql.Tx, dbPath string) (mode
return err
}
if len(textParts) > 0 {
+ text, kind := normalizeText(strings.Join(textParts, "\n"))
msgs = append(msgs, models.Message{
UUID: openCodeUUID("message", currentMsgID),
SessionID: currentSession, Cwd: currentCwd,
Role: normalizeOpenCodeRole(currentRole),
Origin: openCodeOriginForRole(currentRole),
- Content: strings.Join(textParts, "\n"),
- ContentType: "text",
+ Content: text,
+ ContentType: kind,
Timestamp: time.UnixMilli(currentTime),
})
}
diff --git a/internal/readers/opencode_reader_test.go b/internal/readers/opencode_reader_test.go
index 2cf071a..fb58b3f 100644
--- a/internal/readers/opencode_reader_test.go
+++ b/internal/readers/opencode_reader_test.go
@@ -304,8 +304,8 @@ func TestOpenCodeReader_Parse_MultipleTextParts(t *testing.T) {
if len(pf.Records) != 1 {
t.Fatalf("expected 1 record, got %d", len(pf.Records))
}
- if pf.Records[0].Content != "line1\nline2" {
- t.Errorf("Content = %q, want %q", pf.Records[0].Content, "line1\nline2")
+ if pf.Records[0].Content != "line1 line2" {
+ t.Errorf("Content = %q, want %q", pf.Records[0].Content, "line1 line2")
}
}
diff --git a/internal/readers/pi_reader.go b/internal/readers/pi_reader.go
index 6c80574..48f0b48 100644
--- a/internal/readers/pi_reader.go
+++ b/internal/readers/pi_reader.go
@@ -228,11 +228,11 @@ func extractPiMessages(ctx context.Context, rec piRecord, indexReasoning bool) (
// content as a plain string
var s string
if err := json.Unmarshal(rec.Message.Content, &s); err == nil {
- text := sync.CleanContent(s)
+ text, kind := cleanText(s)
if text == "" {
return nil, nil
}
- return []models.Message{{Role: role, Origin: piMessageOrigin(role), Content: text, ContentType: classifyText(text), Timestamp: ts, UUID: piRecordIdentity(rec.ID, "text")}}, nil
+ return []models.Message{{Role: role, Origin: piMessageOrigin(role), Content: text, ContentType: kind, Timestamp: ts, UUID: piRecordIdentity(rec.ID, "text")}}, nil
}
// content as an array of blocks
@@ -242,14 +242,18 @@ func extractPiMessages(ctx context.Context, rec piRecord, indexReasoning bool) (
}
var out []models.Message
var textParts []string
+ textKind := "text"
for i, b := range blocks {
if err := ctx.Err(); err != nil {
return nil, err
}
switch b.Type {
case "text":
- if c := sync.CleanContent(b.Text); c != "" {
+ if c, kind := cleanText(b.Text); c != "" {
textParts = append(textParts, c)
+ if kind == "code" {
+ textKind = kind
+ }
}
case "toolCall":
if t := SerializeToolInput(b.Name, b.Arguments); strings.TrimSpace(t) != "" {
@@ -274,7 +278,7 @@ func extractPiMessages(ctx context.Context, rec piRecord, indexReasoning bool) (
}
if len(textParts) > 0 {
text := strings.TrimSpace(strings.Join(textParts, " "))
- out = append([]models.Message{{Role: role, Origin: piMessageOrigin(role), Content: text, ContentType: classifyText(text), Timestamp: ts, UUID: piRecordIdentity(rec.ID, "text")}}, out...)
+ out = append([]models.Message{{Role: role, Origin: piMessageOrigin(role), Content: text, ContentType: textKind, Timestamp: ts, UUID: piRecordIdentity(rec.ID, "text")}}, out...)
}
return out, nil
}
diff --git a/internal/readers/reader.go b/internal/readers/reader.go
index 5f0b072..e91dbd0 100644
--- a/internal/readers/reader.go
+++ b/internal/readers/reader.go
@@ -8,9 +8,11 @@ import (
"encoding/hex"
"fmt"
"strconv"
+ "strings"
"github.com/pablontiv/backscroll/internal/input_config"
"github.com/pablontiv/backscroll/internal/models"
+ "github.com/pablontiv/backscroll/internal/sync"
)
// SessionReader abstracts reading sessions from a source (file, database, etc.).
@@ -128,3 +130,19 @@ func messageOriginForRole(role string) models.MessageOrigin {
return models.OriginUnknown
}
}
+
+// cleanText retains the established JSONL provider removals before applying the
+// common text policy. Native decoders handle their structural wrappers first.
+func cleanText(text string) (string, string) {
+ return normalizeText(sync.RemoveContentNoise(text))
+}
+
+// normalizeText classifies decoded text before collapsing whitespace. OpenCode
+// opts into this whitespace-only policy after selecting its native text parts.
+func normalizeText(text string) (string, string) {
+ kind := "text"
+ if strings.Contains(text, "```") {
+ kind = "code"
+ }
+ return sync.NormalizeWhitespace(text), kind
+}
diff --git a/internal/readers/reader_test.go b/internal/readers/reader_test.go
index fefdada..068ea9f 100644
--- a/internal/readers/reader_test.go
+++ b/internal/readers/reader_test.go
@@ -135,3 +135,24 @@ func TestIngestionFingerprintCoversSupportedSemantics(t *testing.T) {
})
}
}
+
+func TestIngestionFingerprintNormalizationEpochs(t *testing.T) {
+ for _, tc := range []struct {
+ reader SessionReader
+ epoch uint64
+ }{
+ {&ClaudeReader{}, 1},
+ {&PiReader{}, 2},
+ {&CodexReader{}, 2},
+ {&OMPReader{}, 1},
+ {&OpenCodeReader{}, 3},
+ } {
+ t.Run(tc.reader.Name(), func(t *testing.T) {
+ def := input_config.InputDefinition{Source: "session"}
+ want := IngestionFingerprint(&mockReader{name: tc.reader.Name(), epoch: tc.epoch}, def)
+ if got := IngestionFingerprint(tc.reader, def); got != want {
+ t.Fatalf("fingerprint=%q, want epoch %d fingerprint %q", got, tc.epoch, want)
+ }
+ })
+ }
+}
diff --git a/internal/sync/sync.go b/internal/sync/sync.go
index 465343c..24a8b2e 100644
--- a/internal/sync/sync.go
+++ b/internal/sync/sync.go
@@ -15,8 +15,19 @@ func IsNoiseType(typ string) bool {
return noiseTypes[typ]
}
-// CleanContent removes noise patterns from content
+// CleanContent removes provider noise and normalizes searchable whitespace.
func CleanContent(content string) string {
+ return NormalizeWhitespace(RemoveContentNoise(content))
+}
+
+// NormalizeWhitespace collapses text whitespace without removing provider content.
+func NormalizeWhitespace(content string) string {
+ return strings.Join(strings.Fields(content), " ")
+}
+
+// RemoveContentNoise applies the established JSONL reader removals while keeping
+// whitespace intact for classification. Readers opt into this provider policy.
+func RemoveContentNoise(content string) string {
if content == "" {
return ""
}
@@ -61,10 +72,5 @@ func CleanContent(content string) string {
content = strings.TrimPrefix(content, "Caveat: ")
content = strings.ReplaceAll(content, "Request interrupted", "")
- // Clean extra whitespace
- content = strings.TrimSpace(content)
- // Collapse multiple spaces
- content = strings.Join(strings.Fields(content), " ")
-
return content
}
diff --git a/internal/sync/sync_test.go b/internal/sync/sync_test.go
index de369d3..3defe31 100644
--- a/internal/sync/sync_test.go
+++ b/internal/sync/sync_test.go
@@ -49,6 +49,16 @@ func TestCleanContent(t *testing.T) {
input: "hello world test",
want: "hello world test",
},
+ {
+ name: "multiline fenced whitespace collapse",
+ input: " before\n```go\nvalue := 167\n```\n after\tmarker\u00a0 ",
+ want: "before ```go value := 167 ``` after marker",
+ },
+ {
+ name: "provider removal before whitespace",
+ input: "Caveat: before ```hidden```\n\t after",
+ want: "before after",
+ },
{
name: "empty string",
input: "",