diff --git a/cmd/backscroll/mixed_provider_e2e_test.go b/cmd/backscroll/mixed_provider_e2e_test.go index c4acecf..d85024a 100644 --- a/cmd/backscroll/mixed_provider_e2e_test.go +++ b/cmd/backscroll/mixed_provider_e2e_test.go @@ -1,10 +1,12 @@ package main import ( + "bytes" "database/sql" "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "reflect" "strings" @@ -197,7 +199,39 @@ func TestMixedProviderNativeOwnershipReplayAndRecovery(t *testing.T) { available.run("rebuild") equal(query(available.database, snapshot), stable) equal(query(available.database, outcomes), signals) - for _, path := range []string{omp, pi, codex, oc} { + // A mixed sync keeps perennial records from both orphaned native groups + // and an expired filesystem capability. + if err := os.Remove(omp); err != nil { + t.Fatal(err) + } + native, err = sql.Open("sqlite", oc) + if err != nil { + t.Fatal(err) + } + if _, err := native.Exec(`DELETE FROM session WHERE id='s0'`); err != nil { + t.Fatal(err) + } + if err := native.Close(); err != nil { + t.Fatal(err) + } + cmd := exec.Command(e.binary, "search", "--text", "mixedproviderquartz", "--all-projects", "--json") + cmd.Dir, cmd.Env = e.root, e.environment + var stderr bytes.Buffer + cmd.Stderr = &stderr + output, err := cmd.Output() + if err != nil { + t.Fatalf("mixed orphan sync: %v stderr=%s", err, &stderr) + } + if strings.Count(stderr.String(), "warning: opencode: skipped 1 messages without native session metadata\n") != 1 { + t.Fatalf("orphan warning: %s", &stderr) + } + var retained []json.RawMessage + if err := json.Unmarshal(output, &retained); err != nil || len(retained) != 5 { + t.Fatalf("stdout JSON: %s err=%v", output, err) + } + equal(query(e.database, snapshot), stable) + equal(query(e.database, outcomes), signals) + for _, path := range []string{pi, codex, oc} { if err := os.Remove(path); err != nil { t.Fatal(err) } diff --git a/cmd/backscroll/startup_policy.go b/cmd/backscroll/startup_policy.go index 607a8ea..fb34594 100644 --- a/cmd/backscroll/startup_policy.go +++ b/cmd/backscroll/startup_policy.go @@ -252,6 +252,7 @@ query merges both by rank position (RRF).`, return nil } ctx := cmd.Context() + ctx = context.WithValue(ctx, startupWarningWriterKey{}, stderr) if syncService.diagnostics { ctx = context.WithValue(ctx, startupDiagnosticsWriterKey{}, stderr) } diff --git a/cmd/backscroll/sync_helpers.go b/cmd/backscroll/sync_helpers.go index d538bf8..bcb07b4 100644 --- a/cmd/backscroll/sync_helpers.go +++ b/cmd/backscroll/sync_helpers.go @@ -390,6 +390,8 @@ func isRacyCleanFile(fileMtime string, lastIndexed string) bool { return fileMt.After(indexTime.Add(-time.Duration(racyMarginSeconds) * time.Second)) } +type startupWarningWriterKey struct{} + // maybeAutoSync preserves the legacy non-cancellable entry point by creating // production dependencies for this invocation rather than using shared state. func maybeAutoSync(cfg *config.Config, progress io.Writer) error { @@ -644,6 +646,7 @@ func (s *startupSyncService) syncWithProgress(ctx context.Context, cfg *config.C // selected replays are deterministic even when discovery order changes. var indexedFiles []storage.IndexedFile semanticsReplaysDone, staleReplaysDone := 0, 0 + skippedOpenCodeMessages := 0 for _, claim := range uniqueClaims { if err := ctx.Err(); err != nil { return err @@ -671,6 +674,9 @@ func (s *startupSyncService) syncWithProgress(ctx context.Context, cfg *config.C if err != nil { return fmt.Errorf("parse %s: %w", ref, err) } + if reader.Name() == "opencode" { + skippedOpenCodeMessages += pf.SkippedMessages + } if err := ctx.Err(); err != nil { return err } @@ -821,5 +827,13 @@ func (s *startupSyncService) syncWithProgress(ctx context.Context, cfg *config.C return err } + if skippedOpenCodeMessages > 0 { + warnings, ok := ctx.Value(startupWarningWriterKey{}).(io.Writer) + if !ok { + warnings = progress + } + _, _ = fmt.Fprintf(warnings, "warning: opencode: skipped %d messages without native session metadata\n", skippedOpenCodeMessages) + } + return nil } diff --git a/docs/input-contract.md b/docs/input-contract.md index 3057195..cb1320e 100644 --- a/docs/input-contract.md +++ b/docs/input-contract.md @@ -617,10 +617,15 @@ A single SQLite capability remains one `source_path`; it can contain many native sessions. Each indexed record carries the structured `message.session_id` and, when available, the joined `session.directory`. A present directory is used for project-registry identification. When native cwd is absent, project proof remains -unknown; the database location supplies no native project attribution. Missing rows in a present -session table, empty identities, contradictory message/part session IDs, and -relative or NUL-bearing directories fail the parse conservatively. Message/part-only -exports remain supported without inventing native cwd. +unknown; the database location supplies no native project attribution. When the +session table exists, messages with a matching native session row and their parts +are eligible for indexing. Messages whose native session row is missing are +skipped together with their parts, counted once per message, and reported on +stderr during startup sync. Previously indexed native records retain their +payloads and anchors. Empty identities, contradictory message/part session IDs, +parts without a message row, and relative or NUL-bearing directories fail the +parse conservatively. Message/part-only exports remain supported without +inventing native cwd. Text identity is a deterministic UUIDv5 over a namespace-qualified native message ID. Tool identity is a deterministic UUIDv5 over native message and part IDs, with diff --git a/internal/models/models.go b/internal/models/models.go index c598121..d5c68b4 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -26,10 +26,11 @@ type SearchResult struct { // ParsedFile represents a parsed session or plan file. type ParsedFile struct { - Path string - Hash string // SHA-256 hex - Records []Message - Cwd string // working directory, used for project identification + Path string + Hash string // SHA-256 hex + Records []Message + Cwd string // working directory, used for project identification + SkippedMessages int // messages skipped because native session metadata is unavailable } // Message represents a message in a session. diff --git a/internal/readers/opencode_native_test.go b/internal/readers/opencode_native_test.go index e550b65..28b8f1e 100644 --- a/internal/readers/opencode_native_test.go +++ b/internal/readers/opencode_native_test.go @@ -115,8 +115,9 @@ func TestOpenCodeNativeToolIdentityAndMalformedMetadata(t *testing.T) { if _, err := db.Exec(`DELETE FROM session`); err != nil { t.Fatal(err) } - if _, err := reader.Parse(context.Background(), path, input_config.InputDefinition{}); err == nil { - t.Fatal("orphan session silently fell back") + orphaned, err := reader.Parse(context.Background(), path, input_config.InputDefinition{}) + if err != nil || len(orphaned.Records) != 0 || orphaned.SkippedMessages != 4 { + t.Fatalf("orphan groups: %+v err=%v", orphaned, err) } } diff --git a/internal/readers/opencode_reader.go b/internal/readers/opencode_reader.go index e0765db..4c3d315 100644 --- a/internal/readers/opencode_reader.go +++ b/internal/readers/opencode_reader.go @@ -174,13 +174,11 @@ func parseOpenCodeSnapshot(ctx context.Context, tx *sql.Tx, dbPath string) (mode if invalid { return models.ParsedFile{}, fmt.Errorf("invalid OpenCode message/part relationship") } + var skippedMessages int if hasSession { - if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM message m LEFT JOIN session s ON s.id=m.session_id WHERE s.id IS NULL)`).Scan(&invalid); err != nil { + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM message m LEFT JOIN session s ON s.id=m.session_id WHERE s.id IS NULL`).Scan(&skippedMessages); err != nil { return models.ParsedFile{}, err } - if invalid { - return models.ParsedFile{}, fmt.Errorf("OpenCode message has no native session metadata") - } } directory, join, sessionColumn := "NULL", "", "NULL" if hasSession { @@ -253,12 +251,12 @@ func parseOpenCodeSnapshot(ctx context.Context, tx *sql.Tx, dbPath string) (mode return models.ParsedFile{}, err } - if hasSession && !nativeSession.Valid { - return models.ParsedFile{}, fmt.Errorf("OpenCode message has no native session metadata") - } if !validOpenCodeID(msgID) || !validOpenCodeID(partID) || !validOpenCodeID(sessionID) || partSession != sessionID { return models.ParsedFile{}, fmt.Errorf("invalid OpenCode native identity or session relationship") } + if hasSession && !nativeSession.Valid { + continue + } if cwd.Valid && cwd.String != "" && (!filepath.IsAbs(cwd.String) || strings.IndexByte(cwd.String, 0) >= 0) { return models.ParsedFile{}, fmt.Errorf("invalid OpenCode native directory") } @@ -317,9 +315,10 @@ func parseOpenCodeSnapshot(ctx context.Context, tx *sql.Tx, dbPath string) (mode } return models.ParsedFile{ - Path: dbPath, - Hash: hash, - Records: msgs, + Path: dbPath, + Hash: hash, + Records: msgs, + SkippedMessages: skippedMessages, }, nil } diff --git a/internal/readers/opencode_snapshot_test.go b/internal/readers/opencode_snapshot_test.go index 8494041..0e997f0 100644 --- a/internal/readers/opencode_snapshot_test.go +++ b/internal/readers/opencode_snapshot_test.go @@ -149,3 +149,42 @@ func TestOpenCodeSnapshotCancellation(t *testing.T) { t.Fatal("Parse ignored cancellation") } } + +func TestOpenCodeSnapshotSkipsMissingSessionGroups(t *testing.T) { + path := createOpenCodeDB(t) + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE session(id TEXT PRIMARY KEY,directory TEXT); INSERT INTO session VALUES('sess-1','/projects/native')`); err != nil { + t.Fatal(err) + } + reader := &OpenCodeReader{} + before, err := reader.Parse(context.Background(), path, input_config.InputDefinition{}) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO message VALUES('orphan-a','missing',0,0,'{"role":"user"}'),('orphan-b','missing',0,0,'{"role":"assistant"}'),('orphan-empty','missing',0,0,'{"role":"user"}'); INSERT INTO part VALUES('orphan-p1','orphan-a','missing',0,0,'{"type":"text","text":"orphan"}'),('orphan-p2','orphan-a','missing',0,0,'{"type":"text","text":"also orphan"}'),('orphan-p3','orphan-b','missing',0,0,'{"type":"tool","tool":"Bash","state":{"input":{"command":"echo orphan"},"output":"orphan"}}')`); err != nil { + t.Fatal(err) + } + after, err := reader.Parse(context.Background(), path, input_config.InputDefinition{}) + if err != nil { + t.Fatal(err) + } + if after.SkippedMessages != 3 || len(after.Records) != len(before.Records) { + t.Fatalf("skipped groups: %+v", after) + } + for i, message := range after.Records { + want := before.Records[i] + if message != want || message.UUID == "" || message.SessionID != "sess-1" || message.Cwd != "/projects/native" || message.Origin == "" { + t.Fatalf("native record changed: %+v want %+v", message, want) + } + } + if _, err := db.Exec(`INSERT INTO part VALUES('broken','absent-message','missing',0,0,'{"type":"text","text":"broken"}')`); err != nil { + t.Fatal(err) + } + if _, err := reader.Parse(context.Background(), path, input_config.InputDefinition{}); err == nil || err.Error() != "invalid OpenCode message/part relationship" { + t.Fatalf("relationship error: %v", err) + } +}