Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion cmd/backscroll/mixed_provider_e2e_test.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package main

import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
Expand Down Expand Up @@ -197,7 +199,39 @@ func TestMixedProviderNativeOwnershipReplayAndRecovery(t *testing.T) {
available.run("rebuild")
equal(query(available.database, snapshot), stable)
equal(query(available.database, outcomes), signals)
for _, path := range []string{omp, pi, codex, oc} {
// A mixed sync keeps perennial records from both orphaned native groups
// and an expired filesystem capability.
if err := os.Remove(omp); err != nil {
t.Fatal(err)
}
native, err = sql.Open("sqlite", oc)
if err != nil {
t.Fatal(err)
}
if _, err := native.Exec(`DELETE FROM session WHERE id='s0'`); err != nil {
t.Fatal(err)
}
if err := native.Close(); err != nil {
t.Fatal(err)
}
cmd := exec.Command(e.binary, "search", "--text", "mixedproviderquartz", "--all-projects", "--json")
cmd.Dir, cmd.Env = e.root, e.environment
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
t.Fatalf("mixed orphan sync: %v stderr=%s", err, &stderr)
}
if strings.Count(stderr.String(), "warning: opencode: skipped 1 messages without native session metadata\n") != 1 {
t.Fatalf("orphan warning: %s", &stderr)
}
var retained []json.RawMessage
if err := json.Unmarshal(output, &retained); err != nil || len(retained) != 5 {
t.Fatalf("stdout JSON: %s err=%v", output, err)
}
equal(query(e.database, snapshot), stable)
equal(query(e.database, outcomes), signals)
for _, path := range []string{pi, codex, oc} {
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
Expand Down
1 change: 1 addition & 0 deletions cmd/backscroll/startup_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ query merges both by rank position (RRF).`,
return nil
}
ctx := cmd.Context()
ctx = context.WithValue(ctx, startupWarningWriterKey{}, stderr)
if syncService.diagnostics {
ctx = context.WithValue(ctx, startupDiagnosticsWriterKey{}, stderr)
}
Expand Down
14 changes: 14 additions & 0 deletions cmd/backscroll/sync_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,8 @@ func isRacyCleanFile(fileMtime string, lastIndexed string) bool {
return fileMt.After(indexTime.Add(-time.Duration(racyMarginSeconds) * time.Second))
}

type startupWarningWriterKey struct{}

// maybeAutoSync preserves the legacy non-cancellable entry point by creating
// production dependencies for this invocation rather than using shared state.
func maybeAutoSync(cfg *config.Config, progress io.Writer) error {
Expand Down Expand Up @@ -644,6 +646,7 @@ func (s *startupSyncService) syncWithProgress(ctx context.Context, cfg *config.C
// selected replays are deterministic even when discovery order changes.
var indexedFiles []storage.IndexedFile
semanticsReplaysDone, staleReplaysDone := 0, 0
skippedOpenCodeMessages := 0
for _, claim := range uniqueClaims {
if err := ctx.Err(); err != nil {
return err
Expand Down Expand Up @@ -671,6 +674,9 @@ func (s *startupSyncService) syncWithProgress(ctx context.Context, cfg *config.C
if err != nil {
return fmt.Errorf("parse %s: %w", ref, err)
}
if reader.Name() == "opencode" {
skippedOpenCodeMessages += pf.SkippedMessages
}
if err := ctx.Err(); err != nil {
return err
}
Expand Down Expand Up @@ -821,5 +827,13 @@ func (s *startupSyncService) syncWithProgress(ctx context.Context, cfg *config.C
return err
}

if skippedOpenCodeMessages > 0 {
warnings, ok := ctx.Value(startupWarningWriterKey{}).(io.Writer)
if !ok {
warnings = progress
}
_, _ = fmt.Fprintf(warnings, "warning: opencode: skipped %d messages without native session metadata\n", skippedOpenCodeMessages)
}

return nil
}
13 changes: 9 additions & 4 deletions docs/input-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -617,10 +617,15 @@ A single SQLite capability remains one `source_path`; it can contain many native
sessions. Each indexed record carries the structured `message.session_id` and,
when available, the joined `session.directory`. A present directory is used for
project-registry identification. When native cwd is absent, project proof remains
unknown; the database location supplies no native project attribution. Missing rows in a present
session table, empty identities, contradictory message/part session IDs, and
relative or NUL-bearing directories fail the parse conservatively. Message/part-only
exports remain supported without inventing native cwd.
unknown; the database location supplies no native project attribution. When the
session table exists, messages with a matching native session row and their parts
are eligible for indexing. Messages whose native session row is missing are
skipped together with their parts, counted once per message, and reported on
stderr during startup sync. Previously indexed native records retain their
payloads and anchors. Empty identities, contradictory message/part session IDs,
parts without a message row, and relative or NUL-bearing directories fail the
parse conservatively. Message/part-only exports remain supported without
inventing native cwd.

Text identity is a deterministic UUIDv5 over a namespace-qualified native message
ID. Tool identity is a deterministic UUIDv5 over native message and part IDs, with
Expand Down
9 changes: 5 additions & 4 deletions internal/models/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@ type SearchResult struct {

// ParsedFile represents a parsed session or plan file.
type ParsedFile struct {
Path string
Hash string // SHA-256 hex
Records []Message
Cwd string // working directory, used for project identification
Path string
Hash string // SHA-256 hex
Records []Message
Cwd string // working directory, used for project identification
SkippedMessages int // messages skipped because native session metadata is unavailable
}

// Message represents a message in a session.
Expand Down
5 changes: 3 additions & 2 deletions internal/readers/opencode_native_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,9 @@ func TestOpenCodeNativeToolIdentityAndMalformedMetadata(t *testing.T) {
if _, err := db.Exec(`DELETE FROM session`); err != nil {
t.Fatal(err)
}
if _, err := reader.Parse(context.Background(), path, input_config.InputDefinition{}); err == nil {
t.Fatal("orphan session silently fell back")
orphaned, err := reader.Parse(context.Background(), path, input_config.InputDefinition{})
if err != nil || len(orphaned.Records) != 0 || orphaned.SkippedMessages != 4 {
t.Fatalf("orphan groups: %+v err=%v", orphaned, err)
}
}

Expand Down
19 changes: 9 additions & 10 deletions internal/readers/opencode_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,13 +174,11 @@ func parseOpenCodeSnapshot(ctx context.Context, tx *sql.Tx, dbPath string) (mode
if invalid {
return models.ParsedFile{}, fmt.Errorf("invalid OpenCode message/part relationship")
}
var skippedMessages int
if hasSession {
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM message m LEFT JOIN session s ON s.id=m.session_id WHERE s.id IS NULL)`).Scan(&invalid); err != nil {
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM message m LEFT JOIN session s ON s.id=m.session_id WHERE s.id IS NULL`).Scan(&skippedMessages); err != nil {
return models.ParsedFile{}, err
}
if invalid {
return models.ParsedFile{}, fmt.Errorf("OpenCode message has no native session metadata")
}
}
directory, join, sessionColumn := "NULL", "", "NULL"
if hasSession {
Expand Down Expand Up @@ -253,12 +251,12 @@ func parseOpenCodeSnapshot(ctx context.Context, tx *sql.Tx, dbPath string) (mode
return models.ParsedFile{}, err
}

if hasSession && !nativeSession.Valid {
return models.ParsedFile{}, fmt.Errorf("OpenCode message has no native session metadata")
}
if !validOpenCodeID(msgID) || !validOpenCodeID(partID) || !validOpenCodeID(sessionID) || partSession != sessionID {
return models.ParsedFile{}, fmt.Errorf("invalid OpenCode native identity or session relationship")
}
if hasSession && !nativeSession.Valid {
continue
}
if cwd.Valid && cwd.String != "" && (!filepath.IsAbs(cwd.String) || strings.IndexByte(cwd.String, 0) >= 0) {
return models.ParsedFile{}, fmt.Errorf("invalid OpenCode native directory")
}
Expand Down Expand Up @@ -317,9 +315,10 @@ func parseOpenCodeSnapshot(ctx context.Context, tx *sql.Tx, dbPath string) (mode
}

return models.ParsedFile{
Path: dbPath,
Hash: hash,
Records: msgs,
Path: dbPath,
Hash: hash,
Records: msgs,
SkippedMessages: skippedMessages,
}, nil
}

Expand Down
39 changes: 39 additions & 0 deletions internal/readers/opencode_snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,3 +149,42 @@ func TestOpenCodeSnapshotCancellation(t *testing.T) {
t.Fatal("Parse ignored cancellation")
}
}

func TestOpenCodeSnapshotSkipsMissingSessionGroups(t *testing.T) {
path := createOpenCodeDB(t)
db, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`CREATE TABLE session(id TEXT PRIMARY KEY,directory TEXT); INSERT INTO session VALUES('sess-1','/projects/native')`); err != nil {
t.Fatal(err)
}
reader := &OpenCodeReader{}
before, err := reader.Parse(context.Background(), path, input_config.InputDefinition{})
if err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`INSERT INTO message VALUES('orphan-a','missing',0,0,'{"role":"user"}'),('orphan-b','missing',0,0,'{"role":"assistant"}'),('orphan-empty','missing',0,0,'{"role":"user"}'); INSERT INTO part VALUES('orphan-p1','orphan-a','missing',0,0,'{"type":"text","text":"orphan"}'),('orphan-p2','orphan-a','missing',0,0,'{"type":"text","text":"also orphan"}'),('orphan-p3','orphan-b','missing',0,0,'{"type":"tool","tool":"Bash","state":{"input":{"command":"echo orphan"},"output":"orphan"}}')`); err != nil {
t.Fatal(err)
}
after, err := reader.Parse(context.Background(), path, input_config.InputDefinition{})
if err != nil {
t.Fatal(err)
}
if after.SkippedMessages != 3 || len(after.Records) != len(before.Records) {
t.Fatalf("skipped groups: %+v", after)
}
for i, message := range after.Records {
want := before.Records[i]
if message != want || message.UUID == "" || message.SessionID != "sess-1" || message.Cwd != "/projects/native" || message.Origin == "" {
t.Fatalf("native record changed: %+v want %+v", message, want)
}
}
if _, err := db.Exec(`INSERT INTO part VALUES('broken','absent-message','missing',0,0,'{"type":"text","text":"broken"}')`); err != nil {
t.Fatal(err)
}
if _, err := reader.Parse(context.Background(), path, input_config.InputDefinition{}); err == nil || err.Error() != "invalid OpenCode message/part relationship" {
t.Fatalf("relationship error: %v", err)
}
}
Loading