diff --git a/cmd/opensave-app/frontend/src/components/ConflictModal.svelte b/cmd/opensave-app/frontend/src/components/ConflictModal.svelte index 4a9311ff..f24a7baa 100644 --- a/cmd/opensave-app/frontend/src/components/ConflictModal.svelte +++ b/cmd/opensave-app/frontend/src/components/ConflictModal.svelte @@ -1,6 +1,6 @@
@@ -99,12 +108,22 @@ {peer.status} + {#if peer.needsUpdate} + + needs update + + {/if}
{peer.address === 'relay' ? 'internet relay' : `${peer.address}:${peer.port}`} · last synced {fmtTime(peer.lastSynced)} {#if peer.appVersion}· OpenSave {peer.appVersion}{/if}
+ {#if peer.link} +
+ {linkLabel(peer.link)} +
+ {/if}
diff --git a/cmd/opensave-app/frontend/src/views/Home.svelte b/cmd/opensave-app/frontend/src/views/Home.svelte index bcfd5091..afad2c09 100644 --- a/cmd/opensave-app/frontend/src/views/Home.svelte +++ b/cmd/opensave-app/frontend/src/views/Home.svelte @@ -5,6 +5,7 @@ import { gameStatus, conflictedIds } from '../lib/gamestatus.js'; import OfferedGames from './home/OfferedGames.svelte'; import AddGameCard from './home/AddGameCard.svelte'; + import TrackGameDialog from './home/TrackGameDialog.svelte'; import ScanDialog from './home/ScanDialog.svelte'; import HomeSummary from './home/HomeSummary.svelte'; import LibraryGrid from './home/LibraryGrid.svelte'; @@ -30,6 +31,8 @@ // on the params object, so each request — the sidebar's +, Ctrl+K — opens // it once, and not again on every re-render. let showAdd = false; + // Tracking a game the game database knows (TrackGameDialog). + let showTrackGame = false; let addPath = ''; let handledAdd = null; $: if (params.add && params !== handledAdd) { @@ -108,6 +111,7 @@ +
@@ -118,6 +122,10 @@ (showAdd = false)} /> {/if} +{#if showTrackGame} + (showTrackGame = false)} /> +{/if} + {#if !$stateLoaded} diff --git a/cmd/opensave-app/frontend/src/views/game/ConfigTab.svelte b/cmd/opensave-app/frontend/src/views/game/ConfigTab.svelte index 459703f1..8d6f8bfb 100644 --- a/cmd/opensave-app/frontend/src/views/game/ConfigTab.svelte +++ b/cmd/opensave-app/frontend/src/views/game/ConfigTab.svelte @@ -25,8 +25,12 @@ autoSync: game.autoSync ?? true, maxSnapshots: game.maxSnapshots ?? 5, maxManualSnapshots: game.maxManualSnapshots ?? 0, - syncIgnore: game.syncIgnore ?? '' + syncIgnore: game.syncIgnore ?? '', + syncDeviceSettings: game.syncDeviceSettings ?? false }; + // The files the game database names as this game's settings rather than its + // save (graphics, resolution): left out of syncing unless asked for. + $: deviceSettings = game.deviceSettings ?? []; let locations = []; // Cover preview: a custom URL wins, else the proxied Steam art for the App @@ -76,7 +80,8 @@ autoSync: cfg.autoSync, maxSnapshots: Number(cfg.maxSnapshots), maxManualSnapshots: Number(cfg.maxManualSnapshots), - syncIgnore: cfg.syncIgnore ?? '' + syncIgnore: cfg.syncIgnore ?? '', + syncDeviceSettings: !!cfg.syncDeviceSettings }) ); } @@ -160,6 +165,32 @@ can't push them out. 0 = keep forever (the default). + {#if deviceSettings.length > 0} +
+ +
    + {#each deviceSettings as s} +
  • + {s.pattern.replace(/^!?\//, '')} + + {#if s.source === 'database'}game database{:else if s.source === 'detected'}detected + — {s.reason}{:else}syncs again — {s.reason}{/if} + +
  • + {/each} +
+
+ + + These are this game's settings, not its save — named by the game database or found by + what they hold — so each device keeps its own (graphics, resolution). They are still in + every snapshot. A file that turns out to change along with the save syncs again by + itself; to sync one yourself, untick it under “Pick from your save folder” below. + + {/if} 0} />
@@ -206,6 +237,29 @@ .fields .field { margin-bottom: 14px; } + .device-settings { + margin: 4px 0 6px; + padding-left: 18px; + color: var(--text-dim, inherit); + } + .device-settings code { + font-size: 0.9em; + } + .device-settings .source { + font-size: 0.82em; + color: var(--text-faint); + margin-left: 6px; + } + .device-settings .given-back code { + text-decoration: line-through; + } + .device-block { + margin-bottom: 4px !important; + } + .device-hint { + display: block; + margin-bottom: 14px; + } .hint-warn { color: var(--warn); } diff --git a/cmd/opensave-app/frontend/src/views/game/GameHeader.svelte b/cmd/opensave-app/frontend/src/views/game/GameHeader.svelte index 19750020..8718ad38 100644 --- a/cmd/opensave-app/frontend/src/views/game/GameHeader.svelte +++ b/cmd/opensave-app/frontend/src/views/game/GameHeader.svelte @@ -10,13 +10,17 @@ import { timeAgo } from '../../lib/timeago.js'; import { playLength } from '../../lib/format.js'; import { canLaunch } from '../../lib/gameactions.js'; + import Spinner from '../../components/ui/Spinner.svelte'; + import { restoreProgressText } from '../../lib/restoreprogress.js'; import ArrowLeft from 'lucide-svelte/icons/arrow-left'; import Play from 'lucide-svelte/icons/play'; import RefreshCw from 'lucide-svelte/icons/refresh-cw'; + import Upload from 'lucide-svelte/icons/upload'; import FolderOpen from 'lucide-svelte/icons/folder-open'; import Pencil from 'lucide-svelte/icons/pencil'; import Star from 'lucide-svelte/icons/star'; import TriangleAlert from 'lucide-svelte/icons/triangle-alert'; + import Info from 'lucide-svelte/icons/info'; import { collections, isFavourite, toggleFavourite } from '../../lib/collections.js'; export let game; @@ -24,6 +28,14 @@ export let runner; const { busy, run } = runner; + // A game OpenSave cannot find here: its program, chosen by hand, says where + // it is installed and what runs when it is played (sessions, launch). + async function pickProgram() { + const exe = await native.selectFile(`Program of “${game.name}”`); + if (!exe) return; + await run(`Program set for ${game.name}`, () => api.patch(`/api/games/${game.id}`, { exePath: exe })); + } + $: activity = $syncActivity[game.id]; // Is my Deck up to date with THIS save? One entry per paired device, in @@ -42,6 +54,20 @@ const syncNow = () => run('Sync triggered', () => api.post(`/api/games/${game.id}/sync`)); const launchGame = () => run('Launching…', () => api.post(`/api/games/${game.id}/launch`)); + // For when the devices hold states nobody can rank — a copy that arrived + // only half-way, a mixture of two saves — and the person knows which device + // has the right one. Every other device takes this save whole, keeping a + // snapshot of its own first; one that changed the save on its own since is + // still asked. + async function useEverywhere() { + const ok = await askConfirm( + `Make this device's save of ${game.name} the one every other device uses? They each replace theirs with it — every file, including removing files this device does not have — and keep a snapshot of their own first. A device where the save was changed after this update is asked instead.`, + { title: 'Use this save everywhere?', confirmText: 'Use this save everywhere', danger: true } + ); + if (!ok) return; + await run(`${game.name}: this save goes to your other devices`, () => api.post(`/api/games/${game.id}/use-everywhere`)); + } + let editPath = false; let pathDraft = ''; @@ -112,13 +138,24 @@ {#if activity?.state === 'running'} · syncing {activity.percentage ?? 0}% {/if} + {#if game.restoring && !answering} + · {restoreProgressText(game.restoring)} + {/if} + {#if game.holdingChanges && !game.playingSince} + · changes held back while it seems to be played — synced once its save is quiet + {/if} {#if game.playingSince} · in session since {new Date(game.playingSince).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} {:else if game.lastPlayedAt} · played {timeAgo(game.lastPlayedAt, now)}, {playLength(game.playtimeMs)} in all {/if} {#if game.installed === 'not-found'} - · not found on this PC + · {/if}
{#if syncedWith.length > 0} @@ -137,6 +174,7 @@ {#if canLaunch(game)} {/if} + @@ -163,9 +201,18 @@
- {#if game.emptied.state === 'fetching'} + {#if answering} + Putting the files back{game.emptied.putBackFrom ? ` from the snapshot of ${whenLabel(game.emptied.putBackFromTime)}` : ''}… + {#if game.restoring}{restoreProgressText(game.restoring)}.{:else}A large save takes a few minutes.{/if} + Anything newer comes from your other devices afterwards. + {:else if game.emptied.state === 'fetching'} Putting the files back. - Some are still to come from your other devices; this game syncs as usual once they're all here. + {#if activity?.state === 'running'} + Fetching the rest from your other devices — {activity.percentage ?? 0}%. + {:else} + Some are still to come from your other devices: waiting for one of them to send them — it has to be + online. This game syncs as usual once they're all here. + {/if} {:else} Every save file in {emptiedWhere(game.emptied.locations)} was deleted on this device. Your other devices still have theirs, and nothing is synced until you choose. @@ -188,7 +235,18 @@
{/if} -{#if game.savePathMissing && !editPath} +{#if game.savePathMissing && game.saveDriveMissing && !editPath} + +
+ +
+ This game isn't on this device. + Its saves are on {game.savePath.slice(0, 2)}, which this PC doesn't have, so it's skipped here and keeps syncing + between your other devices. If it is installed here, point the game at its save folder with Edit. +
+
+{:else if game.savePathMissing && !editPath} @@ -204,6 +262,15 @@ {/if} diff --git a/cmd/opensave-app/main.go b/cmd/opensave-app/main.go index 09e46f07..4f8ca678 100644 --- a/cmd/opensave-app/main.go +++ b/cmd/opensave-app/main.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "runtime" + "runtime/debug" "github.com/wailsapp/wails/v2" "github.com/wailsapp/wails/v2/pkg/options" @@ -45,6 +46,8 @@ func main() { // removes its leftover binary) before claiming the single-instance lock. cleanupReplacedBinary() + tuneGC() + app := NewApp() err := wails.Run(&options.App{ @@ -97,3 +100,17 @@ func main() { os.Exit(1) } } + +// tuneGC trades a little CPU for a smaller footprint. OpenSave is a +// background app, and Go's default (let the heap double before collecting) +// meant a large save's manifests showed up as twice their size in Task +// Manager. The soft limit makes the runtime collect and hand memory back to +// the OS harder as it approaches 1GiB. Explicit GOGC / GOMEMLIMIT win. +func tuneGC() { + if os.Getenv("GOGC") == "" { + debug.SetGCPercent(50) + } + if os.Getenv("GOMEMLIMIT") == "" { + debug.SetMemoryLimit(1 << 30) + } +} diff --git a/e2e/batch_bench_test.go b/e2e/batch_bench_test.go new file mode 100644 index 00000000..0299c4e8 --- /dev/null +++ b/e2e/batch_bench_test.go @@ -0,0 +1,59 @@ +package e2e + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/opensave/opensave/internal/p2p" + "github.com/opensave/opensave/internal/p2p/syncengine" + "github.com/opensave/opensave/testutil" +) + +// TestBench_ManySmallFiles times pulling a save of many small files between +// two real daemons, batched and file by file. Opt-in, because it is a +// measurement rather than a check: OPENSAVE_BENCH_FILES=5000 go test ./e2e -run TestBench -v +func TestBench_ManySmallFiles(t *testing.T) { + n, _ := strconv.Atoi(os.Getenv("OPENSAVE_BENCH_FILES")) + if n <= 0 { + t.Skip("set OPENSAVE_BENCH_FILES to run") + } + for _, mode := range []struct { + name string + proto int + }{{"batched", syncengine.ProtoBatchFiles}, {"file-by-file", syncengine.ProtoMultiRoot}} { + t.Run(mode.name, func(t *testing.T) { + restore := p2p.SetServedProto(mode.proto) + defer p2p.SetServedProto(restore) + + a := testutil.NewTestDaemon(t, "Bench-A") + b := testutil.NewTestDaemon(t, "Bench-B") + a.PairWith(b) + chunk := strings.Repeat("x", 3500) // a Project Zomboid map chunk is ~3.5KB + for i := 0; i < n; i++ { + a.WriteSave(fmt.Sprintf("map/map_%d_%d.bin", i/100, i%100), chunk+strconv.Itoa(i)) + } + restoreSyncOnTrack := suppressSyncOnTrack(a, b) + gameID := a.TrackGame("Bench") + b.API(http.MethodPost, "/api/games", map[string]string{"name": "Bench", "savePath": b.SaveDir}, nil) + restoreSyncOnTrack() + + start := time.Now() + b.API(http.MethodPost, "/api/games/"+gameID+"/sync", nil, nil) + count := func() int { + m, _ := filepath.Glob(filepath.Join(b.SaveDir, "map", "*.bin")) + return len(m) + } + if !testutil.WaitFor(60*time.Minute, func() bool { return count() >= n }) { + t.Fatalf("only %d of %d files arrived", count(), n) + } + el := time.Since(start) + t.Logf("%s: %d files in %s = %.2f ms/file", mode.name, n, el.Round(time.Millisecond), float64(el.Microseconds())/1000/float64(n)) + }) + } +} diff --git a/e2e/cli_features_test.go b/e2e/cli_features_test.go index 6c656fa1..15877e00 100644 --- a/e2e/cli_features_test.go +++ b/e2e/cli_features_test.go @@ -176,10 +176,10 @@ func TestCLI_SnapshotAndRollbackRestoresEverything(t *testing.T) { c.mustRun("add", "History Game", dir) c.mustRun("snapshot", "history-game", "-m", "known good") - // Two: the one tracking took automatically, and the one just asked for. - // The manual one is newest, and is the one to roll back to. + // One: the one asked for holds the same files as the one tracking took, + // and takes its place (snapshot/content.go). It is the one to roll back to. ids := c.snapshotIDs("history-game") - if len(ids) != 2 { + if len(ids) != 1 { t.Fatalf("expected the initial snapshot plus the manual one, got %v", ids) } snap := ids[0] @@ -410,6 +410,9 @@ func TestCLI_SnapshotAll(t *testing.T) { c.mustRun("add", "All One", one) c.mustRun("add", "All Two", two) before := len(c.snapshotIDs("all-one")) + len(c.snapshotIDs("all-two")) + // Played on: a snapshot of the same files would be the one already there. + c.saveDir("all-one", map[string]string{"a.sav": "1, played on"}) + c.saveDir("all-two", map[string]string{"b.sav": "2, played on"}) out := c.mustRun("snapshot", "--all", "before", "the", "reinstall") if !strings.Contains(out, "2 game") { @@ -630,7 +633,9 @@ func TestCLI_SnapshotDeleteAndPrune(t *testing.T) { dir := c.saveDir("pruning", map[string]string{"slot1.sav": "v1"}) c.mustRun("add", "Pruning", dir) - for _, v := range []string{"v1", "v2", "v3"} { + // Each a change: a snapshot of the same files as one already there would + // be that one (snapshot/content.go). + for _, v := range []string{"v2", "v3", "v4"} { c.saveDir("pruning", map[string]string{"slot1.sav": v}) c.mustRun("snapshot", "pruning", "-m", v) } @@ -671,6 +676,7 @@ func TestCLI_UntrackAllRequiresConfirmation(t *testing.T) { dir := c.saveDir("guard", map[string]string{"slot1.sav": "keep me"}) c.mustRun("add", "Guarded", dir) + c.saveDir("guard", map[string]string{"slot1.sav": "keep me, played on"}) c.mustRun("snapshot", "guarded") c.mustFail("untrack-all") @@ -694,7 +700,7 @@ func TestCLI_UntrackAllRequiresConfirmation(t *testing.T) { if out := c.mustRun("status"); strings.Contains(out, "Guarded") { t.Errorf("untrack-all --yes did not clear the library:\n%s", out) } - if got := c.readSave(dir, "slot1.sav"); got != "keep me" { + if got := c.readSave(dir, "slot1.sav"); got != "keep me, played on" { t.Errorf("untrack-all touched the save on disk: %q", got) } } diff --git a/e2e/cloud_and_peers_test.go b/e2e/cloud_and_peers_test.go index eaf6ee1f..5fdb30a5 100644 --- a/e2e/cloud_and_peers_test.go +++ b/e2e/cloud_and_peers_test.go @@ -197,6 +197,9 @@ func TestCloud_SyncLocalRepairsATruncatedRemoteCopy(t *testing.T) { a.WriteSave("slot1.sav", strings.Repeat("save data ", 500)) gameID := a.TrackGame("Repair Game") + // A change first: a snapshot of the same files as the one tracking took + // would be that one (snapshot/content.go). + a.WriteSave("slot1.sav", strings.Repeat("more save data ", 500)) a.API(http.MethodPost, "/api/games/"+gameID+"/snapshot", map[string]any{"comment": "full"}, nil) name := waitForUpload(t, cloudDir)[0] diff --git a/e2e/emptied_folder_test.go b/e2e/emptied_folder_test.go index 85ecf78d..456d1daf 100644 --- a/e2e/emptied_folder_test.go +++ b/e2e/emptied_folder_test.go @@ -4,6 +4,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "testing" "time" @@ -248,3 +249,85 @@ func TestEmptiedFolder_AnUnconfirmedEmptyFolderIsNotTakenAsDeletions(t *testing. } } } + +// A save put back is the other device's: every file it holds differently, +// even one whose copy here looks newer by its date, comes from there, and +// this device ends on the other's version rather than one of its own that +// every other device would then be asked to take. +func TestEmptiedFolder_PutBackTakesTheOtherDevicesSaveAndVersion(t *testing.T) { + a, b, gameID := emptiedAndHeld(t, "Emptied Source Of Truth Game") + + // The other device moves on meanwhile: one file changed, and written + // with a date older than the copy in the snapshot here. + path := filepath.Join(b.SaveDir, "slot2.sav") + if err := os.WriteFile(path, []byte("played on since"), 0o666); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-48 * time.Hour) + if err := os.Chtimes(path, old, old); err != nil { + t.Fatal(err) + } + syncBothWays(t, gameID, b, a) + + before := len(b.Daemon.Log.History()) + a.API(http.MethodPost, "/api/games/"+gameID+"/emptied", map[string]string{"answer": "restore"}, nil) + syncBothWays(t, gameID, a, b) + + if got := a.ReadSave("slot2.sav"); got != "played on since" { + t.Errorf("put back, slot2.sav is %q: the copy here won over the other device's", got) + } + if got := b.ReadSave("slot2.sav"); got != "played on since" { + t.Errorf("the other device's slot2.sav became %q", got) + } + va, _ := a.Daemon.Store.GetGameVersion(gameID) + vb, _ := b.Daemon.Store.GetGameVersion(gameID) + if va.Vector != vb.Vector { + t.Errorf("versions differ after the put-back: here %s, there %s", va.Vector, vb.Vector) + } + for _, e := range b.Daemon.Log.History()[before:] { + if strings.Contains(e.Message, "newer version") && strings.Contains(e.Message, "taking") { + t.Errorf("the other device took a version from the one put back: %s", e.Message) + } + } +} + +// Emptying a save and putting it back leaves no snapshot of the empty folder +// or of the save part-way back; the save whole again is kept as "Put back". +func TestEmptiedFolder_NoSnapshotOfTheEmptyOrHalfSave(t *testing.T) { + a, b, gameID := emptiedAndHeld(t, "Emptied Snapshots Game") + a.API(http.MethodPost, "/api/games/"+gameID+"/emptied", map[string]string{"answer": "restore"}, nil) + syncBothWays(t, gameID, a, b) + + var games map[string]struct { + Branches map[string]struct { + Snapshots []struct { + Comment string `json:"comment"` + SizeBytes int64 `json:"sizeBytes"` + } `json:"snapshots"` + } `json:"branches"` + } + var comments []string + if !testutil.WaitFor(15*time.Second, func() bool { + a.API(http.MethodGet, "/api/games", nil, &games) + comments = comments[:0] + putBack := false + for _, br := range games[gameID].Branches { + for _, s := range br.Snapshots { + comments = append(comments, s.Comment) + if s.Comment == "Put back" { + putBack = true + } + } + } + return putBack + }) { + t.Errorf("no snapshot of the save put back; snapshots: %q", comments) + } + for _, br := range games[gameID].Branches { + for _, s := range br.Snapshots { + if s.SizeBytes < 100 { + t.Errorf("a snapshot of the emptied folder was kept: %q, %d bytes", s.Comment, s.SizeBytes) + } + } + } +} diff --git a/e2e/history_test.go b/e2e/history_test.go index b2b7bcff..e5e98b70 100644 --- a/e2e/history_test.go +++ b/e2e/history_test.go @@ -1,11 +1,15 @@ package e2e import ( + "archive/zip" "fmt" + "io" "net/http" "testing" "time" + "github.com/opensave/opensave/internal/snapshot" + "github.com/opensave/opensave/testutil" ) @@ -14,6 +18,7 @@ type snapshotWire struct { Comment string `json:"comment"` Branch string `json:"branch"` IsSystemAuto bool `json:"isSystemAuto"` + ZipPath string `json:"zipPath"` } type gameWire struct { @@ -82,11 +87,49 @@ func TestHistory_SnapshotAndRollback(t *testing.T) { } // Rolling back is itself non-destructive: the pre-rollback state must - // still be reachable, or "undo" would be a one-way door. - snaps := snapshotsOn(a, gameID, "main") - if len(snaps) < 3 { - t.Errorf("expected a safety snapshot of the pre-rollback state, have %d snapshots", len(snaps)) + // still be reachable, or "undo" would be a one-way door. Asked of the + // snapshots themselves, not counted: a state already snapshotted is not + // archived a second time. + if !snapshotHolds(t, a, gameID, "slot1.sav", "level-9") || !snapshotHolds(t, a, gameID, "inventory.dat", "cursed sword") { + t.Error("the pre-rollback state is in no snapshot") + } +} + +// snapshotHolds reports whether any snapshot on a game's main branch holds the +// archive entry name (a path in the save; ".opensave-locations//" +// in another location) with exactly content. +func snapshotHolds(t *testing.T, td *testutil.TestDaemon, gameID, name, content string) bool { + t.Helper() + for _, s := range snapshotsOn(td, gameID, "main") { + path, done, err := snapshot.OpenArchive(s.ZipPath) + if err != nil { + continue + } + zr, err := zip.OpenReader(path) + if err != nil { + done() + continue + } + for _, f := range zr.File { + if f.Name != name { + continue + } + rc, err := f.Open() + if err != nil { + continue + } + b, _ := io.ReadAll(rc) + rc.Close() + if string(b) == content { + zr.Close() + done() + return true + } + } + zr.Close() + done() } + return false } // Branches let two playthroughs coexist. Switching between them has to carry diff --git a/e2e/keep_both_test.go b/e2e/keep_both_test.go index 897bedd8..c3b5f35e 100644 --- a/e2e/keep_both_test.go +++ b/e2e/keep_both_test.go @@ -54,9 +54,12 @@ func branchHolds(t *testing.T, st branchState, branch, rel string) string { if len(snaps) == 0 { return "" } - path, done, err := snapshot.OpenArchive(snaps[0].ZipPath) + // The API lists a branch's snapshots oldest first (gamePayload), so the + // newest — what a switch puts back — is the last. + newest := snaps[len(snaps)-1] + path, done, err := snapshot.OpenArchive(newest.ZipPath) if err != nil { - t.Fatalf("open %s: %v", snaps[0].ZipPath, err) + t.Fatalf("open %s: %v", newest.ZipPath, err) } defer done() zr, err := zip.OpenReader(path) diff --git a/e2e/multiroot_scenarios_test.go b/e2e/multiroot_scenarios_test.go index e6659650..3ab77668 100644 --- a/e2e/multiroot_scenarios_test.go +++ b/e2e/multiroot_scenarios_test.go @@ -10,6 +10,7 @@ import ( "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/p2p/syncengine" + "github.com/opensave/opensave/internal/snapshot" "github.com/opensave/opensave/testutil" ) @@ -426,9 +427,11 @@ func TestMultiRootScenario_ResolveALocationConflictKeepingTheirs(t *testing.T) { if len(a.Daemon.P2P.Sync.ActiveRootConflicts()) != 0 { t.Error("the conflict is still open after being resolved") } - // Undoable: a snapshot was taken before anything was overwritten. - if after := len(snapshotsOn(a, gameID, "main")); after <= before { - t.Errorf("snapshots %d -> %d; no safety snapshot was taken before overwriting", before, after) + // Undoable: what was overwritten is in a snapshot — taken before the + // overwrite, or already there from when it was written. + _ = before + if !snapshotHolds(t, a, gameID, snapshot.RootPrefix+"config/settings.ini", "A-version") { + t.Error("the overwritten version is in no snapshot") } if got := a.ReadSave("save.sav"); got != "primary v1" { t.Errorf("resolving a location changed the save folder: %q", got) diff --git a/e2e/presence_timing_test.go b/e2e/presence_timing_test.go new file mode 100644 index 00000000..4d9402d6 --- /dev/null +++ b/e2e/presence_timing_test.go @@ -0,0 +1,53 @@ +package e2e + +import ( + "os" + "testing" + "time" + + "github.com/opensave/opensave/testutil" +) + +// TestPresence_GoneDeviceShowsOffline measures how long a device that went +// away keeps being shown online. Opt-in measurement: +// OPENSAVE_PRESENCE_TIMING=1 go test ./e2e -run TestPresence_ -v +func TestPresence_GoneDeviceShowsOffline(t *testing.T) { + if os.Getenv("OPENSAVE_PRESENCE_TIMING") == "" { + t.Skip("set OPENSAVE_PRESENCE_TIMING to run") + } + a := testutil.NewTestDaemon(t, "Presence-A") + b := testutil.NewTestDaemon(t, "Presence-B") + a.PairWith(b) + bID := b.NodeID() + + status := func() string { + p, err := a.Daemon.Store.GetPeer(bID) + if err != nil { + return "?" + } + return p.Status + } + if !testutil.WaitFor(60*time.Second, func() bool { return status() == "online" }) { + t.Fatalf("B never showed online on A: %q", status()) + } + + gone := time.Now() + b.Server.Stop() + b.Daemon.Stop() + + last := status() + for time.Since(gone) < 10*time.Minute { + if s := status(); s != last { + t.Logf("%6.1fs after B stopped: %s -> %s", time.Since(gone).Seconds(), last, s) + last = s + } + if last == "offline" { + break + } + time.Sleep(500 * time.Millisecond) + } + t.Logf("B shown online for %.1fs after it stopped", time.Since(gone).Seconds()) + if last != "offline" { + t.Fatalf("B still %q 10 minutes after it stopped", last) + } +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 65a28d0c..6ccc81e0 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "errors" "fmt" @@ -29,6 +30,10 @@ func (s *Server) routes(r chi.Router) { r.Get("/api/games", s.handleListGames) r.Post("/api/games", s.handleTrackGame) + // The game database, for tracking a game by hand (daemon/gamedb.go). + r.Get("/api/gamedb/search", s.handleGameDBSearch) + r.Get("/api/gamedb/identify", s.handleGameDBIdentify) + r.Get("/api/gamedb/running", s.handleGameDBRunning) r.Get("/api/suggest-name", s.handleSuggestName) r.Post("/api/games/untrack-bulk", s.handleBulkUntrack) @@ -49,6 +54,7 @@ func (s *Server) routes(r chi.Router) { r.Post("/api/games/{gameId}/snapshot", s.handleCreateSnapshot) r.Post("/api/games/{gameId}/rollback", s.handleRollback) + r.Post("/api/games/{gameId}/use-everywhere", s.handleUseSaveEverywhere) r.Get("/api/games/{gameId}/save-files", s.handleGameSaveFiles) r.Get("/api/games/{gameId}/snapshot/{snapshotId}/files", s.handleSnapshotFiles) r.Post("/api/games/{gameId}/snapshot/{snapshotId}/restore-file", s.handleRestoreFile) @@ -56,6 +62,7 @@ func (s *Server) routes(r chi.Router) { r.Patch("/api/games/{gameId}/snapshot/{snapshotId}", s.handleEditSnapshot) r.Get("/api/games/{gameId}/sessions", s.handleGameSessions) r.Get("/api/snapshots/check", s.handleSnapshotChecks) + r.Post("/api/snapshots/merge-duplicates", s.handleMergeDuplicates) r.Get("/api/games/{gameId}/snapshot/{snapshotId}/compare/{otherId}", s.handleCompareSnapshots) r.Post("/api/snapshots/check", s.handleVerifySnapshots) r.Post("/api/games/{gameId}/session", s.handleMarkSession) @@ -342,6 +349,8 @@ func (s *Server) handleTrackGame(w http.ResponseWriter, r *http.Request) { Name string `json:"name"` SavePath string `json:"savePath"` AppID string `json:"appId"` + // ExePath: the game's program, when it was picked by it. + ExePath string `json:"exePath"` } if err := readJSON(r, &body); err != nil { writeError(w, http.StatusBadRequest, err.Error()) @@ -352,7 +361,7 @@ func (s *Server) handleTrackGame(w http.ResponseWriter, r *http.Request) { return } - game, err := s.Daemon.TrackGame(store.Game{Name: body.Name, SavePath: body.SavePath, AppID: body.AppID}) + game, err := s.Daemon.TrackGame(store.Game{Name: body.Name, SavePath: body.SavePath, AppID: body.AppID, ExePath: body.ExePath}) if err != nil { // Duplicates (id or path) are conflicts; anything else the daemon // rejects is bad input. @@ -377,7 +386,7 @@ func (s *Server) handleUpdateGame(w http.ResponseWriter, r *http.Request) { oldSavePath := game.SavePath oldAutoSync := game.AutoSync - oldIgnore := game.SyncIgnore + oldIgnore := s.Daemon.IgnoreText(game) oldAppID := game.AppID if err := readJSON(r, &game); err != nil { writeError(w, http.StatusBadRequest, err.Error()) @@ -418,7 +427,7 @@ func (s *Server) handleUpdateGame(w http.ResponseWriter, r *http.Request) { forgetCoverMiss(game.AppID) } - if game.SyncIgnore != oldIgnore { + if s.Daemon.IgnoreText(game) != oldIgnore { // The merge bases were computed over a save that included files the // new rules exclude, so neither device could ever match them again — // which reads as permanent divergence and prompts a conflict on every @@ -454,6 +463,9 @@ func (s *Server) handleUpdateGame(w http.ResponseWriter, r *http.Request) { writeError(w, notFoundToStatus(err), err.Error()) return } + if s.Daemon.IgnoreText(game) != oldIgnore { + s.Daemon.NoteExclusions(game) + } // Re-watch if the save location or autoSync flag changed. if game.SavePath != oldSavePath || game.AutoSync != oldAutoSync { @@ -637,6 +649,50 @@ func (s *Server) handleRollback(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, snap) } +// snapshotID is the request's snapshot id, followed when it is kept as an +// alias of the snapshot holding the same files (snapshot/content.go). For +// reading a snapshot only: deleting or editing takes the id as given, so an +// old id can never reach the snapshot that took its place. +func (s *Server) snapshotID(r *http.Request) string { + return s.resolveID(chi.URLParam(r, "snapshotId")) +} + +func (s *Server) resolveID(id string) string { + if resolved, ok := s.Daemon.Store.ResolveSnapshotID(id); ok { + return resolved + } + return id +} + +// handleMergeDuplicates keeps every game's identical snapshots once, now +// rather than at the next start (daemon.MergeSnapshotDuplicates). +func (s *Server) handleMergeDuplicates(w http.ResponseWriter, r *http.Request) { + merged, freed := s.Daemon.MergeSnapshotDuplicates(true) + s.BroadcastGamesUpdate() + writeJSON(w, http.StatusOK, map[string]any{"merged": merged, "freedBytes": freed}) +} + +// handleUseSaveEverywhere makes this device's save the one every other device +// takes (syncengine.UseSaveEverywhere), and starts a sync to hand it out. The +// sync runs in the background: a large save takes a while, and the answer the +// person needs is that it was accepted. A paused device hands it out when it +// resumes. +func (s *Server) handleUseSaveEverywhere(w http.ResponseWriter, r *http.Request) { + gameID := chi.URLParam(r, "gameId") + vec, err := s.Daemon.P2P.Sync.UseSaveEverywhere(gameID) + if err != nil { + writeError(w, http.StatusConflict, err.Error()) + return + } + s.Daemon.P2P.GoSync(func(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + _, _ = s.Daemon.P2P.SyncGame(ctx, gameID) + }) + s.BroadcastGamesUpdate() + writeJSON(w, http.StatusOK, map[string]any{"version": vec.String()}) +} + func (s *Server) handleCreateBranch(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "gameId") // copyCurrentSave defaults to true when the field is absent: a caller that @@ -973,7 +1029,7 @@ func (s *Server) handleSyncResume(w http.ResponseWriter, r *http.Request) { // handlePreviewRestore says what restoring a snapshot would change, file by // file, without changing anything. func (s *Server) handlePreviewRestore(w http.ResponseWriter, r *http.Request) { - preview, err := s.Daemon.Snapshots.PreviewRestore(chi.URLParam(r, "gameId"), chi.URLParam(r, "snapshotId")) + preview, err := s.Daemon.Snapshots.PreviewRestore(chi.URLParam(r, "gameId"), s.snapshotID(r)) if err != nil { writeError(w, notFoundToStatus(err), err.Error()) return diff --git a/internal/api/routes_backup.go b/internal/api/routes_backup.go index 2be3bcaa..0b7df6cd 100644 --- a/internal/api/routes_backup.go +++ b/internal/api/routes_backup.go @@ -25,7 +25,7 @@ import ( // granular-restore browser in the UI). func (s *Server) handleSnapshotFiles(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "gameId") - snapshotID := chi.URLParam(r, "snapshotId") + snapshotID := s.snapshotID(r) snap, err := s.Daemon.Store.GetSnapshot(snapshotID) if err != nil || snap.GameID != gameID { @@ -59,7 +59,7 @@ func (s *Server) handleSnapshotFiles(w http.ResponseWriter, r *http.Request) { // save location, taking a safety snapshot first. func (s *Server) handleRestoreFile(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "gameId") - snapshotID := chi.URLParam(r, "snapshotId") + snapshotID := s.snapshotID(r) var body struct { RelPath string `json:"relPath"` @@ -723,7 +723,7 @@ func (s *Server) importBackupV2(zr *zip.Reader, manifest *backupManifest, mode s if err == nil && mode == "overwrite" { // A backup file may come from another device: this one's // excluded files stay its own (Manager.RestoreKeeping). - _, err = s.Daemon.Snapshots.RestoreKeeping(gameID, snapID, ignore.Parse(local.SyncIgnore)) + _, err = s.Daemon.Snapshots.RestoreKeeping(gameID, snapID, ignore.Parse(s.Daemon.IgnoreText(local))) res.Path = local.SavePath res.Action = "restored" } else if err == nil { diff --git a/internal/api/routes_cloud.go b/internal/api/routes_cloud.go index d8253cee..05e868bd 100644 --- a/internal/api/routes_cloud.go +++ b/internal/api/routes_cloud.go @@ -346,7 +346,7 @@ func (s *Server) handleCloudRestore(w http.ResponseWriter, r *http.Request) { // A cloud copy may be another device's: this device's excluded files // stay its own (Manager.RestoreKeeping). - if _, err := s.Daemon.Snapshots.RestoreKeeping(gameID, snapID, ignore.Parse(game.SyncIgnore)); err != nil { + if _, err := s.Daemon.Snapshots.RestoreKeeping(gameID, snapID, ignore.Parse(s.Daemon.IgnoreText(game))); err != nil { writeError(w, http.StatusInternalServerError, fmt.Sprintf("downloaded but restore failed: %v", err)) return } diff --git a/internal/api/routes_gamedb.go b/internal/api/routes_gamedb.go new file mode 100644 index 00000000..e75b1995 --- /dev/null +++ b/internal/api/routes_gamedb.go @@ -0,0 +1,36 @@ +package api + +import ( + "net/http" + + "github.com/opensave/opensave/internal/daemon" + "github.com/opensave/opensave/internal/presets" +) + +// GET /api/gamedb/search?q= — games in the game database by name. +func (s *Server) handleGameDBSearch(w http.ResponseWriter, r *http.Request) { + out := s.Daemon.SearchGames(r.URL.Query().Get("q")) + if out == nil { + out = []presets.GameMatch{} + } + writeJSON(w, http.StatusOK, map[string]any{"games": out}) +} + +// GET /api/gamedb/identify?path= — the game a program or folder belongs to. +func (s *Server) handleGameDBIdentify(w http.ResponseWriter, r *http.Request) { + m, ok := s.Daemon.IdentifyGame(r.URL.Query().Get("path")) + if !ok { + writeJSON(w, http.StatusOK, map[string]any{"found": false}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"found": true, "game": m}) +} + +// GET /api/gamedb/running — games running on this device now. +func (s *Server) handleGameDBRunning(w http.ResponseWriter, r *http.Request) { + out := s.Daemon.RunningGames() + if out == nil { + out = []daemon.RunningGame{} + } + writeJSON(w, http.StatusOK, map[string]any{"games": out}) +} diff --git a/internal/api/routes_peers.go b/internal/api/routes_peers.go index 4eb9b594..256cd05b 100644 --- a/internal/api/routes_peers.go +++ b/internal/api/routes_peers.go @@ -243,13 +243,15 @@ func (s *Server) handleProbePeer(w http.ResponseWriter, r *http.Request) { } // syncReason names why a sync did not happen, for a client to act on rather -// than parse the message: "paused" (this device paused syncing), "held" (the +// than parse the message: "paused" (this device paused syncing), "playing" (the game is being played here; it syncs when the session ends), "held" (the // game's save was emptied here and waits for an answer), "offline" (no other // device answered), or "error" for anything else. func syncReason(err error) string { switch { case errors.Is(err, syncengine.ErrPaused): return "paused" + case errors.Is(err, syncengine.ErrPlayingHere): + return "playing" case errors.Is(err, syncengine.ErrHeld): return "held" case errors.Is(err, p2p.ErrNoPeersOnline): diff --git a/internal/api/routes_savefiles.go b/internal/api/routes_savefiles.go index 663b63fc..7f1ae236 100644 --- a/internal/api/routes_savefiles.go +++ b/internal/api/routes_savefiles.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/go-chi/chi/v5" + "github.com/opensave/opensave/internal/daemon" "github.com/opensave/opensave/internal/ignore" ) @@ -37,6 +38,11 @@ type saveFile struct { MtimeMs int64 `json:"mtimeMs"` // Excluded reports whether the current rules stop this file syncing. Excluded bool `json:"excluded"` + // Why: "rule" (the game's own exclusion list), "database" (the game + // database names it as the game's settings) or "detected" (found to be + // its settings by what it holds). Empty when it syncs. + Source string `json:"source,omitempty"` + Reason string `json:"reason,omitempty"` } // saveFileListCap bounds the response. A save folder with more files than @@ -60,11 +66,31 @@ func (s *Server) handleGameSaveFiles(w http.ResponseWriter, r *http.Request) { return } - text := game.SyncIgnore if raw, ok := r.URL.Query()["rules"]; ok { - text = raw[0] // present-but-empty means "no rules", not "use the saved ones" + game.SyncIgnore = raw[0] // present-but-empty means "no rules", not "use the saved ones" } + // With the game's device settings, which are left out the same way. + text := s.Daemon.IgnoreText(game) rules := ignore.Parse(text) + ownRules := ignore.Parse(game.SyncIgnore) + var settings []daemon.DeviceSetting + if !game.SyncDeviceSettings { + settings = s.Daemon.DeviceSettings(game) + } + // which says what excludes a file the rules exclude: the game's own + // rules first, as they apply last; then the last device-settings entry + // that names it. + which := func(path string) (string, string) { + if !ownRules.Empty() && ownRules.Match(path) { + return "rule", "" + } + for i := len(settings) - 1; i >= 0; i-- { + if settings[i].Source != "save" && ignore.Parse(settings[i].Pattern).Match(path) { + return settings[i].Source, settings[i].Reason + } + } + return "rule", "" + } roots := []struct{ name, path string }{{"", game.SavePath}} if extra, rootsErr := s.Daemon.Store.ListGameRoots(gameID); rootsErr == nil { @@ -94,6 +120,11 @@ func (s *Server) handleGameSaveFiles(w http.ResponseWriter, r *http.Request) { for _, f := range files { f.Location = root.name f.Excluded = rules.Match(f.Path) + if f.Excluded && root.name == "" { + f.Source, f.Reason = which(f.Path) + } else if f.Excluded { + f.Source = "rule" + } out = append(out, f) } } diff --git a/internal/api/routes_verify.go b/internal/api/routes_verify.go index af9641ea..e3732368 100644 --- a/internal/api/routes_verify.go +++ b/internal/api/routes_verify.go @@ -31,7 +31,7 @@ func (s *Server) handleVerifySnapshots(w http.ResponseWriter, r *http.Request) { // handleCompareSnapshots says what changed from one of a game's snapshots to // another (snapshot.Manager.Compare). func (s *Server) handleCompareSnapshots(w http.ResponseWriter, r *http.Request) { - c, err := s.Daemon.Snapshots.Compare(chi.URLParam(r, "gameId"), chi.URLParam(r, "snapshotId"), chi.URLParam(r, "otherId")) + c, err := s.Daemon.Snapshots.Compare(chi.URLParam(r, "gameId"), s.snapshotID(r), s.resolveID(chi.URLParam(r, "otherId"))) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/api/server.go b/internal/api/server.go index 66a32562..f9bb82bb 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -98,12 +98,20 @@ func (s *Server) peersPayload() map[string]any { AppVersion string `json:"appVersion,omitempty"` BuildTimeMs int64 `json:"buildTimeMs,omitempty"` HasNewerBuild bool `json:"hasNewerBuild,omitempty"` + // NeedsUpdate: its OpenSave keeps no save versions, so this + // device takes nothing from it until it is updated. + NeedsUpdate bool `json:"needsUpdate,omitempty"` + // Link is how fast this device has found its connection to it, + // and what kind of address it is reached on: what decides the + // order devices are synced in. + Link syncengine.LinkStat `json:"link"` // What actually protects traffic with this device. Sent per peer // rather than described once in the interface, because the answer // differs per pairing and the reader cannot work out which case // they are in from a general statement. p2p.PeerProtection - }{Peer: p, PeerProtection: s.Daemon.P2P.PeerProtection(p)} + }{Peer: p, PeerProtection: s.Daemon.P2P.PeerProtection(p), NeedsUpdate: s.Daemon.P2P.Sync.PeerNeedsUpdate(p.ID), + Link: s.Daemon.P2P.Sync.Link(syncengine.Peer{ID: p.ID, Name: p.Name, Address: p.Address, Port: p.Port, IsWan: p.Address == "relay"})} if b, ok := builds[p.ID]; ok { entry.AppVersion = b.AppVersion entry.BuildTimeMs = b.BuildTimeMs @@ -559,18 +567,31 @@ func (s *Server) gamePayload(g store.Game) map[string]any { // Whether the art cached for this game is explicit, so the client can // blur it until someone asks to see it. An cannot read a // response header, so it travels with the game rather than the image. - "coverExplicit": s.CoverIsExplicit(coverKeyFor(g)), - "syncIgnore": g.SyncIgnore, - "branches": branches, - "createdAt": g.CreatedAt, - "lastSyncedWith": lastSyncedWith, + "coverExplicit": s.CoverIsExplicit(coverKeyFor(g)), + "syncIgnore": g.SyncIgnore, + // The files the game database names as this game's settings, left + // out of syncing unless syncDeviceSettings (daemon/devicesettings.go). + "deviceSettings": s.Daemon.DeviceSettings(g), + "syncDeviceSettings": g.SyncDeviceSettings, + "branches": branches, + "createdAt": g.CreatedAt, + "lastSyncedWith": lastSyncedWith, // The save folder is not there — gone, moved, or on a drive not // plugged in. Nothing is watched or synced for it until it is back. "savePathMissing": daemon.SaveFolderMissing(g.SavePath), - "lastPlayedAt": lastPlayed, - "playingSince": playingSince, - "playtimeMs": play.PlaytimeMs, - "playSessions": play.Sessions, + // …and of those, the ones whose whole drive is absent here: the game + // lives on another device's D:\ or E:\, which is not a problem to + // warn about (daemon.SaveDriveMissing). + "saveDriveMissing": daemon.SaveFolderMissing(g.SavePath) && daemon.SaveDriveMissing(g.SavePath), + "lastPlayedAt": lastPlayed, + "playingSince": playingSince, + // Changes to it are held back as play though no session of it is seen + // (daemon/sessions.go): said, so nothing seems stuck. + "holdingChanges": s.Daemon.HoldingChanges(g.ID), + // A restore of it under way, and how far it has come. + "restoring": s.Daemon.RestoringNow(g.ID), + "playtimeMs": play.PlaytimeMs, + "playSessions": play.Sessions, // Whether the game is installed on this device: "found", "not-found", // or "" when there is no telling (daemon.InstallState). A game that is // not here is one nobody plays here, which is why it shows no play. diff --git a/internal/api/server_test.go b/internal/api/server_test.go index f7a9ebd5..dfefa6a0 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -578,7 +578,11 @@ func TestDeleteSnapshotAndBranch(t *testing.T) { ts.do(t, http.MethodPost, "/api/games", map[string]string{"name": "Del Game", "savePath": ts.saveDir}) waitInitialSnapshot(t, ts, "del-game") - // A manual second snapshot so we have one to delete. + // A manual second snapshot so we have one to delete — of a changed save: + // one of the same files would be the first snapshot, kept once. + if err := os.WriteFile(filepath.Join(ts.saveDir, "s.sav"), []byte("y"), 0o666); err != nil { + t.Fatal(err) + } ts.do(t, http.MethodPost, "/api/games/del-game/snapshot", map[string]string{"comment": "manual"}) snaps, _ := ts.daemon.Store.ListSnapshots("del-game", "main") if len(snaps) < 2 { diff --git a/internal/cliapp/syncsummary.go b/internal/cliapp/syncsummary.go index b5518341..ff13d852 100644 --- a/internal/cliapp/syncsummary.go +++ b/internal/cliapp/syncsummary.go @@ -101,6 +101,12 @@ func outcomeFromPeers(gameID string, peers map[string]peerSyncResult) syncOutcom waiting = who + " is waiting for a folder to be chosen for it" case "peer_holding": waiting = who + " is holding it back: its save was emptied there" + case "peer_outdated_app": + waiting = who + " runs an older OpenSave without save versions — update it there" + case "waiting": + if waiting == "" { + waiting = "a newer version is on a device that is not online" + } case "peer_missing": if waiting == "" { waiting = who + " does not track it" diff --git a/internal/daemon/cloudsync.go b/internal/daemon/cloudsync.go index 6b4dbbf5..70a55b01 100644 --- a/internal/daemon/cloudsync.go +++ b/internal/daemon/cloudsync.go @@ -401,7 +401,7 @@ func (d *Daemon) currentContentHash(game store.Game) (string, error) { if err != nil { return "", err } - return watcher.ContentHash(m, game.SyncIgnore), nil + return watcher.ContentHash(m, d.IgnoreText(game)), nil } // CheckCloud reads the other devices' heads and acts on the ones ahead of @@ -633,7 +633,7 @@ func (d *Daemon) pullFromCloud(game store.Game, fileName, deviceName string) err } // Another device's save: the files this device keeps for itself stay // its own. See Manager.RestoreKeeping. - if _, err := d.Snapshots.RestoreKeeping(game.ID, snapID, ignore.Parse(game.SyncIgnore)); err != nil { + if _, err := d.Snapshots.RestoreKeeping(game.ID, snapID, ignore.Parse(d.IgnoreText(game))); err != nil { return err } // What is on disk now is that snapshot, so record it as the baseline. diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 306ce6ff..e3f41d4a 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -38,6 +38,11 @@ type Options struct { // DisableDiscovery skips UDP LAN discovery (tests pair peers manually // and don't want broadcast traffic). DisableDiscovery bool + // SyncEveryChange keeps and syncs every change made here as it comes, + // never holding a stretch of changes back as play when no session is seen + // (sessions.go, noteActivity). For tests of syncing itself, which change a + // save several times a minute and expect each change to travel. + SyncEveryChange bool } // Daemon is the assembled core. Access subsystems directly for operations @@ -62,10 +67,20 @@ type Daemon struct { // Games whose save folder is not there; see missing.go. missingMu sync.Mutex missing map[string]bool + // relocateChecked is when each missing save folder was last looked for on + // another drive (missing.go). + relocateChecked map[string]time.Time // Play sessions; see sessions.go. sessions sessionState + // Restores under way; see restoreprogress.go. + restoring restoreState + + // Device settings; see devicesettings.go. + devMu sync.Mutex + dev devSettingsCache + // OnCloudOffers receives the saves from other devices' cloud backups // that are waiting for an answer, whenever that list changes, and // OnCloudPulled each one put in place without asking. See cloudsync.go. @@ -80,6 +95,9 @@ type Daemon struct { newGames newGameState // scanMu runs one save scan at a time. See ScanForSaves. scanMu sync.Mutex + // mergeMu runs one merge of identical snapshots at a time + // (MergeSnapshotDuplicates). + mergeMu sync.Mutex // uploads counts cloud mirrors still running, so Stop can wait for them // rather than letting process exit truncate one. @@ -145,6 +163,9 @@ func New(opts Options) (*Daemon, error) { snaps := snapshot.New(s) snaps.Log = log.Log + // Each new snapshot takes the place of older automatic ones it holds + // whole (snapshot/content.go). + snaps.PruneOnCreate = true d := &Daemon{ Paths: paths, @@ -159,6 +180,18 @@ func New(opts Options) (*Daemon, error) { d.initSessions() + // How far a restore has come, for the window (restoreprogress.go). + snaps.OnRestoreProgress = d.noteRestoreProgress + + // A game's device settings are left out of syncing (devicesettings.go). + d.P2P.Sync.DeviceSettingsFor = d.deviceSettingsFor + d.P2P.Sync.ObserveSettings = d.observeSettings + // A game being played here is synced when its session ends + // (syncengine/playing.go). + d.P2P.Sync.Playing = d.playingHere + // An emptied save whole again is kept (emptied.go). + d.P2P.Sync.OnPutBack = d.snapshotPutBack + // A restore or a branch switch rewrites a save folder; nothing syncing it // may read it half-way (syncengine/settle.go). snaps.WriteGate = d.P2P.Sync.Writing @@ -215,7 +248,7 @@ func New(opts Options) (*Daemon, error) { if err != nil { return "" } - return game.SyncIgnore + return d.IgnoreText(game) }, GetLastManifestHash: func(gameID string) (string, error) { game, err := s.GetGame(gameID) @@ -226,10 +259,30 @@ func New(opts Options) (*Daemon, error) { }, SetLastManifestHash: s.SetLastManifestHash, CreateSnapshot: func(gameID string) error { - _, err := snaps.Create(gameID, "", true) + // While a game is played, the watcher only snapshots at a + // checkpoint (sessions.go), named so, and gone once the + // session's end is kept. + comment := "" + if d.playingHere(gameID) { + comment = checkpointComment + } + _, err := snaps.Create(gameID, comment, true) return err }, + SyncWriting: func(gameID string) bool { + return d.P2P != nil && d.P2P.Sync != nil && d.P2P.Sync.BeingWritten(gameID) + }, + SyncWritingNow: func(gameID string) bool { + return d.P2P != nil && d.P2P.Sync != nil && d.P2P.Sync.WritingNow(gameID) + }, + HoldSnapshot: d.holdSnapshotWhilePlaying, OnChanged: func(gameID string) { + // A change made here, by the game or the user: a new version of + // the save, recorded before anyone is offered it. + d.P2P.Sync.NoteLocalChange(gameID) + // How the files left out as its settings changed with it + // (detectsettings.go). + d.observeLocalSettings(gameID) // An emptied save is noticed as it happens, and said on screen, // even with no other device online to hold it back from. _, _ = d.P2P.Sync.CheckHold(gameID, false) @@ -266,6 +319,10 @@ func (d *Daemon) Start() error { if err != nil { return err } + // Snapshots that hold the same files are kept once (snapshot/content.go). + // In the background, once things have settled: naming a snapshot taken + // before contents were recorded reads its whole archive. + time.AfterFunc(2*time.Minute, func() { d.MergeSnapshotDuplicates(false) }) // Switch games tracked under a made-up name get their real one, when an // emulator here knows it by now. d.nameSwitchGames() @@ -276,6 +333,9 @@ func (d *Daemon) Start() error { // then evicts entries it is about to want and starts re-reading saves. delta.SetHashCacheBudgetForGames(len(games)) + // Before anything is watched or compared. + d.adoptExclusionView(games) + for _, game := range games { // Backfill cover art for games tracked before covers existed (or // migrated from the JS app without one). @@ -366,6 +426,44 @@ func (d *Daemon) Start() error { } }) + // The database file gives back the space of what was deleted from it + // (store/compact.go), hourly. + d.P2P.GoSync(func(ctx context.Context) { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + d.Store.Compact() + } + } + }) + + // Find each game's settings by what they hold (detectsettings.go): a few + // minutes after start, then hourly for whatever is due. + d.P2P.GoSync(func(ctx context.Context) { + first := time.NewTimer(newGameFirstScan) + defer first.Stop() + select { + case <-ctx.Done(): + return + case <-first.C: + d.detectSettingsDue(ctx) + } + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + d.detectSettingsDue(ctx) + } + } + }) + // Look for newly installed games: a few minutes after start, then every // hour. See newgames.go. d.P2P.GoSync(func(ctx context.Context) { @@ -685,6 +783,12 @@ func (d *Daemon) TrackGame(game store.Game) (store.Game, error) { return store.Game{}, err } game.SavePath = abs + // Nor a folder inside another game's, or holding one: the same files + // would be watched, snapshotted and synced twice, under two games that + // each think them theirs. + if other, ok := d.overlappingGame(abs); ok { + return store.Game{}, fmt.Errorf("%q already tracks %s, which this folder is inside or holds — track one or the other", other.Name, other.SavePath) + } // Fresh track from the UI (no id supplied): derive the id from the name, // but disambiguate collisions so a second save location for a same-named @@ -985,6 +1089,40 @@ func (d *Daemon) checkSavePathShape(abs string) error { return nil } +// MergeSnapshotDuplicates keeps every game's identical snapshots once +// (snapshot.Manager.MergeDuplicates), one game at a time. Returns how many +// were merged and the bytes freed. all looks at every game; otherwise only at +// games with snapshots whose content is not named yet. +func (d *Daemon) MergeSnapshotDuplicates(all bool) (merged int, freed int64) { + d.mergeMu.Lock() + defer d.mergeMu.Unlock() + games, err := d.Store.ListGames() + if err != nil { + return 0, 0 + } + for _, g := range games { + // Once per game: snapshots taken since are named as they are taken, + // and checked against the older ones then (PruneOnCreate). A game + // whose snapshots all have their content named has had its pass. + if unnamed, err := d.Store.SnapshotsWithoutContentHash(g.ID); !all && (err != nil || len(unnamed) == 0) { + continue + } + if n, f, err := d.Snapshots.MergeDuplicates(g.ID); err == nil { + merged += n + freed += f + } + // And steps on the way to a newer snapshot that holds them whole. + if n, f, err := d.Snapshots.PruneContained(g.ID); err == nil { + merged += n + freed += f + } + } + if merged > 0 { + d.Log.Log("info", fmt.Sprintf("snapshots cleaned up: %d identical or held whole by a newer one, %d MB freed", merged, freed>>20)) + } + return merged, freed +} + // EnsureImportedSnapshot registers a snapshot restored from an .sscb // backup: creates the branch if missing and inserts the metadata row // unless that snapshot id is already known (idempotent re-imports). @@ -1246,6 +1384,14 @@ func (d *Daemon) retrackFromPeer(gameID string) { d.Log.Log("info", fmt.Sprintf("game %q re-tracked on a paired device; its old folder here (%s) is gone, so it will be placed afresh", gameID, savePath)) return } + // Nor at one that holds no save: a game untracked because it was tracked + // where it never saved (a Steam folder of Steam's own files) and tracked + // again where it does on the other device must not come back here at + // the empty folder. Placed afresh, it follows the other device's. + if holdsNoSave(savePath) { + d.Log.Log("info", fmt.Sprintf("game %q re-tracked on a paired device; its old folder here (%s) held no save, so it will be placed afresh", gameID, savePath)) + return + } if name == "" { name = gameID } diff --git a/internal/daemon/detectsettings.go b/internal/daemon/detectsettings.go new file mode 100644 index 00000000..f3cfd998 --- /dev/null +++ b/internal/daemon/detectsettings.go @@ -0,0 +1,257 @@ +package daemon + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/ignore" + "github.com/opensave/opensave/internal/store" +) + +// Finding a game's settings by what they hold, and giving back what turns out +// to be a save. +// +// Detection (presets/detectsettings.go) takes a file for settings by its +// contents. Before anything is left out, and again at every later run, the +// game's snapshots are asked how the file behaved (snapshot/history.go). And +// as long as a file is left out, how it behaves is watched: in this device's +// automatic snapshots, and in what every other device sends (observeSettings). +// A settings file is changed now and then; a save changes with the sessions +// played. One that changes along with the rest of the save in most of them is +// a save after all: it syncs again, for good, whatever named it — this +// detection or the game database — and the log says which and why. +// +// Errors cost little either way. A settings file taken for a save syncs, as +// every file did before. A save taken for settings is still in every +// snapshot, and comes back to syncing at the latest when it has been seen to +// change with the save a few times. + +// detectorVersion moves when detection changes, so every game is judged again. +const detectorVersion = "1" + +const ( + observeMinMoved = 3 // changes along with the save before a file is given back + detectEvery = 24 * time.Hour +) + +func settingsDetectMark(gameID string) string { return "settings_detect:" + gameID } + +// detectSettingsDue runs detection for every game not judged under this +// detector and folder yet, or not for a day. In the background, from Start. +func (d *Daemon) detectSettingsDue(ctx context.Context) { + games, err := d.Store.ListGames() + if err != nil { + return + } + for _, game := range games { + if ctx.Err() != nil { + return + } + mark := d.Store.Mark(settingsDetectMark(game.ID)) + want := detectorVersion + "|" + game.SavePath + if prev, at, ok := strings.Cut(mark, "@"); ok && prev == want { + if t, err := time.Parse(time.RFC3339, at); err == nil && time.Since(t) < detectEvery { + continue + } + } + game = d.linkToGameDatabase(game) + d.DetectSettings(game) + _ = d.Store.SetMark(settingsDetectMark(game.ID), want+"@"+time.Now().UTC().Format(time.RFC3339)) + } +} + +// DetectSettings judges a game's files, records what is new, and re-asks the +// snapshots about everything taken for settings so far. +func (d *Daemon) DetectSettings(game store.Game) { + if d.Scanner == nil || game.SavePath == "" || SaveFolderMissing(game.SavePath) { + return + } + d.devMu.Lock() + db := d.devDBLocked() + d.devMu.Unlock() + + known := map[string]store.SettingsFile{} + if verdicts, err := d.Store.SettingsFiles(game.ID); err == nil { + for _, v := range verdicts { + known[v.Path] = v + } + } + found := db.DetectSettings(game.Name, game.AppID, game.SavePath) + var ask []string + for _, f := range found { + if _, ok := known[f.Path]; !ok { + ask = append(ask, f.Path) + } + } + for p, v := range known { + if v.Verdict == store.VerdictSettings { + ask = append(ask, p) + } + } + saves := d.Snapshots.ChangesLikeASave(game.ID, ask) + + changed := false + var taken, given []string + for _, f := range found { + if _, ok := known[f.Path]; ok { + continue + } + v := store.SettingsFile{GameID: game.ID, Path: f.Path, Verdict: store.VerdictSettings, Reason: f.Reason} + if why, isSave := saves[f.Path]; isSave { + v.Verdict, v.Reason = store.VerdictSave, why + } else { + taken = append(taken, f.Path) + } + if d.Store.SetSettingsFile(v) == nil { + changed = true + } + } + for p, v := range known { + if why, isSave := saves[p]; isSave && v.Verdict == store.VerdictSettings { + v.Verdict, v.Reason, v.AtMs = store.VerdictSave, why, 0 + if d.Store.SetSettingsFile(v) == nil { + changed = true + given = append(given, p+" ("+why+")") + } + } + } + if !changed { + return + } + d.exclusionsChanged(game) + if len(taken) > 0 && !game.SyncDeviceSettings { + d.Log.Log("info", fmt.Sprintf("%q: %s %s this device's settings, not the save — each device keeps its own. "+ + "Untick %s under the game's \"Files that shouldn't sync\" to sync %s after all.", + game.Name, strings.Join(taken, ", "), isAre(len(taken)), itThem(len(taken)), itThem(len(taken)))) + } + if len(given) > 0 { + d.Log.Log("warn", fmt.Sprintf("%q: %s changed along with the save, so %s part of the save after all — syncing again.", + game.Name, strings.Join(given, ", "), itThey(len(given)))) + } +} + +// exclusionsChanged carries a game's recorded hashes over to its exclusion +// list as it is now (adoptExclusionView), after detection changed it. +func (d *Daemon) exclusionsChanged(game store.Game) { + d.forgetVerdicts(game.ID) + if fresh, err := d.Store.GetGame(game.ID); err == nil { + game = fresh + } + d.adoptExclusionView([]store.Game{game}) +} + +// observeSettings is told what a device holds of a game — another device's +// manifest as it arrives, or this device's own at an automatic snapshot — and +// keeps count, for every file left out as this game's settings, of how often +// it changed when the rest of the save did. peerID is "" for this device. +func (d *Daemon) observeSettings(game store.Game, peerID string, m delta.Manifest) { + if game.SyncDeviceSettings { + return + } + settings := d.deviceSettingsFor(game) + if len(settings) == 0 { + return + } + asSettings := ignore.Parse(strings.Join(settings, "\n")) + var watched []string + rest := delta.Manifest{Files: map[string]delta.FileEntry{}} + for p, f := range m.Files { + if asSettings.Match(p) { + watched = append(watched, p) + } else { + rest.Files[p] = f + } + } + if len(watched) == 0 { + return + } + user := ignore.Parse(game.SyncIgnore) + for p := range rest.Files { + if delta.NeverSynced(p) || (!user.Empty() && user.Match(p)) { + delete(rest.Files, p) + } + } + restHash := rest.ManifestHash() + + // The rest of the save, as last seen from this device. + r := d.Store.GetSettingsObservation(game.ID, peerID, "") + saveMoved := r.LastHash != "" && r.LastHash != restHash + first := r.LastHash == "" + r.LastHash = restHash + _ = d.Store.SaveSettingsObservation(r) + + var given []string + for _, p := range watched { + o := d.Store.GetSettingsObservation(game.ID, peerID, p) + h := m.Files[p].Hash + if saveMoved && o.LastHash != "" { + if o.LastHash != h { + o.Moved++ + } else { + o.Still++ + } + } + o.LastHash = h + _ = d.Store.SaveSettingsObservation(o) + if first || !saveMoved { + continue + } + moved, still := 0, 0 + if all, err := d.Store.SettingsObservations(game.ID, p); err == nil { + for _, x := range all { + moved += x.Moved + still += x.Still + } + } + if moved >= observeMinMoved && moved > still { + why := fmt.Sprintf("changed in %d of the %d times the save changed", moved, moved+still) + if d.Store.SetSettingsFile(store.SettingsFile{GameID: game.ID, Path: p, Verdict: store.VerdictSave, Reason: why}) == nil { + given = append(given, p+" ("+why+")") + } + } + } + if len(given) > 0 { + d.exclusionsChanged(game) + d.Log.Log("warn", fmt.Sprintf("%q: %s changed along with the save, so %s part of the save after all — syncing again.", + game.Name, strings.Join(given, ", "), itThey(len(given)))) + } +} + +// observeLocalSettings is observeSettings for this device's own save, at an +// automatic snapshot. +func (d *Daemon) observeLocalSettings(gameID string) { + game, err := d.Store.GetGame(gameID) + if err != nil { + return + } + m, err := delta.BuildManifest(filepath.Clean(game.SavePath)) + if err != nil { + return + } + d.observeSettings(game, "", m) +} + +func isAre(n int) string { + if n == 1 { + return "is" + } + return "are" +} + +func itThey(n int) string { + if n == 1 { + return "it is" + } + return "they are" +} + +func itThem(n int) string { + if n == 1 { + return "it" + } + return "them" +} diff --git a/internal/daemon/devicesettings.go b/internal/daemon/devicesettings.go new file mode 100644 index 00000000..6ef76c1c --- /dev/null +++ b/internal/daemon/devicesettings.go @@ -0,0 +1,189 @@ +package daemon + +import ( + "strings" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/presets" + "github.com/opensave/opensave/internal/store" + "github.com/opensave/opensave/internal/watcher" +) + +// Device settings: the files the game database names as a game's settings +// rather than its save are left out of syncing (presets/devicesettings.go), +// through the game's exclusion list (syncengine.IgnoreText). +// +// Worked out once per run and kept: the database is read when first needed, +// and a game's answer only changes when its name, App ID or folder does — +// which makes it a different key here. A database update takes effect at the +// next start, where adoptExclusionView carries every recorded hash over. + +type devSettingsCache struct { + loaded bool + db *presets.DeviceSettings + // patterns: the game database's, by game name, App ID and folder. + patterns map[string][]string + // verdicts: what was found here (detectsettings.go), by game ID. + verdicts map[string][]store.SettingsFile +} + +// devDB returns the game database for device settings, read when first +// needed. Called with devMu held. +func (d *Daemon) devDBLocked() *presets.DeviceSettings { + if !d.dev.loaded { + d.dev.db = d.Scanner.DeviceSettings() + d.dev.loaded = true + } + if d.dev.patterns == nil { + d.dev.patterns = map[string][]string{} + } + if d.dev.verdicts == nil { + d.dev.verdicts = map[string][]store.SettingsFile{} + } + return d.dev.db +} + +// DeviceSetting is one entry of a game's device settings, for the window. +type DeviceSetting struct { + Pattern string `json:"pattern"` + // Source: "database" (the game database names it), "detected" (found by + // what it holds), or "save" (shown to change with the save, so it syncs + // whatever named it). + Source string `json:"source"` + Reason string `json:"reason,omitempty"` +} + +// deviceSettingsList is every entry, in the order they apply: the game +// database's, what was detected here, then what was shown to be a save — +// as "!" lines, which bring a file back whatever named it before. +func (d *Daemon) deviceSettingsList(game store.Game) []DeviceSetting { + if d.Scanner == nil { + return nil + } + key := game.Name + "\x00" + game.AppID + "\x00" + game.SavePath + d.devMu.Lock() + defer d.devMu.Unlock() + db := d.devDBLocked() + pats, ok := d.dev.patterns[key] + if !ok { + pats = db.Patterns(game.Name, game.AppID, []string{game.SavePath}) + d.dev.patterns[key] = pats + } + verdicts, ok := d.dev.verdicts[game.ID] + if !ok { + verdicts, _ = d.Store.SettingsFiles(game.ID) + d.dev.verdicts[game.ID] = verdicts + } + var out []DeviceSetting + for _, p := range pats { + out = append(out, DeviceSetting{Pattern: p, Source: "database"}) + } + for _, v := range verdicts { + if v.Verdict == store.VerdictSettings { + out = append(out, DeviceSetting{Pattern: "/" + v.Path, Source: "detected", Reason: v.Reason}) + } + } + for _, v := range verdicts { + if v.Verdict == store.VerdictSave { + out = append(out, DeviceSetting{Pattern: "!/" + v.Path, Source: "save", Reason: v.Reason}) + } + } + return out +} + +func (d *Daemon) deviceSettingsFor(game store.Game) []string { + list := d.deviceSettingsList(game) + out := make([]string, 0, len(list)) + for _, s := range list { + out = append(out, s.Pattern) + } + return out +} + +// forgetVerdicts drops a game's cached verdicts after they changed. +func (d *Daemon) forgetVerdicts(gameID string) { + d.devMu.Lock() + delete(d.dev.verdicts, gameID) + d.devMu.Unlock() +} + +// IgnoreText is the exclusion list a game is synced under (see +// syncengine.Engine.IgnoreText). +func (d *Daemon) IgnoreText(game store.Game) string { + if d.P2P == nil || d.P2P.Sync == nil { + return game.SyncIgnore + } + return d.P2P.Sync.IgnoreText(game) +} + +// DeviceSettings returns a game's device settings, whether or not it is set +// to sync them — what the window lists. +func (d *Daemon) DeviceSettings(game store.Game) []DeviceSetting { + return d.deviceSettingsList(game) +} + +// exclusionsMark records, per game, the exclusion list its recorded hashes +// were last taken under, and the never-synced list of the build that took +// them. +func exclusionsMark(gameID string) string { return "exclusions:" + gameID } + +func exclusionsValue(text string) string { + return delta.NeverSyncedList + "\n--\n" + text +} + +// NoteExclusions records that a game's hashes are now taken under its +// current exclusion list — after the rules were changed and the hashes +// rebased by whoever changed them. +func (d *Daemon) NoteExclusions(game store.Game) { + _ = d.Store.SetMark(exclusionsMark(game.ID), exclusionsValue(d.IgnoreText(game))) +} + +// adoptExclusionView carries every game's recorded hashes over to what the +// game is excluded by now, when that changed without anyone writing a rule: +// a build with a new delta.NeverSyncedList, or the game database naming +// settings files. Each is re-taken only when the save is the one it was +// recorded for, so a real change still counts. +// +// Without it, a build that starts excluding a file sees every save holding +// one as changed: an auto-snapshot of a game nobody played, and a new version +// of its own on every device at once, which the devices then disagree over. +func (d *Daemon) adoptExclusionView(games []store.Game) { + neverSyncedDone := d.Store.Mark("never_synced") == delta.NeverSyncedList + for _, game := range games { + now := d.IgnoreText(game) + oldText, beforeNeverSynced := game.SyncIgnore, !neverSyncedDone + if prev := d.Store.Mark(exclusionsMark(game.ID)); prev != "" { + list, text, _ := strings.Cut(prev, "\n--\n") + if list == delta.NeverSyncedList && text == now { + continue + } + oldText, beforeNeverSynced = text, list != delta.NeverSyncedList + } else if !beforeNeverSynced && oldText == now { + _ = d.Store.SetMark(exclusionsMark(game.ID), exclusionsValue(now)) + continue + } + + extra, err := d.Store.GameRootPaths(game.ID) + if err != nil { + extra = nil + } + m, failures, err := delta.BuildMultiManifest(game.SavePath, extra) + if err != nil || len(failures) > 0 { + continue // unreadable now: tried again at the next start + } + was := watcher.ContentHash(m, oldText) + if beforeNeverSynced { + was = watcher.ContentHashBeforeNeverSynced(m, oldText) + } + if game.LastManifestHash != "" && game.LastManifestHash == was { + _ = d.Store.SetLastManifestHash(game.ID, watcher.ContentHash(m, now)) + } + if d.P2P != nil && d.P2P.Sync != nil { + primary := m + primary.Extra = nil + d.P2P.Sync.AdoptExclusionView(game.ID, primary, oldText, beforeNeverSynced) + } + _ = d.Store.SetMark(exclusionsMark(game.ID), exclusionsValue(now)) + } + _ = d.Store.SetMark("never_synced", delta.NeverSyncedList) +} diff --git a/internal/daemon/devicesettings_test.go b/internal/daemon/devicesettings_test.go new file mode 100644 index 00000000..57d87ced --- /dev/null +++ b/internal/daemon/devicesettings_test.go @@ -0,0 +1,174 @@ +package daemon + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/ignore" + "github.com/opensave/opensave/internal/store" + "github.com/opensave/opensave/internal/watcher" +) + +// When the game database starts naming a game's settings, the hash its last +// snapshot recorded is carried over — only where the save is the one it was +// recorded for — so the watcher does not take the new view for a change and +// snapshot a game nobody played. +func TestAdoptExclusionView_CarriesTheLastSnapshotHashOver(t *testing.T) { + d := newTestDaemon(t) + mk := func(id string, files map[string]string) store.Game { + dir := filepath.Join(t.TempDir(), id) + for name, data := range files { + if err := os.MkdirAll(dir, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0o666); err != nil { + t.Fatal(err) + } + } + g := store.Game{ID: id, Name: id, SavePath: dir, ActiveBranch: "main", MaxSnapshots: 5} + if err := d.Store.CreateGame(g); err != nil { + t.Fatal(err) + } + m, err := delta.BuildManifest(dir) + if err != nil { + t.Fatal(err) + } + if err := d.Store.SetLastManifestHash(id, watcher.ContentHash(m, "")); err != nil { + t.Fatal(err) + } + g, _ = d.Store.GetGame(id) + return g + } + same := mk("same", map[string]string{"slot1.sav": "a", "graphics.xml": "4k"}) + changed := mk("changed", map[string]string{"slot1.sav": "a", "graphics.xml": "4k"}) + // A real change since the last snapshot, not yet seen by the watcher. + if err := os.WriteFile(filepath.Join(changed.SavePath, "slot1.sav"), []byte("b"), 0o666); err != nil { + t.Fatal(err) + } + delta.InvalidateRoot(changed.SavePath) + _ = d.Store.SetMark("never_synced", delta.NeverSyncedList) + + d.devMu.Lock() + d.dev = devSettingsCache{loaded: true, patterns: map[string][]string{}, verdicts: map[string][]store.SettingsFile{}} + for _, g := range []store.Game{same, changed} { + d.dev.patterns[g.Name+"\x00"+g.AppID+"\x00"+g.SavePath] = []string{"/graphics.xml"} + } + d.devMu.Unlock() + + d.adoptExclusionView([]store.Game{same, changed}) + + hashNow := func(g store.Game) string { + m, _ := delta.BuildManifest(g.SavePath) + return watcher.ContentHash(m, d.IgnoreText(g)) + } + if g, _ := d.Store.GetGame("same"); g.LastManifestHash != hashNow(g) { + t.Error("an unchanged save's last-snapshot hash was not carried over: the watcher will snapshot it") + } + if g, _ := d.Store.GetGame("changed"); g.LastManifestHash == hashNow(g) { + t.Error("a changed save's hash was carried over: its change would never be snapshotted") + } + if d.Store.Mark(exclusionsMark("same")) == "" { + t.Error("nothing recorded: this would run again at every start") + } +} + +// A file taken for settings that another device's copy shows changing with +// every change of the save is given back to syncing; one that stays put while +// the save moves stays left out. +func TestObserveSettings_GivesBackWhatChangesLikeASave(t *testing.T) { + d := newTestDaemon(t) + dir := t.TempDir() + game := store.Game{ID: "g", Name: "g", SavePath: dir, ActiveBranch: "main", MaxSnapshots: 5} + if err := d.Store.CreateGame(game); err != nil { + t.Fatal(err) + } + for _, p := range []string{"progress.cfg", "video.cfg"} { + if err := d.Store.SetSettingsFile(store.SettingsFile{GameID: "g", Path: p, Verdict: store.VerdictSettings, Reason: "test"}); err != nil { + t.Fatal(err) + } + } + d.devMu.Lock() + d.dev = devSettingsCache{loaded: true, patterns: map[string][]string{}, verdicts: map[string][]store.SettingsFile{}} + d.devMu.Unlock() + excluded := func(p string) bool { return ignore.Parse(d.IgnoreText(game)).Match(p) } + if !excluded("progress.cfg") || !excluded("video.cfg") { + t.Fatal("setup: the detected files are not left out") + } + + manifest := func(i int, video string) delta.Manifest { + return delta.Manifest{Files: map[string]delta.FileEntry{ + "slot1.sav": {Hash: fmt.Sprint("save", i)}, + "progress.cfg": {Hash: fmt.Sprint("progress", i)}, + "video.cfg": {Hash: video}, + }} + } + d.observeSettings(game, "peerA", manifest(0, "1080p")) + for i := 1; i <= 2; i++ { + d.observeSettings(game, "peerA", manifest(i, "1080p")) + } + if !excluded("progress.cfg") { + t.Fatal("given back after two changes: too eager") + } + d.observeSettings(game, "peerA", manifest(3, "1080p")) + if excluded("progress.cfg") { + t.Error("a file that changed with the save three times out of three is still left out") + } + if !excluded("video.cfg") { + t.Error("a settings file that stayed put was given back") + } + v, _ := d.Store.SettingsFiles("g") + for _, f := range v { + if f.Path == "progress.cfg" && f.Verdict != store.VerdictSave { + t.Errorf("verdict %q, want save", f.Verdict) + } + } +} + +// A game re-tracked on another device is not restored here at a folder that +// never held a save — only Steam's own file — but at one that does. +func TestRetrackFromPeer_NotAtAFolderWithoutASave(t *testing.T) { + d := newTestDaemon(t) + empty := t.TempDir() + if err := os.WriteFile(filepath.Join(empty, "remotecache.vdf"), []byte("steam"), 0o666); err != nil { + t.Fatal(err) + } + _ = d.Store.AddUntrackedTombstone("pal", "Pal", empty) + d.retrackFromPeer("pal") + if _, err := d.Store.GetGame("pal"); err == nil { + t.Error("restored at a folder holding only Steam's file") + } + + saves := t.TempDir() + if err := os.WriteFile(filepath.Join(saves, "world.sav"), []byte("progress"), 0o666); err != nil { + t.Fatal(err) + } + _ = d.Store.AddUntrackedTombstone("real", "Real", saves) + d.retrackFromPeer("real") + if _, err := d.Store.GetGame("real"); err != nil { + t.Error("not restored at a folder holding its save") + } +} + +// A folder inside a tracked game's folder, or holding it, is not tracked as +// another game: the same files would sync twice. +func TestTrackGame_RefusesAFolderInsideAnother(t *testing.T) { + d := newTestDaemon(t) + outer := t.TempDir() + inner := filepath.Join(outer, "Saves") + if err := os.MkdirAll(inner, 0o777); err != nil { + t.Fatal(err) + } + if _, err := d.TrackGame(store.Game{Name: "Duckov", SavePath: outer}); err != nil { + t.Fatal(err) + } + if _, err := d.TrackGame(store.Game{Name: "Escape From Duckov", SavePath: inner}); err == nil { + t.Error("a folder inside a tracked game's was tracked as another game") + } + other := t.TempDir() + if _, err := d.TrackGame(store.Game{Name: "Unrelated", SavePath: other}); err != nil { + t.Errorf("an unrelated folder was refused: %v", err) + } +} diff --git a/internal/daemon/emptied.go b/internal/daemon/emptied.go index 3ca01b64..6fc17bc9 100644 --- a/internal/daemon/emptied.go +++ b/internal/daemon/emptied.go @@ -194,3 +194,22 @@ func (d *Daemon) AnswerEmptied(gameID, answer string) (EmptiedAnswer, error) { }) return out, nil } + +// snapshotPutBack keeps a save once every file of it is back after it was +// emptied: nothing was snapshotted while it was empty or part-way back. +func (d *Daemon) snapshotPutBack(gameID string) { + game, err := d.Store.GetGame(gameID) + if err != nil { + return + } + if _, err := d.Snapshots.Create(gameID, "Put back", true); err != nil { + d.Log.Log("warn", fmt.Sprintf("could not keep %q as it was put back: %v", game.Name, err)) + return + } + if hash, err := d.currentContentHash(game); err == nil { + _ = d.Store.SetLastManifestHash(gameID, hash) + } + if d.OnGameChanged != nil { + d.OnGameChanged(gameID) + } +} diff --git a/internal/daemon/gamedb.go b/internal/daemon/gamedb.go new file mode 100644 index 00000000..3f97d0c2 --- /dev/null +++ b/internal/daemon/gamedb.go @@ -0,0 +1,205 @@ +package daemon + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/presets" + "github.com/opensave/opensave/internal/sessions" + "github.com/opensave/opensave/internal/store" +) + +// Tracking a game by hand, with what the game database knows about it: a +// game picked from those running, found by name, or recognised from a +// program or folder chosen. See presets/gamedb.go. + +func (d *Daemon) gameDB() *presets.DeviceSettings { + if d.Scanner == nil { + return nil + } + d.devMu.Lock() + defer d.devMu.Unlock() + return d.devDBLocked() +} + +// SearchGames finds games in the game database by name. +func (d *Daemon) SearchGames(query string) []presets.GameMatch { + out := d.gameDB().Search(query, 20) + for i := range out { + d.markTracked(&out[i]) + } + return out +} + +// IdentifyGame says which game a program or folder belongs to: a program by +// where it is installed, a folder by the game whose save location it is. +func (d *Daemon) IdentifyGame(path string) (presets.GameMatch, bool) { + db := d.gameDB() + info, err := os.Stat(path) + if err != nil { + return presets.GameMatch{}, false + } + var m presets.GameMatch + var ok bool + switch { + case !info.IsDir() && strings.EqualFold(filepath.Ext(path), ".exe"): + m, ok = db.IdentifyProgram(path) + case !info.IsDir(): + m, ok = db.IdentifyFolder(filepath.Dir(path)) // a save file: its folder + default: + m, ok = db.IdentifyFolder(path) + } + if ok { + d.markTracked(&m) + } + return m, ok +} + +// RunningGame is a game found running on this device. +type RunningGame struct { + presets.GameMatch + Exe string `json:"exe"` + Tracked bool `json:"tracked"` +} + +// runningMinMemory is what a program must hold to be offered as a game being +// played: a game holds hundreds of megabytes, and what is small is a tool, a +// launcher, or something a game left behind. +const runningMinMemory = 200 << 20 + +// RunningGames lists the programs running now that the game database knows +// as games, largest first, once per game. +func (d *Daemon) RunningGames() []RunningGame { + if d.sessions.list == nil { + return nil + } + procs, err := d.sessions.list() + if err != nil { + return nil + } + sort.Slice(procs, func(i, j int) bool { return procs[i].Memory > procs[j].Memory }) + db := d.gameDB() + winDir := strings.ToLower(os.Getenv("WINDIR")) + tracked := d.trackedGameKeys() + seen := map[string]bool{} + var out []RunningGame + for _, p := range procs { + if !p.Measured || p.Memory < runningMinMemory || p.Exe == "" { + continue + } + if winDir != "" && strings.HasPrefix(strings.ToLower(p.Exe), winDir+string(filepath.Separator)) { + continue + } + m, ok := db.IdentifyProgram(p.Exe) + if !ok || seen[m.Name] { + continue + } + seen[m.Name] = true + d.markTracked(&m) + out = append(out, RunningGame{GameMatch: m, Exe: p.Exe, Tracked: m.TrackedID != "" || tracked[m.AppID] || tracked[sessions.FolderName(m.Name)]}) + } + return out +} + +// trackedGameKeys holds every tracked game's App ID and name, to say which +// running games are tracked already. +func (d *Daemon) trackedGameKeys() map[string]bool { + keys := map[string]bool{} + games, err := d.Store.ListGames() + if err != nil { + return keys + } + for _, g := range games { + if g.AppID != "" { + keys[g.AppID] = true + } + keys[sessions.FolderName(g.Name)] = true + } + return keys +} + +// holdsNoSave reports a folder with no file any save is made of: empty, or +// only what a launcher keeps there (delta.NeverSynced). +func holdsNoSave(path string) bool { + m, err := delta.BuildManifest(path) + if err != nil { + return false // cannot tell: not "nothing" + } + for p := range m.Files { + if !delta.NeverSynced(p) { + return false + } + } + return true +} + +// overlappingGame is the tracked game whose folder holds path, or is inside +// it, if there is one. +func (d *Daemon) overlappingGame(path string) (store.Game, bool) { + games, err := d.Store.ListGames() + if err != nil { + return store.Game{}, false + } + for _, g := range games { + if g.SavePath != "" && store.PathsOverlap(path, g.SavePath) { + return g, true + } + } + return store.Game{}, false +} + +// markTracked fills in which tracked game, if any, a match already is. +func (d *Daemon) markTracked(m *presets.GameMatch) { + games, err := d.Store.ListGames() + if err != nil { + return + } + for _, g := range games { + same := m.AppID != "" && g.AppID == m.AppID + for _, p := range m.SavePaths { + if g.SavePath != "" && store.PathsOverlap(p, g.SavePath) { + same = true + } + } + if same { + m.TrackedID, m.TrackedName = g.ID, g.Name + return + } + } +} + +func gameDBMark(gameID string) string { return "gamedb:" + gameID } + +// linkToGameDatabase gives a game tracked without an App ID the one the game +// database knows its folder by, strictly (IdentifyFolderStrictly): a folder +// tracked by hand, or found under a name of the scan's own making ("Pal +// (Epic/Unreal Save)"), gets cover art, launching, session detection and its +// device settings. The name is the user's and is left as it is. Once per +// game and folder. +func (d *Daemon) linkToGameDatabase(game store.Game) store.Game { + if game.AppID != "" || game.SavePath == "" { + return game + } + if d.Store.Mark(gameDBMark(game.ID)) == game.SavePath { + return game + } + _ = d.Store.SetMark(gameDBMark(game.ID), game.SavePath) + m, ok := d.gameDB().IdentifyFolderStrictly(game.SavePath) + if !ok || m.AppID == "" { + return game + } + game.AppID = m.AppID + if game.CoverURL == "" { + game.CoverURL = SteamCoverURL(m.AppID) + } + if err := d.Store.UpdateGame(game); err != nil { + return game + } + d.Log.Log("info", fmt.Sprintf("%q is %s in the game database: cover art, its settings and seeing it played come from there", game.Name, m.Name)) + d.exclusionsChanged(game) + return game +} diff --git a/internal/daemon/missing.go b/internal/daemon/missing.go index aa083447..b55c7b1f 100644 --- a/internal/daemon/missing.go +++ b/internal/daemon/missing.go @@ -6,6 +6,8 @@ import ( "io/fs" "os" "path/filepath" + "strings" + "time" "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/logging" @@ -34,10 +36,126 @@ func SaveFolderMissing(savePath string) bool { return errors.Is(err, fs.ErrNotExist) } +// SaveDriveMissing reports whether a save path lives on a drive this device +// does not have at all — a D:\ or E:\ Steam library that exists on another +// PC. That is where a game auto-tracked from a peer lands when the peer's +// library is on a drive letter this machine lacks, and it is not something +// to warn about: nothing was lost here, the game just isn't on this device. +// Only a volume that is plainly absent counts; an unreachable network share +// reports some other error and stays a warning. +func SaveDriveMissing(savePath string) bool { + vol := filepath.VolumeName(savePath) + if vol == "" { + return false + } + _, err := os.Stat(vol + string(filepath.Separator)) + return errors.Is(err, fs.ErrNotExist) +} + +// The same save folder under another drive letter. +// +// A save inside a game's install folder — a Steam library, most often — sits +// on whichever drive that library is on, and that differs between devices: D: +// on one, E: on another. A game tracked on one device and auto-tracked on +// another arrives with the first device's drive letter, and on the second the +// folder is simply "missing" though it is right there on another drive. +// +// So a missing folder is looked for at the identical path on every other drive +// letter. Exactly one match is adopted for this device, and said once. None, or +// more than one (which would be a guess), is said once too, and the search is +// not repeated more often than relocateRecheck: it touches every drive, and the +// minute-long watch reconcile asks about missing folders constantly. +const relocateRecheck = time.Hour + +// relocateCandidates lists the folders at savePath's exact path under the other +// drive letters that exist here. Windows drive letters only. +func relocateCandidates(savePath string) []string { + vol := filepath.VolumeName(savePath) + if len(vol) != 2 || vol[1] != ':' { + return nil + } + rest := savePath[2:] + own := strings.ToUpper(vol[:1]) + var found []string + for c := 'C'; c <= 'Z'; c++ { + letter := string(c) + if letter == own { + continue + } + candidate := letter + ":" + rest + if !SaveFolderMissing(candidate) { + found = append(found, candidate) + } + } + return found +} + +// relocateToOtherDrive switches a game whose save folder is missing to the same +// path on another drive, when there is exactly one. Reports whether it did. +func (d *Daemon) relocateToOtherDrive(gameID, name, savePath string) bool { + d.missingMu.Lock() + last, checked := d.relocateChecked[gameID] + if checked && time.Since(last) < relocateRecheck { + d.missingMu.Unlock() + return false + } + if d.relocateChecked == nil { + d.relocateChecked = map[string]time.Time{} + } + d.relocateChecked[gameID] = time.Now() + d.missingMu.Unlock() + + if name == "" { + name = gameID + } + found := relocateCandidates(savePath) + switch { + case len(found) == 0: + if !checked { + d.Log.Log("info", fmt.Sprintf("the save folder of %q is not on another drive here either (looked for %s on every drive letter)", + name, logging.Quote(savePath[min(2, len(savePath)):]))) + } + return false + case len(found) > 1: + if !checked { + d.Log.Log("warn", fmt.Sprintf("the save folder of %q is missing at %s, and the same path exists on %d other drives (%s) — not guessing which; set it with Edit", + name, logging.Quote(savePath), len(found), strings.Join(found, ", "))) + } + return false + } + + newPath, err := d.ValidateSavePath(found[0]) + if err != nil { + d.Log.Log("warn", fmt.Sprintf("the save folder of %q exists at %s, but it cannot be used: %v", name, logging.Quote(found[0]), err)) + return false + } + game, err := d.Store.GetGame(gameID) + if err != nil || game.SavePath != savePath { + return false // changed meanwhile; the next reconcile sees the new path + } + game.SavePath = newPath + if err := d.Store.UpdateGame(game); err != nil { + d.Log.Log("warn", fmt.Sprintf("could not switch %q to %s: %v", name, logging.Quote(newPath), err)) + return false + } + d.Log.Log("success", fmt.Sprintf("the save folder of %q was not at %s but is at %s on this device — using that", + name, logging.Quote(savePath), logging.Quote(newPath))) + d.missingMu.Lock() + delete(d.missing, gameID) + d.missingMu.Unlock() + if d.OnGameChanged != nil { + d.OnGameChanged(gameID) + } + return true +} + // noteMissing records whether a game's save folder is missing, and says so // once when that changes — not on every minute's check — so the log and the // screen follow the folder going and coming back. func (d *Daemon) noteMissing(gameID, name, savePath string, missing bool) { + if missing && d.relocateToOtherDrive(gameID, name, savePath) { + return // found under another drive letter and switched to it + } d.missingMu.Lock() was := d.missing[gameID] if missing { @@ -55,7 +173,10 @@ func (d *Daemon) noteMissing(gameID, name, savePath string, missing bool) { if name == "" { name = gameID } - if missing { + if missing && SaveDriveMissing(savePath) { + d.Log.Log("info", fmt.Sprintf("the save folder of %q is on %s, which this device doesn't have (%s) — the game is skipped here "+ + "and keeps syncing between the devices that have it", name, filepath.VolumeName(savePath), logging.Quote(savePath))) + } else if missing { d.Log.Log("warn", fmt.Sprintf("the save folder of %q is not there (%s) — it is not watched or synced until it is back; "+ "OpenSave will not create it, since an empty folder in its place would read as every file deleted", name, logging.Quote(savePath))) } else { diff --git a/internal/daemon/missing_test.go b/internal/daemon/missing_test.go new file mode 100644 index 00000000..0b5fdf6c --- /dev/null +++ b/internal/daemon/missing_test.go @@ -0,0 +1,40 @@ +package daemon + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +// A save path on a drive letter this machine does not have is "not on this +// device", not "folder gone": the UI informs rather than warns about it. +func TestSaveDriveMissing(t *testing.T) { + existing := t.TempDir() + if SaveDriveMissing(filepath.Join(existing, "saves")) { + t.Error("a folder on an existing drive was reported as on a missing drive") + } + if SaveDriveMissing("relative/path") { + t.Error("a path without a volume was reported as on a missing drive") + } + + if runtime.GOOS != "windows" { + return + } + // Find a drive letter that is not mounted. + for c := 'Z'; c >= 'D'; c-- { + root := string(c) + `:\` + if _, err := os.Stat(root); err == nil { + continue + } + p := string(c) + `:\SteamLibrary\steamapps\common\Game\save` + if !SaveDriveMissing(p) { + t.Errorf("%s: drive %c: is absent but was not reported missing", p, c) + } + if !SaveFolderMissing(p) { + t.Errorf("%s: SaveFolderMissing should still report the folder missing", p) + } + return + } + t.Skip("every drive letter D-Z is mounted") +} diff --git a/internal/daemon/relocate_test.go b/internal/daemon/relocate_test.go new file mode 100644 index 00000000..152755fc --- /dev/null +++ b/internal/daemon/relocate_test.go @@ -0,0 +1,82 @@ +package daemon + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/opensave/opensave/internal/store" +) + +// freeDriveLetter returns a drive letter not mounted here, or "". +func freeDriveLetter() string { + for c := 'Z'; c >= 'D'; c-- { + if _, err := os.Stat(string(c) + `:\`); err != nil { + return string(c) + } + } + return "" +} + +// A game tracked elsewhere with its save on a drive this device lacks, while +// the identical path exists on another drive here, is switched to it — once. +func TestRelocateToOtherDrive(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("drive letters are a Windows thing") + } + free := freeDriveLetter() + if free == "" { + t.Skip("no free drive letter") + } + d := newTestDaemon(t) + real := filepath.Join(t.TempDir(), "SteamLibrary", "steamapps", "common", "Game", "save") + if err := os.MkdirAll(real, 0o777); err != nil { + t.Fatal(err) + } + elsewhere := free + ":" + real[2:] // same path, absent drive + if err := d.Store.CreateGame(store.Game{ID: "g", Name: "Game", SavePath: elsewhere, ActiveBranch: "main", AutoSync: true}); err != nil { + t.Fatal(err) + } + + d.noteMissing("g", "Game", elsewhere, true) + + g, err := d.Store.GetGame("g") + if err != nil { + t.Fatal(err) + } + if !strings.EqualFold(g.SavePath, real) { + t.Fatalf("save path = %s, want it switched to %s", g.SavePath, real) + } + d.missingMu.Lock() + stillMissing := d.missing["g"] + d.missingMu.Unlock() + if stillMissing { + t.Error("the game is still marked missing after being found") + } +} + +// Not found: said once, and not searched again on every reconcile. +func TestRelocateNotFound_IsNotRepeated(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("drive letters are a Windows thing") + } + free := freeDriveLetter() + if free == "" { + t.Skip("no free drive letter") + } + d := newTestDaemon(t) + path := free + `:\NoSuchLibrary\steamapps\common\Nothing\save` + if err := d.Store.CreateGame(store.Game{ID: "n", Name: "Nothing", SavePath: path, ActiveBranch: "main", AutoSync: true}); err != nil { + t.Fatal(err) + } + if d.relocateToOtherDrive("n", "Nothing", path) { + t.Fatal("relocated to a folder that does not exist") + } + first := d.relocateChecked["n"] + d.relocateToOtherDrive("n", "Nothing", path) + if !d.relocateChecked["n"].Equal(first) { + t.Error("searched again straight away; it should wait relocateRecheck") + } +} diff --git a/internal/daemon/restoreprogress.go b/internal/daemon/restoreprogress.go new file mode 100644 index 00000000..3801df5d --- /dev/null +++ b/internal/daemon/restoreprogress.go @@ -0,0 +1,79 @@ +package daemon + +import ( + "sync" + "time" +) + +// How far a restore has come, for the window: a large save is put back in +// minutes, and a spinner alone left it unclear whether anything was +// happening at all. + +// RestoreProgress is a restore under way. +type RestoreProgress struct { + Phase string `json:"phase"` // snapshot.PhaseRebuilding, PhaseChecking, PhaseWriting + Done int `json:"done"` + Total int `json:"total"` + // SecondsLeft estimates how long the phase still takes; 0 until there + // is enough to go on. + SecondsLeft int `json:"secondsLeft,omitempty"` +} + +type restoreState struct { + mu sync.Mutex + games map[string]*restoreEntry +} + +type restoreEntry struct { + RestoreProgress + phaseAt time.Time // when the phase began + sentAt time.Time // when the window was last told +} + +// noteRestoreProgress records a restore's progress (snapshot.Manager's +// OnRestoreProgress), telling the window about once a second. +func (d *Daemon) noteRestoreProgress(gameID, phase string, done, total int) { + now := time.Now() + d.restoring.mu.Lock() + if d.restoring.games == nil { + d.restoring.games = map[string]*restoreEntry{} + } + tell := false + if phase == "" { + delete(d.restoring.games, gameID) + tell = true + } else { + e := d.restoring.games[gameID] + if e == nil || e.Phase != phase { + e = &restoreEntry{phaseAt: now} + d.restoring.games[gameID] = e + tell = true + } + e.Phase, e.Done, e.Total = phase, done, total + e.SecondsLeft = 0 + if elapsed := now.Sub(e.phaseAt); done > 0 && total > done && elapsed > 2*time.Second { + e.SecondsLeft = int(elapsed.Seconds() / float64(done) * float64(total-done)) + } + if now.Sub(e.sentAt) >= time.Second { + tell = true + } + if tell { + e.sentAt = now + } + } + d.restoring.mu.Unlock() + if tell && d.OnGameChanged != nil { + d.OnGameChanged(gameID) + } +} + +// RestoringNow is the progress of a restore of gameID under way, or nil. +func (d *Daemon) RestoringNow(gameID string) *RestoreProgress { + d.restoring.mu.Lock() + defer d.restoring.mu.Unlock() + if e := d.restoring.games[gameID]; e != nil { + p := e.RestoreProgress + return &p + } + return nil +} diff --git a/internal/daemon/sessions.go b/internal/daemon/sessions.go index 67775f99..07825a60 100644 --- a/internal/daemon/sessions.go +++ b/internal/daemon/sessions.go @@ -10,6 +10,7 @@ import ( "time" "unicode" + "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/sessions" "github.com/opensave/opensave/internal/store" ) @@ -42,6 +43,25 @@ type sessionState struct { list func() ([]sessions.Proc, error) mu sync.Mutex + // checkpoint is when each playing game last had a snapshot taken while + // it was played (holdSnapshotWhilePlaying). + checkpoint map[string]time.Time + // programs: what programsIn found in each install folder. + programs map[string]programList + // active: stretches of changes no session accounts for (noteActivity); + // warned: games already said to change with no program seen. + active map[string]*activity + warned map[string]bool + // lastChange: when each game's save last changed here. + lastChange map[string]time.Time + // gap: the longest time seen between two changes of each game's save in + // a stretch (activity.maxGap). + gap map[string]time.Duration + // cpu: each process's processor time at the last poll, and when it last + // used any (activeProcesses). + cpu map[int]procSeen + // endedAt: when each game's last session ended (holdSnapshotWhilePlaying). + endedAt map[string]time.Time // startHash is each playing game's save as it was when play began, to // tell afterwards whether the session changed it. startHash map[string]string @@ -89,7 +109,9 @@ func (d *Daemon) PollSessions() { if err != nil { return } + procs = d.activeProcesses(procs) running := sessions.Running(procs, targets) + d.rememberInstallFolders(procs, targets, running) d.notePrograms(procs, targets, learn, running) d.sessions.tracker.Poll(running, time.Now()) } @@ -145,9 +167,16 @@ func (d *Daemon) sessionTargets() ([]sessions.Target, map[string]bool, error) { } } dir, viaSteam := installFolder(g, byAppID, byFolder) + if dir == "" { + dir = d.knownInstallFolder(g.ID) + } + t.FolderNames = d.installFolderNames(g) if dir != "" { t.Dirs = append(t.Dirs, dir) } + for _, dir := range t.Dirs { + t.Programs = append(t.Programs, d.programsIn(dir)...) + } // Steam launches the games it has installed; the rest need a // program to start them, and none was given. if dir != "" && !viaSteam && exe == "" { @@ -197,6 +226,9 @@ func (d *Daemon) InstallState(g store.Game) string { if dir, _ := installFolder(g, byAppID, byFolder); dir != "" { return InstallFound } + if d.knownInstallFolder(g.ID) != "" { + return InstallFound + } if g.AppID != "" && steam { return InstallNotFound } @@ -362,9 +394,19 @@ func (d *Daemon) sessionStarted(gameID string, at time.Time) { } func (d *Daemon) sessionEnded(gameID string, started, ended time.Time) { + // The session's end keeps the save and syncs it; a stretch of changes + // it overlapped would otherwise hold the sync back for quietAfter more. + d.endActivity(gameID) + d.sessions.mu.Lock() + if d.sessions.endedAt == nil { + d.sessions.endedAt = map[string]time.Time{} + } + d.sessions.endedAt[gameID] = time.Now() + d.sessions.mu.Unlock() d.sessions.mu.Lock() startHash := d.sessions.startHash[gameID] delete(d.sessions.startHash, gameID) + delete(d.sessions.checkpoint, gameID) seen := d.sessions.seen[gameID] delete(d.sessions.seen, gameID) d.sessions.mu.Unlock() @@ -397,6 +439,7 @@ func (d *Daemon) sessionEnded(gameID string, started, ended time.Time) { d.Log.Log("warn", fmt.Sprintf("could not keep %q as it was left: %v", game.Name, err)) return } + d.dropCheckpoints(game, started) // Onward to the other devices, so it is there on the next one picked up. go func() { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) @@ -455,3 +498,552 @@ func (d *Daemon) endOpenSessions() { } } } + +// checkpointEvery is how often a game being played gets a snapshot of its +// save: one per autosave pushed the snapshots worth keeping out, and none +// until the session ends leaves nothing behind a crash. +const checkpointEvery = 30 * time.Minute + +// holdSnapshotWhilePlaying tells the watcher not to snapshot a change to a +// game being played — in a session, or changing with no session seen +// (noteActivity) — unless checkpointEvery has passed since play began or +// since the last checkpoint. The end of the session, or of the stretch of +// changes, keeps the save as it was left. +func (d *Daemon) holdSnapshotWhilePlaying(gameID string) bool { + // An emptied save, or one held back after being emptied (hold.go), is not + // play: its own rules ask, and fetch it back when told to. + // + // Nor is it snapshotted: an emptied folder holds nothing worth keeping, + // and one part-way through being put back — a restore stopped half-way, + // files still coming from the other devices — is a save no device ever + // had. The save is kept once it is whole again (snapshotPutBack). Asked + // about at once all the same. + if d.emptiedOrHeld(gameID) { + d.endActivity(gameID) + _, _ = d.P2P.Sync.CheckHold(gameID, false) + return true + } + // A game known to be closed: kept and synced at once. + if d.notBeingPlayed(gameID) { + return false + } + // In a session, its rules alone: a checkpoint every checkpointEvery from + // its start. Only a change no session accounts for is a stretch + // (noteActivity) — whose timer once ended inside a session, cleared the + // session's last checkpoint, and so let nearly every autosave after the + // first half hour through as one. + since := d.PlayingSince(gameID) + if since.IsZero() { + start, stretch := d.noteActivity(gameID) + if !stretch { + return false + } + since = start + } + d.sessions.mu.Lock() + defer d.sessions.mu.Unlock() + if d.sessions.checkpoint == nil { + d.sessions.checkpoint = map[string]time.Time{} + } + now := time.Now() + if !checkpointDue(since, d.sessions.checkpoint[gameID], now) { + return true + } + d.sessions.checkpoint[gameID] = now + return false +} + +// checkpointDue says whether checkpointEvery has passed since play began, or +// since the last checkpoint when there was one. +func checkpointDue(since, last, now time.Time) bool { + if last.Before(since) { + last = since + } + return now.Sub(last) >= checkpointEvery +} + +// Programs that come with a game without being it: installers, crash +// reporters, redistributables, anti-cheat services. A process known only by +// name is never matched by one of these — every game ships the same few. +var notTheGame = []string{ + "unins", "setup", "install", "crash", "report", "redist", "vcredist", "vc_redist", "dxsetup", + "directx", "prereq", "launcher", "easyanticheat", "eac", "battleye", "beservice", "update", + "patch", "config", "settings", "helper", "service", "cef", "webhelper", "dotnet", "physx", +} + +// programsIn lists the programs in a game's install folder by file name, +// lower-case, for a process known only by name (sessions.Proc.Name). Read at +// most every installDirsFresh, three folders deep. +func (d *Daemon) programsIn(dir string) []string { + d.sessions.mu.Lock() + if c, ok := d.sessions.programs[dir]; ok && time.Since(c.at) < installDirsFresh { + d.sessions.mu.Unlock() + return c.names + } + d.sessions.mu.Unlock() + + var names []string + root := filepath.Clean(dir) + depth := strings.Count(root, string(filepath.Separator)) + _ = filepath.WalkDir(root, func(p string, e os.DirEntry, err error) error { + if err != nil { + return nil + } + if e.IsDir() { + if strings.Count(filepath.Clean(p), string(filepath.Separator))-depth >= 3 { + return filepath.SkipDir + } + return nil + } + name := strings.ToLower(e.Name()) + if !strings.HasSuffix(name, ".exe") || len(names) >= 64 { + return nil + } + for _, w := range notTheGame { + if strings.Contains(name, w) { + return nil + } + } + names = append(names, name) + return nil + }) + + d.sessions.mu.Lock() + if d.sessions.programs == nil { + d.sessions.programs = map[string]programList{} + } + d.sessions.programs[dir] = programList{at: time.Now(), names: names} + d.sessions.mu.Unlock() + return names +} + +type programList struct { + at time.Time + names []string +} + +// quietAfter is how long a game's save must stay unchanged before a stretch +// of changes no session accounts for is over: longer than the gap between +// two autosaves of the games measured (Crimson Desert: up to six minutes). +const quietAfter = 10 * time.Minute + +// activity is a stretch of changes to a game's save while no session of it +// was seen: a game whose program cannot be told apart, or is not installed +// where this device looks. It is handled as a session — not synced either +// way, a checkpoint at most every checkpointEvery — and when the save has +// stayed unchanged for quietAfter, it is kept and synced as a session's end +// would be. +type activity struct { + start, last time.Time + timer *time.Timer + // changes counts the changes in it; a change within quietAfter of the + // one before, even one already sent, counts as a second. + changes int + // maxGap is the longest time between two of its changes, or the one + // remembered for the game: a game that autosaves every ten minutes must + // not have each autosave taken for the end of a stretch. + maxGap time.Duration +} + +// firstChangeWait is how long a change on its own is held before it is kept +// and synced: long enough that the next autosave of a game being played +// arrives first (Crimson Desert: 3.6 minutes between its first two), so a +// session not seen still sends one version, at its end. A game that saves +// once, as it closes, reaches the other devices this much later — when its +// session is not seen; the end of a session seen is sent at once. +const firstChangeWait = 5 * time.Minute + +// maxQuiet caps how long a stretch can be held after its last change. +const maxQuiet = 45 * time.Minute + +// quietFor is how long the save must stay unchanged to end the stretch: the +// base wait, or twice the longest gap between the game's changes when that +// is longer. +func (a *activity) quietFor() time.Duration { + wait := quietAfter + if a.changes <= 1 { + wait = firstChangeWait + } + if w := 2 * a.maxGap; w > wait { + wait = w + } + if wait > maxQuiet { + wait = maxQuiet + } + return wait +} + +// noteActivity records a change to a game's save made here (not by a sync) +// and returns when the stretch it belongs to began, and whether it is held +// as one. Every change is: a change on its own is kept and synced once the +// save has stayed unchanged for firstChangeWait, a second within quietAfter +// of the one before makes it a stretch that ends after quietAfter unchanged. +// Only SyncEveryChange sends each change as it comes. +func (d *Daemon) noteActivity(gameID string) (time.Time, bool) { + now := time.Now() + d.sessions.mu.Lock() + defer d.sessions.mu.Unlock() + if d.sessions.active == nil { + d.sessions.active = map[string]*activity{} + } + if d.sessions.lastChange == nil { + d.sessions.lastChange = map[string]time.Time{} + } + prev, seen := d.sessions.lastChange[gameID] + d.sessions.lastChange[gameID] = now + if d.opts.SyncEveryChange { + return now, false + } + a := d.sessions.active[gameID] + if a == nil { + a = &activity{start: now, changes: 1, maxGap: d.sessions.gap[gameID]} + if seen && now.Sub(prev) < maxQuiet && now.Sub(prev) < 2*(a.maxGap+quietAfter/2) { + a.start, a.changes = prev, 2 + if g := now.Sub(prev); g > a.maxGap { + a.maxGap = g + } + } + d.sessions.active[gameID] = a + a.timer = time.AfterFunc(a.quietFor(), func() { d.activityQuiet(gameID) }) + } else { + a.changes++ + if g := now.Sub(a.last); g > a.maxGap { + a.maxGap = g + } + a.timer.Reset(a.quietFor()) + } + if d.sessions.gap == nil { + d.sessions.gap = map[string]time.Duration{} + } + if a.maxGap > d.sessions.gap[gameID] { + d.sessions.gap[gameID] = a.maxGap + } + a.last = now + if a.changes == 2 && d.PlayingSince(gameID).IsZero() && !d.sessions.warned[gameID] { + if d.sessions.warned == nil { + d.sessions.warned = map[string]bool{} + } + d.sessions.warned[gameID] = true + name := gameID + if g, err := d.Store.GetGame(gameID); err == nil { + name = g.Name + } + go d.Log.Log("info", fmt.Sprintf("%q is changing its save, but its program is not seen running here — "+ + "handled as being played: kept and synced once it has not changed for %s", name, spokenLength(quietAfter))) + } + return a.start, true +} + +// changingNow reports a game in a stretch of changes (noteActivity). +func (d *Daemon) changingNow(gameID string) bool { + d.sessions.mu.Lock() + defer d.sessions.mu.Unlock() + _, ok := d.sessions.active[gameID] + return ok +} + +// endActivity forgets a stretch of changes; a session's end takes over. +func (d *Daemon) endActivity(gameID string) { + d.sessions.mu.Lock() + if a := d.sessions.active[gameID]; a != nil { + a.timer.Stop() + delete(d.sessions.active, gameID) + } + d.sessions.mu.Unlock() +} + +// activityQuiet ends a stretch of changes once the save has stayed unchanged +// for quietAfter: the save as it was left is kept and goes to the other +// devices, as at the end of a session. +func (d *Daemon) activityQuiet(gameID string) { + d.sessions.mu.Lock() + a := d.sessions.active[gameID] + if a == nil { + d.sessions.mu.Unlock() + return + } + if wait := a.quietFor() - time.Since(a.last); wait > 0 { + a.timer.Reset(wait) + d.sessions.mu.Unlock() + return + } + delete(d.sessions.active, gameID) + if !d.PlayingSince(gameID).IsZero() { + // A session is open after all: its end keeps the save, and its + // checkpoints are its own. + d.sessions.mu.Unlock() + return + } + delete(d.sessions.checkpoint, gameID) + start, last, stretch := a.start, a.last, a.changes > 1 + d.sessions.mu.Unlock() + game, err := d.Store.GetGame(gameID) + if err != nil { + return + } + if hash, err := d.currentContentHash(game); err == nil && hash != game.LastManifestHash { + comment := "Auto backup" + if last.Sub(start) >= shortestSession { + comment = fmt.Sprintf("After playing (%s)", spokenLength(last.Sub(start))) + } + if err := d.markSessionEnd(game, hash, start, comment); err != nil { + d.Log.Log("warn", fmt.Sprintf("could not keep %q as it was left: %v", game.Name, err)) + return + } + d.dropCheckpoints(game, start) + } + d.P2P.Sync.NoteLocalChange(gameID) + if stretch { + d.Log.Log("info", fmt.Sprintf("%q has not changed its save for %s; sending it to your other devices", + game.Name, spokenLength(quietAfter))) + } + if d.OnGameChanged != nil { + d.OnGameChanged(gameID) + } + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + _, _ = d.P2P.SyncGame(ctx, gameID) + }() +} + +// Games installed where no library lists them — a folder of the user's own, +// "G:\Games\Crimson Desert" — are found by the name of their install folder: +// the names the game database knows it by, and the game's own name when it +// is specific enough not to be a folder of something else. Once a session +// has been seen that way, the folder is remembered, so the game is known to +// be installed and its programs can be told by name (programsIn). + +// genericFolders are names no game's folder is recognised by alone. +var genericFolders = map[string]bool{ + "games": true, "game": true, "steam": true, "steamapps": true, "common": true, "program files": true, + "program files x86": true, "bin": true, "bin64": true, "win64": true, "binaries": true, "launcher": true, + "data": true, "x64": true, "app": true, "client": true, "epic games": true, "gog games": true, +} + +func (d *Daemon) installFolderNames(g store.Game) []string { + var names []string + add := func(n string) { + n = sessions.FolderName(n) + if n == "" || genericFolders[n] { + return + } + for _, have := range names { + if have == n { + return + } + } + names = append(names, n) + } + d.devMu.Lock() + db := d.devDBLocked() + d.devMu.Unlock() + for _, n := range db.InstallNames(g.Name, g.AppID) { + if len(sessions.FolderName(n)) >= 4 { + add(n) + } + } + // The game's own name only when it could hardly name anything else: + // two words, or one long one. "Rust" is a folder in every Rust toolchain. + if n := sessions.FolderName(g.Name); strings.Contains(n, " ") && len(n) >= 6 || len(n) >= 10 { + add(g.Name) + } + return names +} + +func installFolderMark(gameID string) string { return "installdir:" + gameID } + +// knownInstallFolder is the install folder a session of the game was seen +// running from, when it is still there. +func (d *Daemon) knownInstallFolder(gameID string) string { + dir := d.Store.Mark(installFolderMark(gameID)) + if dir == "" { + return "" + } + if info, err := os.Stat(dir); err != nil || !info.IsDir() { + return "" + } + return dir +} + +// rememberInstallFolders records, for each game found running by the name of +// its install folder, which folder that was. +func (d *Daemon) rememberInstallFolders(procs []sessions.Proc, targets []sessions.Target, running map[string]int) { + for _, t := range targets { + pid, ok := running[t.GameID] + if !ok || len(t.Dirs) > 0 { + continue + } + for _, p := range procs { + if p.PID != pid { + continue + } + if dir := sessions.FolderOf(p, t); dir != "" && d.Store.Mark(installFolderMark(t.GameID)) != dir { + _ = d.Store.SetMark(installFolderMark(t.GameID), dir) + } + } + } +} + +// playingHere says whether a game counts as being played on this device, for +// the sync engine (syncengine/playing.go): a session is open, a stretch of +// changes is (noteActivity) — or one is about to be: a change made here waits +// out the watcher's debounce. A sync landing in that gap would otherwise +// send the change on the moment before it is held. +func (d *Daemon) playingHere(gameID string) bool { + if d.PlayingSince(gameID).IsZero() && d.emptiedOrHeld(gameID) { + d.endActivity(gameID) + return false + } + if !d.PlayingSince(gameID).IsZero() || d.changingNow(gameID) { + return true + } + if d.notBeingPlayed(gameID) { + return false + } + return !d.opts.SyncEveryChange && d.Watcher != nil && d.Watcher.ChangePending(gameID) +} + +// activeProcesses leaves out the processes doing nothing: next to no memory, +// and no processor time since the last poll. A game leaves such a process +// behind when it closes — a crash handler, a stub of the same name — and it +// kept a session open for as long as it lingered, holding the save back +// from syncing after the game was long gone. +func (d *Daemon) activeProcesses(procs []sessions.Proc) []sessions.Proc { + return d.activeProcessesAt(procs, time.Now()) +} + +// procSeen is a process's processor time at the last poll, and when it was +// last seen using any. +type procSeen struct { + cpu time.Duration + busy time.Time +} + +// idleFor is how long a process may use no processor time at all before it +// is no longer the game, however much memory it holds: what a game leaves +// behind when it is closed or killed can keep its memory, and kept a session +// open with it. A game being played uses some all the time — a paused one +// still draws its menu. +const idleFor = time.Minute + +// busyCPU is the processor time between two polls that counts as using any. +const busyCPU = 20 * time.Millisecond + +func (d *Daemon) activeProcessesAt(procs []sessions.Proc, now time.Time) []sessions.Proc { + d.sessions.mu.Lock() + defer d.sessions.mu.Unlock() + prev := d.sessions.cpu + next := make(map[int]procSeen, len(procs)) + out := procs[:0:0] + for _, p := range procs { + before, seen := prev[p.PID] + if sessions.Idle(p, before.cpu, seen) { + if p.Measured { + next[p.PID] = procSeen{cpu: p.CPU, busy: before.busy} + } + continue + } + if !p.Measured { + out = append(out, p) // cannot be told: counted + continue + } + s := procSeen{cpu: p.CPU, busy: before.busy} + if !seen || p.CPU-before.cpu >= busyCPU { + s.busy = now // new, or working + } + next[p.PID] = s + if now.Sub(s.busy) > idleFor { + continue // holding memory, doing nothing + } + out = append(out, p) + } + d.sessions.cpu = next + return out +} + +// sessionEndGrace is how long after a session ends a change to its save is +// still the session's — the game writing as it closes — and sent at once. +const sessionEndGrace = 2 * time.Minute + +// recognised reports a game whose sessions this device has seen: one has +// been recorded, or the folder it ran from is known. While such a game is not +// running, a change to its save is not play, and is sent at once. +func (d *Daemon) recognised(gameID string) bool { + if d.knownInstallFolder(gameID) != "" { + return true + } + st, err := d.Store.PlayStatsFor(gameID) + return err == nil && st.Sessions > 0 +} + +// notBeingPlayed reports a change that is certainly not play: no session is +// open, and either one ended moments ago, or the game is one whose sessions +// are seen here. +func (d *Daemon) notBeingPlayed(gameID string) bool { + if !d.PlayingSince(gameID).IsZero() { + return false + } + d.sessions.mu.Lock() + ended, ok := d.sessions.endedAt[gameID] + d.sessions.mu.Unlock() + if ok && time.Since(ended) < sessionEndGrace { + return true + } + return d.recognised(gameID) +} + +// checkpointComment names the snapshots taken while a game is played. +const checkpointComment = "Checkpoint while playing" + +// dropCheckpoints removes a session's checkpoints once the save as the +// session left it is kept: they were there in case the session never got to +// end — a crash, the power going — and after it did, they are only clutter +// between one session's snapshot and the next. Pinned ones stay. +func (d *Daemon) dropCheckpoints(game store.Game, since time.Time) { + snaps, err := d.Store.ListSnapshots(game.ID, game.ActiveBranch) + if err != nil { + return + } + for _, s := range snaps { + if !s.IsSystemAuto || s.Pinned || s.Comment != checkpointComment { + continue + } + at, err := time.Parse(time.RFC3339Nano, s.Timestamp) + if err != nil || at.Before(since) { + continue + } + _, _ = d.Snapshots.DeleteSnapshot(game.ID, s.ID) + } +} + +// emptiedOrHeld reports a game whose save folder holds no save at its top — +// emptied — or that is held back after being emptied (a deletion hold, in any +// state). Neither is play: a stretch over it held the save back from being +// asked about, and a save being fetched back from the other devices from +// being fetched. +func (d *Daemon) emptiedOrHeld(gameID string) bool { + if _, ok, err := d.Store.GetDeletionHold(gameID); err == nil && ok { + return true + } + game, err := d.Store.GetGame(gameID) + if err != nil || game.SavePath == "" { + return false + } + entries, err := os.ReadDir(game.SavePath) + if err != nil { + return false // a single-file save, or not there: not this case + } + for _, e := range entries { + if !delta.NeverSynced(e.Name()) { + return false + } + } + return true +} + +// HoldingChanges reports a game whose changes are held back as play while no +// session of it is seen (noteActivity), for the window to say so. +func (d *Daemon) HoldingChanges(gameID string) bool { + return d.PlayingSince(gameID).IsZero() && d.changingNow(gameID) +} diff --git a/internal/daemon/sessions_test.go b/internal/daemon/sessions_test.go index 362c27c7..f8519286 100644 --- a/internal/daemon/sessions_test.go +++ b/internal/daemon/sessions_test.go @@ -193,3 +193,250 @@ func TestSpokenLength(t *testing.T) { } } } + +// While a game is played, a change to its save is snapshotted once per +// checkpointEvery, not at every autosave; and it is not synced either way. +func TestSessionHoldsSnapshotsAndSyncWhilePlaying(t *testing.T) { + d := newTestDaemon(t) + g, _ := sessionGame(t, d, "autosaving") + d.sessions.list = func() ([]sessions.Proc, error) { return []sessions.Proc{{PID: 7, Exe: g.ExePath}}, nil } + d.PollSessions() + if d.PlayingSince(g.ID).IsZero() { + t.Fatal("setup: not playing") + } + if !d.holdSnapshotWhilePlaying(g.ID) { + t.Error("an autosave right after the game started was snapshotted") + } + if !d.P2P.Sync.PlayingHere(g.ID) { + t.Error("the sync engine does not know the game is being played") + } + // Checkpoints: half an hour after play began, then half an hour after + // the last one. + start := time.Now() + for _, c := range []struct { + last time.Time + after time.Duration + want bool + }{ + {time.Time{}, 10 * time.Minute, false}, + {time.Time{}, checkpointEvery, true}, + {start.Add(40 * time.Minute), 50 * time.Minute, false}, + {start.Add(40 * time.Minute), 70 * time.Minute, true}, + } { + if got := checkpointDue(start, c.last, start.Add(c.after)); got != c.want { + t.Errorf("checkpoint due %v after start (last %v) = %v, want %v", c.after, c.last, got, c.want) + } + } +} + +// A game whose program is not seen: a change is held, not synced, and kept +// once the save has stayed unchanged for firstChangeWait; a second soon after +// makes it a stretch, which ends after quietAfter unchanged. +func TestActivityWithoutASessionIsHandledAsPlay(t *testing.T) { + d := newTestDaemon(t) + g, dir := sessionGame(t, d, "unseen") + if !d.holdSnapshotWhilePlaying(g.ID) { + t.Fatal("a single change was not held") + } + d.sessions.mu.Lock() + if q := d.sessions.active[g.ID].quietFor(); q != firstChangeWait { + t.Errorf("a single change waits %v, want %v", q, firstChangeWait) + } + d.sessions.mu.Unlock() + if !d.holdSnapshotWhilePlaying(g.ID) { + t.Fatal("a second change soon after was not held") + } + d.sessions.mu.Lock() + if q := d.sessions.active[g.ID].quietFor(); q != quietAfter { + t.Errorf("a stretch waits %v, want %v", q, quietAfter) + } + d.sessions.mu.Unlock() + if !d.changingNow(g.ID) || !d.P2P.Sync.PlayingHere(g.ID) { + t.Fatal("a game changing its save is not handled as being played") + } + before := len(snapComments(t, d, g.ID)) + if err := os.WriteFile(filepath.Join(dir, "slot1.sav"), []byte("left like this"), 0o666); err != nil { + t.Fatal(err) + } + + // Quiet for longer than quietAfter. + d.sessions.mu.Lock() + d.sessions.active[g.ID].last = time.Now().Add(-quietAfter - time.Second) + d.sessions.mu.Unlock() + d.activityQuiet(g.ID) + if d.changingNow(g.ID) || d.P2P.Sync.PlayingHere(g.ID) { + t.Error("still handled as being played after the save went quiet") + } + if got := snapComments(t, d, g.ID); len(got) != before+1 { + t.Errorf("snapshots %q: want one more, keeping the save as it was left", got) + } +} + +// The names a game's install folder is recognised by: never a generic folder, +// and the game's own name only when it could hardly be another folder's. +func TestInstallFolderNames(t *testing.T) { + d := newTestDaemon(t) + for name, want := range map[string]bool{"Crimson Desert": true, "Rust": false, "Games": false, "Satisfactory": true} { + got := d.installFolderNames(store.Game{ID: name, Name: name}) + if (len(got) > 0) != want { + t.Errorf("%q: folder names %v, want some = %v", name, got, want) + } + } +} + +// A process doing nothing does not keep a session open; once a game's +// sessions are seen here, a change while it is closed is sent at once. +func TestSessionEndsWhenOnlyAStubIsLeft(t *testing.T) { + d := newTestDaemon(t) + g, _ := sessionGame(t, d, "stub") + running := sessions.Proc{PID: 9, Exe: g.ExePath, Measured: true, Memory: 2 << 30, CPU: time.Second} + d.sessions.list = func() ([]sessions.Proc, error) { return []sessions.Proc{running}, nil } + d.PollSessions() + if d.PlayingSince(g.ID).IsZero() { + t.Fatal("setup: the game was not seen running") + } + // The game closed; a stub of the same program lingers, doing nothing. + stub := sessions.Proc{PID: 9, Exe: g.ExePath, Measured: true, Memory: 800 << 10, CPU: time.Second} + d.sessions.list = func() ([]sessions.Proc, error) { return []sessions.Proc{stub}, nil } + d.PollSessions() + since := d.PlayingSince(g.ID) + d.sessions.tracker.Finish(g.ID, since.Add(time.Hour)) // past the grace the tracker allows + d.PollSessions() + if !d.PlayingSince(g.ID).IsZero() { + t.Fatal("a lingering stub keeps the session open") + } + if d.holdSnapshotWhilePlaying(g.ID) { + t.Error("a change right after the session ended was held") + } + if !d.recognised(g.ID) { + t.Error("a game with a recorded session is not recognised") + } +} + +// A session's checkpoints are gone once the save it left is kept: one +// snapshot per session, not one per half hour. +func TestSessionEndDropsItsCheckpoints(t *testing.T) { + d := newTestDaemon(t) + g, dir := sessionGame(t, d, "checkpointed") + d.sessions.list = func() ([]sessions.Proc, error) { return []sessions.Proc{{PID: 3, Exe: g.ExePath}}, nil } + d.PollSessions() + since := d.PlayingSince(g.ID) + if since.IsZero() { + t.Fatal("setup: not playing") + } + for i, data := range []string{"half an hour in", "an hour in"} { + if err := os.WriteFile(filepath.Join(dir, "slot1.sav"), []byte(data), 0o666); err != nil { + t.Fatal(err) + } + if _, err := d.Snapshots.Create(g.ID, checkpointComment, true); err != nil { + t.Fatalf("checkpoint %d: %v", i, err) + } + } + if err := os.WriteFile(filepath.Join(dir, "slot1.sav"), []byte("as it was left"), 0o666); err != nil { + t.Fatal(err) + } + d.sessions.tracker.Finish(g.ID, since.Add(90*time.Minute)) + + got := snapComments(t, d, g.ID) + if len(got) != 1 || got[0] != "After playing (1 h 30 min)" { + t.Errorf("snapshots = %q, want only the session's", got) + } +} + +// In a session, a change is the session's: no stretch is started alongside +// it, whose timer once cleared the session's last checkpoint and let nearly +// every autosave after the first half hour through as a new one. +func TestSessionChangesStartNoStretch(t *testing.T) { + d := newTestDaemon(t) + g, _ := sessionGame(t, d, "in-session") + d.sessions.list = func() ([]sessions.Proc, error) { return []sessions.Proc{{PID: 4, Exe: g.ExePath}}, nil } + d.PollSessions() + if d.PlayingSince(g.ID).IsZero() { + t.Fatal("setup: not playing") + } + for i := 0; i < 3; i++ { + if !d.holdSnapshotWhilePlaying(g.ID) { + t.Fatalf("autosave %d in a fresh session was snapshotted", i) + } + } + if d.changingNow(g.ID) { + t.Error("a stretch runs alongside the session") + } +} + +// A game that autosaves every ten minutes keeps its stretch open: the wait +// is twice the longest gap seen, within bounds. +func TestStretchWaitAdaptsToAutosaves(t *testing.T) { + for _, c := range []struct { + a activity + want time.Duration + }{ + {activity{changes: 1}, firstChangeWait}, + {activity{changes: 2, maxGap: time.Minute}, quietAfter}, + {activity{changes: 3, maxGap: 10 * time.Minute}, 20 * time.Minute}, + {activity{changes: 1, maxGap: 10 * time.Minute}, 20 * time.Minute}, + {activity{changes: 4, maxGap: time.Hour}, maxQuiet}, + } { + if got := c.a.quietFor(); got != c.want { + t.Errorf("%d changes, gap %v: wait %v, want %v", c.a.changes, c.a.maxGap, got, c.want) + } + } +} + +// An emptied save is not play — no stretch holds it back from being asked +// about and fetched back — and is not snapshotted either. +func TestEmptiedSaveIsNotPlay(t *testing.T) { + d := newTestDaemon(t) + g, dir := sessionGame(t, d, "emptied") + d.holdSnapshotWhilePlaying(g.ID) + if !d.changingNow(g.ID) { + t.Fatal("setup: no stretch") + } + if err := os.Remove(filepath.Join(dir, "slot1.sav")); err != nil { + t.Fatal(err) + } + // Not snapshotted (an empty folder is no state worth keeping), but not + // as play: no stretch begins over it. + if !d.holdSnapshotWhilePlaying(g.ID) { + t.Error("the emptied save was snapshotted") + } + if d.changingNow(g.ID) || d.P2P.Sync.PlayingHere(g.ID) { + t.Error("an emptied save is still handled as being played") + } +} + +// What a killed game leaves behind can keep its memory; it is no longer the +// game once it has used no processor time for a minute. One still working — +// a paused game draws its menu — stays. +func TestActiveProcesses_HoldingMemoryDoingNothing(t *testing.T) { + d := newTestDaemon(t) + t0 := time.Now() + left := func(cpu time.Duration) sessions.Proc { + return sessions.Proc{PID: 1, Exe: `G:\Games\Crimson Desert\bin64\CrimsonDesert.exe`, Measured: true, Memory: 2 << 30, CPU: cpu} + } + playing := func(cpu time.Duration) sessions.Proc { + return sessions.Proc{PID: 2, Exe: `G:\Games\Other\game.exe`, Measured: true, Memory: 2 << 30, CPU: cpu} + } + count := func(ps []sessions.Proc) map[int]bool { + m := map[int]bool{} + for _, p := range ps { + m[p.PID] = true + } + return m + } + got := count(d.activeProcessesAt([]sessions.Proc{left(5 * time.Second), playing(time.Second)}, t0)) + if !got[1] || !got[2] { + t.Fatalf("first seen: %v, want both", got) + } + got = count(d.activeProcessesAt([]sessions.Proc{left(5 * time.Second), playing(2 * time.Second)}, t0.Add(30*time.Second))) + if !got[1] || !got[2] { + t.Errorf("half a minute in: %v, want both", got) + } + got = count(d.activeProcessesAt([]sessions.Proc{left(5 * time.Second), playing(3 * time.Second)}, t0.Add(70*time.Second))) + if got[1] { + t.Error("a process doing nothing for over a minute still counts as the game") + } + if !got[2] { + t.Error("a game using the processor was dropped") + } +} diff --git a/internal/daemon/verify.go b/internal/daemon/verify.go index ebf57cbd..47552baf 100644 --- a/internal/daemon/verify.go +++ b/internal/daemon/verify.go @@ -64,6 +64,11 @@ func (d *Daemon) VerifySnapshots(ctx context.Context, gameID string, pace time.D } was := snap.Problem if err := d.Snapshots.Verify(snap); err != nil { + if _, gerr := d.Store.GetSnapshot(snap.ID); gerr != nil { + // Removed while this ran (a newer snapshot holding + // the same files took its place): not damaged, gone. + continue + } report.Damaged = append(report.Damaged, DamagedSnapshot{ GameID: g.ID, GameName: g.Name, SnapshotID: snap.ID, Timestamp: snap.Timestamp, Problem: err.Error(), }) diff --git a/internal/delta/hashcache.go b/internal/delta/hashcache.go index 5ee3b611..61adcda8 100644 --- a/internal/delta/hashcache.go +++ b/internal/delta/hashcache.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "runtime" + "sort" "strings" "sync" "time" @@ -64,11 +65,19 @@ type cacheEntry struct { // the old behaviour. const ( // cacheMinBytes is the floor, and the value a small library gets. - cacheMinBytes = 64 << 20 + // + // A budget, not an allocation: the cache only ever holds entries for + // files that exist, so a small library never comes near it. It has to + // cover the rare library with one enormous save, though — a game that + // writes every map chunk as its own file (Project Zomboid: ~240k files, + // ~53MB by entryCost) filled a 64MB budget on its own, and the cache then + // evicted and re-read that save on every pass, which is the churn it + // exists to prevent. + cacheMinBytes = 256 << 20 // cacheMaxCeiling is the most this will ever hold, however many games are // tracked. A cache is meant to replace repeated reading, not to become // the memory problem it was added to fix. - cacheMaxCeiling = 384 << 20 + cacheMaxCeiling = 512 << 20 // cachePerGameBytes is how much the budget grows per tracked game. // // From measurement rather than taste: a game's manifest costs roughly @@ -77,7 +86,10 @@ const ( // library and is not enough for someone tracking 350 games — the cache // sits at its cap and evicts entries it is about to want again, which // quietly reinstates the repeated reading it exists to remove. - cachePerGameBytes = 512 << 10 + // + // Doubled alongside the floor so a large library still gets more than a + // small one (350 games -> 350MB). + cachePerGameBytes = 1 << 20 ) // cacheMaxBytes is the current budget. Not a constant: it scales with how many @@ -193,6 +205,12 @@ func FileEntryFor(path string) (FileEntry, error) { return cachedHashFile(path, info) } +// FileEntryForInfo is FileEntryFor with the FileInfo a directory walk already +// has, so a cache hit costs no syscall at all. +func FileEntryForInfo(path string, info os.FileInfo) (FileEntry, error) { + return cachedHashFile(path, info) +} + func cachedHashFile(path string, info os.FileInfo) (FileEntry, error) { key := cacheKeyFor(path) stamp := cacheStamp{size: info.Size(), mtimeNs: info.ModTime().UnixNano()} @@ -302,13 +320,10 @@ func evictLocked() { for k, v := range hashCache.entries { all = append(all, aged{k, v.usedAt}) } - // Partial selection would be faster, but this runs only on overflow and - // clarity is worth more here than the microseconds. - for i := 1; i < len(all); i++ { - for j := i; j > 0 && all[j].at.Before(all[j-1].at); j-- { - all[j], all[j-1] = all[j-1], all[j] - } - } + // sort.Slice, not an insertion sort: map iteration order is random, so + // insertion sort was quadratic in the entry count — ~10^10 comparisons for + // a quarter-million-file save, on every overflow. + sort.Slice(all, func(i, j int) bool { return all[i].at.Before(all[j].at) }) target := cacheMaxBytes * 3 / 4 for _, a := range all { if hashCache.bytes <= target { diff --git a/internal/delta/manifest.go b/internal/delta/manifest.go index f70016e5..c4f195bc 100644 --- a/internal/delta/manifest.go +++ b/internal/delta/manifest.go @@ -10,7 +10,9 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "os" + "path" "path/filepath" "sort" "strconv" @@ -28,6 +30,32 @@ import ( // peer as a real file (and cascade into name.opensave.tmp.opensave.tmp). const TmpSuffix = ".opensave.tmp" +// NeverSynced reports whether a file in a save folder is never part of the +// save, on any device. +// +// remotecache.vdf is Steam's record of what Steam Cloud has uploaded for a +// game, kept beside the game's files in Steam's userdata folder — which is +// where a game whose saves live in Steam Cloud is tracked. It describes this +// device's Steam, not the save: syncing it hands Steam another machine's +// bookkeeping. And that folder is under Program Files, where writing needs +// rights OpenSave does not have, so the copy failed with "access denied" on +// every pass, from every device. +// +// Left out where decisions are made — what to sync, what a version is, what +// a snapshot holds — and NOT from the manifest a device builds and serves: a +// device on an older build would read the gap as the file deleted, and +// delete its own copy (the same reasoning as exclusion rules, +// syncengine/ignore.go). +func NeverSynced(rel string) bool { + return strings.EqualFold(path.Base(rel), "remotecache.vdf") +} + +// NeverSyncedList names what NeverSynced leaves out. It goes into the tag of +// a version's hash, so changing the list re-takes the hash quietly on every +// device, like changing a game's exclusion rules — instead of every device +// calling the same save a new version of its own at once. +const NeverSyncedList = "remotecache.vdf" + // staleTmpAge is how old a leftover TmpSuffix file must be before the // manifest walk garbage-collects it. Generous enough that a patch actively // writing its temp file is never deleted out from under it. @@ -217,9 +245,17 @@ func HashFile(path string) (FileEntry, error) { } } + wholeHash := hex.EncodeToString(whole.Sum(nil)) + // A single-block file's block hash is the whole-file hash; share the one + // string. Most save files are single-block, and on a save of a quarter + // million of them the duplicate hex strings cost ~20MB per manifest. + if len(blocks) == 1 { + blocks[0].Hash = wholeHash + } + return FileEntry{ Size: info.Size(), - Hash: hex.EncodeToString(whole.Sum(nil)), + Hash: wholeHash, Blocks: blocks, BlockSize: blockSize, MtimeMs: Milli(info.ModTime().UnixMilli()), @@ -263,7 +299,20 @@ func buildManifest(root string) (Manifest, error) { } var dirs []string - err = filepath.Walk(root, func(path string, walkInfo os.FileInfo, walkErr error) error { + // WalkDir, not Walk: Walk issues an extra Lstat per entry, which on + // Windows opens every file and dominated the cost of a warm build of a + // large save (13-19s vs ~1.2s for 240k files). WalkDir takes size and + // mtime from the directory listing itself. + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + var walkInfo os.FileInfo + if walkErr == nil && path != root { + // Reparse points are rejected below from the listing's type bits, + // before Info is asked for anything. + if d.Type()&(os.ModeSymlink|os.ModeIrregular) != 0 { + return nil + } + walkInfo, walkErr = d.Info() + } if walkErr != nil { // The save folder itself could not be read — its listing refused, // or the folder gone since it was looked at. That is not an empty @@ -282,7 +331,7 @@ func buildManifest(root string) (Manifest, error) { // skipped instead of failing the whole manifest — one // unreadable directory must not abort every sync of the game. if os.IsPermission(walkErr) { - if walkInfo != nil && walkInfo.IsDir() { + if d != nil && d.IsDir() { return filepath.SkipDir } return nil @@ -341,7 +390,6 @@ func buildManifest(root string) (Manifest, error) { } return nil } - entry, err := cachedHashFile(path, walkInfo) if err != nil { // Deleted after the walk listed it and before it could be read. diff --git a/internal/delta/neversynced_test.go b/internal/delta/neversynced_test.go new file mode 100644 index 00000000..1bcf8c21 --- /dev/null +++ b/internal/delta/neversynced_test.go @@ -0,0 +1,17 @@ +package delta + +import "testing" + +func TestNeverSynced(t *testing.T) { + for p, want := range map[string]bool{ + "remotecache.vdf": true, + "remote/RemoteCache.VDF": true, + "remote/save.sav": false, + "remotecache.vdf.bak": false, + "myremotecache.vdf": false, + } { + if got := NeverSynced(p); got != want { + t.Errorf("NeverSynced(%q) = %v, want %v", p, got, want) + } + } +} diff --git a/internal/delta/patch.go b/internal/delta/patch.go index 7976fc45..98e0104b 100644 --- a/internal/delta/patch.go +++ b/internal/delta/patch.go @@ -9,6 +9,7 @@ import ( "time" "github.com/opensave/opensave/internal/fsx" + "github.com/opensave/opensave/internal/owntouch" ) // BlockSource supplies the bytes for one block index, either from the @@ -77,7 +78,11 @@ func replaceWithRetry(tmpPath, filePath string) error { delay *= 2 } clearReadOnlyIfSet(filePath) + // Marked before the rename, so the watcher's event finds it already + // known as a change OpenSave made (owntouch). + owntouch.Mark(filePath) if err = renameFile(tmpPath, filePath); err == nil { + owntouch.Settled(filePath) return nil } } diff --git a/internal/iopar/drive.go b/internal/iopar/drive.go new file mode 100644 index 00000000..14d9a0f5 --- /dev/null +++ b/internal/iopar/drive.go @@ -0,0 +1,52 @@ +// Package iopar runs many small file operations — a save of a quarter of a +// million files restored, checked, written — as many at a time as the drive +// they are on does best: one at a time on a spinning disk, whose head would +// otherwise be sent back and forth across it; many at once on an SSD, which +// works through several requests in parallel and is otherwise left waiting +// on each file's own small delays (its creation, a virus scanner's look at +// it). How many is a starting point by kind of drive, adjusted as the work +// goes by what it actually achieves (Run). +package iopar + +// Kind is the kind of drive a path is on. +type Kind int + +const ( + // Unknown: it could not be told. Treated as a SATA SSD — parallel, but + // modestly so. + Unknown Kind = iota + HDD + SATA + NVMe +) + +func (k Kind) String() string { + switch k { + case HDD: + return "hard disk" + case SATA: + return "SATA SSD" + case NVMe: + return "NVMe SSD" + } + return "unknown drive" +} + +// start is how many operations run at once to begin with. +func (k Kind) start() int { + switch k { + case HDD: + return 1 + case NVMe: + return 32 + } + return 8 +} + +// adapts reports whether the number is adjusted as the work goes. A spinning +// disk is not: it does best one at a time, and finding that out would cost +// the seeking it exists to avoid. +func (k Kind) adapts() bool { return k != HDD } + +// DriveKind says what kind of drive path is on. +func DriveKind(path string) Kind { return driveKind(path) } diff --git a/internal/iopar/drive_other.go b/internal/iopar/drive_other.go new file mode 100644 index 00000000..77cf455b --- /dev/null +++ b/internal/iopar/drive_other.go @@ -0,0 +1,56 @@ +//go:build !windows + +package iopar + +import ( + "os" + "path/filepath" + "strings" + "syscall" +) + +// On Linux the kernel says whether a block device rotates; an NVMe drive is +// named so. Elsewhere it is not told. +func driveKind(path string) Kind { + var st syscall.Stat_t + if err := syscall.Stat(path, &st); err != nil { + return Unknown + } + major, minor := (st.Dev>>8)&0xfff, (st.Dev&0xff)|((st.Dev>>12)&0xfff00) + dev, err := os.Readlink("/sys/dev/block/" + itoa(uint64(major)) + ":" + itoa(uint64(minor))) + if err != nil { + return Unknown + } + // A partition: its disk is the folder above it. + disk := filepath.Base(dev) + sys := filepath.Join("/sys/class/block", disk) + if _, err := os.Stat(filepath.Join(sys, "partition")); err == nil { + disk = filepath.Base(filepath.Dir(dev)) + sys = filepath.Join("/sys/class/block", disk) + } + if strings.HasPrefix(disk, "nvme") { + return NVMe + } + b, err := os.ReadFile(filepath.Join(sys, "queue", "rotational")) + if err != nil { + return Unknown + } + if strings.TrimSpace(string(b)) == "1" { + return HDD + } + return SATA +} + +func itoa(n uint64) string { + if n == 0 { + return "0" + } + var b [20]byte + i := len(b) + for n > 0 { + i-- + b[i] = byte('0' + n%10) + n /= 10 + } + return string(b[i:]) +} diff --git a/internal/iopar/drive_windows.go b/internal/iopar/drive_windows.go new file mode 100644 index 00000000..8bb019cf --- /dev/null +++ b/internal/iopar/drive_windows.go @@ -0,0 +1,94 @@ +//go:build windows + +package iopar + +import ( + "path/filepath" + "strings" + "unsafe" + + "golang.org/x/sys/windows" +) + +// Asked of the volume a path is on, as Windows' own optimiser asks it: does +// it incur a seek penalty (a spinning disk), and over which bus is it +// attached (NVMe or otherwise). + +const ( + ioctlStorageQueryProperty = 0x002D1400 + + storageDeviceProperty = 0 + storageDeviceSeekPenaltyProperty = 7 + propertyStandardQuery = 0 + + busTypeNvme = 17 +) + +type storagePropertyQuery struct { + PropertyID uint32 + QueryType uint32 + Additional [1]byte +} + +type deviceSeekPenaltyDescriptor struct { + Version uint32 + Size uint32 + IncursSeekPenalty byte +} + +type storageDeviceDescriptor struct { + Version uint32 + Size uint32 + DeviceType byte + DeviceTypeModifier byte + RemovableMedia byte + CommandQueueing byte + VendorIDOffset uint32 + ProductIDOffset uint32 + ProductRevisionOffset uint32 + SerialNumberOffset uint32 + BusType uint32 + RawPropertiesLength uint32 +} + +func driveKind(path string) Kind { + abs, err := filepath.Abs(path) + if err != nil { + return Unknown + } + vol := filepath.VolumeName(abs) + if len(vol) != 2 || vol[1] != ':' { + return Unknown // a network share or the like: no drive to ask + } + name, err := windows.UTF16PtrFromString(`\\.\` + strings.ToUpper(vol)) + if err != nil { + return Unknown + } + // No access rights are needed to ask these properties. + h, err := windows.CreateFile(name, 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, nil, windows.OPEN_EXISTING, 0, 0) + if err != nil { + return Unknown + } + defer windows.CloseHandle(h) + + var seek deviceSeekPenaltyDescriptor + if query(h, storageDeviceSeekPenaltyProperty, unsafe.Pointer(&seek), uint32(unsafe.Sizeof(seek))) && seek.IncursSeekPenalty != 0 { + return HDD + } + var dev storageDeviceDescriptor + if query(h, storageDeviceProperty, unsafe.Pointer(&dev), uint32(unsafe.Sizeof(dev))) { + if dev.BusType == busTypeNvme { + return NVMe + } + return SATA + } + return Unknown +} + +func query(h windows.Handle, property uint32, out unsafe.Pointer, size uint32) bool { + q := storagePropertyQuery{PropertyID: property, QueryType: propertyStandardQuery} + var returned uint32 + err := windows.DeviceIoControl(h, ioctlStorageQueryProperty, (*byte)(unsafe.Pointer(&q)), uint32(unsafe.Sizeof(q)), + (*byte)(out), size, &returned, nil) + return err == nil && returned >= 8 +} diff --git a/internal/iopar/run.go b/internal/iopar/run.go new file mode 100644 index 00000000..3cd15028 --- /dev/null +++ b/internal/iopar/run.go @@ -0,0 +1,234 @@ +package iopar + +import ( + "context" + "sync" + "sync/atomic" + "time" +) + +// The adjusting: every window the number of operations finished is +// measured, and the number running at once moves the way that made it +// faster — on while it keeps helping, back when it hurt. A drive that slows +// as it goes, its cache full or a scanner busy, gets fewer; one with more to +// give gets more. Once it settles it still looks the other way now and then, +// so a change later in a long run is followed too. + +const ( + // maxAtOnce bounds how many run at once. + maxAtOnce = 64 + // Changes smaller than this are noise, not a reason to move. + better = 1.05 + worse = 0.95 + // probeAfter windows without change before looking the other way. + probeAfter = 4 +) + +// window is how long each measurement runs; a variable, for tests. +var window = 750 * time.Millisecond + +// Stats is how a run went, for the log. +type Stats struct { + Kind Kind + Start int // operations at once to begin with + Settled int // at the end + Peak float64 + Duration time.Duration +} + +// Run calls work for each of n items, as many at a time as suits the drive +// path is on. progress, if not nil, is told how many are done now and then +// and at the end. The first error stops the run and is returned; work +// already started finishes first. +func Run(ctx context.Context, path string, n int, work func(i int) error, progress func(done, total int)) (Stats, error) { + return RunKind(ctx, DriveKind(path), n, work, progress) +} + +// RunFixed runs work with exactly at operations at once, not adjusting: for +// measuring what a drive does at each number. +func RunFixed(ctx context.Context, at, n int, work func(i int) error, progress func(done, total int)) (Stats, error) { + return run(ctx, Unknown, at, false, n, work, progress) +} + +// RunKind is Run for a drive whose kind is known. +func RunKind(ctx context.Context, kind Kind, n int, work func(i int) error, progress func(done, total int)) (Stats, error) { + return run(ctx, kind, kind.start(), kind.adapts(), n, work, progress) +} + +func run(ctx context.Context, kind Kind, start int, adapts bool, n int, work func(i int) error, progress func(done, total int)) (Stats, error) { + started := time.Now() + st := Stats{Kind: kind, Start: start} + if n <= 0 { + return st, nil + } + p := &pool{limit: min(start, n)} + p.cond = sync.NewCond(&p.mu) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var next, done atomic.Int64 + var firstErr error + var errOnce sync.Once + fail := func(err error) { + errOnce.Do(func() { + firstErr = err + cancel() + p.cond.Broadcast() // wake the ones waiting, to stop + }) + } + + workers := min(maxAtOnce, n) + if !adapts { + workers = p.limit + } + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + if !p.acquire(ctx) { + return + } + i := int(next.Add(1) - 1) + if i >= n { + p.release() + return + } + if err := work(i); err != nil { + p.release() + fail(err) + return + } + done.Add(1) + p.release() + } + }() + } + + // Adjusting, and telling progress, until the workers are done. + finished := make(chan struct{}) + go func() { + wg.Wait() + close(finished) + }() + tick := time.NewTicker(window) + defer tick.Stop() + c := climber{dir: 1} + last, lastAt := int64(0), time.Now() +loop: + for { + select { + case <-finished: + break loop + case now := <-tick.C: + d := done.Load() + rate := float64(d-last) / now.Sub(lastAt).Seconds() + last, lastAt = d, now + if rate > st.Peak { + st.Peak = rate + } + if progress != nil { + progress(int(d), n) + } + if adapts { + p.setLimit(c.next(p.getLimit(), rate)) + } + } + } + if progress != nil { + progress(int(done.Load()), n) + } + st.Settled = p.getLimit() + st.Duration = time.Since(started) + if firstErr != nil { + return st, firstErr + } + return st, ctx.Err() +} + +// climber moves the number running at once towards where the most finishes. +type climber struct { + dir int // +1 more, -1 fewer + prevRate float64 + still int // windows without a move +} + +func (c *climber) next(limit int, rate float64) int { + step := max(1, limit/4) + switch { + case c.prevRate == 0: + // The first window: try more. + case rate >= c.prevRate*better: + // That helped: go on the same way. + c.still = 0 + case rate <= c.prevRate*worse: + // That hurt: back the other way. + c.dir = -c.dir + c.still = 0 + default: + // No difference worth the name: stay, and look the other way after + // a while. + c.still++ + if c.still < probeAfter { + c.prevRate = rate + return limit + } + c.still = 0 + c.dir = -c.dir + } + c.prevRate = rate + limit += c.dir * step + if limit < 1 { + limit, c.dir = 1, 1 + } + if limit > maxAtOnce { + limit, c.dir = maxAtOnce, -1 + } + return limit +} + +// pool lets at most limit workers work at once; the limit can move while +// they do. +type pool struct { + mu sync.Mutex + cond *sync.Cond + active int + limit int +} + +func (p *pool) acquire(ctx context.Context) bool { + p.mu.Lock() + defer p.mu.Unlock() + for p.active >= p.limit { + if ctx.Err() != nil { + return false + } + p.cond.Wait() + } + if ctx.Err() != nil { + return false + } + p.active++ + return true +} + +func (p *pool) release() { + p.mu.Lock() + p.active-- + p.mu.Unlock() + p.cond.Broadcast() +} + +func (p *pool) setLimit(n int) { + p.mu.Lock() + p.limit = n + p.mu.Unlock() + p.cond.Broadcast() +} + +func (p *pool) getLimit() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.limit +} diff --git a/internal/iopar/run_test.go b/internal/iopar/run_test.go new file mode 100644 index 00000000..48f3b74d --- /dev/null +++ b/internal/iopar/run_test.go @@ -0,0 +1,117 @@ +package iopar + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +// A drive that finishes the most with best operations at once: each +// operation takes longer the more run beside it, slowly up to best and +// steeply beyond. +func simulated(best int) func(i int) error { + var active atomic.Int64 + return func(int) error { + a := float64(active.Add(1)) + defer active.Add(-1) + over := a / float64(best) + lat := 200 * time.Microsecond + if over > 1 { + lat = time.Duration(float64(lat) * over * over * over) + } + time.Sleep(lat) + return nil + } +} + +// The simulated drive is timed with sub-millisecond sleeps, so a machine busy +// with other work (a full test run on a slow CPU) can make one run's windows +// look alike and leave it far off. Each kind gets three runs and one has to +// settle near the best; an adaptation that is actually broken misses in all. +func TestRun_SettlesNearWhatTheDriveDoesBest(t *testing.T) { + defer func(w time.Duration) { window = w }(window) + window = 60 * time.Millisecond + for _, kind := range []Kind{NVMe, SATA} { + near := false + for attempt := 1; attempt <= 3 && !near; attempt++ { + st, err := RunKind(context.Background(), kind, 40000, simulated(12), nil) + if err != nil { + t.Fatal(err) + } + t.Logf("%s, run %d: started at %d, settled at %d, peak %.0f/s in %v", kind, attempt, st.Start, st.Settled, st.Peak, st.Duration) + near = st.Settled >= 5 && st.Settled <= 28 + } + if !near { + t.Errorf("%s: no run settled near the 12 the drive does best", kind) + } + } +} + +func TestRun_HardDiskOneAtATime(t *testing.T) { + var active, most atomic.Int64 + st, err := RunKind(context.Background(), HDD, 300, func(int) error { + a := active.Add(1) + defer active.Add(-1) + for { + m := most.Load() + if a <= m || most.CompareAndSwap(m, a) { + break + } + } + time.Sleep(100 * time.Microsecond) + return nil + }, nil) + if err != nil { + t.Fatal(err) + } + if most.Load() != 1 || st.Settled != 1 { + t.Errorf("a hard disk ran %d at once (settled %d), want 1", most.Load(), st.Settled) + } +} + +func TestRun_EveryItemOnceAndProgress(t *testing.T) { + const n = 5000 + var seen [n]atomic.Int32 + var lastDone int + _, err := RunKind(context.Background(), NVMe, n, func(i int) error { + seen[i].Add(1) + return nil + }, func(done, total int) { + if total != n || done < lastDone { + t.Errorf("progress %d/%d after %d", done, total, lastDone) + } + lastDone = done + }) + if err != nil { + t.Fatal(err) + } + for i := range seen { + if seen[i].Load() != 1 { + t.Fatalf("item %d done %d times", i, seen[i].Load()) + } + } + if lastDone != n { + t.Errorf("progress ended at %d of %d", lastDone, n) + } +} + +func TestRun_StopsAtTheFirstError(t *testing.T) { + boom := errors.New("disk full") + var calls atomic.Int64 + _, err := RunKind(context.Background(), SATA, 100000, func(i int) error { + calls.Add(1) + if i == 50 { + return boom + } + time.Sleep(50 * time.Microsecond) + return nil + }, nil) + if !errors.Is(err, boom) { + t.Fatalf("err = %v, want the work's error", err) + } + if calls.Load() > 5000 { + t.Errorf("went on for %d items after the error", calls.Load()) + } +} diff --git a/internal/owntouch/owntouch.go b/internal/owntouch/owntouch.go new file mode 100644 index 00000000..e63987a5 --- /dev/null +++ b/internal/owntouch/owntouch.go @@ -0,0 +1,137 @@ +// Package owntouch remembers which paths in save folders OpenSave itself has +// just written or removed, so the watcher can tell those changes apart from +// ones a game or a person made. +// +// Without it every change OpenSave applied — a file pulled from a peer, a +// deletion a peer asked for, the mtimes "keep mine" touches — looked to the +// watcher exactly like the game saving. A peer deleting files one request at +// a time over several minutes produced a full auto-snapshot every couple of +// seconds, each of a save that was neither the old one nor the new one, and +// those filled the retention budget and pushed out the snapshots that mattered. +package owntouch + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" +) + +// window is how long a mark counts. Long enough to cover the watcher's +// debounce and a slow burst of events behind it; short enough that the same +// path changed again by a game a little later is not mistaken for ours. +const window = 2 * time.Minute + +var ( + mu sync.Mutex + marked = map[string]mark{} +) + +// mark is when OpenSave touched a path, whether it removed it, and - once +// OpenSave is done with a file (Settled) - the time and size it left it with. +type mark struct { + at time.Time + removed bool + settled bool + mod time.Time + size int64 +} + +func key(path string) string { + p := filepath.Clean(path) + if runtime.GOOS == "windows" { + p = strings.ToLower(p) + } + return p +} + +// Mark records that OpenSave itself just created, wrote, removed or re-dated +// path. Its parent folder is marked too: creating or removing an entry is an +// event on the folder as well. +func Mark(path string) { record(path, false) } + +// MarkRemoved records that OpenSave itself is about to remove path. Only a +// path marked this way counts as OpenSave's when it is found gone: a file +// OpenSave has just written and a game or a person then deleted is their +// change, and has to sync and be snapshotted as one. +func MarkRemoved(path string) { record(path, true) } + +// Settled records the time and size OpenSave has left a file it marked with, +// once it is done with it (written, renamed into place, re-dated). From then on +// the file is OpenSave's only while it still has exactly those: a game saving +// over it a moment later - inside any slack a time comparison would need - is +// the game's change. +func Settled(path string) { + fi, err := os.Lstat(path) + if err != nil || fi.IsDir() { + return + } + k := key(path) + mu.Lock() + defer mu.Unlock() + if m, ok := marked[k]; ok { + m.settled, m.mod, m.size = true, fi.ModTime(), fi.Size() + marked[k] = m + } +} + +func record(path string, removed bool) { + if path == "" { + return + } + now := time.Now() + mu.Lock() + defer mu.Unlock() + marked[key(path)] = mark{at: now, removed: removed} + marked[key(filepath.Dir(path))] = mark{at: now} + if len(marked) > 50000 { + for k, m := range marked { + if now.Sub(m.at) > window { + delete(marked, k) + } + } + } +} + +// slack is how much later than its mark a file OpenSave wrote may be dated: +// re-dating to now happens just after marking, and some filesystems keep +// times to two seconds. +const slack = 2 * time.Second + +// Recent reports whether the change just seen at path is one OpenSave made: +// the path was marked within the window, and nothing has written it since. +// +// The second half matters as much as the first. A game often saves straight +// after a sync — the same file OpenSave has just put there — and a mark alone +// would take that save for the sync's for two minutes: no snapshot of it, and +// no new version to hand to anyone. Whatever OpenSave leaves behind is dated +// no later than its mark (a pulled file keeps the time it was written to its +// temporary name, or is given the peer's; re-dating sets now), so a file +// dated later was written by someone else. Once OpenSave is done with a file +// (Settled) the test is exact: the time and size it left, or not ours - the +// slack would take a game's save within two seconds of a pull for the pull's. +// A path that is gone is OpenSave's +// only if OpenSave removed it (MarkRemoved): one it wrote and someone then +// deleted is theirs. A folder carries no save of its own, so its mark is +// enough. +func Recent(path string) bool { + mu.Lock() + m, ok := marked[key(path)] + mu.Unlock() + if !ok || time.Since(m.at) > window { + return false + } + fi, err := os.Lstat(path) + if err != nil { + return m.removed + } + if fi.IsDir() { + return true + } + if m.settled { + return fi.ModTime().Equal(m.mod) && fi.Size() == m.size + } + return !fi.ModTime().After(m.at.Add(slack)) +} diff --git a/internal/owntouch/owntouch_test.go b/internal/owntouch/owntouch_test.go new file mode 100644 index 00000000..824f04a4 --- /dev/null +++ b/internal/owntouch/owntouch_test.go @@ -0,0 +1,132 @@ +package owntouch + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestMarkAndRecent(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "sub", "slot1.sav") + if Recent(file) { + t.Fatal("unmarked path reported as ours") + } + Mark(file) + if err := os.MkdirAll(filepath.Dir(file), 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("pulled"), 0o666); err != nil { + t.Fatal(err) + } + if !Recent(file) { + t.Error("marked path not reported as ours") + } + if !Recent(filepath.Dir(file)) { + t.Error("the parent folder of a marked path not reported as ours") + } + if Recent(filepath.Join(dir, "sub", "slot2.sav")) { + t.Error("a sibling of a marked path reported as ours") + } +} + +// A game saving over a file OpenSave has just written is the game's change, +// however soon after. +func TestRecent_AWriteAfterTheMarkIsNotOurs(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "slot1.sav") + Mark(file) + if err := os.WriteFile(file, []byte("pulled"), 0o666); err != nil { + t.Fatal(err) + } + if !Recent(file) { + t.Fatal("the file OpenSave wrote is not reported as ours") + } + // The game saves a little later. + later := time.Now().Add(5 * time.Second) + if err := os.WriteFile(file, []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(file, later, later); err != nil { + t.Fatal(err) + } + if Recent(file) { + t.Error("a save written after OpenSave's was taken for OpenSave's") + } +} + +// A file OpenSave has just written, then deleted by a game or a person, is +// their deletion: it has to sync and be snapshotted. Only a path OpenSave +// removed itself is its own when found gone. +func TestRecent_AFileDeletedAfterOurWriteIsNotOurs(t *testing.T) { + dir := t.TempDir() + pulled := filepath.Join(dir, "pulled.sav") + Mark(pulled) + if err := os.WriteFile(pulled, []byte("from the peer"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.Remove(pulled); err != nil { + t.Fatal(err) + } + if Recent(pulled) { + t.Error("a file deleted by someone else after OpenSave wrote it was taken for OpenSave's") + } + + removed := filepath.Join(dir, "removed.sav") + if err := os.WriteFile(removed, []byte("old"), 0o666); err != nil { + t.Fatal(err) + } + MarkRemoved(removed) + if err := os.Remove(removed); err != nil { + t.Fatal(err) + } + if !Recent(removed) { + t.Error("a file OpenSave removed was not reported as its own") + } +} + +// A game saving over a file OpenSave has just pulled is the game's change +// however soon it comes - inside the slack a time comparison allows too. +func TestRecent_AWriteRightAfterASettledPullIsNotOurs(t *testing.T) { + file := filepath.Join(t.TempDir(), "progress.sav") + Mark(file) + if err := os.WriteFile(file, []byte("from A"), 0o666); err != nil { + t.Fatal(err) + } + Settled(file) + if !Recent(file) { + t.Fatal("the file OpenSave pulled is not reported as ours") + } + // Same size, a moment later: only the time tells them apart. + time.Sleep(20 * time.Millisecond) + if err := os.WriteFile(file, []byte("from B"), 0o666); err != nil { + t.Fatal(err) + } + if Recent(file) { + t.Error("a save written straight after the pull was taken for OpenSave's") + } +} + +// A pull renames a file into place, then re-dates it to the peer's time. In +// between, and before it is Settled again, the file is still OpenSave's. +func TestRecent_ARedatedPullBeforeItSettlesAgainIsOurs(t *testing.T) { + file := filepath.Join(t.TempDir(), "auto.sav") + Mark(file) + if err := os.WriteFile(file, []byte("pulled"), 0o666); err != nil { + t.Fatal(err) + } + Settled(file) + Mark(file) + peers := time.Now().Add(-time.Hour) + if err := os.Chtimes(file, peers, peers); err != nil { + t.Fatal(err) + } + if !Recent(file) { + t.Error("a pulled file re-dated to the peer's time was taken for someone else's change") + } + Settled(file) + if !Recent(file) { + t.Error("a pulled file settled at the peer's time was taken for someone else's change") + } +} diff --git a/internal/p2p/deletion_race_test.go b/internal/p2p/deletion_race_test.go index 09f46f32..952fcb19 100644 --- a/internal/p2p/deletion_race_test.go +++ b/internal/p2p/deletion_race_test.go @@ -58,7 +58,7 @@ func (a *deviceA) FetchManifest(ctx context.Context, peer syncengine.Peer, gameI if err != nil { return syncengine.ManifestResponse{}, err } - return syncengine.ManifestResponse{Manifest: m, ActiveBranch: "main", Proto: syncengine.ProtoMultiRoot}, nil + return syncengine.ManifestResponse{Manifest: m, ActiveBranch: "main", Proto: syncengine.ProtoVersions}, nil } func (a *deviceA) FetchBlocks(ctx context.Context, peer syncengine.Peer, ref syncengine.FileRef, blockIndices []int, blockSize int) ([]syncengine.BlockData, error) { @@ -159,7 +159,7 @@ func writeFile(t *testing.T, path, body string) { // deleteRequest is one of A's deletion requests arriving at B's LAN route. func (f *raceFixture) deleteRequest(t *testing.T, rel, root string) { t.Helper() - body, _ := json.Marshal(map[string]string{"relPath": rel, "root": root}) + body, _ := json.Marshal(map[string]any{"relPath": rel, "root": root, "versioned": true}) r := httptest.NewRequest(http.MethodPost, "/api/p2p/delete-file/"+f.gameID, bytes.NewReader(body)) r.RemoteAddr = "198.51.100.7:40000" // not a paired device's address: no lineage refresh races the test rc := chi.NewRouteContext() @@ -172,6 +172,26 @@ func (f *raceFixture) deleteRequest(t *testing.T, rel, root string) { } } +// A deletion asked for by an OpenSave that keeps no save versions is refused: +// its old file comparison reads whatever its copy lacks as deleted. +func TestADeletionFromAnOldBuildIsRefused(t *testing.T) { + f := newRaceFixture(t, false) + body, _ := json.Marshal(map[string]any{"relPath": "slot1.sav"}) + r := httptest.NewRequest(http.MethodPost, "/api/p2p/delete-file/"+f.gameID, bytes.NewReader(body)) + r.RemoteAddr = "198.51.100.7:40000" + rc := chi.NewRouteContext() + rc.URLParams.Add("gameId", f.gameID) + r = r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, rc)) + w := httptest.NewRecorder() + f.e.handleDeleteFile(w, r) + if w.Code != http.StatusConflict { + t.Errorf("a deletion from an old build: %d %s, want %d", w.Code, w.Body.String(), http.StatusConflict) + } + if !f.has(f.bDir, "slot1.sav") { + t.Error("the file was deleted on an old build's word") + } +} + func (f *raceFixture) has(dir, rel string) bool { _, err := os.Stat(filepath.Join(dir, filepath.FromSlash(rel))) return err == nil @@ -212,7 +232,7 @@ func TestASyncBetweenTwoRelayedDeletionsIsNotAConflict(t *testing.T) { os.Remove(filepath.Join(f.a.dir, "slot1.sav")) os.Remove(filepath.Join(f.a.dir, "slot2.sav")) - body, _ := json.Marshal(map[string]string{"relPath": "slot1.sav"}) + body, _ := json.Marshal(map[string]any{"relPath": "slot1.sav", "versioned": true}) if code, out := w.serveDeleteFile("/delete-file/"+f.gameID, body, "node_unknown"); code != 200 { t.Fatalf("relayed delete: %d %v", code, out) } diff --git a/internal/p2p/engine.go b/internal/p2p/engine.go index b1777e2e..83e68ea7 100644 --- a/internal/p2p/engine.go +++ b/internal/p2p/engine.go @@ -9,11 +9,15 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "net/http" + "os" + "path/filepath" "strings" "sync" "time" + "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/e2ee" "github.com/opensave/opensave/internal/p2p/discovery" "github.com/opensave/opensave/internal/p2p/pairing" @@ -119,6 +123,20 @@ type Engine struct { // that a device has gone away — see offlineStrikes. pingMu sync.Mutex pingMisses map[string]int + // probeMu serializes PingPairedPeers rounds. + probeMu sync.Mutex + // probingLinks: peers whose connection is being speed-tested now + // (probeLinkSoon). + probeLinksMu sync.Mutex + probingLinks map[string]bool + // When each game last got a safety snapshot before a peer's deletions + // (snapshotBeforePeerDeletions). + peerDeleteSnapMu sync.Mutex + peerDeleteSnap map[string]time.Time + // answered holds the peers that answered a probe during this run, and + // startedMs is when the run began (heardThisRun). + answered map[string]bool + startedMs int64 // Lineage refreshes already running after a peer-applied deletion, keyed // by game+peer. Deletions arrive one file at a time, and each one leaves @@ -283,6 +301,7 @@ func New(s *store.Store, snaps *snapshot.Manager, logf func(level, msg string)) Log: logf, ctx: ctx, cancel: cancel, + startedMs: time.Now().UnixMilli(), } e.Wan = newWanClient(e) e.RelayHost = NewRelayHost(logf) @@ -379,9 +398,34 @@ func (e *Engine) clearPingMisses(peerID string) { delete(e.pingMisses, peerID) } +// noteAnswered records that a peer answered a probe during this run. +func (e *Engine) noteAnswered(peerID string) { + e.pingMu.Lock() + defer e.pingMu.Unlock() + if e.answered == nil { + e.answered = map[string]bool{} + } + e.answered[peerID] = true +} + +// heardThisRun reports whether there is any sign of a peer since this engine +// started: a probe it answered, or a sighting (discovery, a request from it, +// the relay) recent enough to postdate the start. +func (e *Engine) heardThisRun(p store.Peer) bool { + e.pingMu.Lock() + answered := e.answered[p.ID] + e.pingMu.Unlock() + return answered || p.LastSeenMs >= e.startedMs +} + // PingPairedPeers probes every paired LAN peer and updates their // online/offline status. func (e *Engine) PingPairedPeers(ctx context.Context) { + // One probe round at a time: the presence loop and a sync starting can + // both ask, and two overlapping rounds would count one outage as two + // misses toward offlineStrikes. + e.probeMu.Lock() + defer e.probeMu.Unlock() peers, err := e.Store.ListPeers() if err != nil { return @@ -414,11 +458,19 @@ func (e *Engine) PingPairedPeers(ctx context.Context) { // failed cannot lose anything: a device that really has gone is // declared offline a few probes later, and the only cost is a sync // attempt that fails the way it would have anyway. + // + // That patience is for a device seen during this run. A status + // carried over from the previous run is not evidence of anything: a + // PC switched off overnight came back "online" from the database on + // every start, and stayed so until three probes had failed. A device + // not heard from since this start goes offline on its first miss. newStatus := p.Status if ok { e.clearPingMisses(p.ID) + e.noteAnswered(p.ID) newStatus = "online" - } else if e.notePingMiss(p.ID) >= offlineStrikes { + e.probeLinkSoon(p) + } else if misses := e.notePingMiss(p.ID); misses >= offlineStrikes || !e.heardThisRun(p) { newStatus = "offline" } @@ -434,6 +486,35 @@ func (e *Engine) PingPairedPeers(ctx context.Context) { } } +// probeLinkSoon tests the connection to a peer that just answered, in the +// background, when nothing has measured it for a while +// (syncengine/linkspeed.go). One test per peer at a time. +func (e *Engine) probeLinkSoon(p store.Peer) { + if e.Sync == nil || p.Address == "relay" { + return + } + e.probeLinksMu.Lock() + if e.probingLinks == nil { + e.probingLinks = map[string]bool{} + } + if e.probingLinks[p.ID] { + e.probeLinksMu.Unlock() + return + } + e.probingLinks[p.ID] = true + e.probeLinksMu.Unlock() + go func() { + defer func() { + e.probeLinksMu.Lock() + delete(e.probingLinks, p.ID) + e.probeLinksMu.Unlock() + }() + if e.Sync.ProbeLinkIfDue(context.Background(), syncengine.Peer{ID: p.ID, Name: p.Name, Address: p.Address, Port: p.Port}) { + e.notifyPeerUpdate() + } + }() +} + // ErrNoPeersOnline is a sync with no other device to sync with: none of the // paired devices answered. Not a failure of anything — the save goes over when // one is back — which is why it is told apart from errors that are. @@ -542,6 +623,8 @@ func (e *Engine) StartResyncLoop() { stop := e.stopRetry e.pendingMu.Unlock() + e.startPresenceLoop(stop) + // Tracked on the engine's lifecycle so a shutdown lands between ticks // rather than half-way through a transfer. e.GoSync(func(ctx context.Context) { @@ -571,6 +654,43 @@ func (e *Engine) StartResyncLoop() { }) } +// presenceInterval is how often paired LAN peers are probed just to know +// whether they are there. +// +// Probing used to happen only as a side effect: before the 60-second +// reconcile, before a retry, before a manual sync. The reconcile runs on the +// same goroutine as the sync-everything pass it starts, so while a big +// library synced the next probe waited minutes, and with offlineStrikes +// that was minutes more before a device that had gone showed as gone. Peers +// seen by UDP broadcast hid this; one reached over a VPN such as Tailscale, +// which does not carry broadcasts, depends on probes alone. +// +// 10s makes "back" show within ~10s and "gone" within ~30s. A probe is one +// small GET per peer. A var only so a test can shorten it. +var presenceInterval = 10 * time.Second + +// startPresenceLoop probes paired peers on its own ticker, independent of +// whatever sync work is running. It stops with the resync loop. +func (e *Engine) startPresenceLoop(stop chan struct{}) { + e.GoSync(func(ctx context.Context) { + // At once, then on the ticker: until the first round, every peer + // shows whatever the previous run left in the database. + e.PingPairedPeers(ctx) + ticker := time.NewTicker(presenceInterval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ctx.Done(): + return + case <-ticker.C: + e.PingPairedPeers(ctx) + } + } + }) +} + // reconcileAllGames refreshes peer status and re-syncs every game — the // periodic backstop against missed sync triggers. func (e *Engine) reconcileAllGames(ctx context.Context) { @@ -602,6 +722,13 @@ func (e *Engine) retryPendingResyncs(ctx context.Context) { if ctx.Err() != nil { return // shutting down; the failsafe picks these up next start } + // A save folder that is not here cannot be synced, and retrying it + // every tick only logs the same failure. Dropped from the queue; the + // periodic reconcile picks it up once the folder is back. + if g, err := e.Store.GetGame(id); err == nil && saveFolderAbsent(g.SavePath) { + e.ClearPendingResync(id) + continue + } e.Log("info", fmt.Sprintf("retrying interrupted sync for %s", id)) results, err := e.Sync.SyncGame(ctx, id, online) if err == nil { @@ -616,6 +743,22 @@ func (e *Engine) retryPendingResyncs(ctx context.Context) { } } +// saveFolderAbsent reports whether a game's save folder is not on this device +// — the same test as daemon.SaveFolderMissing, which this package cannot +// import. For a single-file save it is the folder the file goes in that +// counts: the file itself may not have been written yet. +func saveFolderAbsent(savePath string) bool { + if savePath == "" { + return false + } + folder := savePath + if isFile, err := delta.ResolveLocalSaveFilePath(savePath); err == nil && isFile { + folder = filepath.Dir(savePath) + } + _, err := os.Stat(folder) + return errors.Is(err, fs.ErrNotExist) +} + // SyncAllGames syncs every tracked game (used when a peer comes online). func (e *Engine) SyncAllGames(ctx context.Context) { if e.Pause.Paused() { @@ -633,6 +776,12 @@ func (e *Engine) SyncAllGames(ctx context.Context) { if !g.AutoSync { continue } + // Not on this device (a drive it doesn't have, a folder that is + // gone): nothing to sync, and it is shown as such on screen. Trying + // anyway logged an error for it against every peer, every pass. + if saveFolderAbsent(g.SavePath) { + continue + } results, err := e.Sync.SyncGame(ctx, g.ID, online) if errors.Is(err, syncengine.ErrHeld) { continue // said once, when it was held; asked about on screen diff --git a/internal/p2p/manifest_gzip_test.go b/internal/p2p/manifest_gzip_test.go new file mode 100644 index 00000000..cca409cf --- /dev/null +++ b/internal/p2p/manifest_gzip_test.go @@ -0,0 +1,63 @@ +package p2p + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/p2p/syncengine" +) + +// A large manifest goes over the wire gzipped, and an ordinary Go client — +// which is what every OpenSave version uses — reads it without knowing. +func TestManifestIsCompressedAndReadable(t *testing.T) { + m := delta.Manifest{Files: map[string]delta.FileEntry{}} + for i := 0; i < 5000; i++ { + sum := sha256.Sum256([]byte(fmt.Sprint(i))) // random-looking, like real hashes + h := hex.EncodeToString(sum[:]) + m.Files[fmt.Sprintf("Saves/world/map_%d_%d.bin", i/100, i%100)] = delta.FileEntry{ + Size: 3500, Hash: h, BlockSize: 65536, Blocks: []delta.Block{{Index: 0, Hash: h, Length: 3500}}, + } + } + resp := syncengine.ManifestResponse{Manifest: m, ActiveBranch: "main"} + + var wire atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cw := &countingResponse{ResponseWriter: w} + jsonOKCompressed(cw, r, resp) + wire.Store(cw.n) + })) + defer srv.Close() + + req, _ := http.NewRequest(http.MethodGet, srv.URL, nil) + var got syncengine.ManifestResponse + if err := doJSONWith(lanBulkClient, req, &got); err != nil { + t.Fatal(err) + } + if len(got.Manifest.Files) != len(m.Files) || got.Manifest.ManifestHash() != m.ManifestHash() { + t.Fatalf("manifest came back with %d files and a different hash", len(got.Manifest.Files)) + } + + plain := httptest.NewRecorder() + jsonOK(plain, resp) + if wire.Load()*2 > int64(plain.Body.Len()) { + t.Errorf("compressed %d bytes vs %d plain — expected at least 2x smaller", wire.Load(), plain.Body.Len()) + } + t.Logf("manifest of %d files: %d bytes plain, %d on the wire", len(m.Files), plain.Body.Len(), wire.Load()) +} + +type countingResponse struct { + http.ResponseWriter + n int64 +} + +func (c *countingResponse) Write(b []byte) (int, error) { + n, err := c.ResponseWriter.Write(b) + c.n += int64(n) + return n, err +} diff --git a/internal/p2p/missing_skip_test.go b/internal/p2p/missing_skip_test.go new file mode 100644 index 00000000..b5de9828 --- /dev/null +++ b/internal/p2p/missing_skip_test.go @@ -0,0 +1,28 @@ +package p2p + +import ( + "os" + "path/filepath" + "testing" +) + +func TestSaveFolderAbsent(t *testing.T) { + dir := t.TempDir() + if saveFolderAbsent(dir) { + t.Error("an existing folder was reported absent") + } + if !saveFolderAbsent(filepath.Join(dir, "gone")) { + t.Error("a missing folder was not reported absent") + } + // A single-file save not written yet: its folder is what counts. + file := filepath.Join(dir, "save.sav") + if err := os.WriteFile(file, []byte("x"), 0o666); err != nil { + t.Fatal(err) + } + if saveFolderAbsent(file) { + t.Error("an existing single-file save was reported absent") + } + if saveFolderAbsent("") { + t.Error("an empty path was reported absent") + } +} diff --git a/internal/p2p/presence_test.go b/internal/p2p/presence_test.go new file mode 100644 index 00000000..aca90a06 --- /dev/null +++ b/internal/p2p/presence_test.go @@ -0,0 +1,125 @@ +package p2p + +import ( + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/opensave/opensave/internal/snapshot" + "github.com/opensave/opensave/internal/store" +) + +func waitUntil(d time.Duration, cond func() bool) bool { + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + if cond() { + return true + } + time.Sleep(20 * time.Millisecond) + } + return cond() +} + +// A PC switched off since the last run is in the database as "online". It must +// not stay so for the three probes a device seen this run is allowed: the +// first unanswered probe after start says it is not there. +func TestPresence_StaleOnlineFromLastRunClearsAtOnce(t *testing.T) { + old := presenceInterval + presenceInterval = time.Hour // only the probe at start + t.Cleanup(func() { presenceInterval = old }) + + s, err := store.Open(filepath.Join(t.TempDir(), "opensave.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if err := s.EnsureDefaultSettings(t.TempDir(), t.TempDir()); err != nil { + t.Fatal(err) + } + // Nothing listens here: the PC is off. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + ln.Close() + lastRun := time.Now().Add(-10 * time.Hour).UnixMilli() + if err := s.UpsertPeer(store.Peer{ID: "node_off", Name: "Switched off", Address: "127.0.0.1", Port: port, Status: "online", LastSeenMs: lastRun}); err != nil { + t.Fatal(err) + } + + e := New(s, snapshot.New(s), func(string, string) {}) + stop := make(chan struct{}) + start := time.Now() + e.startPresenceLoop(stop) + t.Cleanup(func() { close(stop); e.cancel(); e.bgSyncWG.Wait() }) + + if !waitUntil(5*time.Second, func() bool { + p, err := s.GetPeer("node_off") + return err == nil && p.Status == "offline" + }) { + t.Fatal("a peer online only in the last run's record was still online after the first probe") + } + t.Logf("shown offline %.1fs after start", time.Since(start).Seconds()) +} + +// A peer's online state follows it on its own ticker, whatever else the +// engine is doing — not only when a sync or the minute-long reconcile happens +// to probe. A peer reached over a VPN gets no UDP broadcasts, so without this +// its indicator lagged by minutes. +func TestPresenceLoop_FollowsPeer(t *testing.T) { + old := presenceInterval + presenceInterval = 50 * time.Millisecond + t.Cleanup(func() { presenceInterval = old }) + + var up atomic.Bool + up.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !up.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok","appVersion":"2.4.0"}`)) + })) + t.Cleanup(srv.Close) + host, portStr, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + + s, err := store.Open(filepath.Join(t.TempDir(), "opensave.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if err := s.EnsureDefaultSettings(t.TempDir(), t.TempDir()); err != nil { + t.Fatal(err) + } + if err := s.UpsertPeer(store.Peer{ID: "node_vpn", Name: "Over VPN", Address: host, Port: port, Status: "offline"}); err != nil { + t.Fatal(err) + } + + e := New(s, snapshot.New(s), func(string, string) {}) + stop := make(chan struct{}) + e.startPresenceLoop(stop) + t.Cleanup(func() { close(stop); e.cancel(); e.bgSyncWG.Wait() }) + + status := func() string { + p, err := s.GetPeer("node_vpn") + if err != nil { + return "" + } + return p.Status + } + if !waitUntil(3*time.Second, func() bool { return status() == "online" }) { + t.Fatalf("peer answering pings still %q", status()) + } + up.Store(false) + if !waitUntil(3*time.Second, func() bool { return status() == "offline" }) { + t.Fatalf("peer no longer answering still %q", status()) + } +} diff --git a/internal/p2p/routes.go b/internal/p2p/routes.go index 67d11a5c..84c7ecab 100644 --- a/internal/p2p/routes.go +++ b/internal/p2p/routes.go @@ -1,7 +1,9 @@ package p2p import ( + "compress/gzip" "context" + "crypto/rand" "encoding/json" "errors" "fmt" @@ -11,12 +13,14 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync/atomic" "time" "github.com/go-chi/chi/v5" "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/logging" + "github.com/opensave/opensave/internal/owntouch" "github.com/opensave/opensave/internal/p2p/pairing" "github.com/opensave/opensave/internal/p2p/syncengine" "github.com/opensave/opensave/internal/store" @@ -40,18 +44,48 @@ func (e *Engine) RegisterRoutes(r chi.Router) { r.Get("/api/p2p/games", e.handlePeerGameList) r.Post("/api/p2p/sync-event/{gameId}", e.handleSyncEvent) r.Get("/api/p2p/app-binary", e.handleAppBinary) + r.Get("/api/p2p/speedtest", e.handleSpeedTest) // What moves save data, or starts a sync: refused while paused. r.Group(func(r chi.Router) { r.Use(e.refuseWhilePaused) r.Get("/api/p2p/manifest/{gameId}", e.handleManifest) r.Post("/api/p2p/blocks/{gameId}", e.handleBlocks) + r.Post("/api/p2p/files/{gameId}", e.handleFileBatch) r.Post("/api/p2p/delete-file/{gameId}", e.handleDeleteFile) r.Get("/api/sync/trigger/{gameId}", e.handleSyncTrigger) }) }) } +// speedTestMax bounds one speed test, and speedTestData is what it sends: +// random, so nothing along the way can compress it into a better figure. +const speedTestMax = 8 << 20 + +var speedTestData = func() []byte { + b := make([]byte, 1<<20) + _, _ = rand.Read(b) + return b +}() + +// handleSpeedTest sends n bytes (at most speedTestMax) for a peer to time: +// how it learns how close this device is (syncengine/linkspeed.go). +func (e *Engine) handleSpeedTest(w http.ResponseWriter, r *http.Request) { + n, _ := strconv.Atoi(r.URL.Query().Get("n")) + if n <= 0 || n > speedTestMax { + n = speedTestMax + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(n)) + for n > 0 { + chunk := min(n, len(speedTestData)) + if _, err := w.Write(speedTestData[:chunk]); err != nil { + return + } + n -= chunk + } +} + func clientIP(r *http.Request) string { host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { @@ -678,6 +712,11 @@ func (e *Engine) handleManifest(w http.ResponseWriter, r *http.Request) { jsonError(w, http.StatusNotFound, syncengine.HeldMessage) return } + // Being played here: synced when the session ends (syncengine/playing.go). + if e.Sync.PlayingHere(game.ID) { + jsonError(w, http.StatusServiceUnavailable, syncengine.PlayingMessage) + return + } // Never describe a save a sync here is writing: part-way through, it is a // mixture no device holds, and the asker would judge it as a save that had // moved (syncengine/settle.go). Held still while it is read; a write in @@ -701,6 +740,10 @@ func (e *Engine) handleManifest(w http.ResponseWriter, r *http.Request) { } manifest, failures, err := delta.BuildMultiManifest(game.SavePath, extra) if err != nil { + if _, statErr := os.Stat(game.SavePath); statErr != nil { + jsonError(w, http.StatusNotFound, syncengine.FolderMissingMessage) + return + } jsonError(w, http.StatusInternalServerError, "manifest build failed: "+err.Error()) return } @@ -720,11 +763,25 @@ func (e *Engine) handleManifest(w http.ResponseWriter, r *http.Request) { ActiveBranch: game.ActiveBranch, Proto: ServedProto(), DeletionConfirmed: e.Sync.DeletionConfirmed(game.ID), + Version: e.Sync.LocalVersion(game, manifest), + Versions: true, } if latest, err := e.Snapshots.LatestSnapshot(gameID, ""); err == nil { - resp.LatestSnapshot = &syncengine.SnapshotInfo{ID: latest.ID, Timestamp: latest.Timestamp, Comment: latest.Comment} - } - jsonOK(w, resp) + resp.LatestSnapshot = &syncengine.SnapshotInfo{ID: latest.ID, Timestamp: latest.Timestamp, Comment: latest.Comment, ContentHash: latest.ContentHash} + } + // The asker already holds the state both devices last agreed on and says + // so; if this device still holds it too, "unchanged" is the whole answer. + // For a save of a quarter-million files the full manifest is tens of MB + // of JSON, and the minute-long reconcile used to fetch it from every peer + // every minute to learn that nothing had changed. Single-location games + // only: extra locations carry their own bases (see RootHash). + if ifHash := r.URL.Query().Get("ifHash"); ifHash != "" && len(manifest.Extra) == 0 && + manifest.ManifestHash() == ifHash { + resp.Manifest = delta.Manifest{} + resp.Unchanged = true + resp.ManifestHash = ifHash + } + jsonOKCompressed(w, r, resp) } // holdForServing holds the game's save still for a manifest to be served from @@ -787,11 +844,141 @@ func (e *Engine) handleBlocks(w http.ResponseWriter, r *http.Request) { jsonOK(w, map[string]any{"blocks": encodeBlocks(blocks, wantsGzip(body.Encodings))}) } +// Bounds on one batch request, so a peer cannot make this device read and +// hold an unbounded amount in a single response. The requester stays well +// under them (syncengine.batchMaxFiles / batchMaxBytes). +const ( + fileBatchMaxFiles = 1024 + fileBatchMaxBytes = 16 << 20 +) + +// handleFileBatch serves the blocks of many files of one save location in a +// single response: handleBlocks for a list. One round trip per file was what +// made a save of many small files take hours; see syncengine.ProtoBatchFiles. +// +// A file that cannot be read is reported in its own entry rather than failing +// the batch, so the requester can say which one it was. +func (e *Engine) handleFileBatch(w http.ResponseWriter, r *http.Request) { + gameID := chi.URLParam(r, "gameId") + var body struct { + Root string `json:"root"` + Files []syncengine.FileBlocksRequest `json:"files"` + Encodings []string `json:"encodings"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || len(body.Files) == 0 { + jsonError(w, http.StatusBadRequest, "files are required") + return + } + if len(body.Files) > fileBatchMaxFiles { + jsonError(w, http.StatusBadRequest, fmt.Sprintf("at most %d files per batch", fileBatchMaxFiles)) + return + } + game, err := e.trackedGameForPeer(gameID) + if err != nil { + jsonError(w, http.StatusNotFound, "Game not found.") + return + } + base, ok := e.resolveServeRoot(gameID, game, body.Root) + if !ok { + jsonError(w, http.StatusNotFound, "This device has no save location named "+strconv.Quote(body.Root)+" for that game.") + return + } + singleFile, _ := delta.ResolveLocalSaveFilePath(base) + + // What the answer will hold: each file's requested blocks at its block + // size, but never more than the file itself. Counting every block at full + // size took 256 files of 4 KB for 16 MB — the whole limit — and refused + // any batch of small files with a few larger ones among them, which failed + // whole pulls of saves made of many small files. + var requested int64 + for _, f := range body.Files { + bs := f.BlockSize + if bs <= 0 { + bs = 64 * 1024 + } + n := int64(len(f.BlockIndices)) * int64(bs) + if f.RelPath != "" && delta.IsSafePath(base, f.RelPath) { + p := delta.LocalNameFor(base, f.RelPath) + if singleFile { + p = base + } + if info, err := os.Stat(p); err == nil && info.Size() < n { + n = info.Size() + } + } + requested += n + } + if requested > fileBatchMaxBytes { + jsonError(w, http.StatusBadRequest, "batch too large") + return + } + gz := wantsGzip(body.Encodings) + + out := make([]syncengine.FileBlocks, 0, len(body.Files)) + for _, f := range body.Files { + entry := syncengine.FileBlocks{RelPath: f.RelPath} + if f.RelPath == "" || !delta.IsSafePath(base, f.RelPath) { + entry.Error = "invalid path" + out = append(out, entry) + continue + } + // Resolved, not joined — see handleBlocks. + fullPath := delta.LocalNameFor(base, f.RelPath) + if singleFile { + fullPath = base + } + blocks, err := delta.ReadBlocks(fullPath, f.BlockIndices, f.BlockSize) + if err != nil { + entry.Error = "read blocks failed: " + err.Error() + } else { + entry.Blocks = encodeBlocks(blocks, gz) + } + out = append(out, entry) + } + jsonOK(w, map[string]any{"files": out}) +} + +// peerDeleteSnapshotEvery is how far apart two safety snapshots before a +// peer's deletions may be: one for a burst of deletions, however many files +// it removes one request at a time. +const peerDeleteSnapshotEvery = 10 * time.Minute + +// snapshotBeforePeerDeletions takes a safety snapshot of a game before the +// first of a burst of deletions a peer asks for. +func (e *Engine) snapshotBeforePeerDeletions(gameID string, r *http.Request) { + if e.Snapshots == nil { + return + } + e.peerDeleteSnapMu.Lock() + if e.peerDeleteSnap == nil { + e.peerDeleteSnap = map[string]time.Time{} + } + if last, ok := e.peerDeleteSnap[gameID]; ok && time.Since(last) < peerDeleteSnapshotEvery { + e.peerDeleteSnapMu.Unlock() + return + } + e.peerDeleteSnap[gameID] = time.Now() + e.peerDeleteSnapMu.Unlock() + + asker := "another device" + if peer, ok := e.peerByAddress(clientIP(r)); ok { + asker = peer.Name + } + if _, err := e.Snapshots.CreateBeforeReplacing(gameID, fmt.Sprintf("Before %s deleted files here", asker)); err != nil { + e.Log("warn", fmt.Sprintf("safety snapshot before %s's deletions failed: %v", asker, err)) + } +} + func (e *Engine) handleDeleteFile(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "gameId") var body struct { RelPath string `json:"relPath"` Root string `json:"root"` + // Versioned: the asking device keeps save versions. One that does + // not decides deletions by the old file comparison, which reads + // whatever its copy lacks as deleted — the deletions that emptied + // whole saves when a copy had arrived only half-way. + Versioned bool `json:"versioned"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.RelPath == "" { jsonError(w, http.StatusBadRequest, "relPath is required.") @@ -803,6 +990,19 @@ func (e *Engine) handleDeleteFile(w http.ResponseWriter, r *http.Request) { jsonError(w, http.StatusNotFound, "Game not found.") return } + if e.Sync.PlayingHere(game.ID) { + jsonError(w, http.StatusServiceUnavailable, syncengine.PlayingMessage) + return + } + if game.AutoSync && !body.Versioned { + from := clientIP(r) + if p, ok := e.peerByAddress(from); ok { + from = p.Name + } + e.Sync.RefusedOldBuildDelete(game.Name, from) + jsonError(w, http.StatusConflict, syncengine.OldBuildDeleteMessage) + return + } base, ok := e.resolveServeRoot(gameID, game, body.Root) if !ok { jsonError(w, http.StatusNotFound, "This device has no save location named "+strconv.Quote(body.Root)+" for that game.") @@ -821,6 +1021,11 @@ func (e *Engine) handleDeleteFile(w http.ResponseWriter, r *http.Request) { _ = os.Chmod(full, 0o666) deleting := time.Now() if info, statErr := os.Stat(full); statErr == nil { + // One copy of the save before a peer starts deleting from it, not + // one per file: the watcher no longer snapshots changes OpenSave + // applies (owntouch), and it was those, one every few seconds of a + // half-deleted save, that had been the only copy before. + e.snapshotBeforePeerDeletions(game.ID, r) // What is removed is remembered, so a sync of this device's own that // lands before the rest of the batch does not take it for a change // made here (syncengine/peerdeleted.go). @@ -829,6 +1034,8 @@ func (e *Engine) handleDeleteFile(w http.ResponseWriter, r *http.Request) { entry, _ = delta.FileEntryFor(full) } // Empty dirs only, for a folder, like rmdirSync. + // A change made at a peer's request, not by the game (owntouch). + owntouch.MarkRemoved(full) if os.Remove(full) == nil && (info.IsDir() || entry.Hash != "") { e.Sync.NotePeerDeletion(game.ID, body.Root, body.RelPath, entry, info.IsDir()) } @@ -939,6 +1146,10 @@ func (e *Engine) handleSyncEvent(w http.ResponseWriter, r *http.Request) { if e.Sync.Progress.OnSyncError != nil { e.Sync.Progress.OnSyncError(gameID, ev) } + // A sync here waiting for this peer to finish need not wait on. + if peer, ok := e.peerByAddress(clientIP(r)); ok { + e.Sync.NotePeerPulled(gameID, peer.ID) + } } jsonOK(w, map[string]any{"success": true}) } @@ -979,6 +1190,32 @@ func progressEventFromMap(data map[string]any) syncengine.ProgressEvent { return ev } +// jsonOKCompressed is jsonOK, gzipped when the asker accepts it. +// +// For manifests. A save of a few hundred thousand files makes a manifest of +// tens of MB of JSON, which over a VPN did not arrive inside a peer's request +// timeout — so the peer never learned what this device holds, every sync of +// that game failed, and the two devices never converged. Manifest JSON (paths +// and hex hashes) compresses several times over. Go's HTTP client asks for +// gzip and unpacks it on its own unless told not to, so every version of +// OpenSave reads these, including ones that predate this. +func jsonOKCompressed(w http.ResponseWriter, r *http.Request, v any) { + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + jsonOK(w, v) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + w.Header().Add("Vary", "Accept-Encoding") + w.WriteHeader(http.StatusOK) + zw, err := gzip.NewWriterLevel(w, gzip.BestSpeed) + if err != nil { + return + } + _ = json.NewEncoder(zw).Encode(v) + _ = zw.Close() +} + func jsonOK(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -1019,7 +1256,7 @@ func (e *Engine) resolveServeRoot(gameID string, game store.Game, root string) ( // argument — the read still happens concurrently. var servedProto atomic.Int64 -func init() { servedProto.Store(syncengine.ProtoMultiRoot) } +func init() { servedProto.Store(syncengine.ProtoVersions) } // ServedProto reports the protocol revision advertised to peers. func ServedProto() int { return int(servedProto.Load()) } @@ -1040,6 +1277,8 @@ func SetServedProto(v int) int { // freshly-pushed files deliberately stay out of it (see persistLineage), so // deleting one here would pull it back instead of propagating the delete. func (e *Engine) peerFinishedPulling(gameID string, peer syncengine.Peer, data map[string]any) { + // A sync here waiting for this peer before slower ones goes on. + defer e.Sync.NotePeerPulled(gameID, peer.ID) // Recorded first, and synchronously: the peer said exactly which files it // wrote, so the lineage can be updated now rather than after a manifest // round trip. That round trip is what left a window in which deleting a diff --git a/internal/p2p/syncengine/batch.go b/internal/p2p/syncengine/batch.go new file mode 100644 index 00000000..c1b1ff23 --- /dev/null +++ b/internal/p2p/syncengine/batch.go @@ -0,0 +1,301 @@ +package syncengine + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/owntouch" +) + +// Pulling many small files, many to a request. +// +// A pull used to cost one HTTP round trip per file, one after the other. +// Measured on a Windows LAN, that came to ~200ms a file, of which writing the +// file (temp, fsync, rename) was ~2ms — so a save made of a quarter-million +// small files took half a day to move 50MB. Batching removes the round trips; +// the writing is the same as before, file by file, and just as safe: each +// file still goes through a PatchWriter (temp file, content verified against +// the manifest's hash, fsync, rename) and gets the peer's mtime. + +const ( + // batchFileMaxBytes is the most a file may need fetched to go into a + // batch. Anything bigger keeps the per-file path, which already fetches + // its blocks in parallel and has nothing to gain here. + batchFileMaxBytes = 256 << 10 + // batchMaxFiles and batchMaxBytes bound one request, well inside what the + // responder accepts (p2p.fileBatchMaxFiles / fileBatchMaxBytes). + batchMaxFiles = 256 + batchMaxBytes = 4 << 20 + // batchMaxClaimed bounds a request as the responder measures it: every + // block requested at the file's full block size, whatever the file's real + // size. 256 files of 4 KB claim 16 MB that way, which is the responder's + // whole limit (p2p.fileBatchMaxBytes), so any file of two blocks in such + // a batch had it refused as "batch too large" — and the whole pull with + // it, over and over, on saves of many small files. Kept under the limit + // with room to spare. + batchMaxClaimed = 12 << 20 +) + +// claimedBytes is what a file costs a batch as the responder counts it. +func claimedBytes(f delta.FileEntry, indices []int) int64 { + bs := f.BlockSize + if bs <= 0 { + bs = 64 * 1024 + } + return int64(len(indices)) * int64(bs) +} + +// batchWorkers is how many batch requests are in flight at once: enough to +// keep the link and the responder's disk busy while this side writes. A var +// only so a test can make the order deterministic. +var batchWorkers = 4 + +type batchJob struct { + relPath string + localPath string + remote delta.FileEntry + indices []int +} + +// changedBytes is how much of a file a pull has to fetch. +func changedBytes(f delta.FileEntry, indices []int) int64 { + var n int64 + for _, idx := range indices { + if idx >= 0 && idx < len(f.Blocks) { + n += int64(f.Blocks[idx].Length) + } + } + return n +} + +// groupBatches splits jobs into requests of at most batchMaxFiles files, +// batchMaxBytes of blocks, and batchMaxClaimed as the responder counts them. +func groupBatches(jobs []batchJob) [][]batchJob { + var out [][]batchJob + var cur []batchJob + var curBytes, curClaimed int64 + for _, j := range jobs { + b := changedBytes(j.remote, j.indices) + c := claimedBytes(j.remote, j.indices) + if len(cur) > 0 && (len(cur) >= batchMaxFiles || curBytes+b > batchMaxBytes || curClaimed+c > batchMaxClaimed) { + out = append(out, cur) + cur, curBytes, curClaimed = nil, 0, 0 + } + cur = append(cur, j) + curBytes += b + curClaimed += c + } + if len(cur) > 0 { + out = append(out, cur) + } + return out +} + +// pullBatched fetches and writes jobs, many files per request. It returns the +// paths it wrote, including on error, since those files are on disk. +func (e *Engine) pullBatched(ctx context.Context, batcher BatchFetcher, peer Peer, gameID, root string, + jobs []batchJob, throttle *throttler, tracker *progressTracker, onProgress func(force bool)) ([]string, error) { + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + mu sync.Mutex + written []string + firstErr error + ) + fail := func(err error) { + mu.Lock() + if firstErr == nil { + firstErr = err + cancel() + } + mu.Unlock() + } + + batches := groupBatches(jobs) + workers := batchWorkers + if workers > len(batches) { + workers = len(batches) + } + work := make(chan []batchJob) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for batch := range work { + done, err := e.pullOneBatch(ctx, batcher, peer, gameID, root, batch, throttle, tracker, onProgress) + mu.Lock() + written = append(written, done...) + mu.Unlock() + if err != nil { + fail(err) + return + } + // One line per batch instead of one per file: a quarter-million + // "file updated" lines rotated every other message out of the log. + if len(done) > 0 { + e.Log("info", fmt.Sprintf("%d files updated (%s … %s)", len(done), done[0], done[len(done)-1])) + } + onProgress(true) + } + }() + } + for _, b := range batches { + select { + case work <- b: + case <-ctx.Done(): + } + } + close(work) + wg.Wait() + + mu.Lock() + defer mu.Unlock() + if firstErr != nil { + return written, firstErr + } + return written, ctx.Err() +} + +// pullOneBatch fetches one batch and writes its files. +func (e *Engine) pullOneBatch(ctx context.Context, batcher BatchFetcher, peer Peer, gameID, root string, + batch []batchJob, throttle *throttler, tracker *progressTracker, onProgress func(force bool)) ([]string, error) { + + req := make([]FileBlocksRequest, len(batch)) + for i, j := range batch { + req[i] = FileBlocksRequest{RelPath: j.relPath, BlockIndices: j.indices, BlockSize: j.remote.BlockSize} + } + + var resp []FileBlocks + var err error + const maxAttempts = 3 + for attempt := 1; attempt <= maxAttempts; attempt++ { + resp, err = batcher.FetchFileBatch(ctx, peer, gameID, root, req) + if err == nil || errors.Is(err, ErrBatchUnsupported) { + break + } + // Refused as too large — a responder that counts or limits batches + // differently: the same request will be refused again, so it goes + // as two halves instead. Asking again unchanged failed the whole pull. + if strings.Contains(err.Error(), "batch too large") && len(batch) > 1 { + half := len(batch) / 2 + first, err := e.pullOneBatch(ctx, batcher, peer, gameID, root, batch[:half], throttle, tracker, onProgress) + if err != nil { + return first, err + } + second, err := e.pullOneBatch(ctx, batcher, peer, gameID, root, batch[half:], throttle, tracker, onProgress) + return append(first, second...), err + } + e.Log("warn", fmt.Sprintf("batch fetch attempt %d/%d of %d files failed: %v", attempt, maxAttempts, len(batch), err)) + if attempt < maxAttempts { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Duration(attempt) * time.Second): + } + } + } + if errors.Is(err, ErrBatchUnsupported) { + return e.pullBatchOneByOne(ctx, peer, gameID, root, batch, throttle, tracker, onProgress) + } + if err != nil { + return nil, fmt.Errorf("fetch batch of %d files: %w", len(batch), err) + } + + byPath := make(map[string]FileBlocks, len(resp)) + for _, f := range resp { + byPath[f.RelPath] = f + } + + var written []string + for _, j := range batch { + if ctx.Err() != nil { + return written, ctx.Err() + } + f, ok := byPath[j.relPath] + if !ok { + return written, fmt.Errorf("fetch blocks for %s: %s did not send it", j.relPath, peer.Name) + } + if f.Error != "" { + return written, fmt.Errorf("fetch blocks for %s: %s", j.relPath, f.Error) + } + if err := writeBatchedFile(j, f.Blocks); err != nil { + return written, err + } + written = append(written, j.relPath) + n := changedBytes(j.remote, j.indices) + tracker.add(n) + onProgress(false) + throttle.wait(ctx, n) + } + return written, nil +} + +// writeBatchedFile writes one file from its fetched blocks, exactly as +// pullFile does: unchanged blocks from the copy on disk, the rest from the +// peer, verified and renamed into place by the PatchWriter. +func writeBatchedFile(j batchJob, blocks []BlockData) error { + writer, err := delta.NewPatchWriter(j.localPath, j.remote) + if err != nil { + return fmt.Errorf("patch %s: %w", j.relPath, err) + } + committed := false + defer func() { + if !committed { + writer.Abort() + } + }() + incoming := make(map[int]bool, len(j.indices)) + for _, idx := range j.indices { + incoming[idx] = true + } + if err := writer.SeedUnchanged(j.localPath, incoming); err != nil { + return fmt.Errorf("patch %s: %w", j.relPath, err) + } + for _, b := range blocks { + if err := writer.WriteBlock(b.Index, b.Data); err != nil { + return fmt.Errorf("patch %s: %w", j.relPath, err) + } + } + if err := writer.Commit(); err != nil { + return fmt.Errorf("patch %s: %w", j.relPath, err) + } + committed = true + if j.remote.MtimeMs > 0 { + mtime := time.UnixMilli(int64(j.remote.MtimeMs)) + owntouch.Mark(j.localPath) // see pullFiles: re-dated, then Settled + _ = os.Chtimes(j.localPath, mtime, mtime) + owntouch.Settled(j.localPath) + } + return nil +} + +// pullBatchOneByOne is the fallback when the transport cannot batch for this +// peer after all: the same files, through the per-file path. +func (e *Engine) pullBatchOneByOne(ctx context.Context, peer Peer, gameID, root string, + batch []batchJob, throttle *throttler, tracker *progressTracker, onProgress func(force bool)) ([]string, error) { + + var written []string + for _, j := range batch { + if err := e.pullFile(ctx, peer, FileRef{GameID: gameID, Root: root, RelPath: j.relPath}, j.localPath, + j.remote, j.indices, throttle, tracker, onProgress); err != nil { + return written, err + } + if j.remote.MtimeMs > 0 { + mtime := time.UnixMilli(int64(j.remote.MtimeMs)) + owntouch.Mark(j.localPath) + _ = os.Chtimes(j.localPath, mtime, mtime) + owntouch.Settled(j.localPath) + } + written = append(written, j.relPath) + } + return written, nil +} diff --git a/internal/p2p/syncengine/batch_test.go b/internal/p2p/syncengine/batch_test.go new file mode 100644 index 00000000..c468cf6c --- /dev/null +++ b/internal/p2p/syncengine/batch_test.go @@ -0,0 +1,237 @@ +package syncengine + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/snapshot" +) + +// batchTransport is fakeTransport plus batched fetches, answering manifests +// with a chosen protocol revision. +type batchTransport struct { + *fakeTransport + proto int + + mu sync.Mutex + batchCalls int + blockCalls int + failPath string // answered with a per-file error + // maxFiles, when set, refuses bigger batches the way a responder that + // counts differently does. + maxFiles int +} + +func (b *batchTransport) FetchManifest(ctx context.Context, peer Peer, gameID string, q ManifestQuery) (ManifestResponse, error) { + resp, err := b.fakeTransport.FetchManifest(ctx, peer, gameID, q) + resp.Proto = b.proto + return resp, err +} + +func (b *batchTransport) FetchBlocks(ctx context.Context, peer Peer, ref FileRef, idx []int, bs int) ([]BlockData, error) { + b.mu.Lock() + b.blockCalls++ + b.mu.Unlock() + return b.fakeTransport.FetchBlocks(ctx, peer, ref, idx, bs) +} + +func (b *batchTransport) FetchFileBatch(ctx context.Context, peer Peer, gameID, root string, files []FileBlocksRequest) ([]FileBlocks, error) { + b.mu.Lock() + b.batchCalls++ + b.mu.Unlock() + if len(files) > batchMaxFiles { + return nil, fmt.Errorf("batch of %d files is over the limit", len(files)) + } + if b.maxFiles > 0 && len(files) > b.maxFiles { + return nil, fmt.Errorf(`peer returned 400: {"error":"batch too large"}`) + } + out := make([]FileBlocks, 0, len(files)) + for _, f := range files { + if f.RelPath == b.failPath { + out = append(out, FileBlocks{RelPath: f.RelPath, Error: "read blocks failed: locked"}) + continue + } + blocks, err := b.fakeTransport.FetchBlocks(ctx, peer, FileRef{GameID: gameID, Root: root, RelPath: f.RelPath}, f.BlockIndices, f.BlockSize) + if err != nil { + out = append(out, FileBlocks{RelPath: f.RelPath, Error: err.Error()}) + continue + } + out = append(out, FileBlocks{RelPath: f.RelPath, Blocks: blocks}) + } + return out, nil +} + +func setupBatchEngine(t *testing.T, proto int) (*engineEnv, *batchTransport) { + t.Helper() + env := setupEngine(t) + bt := &batchTransport{fakeTransport: env.transport, proto: proto} + env.engine = New(env.store, snapshot.New(env.store), bt) + return env, bt +} + +// A save of many small files, plus one big one, in two folders. +func writeManyFiles(t *testing.T, dir string, n int) { + t.Helper() + for i := 0; i < n; i++ { + write(t, dir, fmt.Sprintf("map/chunk_%04d.bin", i), strings.Repeat(fmt.Sprintf("chunk %d;", i), 20)) + } + big := bytes.Repeat([]byte("large save data "), (512<<10)/16) // 512KB: per-file path + if err := os.WriteFile(filepath.Join(dir, "big.sav"), big, 0o666); err != nil { + t.Fatal(err) + } +} + +func assertSameTree(t *testing.T, want, got string) { + t.Helper() + wm, err := delta.BuildManifest(want) + if err != nil { + t.Fatal(err) + } + gm, err := delta.BuildManifest(got) + if err != nil { + t.Fatal(err) + } + if wm.ManifestHash() != gm.ManifestHash() { + t.Fatalf("trees differ: %d files remote, %d local", len(wm.Files), len(gm.Files)) + } + for p, f := range wm.Files { + if gm.Files[p].MtimeMs != f.MtimeMs { + t.Errorf("%s: mtime %d, want the peer's %d", p, gm.Files[p].MtimeMs, f.MtimeMs) + break + } + } +} + +func TestPull_BatchesSmallFiles(t *testing.T) { + env, bt := setupBatchEngine(t, ProtoBatchFiles) + writeManyFiles(t, env.remoteDir, 600) + + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil { + t.Fatalf("SyncWithPeer: %v", err) + } + if res.Status != "updated" || res.Direction != "pull" { + t.Fatalf("result = %+v, want updated/pull", res) + } + assertSameTree(t, env.remoteDir, env.localDir) + + // 600 one-block files: each claims a 64 KB block as the responder counts, + // so 192 fit a request (batchMaxClaimed) -> 4 requests; the big file alone + // goes block by block. + if bt.batchCalls != 4 { + t.Errorf("batch requests = %d, want 4", bt.batchCalls) + } + if bt.blockCalls != 1 { + t.Errorf("per-file block requests = %d, want 1 (the big file)", bt.blockCalls) + } + if len(env.transport.syncEvents) == 0 || env.transport.syncEvents[len(env.transport.syncEvents)-1] != "sync-complete" { + t.Errorf("sync did not complete: %v", env.transport.syncEvents) + } +} + +// A peer that predates batching is asked file by file, as before. +func TestPull_OldPeerIsNotBatched(t *testing.T) { + env, bt := setupBatchEngine(t, ProtoMultiRoot) + writeManyFiles(t, env.remoteDir, 50) + + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatalf("SyncWithPeer: %v", err) + } + assertSameTree(t, env.remoteDir, env.localDir) + if bt.batchCalls != 0 { + t.Errorf("batch requests to an old peer = %d, want 0", bt.batchCalls) + } + if bt.blockCalls != 51 { + t.Errorf("per-file requests = %d, want 51", bt.blockCalls) + } +} + +// One file the peer cannot read fails the pull and names the file, like the +// per-file path does; nothing half-written is left in its place. +func TestPull_BatchFileErrorNamesTheFile(t *testing.T) { + env, bt := setupBatchEngine(t, ProtoBatchFiles) + writeManyFiles(t, env.remoteDir, 10) + bt.failPath = "map/chunk_0003.bin" + + _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err == nil || !strings.Contains(err.Error(), "map/chunk_0003.bin") { + t.Fatalf("err = %v, want one naming map/chunk_0003.bin", err) + } + if _, statErr := os.Stat(filepath.Join(env.localDir, "map", "chunk_0003.bin")); statErr == nil { + t.Error("the file the peer could not serve was written anyway") + } + leftovers, _ := filepath.Glob(filepath.Join(env.localDir, "map", "*"+delta.TmpSuffix)) + if len(leftovers) > 0 { + t.Errorf("temp files left behind: %v", leftovers) + } +} + +func TestGroupBatches_RespectsLimits(t *testing.T) { + var jobs []batchJob + for i := 0; i < 600; i++ { + jobs = append(jobs, batchJob{relPath: fmt.Sprint(i), + remote: delta.FileEntry{Blocks: []delta.Block{{Index: 0, Length: 32 << 10}}}, + indices: []int{0}}) + } + for _, b := range groupBatches(jobs) { + var n int64 + for _, j := range b { + n += changedBytes(j.remote, j.indices) + } + if len(b) > batchMaxFiles || n > batchMaxBytes { + t.Errorf("batch of %d files / %d bytes is over the limit", len(b), n) + } + } +} + +// Many small files, some of several blocks: what the responder counts — +// every block at full block size — must stay under its limit too. 256 files +// of one 64 KB block already claim 16 MB, and each file of more blocks tipped +// a batch over it: "batch too large", and the whole pull failed. +func TestGroupBatches_StaysUnderWhatTheResponderCounts(t *testing.T) { + var jobs []batchJob + for i := 0; i < 2000; i++ { + blocks := []delta.Block{{Index: 0, Length: 4 << 10}} + indices := []int{0} + if i%10 == 0 { + blocks = []delta.Block{{Index: 0, Length: 64 << 10}, {Index: 1, Length: 64 << 10}, {Index: 2, Length: 10 << 10}} + indices = []int{0, 1, 2} + } + jobs = append(jobs, batchJob{relPath: fmt.Sprint(i), + remote: delta.FileEntry{BlockSize: 64 << 10, Blocks: blocks}, indices: indices}) + } + for _, b := range groupBatches(jobs) { + var claimed int64 + for _, j := range b { + claimed += claimedBytes(j.remote, j.indices) + } + if claimed > 16<<20 { + t.Errorf("a batch of %d files claims %d bytes, over the responder's limit", len(b), claimed) + } + } +} + +// A responder that refuses a batch as too large gets it again in halves, +// rather than the pull failing. +func TestPull_ABatchRefusedAsTooLargeIsSplit(t *testing.T) { + env, bt := setupBatchEngine(t, ProtoBatchFiles) + bt.maxFiles = 10 + for i := 0; i < 40; i++ { + write(t, env.remoteDir, fmt.Sprintf("chunks/c%02d.bin", i), fmt.Sprint("chunk ", i)) + } + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatalf("sync failed though the batches could be split: %v", err) + } + for i := 0; i < 40; i++ { + if _, err := os.Stat(filepath.Join(env.localDir, "chunks", fmt.Sprintf("c%02d.bin", i))); err != nil { + t.Fatalf("c%02d.bin did not arrive", i) + } + } +} diff --git a/internal/p2p/syncengine/changedsince_test.go b/internal/p2p/syncengine/changedsince_test.go new file mode 100644 index 00000000..59c57c10 --- /dev/null +++ b/internal/p2p/syncengine/changedsince_test.go @@ -0,0 +1,93 @@ +package syncengine + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/owntouch" +) + +// A pull that finished and whose file the game then saved over is not a +// pull that stopped part-way: resuming it would replace the new save with +// the peer's. +func TestChangedSincePulled(t *testing.T) { + dir := t.TempDir() + pulled := filepath.Join(dir, "progress.sav") + pull := func(content string) { + owntouch.Mark(pulled) + if err := os.WriteFile(pulled, []byte(content), 0o666); err != nil { + t.Fatal(err) + } + owntouch.Settled(pulled) + } + build := func() delta.Manifest { + m, err := delta.BuildManifest(dir) + if err != nil { + t.Fatal(err) + } + return m + } + d := Decision{FilesToPull: []string{"progress.sav"}} + + pull("from A") + remote := build() + + // The game saves straight after the pull. + time.Sleep(20 * time.Millisecond) + if err := os.WriteFile(pulled, []byte("from B"), 0o666); err != nil { + t.Fatal(err) + } + delta.InvalidateRoot(dir) + if !changedSincePulled(dir, build(), remote, d) { + t.Error("a save written over a finished pull was taken for the pull stopping part-way") + } + + // The pull wrote an old copy and stopped: still as OpenSave left it. + pull("old") + delta.InvalidateRoot(dir) + if changedSincePulled(dir, build(), remote, d) { + t.Error("a pull that stopped part-way was taken for a change made here") + } + + // A file the pull did not touch was the same on both sides when it was + // planned; differing now, it was changed here meanwhile. + if !changedSincePulled(dir, build(), remote, Decision{}) { + t.Error("a file the pull did not touch, changed since, was taken for the pull stopping part-way") + } +} + +// A file the pull did not touch, deleted here the moment the pull is done +// (TestFullSyncFlow on CI): a deletion made here, not a part of the pull to +// fetch again. +func TestChangedSincePulled_AnUntouchedFileDeleted(t *testing.T) { + dir := t.TempDir() + for name, content := range map[string]string{"slot1.sav": "from B", "config/video.ini": "fullscreen=1"} { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o666); err != nil { + t.Fatal(err) + } + } + owntouch.Mark(filepath.Join(dir, "slot1.sav")) + owntouch.Settled(filepath.Join(dir, "slot1.sav")) + remote, err := delta.BuildManifest(dir) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(dir, "config", "video.ini")); err != nil { + t.Fatal(err) + } + delta.InvalidateRoot(dir) + fresh, err := delta.BuildManifest(dir) + if err != nil { + t.Fatal(err) + } + if !changedSincePulled(dir, fresh, remote, Decision{FilesToPull: []string{"slot1.sav"}}) { + t.Error("a file deleted here right after the pull was taken for the pull stopping part-way, and would be fetched back") + } +} diff --git a/internal/p2p/syncengine/conflict.go b/internal/p2p/syncengine/conflict.go index 66293b5a..742c35f8 100644 --- a/internal/p2p/syncengine/conflict.go +++ b/internal/p2p/syncengine/conflict.go @@ -8,6 +8,7 @@ import ( "time" "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/owntouch" "github.com/opensave/opensave/internal/snapshot" ) @@ -41,9 +42,12 @@ func (e *Engine) ResolveConflict(ctx context.Context, gameID, peerID, resolution // see both sides still differing from the stale base and re-conflict // immediately — the "keep looping" bug. e.Log("info", fmt.Sprintf("conflict on %q resolved: keep LOCAL — our version becomes the shared state", gameID)) - if err := e.markResolvedLocal(gameID, peer); err != nil { + if err := e.markResolvedLocal(ctx, gameID, peer); err != nil { return "", err } + // This device's version becomes newer than both, so the peer — and any + // device holding either side's — takes it whole (version.go). + e.versionAfterResolution(gameID, conflict.remoteVersion, true) e.clearConflict(gameID) e.Transport.TriggerPeerPull(peer, gameID) return "", nil @@ -63,8 +67,18 @@ func (e *Engine) ResolveConflict(ctx context.Context, gameID, peerID, resolution e.Log("warn", fmt.Sprintf("safety snapshot before keep-remote failed: %v", err)) } if err := e.overwriteLocalWithRemote(ctx, gameID, peer, remoteData, "Resolved conflict: Overwrite with remote"); err != nil { + // This device's own version is given up either way; what is still + // missing comes from the peer like any outdated device's would. + e.discardLocalVersion(gameID, remoteData.Version) return "", err } + if game, err := e.Store.GetGame(gameID); err == nil { + if now, err := delta.BuildManifest(game.SavePath); err == nil && sameFiles(now, remoteData.Manifest) { + e.versionAfterResolution(gameID, remoteData.Version, false) + } else { + e.discardLocalVersion(gameID, remoteData.Version) + } + } e.markResolvedConverged(gameID, peer) e.clearConflict(gameID) return "", nil @@ -103,9 +117,10 @@ func (e *Engine) ResolveConflict(ctx context.Context, gameID, peerID, resolution // From here it is "keep mine": this device's version becomes the // shared one, and the peer is asked to take it — which, if it has // changes of its own, asks the person there in turn. - if err := e.markResolvedLocal(gameID, peer); err != nil { + if err := e.markResolvedLocal(ctx, gameID, peer); err != nil { return "", err } + e.versionAfterResolution(gameID, conflict.remoteVersion, true) e.clearConflict(gameID) e.Transport.TriggerPeerPull(peer, gameID) return created, nil @@ -160,7 +175,20 @@ func (e *Engine) markResolvedConverged(gameID string, peer Peer) { // unambiguously the newest, so the sync direction is a push. The peer // then receives our version (its own engine snapshots first, or re-raises // its own conflict if it has newer unsynced work of its own). -func (e *Engine) markResolvedLocal(gameID string, peer Peer) error { +// +// The lineage, though, must say only what both sides verifiably hold. It used +// to record every local file as shared (persistLineage(local, local)) before +// the peer had pulled any of it. When that pull did not finish — a large save, +// a request that timed out, a device that went away — the next sync found +// those files "shared before and missing on the peer now", read that as the +// peer having deleted them, and deleted them HERE: on the device whose version +// the person had just chosen to keep. A save of a quarter-million files lost +// tens of thousands that way, and the folders differing again re-raised the +// conflict, inviting the same answer. Now the record is the intersection of +// the two saves as they stand; a file only this device has is new to the +// peer, not deleted by it, and goes over on the next sync. If the peer cannot +// be asked, nothing is recorded as shared, so nothing can be read as deleted. +func (e *Engine) markResolvedLocal(ctx context.Context, gameID string, peer Peer) error { game, err := e.Store.GetGame(gameID) if err != nil { return err @@ -170,7 +198,18 @@ func (e *Engine) markResolvedLocal(gameID string, peer Peer) error { if err != nil { return err } - e.persistLineage(gameID, peer.ID, local, local) + var files, dirs []string + if remote, err := e.peerStateForResolution(ctx, gameID, peer); err == nil { + files, dirs = IntersectLineage(local, remote.Manifest) + } else { + e.Log("info", fmt.Sprintf("could not read %s's save while resolving %q (%v) — nothing is recorded as shared, so nothing it lacks is taken as deleted", peer.Name, gameID, err)) + } + rules := e.rulesFor(gameID) + files = filterPathList(files, rules) + dirs = filterPathList(dirs, rules) + if err := e.Store.SetSyncState(gameID, peer.ID, files, dirs); err != nil { + e.Log("warn", fmt.Sprintf("persist sync lineage failed: %v", err)) + } _ = e.Store.SetAgreedHash(gameID, peer.ID, local.ManifestHash()) _ = e.Store.SetLastManifestHash(gameID, e.contentHashOf(gameID, local, game.SavePath)) e.recordSynced(gameID, peer.ID) @@ -197,14 +236,18 @@ func (e *Engine) touchSaveMtimes(gameID, root string) { return } if !info.IsDir() { + owntouch.Mark(root) _ = os.Chtimes(root, now, now) + owntouch.Settled(root) return } _ = filepath.Walk(root, func(path string, fi os.FileInfo, walkErr error) error { if walkErr != nil || fi.IsDir() { return nil } + owntouch.Mark(path) _ = os.Chtimes(path, now, now) + owntouch.Settled(path) return nil }) } @@ -337,6 +380,7 @@ func (e *Engine) overwriteLocalWithRemote(ctx context.Context, gameID string, pe } full := filepath.Join(game.SavePath, filepath.FromSlash(relPath)) _ = os.Chmod(full, 0o666) + owntouch.MarkRemoved(full) _ = os.Remove(full) } diff --git a/internal/p2p/syncengine/devicesettings_test.go b/internal/p2p/syncengine/devicesettings_test.go new file mode 100644 index 00000000..c9691e07 --- /dev/null +++ b/internal/p2p/syncengine/devicesettings_test.go @@ -0,0 +1,94 @@ +package syncengine + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/ignore" + "github.com/opensave/opensave/internal/store" +) + +func withDeviceSettings(env *engineEnv, patterns ...string) { + env.engine.DeviceSettingsFor = func(store.Game) []string { return patterns } +} + +// Each device keeps its own graphics settings: a differing settings file is +// not a difference, in either direction — until the game is set to sync them. +func TestDeviceSettings_KeptPerDevice(t *testing.T) { + env := setupEngine(t) + withDeviceSettings(env, "/graphics.xml") + write(t, env.localDir, "slot1.sav", "progress") + write(t, env.localDir, "graphics.xml", "4k ultra") + write(t, env.remoteDir, "slot1.sav", "progress") + write(t, env.remoteDir, "graphics.xml", "720p low") + + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil || res.Status != "in_sync" { + t.Fatalf("status = %q, %v; want in_sync", res.Status, err) + } + if got, _ := os.ReadFile(filepath.Join(env.localDir, "graphics.xml")); string(got) != "4k ultra" { + t.Errorf("this device's settings became %q", got) + } + if got, _ := os.ReadFile(filepath.Join(env.remoteDir, "graphics.xml")); string(got) != "720p low" { + t.Errorf("the peer's settings became %q", got) + } + + // A save change still travels; the settings beside it still do not. + write(t, env.remoteDir, "slot1.sav", "more progress") + res, err = env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil || res.Status != "updated" { + t.Fatalf("status = %q, %v; want updated", res.Status, err) + } + if got, _ := os.ReadFile(filepath.Join(env.localDir, "slot1.sav")); string(got) != "more progress" { + t.Errorf("the save did not arrive: %q", got) + } + if got, _ := os.ReadFile(filepath.Join(env.localDir, "graphics.xml")); string(got) != "4k ultra" { + t.Errorf("this device's settings became %q", got) + } + + game, _ := env.store.GetGame("game1") + game.SyncDeviceSettings = true + if env.engine.IgnoreText(game) != game.SyncIgnore { + t.Error("set to sync its settings, the game still leaves them out") + } +} + +// The game's own rules come after the settings, so "!" brings one back. +func TestDeviceSettings_OwnRuleCanBringOneBack(t *testing.T) { + env := setupEngine(t) + withDeviceSettings(env, "/graphics.xml", "/input.cfg") + game, _ := env.store.GetGame("game1") + game.SyncIgnore = "!input.cfg" + rules := env.engine.IgnoreText(game) + if r := ignore.Parse(rules); !r.Match("graphics.xml") || r.Match("input.cfg") { + t.Errorf("rules %q: graphics.xml excluded %v, input.cfg excluded %v", rules, r.Match("graphics.xml"), r.Match("input.cfg")) + } +} + +// Leaving the settings out where they were synced before is not a change of +// the save: the recorded hash is re-taken, no version is named. +func TestDeviceSettings_StartingToLeaveThemOutIsNotAVersion(t *testing.T) { + env := setupEngine(t) + game, _ := env.store.GetGame("game1") + game.AutoSync = true + _ = env.store.UpdateGame(game) + write(t, env.localDir, "slot1.sav", "progress") + write(t, env.localDir, "graphics.xml", "4k ultra") + env.engine.NoteLocalChange("game1") + before, _ := env.store.GetGameVersion("game1") + + withDeviceSettings(env, "/graphics.xml") + m, _ := delta.BuildManifest(env.localDir) + env.engine.AdoptExclusionView("game1", m, "", false) + env.engine.NoteLocalChange("game1") + after, _ := env.store.GetGameVersion("game1") + if after.Vector != before.Vector { + t.Fatalf("version moved from %s to %s", before.Vector, after.Vector) + } + if after.Hash != env.engine.versionHashOf("game1", m) { + t.Error("the recorded hash was not re-taken") + } +} diff --git a/internal/p2p/syncengine/engine.go b/internal/p2p/syncengine/engine.go index e3b02b00..2a1c5b96 100644 --- a/internal/p2p/syncengine/engine.go +++ b/internal/p2p/syncengine/engine.go @@ -12,6 +12,7 @@ import ( "time" "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/owntouch" "github.com/opensave/opensave/internal/snapshot" "github.com/opensave/opensave/internal/store" "github.com/opensave/opensave/internal/syncpause" @@ -27,6 +28,17 @@ type Conflict struct { RemoteStats SideStats `json:"remoteStats"` DiffFiles []DiffFile `json:"diffFiles"` // capped; DiffTotal is the real count DiffTotal int `json:"diffTotal"` + // The uncapped counts by kind. DiffFiles stops at 100, and counting from + // it told someone choosing "keep theirs" on a save of a quarter-million + // files that it would remove a few dozen, when it removed tens of + // thousands: every OnlyLocal file goes. + OnlyLocalTotal int `json:"onlyLocalTotal"` + OnlyRemoteTotal int `json:"onlyRemoteTotal"` + ChangedTotal int `json:"changedTotal"` + + // remoteVersion is the peer's version when the conflict was found, which + // the answer is recorded against (version.go). + remoteVersion *VersionInfo } // SideStats summarises one side's save state for the conflict UI. @@ -73,7 +85,7 @@ type Result struct { // back until told whether that was meant (hold.go). peer_busy: the peer was // still writing the game for a sync of its own; this one runs again when it // has finished (settle.go). - Status string `json:"status"` // in_sync | updated | updated_bidirectional | deletions_synced | triggered_peer_pull | conflict | peer_missing | peer_awaiting_folder | peer_holding | peer_busy + Status string `json:"status"` // in_sync | updated | updated_bidirectional | deletions_synced | triggered_peer_pull | conflict | peer_missing | peer_awaiting_folder | peer_holding | peer_busy | unwritable Direction string `json:"direction"` PeerID string `json:"peerId,omitempty"` PeerName string `json:"peerName,omitempty"` @@ -119,6 +131,8 @@ type Engine struct { // on this device right now (settle.go). settleMu sync.Mutex gates map[string]*saveGate + // writtenAt is when a sync last stopped writing each game (BeingWritten). + writtenAt map[string]time.Time // servedMu guards served: the save states this device recently handed // each peer, per game (served.go). servedMu sync.Mutex @@ -136,6 +150,40 @@ type Engine struct { // rootConflicts holds divergences in a game's EXTRA save locations, keyed // by game and location so several can wait on a decision at once. rootConflicts map[string]*RootConflict + + // linkMu guards links, the measured connection to each peer, and + // pullWaits, syncs waiting for a close peer to finish taking a save + // (linkspeed.go). + linkMu sync.Mutex + links map[string]store.PeerLink + pullWaits map[string]chan struct{} + // versionMu serialises reading and writing the save versions + // (version.go); pullingNow is the games this run is pulling a newer + // version of, and loggedOnce the last "waiting" line said per game/peer. + versionMu sync.Mutex + pullingNow map[string]bool + loggedOnce map[string]string + // peerVersions: per peer, whether its OpenSave keeps save versions + // (PeerNeedsUpdate). + peerVersions map[string]bool + // DeviceSettingsFor returns the exclusion patterns for a game's device + // settings (presets/devicesettings.go), set by the daemon; nil leaves + // them syncing. See IgnoreText. + DeviceSettingsFor func(store.Game) []string + // ObserveSettings is told what another device holds of a game, as its + // manifest arrives, to watch how the files left out as the game's + // settings change (daemon/detectsettings.go). Set by the daemon. + ObserveSettings func(game store.Game, peerID string, remote delta.Manifest) + // Playing reports whether a game is being played on this device + // (playing.go). Set by the daemon from its play sessions. + Playing func(gameID string) bool + // OnPutBack runs when every file of an emptied save is back, put back or + // by hand: no snapshot was taken of it meanwhile, so the save is kept as + // it is now. Set by the daemon. + OnPutBack func(gameID string) + // unwritable: games whose save folder could not be written, and until + // when pulls into it wait (version.go). + unwritable map[string]time.Time } // New creates an Engine. @@ -195,6 +243,13 @@ func isGameNotFound(err error) bool { // into one. const AwaitingFolderMessage = "This device has not been given a folder for this game yet" +// FolderMissingMessage is what a device says when the folder it tracks a game +// at is not there now — on a drive not plugged in, or moved. Nothing to +// compare with, and nothing wrong to report over and over: it contains +// AwaitingFolderMessage, so the asking device waits for it as it does for +// a folder not yet chosen. +const FolderMissingMessage = AwaitingFolderMessage + " here: the folder it was given is not on this device now" + // isAwaitingFolder reports whether the peer is holding this game until someone // chooses where it lives there — as opposed to not tracking it at all. func isAwaitingFolder(err error) bool { @@ -235,6 +290,10 @@ func (e *Engine) SyncGame(ctx context.Context, gameID string, onlinePeers []Peer if e.Paused != nil && e.Paused() { return nil, ErrPaused } + // A game being played here is synced when the session ends (playing.go). + if e.PlayingHere(gameID) { + return nil, ErrPlayingHere + } // A save emptied here is not synced until someone says whether that was // meant (hold.go). if held, err := e.CheckHold(gameID, false); err == nil && held { @@ -303,13 +362,32 @@ func (e *Engine) SyncGame(ctx context.Context, gameID string, onlinePeers []Peer results := map[string]Result{} busy := false - for _, peer := range onlinePeers { + // Fastest connections first (linkspeed.go): a device that is behind takes + // the newer save from the closest device holding it, and a newer save + // here reaches the close devices before the slow ones. + onlinePeers = e.OrderBySpeed(onlinePeers) + for i, peer := range onlinePeers { // Hard per-peer cap: a wedged transport must never hold // activeSyncs forever (which would silently block every future // sync of this game until an app restart). peerCtx, cancel := context.WithTimeout(ctx, perPeerSyncTimeout) res, err := e.SyncWithPeer(peerCtx, gameID, peer) cancel() + // A close device asked to take this save is given the time to, before + // a much slower one is asked: it gets it at full speed, and the slow + // one can then take it from whichever device is nearest to it. + if err == nil && res.Status == "triggered_peer_pull" && e.waitsForClose(onlinePeers, i) { + var size int64 + if game, gerr := e.Store.GetGame(gameID); gerr == nil { + if m, merr := e.ReadManifest(ctx, gameID, game.SavePath); merr == nil { + for _, f := range m.Files { + size += f.Size + } + } + } + e.Log("info", fmt.Sprintf("waiting for %s (close by) to take %s before the slower devices", peer.Name, gameID)) + e.waitForPull(ctx, gameID, peer, size) + } if err == nil && res.Status == "peer_busy" { // Nothing was compared, so nothing is stamped, and it is not a // failure: the peer was writing this game for a sync of its own. @@ -334,7 +412,7 @@ func (e *Engine) SyncGame(ctx context.Context, gameID string, onlinePeers []Peer // divergence from the NEXT sync, causing the peer to silently overwrite // its own changes instead of detecting the conflict and asking. switch res.Status { - case "conflict", "error": + case "conflict", "error", versionStatusWaiting, StatusPeerOutdatedApp, "unwritable", "peer_playing": case "peer_missing", "peer_awaiting_folder", "peer_holding": // The two devices talked and finished, which is what the // per-device stamp has always recorded. But nothing of THIS game @@ -397,15 +475,34 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re if err != nil { return Result{}, err } - e.Log("info", fmt.Sprintf("syncing %q with %q (%s)", game.Name, peer.Name, - map[bool]string{true: "WAN relay", false: "direct LAN"}[peer.Wan()])) - - // 1. Fetch remote manifest + branch info. isFile, _ := delta.ResolveLocalSaveFilePath(game.SavePath) - remoteData, err := e.Transport.FetchManifest(ctx, peer, gameID, ManifestQuery{ + query := ManifestQuery{ Name: game.Name, SavePath: game.SavePath, IsFile: isFile, AppID: game.AppID, CoverURL: game.CoverURL, - }) + } + + // 0. Nothing changed on either side since they last agreed: say so and + // stop, without moving either manifest. Quietly, too — this is the + // answer for nearly every game on every periodic pass. + res, done, prefetched := e.quickInSync(ctx, gameID, game, peer, query) + if done { + return res, nil + } + + // Said once there is something to do (after the in-sync check below), + // not here: a line per game per peer per periodic pass, most of them + // followed by "already in sync", rotated everything else out of the log + // within hours. + syncingLine := fmt.Sprintf("syncing %q with %q (%s)", game.Name, peer.Name, + map[bool]string{true: "WAN relay", false: "direct LAN"}[peer.Wan()]) + + // 1. Fetch remote manifest + branch info. + var remoteData ManifestResponse + if prefetched != nil { + remoteData = *prefetched + } else { + remoteData, err = e.Transport.FetchManifest(ctx, peer, gameID, query) + } if err != nil { // The peer simply isn't tracking this game (they untracked it, or // never had it). That's a stable state, not a transient network @@ -421,6 +518,12 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re // The peer is still writing this game for a sync of its own, and // would otherwise have described a save half-way between two states // (settle.go). SyncGame asks again once it has finished. + // Being played there: its save comes when the session ends, sent by + // that device. Before isSettling, whose message this one contains. + if isPlaying(err) { + e.logOnce(gameID+"|playing|"+peer.ID, fmt.Sprintf("%s is playing %q right now; its save follows when the session ends", peer.Name, game.Name)) + return Result{Status: "peer_playing", PeerID: peer.ID, PeerName: peer.Name}, nil + } if isSettling(err) { e.Log("info", fmt.Sprintf("%s is still applying a sync of %q; asking again when it has finished", peer.Name, game.Name)) return Result{Status: "peer_busy", PeerID: peer.ID, PeerName: peer.Name}, nil @@ -436,6 +539,19 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re return Result{}, fmt.Errorf("fetch remote manifest: %w", err) } + // 1b. A peer whose OpenSave keeps no save versions decides by the old file + // comparison: whatever its copy lacks, it reads as deleted, and a copy that + // arrived half-way spreads its gaps that way. Nothing is taken from it + // until it is updated — it can still take from here, and its deletions are + // refused where they arrive (handleDeleteFile). + e.notePeerApp(peer.ID, remoteData.KeepsVersions()) + if game.AutoSync && !remoteData.KeepsVersions() { + e.logOnce(gameID+"|"+peer.ID, fmt.Sprintf( + "%q: %s runs an OpenSave without save versions — nothing is taken from it until it is updated", + game.Name, peer.Name)) + return Result{Status: StatusPeerOutdatedApp, PeerID: peer.ID, PeerName: peer.Name}, nil + } + // 2. Branch alignment: local follows the remote's active branch. // // From the switch until the peer's save has been pulled into the branch, @@ -495,18 +611,48 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re // would propagate a deletion of the very file being protected. ignoreRules := e.rulesFor(gameID) unfilteredLocal, unfilteredRemote := localManifest, remoteData.Manifest - if !ignoreRules.Empty() { - localManifest = filterManifest(localManifest, ignoreRules) - remoteData.Manifest = filterManifest(remoteData.Manifest, ignoreRules) + if e.ObserveSettings != nil && len(unfilteredRemote.Files) > 0 { + e.ObserveSettings(game, peer.ID, unfilteredRemote) } + localManifest = filterManifest(localManifest, ignoreRules) + remoteData.Manifest = filterManifest(remoteData.Manifest, ignoreRules) - // 3. Existing unresolved conflict blocks further syncing. + // 3. Existing unresolved conflict blocks further syncing — unless a + // version newer than both sides of it has turned up since (version.go): + // it was settled elsewhere, by an answer or "Use this save everywhere", + // and the question here no longer stands. e.mu.Lock() - if existing := e.activeConflicts[gameID]; existing != nil { - e.mu.Unlock() - return Result{Status: "conflict", PeerID: peer.ID, PeerName: peer.Name}, nil - } + existing := e.activeConflicts[gameID] e.mu.Unlock() + if existing != nil { + if remoteData.Version == nil || !game.AutoSync || !e.supersedes(gameID, *remoteData.Version) { + return Result{Status: "conflict", PeerID: peer.ID, PeerName: peer.Name}, nil + } + e.Log("info", fmt.Sprintf("%q: %s holds a version newer than both sides of the open conflict — taking it instead of asking", + game.Name, peer.Name)) + e.clearConflict(gameID) + } + + // 3b. The versions decide, when both devices keep them (version.go): who + // is behind takes the newer save whole, from a device that holds it, and + // nothing is ever taken from — or deleted on the word of — a device that + // is behind. Left to the file comparison below only when the two hold + // the same save, or both saves predate versions. + // + // Not while this device is fetching back a save someone emptied here and + // asked to have put back (hold.go): the emptying was recorded as a version + // of its own, and by versions this device would be the newer one, with + // nothing to fetch. The hold's own rules bring the files back. + fetchingBack := false + if h, ok, err := e.Store.GetDeletionHold(gameID); err == nil && ok && h.State == store.HoldFetching { + fetchingBack = true + } + if remoteData.Version != nil && game.AutoSync && !fetchingBack { + if res, handled, err := e.syncByVersion(ctx, game, peer, localManifest, unfilteredLocal, remoteData); handled { + releaseAligned() + return res, err + } + } // 4. Conflict detection (lineage + skew-tolerant mtimes). lastSyncMs := e.lastSyncTimeMs(peer.ID) @@ -582,7 +728,7 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re // side's UNFILTERED state still hashes to the base then that side has not // changed since, whichever view the base was written in, and its filtered // hash is the same fact expressed in today's terms. - if !ignoreRules.Empty() && agreedHash != "" { + if agreedHash != "" { switch agreedHash { case unfilteredLocal.ManifestHash(): agreedHash = localManifest.ManifestHash() @@ -607,14 +753,14 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re // path is still recorded as shared. Leave it there and the decision reads // "we both had this and now I do not" — and propagates a deletion of the // file the rule exists to protect. - if rules := e.rulesFor(gameID); !rules.Empty() { - lineageFiles = filterLineage(lineageFiles, rules) - lineageDirs = filterLineage(lineageDirs, rules) - } + rules := ignoreRules + lineageFiles = filterLineage(lineageFiles, rules) + lineageDirs = filterLineage(lineageDirs, rules) // A side that is merely behind the other has not diverged from it, whatever // the clocks and the base say (OnlyBehind). - if DetectConflict(judged, remoteData.Manifest, lastSyncMs, agreedHash) && + // Not while being put back: the other device's save is what is wanted. + if !fetchingBack && DetectConflict(judged, remoteData.Manifest, lastSyncMs, agreedHash) && !OnlyBehind(judged, remoteData.Manifest, lineageFiles) { e.registerConflict(gameID, peer, localManifest, remoteData) return Result{Status: "conflict", PeerID: peer.ID, PeerName: peer.Name}, nil @@ -634,17 +780,17 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re // Excluded paths are filtered out for the same reason the lineage is: a // rule change must never be able to reach across and remove a peer's file. deleted := e.recordedDeletions(gameID, delta.PrimaryRoot) - if rules := e.rulesFor(gameID); !rules.Empty() { - for path := range deleted { - if rules.Match(path) { - delete(deleted, path) - } + for path := range deleted { + if excluded(rules, path) { + delete(deleted, path) } } decision := ComputeWithDeletions(localManifest, remoteData.Manifest, lineageFiles, lineageDirs, agreedHash, deleted) + if fetchingBack { + decision = putBackDecision(decision, remoteData.Manifest) + } if !decision.HasChanges() { - e.Log("success", fmt.Sprintf("%q already in sync with %q", game.Name, peer.Name)) e.persistLineage(gameID, peer.ID, localManifest, remoteData.Manifest) // Both sides verifiably identical: this is a convergence point. _ = e.Store.SetAgreedHash(gameID, peer.ID, localManifest.ManifestHash()) @@ -663,8 +809,23 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re // are read through the gate — so the branch hold goes first. releaseAligned() e.syncExtraRoots(ctx, gameID, game, peer, remoteData) + // Share the peer's latest history entry here too. A pull does this; + // a sync that found both sides already identical used to skip it, + // so a game whose saves were identical everywhere from the start + // (Steam Cloud, a copied folder) showed a snapshot on the device + // that tracked it and none on any other. The local files ARE the + // peer's at this point, so the mirror describes the same content. + // Once per peer snapshot: recordMirrorSnapshot skips an id it has. + // + // Last, after the other locations have synced: it zips every location, + // and taking that time before them delayed their sync for nothing. + if remoteData.LatestSnapshot != nil && len(localManifest.Files) > 0 && !e.hasSnapshot(gameID, game) { + e.recordMirrorSnapshot(gameID, game, peer, *remoteData.LatestSnapshot, + fmt.Sprintf("Synced from peer: %s (%s)", peer.Name, remoteData.LatestSnapshot.Comment)) + } return Result{Status: "in_sync", Direction: "none"}, nil } + e.Log("info", syncingLine) if emptiedUnconfirmed(remoteData.Manifest.Files, decision, remoteData.DeletionConfirmed) { e.Log("info", fmt.Sprintf("%q holds none of %q's save files now, and has not confirmed deleting them — keeping this device's copies", @@ -897,10 +1058,9 @@ func (e *Engine) persistLineage(gameID, peerID string, local, remote delta.Manif // RefreshLineage rebuilds this from unfiltered manifests, so without a // filter here an excluded file would be written back in — and the next // sync would read it as shared, then as deleted, and propagate that. - if rules := e.rulesFor(gameID); !rules.Empty() { - files = filterPathList(files, rules) - dirs = filterPathList(dirs, rules) - } + rules := e.rulesFor(gameID) + files = filterPathList(files, rules) + dirs = filterPathList(dirs, rules) if err := e.Store.SetSyncState(gameID, peerID, files, dirs); err != nil { e.Log("warn", fmt.Sprintf("persist sync lineage failed: %v", err)) } @@ -1032,9 +1192,7 @@ func (e *Engine) AddConfirmedLineageForRoot(gameID, peerID, root string, files [ // Excluded paths must not enter the record of what both sides hold, for // the same reason persistLineage filters them: the next sync would read an // excluded file as shared, then as deleted, and propagate that. - if rules := e.rulesFor(gameID); !rules.Empty() { - merged = filterPathList(merged, rules) - } + merged = filterPathList(merged, e.rulesFor(gameID)) sort.Strings(merged) if err := e.Store.SetSyncStateForRoot(gameID, peerID, root, merged, existingDirs); err != nil { e.Log("warn", fmt.Sprintf("recording confirmed lineage failed: %v", err)) @@ -1260,6 +1418,10 @@ func (e *Engine) registerConflict(gameID string, peer Peer, localManifest delta. // Capture comparison data while we hold both manifests, so the UI can // show which side is further along and exactly what differs. diffs := diffManifests(localManifest, remoteData.Manifest) + counts := map[string]int{} + for _, d := range diffs { + counts[d.Status]++ + } const maxDiffFiles = 100 total := len(diffs) if len(diffs) > maxDiffFiles { @@ -1269,10 +1431,12 @@ func (e *Engine) registerConflict(gameID string, peer Peer, localManifest delta. e.mu.Lock() e.activeConflicts[gameID] = &Conflict{ Peer: peer, LocalSnap: localSnap, RemoteSnap: remoteSnap, - LocalStats: manifestStats(localManifest), - RemoteStats: manifestStats(remoteData.Manifest), - DiffFiles: diffs, - DiffTotal: total, + LocalStats: manifestStats(localManifest), + RemoteStats: manifestStats(remoteData.Manifest), + DiffFiles: diffs, + DiffTotal: total, + OnlyLocalTotal: counts["only-local"], OnlyRemoteTotal: counts["only-remote"], ChangedTotal: counts["changed"], + remoteVersion: remoteData.Version, } e.mu.Unlock() @@ -1371,6 +1535,7 @@ func (e *Engine) applyLocalDeletions(gameID string, root syncRoot, d Decision) { // spelling this disk actually holds. full := delta.LocalNameFor(root.Path, relPath) _ = os.Chmod(full, 0o666) + owntouch.MarkRemoved(full) if err := os.Remove(full); err == nil { e.Log("info", "deleted locally (peer deleted): "+relPath) } @@ -1385,6 +1550,7 @@ func (e *Engine) applyLocalDeletions(gameID string, root syncRoot, d Decision) { } full := delta.LocalNameFor(root.Path, relDir) if info, err := os.Stat(full); err == nil && info.IsDir() { + owntouch.MarkRemoved(full) if err := os.Remove(full); err == nil { // only removes empty dirs, matching rmdirSync e.Log("info", "deleted directory locally (peer deleted): "+relDir) } @@ -1427,7 +1593,9 @@ func (e *Engine) createPulledDirsIn(gameID string, root syncRoot, dirsToPull []s if !delta.IsSafePath(root.Path, relDir) { continue } - _ = os.MkdirAll(filepath.Join(root.Path, filepath.FromSlash(relDir)), 0o777) + full := filepath.Join(root.Path, filepath.FromSlash(relDir)) + owntouch.Mark(full) + _ = os.MkdirAll(full, 0o777) } } @@ -1475,7 +1643,11 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s // Make sure every remote directory exists before patching into it. for _, dir := range remoteData.Manifest.Dirs { if delta.IsSafePath(root.Path, dir) { - _ = os.MkdirAll(filepath.Join(root.Path, filepath.FromSlash(dir)), 0o777) + full := filepath.Join(root.Path, filepath.FromSlash(dir)) + if _, err := os.Stat(full); err != nil { + owntouch.Mark(full) + _ = os.MkdirAll(full, 0o777) + } } } @@ -1523,6 +1695,24 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s tracker := newProgressTracker(totalBytes) throttle := e.throttleFor(peer.Wan()) + // What this transfer says about the connection (linkspeed.go): every + // measureWindow while it runs — a pull of a large save takes an hour over a + // slow link, and one measurement at its end left the link unmeasured all + // that time — and what is left when it ends, however it ends: a pull cut + // off half-way still moved what it moved. + const measureWindow = 30 * time.Second + var measureMu sync.Mutex + measuredAt, measuredBytes := time.Now(), int64(0) + measure := func(final bool) { + moved, _, _ := tracker.stats() + measureMu.Lock() + defer measureMu.Unlock() + if d := time.Since(measuredAt); final || d >= measureWindow { + e.noteLinkRate(peer, moved-measuredBytes, d) + measuredAt, measuredBytes = time.Now(), moved + } + } + defer measure(true) // Progress reporter shared by the per-file loop and the block-group // loop inside each file. Without in-file reporting, a single large @@ -1539,6 +1729,7 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s } lastReport = time.Now() reportMu.Unlock() + measure(false) bytesPulled, speed, pct := tracker.stats() ev := ProgressEvent{PeerName: peer.Name, BytesTransferred: bytesPulled, TotalBytes: totalBytes, SpeedBytesPerSec: speed, Percentage: pct} @@ -1561,6 +1752,11 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s // can record the lineage from confirmed fact rather than rediscovering it // with a manifest round trip. See the sync-complete event below. var pulled []string + // Small files are collected here and fetched many to a request once the + // per-file checks below have passed; see pullBatched. + batcher, canBatch := e.Transport.(BatchFetcher) + canBatch = canBatch && !peer.Wan() && remoteData.Proto >= ProtoBatchFiles + var batched []batchJob for _, relPath := range filesToPull { if !delta.IsSafePath(root.Path, relPath) { return fmt.Errorf("path traversal attempt on pulled file %s", relPath) @@ -1608,13 +1804,21 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s if isFile, _ := delta.ResolveLocalSaveFilePath(root.Path); isFile { localFilePath = root.Path // single-file save mode } + if canBatch && len(indices) > 0 && changedBytes(remoteFile, indices) <= batchFileMaxBytes { + batched = append(batched, batchJob{relPath: relPath, localPath: localFilePath, remote: remoteFile, indices: indices}) + continue + } if err := e.pullFile(ctx, peer, FileRef{GameID: gameID, Root: root.Name, RelPath: relPath}, localFilePath, remoteFile, indices, throttle, tracker, reportProgress); err != nil { return err } if remoteFile.MtimeMs > 0 { mtime := time.UnixMilli(int64(remoteFile.MtimeMs)) + // Marked again first: until Settled below, the time the rename left + // is not the one to compare (owntouch). + owntouch.Mark(localFilePath) _ = os.Chtimes(localFilePath, mtime, mtime) + owntouch.Settled(localFilePath) } pulled = append(pulled, relPath) e.Log("info", "file updated: "+relPath) @@ -1622,6 +1826,13 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s // File-boundary progress reporting (always fires). reportProgress(true) } + if len(batched) > 0 { + done, err := e.pullBatched(ctx, batcher, peer, gameID, root.Name, batched, throttle, tracker, reportProgress) + pulled = append(pulled, done...) + if err != nil { + return err + } + } // Said once, plainly, at the end. A sync that quietly leaves files behind // is worse than one that fails: the save looks synced and is not, and the @@ -1822,11 +2033,37 @@ func fetchWithRetry(ctx context.Context, t Transport, peer Peer, ref FileRef, return nil, lastErr } +// hasSnapshot reports whether the game has any snapshot on its active branch +// here. +// +// A sync that finds both sides already identical records the peer's latest +// snapshot only for a game with no history at all — saves identical from the +// start (Steam Cloud, a copied folder) would otherwise never get one. It used +// to record it for every peer it compared with: a full copy of the same save +// per device, again with each new snapshot over there, which for a save of a +// quarter-million files was gigabytes of identical archives. +func (e *Engine) hasSnapshot(gameID string, game store.Game) bool { + if e.Snapshots == nil { + return false + } + _, err := e.Snapshots.LatestSnapshot(gameID, game.ActiveBranch) + return err == nil +} + // recordMirrorSnapshot zips the (just-updated) local save under the peer's // snapshot id so both devices show the same history entry. func (e *Engine) recordMirrorSnapshot(gameID string, game store.Game, peer Peer, remoteSnap SnapshotInfo, comment string) { - if _, err := e.Store.GetSnapshot(remoteSnap.ID); err == nil { - return // already mirrored + if _, known := e.Store.ResolveSnapshotID(remoteSnap.ID); known { + return // already mirrored, or known as the same files as one here + } + // The peer says what its snapshot holds: if a snapshot here holds exactly + // that, it is the same snapshot, and the save is not archived again. + if remoteSnap.ContentHash != "" { + if existing, ok, _ := e.Store.SnapshotByContent(gameID, game.ActiveBranch, remoteSnap.ContentHash); ok && + snapshot.ArchiveExists(existing.ZipPath) { + _ = e.Store.AddSnapshotAlias(remoteSnap.ID, existing.ID) + return + } } settings, err := e.Store.GetSettings() @@ -1851,16 +2088,40 @@ func (e *Engine) recordMirrorSnapshot(gameID string, game store.Game, peer Peer, if rootsErr != nil { mirrorRoots = nil } - if _, err := snapshot.ZipRoots(game.SavePath, mirrorRoots, zipPath); err != nil { + var skipped []string + var captured []store.CapturedFile + if e.Snapshots != nil { + // Unchanged files copied from the newest snapshot, not compressed + // again: seconds rather than minutes for a large save. + skipped, captured, err = e.Snapshots.ArchiveSaveTo(gameID, game.ActiveBranch, zipPath) + } else { + skipped, captured, err = snapshot.ZipRootsCapturing(game.SavePath, mirrorRoots, zipPath) + } + if err != nil { + os.Remove(zipPath) e.Log("warn", fmt.Sprintf("mirror snapshot zip failed: %v", err)) return } + // Incomplete is not kept (snapshot.IncompleteError); the next sync with + // the peer records it again. + if len(skipped) > 0 { + os.Remove(zipPath) + e.Log("info", fmt.Sprintf("snapshot of %q from %s discarded: %d file(s) could not be read; taken again at the next sync", + game.Name, peer.Name, len(skipped))) + return + } + key := snapshot.ContentKey(captured) + if existing, ok, _ := e.Store.SnapshotByContent(gameID, game.ActiveBranch, key); ok && snapshot.ArchiveExists(existing.ZipPath) { + os.Remove(zipPath) + _ = e.Store.AddSnapshotAlias(remoteSnap.ID, existing.ID) + return + } info, err := os.Stat(zipPath) if err != nil { return } - _ = e.Store.CreateSnapshot(store.Snapshot{ + if err := e.Store.CreateSnapshot(store.Snapshot{ ID: remoteSnap.ID, GameID: gameID, BranchName: game.ActiveBranch, @@ -1869,7 +2130,15 @@ func (e *Engine) recordMirrorSnapshot(gameID string, game store.Game, peer Peer, IsSystemAuto: true, ZipPath: zipPath, SizeBytes: info.Size(), - }) + ContentHash: key, + }); err == nil { + _ = e.Store.RecordSnapshotFiles(remoteSnap.ID, captured) + if e.Snapshots != nil { + if snap, gerr := e.Store.GetSnapshot(remoteSnap.ID); gerr == nil { + e.Snapshots.PruneContainedBy(snap, captured) + } + } + } } // humanBytes formats a byte count as a short human-readable string. diff --git a/internal/p2p/syncengine/engine_test.go b/internal/p2p/syncengine/engine_test.go index 1ed654ff..22a2be33 100644 --- a/internal/p2p/syncengine/engine_test.go +++ b/internal/p2p/syncengine/engine_test.go @@ -224,6 +224,59 @@ func TestSync_PullNewRemoteFile(t *testing.T) { } } +// Saves identical on both sides from the start (Steam Cloud, a copied folder) +// never pull anything — and used to never share the peer's history entry +// either, so the game showed a snapshot on one device and none on the other. +func TestSync_InSyncMirrorsPeerSnapshot(t *testing.T) { + env := setupEngine(t) + write(t, env.localDir, "save.dat", "same progress") + write(t, env.remoteDir, "save.dat", "same progress") + env.transport.latestSnap = &SnapshotInfo{ID: "snap_888", Timestamp: "2026-06-01T00:00:00.000Z", Comment: "Initial snapshot"} + + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil { + t.Fatalf("SyncWithPeer error = %v", err) + } + if res.Status != "in_sync" { + t.Fatalf("result = %+v, want in_sync", res) + } + snap, err := env.store.GetSnapshot("snap_888") + if err != nil { + t.Fatalf("peer snapshot not mirrored on an in-sync pass: %v", err) + } + if !strings.Contains(snap.Comment, "Synced from peer: Remote Device") { + t.Errorf("mirror comment = %q", snap.Comment) + } + + // A second in-sync pass must not take it again. + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatal(err) + } + game, err := env.store.GetGame("game1") + if err != nil { + t.Fatal(err) + } + snaps, err := env.store.ListSnapshots("game1", game.ActiveBranch) + if err != nil { + t.Fatal(err) + } + if len(snaps) != 1 { + t.Errorf("snapshots after two in-sync passes = %d, want 1", len(snaps)) + } +} + +// An empty save folder in sync with an empty peer has nothing to mirror. +func TestSync_InSyncEmptyFolderTakesNoMirror(t *testing.T) { + env := setupEngine(t) + env.transport.latestSnap = &SnapshotInfo{ID: "snap_999", Timestamp: "2026-06-01T00:00:00.000Z", Comment: "Initial snapshot"} + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatal(err) + } + if _, err := env.store.GetSnapshot("snap_999"); err == nil { + t.Error("an empty folder was mirrored") + } +} + func TestSync_InsufficientDiskSpace(t *testing.T) { env := setupEngine(t) write(t, env.remoteDir, "save.dat", "remote progress that needs space") diff --git a/internal/p2p/syncengine/hold.go b/internal/p2p/syncengine/hold.go index 7060c241..1e794a8d 100644 --- a/internal/p2p/syncengine/hold.go +++ b/internal/p2p/syncengine/hold.go @@ -85,6 +85,11 @@ type emptiness struct { func (x emptiness) back(held map[string][]string) bool { for name, files := range held { for _, f := range files { + if delta.NeverSynced(f) { + // Recorded by a build that still counted it; never part of + // the save, so not something that has to come back. + continue + } if _, ok := x.present[name][f]; !ok { return false } @@ -295,6 +300,11 @@ func (e *Engine) CheckHold(gameID string, serving bool) (held bool, err error) { case has: if x.back(hold.Files()) { _ = e.Store.ClearDeletionHold(gameID) + // The save is whole again: kept as it is now (OnPutBack), the + // first snapshot of it since it was emptied. + if e.OnPutBack != nil { + go e.OnPutBack(gameID) + } if hold.State == store.HoldAsking { e.Log("success", fmt.Sprintf("every save file of %q is back, so nothing is held back any more", game.Name)) } else { @@ -382,8 +392,8 @@ func (e *Engine) PutBack(gameID string) (fetching int, err error) { } for root, files := range hold.Files() { var missing []string + _ = e.Store.ClearDeletedFiles(gameID, root, files) for _, f := range files { - _ = e.Store.ClearDeletedFile(gameID, root, f) if _, ok := x.present[root][f]; !ok { missing = append(missing, f) } @@ -395,6 +405,9 @@ func (e *Engine) PutBack(gameID string) (fetching int, err error) { } if fetching == 0 { err = e.Store.ClearDeletionHold(gameID) + if err == nil && e.OnPutBack != nil { + go e.OnPutBack(gameID) // whole again already + } } else { err = e.Store.SetDeletionHoldState(gameID, store.HoldFetching) } @@ -530,3 +543,37 @@ func describeLocations(names []string) string { } return strings.Join(parts, " and ") } + +// emptyingUnconfirmed reports a save that holds nothing, or is held back +// after being emptied, while nobody has said the emptying was meant. That is +// not a version of the save: named one, it made this device the newest, so a +// save put back from a snapshot and the other devices came out as this +// device's own new version, which every other device was then asked to take. +// Only "delete them there too" (ConfirmHold) makes it one. +func (e *Engine) emptyingUnconfirmed(gameID string, local delta.Manifest) bool { + hold, has, err := e.Store.GetDeletionHold(gameID) + if err == nil && has { + return hold.State != store.HoldConfirmed + } + for p := range local.Files { + if !delta.NeverSynced(p) { + return false + } + } + return len(local.Extra) == 0 +} + +// putBackDecision turns a decision taken while a save is being put back +// (HoldFetching) into fetching only: every file both devices hold but differ +// on is the other device's, nothing is offered from here, and nothing is +// deleted over there. A file only this device holds is left where it is. +func putBackDecision(d Decision, remote delta.Manifest) Decision { + for _, p := range d.FilesToPush { + if _, ok := remote.Files[p]; ok { + d.FilesToPull = append(d.FilesToPull, p) + } + } + d.FilesToPush, d.DirsToPush = nil, nil + d.FilesToDeleteOnPeer, d.DirsToDeleteOnPeer = nil, nil + return d +} diff --git a/internal/p2p/syncengine/ignore.go b/internal/p2p/syncengine/ignore.go index 96901a0e..dad3e7f0 100644 --- a/internal/p2p/syncengine/ignore.go +++ b/internal/p2p/syncengine/ignore.go @@ -2,6 +2,7 @@ package syncengine import ( "context" + "strings" "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/ignore" @@ -39,15 +40,85 @@ func (e *Engine) rulesFor(gameID string) ignore.Rules { if err != nil { return ignore.Rules{} } - return ignore.Parse(game.SyncIgnore) + return ignore.Parse(e.IgnoreText(game)) +} + +// IgnoreText is the exclusion list a game is synced under: the rules written +// for it, and the files the game database names as its settings +// (DeviceSettingsFor) unless it is set to sync those too. Everything that +// applies a game's rules takes them from here, so the two can never be +// applied by one part of the app and not another. +func (e *Engine) IgnoreText(game store.Game) string { + if game.SyncDeviceSettings || e.DeviceSettingsFor == nil { + return game.SyncIgnore + } + settings := e.DeviceSettingsFor(game) + if len(settings) == 0 { + return game.SyncIgnore + } + text := game.SyncIgnore + if text != "" && !strings.HasSuffix(text, "\n") { + text += "\n" + } + // The game's own rules come last, so a "!" line there can still bring a + // settings file back into syncing. + return "# device settings (game database)\n" + strings.Join(settings, "\n") + "\n" + text +} + +// excluded says whether a path takes no part in syncing: a file no save is +// made of on any device (delta.NeverSynced), or one the game's rules exclude. +// The first is handled exactly like the second — the manifest served still +// lists it, the decision never sees it — so a device on an older build, which +// still counts it, is never told it was deleted. +func excluded(rules ignore.Rules, p string) bool { + return delta.NeverSynced(p) || (!rules.Empty() && rules.Match(p)) +} + +func holdsNeverSynced(m delta.Manifest) bool { + for p := range m.Files { + if delta.NeverSynced(p) { + return true + } + } + for _, root := range m.Extra { + for p := range root.Files { + if delta.NeverSynced(p) { + return true + } + } + } + return false +} + +func anyNeverSynced(paths []string) bool { + for _, p := range paths { + if delta.NeverSynced(p) { + return true + } + } + return false +} + +func anyNeverSyncedIn(paths map[string]struct{}) bool { + for p := range paths { + if delta.NeverSynced(p) { + return true + } + } + return false } // filterManifest returns a copy with every excluded path removed. The // original is untouched: it is still what gets served to peers. func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest { - if rules.Empty() { + if rules.Empty() && !holdsNeverSynced(m) { return m } + return filterManifestBy(m, func(p string) bool { return excluded(rules, p) }) +} + +// filterManifestBy is filterManifest with what to leave out given directly. +func filterManifestBy(m delta.Manifest, leaveOut func(string) bool) delta.Manifest { out := delta.Manifest{ Timestamp: m.Timestamp, Files: make(map[string]delta.FileEntry, len(m.Files)), @@ -61,7 +132,7 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest { // would be told their saves diverged, over a file neither of them is // syncing. for p, entry := range m.Files { - if rules.Match(p) { + if leaveOut(p) { continue } out.Files[p] = entry @@ -70,7 +141,7 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest { } } for _, d := range m.Dirs { - if rules.Match(d) { + if leaveOut(d) { continue } out.Dirs = append(out.Dirs, d) @@ -83,13 +154,13 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest { for name, root := range m.Extra { sub := delta.RootManifest{Files: make(map[string]delta.FileEntry, len(root.Files))} for p, entry := range root.Files { - if rules.Match(p) { + if leaveOut(p) { continue } sub.Files[p] = entry } for _, d := range root.Dirs { - if rules.Match(d) { + if leaveOut(d) { continue } sub.Dirs = append(sub.Dirs, d) @@ -107,12 +178,12 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest { // decision reads "we both had this and now I don't" and propagates a deletion // to the peer. func filterLineage(paths map[string]struct{}, rules ignore.Rules) map[string]struct{} { - if rules.Empty() { + if rules.Empty() && !anyNeverSyncedIn(paths) { return paths } out := make(map[string]struct{}, len(paths)) for p := range paths { - if rules.Match(p) { + if excluded(rules, p) { continue } out[p] = struct{}{} @@ -123,12 +194,12 @@ func filterLineage(paths map[string]struct{}, rules ignore.Rules) map[string]str // filterPathList drops excluded paths from a plain list, for the lineage that // is persisted back after a sync. func filterPathList(paths []string, rules ignore.Rules) []string { - if rules.Empty() { + if rules.Empty() && !anyNeverSynced(paths) { return paths } out := make([]string, 0, len(paths)) for _, p := range paths { - if rules.Match(p) { + if excluded(rules, p) { continue } out = append(out, p) @@ -152,5 +223,5 @@ func (e *Engine) FilteredContentHash(gameID string, game store.Game) string { if err != nil { return "" } - return filterManifest(m, ignore.Parse(game.SyncIgnore)).ManifestHash() + return filterManifest(m, ignore.Parse(e.IgnoreText(game))).ManifestHash() } diff --git a/internal/p2p/syncengine/keeplocal_safety_test.go b/internal/p2p/syncengine/keeplocal_safety_test.go new file mode 100644 index 00000000..064298f5 --- /dev/null +++ b/internal/p2p/syncengine/keeplocal_safety_test.go @@ -0,0 +1,34 @@ +package syncengine + +import ( + "context" + "fmt" + "testing" +) + +// "Keep mine" against a peer holding part of the save, where the peer never +// gets to pull (a pull that times out, a device that goes away). The next +// sync must not read the peer's missing files as deletions and remove them +// from the device whose version the person chose to keep. +func TestKeepLocal_PeerThatNeverPulled_DeletesNothingHere(t *testing.T) { + env := setupEngine(t) + for i := 0; i < 300; i++ { + write(t, env.localDir, fmt.Sprintf("world/map_%04d.bin", i), fmt.Sprintf("chunk %d", i)) + } + for i := 0; i < 100; i++ { + write(t, env.remoteDir, fmt.Sprintf("world/map_%04d.bin", i), fmt.Sprintf("chunk %d", i)) + } + + // The person answers the conflict with "Keep mine". + if err := env.engine.markResolvedLocal(context.Background(), "game1", env.peer); err != nil { + t.Fatal(err) + } + // The peer's pull never happens; its folder stays as it was. + + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatalf("SyncWithPeer: %v", err) + } + if n := countFiles(t, env.localDir); n != 300 { + t.Fatalf("after \"keep mine\" this device went from 300 to %d files — the peer's partial copy was read as deletions", n) + } +} diff --git a/internal/p2p/syncengine/linkspeed.go b/internal/p2p/syncengine/linkspeed.go new file mode 100644 index 00000000..21af61f7 --- /dev/null +++ b/internal/p2p/syncengine/linkspeed.go @@ -0,0 +1,297 @@ +package syncengine + +import ( + "context" + "fmt" + "net" + "sort" + "time" + + "github.com/opensave/opensave/internal/store" +) + +// Network-aware syncing. +// +// Devices are not equally close. Two on the same home network move a save at +// tens of MB a second; one reached over a VPN relay or the internet relay may +// manage a hundred KB. A save that goes to every device at once spends hours +// crawling over the slow links while the device next door, which could have +// had it in seconds and then handed it on, waits in the same queue. +// +// So each device keeps how fast it has found its connection to every other — +// measured from the transfers it makes, and from a short speed test when it +// has nothing recent — and works fastest first: a sync goes to the close +// devices first, and a device that is behind takes the newer save from the +// fastest device that holds it. Before anything is measured, the kind of +// address stands in: a home-network address is taken for fast, a VPN address +// (Tailscale's range) for slower, the internet relay for slowest. + +// LinkStat is what this device knows about its connection to a peer. +type LinkStat struct { + // BytesPerSec is the measured rate, 0 when nothing is measured yet. + BytesPerSec float64 `json:"bytesPerSec"` + MeasuredMs int64 `json:"measuredMs"` + // Kind is what the address says: "lan", "vpn", "relay" or "internet". + Kind string `json:"kind"` +} + +const ( + // minMeasureBytes and minMeasureTime: smaller transfers say more about + // round trips than about the link, and are not counted. + minMeasureBytes = 512 << 10 + minMeasureTime = 500 * time.Millisecond + // probeEvery is how often a link is tested again when no transfer has + // measured it since; probeBytes how much a test moves. + probeEvery = 6 * time.Hour + probeBytes = 2 << 20 + // probeRetry is how soon a speed test that failed is tried again. + probeRetry = 10 * time.Minute +) + +// prior rates, by address kind, used until a link is measured. +var priorRate = map[string]float64{ + "lan": 40 << 20, + "vpn": 2 << 20, + "internet": 1 << 20, + "relay": 256 << 10, +} + +var ( + _, tailscaleNet, _ = net.ParseCIDR("100.64.0.0/10") + privateNets = func() []*net.IPNet { + var out []*net.IPNet + for _, c := range []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.0.0/16", "fc00::/7", "fe80::/10"} { + _, n, _ := net.ParseCIDR(c) + out = append(out, n) + } + return out + }() +) + +// linkKind says what a peer's address is. +func linkKind(peer Peer) string { + if peer.Wan() { + return "relay" + } + ip := net.ParseIP(peer.Address) + if ip == nil { + return "internet" + } + if ip.IsLoopback() { + return "lan" + } + if tailscaleNet.Contains(ip) { + return "vpn" + } + for _, n := range privateNets { + if n.Contains(ip) { + return "lan" + } + } + return "internet" +} + +func (e *Engine) loadLinksLocked() { + if e.links != nil { + return + } + e.links = map[string]store.PeerLink{} + if e.Store == nil { + return + } + if links, err := e.Store.PeerLinks(); err == nil { + e.links = links + } +} + +// Link is what this device knows about its connection to a peer. +func (e *Engine) Link(peer Peer) LinkStat { + e.linkMu.Lock() + defer e.linkMu.Unlock() + e.loadLinksLocked() + l := e.links[peer.ID] + return LinkStat{BytesPerSec: l.BytesPerSec, MeasuredMs: l.MeasuredMs, Kind: linkKind(peer)} +} + +// expectedRate is how fast a transfer with a peer is expected to go: what was +// measured, or the address kind's guess. +func (e *Engine) expectedRate(peer Peer) float64 { + l := e.Link(peer) + if l.BytesPerSec > 0 { + return l.BytesPerSec + } + return priorRate[l.Kind] +} + +// noteLinkRate records a transfer with a peer: bytes in d. +func (e *Engine) noteLinkRate(peer Peer, bytes int64, d time.Duration) { + e.recordLinkRate(peer, bytes, d, minMeasureTime) +} + +// recordLinkRate records bytes in d, when d is at least minTime. +func (e *Engine) recordLinkRate(peer Peer, bytes int64, d, minTime time.Duration) { + if bytes < minMeasureBytes || d < minTime || d <= 0 { + return + } + rate := float64(bytes) / d.Seconds() + e.linkMu.Lock() + defer e.linkMu.Unlock() + e.loadLinksLocked() + l := e.links[peer.ID] + l.PeerID = peer.ID + if l.BytesPerSec > 0 { + // A running average, weighted to the latest: a link changes (a VPN + // that found a direct path, a laptop that moved networks), and the + // order should follow it within a transfer or two. + l.BytesPerSec = 0.4*l.BytesPerSec + 0.6*rate + } else { + l.BytesPerSec = rate + } + l.MeasuredMs = time.Now().UnixMilli() + e.links[peer.ID] = l + if e.Store != nil { + _ = e.Store.SavePeerLink(l) + } +} + +// OrderBySpeed sorts peers fastest first, keeping the given order among +// equals. +func (e *Engine) OrderBySpeed(peers []Peer) []Peer { + out := append([]Peer(nil), peers...) + rates := make(map[string]float64, len(out)) + for _, p := range out { + rates[p.ID] = e.expectedRate(p) + } + sort.SliceStable(out, func(i, j int) bool { return rates[out[i].ID] > rates[out[j].ID] }) + return out +} + +// SpeedProber is implemented by transports that can test a link: fetch n +// bytes of nothing in particular from the peer and say how long it took. +type SpeedProber interface { + ProbeSpeed(ctx context.Context, peer Peer, n int) (int64, time.Duration, error) +} + +// ProbeLinkIfDue tests the link to a peer when nothing has measured it for a +// while. Not over the internet relay, which is the slowest link whatever a +// test says and would only be loaded further. Reports whether it ran. +func (e *Engine) ProbeLinkIfDue(ctx context.Context, peer Peer) bool { + prober, ok := e.Transport.(SpeedProber) + if !ok || peer.Wan() { + return false + } + e.linkMu.Lock() + e.loadLinksLocked() + l := e.links[peer.ID] + now := time.Now() + // A link never measured is tried again soon after a test that failed; + // one measured before only when that has gone stale. + retryAfter := probeEvery + if l.BytesPerSec == 0 { + retryAfter = probeRetry + } + due := now.Sub(time.UnixMilli(l.MeasuredMs)) > probeEvery && now.Sub(time.UnixMilli(l.ProbedMs)) > retryAfter + if due { + l.PeerID = peer.ID + l.ProbedMs = now.UnixMilli() + e.links[peer.ID] = l + if e.Store != nil { + _ = e.Store.SavePeerLink(l) + } + } + e.linkMu.Unlock() + if !due { + return false + } + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + n, d, err := prober.ProbeSpeed(ctx, peer, probeBytes) + if err == nil && d < minMeasureTime { + // Over a home network the test is over before it measures much: + // once more, larger, for a figure worth having. + n, d, err = prober.ProbeSpeed(ctx, peer, 4*probeBytes) + } + if err != nil { + // A link never measured is tried again after probeRetry (see due + // above): the peer may simply not have been ready — or not yet on a + // build that answers a speed test, which once left every link + // unmeasured for hours. + e.Log("info", fmt.Sprintf("speed test with %s did not finish: %v", peer.Name, err)) + return true + } + // A test is a measurement however short: it moved nothing but the test, + // so no round trips of a sync's own are in it. + e.recordLinkRate(peer, n, d, time.Millisecond) + e.Log("info", fmt.Sprintf("connection to %s: %s/s (%s)", peer.Name, humanBytes(int64(e.Link(peer).BytesPerSec)), linkKind(peer))) + return true +} + +// Waiting for close devices before slow ones. + +// fastLink is the rate from which a device counts as close, and slowerBy how +// much slower the next device must be for this one to be waited for first. +const ( + fastLink = 4 << 20 + slowerBy = 4.0 + maxWaitFor = 20 * time.Minute +) + +// NotePeerPulled is told that a peer finished (or gave up) pulling a game from +// this device, and lets a sync waiting for it go on. +func (e *Engine) NotePeerPulled(gameID, peerID string) { + e.linkMu.Lock() + ch := e.pullWaits[gameID+"|"+peerID] + delete(e.pullWaits, gameID+"|"+peerID) + e.linkMu.Unlock() + if ch != nil { + close(ch) + } +} + +// waitForPull waits, after asking a close peer to take this device's save, +// until it has — before the slow devices are asked, so the save reaches the +// close one at full speed and the slow ones can then take it from whichever +// device is closest to them. Bounded by how long the transfer should take, +// and given up when the context ends. +func (e *Engine) waitForPull(ctx context.Context, gameID string, peer Peer, bytes int64) { + ch := make(chan struct{}) + e.linkMu.Lock() + if e.pullWaits == nil { + e.pullWaits = map[string]chan struct{}{} + } + e.pullWaits[gameID+"|"+peer.ID] = ch + e.linkMu.Unlock() + defer func() { + e.linkMu.Lock() + if e.pullWaits[gameID+"|"+peer.ID] == ch { + delete(e.pullWaits, gameID+"|"+peer.ID) + } + e.linkMu.Unlock() + }() + wait := 30*time.Second + time.Duration(3*float64(bytes)/e.expectedRate(peer)*float64(time.Second)) + if wait > maxWaitFor { + wait = maxWaitFor + } + select { + case <-ch: + case <-time.After(wait): + e.Log("info", fmt.Sprintf("%s has not finished taking %s yet; going on with the other devices", peer.Name, gameID)) + case <-ctx.Done(): + } +} + +// waitsForClose reports whether, having asked peers[i] to take this device's +// save, the sync should wait for it before going on: it is a close device, +// and a much slower one is still to come. +func (e *Engine) waitsForClose(peers []Peer, i int) bool { + r := e.expectedRate(peers[i]) + if r < fastLink { + return false + } + for _, p := range peers[i+1:] { + if e.expectedRate(p)*slowerBy <= r { + return true + } + } + return false +} diff --git a/internal/p2p/syncengine/linkspeed_test.go b/internal/p2p/syncengine/linkspeed_test.go new file mode 100644 index 00000000..968dcc87 --- /dev/null +++ b/internal/p2p/syncengine/linkspeed_test.go @@ -0,0 +1,175 @@ +package syncengine + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestLinkKind(t *testing.T) { + cases := map[string]string{ + "192.168.178.20": "lan", + "10.0.0.5": "lan", + "127.0.0.1": "lan", + "100.99.121.82": "vpn", + "203.0.113.9": "internet", + } + for addr, want := range cases { + if got := linkKind(Peer{Address: addr}); got != want { + t.Errorf("%s: %s, want %s", addr, got, want) + } + } + if got := linkKind(Peer{Address: "relay"}); got != "relay" { + t.Errorf("relay: %s", got) + } +} + +// Before anything is measured the kind of address orders the devices; once a +// link is measured, the measurement does. +func TestOrderBySpeed(t *testing.T) { + env := setupEngine(t) + e := env.engine + home := Peer{ID: "home", Address: "192.168.178.20"} + vpn := Peer{ID: "vpn", Address: "100.99.121.82"} + relay := Peer{ID: "relay", Address: "relay"} + + got := e.OrderBySpeed([]Peer{relay, vpn, home}) + if got[0].ID != "home" || got[1].ID != "vpn" || got[2].ID != "relay" { + t.Errorf("by address: %v", ids(got)) + } + + // A VPN link that turns out fast, and a home one that turns out slow. + e.noteLinkRate(vpn, 200<<20, 2*time.Second) + e.noteLinkRate(home, 1<<20, 10*time.Second) + got = e.OrderBySpeed([]Peer{home, vpn, relay}) + if got[0].ID != "vpn" { + t.Errorf("by measurement: %v", ids(got)) + } + // Remembered across restarts. + e2 := New(env.store, nil, env.transport) + if e2.Link(vpn).BytesPerSec == 0 { + t.Error("the measured link was not kept") + } +} + +func TestNoteLinkRate_IgnoresTinyTransfers(t *testing.T) { + env := setupEngine(t) + p := Peer{ID: "p", Address: "192.168.1.2"} + env.engine.noteLinkRate(p, 10<<10, 2*time.Second) + if env.engine.Link(p).BytesPerSec != 0 { + t.Error("a transfer of a few KB was taken as the link's speed") + } +} + +// A close device is waited for before a much slower one, and the wait ends as +// soon as it reports that it has finished. +func TestWaitForClose(t *testing.T) { + env := setupEngine(t) + e := env.engine + home := Peer{ID: "home", Address: "192.168.178.20"} + vpn := Peer{ID: "vpn", Address: "100.99.121.82"} + peers := e.OrderBySpeed([]Peer{vpn, home}) + if !e.waitsForClose(peers, 0) { + t.Fatal("a close device is not waited for before a slow one") + } + if e.waitsForClose(peers, 1) { + t.Error("the last device is waited for, with nothing after it") + } + if e.waitsForClose(e.OrderBySpeed([]Peer{home}), 0) { + t.Error("waited with no slower device to follow") + } + + done := make(chan struct{}) + start := time.Now() + go func() { + e.waitForPull(context.Background(), "game1", home, 1<<30) + close(done) + }() + time.Sleep(100 * time.Millisecond) + e.NotePeerPulled("game1", home.ID) + select { + case <-done: + if time.Since(start) > 5*time.Second { + t.Error("the wait did not end when the device reported") + } + case <-time.After(10 * time.Second): + t.Fatal("the wait did not end when the device reported") + } +} + +type probeTransport struct { + *fakeTransport + fail bool + calls int + // perMB is how long each MB takes; a second when unset. + perMB time.Duration +} + +func (p *probeTransport) ProbeSpeed(ctx context.Context, peer Peer, n int) (int64, time.Duration, error) { + p.calls++ + if p.fail { + return 0, 0, errors.New("peer returned 404") + } + if p.perMB > 0 { + return int64(n), time.Duration(n>>20) * p.perMB, nil + } + return int64(n), time.Second, nil +} + +// A speed test that fails — a peer not yet updated to answer one — is tried +// again after minutes, not hours; a measured link is not tested again soon. +func TestProbeLink_RetriesAFailureSoon(t *testing.T) { + env := setupEngine(t) + pt := &probeTransport{fakeTransport: env.transport, fail: true} + env.engine.Transport = pt + p := Peer{ID: "p", Name: "P", Address: "192.168.1.9"} + + if !env.engine.ProbeLinkIfDue(context.Background(), p) { + t.Fatal("an unmeasured link was not tested") + } + if env.engine.ProbeLinkIfDue(context.Background(), p) { + t.Fatal("tested again straight after a failure") + } + // Eleven minutes later. + env.engine.linkMu.Lock() + l := env.engine.links[p.ID] + l.ProbedMs = time.Now().Add(-11 * time.Minute).UnixMilli() + env.engine.links[p.ID] = l + env.engine.linkMu.Unlock() + pt.fail = false + if !env.engine.ProbeLinkIfDue(context.Background(), p) { + t.Fatal("a failed test was not tried again after ten minutes") + } + if env.engine.Link(p).BytesPerSec == 0 { + t.Fatal("the successful test was not recorded") + } + if env.engine.ProbeLinkIfDue(context.Background(), p) { + t.Error("a freshly measured link was tested again") + } +} + +// A home network finishes the test before it has measured much: it is run +// again larger, and the result is kept — not discarded as too short, which +// left the fastest link unmeasured and sorted behind the slow ones. +func TestProbeLink_AFastLinkIsMeasured(t *testing.T) { + env := setupEngine(t) + pt := &probeTransport{fakeTransport: env.transport, perMB: 20 * time.Millisecond} + env.engine.Transport = pt + p := Peer{ID: "lan", Name: "LAN", Address: "192.168.178.20"} + env.engine.ProbeLinkIfDue(context.Background(), p) + if got := env.engine.Link(p).BytesPerSec; got < 20<<20 { + t.Errorf("a 50 MB/s link measured as %.0f B/s", got) + } + if pt.calls != 2 { + t.Errorf("%d test(s), want the short one and a larger one", pt.calls) + } +} + +func ids(ps []Peer) []string { + out := make([]string, len(ps)) + for i, p := range ps { + out[i] = p.ID + } + return out +} diff --git a/internal/p2p/syncengine/multiroot.go b/internal/p2p/syncengine/multiroot.go index c763a459..544e7e53 100644 --- a/internal/p2p/syncengine/multiroot.go +++ b/internal/p2p/syncengine/multiroot.go @@ -114,24 +114,20 @@ func (e *Engine) syncOneRoot(ctx context.Context, gameID string, game store.Game rules := e.rulesFor(gameID) remote := sr.remote unfilteredLocal, unfilteredRemote := local, remote - if !rules.Empty() { - local = filterManifest(local, rules) - remote = filterManifest(remote, rules) - } + local = filterManifest(local, rules) + remote = filterManifest(remote, rules) fileList, dirList, err := e.Store.GetSyncStateForRoot(gameID, peer.ID, sr.root.Name) if err != nil { return err } lineageFiles, lineageDirs := toSet(fileList), toSet(dirList) - if !rules.Empty() { - // Without this an exclusion added to a location that had already - // synced reads as "we both had this file and now I do not", and the - // deletion travels to the peer — destroying the copy the rule was - // written to protect. - lineageFiles = filterLineage(lineageFiles, rules) - lineageDirs = filterLineage(lineageDirs, rules) - } + // Without this an exclusion added to a location that had already + // synced reads as "we both had this file and now I do not", and the + // deletion travels to the peer — destroying the copy the rule was + // written to protect. + lineageFiles = filterLineage(lineageFiles, rules) + lineageDirs = filterLineage(lineageDirs, rules) decision := Compute(local, remote, lineageFiles, lineageDirs) if !decision.HasChanges() { @@ -172,7 +168,7 @@ func (e *Engine) syncOneRoot(ctx context.Context, gameID string, game store.Game // adds a rule. If a side's UNFILTERED state still hashes to the base then // that side has not changed, and its filtered hash says the same thing in // today's terms. Same translation the main folder does. - if !rules.Empty() && base != "" { + if base != "" { switch base { case unfilteredLocal.RootHash(delta.PrimaryRoot): base = local.RootHash(delta.PrimaryRoot) diff --git a/internal/p2p/syncengine/multiroot_conflict.go b/internal/p2p/syncengine/multiroot_conflict.go index 1fea1ba6..401d69de 100644 --- a/internal/p2p/syncengine/multiroot_conflict.go +++ b/internal/p2p/syncengine/multiroot_conflict.go @@ -140,7 +140,17 @@ func (e *Engine) ResolveRootConflict(ctx context.Context, gameID, peerID, root, if err := e.Store.SetAgreedHashForRoot(gameID, peerID, root, local.RootHash(delta.PrimaryRoot)); err != nil { return err } - files, dirs := IntersectLineage(local, local) + // Shared only what the peer verifiably holds too — see + // markResolvedLocal for what recording the whole local copy as shared + // did once the peer's pull did not finish. + var files, dirs []string + if remoteData, err := e.Transport.FetchManifest(ctx, peer, gameID, ManifestQuery{ + Name: game.Name, SavePath: game.SavePath, AppID: game.AppID, CoverURL: game.CoverURL, + }); err == nil { + if remoteRoot, has := remoteData.Manifest.Extra[root]; has { + files, dirs = IntersectLineage(local, delta.Manifest{Files: remoteRoot.Files, Dirs: remoteRoot.Dirs}) + } + } _ = e.Store.SetSyncStateForRoot(gameID, peerID, root, files, dirs) e.clearRootConflict(gameID, root) e.Transport.TriggerPeerPull(peer, gameID) diff --git a/internal/p2p/syncengine/neversynced_test.go b/internal/p2p/syncengine/neversynced_test.go new file mode 100644 index 00000000..f104089b --- /dev/null +++ b/internal/p2p/syncengine/neversynced_test.go @@ -0,0 +1,137 @@ +package syncengine + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/opensave/opensave/internal/delta" +) + +// Steam's remotecache.vdf is each device's own bookkeeping: never taken from +// a peer, never pushed, never deleted because the peer lacks it. +func TestNeverSynced_LeftAloneOnBothSides(t *testing.T) { + env := setupEngine(t) + write(t, env.localDir, "save.dat", "progress") + write(t, env.localDir, "remotecache.vdf", "this device's steam") + write(t, env.remoteDir, "save.dat", "progress") + write(t, env.remoteDir, "remotecache.vdf", "the other device's steam") + + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil { + t.Fatalf("SyncWithPeer error = %v", err) + } + if res.Status != "in_sync" { + t.Fatalf("status = %q, want in_sync: the saves are the same", res.Status) + } + got, _ := os.ReadFile(filepath.Join(env.localDir, "remotecache.vdf")) + if string(got) != "this device's steam" { + t.Errorf("this device's remotecache.vdf became %q", got) + } + if got, _ := os.ReadFile(filepath.Join(env.remoteDir, "remotecache.vdf")); string(got) != "the other device's steam" { + t.Errorf("the peer's remotecache.vdf became %q", got) + } +} + +// The served manifest keeps listing it: a device on an older build would read +// the gap as the file deleted, and delete its own. +func TestNeverSynced_StillServed(t *testing.T) { + dir := t.TempDir() + write(t, dir, "remotecache.vdf", "steam") + m, err := delta.BuildManifest(dir) + if err != nil { + t.Fatal(err) + } + if _, ok := m.Files["remotecache.vdf"]; !ok { + t.Fatal("the manifest a device serves left remotecache.vdf out") + } +} + +// A build with a new list of such files re-takes the recorded hash quietly +// when the save is the one it was recorded for, and the next change seen is +// not called a new version over it. A save that did change is left to count. +func TestNeverSynced_AdoptingTheViewIsNotAVersion(t *testing.T) { + env := setupEngine(t) + game, err := env.store.GetGame("game1") + if err != nil { + t.Fatal(err) + } + game.AutoSync = true + if err := env.store.UpdateGame(game); err != nil { + t.Fatal(err) + } + write(t, env.localDir, "save.dat", "progress") + write(t, env.localDir, "remotecache.vdf", "steam") + env.engine.NoteLocalChange("game1") // the first version: named by content + + m, err := delta.BuildManifest(env.localDir) + if err != nil { + t.Fatal(err) + } + // As a build before the list recorded it. + rec, err := env.store.GetGameVersion("game1") + if err != nil || rec.Vector == "" { + t.Fatalf("no version recorded: %v", err) + } + vector := rec.Vector + rec.Hash = versionHashBeforeNeverSynced("", m) + if err := env.store.SaveGameVersion(rec); err != nil { + t.Fatal(err) + } + + env.engine.AdoptExclusionView("game1", m, "", true) + env.engine.NoteLocalChange("game1") + rec, _ = env.store.GetGameVersion("game1") + if rec.Vector != vector { + t.Fatalf("version moved from %s to %s over a save nobody changed", vector, rec.Vector) + } + if rec.Hash != env.engine.versionHashOf("game1", m) { + t.Error("the recorded hash was not re-taken") + } + + // Recorded for a save that has since changed: not adopted, so the change + // is a version. + write(t, env.localDir, "save.dat", "more progress") + delta.InvalidateRoot(env.localDir) + changed, _ := delta.BuildManifest(env.localDir) + rec.Hash = versionHashBeforeNeverSynced("", m) + _ = env.store.SaveGameVersion(rec) + env.engine.AdoptExclusionView("game1", changed, "", true) + env.engine.NoteLocalChange("game1") + rec, _ = env.store.GetGameVersion("game1") + if rec.Vector == vector { + t.Error("a real change after the upgrade did not become a version") + } +} + +// Held only by the peer: not taken. Held only here: not pushed, and not +// deleted because the peer lacks it. +func TestNeverSynced_OneSidedIsNotADifference(t *testing.T) { + env := setupEngine(t) + write(t, env.localDir, "save.dat", "progress") + write(t, env.remoteDir, "save.dat", "progress") + write(t, env.remoteDir, "RemoteCache.VDF", "only there") + + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil || res.Status != "in_sync" { + t.Fatalf("status = %q, %v; want in_sync", res.Status, err) + } + if _, err := os.Stat(filepath.Join(env.localDir, "RemoteCache.VDF")); err == nil { + t.Error("the peer's remotecache.vdf was pulled") + } + + os.Remove(filepath.Join(env.remoteDir, "RemoteCache.VDF")) + write(t, env.localDir, "remotecache.vdf", "only here") + delta.InvalidateRoot(env.localDir) + res, err = env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil || res.Status != "in_sync" { + t.Fatalf("status = %q, %v; want in_sync", res.Status, err) + } + if _, err := os.Stat(filepath.Join(env.remoteDir, "remotecache.vdf")); err == nil { + t.Error("this device's remotecache.vdf was pushed") + } + if _, err := os.Stat(filepath.Join(env.localDir, "remotecache.vdf")); err != nil { + t.Error("this device's remotecache.vdf was deleted") + } +} diff --git a/internal/p2p/syncengine/partial_safety_test.go b/internal/p2p/syncengine/partial_safety_test.go new file mode 100644 index 00000000..59588c39 --- /dev/null +++ b/internal/p2p/syncengine/partial_safety_test.go @@ -0,0 +1,113 @@ +package syncengine + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" +) + +// A transfer that stops part-way must never turn into deletions. +// +// Seen in 2.3.1: a device that had received part of a 240k-file save offered +// its partial copy back, the missing files were read as "deleted over there", +// and 23k files were deleted on the device that still had them. 2.4 records +// as shared only what both manifests showed (persistLineage), which should +// rule that out; these hold it there, for the per-file and the batched pull. + +func countFiles(t *testing.T, dir string) int { + t.Helper() + n := 0 + _ = filepath.Walk(dir, func(p string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + n++ + } + return nil + }) + return n +} + +func writeSmallFiles(t *testing.T, dir string, n int) { + t.Helper() + for i := 0; i < n; i++ { + write(t, dir, fmt.Sprintf("world/map_%04d.bin", i), fmt.Sprintf("chunk %d", i)) + } +} + +func interruptedThenResumed(t *testing.T, env *engineEnv, total, stopAfter int) { + t.Helper() + writeSmallFiles(t, env.remoteDir, total) + + ctx, cancel := context.WithCancel(context.Background()) + var fetched atomic.Int64 + env.transport.onFetchBlocks = func() { + if fetched.Add(1) == int64(stopAfter) { + cancel() + } + } + if _, err := env.engine.SyncWithPeer(ctx, "game1", env.peer); err == nil { + t.Fatal("setup: the first sync was meant to be interrupted") + } + env.transport.onFetchBlocks = nil + cancel() + + got := countFiles(t, env.localDir) + if got == 0 || got >= total { + t.Fatalf("setup: after the interruption this side has %d of %d files; want a partial copy", got, total) + } + + // Resume. Nothing may be deleted anywhere, and everything must arrive. + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatalf("resumed sync: %v", err) + } + if len(env.transport.deletedOnPeer) > 0 { + t.Fatalf("the partial copy was propagated as %d deletions on the peer, e.g. %s", + len(env.transport.deletedOnPeer), env.transport.deletedOnPeer[0]) + } + if n := countFiles(t, env.remoteDir); n != total { + t.Fatalf("peer has %d files, want all %d", n, total) + } + if n := countFiles(t, env.localDir); n != total { + t.Fatalf("this side has %d files after resuming, want %d", n, total) + } +} + +func TestPartialPull_InterruptedThenResumed(t *testing.T) { + env := setupEngine(t) + interruptedThenResumed(t, env, 200, 60) +} + +func TestPartialPull_InterruptedThenResumed_Batched(t *testing.T) { + // One batch in flight, so the first batch of 256 is written before the + // interruption lands in the second: a deterministic partial copy. + old := batchWorkers + batchWorkers = 1 + t.Cleanup(func() { batchWorkers = old }) + env, _ := setupBatchEngine(t, ProtoBatchFiles) + // Batches fetch through the same fake reader, one call per file; the + // 300th is inside the second batch. + interruptedThenResumed(t, env, 600, 300) +} + +// The other side of the same situation: this device has everything, the peer +// holds only part of it and never synced. The part it lacks is new to it, not +// deleted by it. +func TestPartialPeer_NeverReadAsDeletions(t *testing.T) { + env := setupEngine(t) + writeSmallFiles(t, env.localDir, 300) + for i := 0; i < 80; i++ { + write(t, env.remoteDir, fmt.Sprintf("world/map_%04d.bin", i), fmt.Sprintf("chunk %d", i)) + } + + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatalf("SyncWithPeer: %v", err) + } + if n := countFiles(t, env.localDir); n != 300 { + t.Fatalf("this side went from 300 to %d files — the peer's partial copy was read as deletions", n) + } + if len(env.transport.deletedOnPeer) > 0 { + t.Fatalf("deleted on peer: %v", env.transport.deletedOnPeer[:1]) + } +} diff --git a/internal/p2p/syncengine/playing.go b/internal/p2p/syncengine/playing.go new file mode 100644 index 00000000..c2abda1c --- /dev/null +++ b/internal/p2p/syncengine/playing.go @@ -0,0 +1,47 @@ +package syncengine + +import "strings" + +// A game being played. +// +// A game autosaves every few minutes, and each autosave is a change of its +// save. Synced as they come, every one of them reached every other device as +// a version of its own, and each of those took a snapshot of it: eleven of +// one game in 26 minutes on a device nobody was sitting at. And this device +// took other devices' versions of a game running here, writing a save under +// a game that was still using it. +// +// So while a game is played here (Playing, from the daemon's sessions), it +// is not synced either way. Another device asking for it is told so, and +// asks again later; nothing is taken for it. When the session ends, the save +// as it was left goes to the other devices in one sync (daemon/sessions.go). + +// PlayingMessage is what a device answers when asked for a game being played +// on it. It contains SettlingMessage, so a device on a build that predates +// this takes it as "busy, ask again" too. +const PlayingMessage = "This device is playing this game right now; its save follows when the session ends. " + SettlingMessage + +const playingMark = "is playing this game right now" + +// isPlaying reports whether an error from a peer is its "playing" answer. +func isPlaying(err error) bool { + return err != nil && strings.Contains(err.Error(), playingMark) +} + +// playingErr is SyncGame declining to sync a game being played here. It +// counts as ErrHeld for callers that only want to know it was not an error. +type playingErr struct{} + +func (playingErr) Error() string { + return "it is being played on this device; it syncs when the session ends" +} + +func (playingErr) Is(target error) bool { return target == ErrHeld || target == ErrPlayingHere } + +// ErrPlayingHere is SyncGame declining to sync a game being played here. +var ErrPlayingHere error = playingErr{} + +// PlayingHere reports whether a game is being played on this device. +func (e *Engine) PlayingHere(gameID string) bool { + return e.Playing != nil && e.Playing(gameID) +} diff --git a/internal/p2p/syncengine/playing_test.go b/internal/p2p/syncengine/playing_test.go new file mode 100644 index 00000000..90112fb3 --- /dev/null +++ b/internal/p2p/syncengine/playing_test.go @@ -0,0 +1,49 @@ +package syncengine + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +// A game being played here is not synced either way until the session ends: +// nothing is taken for it, and callers are not told it failed. +func TestPlaying_NotSyncedWhilePlayedHere(t *testing.T) { + env := setupEngine(t) + write(t, env.localDir, "slot1.sav", "being played") + write(t, env.remoteDir, "slot1.sav", "another device's") + playing := true + env.engine.Playing = func(string) bool { return playing } + + _, err := env.engine.SyncGame(context.Background(), "game1", []Peer{env.peer}) + if !errors.Is(err, ErrPlayingHere) || !errors.Is(err, ErrHeld) { + t.Fatalf("err = %v, want ErrPlayingHere, quiet as ErrHeld", err) + } + if got, _ := os.ReadFile(filepath.Join(env.localDir, "slot1.sav")); string(got) != "being played" { + t.Fatalf("the save of a running game was replaced: %q", got) + } + + playing = false + if _, err := env.engine.SyncGame(context.Background(), "game1", []Peer{env.peer}); err != nil { + t.Fatalf("after the session: %v", err) + } +} + +// Another device playing the game answers "playing": nothing is compared or +// stamped, and asking does not count as a failure. +func TestPlaying_PeerPlayingIsWaitedFor(t *testing.T) { + env := setupEngine(t) + write(t, env.localDir, "slot1.sav", "here") + env.transport.manifestErr = errors.New(PlayingMessage) + + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil || res.Status != "peer_playing" { + t.Fatalf("status = %q, %v; want peer_playing", res.Status, err) + } + // A device on an older build reads the same answer as "busy". + if !isSettling(errors.New(PlayingMessage)) { + t.Error("the playing answer is not recognisable as busy by older builds") + } +} diff --git a/internal/p2p/syncengine/quick.go b/internal/p2p/syncengine/quick.go new file mode 100644 index 00000000..cf0c3906 --- /dev/null +++ b/internal/p2p/syncengine/quick.go @@ -0,0 +1,83 @@ +package syncengine + +import ( + "context" + "fmt" + + "github.com/opensave/opensave/internal/store" +) + +// quickInSync settles the common case of a sync in one small exchange: this +// device's save still hashes to the base both devices last agreed on, and +// the peer, asked whether it does too (ManifestQuery.IfHash), says +// "unchanged". Then there is nothing to decide, and neither manifest needs +// to cross the wire — for a save of a quarter-million files that is tens of +// MB of JSON per peer, which the minute-long reconcile used to fetch every +// minute to learn that nothing had changed. +// +// It only ever answers "in sync" when both sides provably hold the agreed +// state. In every other case it reports not done and the full sync runs; a +// full manifest that came back anyway (a peer that predates the question) +// is handed over in prefetched so it is not fetched twice. +// +// Deliberately narrow: +// - no exclusion rules: the agreed base is taken over the filtered +// manifest, the peer hashes its unfiltered one; +// - no extra save locations: those carry bases of their own (RootHash); +// - no conflict waiting, and the same branch on both sides. +func (e *Engine) quickInSync(ctx context.Context, gameID string, game store.Game, peer Peer, q ManifestQuery) (res Result, done bool, prefetched *ManifestResponse) { + agreed := e.Store.GetAgreedHash(gameID, peer.ID) + if agreed == "" || !e.rulesFor(gameID).Empty() { + return Result{}, false, nil + } + if roots, err := e.Store.GameRootPaths(gameID); err != nil || len(roots) > 0 { + return Result{}, false, nil + } + e.mu.Lock() + conflicted := e.activeConflicts[gameID] != nil + e.mu.Unlock() + if conflicted { + return Result{}, false, nil + } + local, err := e.ReadManifest(ctx, gameID, game.SavePath) + if err != nil || len(local.Extra) > 0 || local.ManifestHash() != agreed { + return Result{}, false, nil + } + + q.IfHash = agreed + resp, err := e.Transport.FetchManifest(ctx, peer, gameID, q) + if err != nil { + // Let the full path fetch again and classify the error as it always has. + return Result{}, false, nil + } + if !resp.Unchanged { + return Result{}, false, &resp + } + if resp.ManifestHash != agreed || (resp.ActiveBranch != "" && resp.ActiveBranch != game.ActiveBranch) { + // Unchanged, but not in a way this shortcut may act on: the full + // exchange decides. + return Result{}, false, nil + } + // The same files; the versions still have to say the same (version.go). + if resp.Version != nil && game.AutoSync { + res, final := e.quickVersionCheck(game, peer, local, *resp.Version) + if !final { + return Result{}, false, nil + } + if res.Status == versionStatusWaiting { + return res, true, nil + } + } + + // Same as the in-sync outcome of a full sync, minus what is already + // recorded: the base is unchanged and the lineage with it. + e.Transport.ReportSyncEvent(peer, gameID, "in-sync", map[string]any{ + "peerName": e.deviceName(), + "manifestHash": agreed, + }) + if resp.LatestSnapshot != nil && len(local.Files) > 0 && !e.hasSnapshot(gameID, game) { + e.recordMirrorSnapshot(gameID, game, peer, *resp.LatestSnapshot, + fmt.Sprintf("Synced from peer: %s (%s)", peer.Name, resp.LatestSnapshot.Comment)) + } + return Result{Status: "in_sync", Direction: "none"}, true, nil +} diff --git a/internal/p2p/syncengine/quick_test.go b/internal/p2p/syncengine/quick_test.go new file mode 100644 index 00000000..3cfdf8d8 --- /dev/null +++ b/internal/p2p/syncengine/quick_test.go @@ -0,0 +1,117 @@ +package syncengine + +import ( + "context" + "testing" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/snapshot" +) + +// ifHashTransport answers IfHash the way a current peer does, and counts +// how many full manifests it had to send. +type ifHashTransport struct { + *fakeTransport + full, unchanged int +} + +func (t *ifHashTransport) FetchManifest(ctx context.Context, peer Peer, gameID string, q ManifestQuery) (ManifestResponse, error) { + resp, err := t.fakeTransport.FetchManifest(ctx, peer, gameID, q) + if err != nil { + return resp, err + } + if q.IfHash != "" && resp.Manifest.ManifestHash() == q.IfHash { + t.unchanged++ + return ManifestResponse{ActiveBranch: resp.ActiveBranch, LatestSnapshot: resp.LatestSnapshot, + Unchanged: true, ManifestHash: q.IfHash, Manifest: delta.Manifest{}}, nil + } + t.full++ + return resp, nil +} + +func setupIfHashEngine(t *testing.T) (*engineEnv, *ifHashTransport) { + t.Helper() + env := setupEngine(t) + tr := &ifHashTransport{fakeTransport: env.transport} + env.engine = New(env.store, snapshot.New(env.store), tr) + return env, tr +} + +func TestQuickInSync_SkipsTheManifestWhenNothingChanged(t *testing.T) { + env, tr := setupIfHashEngine(t) + write(t, env.localDir, "save.dat", "same") + write(t, env.remoteDir, "save.dat", "same") + + // First sync converges and records the agreed base. + if res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil || res.Status != "in_sync" { + t.Fatalf("first sync = %+v, %v", res, err) + } + if tr.full != 1 { + t.Fatalf("setup: full manifests = %d, want 1", tr.full) + } + + // Nothing changed anywhere: settled without a full manifest. + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil || res.Status != "in_sync" { + t.Fatalf("second sync = %+v, %v", res, err) + } + if tr.full != 1 || tr.unchanged != 1 { + t.Errorf("second sync fetched %d full manifests and %d unchanged answers; want 0 more full, 1 unchanged", tr.full-1, tr.unchanged) + } +} + +// A change on either side must go the full way — the shortcut may only ever +// say "in sync" when both sides provably hold the agreed state. +func TestQuickInSync_ChangesStillSync(t *testing.T) { + env, tr := setupIfHashEngine(t) + write(t, env.localDir, "save.dat", "same") + write(t, env.remoteDir, "save.dat", "same") + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatal(err) + } + + // The peer changes. + write(t, env.remoteDir, "new.dat", "made on the other device") + res, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil { + t.Fatal(err) + } + if res.Status != "updated" || res.Direction != "pull" { + t.Fatalf("peer change: result = %+v, want updated/pull", res) + } + + // This side changes. + write(t, env.localDir, "mine.dat", "made here") + before := env.transport.pullTriggers + res, err = env.engine.SyncWithPeer(context.Background(), "game1", env.peer) + if err != nil { + t.Fatal(err) + } + if res.Status == "in_sync" { + t.Fatalf("local change was reported in sync: %+v", res) + } + if env.transport.pullTriggers == before { + t.Error("the peer was not told to pull this side's change") + } + if tr.unchanged != 0 { + t.Errorf("unchanged answers = %d with a change on one side, want 0", tr.unchanged) + } +} + +// A peer that predates the question sends the full manifest; it is used, +// not fetched a second time. +func TestQuickInSync_OldPeerManifestIsNotFetchedTwice(t *testing.T) { + env := setupEngine(t) // plain fake: ignores IfHash + write(t, env.localDir, "save.dat", "same") + write(t, env.remoteDir, "save.dat", "same") + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatal(err) + } + before := env.transport.manifestCalls + if _, err := env.engine.SyncWithPeer(context.Background(), "game1", env.peer); err != nil { + t.Fatal(err) + } + if got := env.transport.manifestCalls - before; got != 1 { + t.Errorf("manifest requests for one sync = %d, want 1", got) + } +} diff --git a/internal/p2p/syncengine/settle.go b/internal/p2p/syncengine/settle.go index 34942e10..a66c1c8d 100644 --- a/internal/p2p/syncengine/settle.go +++ b/internal/p2p/syncengine/settle.go @@ -131,10 +131,44 @@ func (e *Engine) Writing(gameID string) (done func()) { once = true g := e.gateLocked(gameID) g.writers-- + if e.writtenAt == nil { + e.writtenAt = map[string]time.Time{} + } + e.writtenAt[gameID] = time.Now() e.wakeLocked(gameID, g) } } +// writeEcho is how long after a sync stops writing a game its file events may +// still be arriving. +const writeEcho = 10 * time.Second + +// BeingWritten reports whether a sync on this device is writing a game's save +// now, or was a moment ago — what the watcher asks before taking a burst it +// could not attribute (an overflow of file events) for a change of the game's. +// A pull of a save of many files overflows the event queue again and again, +// and each one read as the game saving: an automatic snapshot of a half-pulled +// save every few seconds, which pushed the real ones out of retention. +func (e *Engine) BeingWritten(gameID string) bool { + if e.WritingNow(gameID) { + return true + } + e.settleMu.Lock() + defer e.settleMu.Unlock() + return time.Since(e.writtenAt[gameID]) < writeEcho +} + +// WritingNow reports whether a sync on this device is writing a game's save at +// this moment — what the watcher waits out before looking at a burst, so it +// never judges a save half-way between two states. Not the moments after: +// a change the game makes then is the game's, and its snapshot is not held up. +func (e *Engine) WritingNow(gameID string) bool { + e.settleMu.Lock() + defer e.settleMu.Unlock() + g := e.gates[gameID] + return g != nil && (g.writers > 0 || g.waitingWriters > 0) +} + // Reading holds the game's save still for a sync to read it: it waits until // nothing is writing it, then keeps writers out until done is called. It // returns ErrSettling if ctx ends first. Hold it only for the reading itself — diff --git a/internal/p2p/syncengine/transport.go b/internal/p2p/syncengine/transport.go index c39e315c..c2afbf8d 100644 --- a/internal/p2p/syncengine/transport.go +++ b/internal/p2p/syncengine/transport.go @@ -2,6 +2,7 @@ package syncengine import ( "context" + "errors" "github.com/opensave/opensave/internal/delta" ) @@ -25,12 +26,54 @@ type SnapshotInfo struct { ID string `json:"id"` Timestamp string `json:"timestamp"` Comment string `json:"comment"` + // ContentHash names what the snapshot holds (snapshot.ContentKey): a + // device that already has a snapshot of exactly those files records this + // one's id as an alias of it rather than archiving the save again. + ContentHash string `json:"contentHash,omitempty"` } // ProtoMultiRoot is the protocol revision at which a peer understands games // with more than one save location. const ProtoMultiRoot = 1 +// ProtoBatchFiles is the revision at which a peer serves the blocks of many +// files in one request (FetchFileBatch). Before it, every file cost its own +// round trip — ~200ms each in practice, which for a save of a quarter-million +// small files (Project Zomboid map chunks) meant half a day for 50MB. +const ProtoBatchFiles = 2 + +// ProtoVersions is the revision at which a peer keeps save versions +// (version.go). A peer below it decides by the old file comparison, which +// reads whatever its copy lacks as deleted; nothing is taken from such a peer +// or deleted on its word (OldBuildRefused). +const ProtoVersions = 3 + +// FileBlocksRequest asks for some blocks of one file, as one entry of a batch. +type FileBlocksRequest struct { + RelPath string `json:"relPath"` + BlockIndices []int `json:"blockIndices"` + BlockSize int `json:"blockSize"` +} + +// FileBlocks is one file's answer inside a batch. Error is set, and Blocks +// empty, when the responder could not read that file. +type FileBlocks struct { + RelPath string `json:"relPath"` + Blocks []BlockData `json:"blocks"` + Error string `json:"error,omitempty"` +} + +// BatchFetcher is implemented by transports that can fetch the blocks of +// many files, all in one save location, in a single request. Optional: the +// engine checks for it, and for a peer that advertised ProtoBatchFiles. +type BatchFetcher interface { + FetchFileBatch(ctx context.Context, peer Peer, gameID, root string, files []FileBlocksRequest) ([]FileBlocks, error) +} + +// ErrBatchUnsupported is what a transport returns when it cannot batch for +// this peer (the relay path); the engine then pulls file by file. +var ErrBatchUnsupported = errors.New("batched file fetch is not supported for this peer") + // ManifestResponse is what a peer returns for a manifest request. type ManifestResponse struct { Manifest delta.Manifest `json:"manifest"` @@ -56,6 +99,26 @@ type ManifestResponse struct { // did (see hold.go). Without it an empty location is not taken as every // file in it deleted — see emptiedUnconfirmed. DeletionConfirmed bool `json:"deletionConfirmed,omitempty"` + + // Unchanged answers a request that carried ManifestQuery.IfHash: the + // responder's save still hashes to exactly that, so Manifest is left + // empty. Only ever set when asked; a peer that predates it ignores the + // question and sends the full manifest, which is handled as before. + Unchanged bool `json:"unchanged,omitempty"` + ManifestHash string `json:"manifestHash,omitempty"` + + // Version is the responder's version of the save (version.go), absent + // from builds that predate it and for games it does not version. + Version *VersionInfo `json:"version,omitempty"` + // Versions says the responder keeps save versions at all. The relay + // path sends it instead of a Proto, which there would also claim things + // that path does not serve. + Versions bool `json:"versions,omitempty"` +} + +// KeepsVersions reports whether the responder keeps save versions. +func (r ManifestResponse) KeepsVersions() bool { + return r.Versions || r.Proto >= ProtoVersions } // FileRef identifies one file inside one of a game's save locations. @@ -81,6 +144,9 @@ type ManifestQuery struct { // cover art, instead of a blank tile. AppID string CoverURL string + // IfHash asks the peer to answer "unchanged" instead of its manifest when + // its save still hashes to this — the base both devices last agreed on. + IfHash string } // BlockData is one fetched block. diff --git a/internal/p2p/syncengine/version.go b/internal/p2p/syncengine/version.go new file mode 100644 index 00000000..3ae5855c --- /dev/null +++ b/internal/p2p/syncengine/version.go @@ -0,0 +1,1298 @@ +package syncengine + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "sort" + "strings" + "time" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/ignore" + "github.com/opensave/opensave/internal/owntouch" + "github.com/opensave/opensave/internal/store" +) + +// Save versions. +// +// Every device knows which version of a game's save it holds, as a version +// vector: one counter per device that has ever changed the save, moved only +// when the save really changes there (the watcher saw the game or the user +// write it — never a sync). Comparing two vectors answers, exactly, the +// question the file-by-file comparison could only guess at from what each side +// happens to hold: +// +// - one is older than the other: that device is simply behind. It takes the +// newer save as a whole — every file, and every deletion — and only from a +// device that holds it completely. Nothing is ever taken from it, pushed +// by it, or deleted on its say-so, and it is a source for nobody. +// - equal: the same save. +// - neither: both changed it independently. Only then is anyone asked. +// +// A device also remembers the newest version it has heard of (the target) +// until it holds it. That is what keeps two outdated devices from syncing with +// each other: each knows something newer exists, so each waits for a device +// that holds it, instead of reading the other's half-finished copy as the save +// and passing on what it lacks as deletions. +// +// An empty folder holds no version at all, which is older than every version, +// so a new or emptied device fills itself from a complete one and never raises +// a conflict. A save that existed before this was introduced gets a version +// named after its content (genesisKey), so devices that already held the same +// save agree at once, and ones that did not are compared the way they always +// were, until they agree. + +// VersionVector is writer key → counter. +type VersionVector map[string]int64 + +type versionOrder int + +const ( + versionEqual versionOrder = iota + versionOlder + versionNewer + versionConcurrent +) + +// genesisAll is the entry "Use this save everywhere" adds (UseSaveEverywhere): +// a version holding it counts as having every save from before versions +// existed behind it, whatever its content was named. +const genesisAll = "g:*" + +// holds reports whether v includes entry k at counter n. +func holds(v VersionVector, k string, n int64) bool { + if v[k] >= n { + return true + } + return k != genesisAll && strings.HasPrefix(k, "g:") && v[genesisAll] >= 1 +} + +// compareVersions says how a relates to b. +func compareVersions(a, b VersionVector) versionOrder { + aBehind, bBehind := false, false + for k, bv := range b { + if !holds(a, k, bv) { + aBehind = true + break + } + } + for k, av := range a { + if !holds(b, k, av) { + bBehind = true + break + } + } + switch { + case !aBehind && !bBehind: + return versionEqual + case aBehind && !bBehind: + return versionOlder + case !aBehind && bBehind: + return versionNewer + default: + return versionConcurrent + } +} + +// covers reports whether a is b or newer. +func covers(a, b VersionVector) bool { + o := compareVersions(a, b) + return o == versionEqual || o == versionNewer +} + +func mergeVersions(a, b VersionVector) VersionVector { + out := make(VersionVector, len(a)+len(b)) + for k, v := range a { + out[k] = v + } + for k, v := range b { + if v > out[k] { + out[k] = v + } + } + return out +} + +func (v VersionVector) clone() VersionVector { return mergeVersions(v, nil) } + +// genesisOnly reports whether every entry names a save that predates version +// tracking (genesisKey), i.e. nobody has changed it since. +func (v VersionVector) genesisOnly() bool { + if len(v) == 0 { + return false + } + for k := range v { + if !strings.HasPrefix(k, "g:") { + return false + } + } + return true +} + +// String is a short, stable form for the log. +func (v VersionVector) String() string { + if len(v) == 0 { + return "none" + } + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, len(keys)) + for i, k := range keys { + short := k + if len(short) > 8 { + short = short[:8] + } + parts[i] = fmt.Sprintf("%s=%d", short, v[k]) + } + return strings.Join(parts, ",") +} + +// genesisKey names the version of a save that existed before versions did, +// after its content: devices that hold the same files get the same name. +// Directories are left out — they are created by any sync, and are not what +// makes two saves the same. +func genesisKey(m delta.Manifest) string { + return "g:" + filesKey(m, ignore.Rules{})[:16] +} + +// filesKey hashes which files a save holds and what is in them: paths and +// content hashes, nothing else — not folders, not times. Excluded paths are +// left out. +func filesKey(m delta.Manifest, rules ignore.Rules) string { + return filesKeyOf(m, func(p string) bool { return excluded(rules, p) }) +} + +func filesKeyOf(m delta.Manifest, leaveOut func(string) bool) string { + paths := make([]string, 0, len(m.Files)) + for p := range m.Files { + if !leaveOut(p) { + paths = append(paths, p) + } + } + sort.Strings(paths) + h := sha256.New() + for _, p := range paths { + h.Write([]byte(p)) + h.Write([]byte{0}) + h.Write([]byte(m.Files[p].Hash)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// versionHashOf is what is recorded with a version: the save's files as this +// device syncs them, tagged with the exclusion rules it was taken under. A +// save that no longer hashes to it was changed here since, whether or not the +// watcher said so. +// +// Filtered, because a file nobody syncs — a device's own config — changing +// is not a new version of the save. Tagged, because writing a rule is not one +// either: it changes the filtered hash on every device at once, and each took +// that as a change of its own, so the next sync asked about a divergence +// nobody made. A hash taken under other rules is re-taken, not compared +// (sameSaveAs). The files no save is made of (delta.NeverSyncedList) are in +// the tag for the same reason. +func (e *Engine) versionHashOf(gameID string, primary delta.Manifest) string { + text := "" + if game, err := e.Store.GetGame(gameID); err == nil { + text = e.IgnoreText(game) + } + return versionHashUnder(text, primary) +} + +// versionHashUnder is versionHashOf under a given exclusion list. +func versionHashUnder(text string, primary delta.Manifest) string { + tag := sha256.Sum256([]byte(text + "\x00" + delta.NeverSyncedList)) + return hex.EncodeToString(tag[:4]) + ":" + filesKey(primary, ignore.Parse(text)) +} + +// versionHashBeforeNeverSynced is versionHashOf as builds before +// delta.NeverSynced took it: the rules alone in the tag, those files counted. +func versionHashBeforeNeverSynced(rulesText string, primary delta.Manifest) string { + tag := sha256.Sum256([]byte(rulesText)) + rules := ignore.Parse(rulesText) + return hex.EncodeToString(tag[:4]) + ":" + filesKeyOf(primary, func(p string) bool { + return !rules.Empty() && rules.Match(p) + }) +} + +// AdoptExclusionView re-takes the hash recorded with this device's version of +// a game under what it is excluded by now (IgnoreText), when the save is the +// one it was recorded for under oldText — the files excluded then or now +// aside. Run when the exclusions change without anyone writing a rule (a new +// build's delta.NeverSyncedList, the game database naming settings files), +// before anything compares: otherwise the only difference, which files are +// counted, reads as the save changed here, and every device names the same +// save a new version of its own at once. +// +// beforeNeverSynced says the old hash was taken by a build that predates +// delta.NeverSynced, and counted those files. +func (e *Engine) AdoptExclusionView(gameID string, primary delta.Manifest, oldText string, beforeNeverSynced bool) { + if _, err := e.Store.GetGame(gameID); err != nil { + return + } + // The base agreed with each device, where it is still this save: the + // two agreed on it, and the files now left out leave them agreeing on + // the rest. One that no longer matches is a real change, left to count. + oldRules := ignore.Parse(oldText) + var oldView delta.Manifest + if beforeNeverSynced && oldRules.Empty() { + oldView = primary + } else if beforeNeverSynced { + oldView = filterManifestBy(primary, func(p string) bool { return !oldRules.Empty() && oldRules.Match(p) }) + } else { + oldView = filterManifest(primary, oldRules) + } + if was, now := oldView.ManifestHash(), filterManifest(primary, e.rulesFor(gameID)).ManifestHash(); was != now { + if peers, err := e.Store.ListPeers(); err == nil { + for _, p := range peers { + if e.Store.GetAgreedHash(gameID, p.ID) == was { + _ = e.Store.SetAgreedHash(gameID, p.ID, now) + } + } + } + } + + e.versionMu.Lock() + defer e.versionMu.Unlock() + gv, err := e.loadVersionLocked(gameID) + if err != nil || gv.rec.GameID == "" || gv.rec.Hash == "" { + return + } + was := versionHashUnder(oldText, primary) + if beforeNeverSynced { + was = versionHashBeforeNeverSynced(oldText, primary) + } + if now := e.versionHashOf(gameID, primary); gv.rec.Hash == was && was != now { + gv.rec.Hash = now + _ = e.saveVersionLocked(gv) + } +} + +// sameSaveAs reports whether a hash recorded with a version still describes +// the save that now hashes to current: the same, or taken under different +// exclusion rules, which says nothing about the save. +func sameSaveAs(recorded, current string) bool { + if recorded == current { + return true + } + r, _, okR := strings.Cut(recorded, ":") + c, _, okC := strings.Cut(current, ":") + return okR && okC && r != c +} + +// VersionInfo is a device's version of one game, as it travels in a manifest +// response. Absent from devices that predate it, which are synced the way they +// always were. +type VersionInfo struct { + Vector VersionVector `json:"vector"` + // Target is the newer version the device knows of and does not hold yet. + Target VersionVector `json:"target,omitempty"` + // HasFiles: the device holds at least one save file for the game. + HasFiles bool `json:"hasFiles"` + // Answered is the version someone on the device kept its own save over + // in a conflict (Unix ms in AnsweredAt). Resolution is mutual: the device + // holding that version is asked in turn, not overwritten. + Answered VersionVector `json:"answered,omitempty"` + AnsweredAt int64 `json:"answeredAt,omitempty"` +} + +// Outdated reports whether the device knows its save is behind another's. +func (v VersionInfo) Outdated() bool { + return len(v.Target) > 0 && compareVersions(v.Vector, v.Target) == versionOlder +} + +// gameVersion is the stored record with its vectors decoded. +type gameVersion struct { + rec store.GameVersion + vec VersionVector + target VersionVector + answered VersionVector +} + +func (e *Engine) loadVersionLocked(gameID string) (gameVersion, error) { + rec, err := e.Store.GetGameVersion(gameID) + if err != nil { + return gameVersion{}, err + } + gv := gameVersion{rec: rec, vec: VersionVector{}} + _ = json.Unmarshal([]byte(rec.Vector), &gv.vec) + if gv.vec == nil { + gv.vec = VersionVector{} + } + if rec.Target != "" { + _ = json.Unmarshal([]byte(rec.Target), &gv.target) + } + if rec.Answered != "" { + _ = json.Unmarshal([]byte(rec.Answered), &gv.answered) + } + return gv, nil +} + +func (e *Engine) saveVersionLocked(gv gameVersion) error { + if gv.rec.GameID == "" { + return nil + } + raw, _ := json.Marshal(gv.vec) + gv.rec.Vector = string(raw) + gv.rec.Target = "" + if len(gv.target) > 0 { + t, _ := json.Marshal(gv.target) + gv.rec.Target = string(t) + } + gv.rec.Answered = "" + if len(gv.answered) > 0 { + a, _ := json.Marshal(gv.answered) + gv.rec.Answered = string(a) + } else { + gv.rec.AnsweredAt = 0 + } + return e.Store.SaveGameVersion(gv.rec) +} + +// ensureGenesisLocked names a save that has files and no version yet. Not +// while the device is taking a newer version: its folder is then a part-way +// copy of someone else's, and naming that would make it a version of its own. +func (gv *gameVersion) ensureGenesis(local delta.Manifest, hash string) bool { + if len(gv.vec) > 0 || len(local.Files) == 0 || len(gv.target) > 0 || gv.rec.Pulling { + return false + } + gv.vec = VersionVector{genesisKey(local): 1} + gv.rec.Hash = hash + return true +} + +// refreshLocked brings this device's version up to date with its save before +// anyone compares it: a save that predates versions is named, and one that no +// longer hashes to what its version recorded was changed here — by the game, +// whether or not the watcher has said so yet (it waits for the game to finish +// writing, and while a game holds its files open that can be a long time). It +// is a version of its own from now. Without this, a device whose change was +// still unrecorded looked merely behind, and took the other's save over it. +// +// Not while a pull towards a newer version is unfinished: the folder is then a +// part-way copy of someone else's save, not a change made here. Reports +// whether the record changed. +func (e *Engine) refreshLocked(gv *gameVersion, game store.Game, local delta.Manifest) bool { + hash := e.versionHashOf(game.ID, local) + if gv.ensureGenesis(local, hash) { + return true + } + if gv.rec.Pulling || len(gv.vec) == 0 || gv.rec.Hash == "" || gv.rec.Hash == hash { + return false + } + if e.emptyingUnconfirmed(game.ID, local) { + return false + } + if sameSaveAs(gv.rec.Hash, hash) { + // Taken under other exclusion rules, so it cannot say whether the + // save changed: re-taken under the new ones. A change made at the + // same time still becomes a version — the watcher reports it + // (NoteLocalChange). + gv.rec.Hash = hash + return true + } + e.bumpLocked(gv, game.Name, hash) + return true +} + +// learn takes in what a peer knows: a version newer than this device's +// becomes its target, unless it already knows of one newer still. +func (gv *gameVersion) learn(theirs VersionInfo) { + for _, c := range []VersionVector{theirs.Vector, theirs.Target} { + if len(c) == 0 || compareVersions(gv.vec, c) != versionOlder { + continue + } + if len(gv.target) == 0 || compareVersions(gv.target, c) == versionOlder { + gv.target = c.clone() + } + } + gv.dropReachedTarget() +} + +// dropReachedTarget forgets a target this device's version covers — or has +// gone past it sideways, by changing on its own: that is no longer "behind", +// it is a conflict, and the version comparison says so by itself. +func (gv *gameVersion) dropReachedTarget() { + if len(gv.target) > 0 && compareVersions(gv.vec, gv.target) != versionOlder { + gv.target = nil + } +} + +func (gv gameVersion) info(hasFiles bool) VersionInfo { + v := VersionInfo{Vector: gv.vec.clone(), Target: gv.target.clone(), HasFiles: hasFiles} + if len(gv.answered) > 0 { + v.Answered, v.AnsweredAt = gv.answered.clone(), gv.rec.AnsweredAt + } + return v +} + +// adopt makes this device hold theirs: their version, and their answer to a +// conflict if they carry one — the device holding the version they kept +// theirs over is still to be asked, whoever it meets. +func (gv *gameVersion) adopt(vec VersionVector, theirs VersionInfo) { + gv.vec = vec.clone() + gv.answered, gv.rec.AnsweredAt = nil, 0 + if len(theirs.Answered) > 0 { + gv.answered, gv.rec.AnsweredAt = theirs.Answered.clone(), theirs.AnsweredAt + } + gv.dropReachedTarget() +} + +// LocalVersion is this device's version of a game, for a manifest response. +// manifest is the save being served; a save that predates versions is named +// here if it has not been yet. Nil when the game takes no part in versions: +// one with automatic sync turned off is not watched, so nothing would ever +// move its version — the peer then syncs it the way it always has. +func (e *Engine) LocalVersion(game store.Game, manifest delta.Manifest) *VersionInfo { + if !game.AutoSync { + return nil + } + e.versionMu.Lock() + defer e.versionMu.Unlock() + gv, err := e.loadVersionLocked(game.ID) + if err != nil { + return nil + } + if e.refreshLocked(&gv, game, manifest) { + _ = e.saveVersionLocked(gv) + } + v := gv.info(len(manifest.Files) > 0) + return &v +} + +// NoteLocalChange moves this device's version on: the watcher saw the game +// or the user change the save (never a sync — those are told apart by +// internal/owntouch). Called before the change is offered to anyone. +func (e *Engine) NoteLocalChange(gameID string) { + game, err := e.Store.GetGame(gameID) + if err != nil || !game.AutoSync { + return + } + e.versionMu.Lock() + defer e.versionMu.Unlock() + gv, err := e.loadVersionLocked(gameID) + if err != nil || gv.rec.GameID == "" { + return + } + if gv.rec.Pulling { + // Files a pull towards a newer version wrote — this run's own, still + // arriving, or left by a run that stopped part-way (after a restart + // the watcher reports those as a change: it cannot know who wrote + // them). Not a version: the folder is a part-way copy of someone + // else's, and naming it would hand that copy to every other device. + // The pull finishing is what moves the version. Anything written here + // meanwhile is in the snapshot taken before the pull replaces it. + return + } + m, err := delta.BuildManifest(game.SavePath) + if err != nil { + return + } + hash := e.versionHashOf(gameID, m) + if len(gv.vec) == 0 { + // First change ever seen here, on a save that predates versions or a + // new one. Named after its content, so a device holding the same files + // agrees with it rather than conflicting. + if gv.ensureGenesis(m, hash) { + _ = e.saveVersionLocked(gv) + } + return + } + if e.emptyingUnconfirmed(gameID, m) { + return // not a version until someone says it was meant (hold.go) + } + if hash == gv.rec.Hash { + // Already a version: a sync noticed it first (refreshLocked), or the + // change was only to files nobody syncs. + return + } + // The watcher saw the game or the user write the save: a version, even if + // the exclusion rules changed since the last one (writing a rule raises + // no file event, so that alone never arrives here). + e.bumpLocked(&gv, game.Name, hash) +} + +// bumpLocked gives this device a new version of its own, after hash. +func (e *Engine) bumpLocked(gv *gameVersion, name, hash string) { + gv.rec.Counter++ + gv.vec = gv.vec.clone() + gv.vec[gv.rec.Key] = gv.rec.Counter + gv.rec.Hash = hash + gv.dropReachedTarget() + if err := e.saveVersionLocked(*gv); err == nil { + e.Log("info", fmt.Sprintf("%s changed on this device — version %s", name, gv.vec)) + } +} + +// versionAfterResolution records a conflict's answer as a version. +// +// Keeping the peer's save makes this device hold what both held: newer than +// both, so the peer finds nothing to take and devices behind either take it. +// +// Keeping this device's own is a new version of its own — but deliberately +// not one that includes the peer's: that would make the peer simply behind, +// and its save would be replaced without anyone there being asked. Resolution +// is mutual. The answer is recorded instead (Answered), so this device is not +// asked again about the same version, and the peer, still independent of it, +// asks in turn. Whoever answers last decides (syncByVersion). +func (e *Engine) versionAfterResolution(gameID string, theirs *VersionInfo, keptLocal bool) { + if theirs == nil { + return + } + e.versionMu.Lock() + defer e.versionMu.Unlock() + gv, err := e.loadVersionLocked(gameID) + if err != nil || gv.rec.GameID == "" { + return + } + if keptLocal { + gv.rec.Counter++ + gv.vec = gv.vec.clone() + gv.vec[gv.rec.Key] = gv.rec.Counter + gv.answered, gv.rec.AnsweredAt = theirs.Vector.clone(), time.Now().UnixMilli() + } else { + gv.vec = mergeVersions(gv.vec, theirs.Vector) + gv.answered, gv.rec.AnsweredAt = nil, 0 + } + gv.rec.Pulling = false // whatever was being taken, the answer replaces it + gv.rec.Hash = "" + if game, err := e.Store.GetGame(gameID); err == nil { + if m, err := delta.BuildManifest(game.SavePath); err == nil { + gv.rec.Hash = e.versionHashOf(gameID, m) + } + } + gv.dropReachedTarget() + _ = e.saveVersionLocked(gv) +} + +// discardLocalVersion is for a "keep theirs" that did not finish: this +// device's own version is given up, and it takes the peer's like any outdated +// device would — whole, and only from a device that holds it. +func (e *Engine) discardLocalVersion(gameID string, theirs *VersionInfo) { + if theirs == nil { + return + } + e.versionMu.Lock() + defer e.versionMu.Unlock() + gv, err := e.loadVersionLocked(gameID) + if err != nil || gv.rec.GameID == "" { + return + } + gv.vec = VersionVector{} + gv.target = theirs.Vector.clone() + gv.rec.Hash = "" + _ = e.saveVersionLocked(gv) +} + +// versionStatusWaiting: this device or the peer is behind, and the other +// cannot give it the newer version either. Nothing happens; both wait for a +// device that holds it. +const versionStatusWaiting = "waiting" + +// syncByVersion decides a sync from the two devices' versions. handled false +// leaves it to the file comparison: the two hold the same version and the +// same files, or both saves predate versions and have not met since. +// +// local and remote are filtered by the exclusion rules; unfilteredLocal is +// this device's save as it is on disk, which names a version. +func (e *Engine) syncByVersion(ctx context.Context, game store.Game, peer Peer, + local, unfilteredLocal delta.Manifest, remote ManifestResponse) (Result, bool, error) { + + gameID := game.ID + theirs := *remote.Version + + e.versionMu.Lock() + gv, err := e.loadVersionLocked(gameID) + if err != nil || gv.rec.GameID == "" { + e.versionMu.Unlock() + return Result{}, false, nil + } + before := gv.target.clone() + changed := e.refreshLocked(&gv, game, unfilteredLocal) + gv.learn(theirs) + if changed || compareVersions(before, gv.target) != versionEqual || len(before) != len(gv.target) { + _ = e.saveVersionLocked(gv) + } + mine := gv.info(len(local.Files) > 0) + e.versionMu.Unlock() + + waiting := func(why string) (Result, bool, error) { + e.logOnce(gameID+"|"+peer.ID, why) + return Result{Status: versionStatusWaiting, PeerID: peer.ID, PeerName: peer.Name}, true, nil + } + pushTo := func() (Result, bool, error) { + // Once per state: a peer still taking it is asked again on every + // pass until it has, and that is not news each time. + e.logOnce(gameID+"|"+peer.ID, fmt.Sprintf("%q: %s has an older version (%s, this device %s) — asking it to take this one", + game.Name, peer.Name, theirs.Vector, mine.Vector)) + e.Transport.TriggerPeerPull(peer, gameID) + return Result{Status: "triggered_peer_pull", Direction: "push", PeerID: peer.ID, PeerName: peer.Name}, true, nil + } + + switch { + case mine.Outdated(): + if theirs.Outdated() || !covers(theirs.Vector, mine.Target) { + return waiting(fmt.Sprintf("%q: this device is behind (holds %s, newest known %s); %s does not hold that version either — waiting for a device that does", + game.Name, mine.Vector, mine.Target, peer.Name)) + } + return e.pullVersion(ctx, game, peer, local, remote, theirs.Vector) + case theirs.Outdated(): + if covers(mine.Vector, theirs.Target) { + return pushTo() + } + return waiting(fmt.Sprintf("%q: %s is behind and this device does not hold the version it is waiting for either", + game.Name, peer.Name)) + } + + switch compareVersions(mine.Vector, theirs.Vector) { + case versionNewer: + return pushTo() + case versionOlder: + // learn made this device outdated above, so this is only reached if + // the record could not be read back; the peer holds the newer version + // completely, so take it. + return e.pullVersion(ctx, game, peer, local, remote, theirs.Vector) + case versionEqual: + e.forgetLogOnce(gameID + "|" + peer.ID) + if sameFiles(local, remote.Manifest) { + return Result{}, false, nil + } + // One version, different files. Each side checks its own save against + // its version before answering (refreshLocked), so this is a change + // that landed after the peer answered; its sync carries it. + return waiting(fmt.Sprintf("%q: same version as %s but the files differ — waiting for the change to be recorded", + game.Name, peer.Name)) + } + + // Concurrent: both changed independently. + e.forgetLogOnce(gameID + "|" + peer.ID) + if sameFiles(local, remote.Manifest) { + // …into the same save. Both record the union, which each computes the + // same way, and are equal from then on. + e.versionMu.Lock() + if gv, err := e.loadVersionLocked(gameID); err == nil { + gv.vec = mergeVersions(gv.vec, theirs.Vector) + gv.rec.Hash = e.versionHashOf(gameID, unfilteredLocal) + gv.dropReachedTarget() + _ = e.saveVersionLocked(gv) + } + e.versionMu.Unlock() + return Result{}, false, nil + } + if !mine.HasFiles { + // An empty folder is never one side of a conflict: it takes the save. + return e.pullVersion(ctx, game, peer, local, remote, mergeVersions(mine.Vector, theirs.Vector)) + } + if !theirs.HasFiles { + return pushTo() + } + if mine.Vector.genesisOnly() || theirs.Vector.genesisOnly() { + // A save from before versions existed, never compared with this one + // since: nothing recorded says which is newer, so the files decide + // (transitionNewer) and the older device takes the newer save whole, + // as any device behind does — never file by file, which is how devices + // ended up holding mixtures of two saves that no game ever wrote. Only + // an exact tie is asked about. + switch transitionNewer(local, remote.Manifest, e.Store.GetAgreedHash(gameID, peer.ID)) { + case 1: + return pushTo() + case -1: + e.Log("info", fmt.Sprintf("%q: %s's save holds the more recent work — taking it whole (this device's is kept in a snapshot)", + game.Name, peer.Name)) + return e.pullVersion(ctx, game, peer, local, remote, theirs.Vector) + } + e.Log("warn", fmt.Sprintf("%q differs from %s's and neither holds more recent work — asking", game.Name, peer.Name)) + e.registerConflict(gameID, peer, local, remote) + return Result{Status: "conflict", PeerID: peer.ID, PeerName: peer.Name}, true, nil + } + + // Someone already answered: here, keeping this device's save over the + // peer's version (or one it has since moved on from), or there, keeping + // theirs over this one's. + iAnswered := len(mine.Answered) > 0 && covers(theirs.Vector, mine.Answered) + theyAnswered := len(theirs.Answered) > 0 && covers(mine.Vector, theirs.Answered) + switch { + case iAnswered && theyAnswered: + // Both were answered, each keeping its own. The later answer stands, + // as the person who gave it saw the other one too; the earlier one's + // save is in the snapshot taken before it is replaced. + // Answers in the same millisecond are settled the same way on both + // devices, or each would keep asking the other to take its own. + if mine.AnsweredAt > theirs.AnsweredAt || + (mine.AnsweredAt == theirs.AnsweredAt && mine.Vector.String() > theirs.Vector.String()) { + return pushTo() + } + e.Log("info", fmt.Sprintf("%q: %s's answer to the conflict came later — taking its version", game.Name, peer.Name)) + return e.pullVersion(ctx, game, peer, local, remote, mergeVersions(mine.Vector, theirs.Vector)) + case iAnswered: + // Answered here; the peer is asked in turn when it syncs. + return pushTo() + } + e.Log("warn", fmt.Sprintf("%q was changed independently on this device (%s) and on %s (%s)", + game.Name, mine.Vector, peer.Name, theirs.Vector)) + e.registerConflict(gameID, peer, local, remote) + return Result{Status: "conflict", PeerID: peer.ID, PeerName: peer.Name}, true, nil +} + +// pullVersion makes this device's save exactly the peer's — every file it +// holds, and none it does not — and only then records the peer's version as +// this device's. Until that, the device stays behind and keeps waiting. +// +// local and remote are filtered by the exclusion rules, so excluded files are +// neither taken nor removed. +func (e *Engine) pullVersion(ctx context.Context, game store.Game, peer Peer, + local delta.Manifest, remote ManifestResponse, adopt VersionVector) (Result, bool, error) { + + gameID := game.ID + if until, blocked := e.unwritableUntil(gameID); blocked { + // Said once when it happened (below); waiting is the answer until then. + e.logOnce(gameID+"|unwritable", fmt.Sprintf("%q: waiting until %s before writing to its save folder again", + game.Name, until.Format("15:04"))) + return Result{Status: "unwritable", PeerID: peer.ID, PeerName: peer.Name}, true, nil + } + if len(remote.Manifest.Files) == 0 && len(local.Files) > 0 && !remote.DeletionConfirmed { + // The newer version is an empty folder nobody has confirmed emptying. + e.Log("info", fmt.Sprintf("%q holds none of %q's save files now, and has not confirmed deleting them — keeping this device's copies", + peer.Name, game.Name)) + return Result{Status: "peer_holding", PeerID: peer.ID, PeerName: peer.Name}, true, nil + } + + var d Decision + for p, lf := range local.Files { + rf, ok := remote.Manifest.Files[p] + switch { + case !ok: + d.FilesToDeleteLocally = append(d.FilesToDeleteLocally, p) + case rf.Hash != lf.Hash: + d.FilesToPull = append(d.FilesToPull, p) + } + } + for p := range remote.Manifest.Files { + if _, ok := local.Files[p]; !ok { + d.FilesToPull = append(d.FilesToPull, p) + } + } + localDirs, remoteDirs := toSet(local.Dirs), toSet(remote.Manifest.Dirs) + for dir := range remoteDirs { + if _, ok := localDirs[dir]; !ok { + d.DirsToPull = append(d.DirsToPull, dir) + } + } + for dir := range localDirs { + if _, ok := remoteDirs[dir]; !ok { + d.DirsToDeleteLocally = append(d.DirsToDeleteLocally, dir) + } + } + sort.Strings(d.FilesToPull) + + e.Log("info", fmt.Sprintf("%q: taking %s's newer version (%s): %d file(s) to fetch, %d to remove", + game.Name, peer.Name, adopt, len(d.FilesToPull), len(d.FilesToDeleteLocally))) + + // Whatever this device holds is kept in a snapshot before any of it is + // replaced or removed. + replacing := len(d.FilesToDeleteLocally) + for _, p := range d.FilesToPull { + if _, ok := local.Files[p]; ok { + replacing++ + } + } + if replacing > 0 { + if _, err := e.Snapshots.CreateBeforeReplacing(gameID, fmt.Sprintf("Before taking %s's newer version", peer.Name)); err != nil { + return Result{}, true, fmt.Errorf("refusing to replace %q's files: they could not be snapshotted first: %w", game.Name, err) + } + } + + e.setPulling(gameID, true) + defer e.setPulling(gameID, false) + + applied := e.Writing(gameID) + started := time.Now() + e.applyLocalDeletions(gameID, primaryRootOf(game), d) + if n := len(d.FilesToDeleteLocally); n > 0 { + e.RecordActivity(store.ActivityEvent{GameID: gameID, Kind: store.ActivityDeleted, Device: peer.Name, Files: n}) + e.noteEmptiedByPeer(gameID, started) + } + e.createPulledDirs(gameID, game, d.DirsToPull) + if len(d.FilesToPull) > 0 { + if err := e.pullFiles(ctx, peer, gameID, game, primaryRootOf(game), local, remote, d.FilesToPull); err != nil { + applied() + if errors.Is(err, fs.ErrPermission) { + // Not something the next pass fixes: the folder needs other + // rights (one under Program Files does). Said once, plainly, + // and tried again later rather than on every pass. + e.markUnwritable(gameID) + e.Log("error", fmt.Sprintf("%q: the save folder %s cannot be written (%v). Give OpenSave write access to it, or track the game at another folder. Trying again in %s.", + game.Name, game.SavePath, err, unwritableRetry)) + } + return Result{}, true, fmt.Errorf("taking %s's version of %q did not finish (it is resumed on the next sync): %w", peer.Name, game.Name, err) + } + } + applied() + + // Only a save that is now exactly the peer's takes its version. Read + // straight from disk, not through the gate (settle.go): this sync is the + // one that wrote it, and after following a peer onto a branch it still + // holds the save — waiting for that would wait for itself until the + // sync's time ran out. + fresh, err := delta.BuildManifest(game.SavePath) + if err != nil { + return Result{}, true, err + } + freshHash := e.versionHashOf(gameID, fresh) + fresh = filterManifest(fresh, e.rulesFor(gameID)) + changedSince := false + if !sameFiles(fresh, remote.Manifest) { + if !changedSincePulled(primaryRootOf(game).Path, fresh, remote.Manifest, d) { + return Result{}, true, fmt.Errorf("taking %s's version of %q did not finish: the save still differs; it is resumed on the next sync", peer.Name, game.Name) + } + changedSince = true + } + + e.versionMu.Lock() + if gv, err := e.loadVersionLocked(gameID); err == nil { + theirs := VersionInfo{} + if remote.Version != nil { + theirs = *remote.Version + } + gv.adopt(adopt, theirs) + gv.rec.Hash = freshHash + gv.rec.Pulling = false + e.Log("success", fmt.Sprintf("%q now holds version %s from %s", game.Name, gv.vec, peer.Name)) + if changedSince { + // The peer's version arrived whole, and was then changed here - + // the game saving straight after the pull. That is a version of + // this device's own, made from theirs, and goes back to them. + e.bumpLocked(&gv, game.Name, freshHash) + } + _ = e.saveVersionLocked(gv) + } + e.versionMu.Unlock() + if changedSince { + e.Transport.TriggerPeerPull(peer, gameID) + } + + // The file comparison's own records, so it agrees if it is ever asked. + e.persistLineage(gameID, peer.ID, fresh, remote.Manifest) + _ = e.Store.SetAgreedHash(gameID, peer.ID, remote.Manifest.ManifestHash()) + + e.syncExtraRoots(ctx, gameID, game, peer, remote) + if !d.HasChanges() { + return Result{Status: "in_sync", Direction: "none", PeerID: peer.ID, PeerName: peer.Name}, true, nil + } + return Result{Status: "updated", Direction: "pull", PeerID: peer.ID, PeerName: peer.Name}, true, nil +} + +// unwritableRetry is how long a game whose save folder could not be written +// waits before a pull into it is tried again. +const unwritableRetry = 30 * time.Minute + +func (e *Engine) markUnwritable(gameID string) { + e.versionMu.Lock() + defer e.versionMu.Unlock() + if e.unwritable == nil { + e.unwritable = map[string]time.Time{} + } + e.unwritable[gameID] = time.Now().Add(unwritableRetry) +} + +func (e *Engine) unwritableUntil(gameID string) (time.Time, bool) { + e.versionMu.Lock() + defer e.versionMu.Unlock() + until, ok := e.unwritable[gameID] + if !ok || time.Now().After(until) { + delete(e.unwritable, gameID) + return time.Time{}, false + } + return until, true +} + +// setPulling marks a pull towards a newer version as running, so the files it +// writes are never taken for a change made here. In memory for this run; in +// the record from the start until the pull has completed, which only adopting +// the version clears — a pull that stopped part-way leaves a folder that is +// neither this device's version nor the peer's, and it must not be read as a +// change made here (refreshLocked, NoteLocalChange) while it waits to resume. +func (e *Engine) setPulling(gameID string, on bool) { + e.versionMu.Lock() + defer e.versionMu.Unlock() + if e.pullingNow == nil { + e.pullingNow = map[string]bool{} + } + if !on { + delete(e.pullingNow, gameID) + return + } + e.pullingNow[gameID] = true + if gv, err := e.loadVersionLocked(gameID); err == nil && gv.rec.GameID != "" && !gv.rec.Pulling { + gv.rec.Pulling = true + _ = e.saveVersionLocked(gv) + } +} + +// quickVersionCheck is the version half of quickInSync: both saves are the +// agreed one, so the question is only whether the versions say the same. +// done reports a final answer; otherwise the full sync decides. +func (e *Engine) quickVersionCheck(game store.Game, peer Peer, local delta.Manifest, theirs VersionInfo) (res Result, done bool) { + e.versionMu.Lock() + gv, err := e.loadVersionLocked(game.ID) + if err != nil || gv.rec.GameID == "" { + e.versionMu.Unlock() + return Result{}, false + } + before := gv.target.clone() + changed := e.refreshLocked(&gv, game, local) + gv.learn(theirs) + if changed || compareVersions(before, gv.target) != versionEqual || len(before) != len(gv.target) { + _ = e.saveVersionLocked(gv) + } + mine := gv.info(len(local.Files) > 0) + e.versionMu.Unlock() + + if compareVersions(mine.Vector, theirs.Vector) != versionEqual { + return Result{}, false + } + if mine.Outdated() || theirs.Outdated() { + // The same save, and at least one of the two knows a newer one exists + // that neither holds: nothing to do between these two. + return Result{Status: versionStatusWaiting, PeerID: peer.ID, PeerName: peer.Name}, true + } + return Result{}, true +} + +// logOnce logs a line the first time it is the answer for key, and not again +// while it stays the answer: waiting is re-decided every reconcile. +func (e *Engine) logOnce(key, msg string) { + e.versionMu.Lock() + if e.loggedOnce == nil { + e.loggedOnce = map[string]string{} + } + same := e.loggedOnce[key] == msg + e.loggedOnce[key] = msg + e.versionMu.Unlock() + if !same { + e.Log("info", msg) + } +} + +func (e *Engine) forgetLogOnce(key string) { + e.versionMu.Lock() + delete(e.loggedOnce, key) + e.versionMu.Unlock() +} + +// transitionNewer says which of two saves from before versions is the newer, +// when that is beyond doubt: 1 for local, -1 for remote, 0 when it is not. +// +// Beyond doubt means one of two things. The devices last agreed on a state +// and one of them still holds exactly it: that one has not changed since, so +// the other is newer. Or one save is newer in every file that differs — each +// is newer there or missing on the other side — and the other holds no file +// of its own written after the first's newest. That is what a save someone +// played on looks like next to one nobody touched. +// +// It is deliberately not "the newer file wins, file by file": that makes a +// mixture of two saves no game wrote. And not "the newest file wins", which a +// mixture passes as easily as a real save. Here the device that gives way was +// older in every file that differs, so it never loses anything newer than +// what it takes — at worst it takes a copy that arrived only half-way, and +// then the complete save, newer than both, replaces that too when it is +// seen. States newer each in different files fail the test both ways and are +// left to a person. +func transitionNewer(local, remote delta.Manifest, agreed string) int { + if agreed != "" { + lh, rh := local.ManifestHash(), remote.ManifestHash() + switch { + case lh == agreed && rh != agreed: + return -1 + case rh == agreed && lh != agreed: + return 1 + } + } + // Two saves with no file in common are not two versions of the same + // files — a save kept as one file named differently on each device, say + // — and nothing about their times says one should replace the other. + if len(local.Files) > 0 && len(remote.Files) > 0 && !shareAPath(local, remote) { + return 0 + } + l, r := newerInEveryFile(local, remote), newerInEveryFile(remote, local) + switch { + case l && !r: + return 1 + case r && !l: + return -1 + } + return newestWork(local, remote) +} + +// newestWork settles two saves from before versions that neither rule above +// could: the one holding the most recent work — the latest-written of the +// files where they differ — is the save, and if that is a tie, the one newer +// in more of those files. The other takes it whole, keeping a snapshot of its +// own first. +// +// Before versions, devices that never ran a game held copies relayed from the +// others at different times, and syncs file by file had left some holding +// mixtures; those were asked about, about states nobody made. Taking the one +// with the most recent work is what syncing is for — an older state is never +// what anyone wants handed on — and whatever the other held stays in its +// snapshot. Changes made since versions existed are not decided this way: +// those are versions of their own, and a real conflict between them is asked. +// +// 0 only for an exact tie, which leaves it to a person. +func newestWork(a, b delta.Manifest) int { + var aNewest, bNewest delta.Milli + aCount, bCount := 0, 0 + consider := func(p string) { + af, inA := a.Files[p] + bf, inB := b.Files[p] + if inA && inB && af.Hash == bf.Hash { + return + } + if inA && af.MtimeMs > aNewest { + aNewest = af.MtimeMs + } + if inB && bf.MtimeMs > bNewest { + bNewest = bf.MtimeMs + } + if inA && inB { + switch { + case af.MtimeMs > bf.MtimeMs: + aCount++ + case bf.MtimeMs > af.MtimeMs: + bCount++ + } + } + } + for p := range a.Files { + consider(p) + } + for p := range b.Files { + if _, inA := a.Files[p]; !inA { + consider(p) + } + } + switch { + case aNewest > bNewest: + return 1 + case bNewest > aNewest: + return -1 + case aCount > bCount: + return 1 + case bCount > aCount: + return -1 + } + return 0 +} + +func shareAPath(a, b delta.Manifest) bool { + for p := range a.Files { + if _, ok := b.Files[p]; ok { + return true + } + } + return false +} + +// newerInEveryFile reports whether a is newer than b wherever they differ, +// and they do differ. +func newerInEveryFile(a, b delta.Manifest) bool { + differ := false + var aNewest delta.Milli + for p, af := range a.Files { + if af.MtimeMs > aNewest { + aNewest = af.MtimeMs + } + bf, ok := b.Files[p] + if !ok { + differ = true + continue + } + if af.Hash != bf.Hash { + differ = true + if af.MtimeMs <= bf.MtimeMs { + return false + } + } + } + for p, bf := range b.Files { + if _, ok := a.Files[p]; ok { + continue + } + differ = true + if bf.MtimeMs >= aNewest { + return false + } + } + return differ +} + +// StatusPeerOutdatedApp: the peer runs an OpenSave without save versions. +// Nothing is taken from it until it is updated; it can still take from here. +const StatusPeerOutdatedApp = "peer_outdated_app" + +// OldBuildDeleteMessage is the answer to a deletion asked for by a device +// whose OpenSave does not keep save versions. +const OldBuildDeleteMessage = "this device keeps save versions and takes no deletions from an OpenSave that does not — update OpenSave on the asking device" + +// notePeerApp records whether a peer's OpenSave keeps save versions, as its +// last manifest answer said. +func (e *Engine) notePeerApp(peerID string, versions bool) { + e.versionMu.Lock() + defer e.versionMu.Unlock() + if e.peerVersions == nil { + e.peerVersions = map[string]bool{} + } + e.peerVersions[peerID] = versions +} + +// PeerNeedsUpdate reports whether a peer is known to run an OpenSave without +// save versions — one this device takes nothing from — so the person can be +// told to update it. +func (e *Engine) PeerNeedsUpdate(peerID string) bool { + e.versionMu.Lock() + defer e.versionMu.Unlock() + v, known := e.peerVersions[peerID] + return known && !v +} + +// RefusedOldBuildDelete logs, once per game and device, that a deletion asked +// for by an OpenSave without save versions was refused. +func (e *Engine) RefusedOldBuildDelete(gameID, from string) { + e.logOnce("refused|"+gameID+"|"+from, fmt.Sprintf( + "%s asked to delete files of %q; refused — its OpenSave does not keep save versions and decides by the old file comparison. Update it there.", + from, gameID)) +} + +// supersedes reports whether theirs is newer than this device's version and +// than the version the open conflict is with: both sides of the question are +// behind it. +func (e *Engine) supersedes(gameID string, theirs VersionInfo) bool { + e.mu.Lock() + c := e.activeConflicts[gameID] + e.mu.Unlock() + if c == nil || c.remoteVersion == nil { + return false + } + e.versionMu.Lock() + gv, err := e.loadVersionLocked(gameID) + e.versionMu.Unlock() + if err != nil || len(gv.vec) == 0 { + return false + } + return compareVersions(gv.vec, theirs.Vector) == versionOlder && + compareVersions(c.remoteVersion.Vector, theirs.Vector) == versionOlder +} + +// UseSaveEverywhere makes this device's save of a game the one every other +// device takes: its version becomes newer than every save from before +// versions existed, on any device, so each of those takes this one whole, +// keeping a snapshot of its own first. For when several devices hold states +// nobody can rank — copies that arrived half-way, mixtures of two saves — +// and a person knows which device has the right one. +// +// A change made on another device since versions existed is not overruled: +// that device and this one have each changed the save independently, and +// the person there is asked, as for any conflict. +func (e *Engine) UseSaveEverywhere(gameID string) (VersionVector, error) { + game, err := e.Store.GetGame(gameID) + if err != nil { + return nil, err + } + if !game.AutoSync { + return nil, errors.New("automatic sync is off for this game, so it does not take part in versions") + } + m, err := delta.BuildManifest(game.SavePath) + if err != nil { + return nil, err + } + if len(m.Files) == 0 { + return nil, errors.New("this device holds no save files for this game") + } + e.versionMu.Lock() + gv, err := e.loadVersionLocked(gameID) + if err != nil || gv.rec.GameID == "" { + e.versionMu.Unlock() + if err == nil { + err = errors.New("the game is not tracked") + } + return nil, err + } + gv.vec = mergeVersions(gv.vec, VersionVector{genesisAll: 1}) + gv.rec.Counter++ + gv.vec[gv.rec.Key] = gv.rec.Counter + gv.rec.Hash = e.versionHashOf(gameID, m) + gv.target, gv.answered, gv.rec.AnsweredAt = nil, nil, 0 + gv.rec.Pulling = false + saveErr := e.saveVersionLocked(gv) + vec := gv.vec.clone() + e.versionMu.Unlock() + if saveErr != nil { + return nil, saveErr + } + e.mu.Lock() + _, conflicted := e.activeConflicts[gameID] + e.mu.Unlock() + if conflicted { + e.clearConflict(gameID) + } + e.Log("info", fmt.Sprintf("%q: this device's save is to be used everywhere — version %s", game.Name, vec)) + return vec, nil +} + +// changedSincePulled reports a pull that finished and whose save was then +// changed here: every way the save now differs from the peer's is a change +// someone other than OpenSave made since the pull was planned. A game saving, +// or a person deleting a file, the moment a pull is done leaves exactly that. +// +// - A file the pull did not touch was the same on both sides when the pull +// was planned (anything that differed was to be fetched or removed). If +// it differs now, it was changed here meanwhile. +// - A file the pull wrote or removed differs because of someone else only +// if someone other than OpenSave has written it, or put it back, since +// (owntouch). Still as OpenSave left it, it is the pull's own unfinished +// part: a pull that stopped part-way, resumed on the next sync as before. +// +// Without this, a save the game had just written, or a file just deleted, +// was taken for the unfinished part of the pull and replaced with the peer's +// on the next sync; only the snapshot taken before replacing it still held it. +func changedSincePulled(root string, fresh, remote delta.Manifest, d Decision) bool { + touched := make(map[string]bool, len(d.FilesToPull)+len(d.FilesToDeleteLocally)) + for _, p := range d.FilesToPull { + touched[p] = true + } + for _, p := range d.FilesToDeleteLocally { + touched[p] = true + } + changedByOthers := func(p string) bool { + return !touched[p] || !owntouch.Recent(delta.LocalNameFor(root, p)) + } + differs := 0 + for p, lf := range fresh.Files { + if rf, ok := remote.Files[p]; ok && rf.Hash == lf.Hash { + continue + } + differs++ + if !changedByOthers(p) { + return false + } + } + for p := range remote.Files { + if _, ok := fresh.Files[p]; ok { + continue + } + differs++ + if !changedByOthers(p) { + return false + } + } + return differs > 0 +} diff --git a/internal/p2p/syncengine/version_test.go b/internal/p2p/syncengine/version_test.go new file mode 100644 index 00000000..67e2d189 --- /dev/null +++ b/internal/p2p/syncengine/version_test.go @@ -0,0 +1,803 @@ +package syncengine + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "testing" + "time" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/snapshot" + "github.com/opensave/opensave/internal/store" +) + +func TestCompareVersions(t *testing.T) { + cases := []struct { + a, b VersionVector + want versionOrder + }{ + {VersionVector{}, VersionVector{}, versionEqual}, + {VersionVector{}, VersionVector{"x": 1}, versionOlder}, + {VersionVector{"x": 2}, VersionVector{"x": 1}, versionNewer}, + {VersionVector{"x": 1, "y": 1}, VersionVector{"x": 1, "y": 1}, versionEqual}, + {VersionVector{"x": 2}, VersionVector{"x": 1, "y": 1}, versionConcurrent}, + {VersionVector{"x": 1}, VersionVector{"x": 1, "y": 1}, versionOlder}, + } + for _, c := range cases { + if got := compareVersions(c.a, c.b); got != c.want { + t.Errorf("compare(%v, %v) = %d, want %d", c.a, c.b, got, c.want) + } + } +} + +// "Use this save everywhere" covers every save from before versions, not +// changes made since. +func TestCompareVersions_UseEverywhereCoversEveryOldSave(t *testing.T) { + everywhere := VersionVector{"g:aaaa": 1, genesisAll: 1, "x": 1} + if got := compareVersions(VersionVector{"g:bbbb": 1}, everywhere); got != versionOlder { + t.Errorf("an old save of other content vs everywhere = %d, want older", got) + } + if got := compareVersions(VersionVector{}, everywhere); got != versionOlder { + t.Errorf("an empty folder vs everywhere = %d, want older", got) + } + if got := compareVersions(VersionVector{"g:bbbb": 1, "y": 1}, everywhere); got != versionConcurrent { + t.Errorf("a save changed since versions vs everywhere = %d, want concurrent", got) + } +} + +func setTimes(t *testing.T, dir, rel string, at time.Time) { + t.Helper() + if err := os.Chtimes(filepath.Join(dir, filepath.FromSlash(rel)), at, at); err != nil { + t.Fatal(err) + } +} + +// Two saves from before versions, one played on since they last matched: +// that one is newer in every file that differs, and the other takes it whole +// — no question, and no mixture. +func TestVersions_TransitionTakesTheClearlyNewerSave(t *testing.T) { + m, n := newMesh(t, "played", "old") + played, old := n["played"], n["old"] + t0 := time.Now().Add(-48 * time.Hour) + t1 := time.Now().Add(-1 * time.Hour) + for _, d := range []*meshNode{played, old} { + write(t, d.dir, "slot1.sav", "day one") + write(t, d.dir, "meta.dat", "m1") + write(t, d.dir, "stale.tmp", "gone later") + setTimes(t, d.dir, "slot1.sav", t0) + setTimes(t, d.dir, "meta.dat", t0) + setTimes(t, d.dir, "stale.tmp", t0) + } + // Played on, before the update: changed, added, removed. + write(t, played.dir, "slot1.sav", "day five") + write(t, played.dir, "slot2.sav", "new slot") + _ = os.Remove(filepath.Join(played.dir, "stale.tmp")) + setTimes(t, played.dir, "slot1.sav", t1) + setTimes(t, played.dir, "slot2.sav", t1) + + for _, pair := range [][2]*meshNode{{old, played}, {played, old}} { + res, err := syncPair(t, pair[0], pair[1]) + if err != nil || res.Status == "conflict" { + t.Fatalf("%s↔%s: %+v, %v", pair[0].name, pair[1].name, res, err) + } + } + settleAll(t, played, old) + if !sameSave(old.files(t), played.files(t)) { + t.Errorf("the old device did not take the played save whole: %s", describe(old.files(t))) + } + if m.deletes() != 0 { + t.Errorf("%d deletion(s) were sent", m.deletes()) + } +} + +// Two copies relayed from other devices at different times, each newer in a +// different file — the state syncing file by file used to leave. Nobody is +// asked: the one holding the more recent work is the save, and the other +// takes it whole, keeping its own in a snapshot. Never a mixture. +func TestVersions_TransitionTakesTheMostRecentWorkWithoutAsking(t *testing.T) { + m, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + t0 := time.Now().Add(-48 * time.Hour) + t1 := time.Now().Add(-2 * time.Hour) + t2 := time.Now().Add(-1 * time.Hour) + write(t, a.dir, "one.sav", "a-new") + write(t, a.dir, "two.sav", "old") + write(t, b.dir, "one.sav", "old") + write(t, b.dir, "two.sav", "b-newest") + setTimes(t, a.dir, "one.sav", t1) + setTimes(t, a.dir, "two.sav", t0) + setTimes(t, b.dir, "one.sav", t0) + setTimes(t, b.dir, "two.sav", t2) + want := b.files(t) + + for _, pair := range [][2]*meshNode{{a, b}, {b, a}} { + if res, err := syncPair(t, pair[0], pair[1]); err != nil || res.Status == "conflict" { + t.Fatalf("%s↔%s: %+v, %v — want no question", pair[0].name, pair[1].name, res, err) + } + } + settleAll(t, a, b) + if !sameSave(a.files(t), want) || !sameSave(b.files(t), want) { + t.Errorf("not both on the save with the most recent work: a %s, b %s", describe(a.files(t)), describe(b.files(t))) + } + if m.deletes() != 0 { + t.Errorf("%d deletion(s) were sent", m.deletes()) + } +} + +// Two saves with no file in common — one file, named differently on each +// device — are not two versions of the same files: asked about, never one +// replacing the other because it is newer. +func TestVersions_TransitionAsksWhenNoFileIsShared(t *testing.T) { + _, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + write(t, a.dir, "alice.sav", "A's world") + write(t, b.dir, "bob.sav", "B's world") + setTimes(t, a.dir, "alice.sav", time.Now().Add(-time.Hour)) + setTimes(t, b.dir, "bob.sav", time.Now().Add(-48*time.Hour)) + bBefore := b.files(t) + for _, pair := range [][2]*meshNode{{a, b}, {b, a}} { + if res, err := syncPair(t, pair[0], pair[1]); err != nil || res.Status != "conflict" { + t.Fatalf("%s↔%s: %+v, %v — want a question", pair[0].name, pair[1].name, res, err) + } + } + if !sameSave(b.files(t), bBefore) { + t.Error("b's save was replaced by one with no file in common") + } +} + +// Only an exact tie — the same latest time, newer in as many files — is asked +// about: there is nothing to tell the two apart. +func TestVersions_TransitionAsksOnlyOnAnExactTie(t *testing.T) { + _, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + t1 := time.Now().Add(-1 * time.Hour) + write(t, a.dir, "slot.sav", "a-content") + write(t, b.dir, "slot.sav", "b-content") + setTimes(t, a.dir, "slot.sav", t1) + setTimes(t, b.dir, "slot.sav", t1) + aBefore, bBefore := a.files(t), b.files(t) + + res, err := syncPair(t, a, b) + if err != nil || res.Status != "conflict" { + t.Fatalf("an exact tie: %+v, %v — want a question", res, err) + } + if !sameSave(a.files(t), aBefore) || !sameSave(b.files(t), bBefore) { + t.Error("a save changed before anyone answered") + } +} + +// "Use this save everywhere" on the device with the right save: every other +// device takes it whole — even one already waiting on a question about two +// mixed states — and nobody is asked. +func TestVersions_UseThisSaveEverywhere(t *testing.T) { + m, n := newMesh(t, "right", "mixed1", "mixed2") + right, mixed1, mixed2 := n["right"], n["mixed1"], n["mixed2"] + t0 := time.Now().Add(-48 * time.Hour) + t1 := time.Now().Add(-1 * time.Hour) + writeMany(t, right.dir, "map", 12) + write(t, right.dir, "players.db", "current") + // Two states nothing tells apart: one file each, the same time. + write(t, mixed1.dir, "players.db", "old-1") + write(t, mixed2.dir, "players.db", "old-2") + setTimes(t, mixed1.dir, "players.db", t1) + setTimes(t, mixed2.dir, "players.db", t1) + _ = t0 + + if res, _ := syncPair(t, mixed1, mixed2); res.Status != "conflict" { + t.Fatalf("setup: an exact tie should be asked about, got %+v", res) + } + if _, err := right.eng.UseSaveEverywhere("game1"); err != nil { + t.Fatal(err) + } + settleAll(t, right, mixed1, mixed2) + want := right.files(t) + for _, d := range []*meshNode{mixed1, mixed2} { + if !sameSave(d.files(t), want) { + t.Errorf("%s did not take the save used everywhere: %s", d.name, describe(d.files(t))) + } + if len(d.eng.ActiveConflicts()) != 0 { + t.Errorf("%s still has a conflict open", d.name) + } + } + if m.deletes() != 0 { + t.Errorf("%d deletion(s) were sent", m.deletes()) + } +} + +// A change made on another device after versions existed is not overruled by +// "Use this save everywhere": that is a conflict like any other. +func TestVersions_UseEverywhereDoesNotOverruleALaterChange(t *testing.T) { + _, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + write(t, a.dir, "slot.sav", "start") + settleAll(t, a, b) + b.change(t, func(dir string) { write(t, dir, "slot.sav", "b-played") }) + if _, err := a.eng.UseSaveEverywhere("game1"); err != nil { + t.Fatal(err) + } + bBefore := b.files(t) + settleAll(t, a, b) + if !sameSave(b.files(t), bBefore) { + t.Error("b's change since the update was replaced without anyone being asked") + } + if len(a.eng.ActiveConflicts())+len(b.eng.ActiveConflicts()) == 0 { + t.Error("nobody was asked") + } +} + +// --- a network of real engines ------------------------------------------- + +// meshNode is one device: its own store, save folder and engine. +type meshNode struct { + name string + dir string + st *store.Store + eng *Engine + // blockBudget, when >= 0, is how many more files this device serves + // before every further block request fails: a pull that stops part-way. + blockBudget int + // proto is the protocol revision it answers with; below ProtoVersions it + // stands for an OpenSave that keeps no save versions. + proto int +} + +func (n *meshNode) peer() Peer { + return Peer{ID: "node_" + n.name, Name: n.name, Address: "127.0.0.1", Port: 1} +} + +type mesh struct { + mu sync.Mutex + nodes map[string]*meshNode // by peer id + offline map[string]bool + // deletesSent counts DeleteRemote calls: the version path never makes one. + deletesSent int +} + +type meshTransport struct { + m *mesh + self *meshNode +} + +var errUnreachable = errors.New("peer unreachable") + +func (t *meshTransport) target(peer Peer) (*meshNode, error) { + t.m.mu.Lock() + defer t.m.mu.Unlock() + n := t.m.nodes[peer.ID] + if n == nil || t.m.offline[peer.ID] { + return nil, errUnreachable + } + return n, nil +} + +// FetchManifest answers the way the real route does: the save, and the +// device's version of it. +func (t *meshTransport) FetchManifest(ctx context.Context, peer Peer, gameID string, q ManifestQuery) (ManifestResponse, error) { + n, err := t.target(peer) + if err != nil { + return ManifestResponse{}, err + } + game, err := n.st.GetGame(gameID) + if err != nil { + return ManifestResponse{}, errors.New("Game not found.") + } + m, err := delta.BuildManifest(n.dir) + if err != nil { + return ManifestResponse{}, err + } + return ManifestResponse{ + Manifest: m, + ActiveBranch: game.ActiveBranch, + DeletionConfirmed: n.eng.DeletionConfirmed(gameID), + Version: n.eng.LocalVersion(game, m), + Proto: n.proto, + }, nil +} + +func (t *meshTransport) FetchBlocks(ctx context.Context, peer Peer, ref FileRef, blockIndices []int, blockSize int) ([]BlockData, error) { + n, err := t.target(peer) + if err != nil { + return nil, err + } + t.m.mu.Lock() + if n.blockBudget == 0 { + t.m.mu.Unlock() + return nil, errors.New("connection reset") + } + if n.blockBudget > 0 { + n.blockBudget-- + } + t.m.mu.Unlock() + full := filepath.Join(n.dir, filepath.FromSlash(ref.RelPath)) + entry, err := delta.HashFile(full) + if err != nil { + return nil, err + } + raw, err := os.ReadFile(full) + if err != nil { + return nil, err + } + var out []BlockData + for _, idx := range blockIndices { + if idx >= len(entry.Blocks) { + continue + } + start := idx * entry.BlockSize + end := start + entry.Blocks[idx].Length + if end > len(raw) { + end = len(raw) + } + out = append(out, BlockData{Index: idx, Data: raw[start:end], Length: end - start}) + } + return out, nil +} + +func (t *meshTransport) DeleteRemote(ctx context.Context, peer Peer, ref FileRef) error { + n, err := t.target(peer) + if err != nil { + return err + } + t.m.mu.Lock() + t.m.deletesSent++ + t.m.mu.Unlock() + return os.RemoveAll(filepath.Join(n.dir, filepath.FromSlash(ref.RelPath))) +} + +func (t *meshTransport) TriggerPeerPull(peer Peer, gameID string) {} +func (t *meshTransport) ReportSyncEvent(peer Peer, gameID, eventType string, data map[string]any) {} + +func newMesh(t *testing.T, names ...string) (*mesh, map[string]*meshNode) { + t.Helper() + m := &mesh{nodes: map[string]*meshNode{}, offline: map[string]bool{}} + byName := map[string]*meshNode{} + for _, name := range names { + root := t.TempDir() + dir := filepath.Join(root, "saves") + if err := os.MkdirAll(dir, 0o777); err != nil { + t.Fatal(err) + } + st, err := store.Open(filepath.Join(root, "opensave.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if err := st.EnsureDefaultSettings(root, filepath.Join(root, "backups")); err != nil { + t.Fatal(err) + } + if err := st.CreateGame(store.Game{ID: "game1", Name: "Game One", SavePath: dir, AutoSync: true, MaxSnapshots: 50}); err != nil { + t.Fatal(err) + } + n := &meshNode{name: name, dir: dir, st: st, blockBudget: -1, proto: ProtoVersions} + n.eng = New(st, snapshot.New(st), &meshTransport{m: m, self: n}) + m.nodes[n.peer().ID] = n + byName[name] = n + } + for _, a := range byName { + for _, b := range byName { + if a != b { + p := b.peer() + if err := a.st.UpsertPeer(store.Peer{ID: p.ID, Name: p.Name, Address: p.Address, Port: p.Port, Status: "online"}); err != nil { + t.Fatal(err) + } + } + } + } + return m, byName +} + +func (m *mesh) setOffline(n *meshNode, off bool) { + m.mu.Lock() + m.offline[n.peer().ID] = off + m.mu.Unlock() +} + +func (m *mesh) deletes() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.deletesSent +} + +// syncPair runs one sync of a with b, as a's engine would. +func syncPair(t *testing.T, a, b *meshNode) (Result, error) { + t.Helper() + return a.eng.SyncWithPeer(context.Background(), "game1", b.peer()) +} + +// change edits the save the way a game does, then tells the engine — the +// watcher's part. +func (n *meshNode) change(t *testing.T, edit func(dir string)) { + t.Helper() + edit(n.dir) + n.eng.NoteLocalChange("game1") +} + +func (n *meshNode) files(t *testing.T) map[string]string { + t.Helper() + m, err := delta.BuildManifest(n.dir) + if err != nil { + t.Fatal(err) + } + out := map[string]string{} + for p, f := range m.Files { + out[p] = f.Hash + } + return out +} + +func (n *meshNode) version(t *testing.T) VersionInfo { + t.Helper() + game, err := n.st.GetGame("game1") + if err != nil { + t.Fatal(err) + } + m, err := delta.BuildManifest(n.dir) + if err != nil { + t.Fatal(err) + } + return *n.eng.LocalVersion(game, m) +} + +func sameSave(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for p, h := range a { + if b[p] != h { + return false + } + } + return true +} + +func describe(files map[string]string) string { + paths := make([]string, 0, len(files)) + for p := range files { + paths = append(paths, p) + } + sort.Strings(paths) + if len(paths) > 6 { + return fmt.Sprintf("%d files (%v …)", len(paths), paths[:6]) + } + return fmt.Sprintf("%v", paths) +} + +func writeMany(t *testing.T, dir, prefix string, n int) { + t.Helper() + for i := 0; i < n; i++ { + write(t, dir, fmt.Sprintf("%s/chunk_%03d.bin", prefix, i), fmt.Sprintf("%s-%d", prefix, i)) + } +} + +// settleAll syncs every reachable node with every other, a few rounds, the +// way devices that are all online settle. +func settleAll(t *testing.T, nodes ...*meshNode) { + t.Helper() + for round := 0; round < 4; round++ { + for _, a := range nodes { + for _, b := range nodes { + if a != b { + _, _ = syncPair(t, a, b) + } + } + } + } +} + +// --- scenarios --------------------------------------------------------------- + +// The case this exists for. Three devices share a save; the server changes it +// a lot. One laptop starts taking the new version and is cut off part-way; the +// server then goes offline. The two laptops must not sync with each other in +// the meantime — not the half-finished copy, and not its missing files read as +// deletions — and when the server is back, both end up with its save exactly. +func TestVersions_IncompleteDevicesWaitForTheNewestOne(t *testing.T) { + m, n := newMesh(t, "server", "laptop", "desk") + server, laptop, desk := n["server"], n["laptop"], n["desk"] + + writeMany(t, server.dir, "map", 40) + write(t, server.dir, "players.db", "v1") + settleAll(t, server, laptop, desk) + original := server.files(t) + for _, d := range []*meshNode{laptop, desk} { + if !sameSave(d.files(t), original) { + t.Fatalf("%s did not take the initial save: %s", d.name, describe(d.files(t))) + } + } + + // The desk is switched off for a while; the server plays and the laptop + // takes that. The desk is now two versions behind, the laptop one — + // which makes the laptop newer than the desk, the comparison a device + // part-way through a pull must never win. + m.setOffline(desk, true) + server.change(t, func(dir string) { write(t, dir, "players.db", "v1.5") }) + settleAll(t, server, laptop) + m.setOffline(desk, false) + + // The server plays on: new chunks, some removed, one changed. + server.change(t, func(dir string) { + writeMany(t, dir, "map2", 30) + for i := 0; i < 5; i++ { + _ = os.Remove(filepath.Join(dir, "map", fmt.Sprintf("chunk_%03d.bin", i))) + } + write(t, dir, "players.db", "v2") + }) + newest := server.files(t) + + // The laptop starts taking it and is cut off after 10 files. + server.blockBudget = 10 + if _, err := syncPair(t, laptop, server); err == nil { + t.Fatal("a pull cut off part-way reported success") + } + server.blockBudget = -1 + if !laptop.version(t).Outdated() { + t.Fatalf("the laptop does not know it is behind: %+v", laptop.version(t)) + } + m.setOffline(server, true) + + // Server gone. The laptop and desk meet, repeatedly, both ways. + deskBefore := desk.files(t) + laptopBefore := laptop.files(t) + for i := 0; i < 3; i++ { + for _, pair := range [][2]*meshNode{{desk, laptop}, {laptop, desk}} { + res, err := syncPair(t, pair[0], pair[1]) + if err != nil { + t.Fatalf("%s↔%s: %v", pair[0].name, pair[1].name, err) + } + if res.Status == "conflict" { + t.Fatalf("%s↔%s raised a conflict between two copies of the same history", pair[0].name, pair[1].name) + } + } + } + if !sameSave(desk.files(t), deskBefore) { + t.Errorf("the desk changed while the newest device was away: was %s, now %s", + describe(deskBefore), describe(desk.files(t))) + } + if !sameSave(laptop.files(t), laptopBefore) { + t.Errorf("the laptop's partial copy changed while the newest device was away: was %s, now %s", + describe(laptopBefore), describe(laptop.files(t))) + } + if !desk.version(t).Outdated() { + t.Error("the desk never learned from the laptop that a newer version exists") + } + if m.deletes() != 0 { + t.Errorf("%d deletion(s) were sent to another device", m.deletes()) + } + + // The server returns; everyone ends with exactly its save. + m.setOffline(server, false) + settleAll(t, server, laptop, desk) + for _, d := range []*meshNode{server, laptop, desk} { + if !sameSave(d.files(t), newest) { + t.Errorf("%s does not hold the newest save: %s", d.name, describe(d.files(t))) + } + if v := d.version(t); v.Outdated() { + t.Errorf("%s still thinks it is behind: %+v", d.name, v) + } + } + if m.deletes() != 0 { + t.Errorf("%d deletion(s) were sent to another device", m.deletes()) + } +} + +// An empty folder — a new device, or one whose copy never arrived — is never +// one side of a conflict, and never makes the device with the save lose files. +func TestVersions_AnEmptyDeviceTakesTheSaveWithoutAsking(t *testing.T) { + m, n := newMesh(t, "full", "empty") + full, empty := n["full"], n["empty"] + writeMany(t, full.dir, "map", 100) + want := full.files(t) + + // The device with the save syncs first: nothing of its own may go. + res, err := syncPair(t, full, empty) + if err != nil { + t.Fatal(err) + } + if res.Status == "conflict" { + t.Fatal("a save against an empty folder raised a conflict") + } + if !sameSave(full.files(t), want) { + t.Fatalf("the device with the save lost files to an empty one: %s", describe(full.files(t))) + } + if res, err := syncPair(t, empty, full); err != nil || res.Status == "conflict" { + t.Fatalf("the empty device's sync: %+v, %v", res, err) + } + if !sameSave(empty.files(t), want) { + t.Errorf("the empty device did not take the save: %s", describe(empty.files(t))) + } + if m.deletes() != 0 { + t.Errorf("%d deletion(s) were sent", m.deletes()) + } +} + +// A save that existed before versions did, identical on two devices, is one +// version at once: no conflict, nothing moved. +func TestVersions_IdenticalSavesFromBeforeVersionsAgree(t *testing.T) { + _, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + writeMany(t, a.dir, "s", 5) + writeMany(t, b.dir, "s", 5) + res, err := syncPair(t, a, b) + if err != nil || res.Status != "in_sync" { + t.Fatalf("identical saves: %+v, %v", res, err) + } + if compareVersions(a.version(t).Vector, b.version(t).Vector) != versionEqual { + t.Errorf("identical saves got different versions: %v vs %v", a.version(t).Vector, b.version(t).Vector) + } +} + +// A deletion made in the game is part of the new version, and reaches the +// other device as that — whole — however many files it is. +func TestVersions_DeletionsTravelWithTheVersion(t *testing.T) { + m, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + writeMany(t, a.dir, "map", 20) + settleAll(t, a, b) + a.change(t, func(dir string) { + _ = os.RemoveAll(filepath.Join(dir, "map")) + write(t, dir, "fresh.sav", "new world") + }) + settleAll(t, a, b) + if !sameSave(b.files(t), a.files(t)) { + t.Errorf("b = %s, want %s", describe(b.files(t)), describe(a.files(t))) + } + if m.deletes() != 0 { + t.Errorf("the newer device sent %d deletion(s) itself; the older one takes them", m.deletes()) + } +} + +// Both devices changed the save while apart: that, and only that, is asked +// about. Resolution is mutual — keeping A's on A asks B in turn rather than +// overwriting it, and A is not asked again. Once B agrees, the answer spreads, +// and a third device that had changed nothing takes it without being asked. +func TestVersions_IndependentChangesAskEachSideOnceAndTheAnswerSpreads(t *testing.T) { + _, n := newMesh(t, "a", "b", "c") + a, b, c := n["a"], n["b"], n["c"] + write(t, a.dir, "slot.sav", "start") + settleAll(t, a, b, c) + + a.change(t, func(dir string) { write(t, dir, "slot.sav", "a-played") }) + b.change(t, func(dir string) { write(t, dir, "slot.sav", "b-played") }) + + res, err := syncPair(t, a, b) + if err != nil || res.Status != "conflict" { + t.Fatalf("independent changes: %+v, %v — want a conflict", res, err) + } + if _, err := a.eng.ResolveConflict(context.Background(), "game1", b.peer().ID, "keep-local"); err != nil { + t.Fatal(err) + } + bBefore := b.files(t) + settleAll(t, a, b, c) + if len(a.eng.ActiveConflicts()) != 0 { + t.Error("A was asked again after answering") + } + if len(b.eng.ActiveConflicts()) == 0 { + t.Fatal("B was never asked: A's answer must not decide B's save on its own") + } + if !sameSave(b.files(t), bBefore) { + t.Error("B's save changed before anyone there answered") + } + if _, err := b.eng.ResolveConflict(context.Background(), "game1", a.peer().ID, "keep-remote"); err != nil { + t.Fatal(err) + } + settleAll(t, a, b, c) + for _, d := range []*meshNode{b, c} { + if got := d.files(t)["slot.sav"]; got != a.files(t)["slot.sav"] { + t.Errorf("%s did not take the kept version", d.name) + } + if len(d.eng.ActiveConflicts()) != 0 { + t.Errorf("%s still has a conflict after both answered", d.name) + } + } +} + +// Both sides answer "keep mine" (or "keep both"): the later answer stands and +// both end on one save, instead of asking each other forever. +func TestVersions_TwoAnswersSettleOnTheLater(t *testing.T) { + _, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + write(t, a.dir, "slot.sav", "start") + settleAll(t, a, b) + a.change(t, func(dir string) { write(t, dir, "slot.sav", "a-played") }) + b.change(t, func(dir string) { write(t, dir, "slot.sav", "b-played") }) + if res, _ := syncPair(t, a, b); res.Status != "conflict" { + t.Fatalf("A: %+v, want a conflict", res) + } + if res, _ := syncPair(t, b, a); res.Status != "conflict" { + t.Fatalf("B: %+v, want a conflict", res) + } + if _, err := a.eng.ResolveConflict(context.Background(), "game1", b.peer().ID, "keep-local"); err != nil { + t.Fatal(err) + } + time.Sleep(5 * time.Millisecond) + if _, err := b.eng.ResolveConflict(context.Background(), "game1", a.peer().ID, "keep-local"); err != nil { + t.Fatal(err) + } + settleAll(t, a, b) + if got := a.files(t)["slot.sav"]; got != b.files(t)["slot.sav"] { + t.Fatal("the two never settled on one save") + } + if len(a.eng.ActiveConflicts())+len(b.eng.ActiveConflicts()) != 0 { + t.Error("a conflict is still open after both answered") + } + if got, _ := os.ReadFile(filepath.Join(a.dir, "slot.sav")); string(got) != "b-played" { + t.Errorf("A holds %q; B answered last, so its save should stand", got) + } +} + +// A change the watcher never reported — made while the app was closed — still +// becomes a version once the two devices meet, instead of leaving them on one +// version with different files, each waiting for the other. +func TestVersions_AnUnnoticedChangeIsFoundWhenTheDevicesMeet(t *testing.T) { + _, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + write(t, a.dir, "slot.sav", "start") + settleAll(t, a, b) + write(t, a.dir, "slot.sav", "played offline") // no NoteLocalChange + settleAll(t, b, a) + if got := b.files(t)["slot.sav"]; got != a.files(t)["slot.sav"] { + t.Errorf("the unnoticed change never reached b") + } +} + +// A device whose OpenSave keeps no save versions is never a source: its half +// of a save is not taken, and nothing here changes on its account. It is +// reported as needing an update. +func TestVersions_NothingIsTakenFromAnOldBuild(t *testing.T) { + _, n := newMesh(t, "current", "old") + current, old := n["current"], n["old"] + writeMany(t, current.dir, "map", 10) + writeMany(t, old.dir, "map", 3) + write(t, old.dir, "only-old.bin", "x") + old.proto = ProtoBatchFiles + before := current.files(t) + + res, err := syncPair(t, current, old) + if err != nil || res.Status != StatusPeerOutdatedApp { + t.Fatalf("sync with an old build: %+v, %v — want %s", res, err, StatusPeerOutdatedApp) + } + if !sameSave(current.files(t), before) { + t.Error("the save changed on account of a device without save versions") + } + if !current.eng.PeerNeedsUpdate(old.peer().ID) { + t.Error("the old build is not reported as needing an update") + } + old.proto = ProtoVersions + _, _ = syncPair(t, current, old) + if current.eng.PeerNeedsUpdate(old.peer().ID) { + t.Error("still reported as needing an update after it was updated") + } +} + +// Files a sync writes are not a change made here: the pull-in-progress record +// keeps them from becoming a version, even across a restart. +func TestVersions_PulledFilesAreNotANewVersion(t *testing.T) { + _, n := newMesh(t, "a", "b") + a, b := n["a"], n["b"] + writeMany(t, a.dir, "map", 10) + settleAll(t, a, b) + before := b.version(t).Vector + + a.change(t, func(dir string) { writeMany(t, dir, "map3", 10) }) + a.blockBudget = 3 + _, _ = syncPair(t, b, a) + a.blockBudget = -1 + + // What a restart leaves: the record says a pull was running, and the + // watcher reports the half-written folder as a change. + rec, _ := b.st.GetGameVersion("game1") + rec.Pulling = true + _ = b.st.SaveGameVersion(rec) + b.eng.NoteLocalChange("game1") + if got := b.version(t).Vector; compareVersions(got, before) != versionEqual { + t.Fatalf("a half-finished pull became a version of its own: %v (was %v)", got, before) + } + settleAll(t, a, b) + if !sameSave(b.files(t), a.files(t)) { + t.Errorf("b did not finish taking a's version: %s", describe(b.files(t))) + } +} diff --git a/internal/p2p/transport_lan.go b/internal/p2p/transport_lan.go index 1955b640..44dcf3d6 100644 --- a/internal/p2p/transport_lan.go +++ b/internal/p2p/transport_lan.go @@ -35,6 +35,23 @@ type lanTransport struct { var lanClient = &http.Client{Timeout: 30 * time.Second} +// lanBulkClient is for the two requests whose size grows with the save: a +// manifest, and a batch of files. Thirty seconds was fine for a few hundred +// files and not for a few hundred thousand over a VPN; the manifest then never +// arrived, every sync of that game failed, and the devices never converged. +// The sync's own context still bounds it (perPeerSyncTimeout), and the +// transport still gives up on a peer that does not answer at all. +var lanBulkClient = &http.Client{ + Timeout: 10 * time.Minute, + Transport: func() http.RoundTripper { + t := http.DefaultTransport.(*http.Transport).Clone() + // Headers arrive once the peer has built its manifest, which for a big + // save not hashed recently takes a while — but not this long. + t.ResponseHeaderTimeout = 3 * time.Minute + return t + }(), +} + func peerURL(peer syncengine.Peer, route string) string { return fmt.Sprintf("http://%s:%d/api/p2p%s", peer.Address, peer.Port, route) } @@ -54,9 +71,17 @@ func (t *lanTransport) FetchManifest(ctx context.Context, peer syncengine.Peer, if q.CoverURL != "" { params.Set("coverUrl", q.CoverURL) } + if q.IfHash != "" { + params.Set("ifHash", q.IfHash) + } var resp syncengine.ManifestResponse - err := t.getJSON(ctx, peer, peerURL(peer, "/manifest/"+gameID)+"?"+params.Encode(), &resp) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, peerURL(peer, "/manifest/"+gameID)+"?"+params.Encode(), nil) + if err != nil { + return resp, err + } + t.sign(req, peer, nil) + err = doJSONWith(lanBulkClient, req, &resp) return resp, err } @@ -67,17 +92,95 @@ func (t *lanTransport) FetchBlocks(ctx context.Context, peer syncengine.Peer, re // No encodings advertised: on a LAN the wire is typically faster than the // compressor, so the bytes saved cost more than they're worth. Responses // are still decoded, so a peer that compresses anyway is handled. - err := t.postJSON(ctx, peer, peerURL(peer, "/blocks/"+ref.GameID), map[string]any{ + // + // Given time for what it asks for. "LAN" here is any direct address, + // a VPN one (Tailscale) included, and a few MB of blocks at a VPN relay's + // hundred-odd KB a second outlasted the fixed 30 seconds every request + // used to get: the request failed, and with it the whole pull, over and + // over. Sized like the relay's (slowLinkBytesPerSec), within the bulk + // client's ceiling. + expected := int64(len(blockIndices)) * int64(max(blockSize, 1)) + ctx, cancel := context.WithTimeout(ctx, 30*time.Second+time.Duration(expected/(slowLinkBytesPerSec/2))*time.Second) + defer cancel() + raw, err := json.Marshal(map[string]any{ "relPath": ref.RelPath, "root": ref.Root, "blockIndices": blockIndices, "blockSize": blockSize, - }, &resp) + }) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, peerURL(peer, "/blocks/"+ref.GameID), bytes.NewReader(raw)) if err != nil { return nil, err } + req.Header.Set("Content-Type", "application/json") + t.sign(req, peer, raw) + if err := doJSONWith(lanBulkClient, req, &resp); err != nil { + return nil, err + } return decodeBlocks(resp.Blocks) } +// ProbeSpeed times fetching n bytes from the peer's speed test +// (syncengine.SpeedProber). The clock starts once the answer begins to +// arrive, so the figure is the link's rate rather than its round trip. +func (t *lanTransport) ProbeSpeed(ctx context.Context, peer syncengine.Peer, n int) (int64, time.Duration, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, peerURL(peer, fmt.Sprintf("/speedtest?n=%d", n)), nil) + if err != nil { + return 0, 0, err + } + t.sign(req, peer, nil) + resp, err := lanBulkClient.Do(req) + if err != nil { + return 0, 0, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 0, 0, fmt.Errorf("peer returned %d", resp.StatusCode) + } + start := time.Now() + got, err := io.Copy(io.Discard, resp.Body) + if err != nil { + return got, time.Since(start), err + } + return got, time.Since(start), nil +} + +// FetchFileBatch fetches the blocks of many files in one request. Only asked +// of a peer that advertised syncengine.ProtoBatchFiles. +func (t *lanTransport) FetchFileBatch(ctx context.Context, peer syncengine.Peer, gameID, root string, files []syncengine.FileBlocksRequest) ([]syncengine.FileBlocks, error) { + var resp struct { + Files []syncengine.FileBlocks `json:"files"` + } + raw, err := json.Marshal(map[string]any{"root": root, "files": files}) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, peerURL(peer, "/files/"+gameID), bytes.NewReader(raw)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + t.sign(req, peer, raw) + if err := doJSONWith(lanBulkClient, req, &resp); err != nil { + return nil, err + } + for i := range resp.Files { + if resp.Files[i].Error != "" { + continue + } + blocks, err := decodeBlocks(resp.Files[i].Blocks) + if err != nil { + return nil, fmt.Errorf("%s: %w", resp.Files[i].RelPath, err) + } + resp.Files[i].Blocks = blocks + } + return resp.Files, nil +} + func (t *lanTransport) DeleteRemote(ctx context.Context, peer syncengine.Peer, ref syncengine.FileRef) error { - return t.postJSON(ctx, peer, peerURL(peer, "/delete-file/"+ref.GameID), map[string]any{"relPath": ref.RelPath, "root": ref.Root}, nil) + // "versioned": this device keeps save versions; a peer that does too + // refuses deletions from one that does not (handleDeleteFile). + return t.postJSON(ctx, peer, peerURL(peer, "/delete-file/"+ref.GameID), map[string]any{"relPath": ref.RelPath, "root": ref.Root, "versioned": true}, nil) } // TriggerPeerPull tells a peer that this device holds newer content. @@ -224,7 +327,11 @@ func (t *lanTransport) sign(req *http.Request, peer syncengine.Peer, body []byte } func doJSON(req *http.Request, out any) error { - resp, err := lanClient.Do(req) + return doJSONWith(lanClient, req, out) +} + +func doJSONWith(client *http.Client, req *http.Request, out any) error { + resp, err := client.Do(req) if err != nil { return err } diff --git a/internal/p2p/transport_wan.go b/internal/p2p/transport_wan.go index 859e91ef..b2b5d90d 100644 --- a/internal/p2p/transport_wan.go +++ b/internal/p2p/transport_wan.go @@ -3,6 +3,7 @@ package p2p import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -90,7 +91,7 @@ func (t *wanTransport) FetchBlocks(ctx context.Context, peer syncengine.Peer, re } func (t *wanTransport) DeleteRemote(ctx context.Context, peer syncengine.Peer, ref syncengine.FileRef) error { - _, err := t.wan.Request(ctx, peer.ID, "/delete-file/"+ref.GameID, "POST", map[string]any{"relPath": ref.RelPath, "root": ref.Root}) + _, err := t.wan.Request(ctx, peer.ID, "/delete-file/"+ref.GameID, "POST", map[string]any{"relPath": ref.RelPath, "root": ref.Root, "versioned": true}) return err } @@ -154,6 +155,30 @@ func (t *routingTransport) FetchBlocks(ctx context.Context, peer syncengine.Peer return t.pick(peer).FetchBlocks(ctx, peer, ref, blockIndices, blockSize) } +// FetchFileBatch batches over the LAN only. Relay messages are sealed and +// size-limited per client (see BatchIndices), so the relay keeps pulling file +// by file. +func (t *routingTransport) FetchFileBatch(ctx context.Context, peer syncengine.Peer, gameID, root string, files []syncengine.FileBlocksRequest) ([]syncengine.FileBlocks, error) { + if peer.Wan() { + return nil, syncengine.ErrBatchUnsupported + } + b, ok := t.lan.(syncengine.BatchFetcher) + if !ok { + return nil, syncengine.ErrBatchUnsupported + } + return b.FetchFileBatch(ctx, peer, gameID, root, files) +} + +// ProbeSpeed tests LAN links only (syncengine.SpeedProber); the relay is the +// slowest link whatever a test would say. +func (t *routingTransport) ProbeSpeed(ctx context.Context, peer syncengine.Peer, n int) (int64, time.Duration, error) { + p, ok := t.lan.(syncengine.SpeedProber) + if peer.Wan() || !ok { + return 0, 0, errors.New("no speed test over the relay") + } + return p.ProbeSpeed(ctx, peer, n) +} + func (t *routingTransport) DeleteRemote(ctx context.Context, peer syncengine.Peer, ref syncengine.FileRef) error { return t.pick(peer).DeleteRemote(ctx, peer, ref) } diff --git a/internal/p2p/wanclient_handlers.go b/internal/p2p/wanclient_handlers.go index 95203dd3..766dbcdb 100644 --- a/internal/p2p/wanclient_handlers.go +++ b/internal/p2p/wanclient_handlers.go @@ -172,6 +172,7 @@ func (w *WanClient) handleMessage(ctx context.Context, msg RelayMessage) { if w.engine.Sync.Progress.OnSyncError != nil { w.engine.Sync.Progress.OnSyncError(msg.GameID, ev) } + w.engine.Sync.NotePeerPulled(msg.GameID, msg.From) } case "request": @@ -603,6 +604,9 @@ func (w *WanClient) serveManifest(route string, body json.RawMessage, peerID str if w.engine.holdingBack(game) { return 404, map[string]string{"error": syncengine.HeldMessage} } + if w.engine.Sync.PlayingHere(game.ID) { + return 503, map[string]string{"error": syncengine.PlayingMessage} + } // As on the LAN: a save a sync here is writing is not described // (syncengine/settle.go). Each request runs on its own goroutine, so the // wait does not hold up the relay traffic that finishes the write. @@ -614,15 +618,26 @@ func (w *WanClient) serveManifest(route string, body json.RawMessage, peerID str manifest, err := delta.BuildManifest(game.SavePath) if err != nil { + if _, statErr := os.Stat(game.SavePath); statErr != nil { + return 404, map[string]string{"error": syncengine.FolderMissingMessage} + } return 500, map[string]string{"error": err.Error()} } w.engine.Sync.NoteServed(game.ID, peerID, manifest) // syncengine/served.go + // "versions" says this device keeps save versions, and "version" is its + // version of this save (syncengine/version.go). Not "proto": over the + // relay that would also claim extra save locations and batched blocks, + // which this path does not serve. resp := map[string]any{ "gameId": gameID, "activeBranch": game.ActiveBranch, "manifest": manifest, "deletionConfirmed": w.engine.Sync.DeletionConfirmed(game.ID), + "versions": true, + } + if v := w.engine.Sync.LocalVersion(game, manifest); v != nil { + resp["version"] = v } if latest, err := w.engine.Snapshots.LatestSnapshot(gameID, ""); err == nil { - resp["latestSnapshot"] = syncengine.SnapshotInfo{ID: latest.ID, Timestamp: latest.Timestamp, Comment: latest.Comment} + resp["latestSnapshot"] = syncengine.SnapshotInfo{ID: latest.ID, Timestamp: latest.Timestamp, Comment: latest.Comment, ContentHash: latest.ContentHash} } else { resp["latestSnapshot"] = nil } @@ -678,6 +693,9 @@ func (w *WanClient) serveDeleteFile(route string, rawBody json.RawMessage, fromP gameID := route[strings.LastIndex(route, "/")+1:] var body struct { RelPath string `json:"relPath"` + // Versioned: the asking device keeps save versions (see the LAN + // route, handleDeleteFile). + Versioned bool `json:"versioned"` } if err := json.Unmarshal(rawBody, &body); err != nil || body.RelPath == "" { return 400, map[string]string{"error": "relPath is required."} @@ -686,6 +704,17 @@ func (w *WanClient) serveDeleteFile(route string, rawBody json.RawMessage, fromP if err != nil { return 404, map[string]string{"error": "Game not found."} } + if w.engine.Sync.PlayingHere(game.ID) { + return 503, map[string]string{"error": syncengine.PlayingMessage} + } + if game.AutoSync && !body.Versioned { + from := fromPeerID + if p, err := w.engine.Store.GetPeer(fromPeerID); err == nil && p.Name != "" { + from = p.Name + } + w.engine.Sync.RefusedOldBuildDelete(game.Name, from) + return 409, map[string]string{"error": syncengine.OldBuildDeleteMessage} + } if !delta.IsSafePath(game.SavePath, body.RelPath) { return 403, map[string]string{"error": "invalid path"} } diff --git a/internal/presets/detectsettings.go b/internal/presets/detectsettings.go new file mode 100644 index 00000000..5fe85f2b --- /dev/null +++ b/internal/presets/detectsettings.go @@ -0,0 +1,361 @@ +package presets + +import ( + "bytes" + "io/fs" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode/utf16" +) + +// Detecting a game's settings files by what they hold, for games the game +// database does not cover (DeviceSettings covers the ones it does). +// +// What tells a device's settings from a save is its contents: resolution, +// vsync, shadow quality, upscaler — the vocabulary of one machine's display, +// which no save has a use for. Names were measured against the Ludusavi +// manifest's own labels (16,700 settings entries, 9,000 save entries) and do +// not tell them apart on their own: "settings" names 1,521 settings files and +// 70 saves, ".ini" 1,831 and 137, ".json" 359 and 216. So a name only adds +// weight to what the contents say, by how reliably the manifest shows it to +// mean settings; a word that names saves rules a file out. +// +// What this misses stays syncing, which is how every file synced before; what +// it wrongly takes is caught by how a save behaves — it changes with every +// session — and given back (daemon/detectsettings.go). + +// DetectedSettings is one file found to be a game's settings. +type DetectedSettings struct { + Path string // relative to the save folder, slash-separated + Reason string +} + +const ( + detectMaxSize = 256 << 10 + detectMaxFiles = 4000 // text-like files read per folder, at most +) + +var detectExts = map[string]bool{ + "ini": true, "cfg": true, "conf": true, "config": true, "xml": true, "json": true, "lua": true, + "txt": true, "yaml": true, "yml": true, "toml": true, "vdf": true, "set": true, "settings": true, + "opt": true, "prefs": true, "local": true, "properties": true, +} + +// Terms of a machine's display and graphics, as sequences of words. Matched +// in identifiers split at case changes and separators, so +// "_enableFrameGeneration", "r_materialAniso" and "Shadow Quality" all count. +var strongTerms = [][]string{ + {"resolution"}, {"fullscreen"}, {"windowed"}, {"borderless"}, {"vsync"}, {"v", "sync"}, + {"antialiasing"}, {"anti", "aliasing"}, {"aniso"}, {"anisotropic"}, {"anisotropy"}, + {"msaa"}, {"fxaa"}, {"taa"}, {"smaa"}, {"ssao"}, {"hbao"}, {"ssr"}, {"bloom"}, {"gamma"}, + {"brightness"}, {"hdr"}, {"dlss"}, {"fsr"}, {"xess"}, {"upscaling"}, {"upscaler"}, + {"monitor"}, {"adapter"}, {"gpu"}, {"vram"}, {"tessellation"}, {"raytracing"}, + {"ray", "tracing"}, {"fov"}, {"field", "view"}, {"lod"}, {"framerate"}, {"frame", "rate"}, + {"refreshrate"}, {"refresh", "rate"}, {"frame", "generation"}, {"frame", "limit"}, + {"fps", "limit"}, {"max", "fps"}, {"texture", "quality"}, {"texture", "filtering"}, + {"shadow", "quality"}, {"shadow", "resolution"}, {"motion", "blur"}, {"depth", "field"}, + {"view", "distance"}, {"draw", "distance"}, {"render", "scale"}, {"resolution", "scale"}, + {"display", "mode"}, {"window", "mode"}, {"screen", "mode"}, {"ambient", "occlusion"}, + {"screen", "width"}, {"screen", "height"}, {"quality", "preset"}, {"graphics", "quality"}, + {"film", "grain"}, {"lens", "flare"}, {"volumetric"}, {"scalability"}, {"supersampling"}, + {"sharpening"}, {"chromatic", "aberration"}, {"vignette"}, {"directx"}, {"vulkan"}, + {"dx11"}, {"dx12"}, {"d3d11"}, {"d3d12"}, +} + +var weakTerms = []string{ + "width", "height", "quality", "preset", "driver", "spec", "shadows", "shadow", "texture", + "textures", "video", "display", "graphics", "render", "screen", "fps", "sync", "filtering", +} + +// Words in a file name, weighted by how reliably the Ludusavi manifest shows +// them to mean settings rather than a save. +var nameWeights = map[string]float64{ + "options": 1.5, "option": 1.5, "graphics": 1.5, "graphic": 1.5, "video": 1.5, "display": 1.5, + "render": 1.5, "engine": 1.5, "scalability": 1.5, "gameusersettings": 1.5, "screen": 1, + "settings": 0.5, "setting": 0.5, "config": 0.5, "configuration": 0.5, "cfg": 0.5, + "prefs": 0.5, "preferences": 0.5, +} + +// Words that name saves somewhere in the manifest often enough that a file +// carrying one is never taken, whatever it holds. Logs are not settings +// either, though leaving them out would harm nothing. +var saveWords = map[string]bool{ + "save": true, "saves": true, "savegame": true, "savedata": true, "slot": true, "progress": true, + "autosave": true, "quicksave": true, "world": true, "player": true, "character": true, + "campaign": true, "level": true, "levels": true, "achievement": true, "achievements": true, + "achievments": true, "stats": true, "unlock": true, "unlocks": true, "inventory": true, + "profile": true, "career": true, "highscore": true, "highscores": true, "score": true, + "scores": true, "records": true, "besttimes": true, "gamedata": true, "log": true, + "logs": true, "crash": true, "output": true, "history": true, +} + +var identRe = regexp.MustCompile(`[A-Za-z][A-Za-z0-9]*`) +var camelRe = regexp.MustCompile(`[A-Z]?[a-z]+|[A-Z]+(?:[^a-z]|$)|[0-9]+`) + +// words splits text into lowercase words: identifiers, then at case changes +// ("FrameGeneration" → frame generation), keeping a whole short identifier +// too ("dx11", "vsync"). +func words(text string) []string { + var out []string + for _, id := range identRe.FindAllString(text, -1) { + parts := camelRe.FindAllString(id, -1) + low := strings.ToLower(id) + if len(parts) > 1 && len(low) <= 6 { + out = append(out, low) + } + for _, p := range parts { + p = strings.ToLower(p) + if p == "of" || p == "the" || p == "b" || p == "r" || p == "e" { + continue // filler, and engine prefixes like r_ and bEnable + } + out = append(out, p) + } + } + return out +} + +// textOf returns a file's text, decoding UTF-16, or false for binary. +func textOf(b []byte) (string, bool) { + switch { + case bytes.HasPrefix(b, []byte{0xFF, 0xFE}), bytes.HasPrefix(b, []byte{0xFE, 0xFF}): + big := b[0] == 0xFE + b = b[2:] + u := make([]uint16, len(b)/2) + for i := range u { + if big { + u[i] = uint16(b[2*i])<<8 | uint16(b[2*i+1]) + } else { + u[i] = uint16(b[2*i+1])<<8 | uint16(b[2*i]) + } + } + return string(utf16.Decode(u)), true + case len(b) >= 4 && b[1] == 0 && b[3] == 0 && b[0] != 0 && b[2] != 0: + // UTF-16LE without a mark, as some engines write their XML. + u := make([]uint16, len(b)/2) + for i := range u { + u[i] = uint16(b[2*i+1])<<8 | uint16(b[2*i]) + } + return string(utf16.Decode(u)), true + case bytes.IndexByte(b, 0) >= 0: + return "", false + } + return string(b), true +} + +// contentScore counts the distinct display terms a text holds. +func contentScore(text string) (strong int, score float64) { + ws := words(text) + present := make(map[string]bool, len(ws)) + pairs := make(map[[2]string]bool, len(ws)) + for i, w := range ws { + present[w] = true + if i+1 < len(ws) { + pairs[[2]string{w, ws[i+1]}] = true + } + } + for _, t := range strongTerms { + if (len(t) == 1 && present[t[0]]) || (len(t) == 2 && pairs[[2]string{t[0], t[1]}]) { + strong++ + } + } + weak := 0.0 + for _, w := range weakTerms { + if present[w] { + weak += 0.5 + } + } + if weak > 2 { + weak = 2 + } + return strong, float64(strong) + weak +} + +// nameSignal weighs a file's name: the support it lends, and whether a word +// in it names saves. +func nameSignal(rel string) (weight float64, saveWord, strongName bool) { + base := path.Base(rel) + ext := strings.ToLower(strings.TrimPrefix(path.Ext(base), ".")) + stem := strings.TrimSuffix(base, path.Ext(base)) + ws := words(stem) + low := strings.ToLower(stem) + for w := range nameWeights { + if len(w) >= 6 && strings.Contains(low, w) { + ws = append(ws, w) + } + } + seen := map[string]bool{} + for _, w := range ws { + if seen[w] { + continue + } + seen[w] = true + if saveWords[w] { + saveWord = true + } + weight += nameWeights[w] + if nameWeights[w] >= 1.5 { + strongName = true + } + } + if strings.Contains(low, "save") || strings.Contains(low, "slot") || ext == "log" { + saveWord = true + } + if ext == "ini" || ext == "cfg" || ext == "conf" { + weight += 0.5 + } + if weight > 2 { + weight = 2 + } + return weight, saveWord, strongName +} + +// unrealSettings reports a file in an Unreal Engine game's settings folder, +// Saved/Config/: the engine writes a machine's settings there and +// nothing else, often as files too short to judge by content. +func unrealSettings(rel string) bool { + segs := strings.Split(strings.ToLower(rel), "/") + if len(segs) < 2 || !strings.HasSuffix(segs[len(segs)-1], ".ini") { + return false + } + for i := 0; i+1 < len(segs)-1; i++ { + if segs[i] == "config" && (strings.HasPrefix(segs[i+1], "windows") || strings.HasPrefix(segs[i+1], "linux") || + strings.HasPrefix(segs[i+1], "mac")) { + return true + } + } + return false +} + +// judgeSettings decides one file from its path and bytes. +func judgeSettings(rel string, b []byte) (bool, string) { + weight, saveWord, strongName := nameSignal(rel) + if saveWord && !strongName { + return false, "" + } + if unrealSettings(rel) && !saveWord { + return true, "Unreal Engine settings folder" + } + text, ok := textOf(b) + if !ok { + return false, "" + } + strong, score := contentScore(text) + switch { + case saveWord: + // "user_engine_option_save.xml": a strong settings word and a save + // word together. Taken only on overwhelming contents. + if strong >= 6 && score+weight >= 8 { + return true, "display settings in its contents" + } + case len(b) > 32<<10: + // A long file holds a lot besides; a save with a settings section is + // long. Only a file that is mostly settings is taken. + if strong >= 6 { + return true, "display settings in its contents" + } + case strong >= 3 && score+weight >= 4: + return true, "display settings in its contents" + case strong >= 2 && weight >= 1.5 && score+weight >= 4: + return true, "display settings in its contents, and its name" + } + return false, "" +} + +// DetectSettings finds the settings files in root by what they hold. The +// game's save locations in the game database (by name or App ID) are never +// taken, whatever they hold. +func (ds *DeviceSettings) DetectSettings(name, appID, root string) []DetectedSettings { + var savePats []string + if ds != nil { + g, ok := ds.byID[appID] + if !ok || appID == "" { + g, ok = ds.byName[normalizeGameName(stripNameSuffixes(name))] + } + if ok { + for _, vars := range ds.sc.ludusaviVarSets(g, ds.protonIx) { + for _, tpl := range g.Paths { + for _, expanded := range expandTemplate(tpl, vars, ds.bases(g.Installs), g.SteamID) { + if rel, ok := relativeUnder(root, expanded); ok { + savePats = append(savePats, rel) + } + } + } + } + } + } + return detectIn(root, savePats) +} + +// isSaveLocation reports whether rel is, or is inside, one of the game +// database's save locations (relative patterns). A location naming a folder +// covers what is in it; one with a file extension names that file. +func isSaveLocation(rel string, savePats []string) bool { + segs := strings.Split(strings.ToLower(rel), "/") + for _, p := range savePats { + ps := strings.Split(p, "/") + if len(ps) > len(segs) { + continue + } + match := true + for i, pc := range ps { + if ok, err := path.Match(pc, segs[i]); err != nil || !ok { + match = false + break + } + } + if !match { + continue + } + if len(ps) < len(segs) || path.Ext(ps[len(ps)-1]) != "" { + return true + } + } + return false +} + +func detectIn(root string, savePats []string) []DetectedSettings { + var out []DetectedSettings + read := 0 + _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() { + if p != root && strings.HasPrefix(d.Name(), ".") { + return filepath.SkipDir + } + return nil + } + ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(p), ".")) + if !detectExts[ext] || read >= detectMaxFiles { + return nil + } + rel, err := filepath.Rel(root, p) + if err != nil { + return nil + } + rel = filepath.ToSlash(rel) + if strings.Contains(strings.ToLower(rel), ".sync-conflict-") || isSaveLocation(rel, savePats) { + return nil + } + info, err := d.Info() + if err != nil || info.Size() > detectMaxSize { + return nil + } + b, err := os.ReadFile(p) + if err != nil { + return nil + } + read++ + if ok, reason := judgeSettings(rel, b); ok { + out = append(out, DetectedSettings{Path: rel, Reason: reason}) + } + return nil + }) + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} diff --git a/internal/presets/detectsettings_test.go b/internal/presets/detectsettings_test.go new file mode 100644 index 00000000..f499e375 --- /dev/null +++ b/internal/presets/detectsettings_test.go @@ -0,0 +1,65 @@ +package presets + +import ( + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf16" +) + +func utf16le(s string) []byte { + u := utf16.Encode([]rune(s)) + b := []byte{0xFF, 0xFE} + for _, r := range u { + b = append(b, byte(r), byte(r>>8)) + } + return b +} + +func TestJudgeSettings(t *testing.T) { + crimson := ` + + +` + doom := "configVersion 9\nr_materialAniso \"16\"\nr_mode \"18\"\nr_fullscreen \"1\"\nr_swapInterval \"1\"\nr_lodScale \"1\"\nr_gamma \"1.2\"\n" + cases := []struct { + name string + path string + data []byte + want bool + }{ + {"display settings", "videoconfig.txt", []byte("setting.fullscreen 1\nsetting.mat_vsync 0\nsetting.gamma 1\nsetting.defaultres 1920\nsetting.ssao 1\n"), true}, + {"camelCase identifiers, save word outweighed", "user_engine_option_save.xml", []byte(crimson), true}, + {"engine abbreviations", "DOOMEternalConfig.local", []byte(doom), true}, + {"UTF-16", "GraphicsConfig.xml", utf16le(`11233`), true}, + {"Unreal settings folder", "Saved/Config/WindowsNoEditor/Input.ini", []byte("[/Script/Engine.InputSettings]\n"), true}, + {"save word, weak settings", "savegame.json", []byte(`{"fullscreen":true,"vsync":true,"resolution":"1080p","gold":12}`), false}, + {"progress in a settings-like file", "config.cfg", []byte("level 4\ngold 120\nhealth 80\n"), false}, + {"a log mentioning the GPU", "output_log.txt", []byte("GPU: RTX; resolution 1920; vsync on; fullscreen; monitor 1; hdr off"), false}, + {"binary", "graphics.cfg", []byte{0x01, 0x00, 0x02, 0x00, 0x03}, false}, + {"long save with a settings section", "state.json", []byte(`{"fullscreen":1,"vsync":1,"resolution":1,"gamma":1,"hdr":0,` + strings.Repeat(`"item":1,`, 5000) + `"x":0}`), false}, + } + for _, c := range cases { + if got, _ := judgeSettings(c.path, c.data); got != c.want { + t.Errorf("%s: %s judged settings=%v, want %v", c.name, c.path, got, c.want) + } + } +} + +// The game database's save locations are never taken, whatever they hold. +func TestDetectIn_SaveLocationsAreNeverTaken(t *testing.T) { + root := t.TempDir() + settings := "fullscreen=1\nvsync=1\nresolution=1920x1080\nshadow_quality=3\nanti_aliasing=2\n" + for _, p := range []string{"video.ini", "slots/1/video.ini", "keep/video.ini"} { + full := filepath.Join(root, filepath.FromSlash(p)) + _ = os.MkdirAll(filepath.Dir(full), 0o777) + if err := os.WriteFile(full, []byte(settings), 0o666); err != nil { + t.Fatal(err) + } + } + got := detectIn(root, []string{"slots", "keep/video.ini"}) + if len(got) != 1 || got[0].Path != "video.ini" { + t.Errorf("detected %+v, want only video.ini", got) + } +} diff --git a/internal/presets/devicesettings.go b/internal/presets/devicesettings.go new file mode 100644 index 00000000..87a75a78 --- /dev/null +++ b/internal/presets/devicesettings.go @@ -0,0 +1,154 @@ +package presets + +import ( + "path" + "sort" + "strings" +) + +// DeviceSettings answers, for a tracked game, which of the files in its save +// folders are its settings rather than its save: what the Ludusavi manifest +// tags "config" and not "save" (indexedGame.Config). A device's graphics +// settings are the usual case — a game that keeps them beside its saves had +// one PC's resolution and quality handed to every other. +// +// The answer is a list of exclusion patterns, in the syntax of a game's own +// exclusion rules and relative to the save folder they fall in, so it is +// applied by exactly the machinery that applies those: never synced in either +// direction, never deleted on another device, still in every snapshot. +type DeviceSettings struct { + sc *Scanner + byID map[string]indexedGame + byName map[string]indexedGame + protonIx map[string][]string + bases func([]string) []string + // all is every game in the index, for looking one up by what is on + // disk (gamedb.go); blocked, the folders no save location may be. + all []indexedGame + blocked map[string]bool + // byInstall: games by install folder name, built when first asked. + byInstall map[string][]indexedGame +} + +// DeviceSettings loads the manifest index once for any number of lookups. +// Nil when there is no index. +func (sc *Scanner) DeviceSettings() *DeviceSettings { + if sc == nil || sc.CacheFile == "" { + return nil + } + idx := sc.loadManifestIndex() + if len(idx) == 0 { + return nil + } + ds := &DeviceSettings{sc: sc, byID: map[string]indexedGame{}, byName: map[string]indexedGame{}} + ds.all = idx + for _, g := range idx { + if len(g.Config) == 0 && len(g.Paths) == 0 { + continue + } + if g.SteamID != "" { + ds.byID[g.SteamID] = g + } + if key := normalizeGameName(g.Name); key != "" { + ds.byName[key] = g + } + } + if sc.goos() == "linux" { + ds.protonIx = sc.protonPrefixIndex() + } + if sc.goos() == "windows" { + ds.blocked = blockedRoots(windowsPathVars()) + } else { + ds.blocked = linuxBlockedRoots(sc.linuxHome()) + } + ds.bases = sc.installBaseCandidates() + return ds +} + +// Patterns returns the exclusion patterns for a game's settings that lie +// inside one of roots (its save folders; the first is the main one), each +// relative to the root it is in. Patterns for any other root than the first +// are not returned: a game's rules are written against its main folder. +func (ds *DeviceSettings) Patterns(name, appID string, roots []string) []string { + if ds == nil || len(roots) == 0 { + return nil + } + g, ok := ds.byID[appID] + if !ok || appID == "" { + g, ok = ds.byName[normalizeGameName(stripNameSuffixes(name))] + } + if !ok { + return nil + } + installBases := ds.bases(g.Installs) + seen := map[string]bool{} + var out []string + for _, vars := range ds.sc.ludusaviVarSets(g, ds.protonIx) { + for _, tpl := range g.Config { + for _, expanded := range expandTemplate(tpl, vars, installBases, g.SteamID) { + rel, ok := relativeUnder(roots[0], expanded) + if !ok || coversEverything(rel) || seen[rel] { + continue + } + seen[rel] = true + out = append(out, "/"+rel) + } + } + } + sort.Strings(out) + return out +} + +// relativeUnder returns pattern relative to root when it names something +// strictly inside it, comparing component by component so that a wildcard the +// template had (a store user id) matches the folder actually tracked. +// Case-insensitive, as exclusion rules are. +func relativeUnder(root, pattern string) (string, bool) { + split := func(p string) []string { + // Either separator on every system: a pattern written for Windows + // (Ludusavi's data, a Proton prefix) is matched on Linux too, and + // path.Match would read a backslash as an escape. + p = strings.ToLower(path.Clean(strings.ReplaceAll(p, `\`, "/"))) + var parts []string + for _, c := range strings.Split(p, "/") { + if c != "" { + parts = append(parts, c) + } + } + return parts + } + r, p := split(root), split(pattern) + if len(r) == 0 || len(p) <= len(r) { + return "", false + } + for i, rc := range r { + if ok, err := path.Match(p[i], rc); err != nil || !ok { + return "", false + } + } + return strings.Join(p[len(r):], "/"), true +} + +// coversEverything reports a relative pattern that would exclude the whole +// folder — a settings entry naming the folder the save is tracked at. Never +// applied: a wrong exclusion loses saves, a missed one only syncs a setting. +func coversEverything(rel string) bool { + first := strings.SplitN(rel, "/", 2)[0] + return strings.Trim(first, "*") == "" || first == "*.*" +} + +// InstallNames returns the names the game database knows a game's install +// folder by ("Crimson Desert"), found by App ID or name. +func (ds *DeviceSettings) InstallNames(name, appID string) []string { + if ds == nil { + return nil + } + g, ok := ds.byID[appID] + if !ok || appID == "" { + g, ok = ds.byName[normalizeGameName(stripNameSuffixes(name))] + } + if !ok { + return nil + } + return append([]string(nil), g.Installs...) +} diff --git a/internal/presets/devicesettings_test.go b/internal/presets/devicesettings_test.go new file mode 100644 index 00000000..019641c4 --- /dev/null +++ b/internal/presets/devicesettings_test.go @@ -0,0 +1,82 @@ +package presets + +import ( + "path/filepath" + "reflect" + "runtime" + "testing" +) + +// Crimson Desert's shape: the save is a per-account folder inside the tracked +// one, and the graphics settings sit beside it. The settings are named; the +// save is not; an entry tagged both counts as save. +func TestDeviceSettings_SettingsInsideTheSaveFolder(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("templates below use Windows placeholders") + } + home := t.TempDir() + t.Setenv("USERPROFILE", home) + t.Setenv("HOME", home) + local := filepath.Join(home, "AppData", "Local") + t.Setenv("LOCALAPPDATA", local) + + sc := manifestScanner(t, ` +Pearl Game: + files: + "/Pearl/CD/save/": + tags: [save] + "/Pearl/CD/save/user_engine_option_save.xml": + tags: [config] + "/Pearl/CD/save/both.cfg": + tags: [config, save] + "/Pearl/CD/save/*": + tags: [config] + "/Pearl/Elsewhere/settings.ini": + tags: [config] + steam: + id: 123 +`) + ds := sc.DeviceSettings() + if ds == nil { + t.Fatal("no index") + } + root := filepath.Join(local, "Pearl", "CD", "save") + want := []string{"/user_engine_option_save.xml"} + if got := ds.Patterns("Pearl Game", "123", []string{root}); !reflect.DeepEqual(got, want) { + t.Errorf("by App ID: %v, want %v", got, want) + } + if got := ds.Patterns("Pearl Game", "", []string{root}); !reflect.DeepEqual(got, want) { + t.Errorf("by name: %v, want %v", got, want) + } + // Tracked at the per-account folder: the settings are not inside it. + if got := ds.Patterns("Pearl Game", "123", []string{filepath.Join(root, "22202")}); len(got) != 0 { + t.Errorf("at the account folder: %v, want none", got) + } + if got := ds.Patterns("Another Game", "999", []string{root}); len(got) != 0 { + t.Errorf("unknown game: %v", got) + } +} + +func TestRelativeUnder(t *testing.T) { + for _, c := range []struct { + root, pattern, want string + ok bool + }{ + {`C:\Users\A\Saved Games\Game`, `C:\Users\A\Saved Games\Game\game.cfg`, "game.cfg", true}, + {`C:\Users\A/Saved Games/Game`, `c:\users\a\saved games\game\Config\x.ini`, "config/x.ini", true}, + {`C:\Users\A\Game\save\22202`, `C:\Users\A\Game\save\*\opt.xml`, "opt.xml", true}, + {`C:\Users\A\Game\save`, `C:\Users\A\*\save\opt.xml`, "opt.xml", true}, + {`C:\Users\A\Game`, `C:\Users\A\Game`, "", false}, + {`C:\Users\A\Game`, `C:\Users\A\Other\x.ini`, "", false}, + } { + got, ok := relativeUnder(c.root, c.pattern) + if got != c.want || ok != c.ok { + t.Errorf("relativeUnder(%q, %q) = %q, %v; want %q, %v", c.root, c.pattern, got, ok, c.want, c.ok) + } + } + for rel, want := range map[string]bool{"*": true, "*.*": true, "**/x": true, "*.cfg": false, "config": false} { + if coversEverything(rel) != want { + t.Errorf("coversEverything(%q) = %v", rel, !want) + } + } +} diff --git a/internal/presets/gamedb.go b/internal/presets/gamedb.go new file mode 100644 index 00000000..6afa159d --- /dev/null +++ b/internal/presets/gamedb.go @@ -0,0 +1,249 @@ +package presets + +import ( + "path" + "path/filepath" + "sort" + "strings" +) + +// Finding a game in the game database by what is on this device: a program +// that is running or was picked, a folder that was picked, or a name typed. +// What it answers is everything tracking needs and a folder picked by hand +// lacked: the game's name, its Steam App ID (cover art, launching, sessions), +// and where its saves are on this device. + +// GameMatch is a game from the game database, as found on this device. +type GameMatch struct { + Name string `json:"name"` + AppID string `json:"appId,omitempty"` + // SavePaths are the game's save locations that exist on this device, + // most likely first. + SavePaths []string `json:"savePaths"` + // InstallDir is where it is installed, when that is known. + InstallDir string `json:"installDir,omitempty"` + // TrackedID and TrackedName name the game already tracked here that + // this one is — its folder is, holds or is held by one of SavePaths, or it + // has the App ID. Filled in by the daemon. + TrackedID string `json:"trackedId,omitempty"` + TrackedName string `json:"trackedName,omitempty"` +} + +// savePathsHere lists a game's save locations that exist on this device. +func (ds *DeviceSettings) savePathsHere(g indexedGame) []string { + if ds == nil || len(g.Paths) == 0 { + return nil + } + return expandGamePaths(g, ds.sc.ludusaviVarSets(g, ds.protonIx), ds.bases, ds.blocked) +} + +func (ds *DeviceSettings) matchOf(g indexedGame, installDir string) GameMatch { + return GameMatch{Name: g.Name, AppID: g.SteamID, SavePaths: ds.savePathsHere(g), InstallDir: installDir} +} + +// Search finds games by name: the whole name first, then names that begin +// with what was typed, then names that contain it. +func (ds *DeviceSettings) Search(query string, limit int) []GameMatch { + q := normalizeGameName(query) + if ds == nil || len(q) < 2 { + return nil + } + type hit struct { + g indexedGame + rank int + } + var hits []hit + for _, g := range ds.all { + n := normalizeGameName(g.Name) + switch { + case n == q: + hits = append(hits, hit{g, 0}) + case strings.HasPrefix(n, q): + hits = append(hits, hit{g, 1}) + case strings.Contains(n, q): + hits = append(hits, hit{g, 2}) + } + } + sort.SliceStable(hits, func(i, j int) bool { + if hits[i].rank != hits[j].rank { + return hits[i].rank < hits[j].rank + } + return len(hits[i].g.Name) < len(hits[j].g.Name) + }) + if len(hits) > limit { + hits = hits[:limit] + } + out := make([]GameMatch, 0, len(hits)) + for _, h := range hits { + out = append(out, ds.matchOf(h.g, "")) + } + return out +} + +// IdentifyProgram finds the game a program belongs to: by the Steam install +// folder it is in, or by the name of a folder above it that is the game's +// install folder in the game database ("G:\Games\Crimson Desert"). +func (ds *DeviceSettings) IdentifyProgram(exe string) (GameMatch, bool) { + if ds == nil || exe == "" { + return GameMatch{}, false + } + exe = filepath.Clean(exe) + byAppID, _ := ds.sc.InstallDirs() + for appID, dir := range byAppID { + if dir != "" && isSubPath(dir, exe) { + if g, ok := ds.gameByID(appID); ok { + return ds.matchOf(g, dir), true + } + } + } + byFolder := ds.installNameIndex() + for dir := filepath.Dir(exe); dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + if gs := byFolder[installKey(filepath.Base(dir))]; len(gs) > 0 { + return ds.matchOf(pickGame(gs), dir), true + } + } + return GameMatch{}, false +} + +// IdentifyFolder finds the game a save folder belongs to: the game whose save +// location in the game database is this folder, or holds it, or is held by it +// (a folder picked one level too high). +func (ds *DeviceSettings) IdentifyFolder(folder string) (GameMatch, bool) { + return ds.identifyFolder(folder, 0) +} + +// IdentifyFolderStrictly is IdentifyFolder for naming a game nobody asked +// about: a folder holding the save location only counts when that is at most +// one folder further down ("Duckov" holding "DuckovSaves"), never a folder +// like Documents that holds many games'. +func (ds *DeviceSettings) IdentifyFolderStrictly(folder string) (GameMatch, bool) { + return ds.identifyFolder(folder, 1) +} + +// identifyFolder: maxDepth limits how far below the folder a save location +// held by it may be, 0 for no limit. +func (ds *DeviceSettings) identifyFolder(folder string, maxDepth int) (GameMatch, bool) { + if ds == nil || folder == "" { + return GameMatch{}, false + } + folder = filepath.Clean(folder) + low := strings.ToLower(folder) + best, bestScore := indexedGame{}, 0 + for _, g := range ds.all { + if len(g.Paths) == 0 { + continue + } + // An install folder is only worked out for a game whose install + // folder name is in the path at all: doing it for every game in the + // database would look on disk tens of thousands of times. + bases := func([]string) []string { return nil } + for _, inst := range g.Installs { + if inst != "" && strings.Contains(low, strings.ToLower(inst)) { + bases = ds.bases + break + } + } + for _, vars := range ds.sc.ludusaviVarSets(g, ds.protonIx) { + for _, tpl := range g.Paths { + for _, expanded := range expandTemplate(tpl, vars, bases(g.Installs), g.SteamID) { + score := 0 + switch { + case sameUnder(folder, expanded): + score = 3 + case under(expanded, folder): + score = 2 // a folder inside the save location + case under(folder, expanded): + // The save location inside the folder picked, counted + // to the folder a file location is in. + rel, _ := relativeUnder(folder, expanded) + if path.Ext(path.Base(rel)) != "" { + rel = path.Dir(rel) + } + if maxDepth == 0 || strings.Count(rel, "/") < maxDepth { + score = 1 + } + } + if score > bestScore || (score == bestScore && score > 0 && better(g, best)) { + best, bestScore = g, score + } + } + } + } + } + if bestScore == 0 { + return GameMatch{}, false + } + return ds.matchOf(best, ""), true +} + +// sameUnder reports whether path is pattern (a template expanded, possibly +// with wildcards), compared component by component. +func sameUnder(path, pattern string) bool { + rel, ok := relativeUnder(path, pattern+string(filepath.Separator)+"x") + return ok && rel == "x" +} + +// under reports whether path lies strictly inside pattern. +func under(pattern, path string) bool { + _, ok := relativeUnder(pattern, path) + return ok +} + +// better prefers a game with a Steam App ID — cover art, launching — when two +// match equally. +func better(g, than indexedGame) bool { + return g.SteamID != "" && than.SteamID == "" +} + +func pickGame(gs []indexedGame) indexedGame { + best := gs[0] + for _, g := range gs[1:] { + if better(g, best) { + best = g + } + } + return best +} + +func (ds *DeviceSettings) gameByID(appID string) (indexedGame, bool) { + if g, ok := ds.byID[appID]; ok { + return g, true + } + for _, g := range ds.all { + if g.SteamID == appID { + return g, true + } + } + return indexedGame{}, false +} + +// installNameIndex maps install folder names, normalised, to the games that +// use them, for names specific enough to say which game a folder is. +func (ds *DeviceSettings) installNameIndex() map[string][]indexedGame { + if ds.byInstall != nil { + return ds.byInstall + } + ds.byInstall = map[string][]indexedGame{} + for _, g := range ds.all { + for _, inst := range g.Installs { + n := installKey(inst) + if len(n) < 4 || genericInstallNames[n] { + continue + } + ds.byInstall[n] = append(ds.byInstall[n], g) + } + } + return ds.byInstall +} + +var genericInstallNames = map[string]bool{ + "game": true, "games": true, "data": true, "bin": true, "binaries": true, "launcher": true, + "client": true, "common": true, "steam": true, "app": true, +} + +// installKey is a folder name as compared with the game database's install +// folder names: normalised, and without spaces, so "Project Zomboid" is the +// "ProjectZomboid" the database knows. +func installKey(name string) string { + return strings.ReplaceAll(normalizeGameName(name), " ", "") +} diff --git a/internal/presets/gamedb_test.go b/internal/presets/gamedb_test.go new file mode 100644 index 00000000..5470d86c --- /dev/null +++ b/internal/presets/gamedb_test.go @@ -0,0 +1,102 @@ +package presets + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestGameDB_FindsGamesByNameFolderAndProgram(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("templates below use Windows placeholders") + } + home := t.TempDir() + t.Setenv("USERPROFILE", home) + t.Setenv("HOME", home) + local := filepath.Join(home, "AppData", "Local") + t.Setenv("LOCALAPPDATA", local) + save := filepath.Join(local, "Pearl", "CD", "save") + if err := os.MkdirAll(filepath.Join(save, "123"), 0o777); err != nil { + t.Fatal(err) + } + + sc := manifestScanner(t, ` +Pearl Game: + files: + "/Pearl/CD/save": + tags: [save] + installDir: + Pearl Game: {} + steam: + id: 3321460 +Project Things: + files: + "/Things/Saves": + tags: [save] + installDir: + ProjectThings: {} +`) + ds := sc.DeviceSettings() + if ds == nil { + t.Fatal("no index") + } + + got := ds.Search("pearl", 5) + if len(got) != 1 || got[0].AppID != "3321460" || len(got[0].SavePaths) != 1 || got[0].SavePaths[0] != save { + t.Errorf("search = %+v", got) + } + + for _, folder := range []string{save, filepath.Join(save, "123"), filepath.Join(local, "Pearl", "CD")} { + if m, ok := ds.IdentifyFolder(folder); !ok || m.Name != "Pearl Game" { + t.Errorf("folder %s: %+v %v", folder, m, ok) + } + } + if _, ok := ds.IdentifyFolder(filepath.Join(home, "Elsewhere")); ok { + t.Error("an unrelated folder was taken for a game") + } + + m, ok := ds.IdentifyProgram(`G:\Games\Pearl Game\bin64\game.exe`) + if !ok || m.Name != "Pearl Game" || m.InstallDir != `G:\Games\Pearl Game` { + t.Errorf("program in a folder of its own = %+v %v", m, ok) + } + if m, ok := ds.IdentifyProgram(`D:\Stuff\Project Things\run.exe`); !ok || m.Name != "Project Things" { + t.Errorf("install folder written with a space the database leaves out = %+v %v", m, ok) + } + if _, ok := ds.IdentifyProgram(`C:\Tools\editor.exe`); ok { + t.Error("an unrelated program was taken for a game") + } +} + +// Naming a game nobody asked about: a folder one level above its save +// location counts, a folder holding many games' does not. +func TestGameDB_IdentifyFolderStrictly(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("templates below use Windows placeholders") + } + home := t.TempDir() + t.Setenv("USERPROFILE", home) + t.Setenv("HOME", home) + low := filepath.Join(home, "AppData", "LocalLow") + if err := os.MkdirAll(filepath.Join(low, "Soda", "Duck", "Saves"), 0o777); err != nil { + t.Fatal(err) + } + sc := manifestScanner(t, ` +Duck Game: + files: + "/AppData/LocalLow/Soda/Duck/Saves/*.sav": + tags: [save] + steam: + id: 3167020 +`) + ds := sc.DeviceSettings() + if m, ok := ds.IdentifyFolderStrictly(filepath.Join(low, "Soda", "Duck")); !ok || m.AppID != "3167020" { + t.Errorf("the folder above the save location: %+v %v", m, ok) + } + if _, ok := ds.IdentifyFolderStrictly(low); ok { + t.Error("a folder holding many games' saves was named after one") + } + if _, ok := ds.IdentifyFolder(low); !ok { + t.Error("asked about, a folder holding the save location further down is not recognised") + } +} diff --git a/internal/presets/ludusavi.go b/internal/presets/ludusavi.go index 7a6ea160..31027066 100644 --- a/internal/presets/ludusavi.go +++ b/internal/presets/ludusavi.go @@ -68,6 +68,11 @@ type indexedGame struct { // index built by an older release, which reads as "none" — the field is // additive, so an index on disk from a previous version still loads. Registry []string `json:"r,omitempty"` + // Config is where the game keeps its settings and nothing else: entries + // the manifest tags "config" and not "save". A device's own graphics + // settings among them; inside a tracked save folder they are left out of + // syncing (DeviceSettingsPatterns). + Config []string `json:"c,omitempty"` } // manifestPaths derives the manifest + index locations from the scanner's @@ -89,7 +94,7 @@ func (sc *Scanner) manifestPaths() (yamlPath, indexPath string) { // reading the file it knows and a newer one builds its own, so a user moving // between the two is never handed an index whose shape their code predates. // Bump this whenever indexedGame gains or loses a field that a scan depends on. -const indexFileName = "ludusavi-index-v2.json" +const indexFileName = "ludusavi-index-v3.json" // scanLudusavi expands the manifest's save-path templates and returns the // locations that actually exist on this machine. @@ -728,13 +733,17 @@ func buildManifestIndex(yamlPath string) []indexedGame { games := make([]indexedGame, 0, len(manifest)) for name, mg := range manifest { - var paths []string + var paths, config []string for tpl, entry := range mg.Files { + if entryIsConfigEntry(tpl, entry) { + config = append(config, tpl) + } if !entryIsSaveEntry(tpl, entry) { continue } paths = append(paths, tpl) } + sort.Strings(config) var regKeys []string for key, entry := range mg.Registry { if !registryEntryIsSave(key, entry) { @@ -763,7 +772,7 @@ func buildManifestIndex(yamlPath string) []indexedGame { continue } sort.Strings(regKeys) // stable index across rebuilds - g := indexedGame{Name: name, Paths: paths, Registry: regKeys, SteamID: steamID} + g := indexedGame{Name: name, Paths: paths, Registry: regKeys, SteamID: steamID, Config: config} for dir := range mg.InstallDir { g.Installs = append(g.Installs, dir) } @@ -799,6 +808,34 @@ func registryEntryIsSave(key string, entry manifestFileEntry) bool { } func entryIsSaveEntry(tpl string, entry manifestFileEntry) bool { + if len(entry.Tags) > 0 && !hasTag(entry, "save") { + return false + } + return entryResolvable(tpl, entry) +} + +// entryIsConfigEntry keeps manifest file entries that are a game's settings +// and nothing else: tagged "config" and not "save". An entry carrying both +// is part of the save, whatever else it holds. +func entryIsConfigEntry(tpl string, entry manifestFileEntry) bool { + if !hasTag(entry, "config") || hasTag(entry, "save") { + return false + } + return entryResolvable(tpl, entry) +} + +func hasTag(entry manifestFileEntry, tag string) bool { + for _, t := range entry.Tags { + if t == tag { + return true + } + } + return false +} + +// entryResolvable reports whether a file template can plausibly resolve on +// a platform this app runs on. +func entryResolvable(tpl string, entry manifestFileEntry) bool { // Placeholders no supported platform can resolve. if strings.Contains(tpl, "") || strings.Contains(tpl, "") { return false @@ -812,19 +849,6 @@ func entryIsSaveEntry(tpl string, entry manifestFileEntry) bool { return false } - if len(entry.Tags) > 0 { - hasSave := false - for _, t := range entry.Tags { - if t == "save" { - hasSave = true - break - } - } - if !hasSave { - return false - } - } - if len(entry.When) == 0 { return true } diff --git a/internal/presets/savestats.go b/internal/presets/savestats.go index 4a750b93..9152feb4 100644 --- a/internal/presets/savestats.go +++ b/internal/presets/savestats.go @@ -7,6 +7,8 @@ import ( "path/filepath" "sync" "time" + + "github.com/opensave/opensave/internal/delta" ) // Measuring is deliberately separate from detection. Scan answers "could a @@ -149,6 +151,12 @@ func measureOne(path string, deadline time.Time) (count int, bytes int64, latest if d.IsDir() { return nil } + // Not part of any save (Steam's remotecache.vdf): a folder holding + // only that is a game Steam knows of, not a save, and was tracked as + // one — twenty-odd games with nothing to sync. + if delta.NeverSynced(d.Name()) { + return nil + } fi, err := d.Info() if err != nil { return nil diff --git a/internal/presets/savestats_test.go b/internal/presets/savestats_test.go index 1d8b544d..8f7d5e41 100644 --- a/internal/presets/savestats_test.go +++ b/internal/presets/savestats_test.go @@ -211,3 +211,17 @@ func TestMeasure_FolderHoldingOnlyAJunctionIsEmpty(t *testing.T) { saves[0].FileCount) } } + +// A Steam userdata folder holding only Steam's own remotecache.vdf is not a +// save: measured, it is empty, and so it is never offered as a new game. +func TestMeasure_OnlyLauncherFilesIsEmpty(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "remotecache.vdf"), []byte("steam"), 0o666); err != nil { + t.Fatal(err) + } + saves := []DiscoveredSave{{SavePath: dir}} + Measure(saves) + if !saves[0].IsEmpty() { + t.Errorf("measured %d file(s): a folder of Steam's bookkeeping counts as a save", saves[0].FileCount) + } +} diff --git a/internal/sessions/procs_windows.go b/internal/sessions/procs_windows.go index 1fb4ceb8..73450d37 100644 --- a/internal/sessions/procs_windows.go +++ b/internal/sessions/procs_windows.go @@ -3,14 +3,18 @@ package sessions import ( + "time" "unsafe" "golang.org/x/sys/windows" ) // listProcesses walks the process table and asks each process for its full -// program path. A process this user may not look at (a service, another -// user's) is skipped: no game of theirs is one of ours. +// program path, how much memory it holds and how much processor time it has +// used. A process whose path cannot be read is listed by its file name alone +// (Proc.Name): a game protected against tampering keeps its path from +// everyone, and matching by name within its install folder's programs is +// still exact enough. func listProcesses() ([]Proc, error) { snap, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0) if err != nil { @@ -27,8 +31,12 @@ func listProcesses() ([]Proc, error) { buf := make([]uint16, windows.MAX_LONG_PATH) for { if pid := entry.ProcessID; pid > 4 { - if exe := imagePath(pid, buf); exe != "" { - out = append(out, Proc{PID: int(pid), Exe: exe}) + if p, ok := inspect(pid, buf); ok { + out = append(out, p) + } else if name := windows.UTF16ToString(entry.ExeFile[:]); name != "" { + // Its path is not ours to read: a game protected against + // tampering, run elevated, or another user's. Known by name. + out = append(out, Proc{PID: int(pid), Name: name}) } } if err := windows.Process32Next(snap, &entry); err != nil { @@ -38,15 +46,46 @@ func listProcesses() ([]Proc, error) { return out, nil } -func imagePath(pid uint32, buf []uint16) string { +var procGetProcessMemoryInfo = windows.NewLazySystemDLL("kernel32.dll").NewProc("K32GetProcessMemoryInfo") + +// processMemoryCounters is PROCESS_MEMORY_COUNTERS. +type processMemoryCounters struct { + cb uint32 + PageFaultCount uint32 + PeakWorkingSetSize uintptr + WorkingSetSize uintptr + QuotaPeakPagedPoolUsage uintptr + QuotaPagedPoolUsage uintptr + QuotaPeakNonPagedPoolUsage uintptr + QuotaNonPagedPoolUsage uintptr + PagefileUsage uintptr + PeakPagefileUsage uintptr +} + +func inspect(pid uint32, buf []uint16) (Proc, bool) { h, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, pid) if err != nil { - return "" + return Proc{}, false } defer windows.CloseHandle(h) n := uint32(len(buf)) if err := windows.QueryFullProcessImageName(h, 0, &buf[0], &n); err != nil { - return "" + return Proc{}, false + } + p := Proc{PID: int(pid), Exe: windows.UTF16ToString(buf[:n])} + + var creation, exit, kernel, user windows.Filetime + timesOK := windows.GetProcessTimes(h, &creation, &exit, &kernel, &user) == nil + var mem processMemoryCounters + mem.cb = uint32(unsafe.Sizeof(mem)) + r, _, _ := procGetProcessMemoryInfo.Call(uintptr(h), uintptr(unsafe.Pointer(&mem)), uintptr(mem.cb)) + if timesOK && r != 0 { + p.Measured = true + p.Memory = uint64(mem.WorkingSetSize) + // FILETIME counts 100-nanosecond intervals. + ticks := (uint64(kernel.HighDateTime)<<32 | uint64(kernel.LowDateTime)) + + (uint64(user.HighDateTime)<<32 | uint64(user.LowDateTime)) + p.CPU = time.Duration(ticks * 100) } - return windows.UTF16ToString(buf[:n]) + return p, true } diff --git a/internal/sessions/sessions.go b/internal/sessions/sessions.go index 1f47e5c3..705f3a35 100644 --- a/internal/sessions/sessions.go +++ b/internal/sessions/sessions.go @@ -25,6 +25,7 @@ import ( "strings" "sync" "time" + "unicode" ) // Proc is a running process, as much of it as matching needs. @@ -32,8 +33,20 @@ type Proc struct { PID int // Exe is the program's full path, where the system says. Exe string + // Name is the program's file name, where the system gives no path: a + // game protected against tampering lets nobody read where it runs from, + // but the process list still says what it is called. + Name string // Args is the command line, where the system says (Linux). Args []string + // Measured says Memory and CPU were read: the program's working set in + // bytes, and the processor time it has used so far. A game being played + // holds hundreds of megabytes and keeps the processor busy; what a game + // leaves behind when it closes — a crash handler, a stub of the same name + // — holds next to nothing and does nothing (Idle). + Measured bool + Memory uint64 + CPU time.Duration // SteamAppID is the Steam app the process was started for, where Steam // tagged it (Linux: SteamAppId in the environment, or AppId=… on the // command line of Steam's launcher). @@ -49,6 +62,14 @@ type Target struct { // Dirs are the game's install folders: anything running from inside one // is the game. Dirs []string + // FolderNames are names the game's install folder goes by, for a game + // installed where no library lists it ("G:GamesCrimson Desert"): + // a program running from inside a folder of one of these names is the + // game. Normalised with FolderName. + FolderNames []string + // Programs are the file names of the programs in the install folders, for + // a process known only by name (Proc.Name). Lower-case. + Programs []string } // Running reports which targets have a process running, from a list of @@ -74,6 +95,20 @@ func matches(p Proc, t Target) bool { if p.Exe != "" && t.Exe != "" && samePath(p.Exe, t.Exe) { return true } + if p.Exe == "" && p.Name != "" { + name := strings.ToLower(p.Name) + if t.Exe != "" && strings.EqualFold(filepath.Base(t.Exe), name) { + return true + } + for _, prog := range t.Programs { + if prog == name { + return true + } + } + } + if p.Exe != "" && FolderOf(p, t) != "" { + return true + } for _, dir := range t.Dirs { if dir == "" { continue @@ -252,3 +287,55 @@ func (t *Tracker) Finish(gameID string, at time.Time) { // List returns the processes running now. Where the system cannot say, it // returns none and an error, and nothing is ever seen playing. func List() ([]Proc, error) { return listProcesses() } + +// FolderName normalises a folder or game name for comparing the two: lower +// case, letters, digits and single spaces. +func FolderName(name string) string { + var b strings.Builder + for _, r := range strings.ToLower(name) { + switch { + case unicode.IsLetter(r) || unicode.IsDigit(r): + b.WriteRune(r) + default: + b.WriteRune(' ') + } + } + return strings.Join(strings.Fields(b.String()), " ") +} + +// FolderOf returns the folder, among those above a process's program, that +// is named as one of a target's install folders (Target.FolderNames), or "". +func FolderOf(p Proc, t Target) string { + if p.Exe == "" || len(t.FolderNames) == 0 { + return "" + } + dir := filepath.Dir(p.Exe) + for dir != "" && dir != filepath.Dir(dir) { + // Without spaces: the game database writes "ProjectZomboid" for a + // folder anyone else calls "Project Zomboid". + name := strings.ReplaceAll(FolderName(filepath.Base(dir)), " ", "") + for _, want := range t.FolderNames { + if name == strings.ReplaceAll(want, " ", "") { + return dir + } + } + dir = filepath.Dir(dir) + } + return "" +} + +// Idle reports a process that holds next to no memory and has used no +// processor time since it was last seen (prevCPU, when it was): not a game +// being played, whatever it is called. A process that could not be measured +// is never called idle. +func Idle(p Proc, prevCPU time.Duration, seenBefore bool) bool { + if !p.Measured || p.Memory >= idleMemory { + return false + } + return !seenBefore || p.CPU-prevCPU < idleCPU +} + +const ( + idleMemory = 32 << 20 // a game holds far more + idleCPU = 100 * time.Millisecond // per poll: 1% of ten seconds +) diff --git a/internal/sessions/sessions_test.go b/internal/sessions/sessions_test.go index c4311de4..87131882 100644 --- a/internal/sessions/sessions_test.go +++ b/internal/sessions/sessions_test.go @@ -105,3 +105,67 @@ func TestListFindsThisProcess(t *testing.T) { t.Errorf("this process (%d, %s) was not found among %d: %v", os.Getpid(), self, len(procs), running) } } + +// A game protected against tampering keeps its path from everyone; its +// process is still recognised by name, among its install folder's programs. +func TestRunning_ProtectedProcessByName(t *testing.T) { + target := Target{GameID: "cd", Dirs: []string{`C:\Games\Crimson Desert`}, Programs: []string{"crimsondesert.exe"}} + procs := []Proc{{PID: 11, Name: "explorer.exe"}, {PID: 12, Name: "CrimsonDesert.exe"}} + if got := Running(procs, []Target{target}); got["cd"] != 12 { + t.Errorf("Running = %v, want the protected process recognised by name", got) + } + if got := Running([]Proc{{PID: 13, Name: "game.exe"}}, []Target{target}); len(got) != 0 { + t.Errorf("a program not in the game's folder was taken for it: %v", got) + } +} + +// A game installed in a folder no library lists is found by the folder's +// name, wherever that folder is. +func TestRunning_ByInstallFolderName(t *testing.T) { + // Paths as this system writes them: a process's program is a native path. + root := string(filepath.Separator) + if runtime.GOOS == "windows" { + root = `G:\` + } + games := filepath.Join(root, "Games") + target := Target{GameID: "cd", FolderNames: []string{FolderName("Crimson Desert")}} + procs := []Proc{ + {PID: 5, Exe: filepath.Join(root, "Windows", "explorer.exe")}, + {PID: 6, Exe: filepath.Join(games, "Crimson Desert", "bin64", "CrimsonDesert.exe")}, + } + if got := Running(procs, []Target{target}); got["cd"] != 6 { + t.Fatalf("Running = %v, want the game found by its folder", got) + } + if dir := FolderOf(procs[1], target); dir != filepath.Join(games, "Crimson Desert") { + t.Errorf("FolderOf = %q", dir) + } + other := []Proc{{PID: 7, Exe: filepath.Join(games, "Crimson Desert Tools", "editor.exe")}} + if got := Running(other, []Target{target}); len(got) != 0 { + t.Errorf("a folder whose name only begins the same was taken for the game: %v", got) + } +} + +// What a game leaves behind when it closes is not the game being played: next +// to no memory, no processor time since it was last seen. +func TestIdle(t *testing.T) { + stub := Proc{PID: 1, Exe: `G:\Games\Crimson Desert\bin64\CrimsonDesert.exe`, Measured: true, Memory: 900 << 10, CPU: 2 * time.Second} + if !Idle(stub, 2*time.Second, true) { + t.Error("a lingering stub counts as the game") + } + if !Idle(stub, 0, false) { + t.Error("a stub seen for the first time counts as the game") + } + busy := stub + busy.CPU = 3 * time.Second + if Idle(busy, 2*time.Second, true) { + t.Error("a small process using the processor is called idle") + } + game := stub + game.Memory = 3 << 30 + if Idle(game, 2*time.Second, true) { + t.Error("a game holding gigabytes is called idle while paused") + } + if Idle(Proc{PID: 2, Name: "CrimsonDesert.exe"}, 0, false) { + t.Error("a process that could not be measured is called idle") + } +} diff --git a/internal/snapshot/content.go b/internal/snapshot/content.go new file mode 100644 index 00000000..ccb7fb73 --- /dev/null +++ b/internal/snapshot/content.go @@ -0,0 +1,397 @@ +package snapshot + +import ( + "archive/zip" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "sort" + "strings" + + "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/store" +) + +// Snapshot content. +// +// A snapshot is named by what it holds: every file's name in the archive and +// the SHA-256 of its content, hashed together. The file hashes are the ones the +// sync compares (the hash cache both use), so the same save gives the same +// value on every device — which is how a device knows the snapshot another +// device took is one it already has, and keeps the save archived once. +// +// Before this, a save of a quarter-million files was archived again for every +// device it was compared with, and again on every new snapshot over there: +// gigabytes of identical archives for one 780 MB save. + +// contentEntry is one file of a snapshot: its name in the archive and its hash. +type contentEntry struct{ name, hash string } + +func contentKeyOf(entries []contentEntry) string { + sort.Slice(entries, func(i, j int) bool { return entries[i].name < entries[j].name }) + h := sha256.New() + for _, e := range entries { + h.Write([]byte(e.name)) + h.Write([]byte{0}) + h.Write([]byte(e.hash)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// archiveName is the name a captured file has in the archive. +func archiveName(c store.CapturedFile) string { + if c.Root == "" { + return c.Path + } + return RootPrefix + c.Root + "/" + c.Path +} + +// ContentKey names what a snapshot holds, from its recorded files. +func ContentKey(files []store.CapturedFile) string { + entries := make([]contentEntry, 0, len(files)) + for _, f := range files { + entries = append(entries, contentEntry{archiveName(f), f.Hash}) + } + return contentKeyOf(entries) +} + +// ContentKeyOfArchive names what a snapshot holds by reading its archive: for +// a snapshot recorded without its file list (one taken for a sync with a peer). +func ContentKeyOfArchive(zipPath string) (string, error) { + files, err := ArchiveFiles(zipPath) + if err != nil { + return "", err + } + return ContentKey(files), nil +} + +// ArchiveFiles reads a snapshot's archive and lists its files with their +// content hashes, as the snapshot would have recorded them. +func ArchiveFiles(zipPath string) ([]store.CapturedFile, error) { + path, done, err := OpenArchive(zipPath) + if err != nil { + return nil, err + } + defer done() + r, err := zip.OpenReader(path) + if err != nil { + return nil, err + } + defer r.Close() + out := make([]store.CapturedFile, 0, len(r.File)) + for _, f := range r.File { + if f.FileInfo().IsDir() || strings.HasSuffix(f.Name, "/") || strings.HasSuffix(f.Name, delta.TmpSuffix) { + continue + } + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("%s: %w", f.Name, err) + } + h := sha256.New() + _, err = io.Copy(h, rc) + rc.Close() + if err != nil { + return nil, fmt.Errorf("%s: %w", f.Name, err) + } + c := store.CapturedFile{Path: f.Name, Hash: hex.EncodeToString(h.Sum(nil))} + if rest, ok := strings.CutPrefix(f.Name, RootPrefix); ok { + if root, p, ok := strings.Cut(rest, "/"); ok { + c.Root, c.Path = root, p + } + } + out = append(out, c) + } + return out, nil +} + +// filesOf is what a snapshot holds, from its recorded file list — read from +// its archive once, and recorded, when it has none. +func (m *Manager) filesOf(snap store.Snapshot) ([]store.CapturedFile, error) { + if files, err := m.Store.SnapshotFiles(snap.ID); err == nil && len(files) > 0 { + return files, nil + } + files, err := ArchiveFiles(snap.ZipPath) + if err != nil { + return nil, err + } + if len(files) > 0 { + _ = m.Store.RecordSnapshotFiles(snap.ID, files) + } + return files, nil +} + +// contentHashOf works out what a snapshot holds (filesOf). +func (m *Manager) contentHashOf(snap store.Snapshot) (string, error) { + files, err := m.filesOf(snap) + if err != nil { + return "", err + } + return ContentKey(files), nil +} + +// PruneContained removes automatic snapshots that a newer snapshot on the same +// branch holds entirely: every file of the older one is in the newer one, with +// the same content. Such a snapshot is only a step on the way to the newer +// one — a pull that stopped part-way, a save that only gained files — and +// restoring it would give nothing the newer one lacks but the absence of what +// was added since. The other way round is kept: a newer snapshot contained in +// an older one is a save something was deleted from, on purpose. Pinned +// snapshots and ones taken by hand are never removed. +// +// The largest snapshots are looked at first as the ones that may hold others — +// they hold the most — and passes repeat until one removes nothing. After +// that, each new snapshot is checked against the older ones as it is taken +// (PruneContainedBy), which is all it takes to stay that way. +func (m *Manager) PruneContained(gameID string) (removed int, freed int64, err error) { + for { + n, f, err := m.pruneContainedPass(gameID) + removed += n + freed += f + if err != nil || n == 0 { + return removed, freed, err + } + } +} + +func (m *Manager) pruneContainedPass(gameID string) (removed int, freed int64, err error) { + snaps, err := m.Store.AllSnapshotsOfGame(gameID) // oldest first + if err != nil { + return 0, 0, err + } + byBranch := map[string][]store.Snapshot{} + counts := map[string]int{} + for _, s := range snaps { + if !ArchiveExists(s.ZipPath) { + continue + } + files, ferr := m.filesOf(s) + if ferr != nil { + continue + } + counts[s.ID] = len(files) + byBranch[s.BranchName] = append(byBranch[s.BranchName], s) + } + for _, list := range byBranch { + order := make(map[string]int, len(list)) // position in time + for i, s := range list { + order[s.ID] = i + } + holders := append([]store.Snapshot(nil), list...) + sort.SliceStable(holders, func(i, j int) bool { return counts[holders[i].ID] > counts[holders[j].ID] }) + gone := map[string]bool{} + for _, holder := range holders { + if gone[holder.ID] || counts[holder.ID] == 0 { + continue + } + holderFiles, ferr := m.filesOf(holder) + if ferr != nil { + continue + } + have := make(map[string]string, len(holderFiles)) + for _, f := range holderFiles { + have[archiveName(f)] = f.Hash + } + for _, older := range list[:order[holder.ID]] { + if gone[older.ID] || older.Pinned || !older.IsSystemAuto || counts[older.ID] == 0 || counts[older.ID] > len(holderFiles) { + continue + } + olderFiles, ferr := m.filesOf(older) + if ferr != nil || !containedIn(olderFiles, have) { + continue + } + identical := len(olderFiles) == len(holderFiles) + if identical { + // The same files: its id still leads to them. Before the + // delete, so no moment exists where the id leads nowhere; + // while the row is there it still resolves to itself. + _ = m.Store.RepointSnapshotAliases(older.ID, holder.ID) + _ = m.Store.AddSnapshotAlias(older.ID, holder.ID) + m.carryName(older, &holder) + } + f, derr := m.DeleteSnapshot(gameID, older.ID) + if derr != nil { + continue + } + gone[older.ID] = true + removed++ + freed += f + if m.Log != nil { + m.Log("info", fmt.Sprintf("snapshot %s removed: every file of it is in the newer %s", older.ID, holder.ID)) + } + } + } + } + return removed, freed, nil +} + +func containedIn(files []store.CapturedFile, have map[string]string) bool { + for _, f := range files { + if have[archiveName(f)] != f.Hash { + return false + } + } + return true +} + +// PruneContainedBy checks the older snapshots of snap's branch against snap, +// just taken, and removes those it holds entirely (PruneContained). Only when +// PruneOnCreate is set; in the background, counted for shutdown. +func (m *Manager) PruneContainedBy(snap store.Snapshot, files []store.CapturedFile) { + if !m.PruneOnCreate || len(files) == 0 { + return + } + m.inFlight.Add() + go func() { + defer m.inFlight.Done() + have := make(map[string]string, len(files)) + for _, f := range files { + have[archiveName(f)] = f.Hash + } + snaps, err := m.Store.ListSnapshots(snap.GameID, snap.BranchName) + if err != nil { + return + } + for _, older := range snaps { + if older.ID == snap.ID || older.Timestamp >= snap.Timestamp || older.Pinned || !older.IsSystemAuto || + !ArchiveExists(older.ZipPath) { + continue + } + olderFiles, ferr := m.filesOf(older) + if ferr != nil || len(olderFiles) == 0 || len(olderFiles) > len(files) || !containedIn(olderFiles, have) { + continue + } + identical := len(olderFiles) == len(files) + if identical { + // Aliased before the delete (see PruneContained). + _ = m.Store.RepointSnapshotAliases(older.ID, snap.ID) + _ = m.Store.AddSnapshotAlias(older.ID, snap.ID) + m.carryName(older, &snap) + } + _, derr := m.DeleteSnapshot(snap.GameID, older.ID) + if derr == nil && m.Log != nil { + m.Log("info", fmt.Sprintf("snapshot %s removed: every file of it is in the newer %s", older.ID, snap.ID)) + } + } + }() +} + +// ArchiveSaveTo archives a game's save — every location — into zipPath the way +// a snapshot here is archived: files unchanged since the branch's newest +// snapshot are copied from it rather than compressed again (reuse.go). For a +// snapshot recorded under another device's id after a sync; compressing a save +// of a quarter-million files from scratch kept a finished pull at 100% for +// minutes. +func (m *Manager) ArchiveSaveTo(gameID, branch, zipPath string) (skipped []string, captured []store.CapturedFile, err error) { + m.inFlight.Add() + defer m.inFlight.Done() + game, err := m.Store.GetGame(gameID) + if err != nil { + return nil, nil, err + } + extra, rootsErr := m.Store.GameRootPaths(gameID) + if rootsErr != nil { + extra = nil + } + reuse := m.openReuseSource(gameID, branch) + defer reuse.Close() + return zipRootsCapturing(game.SavePath, extra, zipPath, reuse) +} + +// isNewest reports whether id is the newest snapshot on a game's branch. +func (m *Manager) isNewest(gameID, branch, id string) bool { + snaps, err := m.Store.ListSnapshots(gameID, branch) + return err == nil && len(snaps) > 0 && snaps[0].ID == id +} + +// MergeDuplicates names every snapshot of a game by its content and keeps each +// content archived once per branch: of snapshots holding the same files, one +// keeps its archive — a pinned one, else one taken by hand, else the newest, +// which retention keeps longest — +// and the others are removed, their ids kept as aliases of it so a device that +// knows a snapshot by one of them still finds it. Pinned snapshots are never +// removed. Returns how many were merged and the bytes freed. +func (m *Manager) MergeDuplicates(gameID string) (merged int, freed int64, err error) { + missing, err := m.Store.SnapshotsWithoutContentHash(gameID) + if err != nil { + return 0, 0, err + } + for _, snap := range missing { + if !ArchiveExists(snap.ZipPath) { + continue + } + key, err := m.contentHashOf(snap) + if err != nil { + if m.Log != nil { + m.Log("warn", fmt.Sprintf("could not read snapshot %s to name its content: %v", snap.ID, err)) + } + continue + } + _ = m.Store.SetSnapshotContentHash(snap.ID, key) + } + + snaps, err := m.Store.AllSnapshotsOfGame(gameID) + if err != nil { + return 0, 0, err + } + groups := map[string][]store.Snapshot{} + for _, s := range snaps { + if s.ContentHash == "" { + continue + } + k := s.BranchName + "\x00" + s.ContentHash + groups[k] = append(groups[k], s) + } + for _, group := range groups { + if len(group) < 2 { + continue + } + keep := group[len(group)-1] // newest (oldest first) + for _, s := range group { + if s.Pinned && !keep.Pinned || s.Pinned == keep.Pinned && !s.IsSystemAuto && keep.IsSystemAuto { + keep = s + } + } + for _, s := range group { + if s.ID == keep.ID || s.Pinned { + continue + } + _ = m.Store.RepointSnapshotAliases(s.ID, keep.ID) + _ = m.Store.AddSnapshotAlias(s.ID, keep.ID) + f, err := m.DeleteSnapshot(gameID, s.ID) + if err != nil { + continue + } + merged++ + freed += f + if m.Log != nil { + m.Log("info", fmt.Sprintf("snapshot %s held the same files as %s; kept once", s.ID, keep.ID)) + } + } + } + return merged, freed, nil +} + +// defaultAutoComment is what an automatic snapshot taken for no particular +// reason is called. +const defaultAutoComment = "Auto backup" + +// carryName keeps what an older snapshot is called when a newer one holding +// exactly the same files takes its place: its reason ("After playing (1 h)", +// "Put back") if the newer one has only the default an automatic snapshot +// gets, and a note someone wrote if the newer one has none. A session's +// named snapshot and the watcher's copy of the same save, taken a moment +// later as the game's last write settled, otherwise left only "Auto backup". +func (m *Manager) carryName(older store.Snapshot, newer *store.Snapshot) { + if newer.Comment == defaultAutoComment && older.Comment != "" && older.Comment != defaultAutoComment { + if m.Store.SetSnapshotComment(newer.ID, older.Comment) == nil { + newer.Comment = older.Comment + } + } + if newer.Note == "" && older.Note != "" { + if m.Store.SetSnapshotNote(newer.ID, older.Note) == nil { + newer.Note = older.Note + } + } +} diff --git a/internal/snapshot/content_test.go b/internal/snapshot/content_test.go new file mode 100644 index 00000000..ab4db8ee --- /dev/null +++ b/internal/snapshot/content_test.go @@ -0,0 +1,259 @@ +package snapshot + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// The content a snapshot records and the content read back from its archive +// are named alike — which is what lets a device that only has another's +// archive, or only its file list, tell they hold the same save. +func TestContentKey_FromFilesAndFromArchiveAgree(t *testing.T) { + env := setup(t) + writeSave(t, env.saveDir, "slot1.sav", "one") + writeSave(t, env.saveDir, "sub/slot2.sav", "two") + snap, err := env.mgr.Create("game1", "", true) + if err != nil { + t.Fatal(err) + } + files, err := env.store.SnapshotFiles(snap.ID) + if err != nil || len(files) == 0 { + t.Fatalf("no file list recorded: %v", err) + } + fromArchive, err := ContentKeyOfArchive(snap.ZipPath) + if err != nil { + t.Fatal(err) + } + if got := ContentKey(files); got != fromArchive { + t.Errorf("from the file list %s, from the archive %s", got, fromArchive) + } + stored, _ := env.store.GetSnapshot(snap.ID) + if stored.ContentHash != fromArchive { + t.Errorf("recorded content %q, want %q", stored.ContentHash, fromArchive) + } +} + +// Steps on the way to a newer snapshot — every file of them in it, unchanged — +// go. A newer snapshot that lacks files (something was deleted) does not make +// the older one redundant; pinned and hand-taken snapshots are never removed. +func TestPruneContained(t *testing.T) { + env := setup(t) + game, err := env.store.GetGame("game1") + if err != nil { + t.Fatal(err) + } + game.MaxSnapshots = 50 + if err := env.store.UpdateGame(game); err != nil { + t.Fatal(err) + } + snap := func(auto bool) string { + t.Helper() + s, err := env.mgr.Create("game1", "", auto) + if err != nil { + t.Fatal(err) + } + return s.ID + } + writeSave(t, env.saveDir, "a.bin", "a") + step1 := snap(true) + writeSave(t, env.saveDir, "b.bin", "b") + step2 := snap(true) + pinnedStep := snap(false) // same files as step2, by hand, then pinned below + _ = env.store.SetSnapshotPinned(pinnedStep, true) + writeSave(t, env.saveDir, "c.bin", "c") + full := snap(true) + // Then a file is deleted on purpose. + if err := os.Remove(filepath.Join(env.saveDir, "c.bin")); err != nil { + t.Fatal(err) + } + writeSave(t, env.saveDir, "a.bin", "a, played on") + after := snap(true) + + removed, _, err := env.mgr.PruneContained("game1") + if err != nil { + t.Fatal(err) + } + gone := func(id string) bool { _, err := env.store.GetSnapshot(id); return err != nil } + if !gone(step1) || !gone(step2) { + t.Errorf("steps on the way to a newer snapshot kept: step1 gone=%v step2 gone=%v", gone(step1), gone(step2)) + } + if gone(pinnedStep) { + t.Error("a pinned snapshot was removed") + } + if gone(full) { + t.Error("removed a snapshot holding a file deleted later on purpose") + } + if gone(after) { + t.Error("removed the newest snapshot") + } + if removed != 2 { + t.Errorf("removed %d, want 2", removed) + } +} + +// With PruneOnCreate, a new snapshot takes the place of older automatic ones it +// holds entirely, as it is taken; one that lacks a file of them does not. +func TestPruneContainedBy_ANewSnapshotReplacesItsSteps(t *testing.T) { + env := setup(t) + env.mgr.PruneOnCreate = true + game, _ := env.store.GetGame("game1") + game.MaxSnapshots = 50 + _ = env.store.UpdateGame(game) + + writeSave(t, env.saveDir, "a.bin", "a") + first, err := env.mgr.Create("game1", "", true) + if err != nil { + t.Fatal(err) + } + writeSave(t, env.saveDir, "b.bin", "b") + if _, err := env.mgr.Create("game1", "", true); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := env.store.GetSnapshot(first.ID); err != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + if _, err := env.store.GetSnapshot(first.ID); err == nil { + t.Fatal("the step the new snapshot holds entirely was kept") + } + + // Now a file goes: the newer snapshot does not hold the older one. + before, _ := env.store.ListSnapshots("game1", "main") + if err := os.Remove(filepath.Join(env.saveDir, "b.bin")); err != nil { + t.Fatal(err) + } + if _, err := env.mgr.Create("game1", "", true); err != nil { + t.Fatal(err) + } + time.Sleep(500 * time.Millisecond) + if _, err := env.store.GetSnapshot(before[0].ID); err != nil { + t.Error("removed the snapshot holding a file deleted afterwards") + } +} + +// A copy taken before a sync replaces a save already archived is that archive, +// not a second one. +func TestBeforeReplacing_AnArchivedSaveIsNotArchivedAgain(t *testing.T) { + env := setup(t) + writeSave(t, env.saveDir, "slot1.sav", "progress") + first, err := env.mgr.Create("game1", "", true) + if err != nil { + t.Fatal(err) + } + copy, err := env.mgr.CreateBeforeReplacing("game1", "before sync") + if err != nil { + t.Fatal(err) + } + if copy.ID != first.ID { + t.Errorf("a second archive %s of the save %s already holds", copy.ID, first.ID) + } + snaps, _ := env.store.ListSnapshots("game1", "main") + if len(snaps) != 1 { + t.Errorf("%d snapshots, want 1", len(snaps)) + } +} + +// Several snapshots of the same files — one per device the save was compared +// with — are kept once: the newest unless one is pinned or taken by hand; +// pinned ones are never removed; the others' ids still lead to it. +func TestMergeDuplicates(t *testing.T) { + env := setup(t) + // Room for every snapshot below: this is about merging, not retention. + game, err := env.store.GetGame("game1") + if err != nil { + t.Fatal(err) + } + game.MaxSnapshots = 50 + if err := env.store.UpdateGame(game); err != nil { + t.Fatal(err) + } + writeSave(t, env.saveDir, "slot1.sav", "the save") + var same []string + for i := 0; i < 3; i++ { + s, err := env.mgr.Create("game1", "", true) + if err != nil { + t.Fatal(err) + } + same = append(same, s.ID) + } + pinned, err := env.mgr.Create("game1", "pinned copy", true) + if err != nil { + t.Fatal(err) + } + if err := env.store.SetSnapshotPinned(pinned.ID, true); err != nil { + t.Fatal(err) + } + writeSave(t, env.saveDir, "slot1.sav", "played on") + other, err := env.mgr.Create("game1", "", true) + if err != nil { + t.Fatal(err) + } + // As if taken before contents were recorded. + for _, id := range append(same, pinned.ID, other.ID) { + _ = env.store.SetSnapshotContentHash(id, "") + } + + merged, _, err := env.mgr.MergeDuplicates("game1") + if err != nil { + t.Fatal(err) + } + if merged != 3 { + t.Errorf("merged %d, want the 3 unpinned copies of the pinned one", merged) + } + if _, err := env.store.GetSnapshot(pinned.ID); err != nil { + t.Error("the pinned snapshot was removed") + } + if _, err := env.store.GetSnapshot(other.ID); err != nil { + t.Error("a snapshot of different content was removed") + } + for _, id := range same { + if got, ok := env.store.ResolveSnapshotID(id); !ok || got != pinned.ID { + t.Errorf("%s leads to %q (%v), want the snapshot kept, %s", id, got, ok, pinned.ID) + } + } + // Run again: nothing more to do. + if again, _, _ := env.mgr.MergeDuplicates("game1"); again != 0 { + t.Errorf("a second pass merged %d more", again) + } +} + +// A session's named snapshot, then the watcher's copy of the same save a +// moment later: the copy takes its place and keeps its name. +func TestPruneContainedBy_AnIdenticalCopyKeepsTheName(t *testing.T) { + env := setup(t) + env.mgr.PruneOnCreate = true + writeSave(t, env.saveDir, "slot1.sav", "after an evening") + named, err := env.mgr.Create("game1", "After playing (1 h)", true) + if err != nil { + t.Fatal(err) + } + if err := env.store.SetSnapshotNote(named.ID, "boss beaten"); err != nil { + t.Fatal(err) + } + copied, err := env.mgr.Create("game1", "", true) + if err != nil { + t.Fatal(err) + } + if copied.ID == named.ID { + t.Skip("the copy was not taken: nothing to replace") + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := env.store.GetSnapshot(named.ID); err != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + kept, err := env.store.GetSnapshot(copied.ID) + if err != nil { + t.Fatal(err) + } + if kept.Comment != "After playing (1 h)" || kept.Note != "boss beaten" { + t.Errorf("the snapshot kept is called %q (note %q), want the name and note of the one it replaced", kept.Comment, kept.Note) + } +} diff --git a/internal/snapshot/extract.go b/internal/snapshot/extract.go new file mode 100644 index 00000000..f7ab5120 --- /dev/null +++ b/internal/snapshot/extract.go @@ -0,0 +1,137 @@ +package snapshot + +import ( + "archive/zip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + + "github.com/opensave/opensave/internal/iopar" +) + +// Progress is told how far a long operation on a save has come: which phase +// ("rebuilding", "checking", "writing"), how many of how many done. +type Progress func(phase string, done, total int) + +// Phases of a restore, as Progress names them. +const ( + PhaseRebuilding = "rebuilding" + PhaseChecking = "checking" + PhaseWriting = "writing" +) + +// extractJob is one archive entry to write: under destDir, at rel. +type extractJob struct { + f *zip.File + destDir string + rel string +} + +// extractAll writes archive entries into place. Every folder is made first, +// once — not looked up again for each of a quarter of a million files — and +// the files are then written as many at a time as suits the drive they go to +// (iopar): one at a time on a spinning disk, many at once on an SSD, where +// written one after another a large save took most of a quarter of an hour +// at a fraction of what the drive can do. +func extractAll(jobs []extractJob, onDrive string, progress Progress) error { + type target struct { + f *zip.File + dest string + } + var files []target + dirs := map[string]struct{}{} + for _, j := range jobs { + dest, err := entryDest(j.f.Name, j.destDir, j.rel) + if err != nil { + return err + } + if j.f.FileInfo().IsDir() { + dirs[dest] = struct{}{} + continue + } + dirs[filepath.Dir(dest)] = struct{}{} + files = append(files, target{j.f, dest}) + } + ordered := make([]string, 0, len(dirs)) + for d := range dirs { + ordered = append(ordered, d) + } + sort.Strings(ordered) // a parent before what is inside it + for _, d := range ordered { + if err := os.MkdirAll(d, 0o777); err != nil { + return err + } + } + + var report func(done, total int) + if progress != nil { + report = func(done, total int) { progress(PhaseWriting, done, total) } + } + st, err := iopar.RunKind(context.Background(), driveKindOf(onDrive), len(files), func(i int) error { + return writeEntry(files[i].f, files[i].dest) + }, report) + lastExtract.Store(&st) + return err +} + +// driveKindOf is iopar.DriveKind; a variable, so a test can compare one at a +// time with many on the same drive. +var driveKindOf = iopar.DriveKind + +// lastExtract is how the last extraction went, for a test to report. +var lastExtract atomic.Pointer[iopar.Stats] + +// entryDest is where an entry is written: rel under destDir, refused if it +// would land outside it (zip-slip). Checked on the name actually written, +// since that is the one that decides where bytes land. +func entryDest(name, destDir, rel string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(rel)) + if strings.HasPrefix(clean, "..") || filepath.IsAbs(clean) { + return "", fmt.Errorf("zip entry %q escapes destination", name) + } + return filepath.Join(destDir, clean), nil +} + +var copyBuffers = sync.Pool{New: func() any { b := make([]byte, 64<<10); return &b }} + +// writeEntry writes one file entry to dest, whose folder exists. +func writeEntry(f *zip.File, dest string) error { + src, err := f.Open() + if err != nil { + return err + } + defer src.Close() + dst, err := os.OpenFile(dest, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o666) + if err != nil { + return err + } + buf := copyBuffers.Get().(*[]byte) + defer copyBuffers.Put(buf) + if _, err := io.CopyBuffer(dst, src, *buf); err != nil { + dst.Close() + return err + } + return dst.Close() +} + +// progressFor is how a restore of gameID tells OnRestoreProgress how far it +// has come; nil when nobody listens. +func (m *Manager) progressFor(gameID string) Progress { + if m.OnRestoreProgress == nil { + return nil + } + return func(phase string, done, total int) { m.OnRestoreProgress(gameID, phase, done, total) } +} + +// progressDone tells OnRestoreProgress the restore of gameID is over. +func (m *Manager) progressDone(gameID string) { + if m.OnRestoreProgress != nil { + m.OnRestoreProgress(gameID, "", 0, 0) + } +} diff --git a/internal/snapshot/history.go b/internal/snapshot/history.go new file mode 100644 index 00000000..b6ebe914 --- /dev/null +++ b/internal/snapshot/history.go @@ -0,0 +1,103 @@ +package snapshot + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" +) + +// How often files changed along with a game's save, read from its +// snapshots: a settings file is changed now and then, when someone changes a +// setting; a save changes with nearly every session. A file taken for a +// game's settings that changes like a save is not its settings +// (daemon/detectsettings.go). + +const ( + historySnapshots = 40 // the most recent ones are read + historyMinSaves = 4 // changes of the save needed before anything is said +) + +// ChangesLikeASave returns, for each of paths (relative to the main save +// folder) that changed in more than half of the snapshots in which the rest +// of the save changed, why it is judged a save. Paths with too little history +// to tell are not in it. +func (m *Manager) ChangesLikeASave(gameID string, paths []string) map[string]string { + out := map[string]string{} + if len(paths) == 0 { + return out + } + watched := make(map[string]bool, len(paths)) + for _, p := range paths { + watched[p] = true + } + snaps, err := m.Store.AllSnapshotsOfGame(gameID) + if err != nil { + return out + } + if len(snaps) > historySnapshots { + snaps = snaps[len(snaps)-historySnapshots:] + } + + // Per snapshot, the hashes of the watched files and one value for the + // rest of the save. + type state struct { + watched map[string]string + rest string + } + var states []state + for _, s := range snaps { + files, err := m.filesOf(s) + if err != nil || len(files) == 0 { + continue + } + st := state{watched: map[string]string{}} + var rest []string + for _, f := range files { + if f.Root != "" { + continue + } + if watched[f.Path] { + st.watched[f.Path] = f.Hash + } else { + rest = append(rest, f.Path+"\x00"+f.Hash) + } + } + st.rest = restKey(rest) + states = append(states, st) + } + + saves := 0 + moved := map[string]int{} + for i := 1; i < len(states); i++ { + a, b := states[i-1], states[i] + if a.rest == b.rest { + continue // the save did not change: nothing to compare with + } + saves++ + for p := range watched { + if a.watched[p] != b.watched[p] { + moved[p]++ + } + } + } + if saves < historyMinSaves { + return out + } + for p, n := range moved { + if 2*n > saves { + out[p] = fmt.Sprintf("changed in %d of the %d snapshots in which the save changed", n, saves) + } + } + return out +} + +func restKey(entries []string) string { + sort.Strings(entries) + h := sha256.New() + for _, e := range entries { + h.Write([]byte(e)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/internal/snapshot/history_test.go b/internal/snapshot/history_test.go new file mode 100644 index 00000000..1c7ea812 --- /dev/null +++ b/internal/snapshot/history_test.go @@ -0,0 +1,52 @@ +package snapshot + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A file that changes with every change of the save is a save; one changed +// once among many is settings; too little history says nothing. +func TestChangesLikeASave(t *testing.T) { + env := setup(t) + write := func(name, data string) { + if err := os.WriteFile(filepath.Join(env.saveDir, name), []byte(data), 0o666); err != nil { + t.Fatal(err) + } + } + snap := func() { + if _, err := env.mgr.Create("game1", "", false); err != nil { + t.Fatal(err) + } + } + write("graphics.cfg", "resolution=1080") + write("progress.cfg", "level=0") + write("slot1.sav", "0") + snap() + ask := []string{"graphics.cfg", "progress.cfg"} + for i := 1; i <= 3; i++ { + write("slot1.sav", strings.Repeat("x", i)) + write("progress.cfg", "level="+strings.Repeat("1", i)) + snap() + } + if got := env.mgr.ChangesLikeASave("game1", ask); len(got) != 0 { + t.Fatalf("3 changes of the save already judged: %v", got) + } + for i := 4; i <= 6; i++ { + write("slot1.sav", strings.Repeat("x", i)) + write("progress.cfg", "level="+strings.Repeat("1", i)) + if i == 5 { + write("graphics.cfg", "resolution=2160") + } + snap() + } + got := env.mgr.ChangesLikeASave("game1", ask) + if _, ok := got["progress.cfg"]; !ok { + t.Errorf("a file that changed with every save is not judged a save: %v", got) + } + if _, ok := got["graphics.cfg"]; ok { + t.Errorf("a file changed once in six saves is judged a save: %v", got) + } +} diff --git a/internal/snapshot/reuse.go b/internal/snapshot/reuse.go new file mode 100644 index 00000000..90da079e --- /dev/null +++ b/internal/snapshot/reuse.go @@ -0,0 +1,124 @@ +package snapshot + +import ( + "archive/zip" + "os" + "sync/atomic" + + "github.com/opensave/opensave/internal/delta" +) + +// Snapshots that re-pack only what changed. +// +// Every snapshot is a complete zip, restorable on its own — that stays. What +// changes is how it is written: a file whose content is exactly what the +// previous snapshot of the same branch holds is copied into the new archive +// as its already-compressed entry (zip.Writer.Copy), not read and deflated +// again. Only new and changed files are compressed. +// +// For a save of a quarter-million small files (Project Zomboid map chunks) +// where a play session touches a few hundred, that turns a snapshot from a +// minute of reading and deflating 0.76GB into writing mostly copied bytes. +// +// Content is identified by SHA-256 — the hash each snapshot already records +// per file (store.SnapshotFiles) and the one the manifest's hash cache holds, +// so an unchanged file is recognised without being read. Size is checked as +// well, cheaply, before the hash is asked for. +type reuseSource struct { + zr *zip.ReadCloser + files map[string]*zip.File // entry name -> previous entry + hashes map[string]string // entry name -> sha256 recorded for it + reused int // entries copied, for tests +} + +// lastReused is how many entries the most recent snapshot copied from its +// predecessor. Tests read it; nothing else does. +var lastReused atomic.Int64 + +// openReuseSource prepares the newest snapshot of a branch for reuse. Any +// problem — no snapshot, no recorded file list (an older snapshot, or a +// mirror), a zip that does not open — means no reuse, never a failed snapshot. +func (m *Manager) openReuseSource(gameID, branch string) *reuseSource { + prev, err := m.LatestSnapshot(gameID, branch) + if err != nil || prev.ZipPath == "" { + return nil + } + recorded, err := m.Store.SnapshotFiles(prev.ID) + if err != nil || len(recorded) == 0 { + return nil + } + zr, err := zip.OpenReader(prev.ZipPath) + if err != nil { + return nil + } + r := &reuseSource{ + zr: zr, + files: make(map[string]*zip.File, len(zr.File)), + hashes: make(map[string]string, len(recorded)), + } + for _, f := range zr.File { + r.files[f.Name] = f + } + for _, c := range recorded { + name := c.Path + if c.Root != "" { + name = RootPrefix + c.Root + "/" + c.Path + } + r.hashes[name] = c.Hash + } + return r +} + +func (r *reuseSource) Close() { + if r == nil { + lastReused.Store(0) + return + } + lastReused.Store(int64(r.reused)) + if r.zr != nil { + r.zr.Close() + } +} + +// addFileEntry adds filePath under entryName, copying the previous +// snapshot's entry when the content is the same, and archiving it afresh +// otherwise. Returns the file's sha256 either way. A nil receiver archives +// afresh, as every snapshot did before. +func (r *reuseSource) addFileEntry(w *zip.Writer, filePath, entryName string, info os.FileInfo) (string, error) { + if r != nil { + if hash, ok := r.reusable(filePath, entryName, info); ok { + if err := w.Copy(r.files[entryName]); err != nil { + return "", err + } + r.reused++ + return hash, nil + } + } + return addFileEntry(w, filePath, entryName) +} + +// reusable reports whether the previous snapshot holds exactly this file's +// current content under the same name. +func (r *reuseSource) reusable(filePath, entryName string, info os.FileInfo) (string, bool) { + prevHash, ok := r.hashes[entryName] + prev := r.files[entryName] + if !ok || prev == nil || prev.Mode().IsDir() { + return "", false + } + if info == nil { + var err error + if info, err = os.Stat(filePath); err != nil { + return "", false + } + } + if uint64(info.Size()) != prev.UncompressedSize64 { + return "", false + } + // From the hash cache when the file is unchanged since it was last + // hashed, which after the watcher's manifest build it nearly always is. + entry, err := delta.FileEntryForInfo(filePath, info) + if err != nil || entry.Hash != prevHash { + return "", false + } + return prevHash, true +} diff --git a/internal/snapshot/reuse_bench_test.go b/internal/snapshot/reuse_bench_test.go new file mode 100644 index 00000000..641a0335 --- /dev/null +++ b/internal/snapshot/reuse_bench_test.go @@ -0,0 +1,61 @@ +package snapshot + +import ( + "archive/zip" + "os" + "path/filepath" + "testing" + "time" + + "github.com/opensave/opensave/internal/delta" +) + +// TestBench_SnapshotReuse times a full snapshot of a real save folder and a +// second one that reuses it. Opt-in: OPENSAVE_BENCH_SAVE=. +// Reads the folder only; archives go to a temp dir. +func TestBench_SnapshotReuse(t *testing.T) { + src := os.Getenv("OPENSAVE_BENCH_SAVE") + if src == "" { + t.Skip("set OPENSAVE_BENCH_SAVE to run") + } + dir := t.TempDir() + + first := filepath.Join(dir, "first.zip") + start := time.Now() + skipped, captured, err := zipRootsCapturing(src, nil, first, nil) + if err != nil { + t.Fatal(err) + } + full := time.Since(start) + t.Logf("full snapshot skipped %d unreadable files", len(skipped)) + + // As in the app: the watcher's manifest build has hashed the save just + // before a snapshot is taken, so the cache is warm. + if _, err := delta.BuildManifest(src); err != nil { + t.Fatal(err) + } + + zr, err := zip.OpenReader(first) + if err != nil { + t.Fatal(err) + } + r := &reuseSource{zr: zr, files: map[string]*zip.File{}, hashes: map[string]string{}} + for _, f := range zr.File { + r.files[f.Name] = f + } + for _, c := range captured { + r.hashes[c.Path] = c.Hash + } + second := filepath.Join(dir, "second.zip") + start = time.Now() + if _, _, err := zipRootsCapturing(src, nil, second, r); err != nil { + t.Fatal(err) + } + incr := time.Since(start) + r.Close() + + fi1, _ := os.Stat(first) + fi2, _ := os.Stat(second) + t.Logf("%d files: full snapshot %s (%d MB), reusing snapshot %s (%d MB), %d entries copied", + len(captured), full.Round(time.Millisecond), fi1.Size()>>20, incr.Round(time.Millisecond), fi2.Size()>>20, lastReused.Load()) +} diff --git a/internal/snapshot/reuse_test.go b/internal/snapshot/reuse_test.go new file mode 100644 index 00000000..104a8efd --- /dev/null +++ b/internal/snapshot/reuse_test.go @@ -0,0 +1,69 @@ +package snapshot + +import ( + "fmt" + "testing" +) + +// A snapshot copies the files that did not change from the previous one, and +// still restores to exactly the save it was taken of. +func TestSnapshotReusesUnchangedFiles(t *testing.T) { + env := setup(t) + for i := 0; i < 50; i++ { + write(t, env.saveDir, fmt.Sprintf("map/chunk_%02d.bin", i), fmt.Sprintf("chunk %d %s", i, "................................................")) + } + first, err := env.mgr.Create("game1", "first", false) + if err != nil { + t.Fatal(err) + } + if got := lastReused.Load(); got != 0 { + t.Errorf("first snapshot reused %d entries from nothing", got) + } + + // A session changes two chunks and adds one. + write(t, env.saveDir, "map/chunk_03.bin", "changed three") + write(t, env.saveDir, "map/chunk_40.bin", "changed forty") + write(t, env.saveDir, "map/chunk_99.bin", "new") + second, err := env.mgr.Create("game1", "second", false) + if err != nil { + t.Fatal(err) + } + if got := lastReused.Load(); got != 48 { + t.Errorf("second snapshot copied %d unchanged entries, want 48", got) + } + + // Each snapshot restores its own state, independently of the other. + write(t, env.saveDir, "map/chunk_03.bin", "ruined") + if _, err := env.mgr.Restore("game1", second.ID); err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{ + "map/chunk_03.bin": "changed three", + "map/chunk_40.bin": "changed forty", + "map/chunk_99.bin": "new", + "map/chunk_07.bin": "chunk 7 ................................................", + } { + if got := read(env.saveDir, name); got != want { + t.Errorf("after restoring the second snapshot, %s = %q, want %q", name, got, want) + } + } + if _, err := env.mgr.Restore("game1", first.ID); err != nil { + t.Fatal(err) + } + if got := read(env.saveDir, "map/chunk_03.bin"); got != "chunk 3 ................................................" { + t.Errorf("after restoring the first snapshot, chunk_03 = %q", got) + } + if got := read(env.saveDir, "map/chunk_99.bin"); got != "" { + t.Errorf("restoring the first snapshot left the later file: %q", got) + } + + // And the recorded contents of the second snapshot are right for the + // copied entries too. + files, err := env.store.SnapshotFiles(second.ID) + if err != nil { + t.Fatal(err) + } + if len(files) != 51 { + t.Errorf("second snapshot records %d files, want 51", len(files)) + } +} diff --git a/internal/snapshot/shared.go b/internal/snapshot/shared.go index 81de0691..b3d8504d 100644 --- a/internal/snapshot/shared.go +++ b/internal/snapshot/shared.go @@ -140,6 +140,12 @@ var reading = struct { // with it. A missing snapshot is an error that matches fs.ErrNotExist; a // compacted one whose shared files are damaged, one that matches ErrDamaged. func OpenArchive(zipPath string) (path string, done func(), err error) { + return OpenArchiveProgress(zipPath, nil) +} + +// OpenArchiveProgress is OpenArchive telling progress how far a rebuild of a +// compacted archive has come. +func OpenArchiveProgress(zipPath string, progress Progress) (path string, done func(), err error) { reading.Lock() if _, statErr := os.Stat(zipPath); statErr == nil { reading.n[zipPath]++ @@ -165,7 +171,7 @@ func OpenArchive(zipPath string) (path string, done func(), err error) { return "", func() {}, err } name := tmp.Name() - err = rebuild(zipPath, tmp) + err = rebuild(zipPath, tmp, progress) if closeErr := tmp.Close(); err == nil { err = closeErr } @@ -250,7 +256,7 @@ func validSum(s string) bool { } // rebuild writes a compacted snapshot's archive back out whole. -func rebuild(zipPath string, out io.Writer) error { +func rebuild(zipPath string, out io.Writer, progress Progress) error { man, err := readManifest(manifestPath(zipPath)) if err != nil { return err @@ -265,7 +271,10 @@ func rebuild(zipPath string, out io.Writer) error { } w := zip.NewWriter(out) next := 0 - for _, e := range man.Entries { + for i, e := range man.Entries { + if progress != nil && i%500 == 0 { + progress(PhaseRebuilding, i, len(man.Entries)) + } if e.Blob != "" { if err := copyShared(w, store, e); err != nil { return err @@ -280,6 +289,9 @@ func rebuild(zipPath string, out io.Writer) error { } next++ } + if progress != nil { + progress(PhaseRebuilding, len(man.Entries), len(man.Entries)) + } return w.Close() } @@ -603,7 +615,7 @@ func sameArchive(zipPath string, orig *zip.ReadCloser) error { return err } defer os.Remove(tmp.Name()) - err = rebuild(zipPath, tmp) + err = rebuild(zipPath, tmp, nil) if closeErr := tmp.Close(); err == nil { err = closeErr } diff --git a/internal/snapshot/shared_test.go b/internal/snapshot/shared_test.go index f125591e..73049548 100644 --- a/internal/snapshot/shared_test.go +++ b/internal/snapshot/shared_test.go @@ -558,7 +558,9 @@ func TestSwitchingToABranchWhoseNewestIsCompacted(t *testing.T) { env := setup(t) snaps, originals := threeSnapshots(t, env) // Over to another branch: the switch keeps the save on main first, and - // that copy is main's newest. + // that copy is main's newest. Changed first, or the copy would hold the + // same files as the third snapshot, and be that snapshot (content.go). + writeSave(t, env.saveDir, "slot_switch.sav", "before the switch") if _, err := env.mgr.CreateBranch("game1", "other", true); err != nil { t.Fatal(err) } diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 77d0cf6f..f9ac955e 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -41,11 +41,27 @@ import ( // before starting one. type UploadHook func(zipPath, remoteFileName string) +// IncompleteError is a snapshot discarded because files of the save could not +// be read — in use by the game, or changing while it was archived. +type IncompleteError struct { + Game string + Skipped []string +} + +func (e *IncompleteError) Error() string { + return fmt.Sprintf("snapshot of %q discarded: %d file(s) could not be read (e.g. %s) — taken again at the next chance", + e.Game, len(e.Skipped), e.Skipped[0]) +} + // Manager performs snapshot/branch operations against the store and // filesystem. type Manager struct { - Store *store.Store - OnUpload UploadHook + Store *store.Store + // OnRestoreProgress is told how far a restore of a game has come: its + // phase (PhaseRebuilding, PhaseChecking, PhaseWriting) and how many of + // how many done; phase "" when it is over. Optional. + OnRestoreProgress func(gameID, phase string, done, total int) + OnUpload UploadHook // OnCreated fires for every snapshot written, before its upload starts. // current is false for a copy kept of a save about to be replaced (see // CreateBeforeReplacing). Optional; runs on the snapshotting goroutine @@ -80,6 +96,10 @@ type Manager struct { // running past shutdown it wrote into a directory that was being deleted // and recorded against a database that was already closed. inFlight drain.Group + // PruneOnCreate checks each new snapshot against the older ones of its + // branch and removes those it holds entirely (content.go, + // PruneContainedBy). Off unless set: the daemon sets it. + PruneOnCreate bool // sharedMu is held by a compaction and by the clean-up of shared files // (shared.go): a compaction names shared files before any list does, and // a clean-up running then would take them for unused. pendingRoots are @@ -236,20 +256,48 @@ func (m *Manager) createOnBranchFrom(gameID, branch, from, comment string, isSys } else { src = from } - skipped, captured, err := ZipRootsCapturing(src, extraRoots, stagingPath) + // The game's own save, onto its branch: unchanged files are copied from + // the branch's previous snapshot rather than compressed again (reuse.go). + var reuse *reuseSource + if from == "" { + reuse = m.openReuseSource(gameID, branch) + } + skipped, captured, err := zipRootsCapturing(src, extraRoots, stagingPath, reuse) + reuse.Close() if err != nil { os.Remove(stagingPath) return store.Snapshot{}, fmt.Errorf("zip save data: %w", err) } defer os.Remove(stagingPath) // no-op once renamed - if len(skipped) > 0 && m.Log != nil { - sample := skipped[0] - m.Log("warn", fmt.Sprintf("snapshot of %q skipped %d unreadable file(s), e.g. %s", game.Name, len(skipped), sample)) + // A snapshot missing files is not kept: it would stand in the history as + // a save that never existed — and, counted against retention, push out a + // complete one. The caller tries again at the next chance (the watcher + // leaves the change unrecorded, so its next pass takes it again; a sync + // that needed the copy before replacing files does not replace them). + if len(skipped) > 0 { + return store.Snapshot{}, &IncompleteError{Game: game.Name, Skipped: skipped} + } + + // A copy taken before a sync replaces the save, of files a snapshot on this + // branch already holds: that one is the copy, and the save is not archived + // again (content.go). Syncs took one of these each time, mostly of a save + // already archived. Only when that snapshot is the branch's newest: a + // branch switch puts back the newest, and retention removes older ones + // first — an older copy standing in would leave the save behind either way. Snapshots the + // game's changes or a person ask for are always taken: the watcher takes + // them only when something changed, and a person asked. + contentKey := ContentKey(captured) + if existing, ok, _ := m.Store.SnapshotByContent(gameID, branch, contentKey); !current && ok && ArchiveExists(existing.ZipPath) && + m.isNewest(gameID, branch, existing.ID) { + if m.Log != nil { + m.Log("info", fmt.Sprintf("%q holds the same files as snapshot %s; not archived again", game.Name, existing.ID)) + } + return existing, nil } if comment == "" { if isSystemAuto { - comment = "Auto backup" + comment = defaultAutoComment } else { comment = "Manual snapshot" } @@ -260,6 +308,11 @@ func (m *Manager) createOnBranchFrom(gameID, branch, from, comment string, isSys return store.Snapshot{}, err } snapshotID := snap.ID + _ = m.Store.SetSnapshotContentHash(snapshotID, contentKey) + snap.ContentHash = contentKey + // Older automatic snapshots this one holds entirely are steps on the way + // to it (content.go). + defer m.PruneContainedBy(snap, captured) // What this snapshot holds, file by file, recorded once and never // revisited. It is what lets a later caller ask whether some exact content @@ -730,7 +783,18 @@ func (m *Manager) Restore(gameID, snapshotID string) (store.Snapshot, error) { if err != nil { return store.Snapshot{}, err } + // An id kept as an alias of the snapshot holding the same files + // (content.go) restores that snapshot. + if resolved, ok := m.Store.ResolveSnapshotID(snapshotID); ok { + snapshotID = resolved + } snap, err := m.Store.GetSnapshot(snapshotID) + if err != nil { + // Removed in favour of the same files just now: its alias is there. + if resolved, ok := m.Store.ResolveSnapshotID(snapshotID); ok { + snap, err = m.Store.GetSnapshot(resolved) + } + } if err != nil || snap.GameID != gameID { return store.Snapshot{}, fmt.Errorf("snapshot %q not found for game %q", snapshotID, gameID) } @@ -740,10 +804,12 @@ func (m *Manager) Restore(gameID, snapshotID string) (store.Snapshot, error) { // restore empties the save folder and then extracts; an archive found // damaged part-way through would leave neither the save that was there // nor this one. - archive, done, err := OpenArchive(snap.ZipPath) + progress := m.progressFor(gameID) + defer m.progressDone(gameID) + archive, done, err := OpenArchiveProgress(snap.ZipPath, progress) if err == nil { defer done() - err = VerifyArchive(archive) + err = VerifyArchiveProgress(archive, progress) } else { err = damagedArchive(snap.ZipPath, err) } @@ -776,7 +842,7 @@ func (m *Manager) Restore(gameID, snapshotID string) (store.Snapshot, error) { if rootsErr != nil { restoreRoots = nil } - unplaced, err := UnzipRoots(restoreZip, game.SavePath, restoreRoots) + unplaced, err := UnzipRootsProgress(restoreZip, game.SavePath, restoreRoots, progress) for _, name := range unplaced { // Named but not placed: the snapshot holds files for a location this // device has no folder for. Saying so is the only honest option — @@ -1188,14 +1254,16 @@ func (m *Manager) recordDeletionsSince(gameID, branch, snapshotID string, captur if err != nil || len(records) == 0 { continue } + var back []string for _, f := range captured { if f.Root != root { continue } if _, remembered := records[f.Path]; remembered { - _ = m.Store.ClearDeletedFile(gameID, root, f.Path) + back = append(back, f.Path) } } + _ = m.Store.ClearDeletedFiles(gameID, root, back) } } diff --git a/internal/snapshot/verify.go b/internal/snapshot/verify.go index 69f1ff47..ecce0bf9 100644 --- a/internal/snapshot/verify.go +++ b/internal/snapshot/verify.go @@ -2,6 +2,7 @@ package snapshot import ( "archive/zip" + "context" "errors" "fmt" "io" @@ -10,6 +11,7 @@ import ( "sync" "time" + "github.com/opensave/opensave/internal/iopar" "github.com/opensave/opensave/internal/store" ) @@ -34,6 +36,14 @@ var ErrDamaged = errors.New("snapshot is damaged") // by its parts: its own small files read in full, and every shared file it // names read back against the hash of the bytes it was stored with. func VerifyArchive(path string) error { + return VerifyArchiveProgress(path, nil) +} + +// VerifyArchiveProgress is VerifyArchive telling progress how far it is. The +// entries are read as many at a time as suits the drive the archive is on +// (iopar): each is a small read of its own, and one after another a large +// save's quarter of a million of them took minutes. +func VerifyArchiveProgress(path string, progress Progress) error { r, err := zip.OpenReader(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -45,15 +55,25 @@ func VerifyArchive(path string) error { return fmt.Errorf("%w: its archive cannot be opened: %v", ErrDamaged, err) } defer r.Close() + var files []*zip.File for _, f := range r.File { - if f.FileInfo().IsDir() { - continue + if !f.FileInfo().IsDir() { + files = append(files, f) } + } + var report func(done, total int) + if progress != nil { + report = func(done, total int) { progress(PhaseChecking, done, total) } + } + _, err = iopar.RunKind(context.Background(), driveKindOf(path), len(files), func(i int) error { + f := files[i] rc, err := f.Open() if err != nil { return fmt.Errorf("%w: %s cannot be read: %v", ErrDamaged, f.Name, err) } - _, err = io.Copy(io.Discard, rc) + buf := copyBuffers.Get().(*[]byte) + _, err = io.CopyBuffer(io.Discard, rc, *buf) + copyBuffers.Put(buf) rc.Close() if err != nil { if errors.Is(err, zip.ErrChecksum) { @@ -61,8 +81,9 @@ func VerifyArchive(path string) error { } return fmt.Errorf("%w: %s cannot be read: %v", ErrDamaged, f.Name, err) } - } - return nil + return nil + }, report) + return err } // Verify checks one snapshot and records what it found. diff --git a/internal/snapshot/zip.go b/internal/snapshot/zip.go index 1ea2920c..53db6e11 100644 --- a/internal/snapshot/zip.go +++ b/internal/snapshot/zip.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "io" + "io/fs" "os" "path/filepath" "strings" @@ -38,6 +39,10 @@ func ZipPath(sourcePath, outPath string) (skipped []string, err error) { // archived so the caller can record what the snapshot holds. The hashes come // free: the bytes pass through a hasher on their way into the archive. func ZipPathCapturing(sourcePath, outPath string) (skipped []string, captured []store.CapturedFile, err error) { + return zipPathCapturing(sourcePath, outPath, nil) +} + +func zipPathCapturing(sourcePath, outPath string, reuse *reuseSource) (skipped []string, captured []store.CapturedFile, err error) { info, err := os.Stat(sourcePath) if err != nil { return nil, nil, fmt.Errorf("source path does not exist: %w", err) @@ -54,7 +59,7 @@ func ZipPathCapturing(sourcePath, outPath string) (skipped []string, captured [] if !info.IsDir() { name := filepath.Base(sourcePath) - hash, addErr := addFileEntry(w, sourcePath, name) + hash, addErr := reuse.addFileEntry(w, sourcePath, name, info) if addErr != nil { return nil, nil, addErr } @@ -62,7 +67,10 @@ func ZipPathCapturing(sourcePath, outPath string) (skipped []string, captured [] } archived := 0 - walkErr := filepath.Walk(sourcePath, func(path string, walkInfo os.FileInfo, walkErr error) error { + // WalkDir, not Walk: Walk's extra Lstat per entry opens every file on + // Windows, which on a save of a quarter-million files cost more than the + // archiving itself once unchanged files are copied (reuse.go). + walkErr := filepath.WalkDir(sourcePath, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { // Unreadable subtree — record it and move on. if path != sourcePath { @@ -80,14 +88,24 @@ func ZipPathCapturing(sourcePath, outPath string) (skipped []string, captured [] } rel = filepath.ToSlash(rel) - if walkInfo.IsDir() { + if d.IsDir() { // Explicit directory entries keep empty dirs restorable. if _, err := w.CreateHeader(&zip.FileHeader{Name: rel + "/", Method: zip.Store}); err != nil { return err } return nil } - hash, addErr := addFileEntry(w, path, rel) + // OpenSave's own temporary file for a file being pulled: never part + // of the save, and gone by the time it is read. + if strings.HasSuffix(rel, delta.TmpSuffix) || delta.NeverSynced(rel) { + return nil + } + info, infoErr := d.Info() + if infoErr != nil { + skipped = append(skipped, path) + return nil + } + hash, addErr := reuse.addFileEntry(w, path, rel, info) if addErr != nil { skipped = append(skipped, path) return nil @@ -248,6 +266,11 @@ func newSnapshotWriter(out io.Writer) *zip.Writer { // directory and extracts into it — both matching unzipDirectory() in the // JS app. func UnzipTo(zipPath, targetPath string) error { + return UnzipToProgress(zipPath, targetPath, nil) +} + +// UnzipToProgress is UnzipTo telling progress how far it is. +func UnzipToProgress(zipPath, targetPath string, progress Progress) error { // Defence in depth, not the load-bearing guard. Extraction does not // restore the archived modification time — the written file gets the // current one — so the hash cache would notice on its own today. It is @@ -308,12 +331,11 @@ func UnzipTo(zipPath, targetPath string) error { } } + jobs := make([]extractJob, 0, len(r.File)) for _, entry := range r.File { - if err := extractEntry(entry, destDir); err != nil { - return err - } + jobs = append(jobs, extractJob{f: entry, destDir: destDir, rel: entry.Name}) } - return nil + return extractAll(jobs, destDir, progress) } func extractEntry(entry *zip.File, destDir string) error { diff --git a/internal/snapshot/zip_roots.go b/internal/snapshot/zip_roots.go index 8f70ff2f..540e538f 100644 --- a/internal/snapshot/zip_roots.go +++ b/internal/snapshot/zip_roots.go @@ -3,6 +3,7 @@ package snapshot import ( "archive/zip" "fmt" + "io/fs" "os" "path/filepath" "sort" @@ -44,8 +45,14 @@ func ZipRoots(primary string, extra map[string]string, outPath string) (skipped // instead of inferred from a single whole-save hash that nothing reliably // kept current. func ZipRootsCapturing(primary string, extra map[string]string, outPath string) (skipped []string, captured []store.CapturedFile, err error) { + return zipRootsCapturing(primary, extra, outPath, nil) +} + +// zipRootsCapturing is ZipRootsCapturing, copying unchanged files straight +// out of reuse (the previous snapshot) when it is given. See reuse.go. +func zipRootsCapturing(primary string, extra map[string]string, outPath string, reuse *reuseSource) (skipped []string, captured []store.CapturedFile, err error) { if len(extra) == 0 { - return ZipPathCapturing(primary, outPath) + return zipPathCapturing(primary, outPath, reuse) } // One archive, written in a single pass. @@ -69,7 +76,7 @@ func ZipRootsCapturing(primary string, extra map[string]string, outPath string) defer f.Close() w := newSnapshotWriter(f) - primarySkipped, primaryFiles, err := archiveInto(w, primary, "", "") + primarySkipped, primaryFiles, err := archiveInto(w, primary, "", "", reuse) skipped = append(skipped, primarySkipped...) captured = append(captured, primaryFiles...) if err != nil { @@ -82,7 +89,7 @@ func ZipRootsCapturing(primary string, extra map[string]string, outPath string) if strings.TrimSpace(path) == "" { continue } - sub, subFiles, subErr := archiveInto(w, path, RootPrefix+name+"/", name) + sub, subFiles, subErr := archiveInto(w, path, RootPrefix+name+"/", name, reuse) skipped = append(skipped, sub...) captured = append(captured, subFiles...) if subErr != nil { @@ -94,14 +101,14 @@ func ZipRootsCapturing(primary string, extra map[string]string, outPath string) } // archiveInto writes one directory tree into an open zip under prefix. -func archiveInto(w *zip.Writer, sourcePath, prefix, root string) (skipped []string, captured []store.CapturedFile, err error) { +func archiveInto(w *zip.Writer, sourcePath, prefix, root string, reuse *reuseSource) (skipped []string, captured []store.CapturedFile, err error) { info, err := os.Stat(sourcePath) if err != nil { return nil, nil, err } if !info.IsDir() { name := filepath.Base(sourcePath) - hash, addErr := addFileEntry(w, sourcePath, prefix+name) + hash, addErr := reuse.addFileEntry(w, sourcePath, prefix+name, info) if addErr != nil { return nil, nil, addErr } @@ -117,7 +124,7 @@ func archiveInto(w *zip.Writer, sourcePath, prefix, root string) (skipped []stri } } - walkErr := filepath.Walk(sourcePath, func(path string, walkInfo os.FileInfo, walkErr error) error { + walkErr := filepath.WalkDir(sourcePath, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { if path != sourcePath { skipped = append(skipped, path) @@ -133,11 +140,20 @@ func archiveInto(w *zip.Writer, sourcePath, prefix, root string) (skipped []stri return err } rel = filepath.ToSlash(rel) - if walkInfo.IsDir() { + if d.IsDir() { _, err := w.CreateHeader(&zip.FileHeader{Name: prefix + rel + "/", Method: zip.Store}) return err } - hash, addErr := addFileEntry(w, path, prefix+rel) + // OpenSave's own temporary file for a file being pulled (zip.go). + if strings.HasSuffix(rel, delta.TmpSuffix) || delta.NeverSynced(rel) { + return nil + } + info, infoErr := d.Info() + if infoErr != nil { + skipped = append(skipped, path) + return nil + } + hash, addErr := reuse.addFileEntry(w, path, prefix+rel, info) if addErr != nil { skipped = append(skipped, path) return nil @@ -196,6 +212,11 @@ func rootOfEntry(entry string) (string, bool) { // restore — the operation someone reaches for when things have already gone // wrong. func UnzipRoots(zipPath, primary string, extra map[string]string) (unplaced []string, err error) { + return UnzipRootsProgress(zipPath, primary, extra, nil) +} + +// UnzipRootsProgress is UnzipRoots telling progress how far it is. +func UnzipRootsProgress(zipPath, primary string, extra map[string]string, progress Progress) (unplaced []string, err error) { // Every location this can write into, dropped from the hash cache. See // UnzipTo for why this is defence in depth rather than the guard that is // currently doing the work. @@ -211,7 +232,7 @@ func UnzipRoots(zipPath, primary string, extra map[string]string) (unplaced []st return nil, err } if len(roots) == 0 { - return nil, UnzipTo(zipPath, primary) + return nil, UnzipToProgress(zipPath, primary, progress) } r, err := zip.OpenReader(zipPath) @@ -248,6 +269,7 @@ func UnzipRoots(zipPath, primary string, extra map[string]string) (unplaced []st cleared := map[string]bool{} missing := map[string]bool{} + var jobs []extractJob for _, f := range r.File { name, isRoot := rootOfEntry(f.Name) target := primary @@ -283,9 +305,10 @@ func UnzipRoots(zipPath, primary string, extra map[string]string) (unplaced []st continue } } - if err := extractEntryAs(f, target, entry); err != nil { - return nil, err - } + jobs = append(jobs, extractJob{f: f, destDir: target, rel: entry}) + } + if err := extractAll(jobs, primary, progress); err != nil { + return nil, err } for name := range missing { diff --git a/internal/store/compact.go b/internal/store/compact.go new file mode 100644 index 00000000..069bb895 --- /dev/null +++ b/internal/store/compact.go @@ -0,0 +1,43 @@ +package store + +// Giving the space of deleted rows back to the disk. +// +// SQLite keeps the pages rows were deleted from, for rows to come; the file +// never shrinks on its own. Snapshots are merged, pruned and replaced all the +// time, and each held a row per file of its save — a quarter of a million +// for one game — so the file grew by what every snapshot ever deleted had +// held: 663 MB on one device, 262 MB of it unused. +// +// With auto_vacuum set to INCREMENTAL the free pages can be handed back a +// few at a time (Compact), never holding the database for long. Setting it +// takes one full VACUUM, done once, when the database is opened and nothing +// else is using it: some seconds for a large one. + +// freePagesWorthCompacting is how many free pages are given back at a time, +// and fewer are not worth the bother: 4 MB at the default page size. +const freePagesWorthCompacting = 1024 + +func (s *Store) compactOnOpen() { + var mode int + if err := s.db.Get(&mode, `PRAGMA auto_vacuum`); err != nil { + return + } + if mode != 2 { // 2 = INCREMENTAL + if _, err := s.db.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil { + return + } + _, _ = s.db.Exec(`VACUUM`) + return + } + s.Compact() +} + +// Compact hands the database file's unused pages back to the disk, when +// there are enough of them to matter. +func (s *Store) Compact() { + var free int + if err := s.db.Get(&free, `PRAGMA freelist_count`); err != nil || free < freePagesWorthCompacting { + return + } + _, _ = s.db.Exec(`PRAGMA incremental_vacuum`) +} diff --git a/internal/store/compact_test.go b/internal/store/compact_test.go new file mode 100644 index 00000000..1cec9f8f --- /dev/null +++ b/internal/store/compact_test.go @@ -0,0 +1,97 @@ +package store + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// Deleted rows give their space back to the disk: the file shrinks, rather +// than holding everything it ever held. +func TestCompactGivesSpaceBack(t *testing.T) { + path := filepath.Join(t.TempDir(), "opensave.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + var mode int + _ = s.db.Get(&mode, `PRAGMA auto_vacuum`) + if mode != 2 { + t.Fatalf("auto_vacuum = %d, want incremental (2)", mode) + } + pad := strings.Repeat("x", 2000) + tx := s.db.MustBegin() + for i := 0; i < 4000; i++ { + tx.MustExec(`INSERT INTO marks (name, value) VALUES (?, ?)`, fmt.Sprint("k", i), pad) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + _, _ = s.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) + full, _ := os.Stat(path) + if _, err := s.db.Exec(`DELETE FROM marks`); err != nil { + t.Fatal(err) + } + s.Compact() + _, _ = s.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) + after, _ := os.Stat(path) + if after.Size() >= full.Size()/2 { + t.Errorf("file %d bytes after deleting everything, %d before: nothing given back", after.Size(), full.Size()) + } + s.Close() +} + +// Clearing a large save's deletion records is one transaction, not one write +// to disk per file. +func TestClearDeletedFilesIsOneTransaction(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "opensave.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err := s.CreateGame(Game{ID: "g", Name: "g", SavePath: t.TempDir(), ActiveBranch: "main"}); err != nil { + t.Fatal(err) + } + var files []DeletedFile + var paths []string + for i := 0; i < 50000; i++ { + p := fmt.Sprintf("map/%d/%d.bin", i/100, i) + files = append(files, DeletedFile{Path: p, Hash: "h"}) + paths = append(paths, p) + } + if err := s.RecordDeletedFiles("g", files); err != nil { + t.Fatal(err) + } + start := time.Now() + if err := s.ClearDeletedFiles("g", "", paths); err != nil { + t.Fatal(err) + } + took := time.Since(start) + left, _ := s.DeletedFiles("g", "") + if len(left) != 0 { + t.Errorf("%d records left", len(left)) + } + // Generous: CI runs this under the race detector on shared runners, where + // one statement per file took 10s; what this guards against took minutes. + if took > 30*time.Second { + t.Errorf("clearing 50,000 records took %v", took) + } + t.Logf("cleared 50,000 deletion records in %v", took) +} + +// The database writes through a write-ahead log, not a rollback journal +// created and deleted for every write. +func TestDatabaseUsesWAL(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "opensave.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var mode string + if err := s.db.Get(&mode, `PRAGMA journal_mode`); err != nil || mode != "wal" { + t.Errorf("journal_mode = %q (%v), want wal", mode, err) + } +} diff --git a/internal/store/db.go b/internal/store/db.go index 9e69aeac..44e17004 100644 --- a/internal/store/db.go +++ b/internal/store/db.go @@ -32,7 +32,14 @@ func Open(path string) (*Store, error) { // _pragma params ensure foreign keys are enforced (SQLite defaults them // off per-connection) and busy_timeout avoids spurious SQLITE_BUSY // errors from the watcher/api/p2p goroutines all touching the DB. - dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)", path) + // + // WAL: a write is appended to one log and the database file is updated + // from it later, rather than every write creating, flushing and deleting a + // rollback journal of its own — which made a quarter of a million small + // writes take most of an hour. synchronous NORMAL is what WAL is meant to + // run with: the database stays whole whatever happens; a power cut can + // only lose the last moments' writes, never corrupt it. + dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)", path) db, err := sqlx.Open("sqlite", dsn) if err != nil { return nil, fmt.Errorf("open sqlite database: %w", err) @@ -51,6 +58,7 @@ func Open(path string) (*Store, error) { db.Close() return nil, err } + s.compactOnOpen() // The file holds the device's private key, the Google tokens and the // vault keys. SQLite creates it with the process umask, which is usually // world-readable; the directory around it is private now, but the file diff --git a/internal/store/deleted_files.go b/internal/store/deleted_files.go index 1cda3429..0a4a0e0e 100644 --- a/internal/store/deleted_files.go +++ b/internal/store/deleted_files.go @@ -107,6 +107,41 @@ func (s *Store) ClearDeletedFile(gameID, root, path string) error { return nil } +// ClearDeletedFiles forgets the deletions recorded for paths of one save +// location, in one transaction: one at a time, each its own write to disk, +// the quarter of a million of a large save took a quarter of an hour. Paths go +// clearDeletedChunk at a time, so that is a few hundred statements, not one +// per file. +func (s *Store) ClearDeletedFiles(gameID, root string, paths []string) error { + if len(paths) == 0 { + return nil + } + tx, err := s.db.Beginx() + if err != nil { + return err + } + defer tx.Rollback() + for len(paths) > 0 { + n := min(len(paths), clearDeletedChunk) + args := make([]any, 0, n+2) + args = append(args, gameID, root) + for _, p := range paths[:n] { + args = append(args, p) + } + q := `DELETE FROM deleted_files WHERE game_id = ? AND root = ? AND path IN (?` + + strings.Repeat(",?", n-1) + `)` + if _, err := tx.Exec(q, args...); err != nil { + return fmt.Errorf("clear deletion records: %w", err) + } + paths = paths[n:] + } + return tx.Commit() +} + +// clearDeletedChunk is how many paths one statement of ClearDeletedFiles +// names: well under SQLite's limit on bound parameters. +const clearDeletedChunk = 500 + // ClearDeletedFilesForGame forgets every deletion recorded for a game, for use // when it is untracked — the records describe a folder this device no longer // has an opinion about. diff --git a/internal/store/games.go b/internal/store/games.go index 52d9c928..18b23ee6 100644 --- a/internal/store/games.go +++ b/internal/store/games.go @@ -72,7 +72,11 @@ type Game struct { // else — snapshots keep capturing every file, so a restore can never be // the thing that deletes an excluded config. SyncIgnore string `db:"sync_ignore" json:"syncIgnore"` - CreatedAt string `db:"created_at" json:"createdAt"` + // SyncDeviceSettings syncs the files the game database names as the + // game's settings, which are otherwise left out of syncing + // (presets/devicesettings.go). Off unless the user turns it on. + SyncDeviceSettings bool `db:"sync_device_settings" json:"syncDeviceSettings"` + CreatedAt string `db:"created_at" json:"createdAt"` } // CreateGame inserts a new game and its default "main" branch in one @@ -159,7 +163,8 @@ func (s *Store) UpdateGame(g Game) error { exe_path = :exe_path, cover_url = :cover_url, last_manifest_hash = :last_manifest_hash, - sync_ignore = :sync_ignore + sync_ignore = :sync_ignore, + sync_device_settings = :sync_device_settings WHERE id = :id`, g) if err != nil { return fmt.Errorf("update game %s: %w", g.ID, err) diff --git a/internal/store/marks.go b/internal/store/marks.go new file mode 100644 index 00000000..5928757a --- /dev/null +++ b/internal/store/marks.go @@ -0,0 +1,22 @@ +package store + +import "fmt" + +// Mark returns the value a one-time step was taken for, empty if never. +// See migrations/0040_marks.sql. +func (s *Store) Mark(name string) string { + var v string + if err := s.db.Get(&v, `SELECT value FROM marks WHERE name = ?`, name); err != nil { + return "" + } + return v +} + +// SetMark records that a one-time step was taken for value. +func (s *Store) SetMark(name, value string) error { + if _, err := s.db.Exec(`INSERT INTO marks (name, value) VALUES (?, ?) + ON CONFLICT(name) DO UPDATE SET value = excluded.value`, name, value); err != nil { + return fmt.Errorf("set mark %s: %w", name, err) + } + return nil +} diff --git a/internal/store/migrations/0038_peer_links.sql b/internal/store/migrations/0038_peer_links.sql new file mode 100644 index 00000000..4cff6cfb --- /dev/null +++ b/internal/store/migrations/0038_peer_links.sql @@ -0,0 +1,15 @@ +-- How fast this device has measured its connection to each paired device: +-- from transfers it made, and from a short speed test when there is nothing +-- recent (see internal/p2p/syncengine/linkspeed.go). Syncs go to the fastest +-- connections first, so a save reaches the devices close to it before it +-- crawls over a slow relay — and those can then hand it on. +-- +-- bytes_per_sec is a running average; measured_ms when it last changed; +-- probed_ms when a speed test last ran, so a slow link is not tested over and +-- over. +CREATE TABLE peer_links ( + peer_id TEXT PRIMARY KEY, + bytes_per_sec REAL NOT NULL DEFAULT 0, + measured_ms INTEGER NOT NULL DEFAULT 0, + probed_ms INTEGER NOT NULL DEFAULT 0 +); diff --git a/internal/store/migrations/0039_game_versions.sql b/internal/store/migrations/0039_game_versions.sql new file mode 100644 index 00000000..3c0c2ce1 --- /dev/null +++ b/internal/store/migrations/0039_game_versions.sql @@ -0,0 +1,42 @@ +-- Which version of a game's save this device holds, as a version vector, and +-- whether it knows of a newer one it has not finished taking yet (see +-- internal/p2p/syncengine/version.go). +-- +-- vector is a JSON object of writer key to counter. A writer key belongs to +-- one device for one game (key below), and its counter moves only when the +-- save really changes there — the game or the user wrote it, never a sync. +-- Comparing two vectors says, without guessing from files or clocks, whether +-- one device is simply behind the other or both changed independently. +-- +-- target is the newest version this device has heard of that is newer than +-- its own, '' when none. While it is set this device is outdated: it takes +-- the save only from a device holding that version, and is a source for +-- nobody — its files are not handed out, and what it lacks is not read as +-- deleted. +-- +-- counter is this device's own counter for the game. It never goes back, even +-- when the device adopts another's version, so the same key and number never +-- name two different saves. +-- +-- pulling is set while a pull towards target is under way, so the files it +-- writes are not mistaken for a new local version if the app stops half-way. +-- +-- hash is the save's content hash when vector was last set ('' unknown). Two +-- devices on one version holding different files means a change went +-- unnoticed on one of them; this says which. +-- +-- answered is the other device's version this one kept its own save over when +-- someone answered a conflict here ('' none), and answered_at when (Unix ms). +-- It travels with the version, so the other device is asked in turn rather +-- than overwritten, and this one is not asked again. +CREATE TABLE game_versions ( + game_id TEXT PRIMARY KEY REFERENCES games(id) ON DELETE CASCADE, + key TEXT NOT NULL, + counter INTEGER NOT NULL DEFAULT 0, + vector TEXT NOT NULL DEFAULT '{}', + target TEXT NOT NULL DEFAULT '', + pulling INTEGER NOT NULL DEFAULT 0, + hash TEXT NOT NULL DEFAULT '', + answered TEXT NOT NULL DEFAULT '', + answered_at INTEGER NOT NULL DEFAULT 0 +); diff --git a/internal/store/migrations/0040_snapshot_content.sql b/internal/store/migrations/0040_snapshot_content.sql new file mode 100644 index 00000000..46e0a102 --- /dev/null +++ b/internal/store/migrations/0040_snapshot_content.sql @@ -0,0 +1,16 @@ +-- What a snapshot holds, as one value: a hash over every file's name in the +-- archive and its content hash (snapshot.ContentKey). The same files give the +-- same value on every device, so two snapshots of one save — taken here, or +-- taken on another device and recorded here as "Synced from peer" — are known +-- to be the same, and the archive is kept once. '' until computed. +ALTER TABLE snapshots ADD COLUMN content_hash TEXT NOT NULL DEFAULT ''; + +CREATE INDEX IF NOT EXISTS idx_snapshots_content ON snapshots(game_id, branch_name, content_hash); + +-- Another snapshot id that names the same content as a snapshot kept here: a +-- peer's snapshot whose save this device already has archived. Known, so it is +-- not archived again, and not asked about again on every sync. +CREATE TABLE snapshot_aliases ( + alias_id TEXT PRIMARY KEY, + snapshot_id TEXT NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE +); diff --git a/internal/store/migrations/0041_marks.sql b/internal/store/migrations/0041_marks.sql new file mode 100644 index 00000000..e9992819 --- /dev/null +++ b/internal/store/migrations/0041_marks.sql @@ -0,0 +1,8 @@ +-- One-time steps this device has already taken, by name, with the value they +-- were taken for. A step whose value changes in a later build runs again. +-- Used for delta.NeverSyncedList: when the files no save is made of change, +-- the hashes recorded under the old list are re-taken once (daemon.go). +CREATE TABLE marks ( + name TEXT PRIMARY KEY, + value TEXT NOT NULL +); diff --git a/internal/store/migrations/0042_sync_device_settings.sql b/internal/store/migrations/0042_sync_device_settings.sql new file mode 100644 index 00000000..a5f9d82e --- /dev/null +++ b/internal/store/migrations/0042_sync_device_settings.sql @@ -0,0 +1,5 @@ +-- Whether a game's device settings (the files the Ludusavi manifest tags as +-- its settings and not its save, presets/devicesettings.go) sync like the rest +-- of its save. Off by default: one PC's resolution and graphics quality are +-- not another's. +ALTER TABLE games ADD COLUMN sync_device_settings INTEGER NOT NULL DEFAULT 0; diff --git a/internal/store/migrations/0043_settings_files.sql b/internal/store/migrations/0043_settings_files.sql new file mode 100644 index 00000000..46dee16b --- /dev/null +++ b/internal/store/migrations/0043_settings_files.sql @@ -0,0 +1,31 @@ +-- Files in a game's main save folder judged to be its settings rather than its +-- save, by what they hold (presets/detectsettings.go), and files shown to be a +-- save after all — by changing with the save, in this device's snapshots or in +-- what another device sent (daemon/detectsettings.go). +-- +-- verdict 'settings' leaves the file out of syncing, as the game database's +-- settings files are. verdict 'save' keeps it syncing for good: detection never +-- takes it again, and a game database entry that named it is overruled. +-- path is relative to the main save folder, slash-separated. +CREATE TABLE settings_files ( + game_id TEXT NOT NULL REFERENCES games(id) ON DELETE CASCADE, + path TEXT NOT NULL, + verdict TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + at_ms INTEGER NOT NULL, + PRIMARY KEY (game_id, path) +); + +-- What another device's copy of an excluded settings file did, sync by sync: +-- moved counts syncs in which the save changed and so did that copy, still +-- those in which the save changed and it did not. A settings file is changed +-- now and then; a save that was taken for one changes with every session. +CREATE TABLE settings_observations ( + game_id TEXT NOT NULL REFERENCES games(id) ON DELETE CASCADE, + peer_id TEXT NOT NULL, + path TEXT NOT NULL, + last_hash TEXT NOT NULL DEFAULT '', + moved INTEGER NOT NULL DEFAULT 0, + still INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (game_id, peer_id, path) +); diff --git a/internal/store/peer_links.go b/internal/store/peer_links.go new file mode 100644 index 00000000..d65b3a37 --- /dev/null +++ b/internal/store/peer_links.go @@ -0,0 +1,37 @@ +package store + +import "fmt" + +// PeerLink is how fast this device has measured its connection to a peer. +// See migrations/0038_peer_links.sql. +type PeerLink struct { + PeerID string `db:"peer_id"` + BytesPerSec float64 `db:"bytes_per_sec"` + MeasuredMs int64 `db:"measured_ms"` + ProbedMs int64 `db:"probed_ms"` +} + +// PeerLinks returns every recorded link, by peer. +func (s *Store) PeerLinks() (map[string]PeerLink, error) { + var rows []PeerLink + if err := s.db.Select(&rows, `SELECT * FROM peer_links`); err != nil { + return nil, fmt.Errorf("list peer links: %w", err) + } + out := make(map[string]PeerLink, len(rows)) + for _, r := range rows { + out[r.PeerID] = r + } + return out, nil +} + +// SavePeerLink records a peer's link, replacing what was there. +func (s *Store) SavePeerLink(l PeerLink) error { + _, err := s.db.Exec(`INSERT INTO peer_links (peer_id, bytes_per_sec, measured_ms, probed_ms) VALUES (?, ?, ?, ?) + ON CONFLICT(peer_id) DO UPDATE SET bytes_per_sec = excluded.bytes_per_sec, + measured_ms = excluded.measured_ms, probed_ms = excluded.probed_ms`, + l.PeerID, l.BytesPerSec, l.MeasuredMs, l.ProbedMs) + if err != nil { + return fmt.Errorf("save peer link %s: %w", l.PeerID, err) + } + return nil +} diff --git a/internal/store/settings_files.go b/internal/store/settings_files.go new file mode 100644 index 00000000..94794a57 --- /dev/null +++ b/internal/store/settings_files.go @@ -0,0 +1,83 @@ +package store + +import ( + "fmt" + "time" +) + +// Verdicts on a file in a game's save folder (migrations/0042_settings_files.sql). +const ( + VerdictSettings = "settings" + VerdictSave = "save" +) + +// SettingsFile is a verdict on one file of a game's main save folder. +type SettingsFile struct { + GameID string `db:"game_id" json:"-"` + Path string `db:"path" json:"path"` + Verdict string `db:"verdict" json:"verdict"` + Reason string `db:"reason" json:"reason"` + AtMs int64 `db:"at_ms" json:"atMs"` +} + +// SettingsFiles returns every verdict recorded for a game. +func (s *Store) SettingsFiles(gameID string) ([]SettingsFile, error) { + var out []SettingsFile + if err := s.db.Select(&out, `SELECT * FROM settings_files WHERE game_id = ? ORDER BY path`, gameID); err != nil { + return nil, fmt.Errorf("list settings files of %s: %w", gameID, err) + } + return out, nil +} + +// SetSettingsFile records a verdict, replacing any earlier one. +func (s *Store) SetSettingsFile(f SettingsFile) error { + if f.AtMs == 0 { + f.AtMs = time.Now().UnixMilli() + } + if _, err := s.db.NamedExec(`INSERT INTO settings_files (game_id, path, verdict, reason, at_ms) + VALUES (:game_id, :path, :verdict, :reason, :at_ms) + ON CONFLICT(game_id, path) DO UPDATE SET verdict = excluded.verdict, reason = excluded.reason, at_ms = excluded.at_ms`, f); err != nil { + return fmt.Errorf("set settings file %s/%s: %w", f.GameID, f.Path, err) + } + return nil +} + +// SettingsObservation is what one device's copy of an excluded settings file +// has done so far. +type SettingsObservation struct { + GameID string `db:"game_id"` + PeerID string `db:"peer_id"` + Path string `db:"path"` + LastHash string `db:"last_hash"` + Moved int `db:"moved"` + Still int `db:"still"` +} + +// SettingsObservations returns what every device's copy of path has done. +func (s *Store) SettingsObservations(gameID, path string) ([]SettingsObservation, error) { + var out []SettingsObservation + if err := s.db.Select(&out, `SELECT * FROM settings_observations WHERE game_id = ? AND path = ?`, gameID, path); err != nil { + return nil, fmt.Errorf("list settings observations: %w", err) + } + return out, nil +} + +// GetSettingsObservation returns one device's record ({} when none). +func (s *Store) GetSettingsObservation(gameID, peerID, path string) SettingsObservation { + var o SettingsObservation + if err := s.db.Get(&o, `SELECT * FROM settings_observations WHERE game_id = ? AND peer_id = ? AND path = ?`, gameID, peerID, path); err != nil { + return SettingsObservation{GameID: gameID, PeerID: peerID, Path: path} + } + return o +} + +// SaveSettingsObservation writes one device's record. +func (s *Store) SaveSettingsObservation(o SettingsObservation) error { + if _, err := s.db.NamedExec(`INSERT INTO settings_observations (game_id, peer_id, path, last_hash, moved, still) + VALUES (:game_id, :peer_id, :path, :last_hash, :moved, :still) + ON CONFLICT(game_id, peer_id, path) DO UPDATE SET last_hash = excluded.last_hash, + moved = excluded.moved, still = excluded.still`, o); err != nil { + return fmt.Errorf("save settings observation: %w", err) + } + return nil +} diff --git a/internal/store/snapshot_content.go b/internal/store/snapshot_content.go new file mode 100644 index 00000000..f9df1be2 --- /dev/null +++ b/internal/store/snapshot_content.go @@ -0,0 +1,92 @@ +package store + +import ( + "database/sql" + "errors" + "fmt" +) + +// Snapshot content: which snapshots hold the same files, so a save is +// archived once however many times — and on however many devices — it is +// snapshotted. See migrations/0039_snapshot_content.sql. + +// SetSnapshotContentHash records what a snapshot holds. +func (s *Store) SetSnapshotContentHash(id, hash string) error { + if _, err := s.db.Exec(`UPDATE snapshots SET content_hash = ? WHERE id = ?`, hash, id); err != nil { + return fmt.Errorf("set content hash of %s: %w", id, err) + } + return nil +} + +// SnapshotByContent finds a snapshot of a game's branch holding exactly the +// content hash names; ok is false when there is none. The newest, so a copy +// kept in its place is the one retention keeps longest. +func (s *Store) SnapshotByContent(gameID, branch, hash string) (snap Snapshot, ok bool, err error) { + if hash == "" { + return Snapshot{}, false, nil + } + err = s.db.Get(&snap, `SELECT * FROM snapshots WHERE game_id = ? AND branch_name = ? AND content_hash = ? + ORDER BY timestamp DESC LIMIT 1`, gameID, branch, hash) + if errors.Is(err, sql.ErrNoRows) { + return Snapshot{}, false, nil + } + if err != nil { + return Snapshot{}, false, fmt.Errorf("find snapshot by content: %w", err) + } + return snap, true, nil +} + +// AddSnapshotAlias records that alias (another device's snapshot id) names the +// same content as snapshotID here. +func (s *Store) AddSnapshotAlias(alias, snapshotID string) error { + if alias == "" || alias == snapshotID { + return nil + } + if _, err := s.db.Exec(`INSERT INTO snapshot_aliases (alias_id, snapshot_id) VALUES (?, ?) + ON CONFLICT(alias_id) DO UPDATE SET snapshot_id = excluded.snapshot_id`, alias, snapshotID); err != nil { + return fmt.Errorf("add snapshot alias %s: %w", alias, err) + } + return nil +} + +// RepointSnapshotAliases moves every alias of from onto to, before from is +// removed in favour of to. +func (s *Store) RepointSnapshotAliases(from, to string) error { + if _, err := s.db.Exec(`UPDATE snapshot_aliases SET snapshot_id = ? WHERE snapshot_id = ?`, to, from); err != nil { + return fmt.Errorf("repoint aliases of %s: %w", from, err) + } + return nil +} + +// ResolveSnapshotID says which snapshot here an id names: itself when it is +// one, the snapshot it is an alias of when it is that; ok false when neither. +func (s *Store) ResolveSnapshotID(id string) (string, bool) { + var found string + if err := s.db.Get(&found, `SELECT id FROM snapshots WHERE id = ?`, id); err == nil { + return found, true + } + if err := s.db.Get(&found, `SELECT snapshot_id FROM snapshot_aliases WHERE alias_id = ?`, id); err == nil { + return found, true + } + return "", false +} + +// SnapshotsWithoutContentHash lists a game's snapshots whose content has not +// been computed yet. +func (s *Store) SnapshotsWithoutContentHash(gameID string) ([]Snapshot, error) { + var out []Snapshot + if err := s.db.Select(&out, `SELECT * FROM snapshots WHERE game_id = ? AND content_hash = '' ORDER BY timestamp`, gameID); err != nil { + return nil, fmt.Errorf("list snapshots without content hash: %w", err) + } + return out, nil +} + +// AllSnapshotsOfGame lists every snapshot of a game, on every branch, oldest +// first. +func (s *Store) AllSnapshotsOfGame(gameID string) ([]Snapshot, error) { + var out []Snapshot + if err := s.db.Select(&out, `SELECT * FROM snapshots WHERE game_id = ? ORDER BY timestamp`, gameID); err != nil { + return nil, fmt.Errorf("list snapshots of %s: %w", gameID, err) + } + return out, nil +} diff --git a/internal/store/snapshots.go b/internal/store/snapshots.go index bf0efc23..19ed403c 100644 --- a/internal/store/snapshots.go +++ b/internal/store/snapshots.go @@ -26,13 +26,16 @@ type Snapshot struct { // if it was not (migration 0032; internal/snapshot/verify.go). CheckedMs int64 `db:"checked_ms" json:"checkedMs"` Problem string `db:"problem" json:"problem"` + // ContentHash names what the snapshot holds (snapshot.ContentKey), the + // same on every device for the same files; '' until computed. + ContentHash string `db:"content_hash" json:"contentHash"` } // CreateSnapshot inserts a new snapshot record. func (s *Store) CreateSnapshot(snap Snapshot) error { _, err := s.db.NamedExec(` - INSERT INTO snapshots (id, game_id, branch_name, timestamp, comment, is_system_auto, zip_path, size_bytes) - VALUES (:id, :game_id, :branch_name, :timestamp, :comment, :is_system_auto, :zip_path, :size_bytes)`, + INSERT INTO snapshots (id, game_id, branch_name, timestamp, comment, is_system_auto, zip_path, size_bytes, content_hash) + VALUES (:id, :game_id, :branch_name, :timestamp, :comment, :is_system_auto, :zip_path, :size_bytes, :content_hash)`, snap) if err != nil { return fmt.Errorf("create snapshot %s: %w", snap.ID, err) diff --git a/internal/store/versions.go b/internal/store/versions.go new file mode 100644 index 00000000..3803db67 --- /dev/null +++ b/internal/store/versions.go @@ -0,0 +1,71 @@ +package store + +import ( + "crypto/rand" + "database/sql" + "encoding/hex" + "errors" + "fmt" +) + +// Save versions: which version of a game's save this device holds. See +// migrations/0037_game_versions.sql for the columns and +// internal/p2p/syncengine/version.go for the rules. The vectors are kept as +// the JSON the sync engine writes; this layer does not interpret them. + +// GameVersion is one game's version record on this device. +type GameVersion struct { + GameID string `db:"game_id"` + Key string `db:"key"` + Counter int64 `db:"counter"` + Vector string `db:"vector"` + Target string `db:"target"` + Pulling bool `db:"pulling"` + Hash string `db:"hash"` + // Answered is the JSON vector this device kept its save over in a + // conflict, AnsweredAt when. + Answered string `db:"answered"` + AnsweredAt int64 `db:"answered_at"` +} + +// GetGameVersion returns a game's version record, creating an empty one — +// with a fresh writer key — the first time it is asked for. A game that is not +// tracked gets a record that is not stored. +func (s *Store) GetGameVersion(gameID string) (GameVersion, error) { + var v GameVersion + err := s.db.Get(&v, `SELECT * FROM game_versions WHERE game_id = ?`, gameID) + if err == nil { + return v, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return GameVersion{}, fmt.Errorf("get game version %s: %w", gameID, err) + } + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return GameVersion{}, err + } + v = GameVersion{GameID: gameID, Key: hex.EncodeToString(b[:]), Vector: "{}"} + // OR IGNORE: two callers creating it at once keep the first key. + if _, err := s.db.Exec(`INSERT OR IGNORE INTO game_versions (game_id, key, vector) + SELECT ?, ?, '{}' WHERE EXISTS (SELECT 1 FROM games WHERE id = ?)`, + gameID, v.Key, gameID); err != nil { + return GameVersion{}, fmt.Errorf("create game version %s: %w", gameID, err) + } + if err := s.db.Get(&v, `SELECT * FROM game_versions WHERE game_id = ?`, gameID); err != nil && + !errors.Is(err, sql.ErrNoRows) { + return GameVersion{}, fmt.Errorf("get game version %s: %w", gameID, err) + } + return v, nil +} + +// SaveGameVersion writes a game's version record. The writer key is never +// changed by this: it is the record's identity. +func (s *Store) SaveGameVersion(v GameVersion) error { + _, err := s.db.Exec(`UPDATE game_versions SET counter = ?, vector = ?, target = ?, pulling = ?, hash = ?, + answered = ?, answered_at = ? WHERE game_id = ?`, + v.Counter, v.Vector, v.Target, v.Pulling, v.Hash, v.Answered, v.AnsweredAt, v.GameID) + if err != nil { + return fmt.Errorf("save game version %s: %w", v.GameID, err) + } + return nil +} diff --git a/internal/watcher/neversynced_test.go b/internal/watcher/neversynced_test.go new file mode 100644 index 00000000..f3d19a75 --- /dev/null +++ b/internal/watcher/neversynced_test.go @@ -0,0 +1,26 @@ +package watcher + +import ( + "testing" + + "github.com/opensave/opensave/internal/delta" +) + +// Steam rewriting remotecache.vdf is not a change of the save; a value +// recorded by an older build, which counted it, is still recognisable. +func TestContentHashLeavesOutNeverSynced(t *testing.T) { + save := delta.Manifest{Files: map[string]delta.FileEntry{"save.sav": {Hash: "a"}}} + withCache := delta.Manifest{Files: map[string]delta.FileEntry{ + "save.sav": {Hash: "a"}, + "remotecache.vdf": {Hash: "steam"}, + }} + if ContentHash(withCache, "") != ContentHash(save, "") { + t.Error("remotecache.vdf counts toward the content hash") + } + if ContentHash(save, "") != save.ContentHash() { + t.Error("a save without such files hashes differently than before") + } + if ContentHashBeforeNeverSynced(withCache, "") != withCache.ContentHash() { + t.Error("the hash an older build recorded cannot be reproduced") + } +} diff --git a/internal/watcher/owntouch_test.go b/internal/watcher/owntouch_test.go new file mode 100644 index 00000000..d984bc29 --- /dev/null +++ b/internal/watcher/owntouch_test.go @@ -0,0 +1,157 @@ +package watcher + +import ( + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/opensave/opensave/internal/owntouch" +) + +// Files OpenSave writes or deletes itself — a pull, a peer's deletions — are +// not a new save. Only the recorded hash moves; no auto-snapshot, and no +// OnChanged echoing the sync back out. +func TestOwnChanges_AreNotSnapshotted(t *testing.T) { + saveDir := t.TempDir() + col := newCollector() + eng := New(col.callbacks()) + defer eng.Stop() + if err := eng.Watch("game1", saveDir); err != nil { + t.Fatal(err) + } + + // A peer's deletions arriving one at a time, as they do, plus pulled files. + for i := 0; i < 5; i++ { + p := filepath.Join(saveDir, fmt.Sprintf("map_%d.bin", i)) + owntouch.Mark(p) + if err := os.WriteFile(p, []byte(fmt.Sprint(i)), 0o666); err != nil { + t.Fatal(err) + } + time.Sleep(300 * time.Millisecond) + } + // Past the debounce, with room to spare. + if !waitFor(t, 8*time.Second, func() bool { + col.mu.Lock() + defer col.mu.Unlock() + return col.manifestHashes["game1"] != "" + }) { + t.Fatal("the recorded hash was not moved to the new content") + } + time.Sleep(3 * time.Second) + if n := col.snapshotCount(); n != 0 { + t.Errorf("%d auto-snapshot(s) for changes OpenSave made itself, want 0", n) + } + if n := col.changedCount(); n != 0 { + t.Errorf("OnChanged fired %d time(s) for changes OpenSave made itself", n) + } + + // The game then saves: that is snapshotted, as always. + if err := os.WriteFile(filepath.Join(saveDir, "slot1.sav"), []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if !waitFor(t, 10*time.Second, func() bool { return col.snapshotCount() >= 1 }) { + t.Fatal("a change the game made was not snapshotted") + } +} + +// A pull that creates folders and writes into them before the watcher has put +// them under watch is still the pull: the rescan that follows registering +// them is not someone else's change. +func TestOwnNewFolders_AreNotSnapshotted(t *testing.T) { + saveDir := t.TempDir() + col := newCollector() + eng := New(col.callbacks()) + defer eng.Stop() + if err := eng.Watch("game1", saveDir); err != nil { + t.Fatal(err) + } + for i := 0; i < 20; i++ { + dir := filepath.Join(saveDir, fmt.Sprintf("slot%02d", i)) + owntouch.Mark(dir) + if err := os.MkdirAll(dir, 0o777); err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "data.sav") + owntouch.Mark(p) + if err := os.WriteFile(p, []byte(fmt.Sprint(i)), 0o666); err != nil { + t.Fatal(err) + } + } + if !waitFor(t, 10*time.Second, func() bool { + col.mu.Lock() + defer col.mu.Unlock() + return col.manifestHashes["game1"] != "" + }) { + t.Fatal("the recorded hash was not moved to the new content") + } + time.Sleep(4 * time.Second) + if n := col.snapshotCount(); n != 0 { + t.Errorf("%d auto-snapshot(s) for folders and files a pull created", n) + } + + // A folder the game creates is a change, as before. + if err := os.MkdirAll(filepath.Join(saveDir, "newslot"), 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(saveDir, "newslot", "data.sav"), []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if !waitFor(t, 10*time.Second, func() bool { return col.snapshotCount() >= 1 }) { + t.Fatal("a folder the game created was not snapshotted") + } +} + +// While a sync is writing the game, the watcher looks at what changed only +// once it has finished — never at a save half-way between two states — and a +// change of the game's is still snapshotted then. +func TestWhileASyncWrites_TheWatcherWaits(t *testing.T) { + saveDir := t.TempDir() + col := newCollector() + var writing atomic.Bool + cb := col.callbacks() + cb.SyncWriting = func(string) bool { return writing.Load() } + cb.SyncWritingNow = cb.SyncWriting + eng := New(cb) + defer eng.Stop() + if err := eng.Watch("game1", saveDir); err != nil { + t.Fatal(err) + } + writing.Store(true) + if err := os.WriteFile(filepath.Join(saveDir, "slot1.sav"), []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + time.Sleep(5 * time.Second) + if n := col.snapshotCount(); n != 0 { + t.Fatalf("%d snapshot(s) while a sync was writing the game", n) + } + writing.Store(false) + if !waitFor(t, 10*time.Second, func() bool { return col.snapshotCount() >= 1 }) { + t.Fatal("the game's change was not snapshotted once the sync had finished") + } +} + +// A burst that mixes OpenSave's own changes with one from the game is a real +// change and is snapshotted. +func TestMixedChanges_AreSnapshotted(t *testing.T) { + saveDir := t.TempDir() + col := newCollector() + eng := New(col.callbacks()) + defer eng.Stop() + if err := eng.Watch("game1", saveDir); err != nil { + t.Fatal(err) + } + own := filepath.Join(saveDir, "pulled.bin") + owntouch.Mark(own) + if err := os.WriteFile(own, []byte("from a peer"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(saveDir, "slot1.sav"), []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if !waitFor(t, 10*time.Second, func() bool { return col.snapshotCount() >= 1 }) { + t.Fatal("a burst including the game's own change was not snapshotted") + } +} diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 612ebfb6..786abb34 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -32,6 +32,7 @@ import ( "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/ignore" "github.com/opensave/opensave/internal/logging" + "github.com/opensave/opensave/internal/owntouch" ) const ( @@ -118,6 +119,30 @@ type Callbacks struct { OnChanged func(gameID string) // Log receives human-readable watcher activity. May be nil. Log func(level, msg string) + // SyncWriting reports whether a sync on this device is writing the game + // now, or was a moment ago: a burst the watcher cannot attribute — an + // overflow of file events — is then taken for the sync's. May be nil. + SyncWriting func(gameID string) bool + // SyncWritingNow reports whether a sync is writing the game at this + // moment: the watcher looks at the burst once it has finished. May be nil. + SyncWritingNow func(gameID string) bool + // HoldSnapshot reports whether a change should not be snapshotted now: + // the game is being played, and a checkpoint was taken recently. The + // change is not recorded as captured either, so the next one, or the end + // of the session, still sees it. May be nil. + HoldSnapshot func(gameID string) bool +} + +// syncWriting reports whether a sync is or was just writing a game +// (Callbacks.SyncWriting). +func (e *Engine) syncWriting(gameID string) bool { + return e.cb.SyncWriting != nil && e.cb.SyncWriting(gameID) +} + +// syncWritingNow reports whether a sync is writing a game at this moment +// (Callbacks.SyncWritingNow). +func (e *Engine) syncWritingNow(gameID string) bool { + return e.cb.SyncWritingNow != nil && e.cb.SyncWritingNow(gameID) } // Engine owns one watch goroutine per tracked game. @@ -188,6 +213,20 @@ type gameWatch struct { // registerFolders when an Add fails. rewatch atomic.Bool + // What the burst since the last snapshot check was made of: changes + // OpenSave applied itself (owntouch), anything else, or both. Read and + // written only by the event loop. + ownEvents, otherEvents bool + // pendingChange is set while a change made here (not by a sync) waits + // out the debounce, for ChangePending: read from other goroutines. + pendingChange atomic.Bool + // foreignFolders: a folder queued for watching since the last rescan was + // not one OpenSave created, or the watch started on a busy folder. The + // rescan that follows registration counts as someone else's change only + // then — a pull creates folders and writes into them before they are + // watched, and that is still the pull. Event loop only. + foreignFolders bool + // folders carries directories to put under watch to registerFolders. See // there for why the event loop does not register them itself. folders chan string @@ -450,6 +489,11 @@ func (e *Engine) WatchWithLocations(gameID, savePath string, extra map[string]st } if busy { gw.rewatch.Store(true) + // What the folder holds is nobody's change OpenSave knows of: this + // rescan must not record it as its own, or a change made while the + // app was closed would become the baseline before the catch-up + // below could see it. Set before the event loop starts. + gw.foreignFolders = true gw.rescan <- struct{}{} // buffered and empty: cannot block } e.games[gameID] = gw @@ -507,6 +551,15 @@ func (e *Engine) Unwatch(gameID string) { } } +// ChangePending reports whether a change made here to a game's save is +// waiting out the debounce: seen, and not yet judged. +func (e *Engine) ChangePending(gameID string) bool { + e.mu.Lock() + gw, ok := e.games[gameID] + e.mu.Unlock() + return ok && gw.pendingChange.Load() +} + // Stop shuts down every watch goroutine. func (e *Engine) Stop() { if e.stopCatch != nil { @@ -676,10 +729,20 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { if !gw.eventRelevant(event) { continue } + own := owntouch.Recent(event.Name) + if own { + gw.ownEvents = true + } else { + gw.otherEvents = true + gw.pendingChange.Store(true) + } // New subdirectory in directory mode: extend the watch — on // registerFolders, never here. See registerFolders. if !gw.isFile && event.Has(fsnotify.Create) { if info, err := os.Stat(event.Name); err == nil && info.IsDir() { + if !own { + gw.foreignFolders = true + } gw.queueFolder(event.Name) } } @@ -689,7 +752,15 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { // Folders just came under watch, or the watch started on a folder // that was busy. Anything written into a folder before its watch // existed raised no event of its own, so read the tree again once - // things are quiet. + // things are quiet. Whose change that is follows from whose + // folders they were: what OpenSave created, it also filled. + if gw.foreignFolders { + gw.otherEvents = true + gw.pendingChange.Store(true) + } else { + gw.ownEvents = true + } + gw.foreignFolders = false resetDebounce() case err, ok := <-gw.fsw.Errors: @@ -718,6 +789,17 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { // directory and allocates no second buffer for one it already // has. gw.rewatch.Store(true) + // The missed events could be anyone's — unless a sync is + // writing this game, which is what overflows the queue: a + // pull of a save of many files does, again and again, and + // each was taken for the game saving, an automatic snapshot + // of a half-pulled save every few seconds. + if e.syncWriting(gw.gameID) { + gw.ownEvents = true + } else { + gw.otherEvents = true + gw.pendingChange.Store(true) + } } else { e.log("warn", fmt.Sprintf("watching %q: %v", gw.gameID, err)) continue @@ -727,6 +809,12 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { case <-debounceC: debounce = nil debounceC = nil + // A sync still writing this game: look at the burst once it has + // finished, not at a save half-way between two states. + if e.syncWritingNow(gw.gameID) { + resetDebounce() + continue + } // Re-register, so a folder that went unwatched is watched from // here on, and read the tree below, so it is found now. Doing only // one of those leaves it correct today and silent tomorrow. @@ -758,7 +846,10 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { for _, path := range gw.extra { delta.InvalidateRoot(path) } - e.handleChange(ctx, gw) + ownOnly := gw.ownEvents && !gw.otherEvents + gw.ownEvents, gw.otherEvents = false, false + gw.pendingChange.Store(false) + e.handleChangeFrom(ctx, gw, ownOnly) } } } @@ -775,6 +866,12 @@ func (gw *gameWatch) eventRelevant(event fsnotify.Event) bool { if strings.HasSuffix(name, ".opensave.tmp") { return false } + // A file the launcher rewrites on its own (Steam's remotecache.vdf, on + // every start) is not a save: it is never synced or snapshotted, so it + // does not wake the watcher either. + if delta.NeverSynced(name) { + return false + } if gw.isFile { return strings.EqualFold( filepath.Clean(event.Name), @@ -788,6 +885,22 @@ func (gw *gameWatch) eventRelevant(event fsnotify.Event) bool { // (gameplay guard), skip if content is unchanged since the last // auto-snapshot, then snapshot with retries and notify. func (e *Engine) handleChange(ctx context.Context, gw *gameWatch) { + e.handleChangeFrom(ctx, gw, false) +} + +// handleChangeFrom is handleChange told whether every change in the burst was +// one OpenSave applied itself — a pull, a deletion a peer asked for, mtimes a +// conflict resolution touched (owntouch). +// +// Those are not a new save. They are a sync, which keeps its own history +// (the mirror snapshot of a pull, the safety snapshot before files are +// replaced). Snapshotting them as well took a full auto-snapshot every couple +// of seconds while a peer deleted files one request at a time, each of a save +// half-way between two states, until they had filled the retention budget and +// pushed out the snapshots worth keeping. So the burst only moves the recorded +// hash, which keeps the next real change measured against what is actually on +// disk; anything not known to be ours is snapshotted as before. +func (e *Engine) handleChangeFrom(ctx context.Context, gw *gameWatch, ownOnly bool) { // Gameplay guard: the game may still be mid-write. for anyFileLocked(gw.savePath) { e.log("info", fmt.Sprintf("save files for %q are in use; waiting (gameplay guard)", gw.gameID)) @@ -821,6 +934,15 @@ func (e *Engine) handleChange(ctx context.Context, gw *gameWatch) { e.log("info", fmt.Sprintf("no content change for %q; skipping auto-snapshot", gw.gameID)) return } + if ownOnly { + if err := e.cb.SetLastManifestHash(gw.gameID, currentHash); err != nil { + e.log("warn", fmt.Sprintf("failed to record manifest hash for %q: %v", gw.gameID, err)) + } + return + } + if e.cb.HoldSnapshot != nil && e.cb.HoldSnapshot(gw.gameID) { + return // being played: kept by the session (a checkpoint, or its end) + } for attempt := 1; attempt <= snapshotMaxRetries; attempt++ { err = e.cb.CreateSnapshot(gw.gameID) @@ -1032,23 +1154,54 @@ func (e *Engine) log(level, msg string) { // against that recorded value, so it has to compute it this way — one // definition, or the question gets two answers. func ContentHash(m delta.Manifest, ignoreRules string) string { - if rules := ignore.Parse(ignoreRules); !rules.Empty() { - m = filterForHash(m, rules) + rules := ignore.Parse(ignoreRules) + leaveOut := func(p string) bool { + return delta.NeverSynced(p) || (!rules.Empty() && rules.Match(p)) + } + if rules.Empty() && !leavesAnyOut(m, leaveOut) { + return m.ContentHash() // the same value, without the copy + } + return filterForHash(m, leaveOut).ContentHash() +} + +func leavesAnyOut(m delta.Manifest, leaveOut func(string) bool) bool { + for p := range m.Files { + if leaveOut(p) { + return true + } + } + for _, root := range m.Extra { + for p := range root.Files { + if leaveOut(p) { + return true + } + } + } + return false +} + +// ContentHashBeforeNeverSynced is ContentHash as builds before +// delta.NeverSynced took it, with those files counted: what a value recorded +// by one of them is compared with, to tell whether the save changed since. +func ContentHashBeforeNeverSynced(m delta.Manifest, ignoreRules string) string { + rules := ignore.Parse(ignoreRules) + if rules.Empty() { + return m.ContentHash() } - return m.ContentHash() + return filterForHash(m, rules.Match).ContentHash() } // filterForHash drops excluded paths before the content hash is taken, so the // value recorded here means the same thing the sync engine means by it. -func filterForHash(m delta.Manifest, rules ignore.Rules) delta.Manifest { +func filterForHash(m delta.Manifest, leaveOut func(string) bool) delta.Manifest { out := delta.Manifest{Files: make(map[string]delta.FileEntry, len(m.Files))} for p, entry := range m.Files { - if !rules.Match(p) { + if !leaveOut(p) { out.Files[p] = entry } } for _, d := range m.Dirs { - if !rules.Match(d) { + if !leaveOut(d) { out.Dirs = append(out.Dirs, d) } } @@ -1057,12 +1210,12 @@ func filterForHash(m delta.Manifest, rules ignore.Rules) delta.Manifest { for name, root := range m.Extra { sub := delta.RootManifest{Files: make(map[string]delta.FileEntry, len(root.Files))} for p, entry := range root.Files { - if !rules.Match(p) { + if !leaveOut(p) { sub.Files[p] = entry } } for _, d := range root.Dirs { - if !rules.Match(d) { + if !leaveOut(d) { sub.Dirs = append(sub.Dirs, d) } } diff --git a/testutil/harness.go b/testutil/harness.go index 9323e7b4..c5d7d043 100644 --- a/testutil/harness.go +++ b/testutil/harness.go @@ -112,7 +112,7 @@ func NewTestDaemon(t *testing.T, name string) *TestDaemon { if err := os.MkdirAll(home, 0o777); err != nil { t.Fatal(err) } - d, err := daemon.New(daemon.Options{HomeOverride: home, DisableDiscovery: true}) + d, err := daemon.New(daemon.Options{HomeOverride: home, DisableDiscovery: true, SyncEveryChange: true}) if err != nil { t.Fatalf("daemon.New(%s): %v", name, err) } @@ -175,6 +175,13 @@ func NewTestDaemon(t *testing.T, name string) *TestDaemon { } td := &TestDaemon{T: t, Daemon: d, Server: srv, Addr: addr, Port: port, SaveDir: saveDir} + t.Cleanup(func() { + if t.Failed() { + for _, e := range d.Log.History() { + t.Logf("[%s] %s %s", name, e.Level, e.Message) + } + } + }) t.Cleanup(func() { srv.Stop() d.Stop()