diff --git a/internal/delta/patch.go b/internal/delta/patch.go index 7976fc45..98e0104b 100644 --- a/internal/delta/patch.go +++ b/internal/delta/patch.go @@ -9,6 +9,7 @@ import ( "time" "github.com/opensave/opensave/internal/fsx" + "github.com/opensave/opensave/internal/owntouch" ) // BlockSource supplies the bytes for one block index, either from the @@ -77,7 +78,11 @@ func replaceWithRetry(tmpPath, filePath string) error { delay *= 2 } clearReadOnlyIfSet(filePath) + // Marked before the rename, so the watcher's event finds it already + // known as a change OpenSave made (owntouch). + owntouch.Mark(filePath) if err = renameFile(tmpPath, filePath); err == nil { + owntouch.Settled(filePath) return nil } } diff --git a/internal/owntouch/owntouch.go b/internal/owntouch/owntouch.go new file mode 100644 index 00000000..e63987a5 --- /dev/null +++ b/internal/owntouch/owntouch.go @@ -0,0 +1,137 @@ +// Package owntouch remembers which paths in save folders OpenSave itself has +// just written or removed, so the watcher can tell those changes apart from +// ones a game or a person made. +// +// Without it every change OpenSave applied — a file pulled from a peer, a +// deletion a peer asked for, the mtimes "keep mine" touches — looked to the +// watcher exactly like the game saving. A peer deleting files one request at +// a time over several minutes produced a full auto-snapshot every couple of +// seconds, each of a save that was neither the old one nor the new one, and +// those filled the retention budget and pushed out the snapshots that mattered. +package owntouch + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" +) + +// window is how long a mark counts. Long enough to cover the watcher's +// debounce and a slow burst of events behind it; short enough that the same +// path changed again by a game a little later is not mistaken for ours. +const window = 2 * time.Minute + +var ( + mu sync.Mutex + marked = map[string]mark{} +) + +// mark is when OpenSave touched a path, whether it removed it, and - once +// OpenSave is done with a file (Settled) - the time and size it left it with. +type mark struct { + at time.Time + removed bool + settled bool + mod time.Time + size int64 +} + +func key(path string) string { + p := filepath.Clean(path) + if runtime.GOOS == "windows" { + p = strings.ToLower(p) + } + return p +} + +// Mark records that OpenSave itself just created, wrote, removed or re-dated +// path. Its parent folder is marked too: creating or removing an entry is an +// event on the folder as well. +func Mark(path string) { record(path, false) } + +// MarkRemoved records that OpenSave itself is about to remove path. Only a +// path marked this way counts as OpenSave's when it is found gone: a file +// OpenSave has just written and a game or a person then deleted is their +// change, and has to sync and be snapshotted as one. +func MarkRemoved(path string) { record(path, true) } + +// Settled records the time and size OpenSave has left a file it marked with, +// once it is done with it (written, renamed into place, re-dated). From then on +// the file is OpenSave's only while it still has exactly those: a game saving +// over it a moment later - inside any slack a time comparison would need - is +// the game's change. +func Settled(path string) { + fi, err := os.Lstat(path) + if err != nil || fi.IsDir() { + return + } + k := key(path) + mu.Lock() + defer mu.Unlock() + if m, ok := marked[k]; ok { + m.settled, m.mod, m.size = true, fi.ModTime(), fi.Size() + marked[k] = m + } +} + +func record(path string, removed bool) { + if path == "" { + return + } + now := time.Now() + mu.Lock() + defer mu.Unlock() + marked[key(path)] = mark{at: now, removed: removed} + marked[key(filepath.Dir(path))] = mark{at: now} + if len(marked) > 50000 { + for k, m := range marked { + if now.Sub(m.at) > window { + delete(marked, k) + } + } + } +} + +// slack is how much later than its mark a file OpenSave wrote may be dated: +// re-dating to now happens just after marking, and some filesystems keep +// times to two seconds. +const slack = 2 * time.Second + +// Recent reports whether the change just seen at path is one OpenSave made: +// the path was marked within the window, and nothing has written it since. +// +// The second half matters as much as the first. A game often saves straight +// after a sync — the same file OpenSave has just put there — and a mark alone +// would take that save for the sync's for two minutes: no snapshot of it, and +// no new version to hand to anyone. Whatever OpenSave leaves behind is dated +// no later than its mark (a pulled file keeps the time it was written to its +// temporary name, or is given the peer's; re-dating sets now), so a file +// dated later was written by someone else. Once OpenSave is done with a file +// (Settled) the test is exact: the time and size it left, or not ours - the +// slack would take a game's save within two seconds of a pull for the pull's. +// A path that is gone is OpenSave's +// only if OpenSave removed it (MarkRemoved): one it wrote and someone then +// deleted is theirs. A folder carries no save of its own, so its mark is +// enough. +func Recent(path string) bool { + mu.Lock() + m, ok := marked[key(path)] + mu.Unlock() + if !ok || time.Since(m.at) > window { + return false + } + fi, err := os.Lstat(path) + if err != nil { + return m.removed + } + if fi.IsDir() { + return true + } + if m.settled { + return fi.ModTime().Equal(m.mod) && fi.Size() == m.size + } + return !fi.ModTime().After(m.at.Add(slack)) +} diff --git a/internal/owntouch/owntouch_test.go b/internal/owntouch/owntouch_test.go new file mode 100644 index 00000000..824f04a4 --- /dev/null +++ b/internal/owntouch/owntouch_test.go @@ -0,0 +1,132 @@ +package owntouch + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestMarkAndRecent(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "sub", "slot1.sav") + if Recent(file) { + t.Fatal("unmarked path reported as ours") + } + Mark(file) + if err := os.MkdirAll(filepath.Dir(file), 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("pulled"), 0o666); err != nil { + t.Fatal(err) + } + if !Recent(file) { + t.Error("marked path not reported as ours") + } + if !Recent(filepath.Dir(file)) { + t.Error("the parent folder of a marked path not reported as ours") + } + if Recent(filepath.Join(dir, "sub", "slot2.sav")) { + t.Error("a sibling of a marked path reported as ours") + } +} + +// A game saving over a file OpenSave has just written is the game's change, +// however soon after. +func TestRecent_AWriteAfterTheMarkIsNotOurs(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "slot1.sav") + Mark(file) + if err := os.WriteFile(file, []byte("pulled"), 0o666); err != nil { + t.Fatal(err) + } + if !Recent(file) { + t.Fatal("the file OpenSave wrote is not reported as ours") + } + // The game saves a little later. + later := time.Now().Add(5 * time.Second) + if err := os.WriteFile(file, []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(file, later, later); err != nil { + t.Fatal(err) + } + if Recent(file) { + t.Error("a save written after OpenSave's was taken for OpenSave's") + } +} + +// A file OpenSave has just written, then deleted by a game or a person, is +// their deletion: it has to sync and be snapshotted. Only a path OpenSave +// removed itself is its own when found gone. +func TestRecent_AFileDeletedAfterOurWriteIsNotOurs(t *testing.T) { + dir := t.TempDir() + pulled := filepath.Join(dir, "pulled.sav") + Mark(pulled) + if err := os.WriteFile(pulled, []byte("from the peer"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.Remove(pulled); err != nil { + t.Fatal(err) + } + if Recent(pulled) { + t.Error("a file deleted by someone else after OpenSave wrote it was taken for OpenSave's") + } + + removed := filepath.Join(dir, "removed.sav") + if err := os.WriteFile(removed, []byte("old"), 0o666); err != nil { + t.Fatal(err) + } + MarkRemoved(removed) + if err := os.Remove(removed); err != nil { + t.Fatal(err) + } + if !Recent(removed) { + t.Error("a file OpenSave removed was not reported as its own") + } +} + +// A game saving over a file OpenSave has just pulled is the game's change +// however soon it comes - inside the slack a time comparison allows too. +func TestRecent_AWriteRightAfterASettledPullIsNotOurs(t *testing.T) { + file := filepath.Join(t.TempDir(), "progress.sav") + Mark(file) + if err := os.WriteFile(file, []byte("from A"), 0o666); err != nil { + t.Fatal(err) + } + Settled(file) + if !Recent(file) { + t.Fatal("the file OpenSave pulled is not reported as ours") + } + // Same size, a moment later: only the time tells them apart. + time.Sleep(20 * time.Millisecond) + if err := os.WriteFile(file, []byte("from B"), 0o666); err != nil { + t.Fatal(err) + } + if Recent(file) { + t.Error("a save written straight after the pull was taken for OpenSave's") + } +} + +// A pull renames a file into place, then re-dates it to the peer's time. In +// between, and before it is Settled again, the file is still OpenSave's. +func TestRecent_ARedatedPullBeforeItSettlesAgainIsOurs(t *testing.T) { + file := filepath.Join(t.TempDir(), "auto.sav") + Mark(file) + if err := os.WriteFile(file, []byte("pulled"), 0o666); err != nil { + t.Fatal(err) + } + Settled(file) + Mark(file) + peers := time.Now().Add(-time.Hour) + if err := os.Chtimes(file, peers, peers); err != nil { + t.Fatal(err) + } + if !Recent(file) { + t.Error("a pulled file re-dated to the peer's time was taken for someone else's change") + } + Settled(file) + if !Recent(file) { + t.Error("a pulled file settled at the peer's time was taken for someone else's change") + } +} diff --git a/internal/p2p/routes.go b/internal/p2p/routes.go index 67d11a5c..526de11e 100644 --- a/internal/p2p/routes.go +++ b/internal/p2p/routes.go @@ -17,6 +17,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/logging" + "github.com/opensave/opensave/internal/owntouch" "github.com/opensave/opensave/internal/p2p/pairing" "github.com/opensave/opensave/internal/p2p/syncengine" "github.com/opensave/opensave/internal/store" @@ -829,6 +830,8 @@ func (e *Engine) handleDeleteFile(w http.ResponseWriter, r *http.Request) { entry, _ = delta.FileEntryFor(full) } // Empty dirs only, for a folder, like rmdirSync. + // A change made at a peer's request, not by the game (owntouch). + owntouch.MarkRemoved(full) if os.Remove(full) == nil && (info.IsDir() || entry.Hash != "") { e.Sync.NotePeerDeletion(game.ID, body.Root, body.RelPath, entry, info.IsDir()) } diff --git a/internal/p2p/syncengine/conflict.go b/internal/p2p/syncengine/conflict.go index 66293b5a..a0babbb7 100644 --- a/internal/p2p/syncengine/conflict.go +++ b/internal/p2p/syncengine/conflict.go @@ -8,6 +8,7 @@ import ( "time" "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/owntouch" "github.com/opensave/opensave/internal/snapshot" ) @@ -197,14 +198,18 @@ func (e *Engine) touchSaveMtimes(gameID, root string) { return } if !info.IsDir() { + owntouch.Mark(root) _ = os.Chtimes(root, now, now) + owntouch.Settled(root) return } _ = filepath.Walk(root, func(path string, fi os.FileInfo, walkErr error) error { if walkErr != nil || fi.IsDir() { return nil } + owntouch.Mark(path) _ = os.Chtimes(path, now, now) + owntouch.Settled(path) return nil }) } @@ -337,6 +342,7 @@ func (e *Engine) overwriteLocalWithRemote(ctx context.Context, gameID string, pe } full := filepath.Join(game.SavePath, filepath.FromSlash(relPath)) _ = os.Chmod(full, 0o666) + owntouch.MarkRemoved(full) _ = os.Remove(full) } diff --git a/internal/p2p/syncengine/engine.go b/internal/p2p/syncengine/engine.go index e3b02b00..f9992707 100644 --- a/internal/p2p/syncengine/engine.go +++ b/internal/p2p/syncengine/engine.go @@ -12,6 +12,7 @@ import ( "time" "github.com/opensave/opensave/internal/delta" + "github.com/opensave/opensave/internal/owntouch" "github.com/opensave/opensave/internal/snapshot" "github.com/opensave/opensave/internal/store" "github.com/opensave/opensave/internal/syncpause" @@ -1371,6 +1372,7 @@ func (e *Engine) applyLocalDeletions(gameID string, root syncRoot, d Decision) { // spelling this disk actually holds. full := delta.LocalNameFor(root.Path, relPath) _ = os.Chmod(full, 0o666) + owntouch.MarkRemoved(full) if err := os.Remove(full); err == nil { e.Log("info", "deleted locally (peer deleted): "+relPath) } @@ -1385,6 +1387,7 @@ func (e *Engine) applyLocalDeletions(gameID string, root syncRoot, d Decision) { } full := delta.LocalNameFor(root.Path, relDir) if info, err := os.Stat(full); err == nil && info.IsDir() { + owntouch.MarkRemoved(full) if err := os.Remove(full); err == nil { // only removes empty dirs, matching rmdirSync e.Log("info", "deleted directory locally (peer deleted): "+relDir) } @@ -1427,7 +1430,9 @@ func (e *Engine) createPulledDirsIn(gameID string, root syncRoot, dirsToPull []s if !delta.IsSafePath(root.Path, relDir) { continue } - _ = os.MkdirAll(filepath.Join(root.Path, filepath.FromSlash(relDir)), 0o777) + full := filepath.Join(root.Path, filepath.FromSlash(relDir)) + owntouch.Mark(full) + _ = os.MkdirAll(full, 0o777) } } @@ -1475,7 +1480,11 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s // Make sure every remote directory exists before patching into it. for _, dir := range remoteData.Manifest.Dirs { if delta.IsSafePath(root.Path, dir) { - _ = os.MkdirAll(filepath.Join(root.Path, filepath.FromSlash(dir)), 0o777) + full := filepath.Join(root.Path, filepath.FromSlash(dir)) + if _, err := os.Stat(full); err != nil { + owntouch.Mark(full) + _ = os.MkdirAll(full, 0o777) + } } } @@ -1614,7 +1623,11 @@ func (e *Engine) pullFiles(ctx context.Context, peer Peer, gameID string, game s } if remoteFile.MtimeMs > 0 { mtime := time.UnixMilli(int64(remoteFile.MtimeMs)) + // Marked again first: until Settled below, the time the rename left + // is not the one to compare (owntouch). + owntouch.Mark(localFilePath) _ = os.Chtimes(localFilePath, mtime, mtime) + owntouch.Settled(localFilePath) } pulled = append(pulled, relPath) e.Log("info", "file updated: "+relPath) diff --git a/internal/watcher/owntouch_test.go b/internal/watcher/owntouch_test.go new file mode 100644 index 00000000..6a1e8134 --- /dev/null +++ b/internal/watcher/owntouch_test.go @@ -0,0 +1,127 @@ +package watcher + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/opensave/opensave/internal/owntouch" +) + +// Files OpenSave writes or deletes itself — a pull, a peer's deletions — are +// not a new save. Only the recorded hash moves; no auto-snapshot, and no +// OnChanged echoing the sync back out. +func TestOwnChanges_AreNotSnapshotted(t *testing.T) { + saveDir := t.TempDir() + col := newCollector() + eng := New(col.callbacks()) + defer eng.Stop() + if err := eng.Watch("game1", saveDir); err != nil { + t.Fatal(err) + } + + // A peer's deletions arriving one at a time, as they do, plus pulled files. + for i := 0; i < 5; i++ { + p := filepath.Join(saveDir, fmt.Sprintf("map_%d.bin", i)) + owntouch.Mark(p) + if err := os.WriteFile(p, []byte(fmt.Sprint(i)), 0o666); err != nil { + t.Fatal(err) + } + time.Sleep(300 * time.Millisecond) + } + // Past the debounce, with room to spare. + if !waitFor(t, 8*time.Second, func() bool { + col.mu.Lock() + defer col.mu.Unlock() + return col.manifestHashes["game1"] != "" + }) { + t.Fatal("the recorded hash was not moved to the new content") + } + time.Sleep(3 * time.Second) + if n := col.snapshotCount(); n != 0 { + t.Errorf("%d auto-snapshot(s) for changes OpenSave made itself, want 0", n) + } + if n := col.changedCount(); n != 0 { + t.Errorf("OnChanged fired %d time(s) for changes OpenSave made itself", n) + } + + // The game then saves: that is snapshotted, as always. + if err := os.WriteFile(filepath.Join(saveDir, "slot1.sav"), []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if !waitFor(t, 10*time.Second, func() bool { return col.snapshotCount() >= 1 }) { + t.Fatal("a change the game made was not snapshotted") + } +} + +// A pull that creates folders and writes into them before the watcher has put +// them under watch is still the pull: the rescan that follows registering +// them is not someone else's change. +func TestOwnNewFolders_AreNotSnapshotted(t *testing.T) { + saveDir := t.TempDir() + col := newCollector() + eng := New(col.callbacks()) + defer eng.Stop() + if err := eng.Watch("game1", saveDir); err != nil { + t.Fatal(err) + } + for i := 0; i < 20; i++ { + dir := filepath.Join(saveDir, fmt.Sprintf("slot%02d", i)) + owntouch.Mark(dir) + if err := os.MkdirAll(dir, 0o777); err != nil { + t.Fatal(err) + } + p := filepath.Join(dir, "data.sav") + owntouch.Mark(p) + if err := os.WriteFile(p, []byte(fmt.Sprint(i)), 0o666); err != nil { + t.Fatal(err) + } + } + if !waitFor(t, 10*time.Second, func() bool { + col.mu.Lock() + defer col.mu.Unlock() + return col.manifestHashes["game1"] != "" + }) { + t.Fatal("the recorded hash was not moved to the new content") + } + time.Sleep(4 * time.Second) + if n := col.snapshotCount(); n != 0 { + t.Errorf("%d auto-snapshot(s) for folders and files a pull created", n) + } + + // A folder the game creates is a change, as before. + if err := os.MkdirAll(filepath.Join(saveDir, "newslot"), 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(saveDir, "newslot", "data.sav"), []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if !waitFor(t, 10*time.Second, func() bool { return col.snapshotCount() >= 1 }) { + t.Fatal("a folder the game created was not snapshotted") + } +} + +// A burst that mixes OpenSave's own changes with one from the game is a real +// change and is snapshotted. +func TestMixedChanges_AreSnapshotted(t *testing.T) { + saveDir := t.TempDir() + col := newCollector() + eng := New(col.callbacks()) + defer eng.Stop() + if err := eng.Watch("game1", saveDir); err != nil { + t.Fatal(err) + } + own := filepath.Join(saveDir, "pulled.bin") + owntouch.Mark(own) + if err := os.WriteFile(own, []byte("from a peer"), 0o666); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(saveDir, "slot1.sav"), []byte("played"), 0o666); err != nil { + t.Fatal(err) + } + if !waitFor(t, 10*time.Second, func() bool { return col.snapshotCount() >= 1 }) { + t.Fatal("a burst including the game's own change was not snapshotted") + } +} diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 612ebfb6..3c9b7ee9 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -32,6 +32,7 @@ import ( "github.com/opensave/opensave/internal/delta" "github.com/opensave/opensave/internal/ignore" "github.com/opensave/opensave/internal/logging" + "github.com/opensave/opensave/internal/owntouch" ) const ( @@ -188,6 +189,17 @@ type gameWatch struct { // registerFolders when an Add fails. rewatch atomic.Bool + // What the burst since the last snapshot check was made of: changes + // OpenSave applied itself (owntouch), anything else, or both. Read and + // written only by the event loop. + ownEvents, otherEvents bool + // foreignFolders: a folder queued for watching since the last rescan was + // not one OpenSave created, or the watch started on a busy folder. The + // rescan that follows registration counts as someone else's change only + // then — a pull creates folders and writes into them before they are + // watched, and that is still the pull. Event loop only. + foreignFolders bool + // folders carries directories to put under watch to registerFolders. See // there for why the event loop does not register them itself. folders chan string @@ -450,6 +462,11 @@ func (e *Engine) WatchWithLocations(gameID, savePath string, extra map[string]st } if busy { gw.rewatch.Store(true) + // What the folder holds is nobody's change OpenSave knows of: this + // rescan must not record it as its own, or a change made while the + // app was closed would become the baseline before the catch-up + // below could see it. Set before the event loop starts. + gw.foreignFolders = true gw.rescan <- struct{}{} // buffered and empty: cannot block } e.games[gameID] = gw @@ -676,10 +693,19 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { if !gw.eventRelevant(event) { continue } + own := owntouch.Recent(event.Name) + if own { + gw.ownEvents = true + } else { + gw.otherEvents = true + } // New subdirectory in directory mode: extend the watch — on // registerFolders, never here. See registerFolders. if !gw.isFile && event.Has(fsnotify.Create) { if info, err := os.Stat(event.Name); err == nil && info.IsDir() { + if !own { + gw.foreignFolders = true + } gw.queueFolder(event.Name) } } @@ -689,7 +715,14 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { // Folders just came under watch, or the watch started on a folder // that was busy. Anything written into a folder before its watch // existed raised no event of its own, so read the tree again once - // things are quiet. + // things are quiet. Whose change that is follows from whose + // folders they were: what OpenSave created, it also filled. + if gw.foreignFolders { + gw.otherEvents = true + } else { + gw.ownEvents = true + } + gw.foreignFolders = false resetDebounce() case err, ok := <-gw.fsw.Errors: @@ -718,6 +751,7 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { // directory and allocates no second buffer for one it already // has. gw.rewatch.Store(true) + gw.otherEvents = true // the missed events could be anyone's } else { e.log("warn", fmt.Sprintf("watching %q: %v", gw.gameID, err)) continue @@ -758,7 +792,9 @@ func (e *Engine) run(ctx context.Context, gw *gameWatch) { for _, path := range gw.extra { delta.InvalidateRoot(path) } - e.handleChange(ctx, gw) + ownOnly := gw.ownEvents && !gw.otherEvents + gw.ownEvents, gw.otherEvents = false, false + e.handleChangeFrom(ctx, gw, ownOnly) } } } @@ -788,6 +824,22 @@ func (gw *gameWatch) eventRelevant(event fsnotify.Event) bool { // (gameplay guard), skip if content is unchanged since the last // auto-snapshot, then snapshot with retries and notify. func (e *Engine) handleChange(ctx context.Context, gw *gameWatch) { + e.handleChangeFrom(ctx, gw, false) +} + +// handleChangeFrom is handleChange told whether every change in the burst was +// one OpenSave applied itself — a pull, a deletion a peer asked for, mtimes a +// conflict resolution touched (owntouch). +// +// Those are not a new save. They are a sync, which keeps its own history +// (the mirror snapshot of a pull, the safety snapshot before files are +// replaced). Snapshotting them as well took a full auto-snapshot every couple +// of seconds while a peer deleted files one request at a time, each of a save +// half-way between two states, until they had filled the retention budget and +// pushed out the snapshots worth keeping. So the burst only moves the recorded +// hash, which keeps the next real change measured against what is actually on +// disk; anything not known to be ours is snapshotted as before. +func (e *Engine) handleChangeFrom(ctx context.Context, gw *gameWatch, ownOnly bool) { // Gameplay guard: the game may still be mid-write. for anyFileLocked(gw.savePath) { e.log("info", fmt.Sprintf("save files for %q are in use; waiting (gameplay guard)", gw.gameID)) @@ -821,6 +873,12 @@ func (e *Engine) handleChange(ctx context.Context, gw *gameWatch) { e.log("info", fmt.Sprintf("no content change for %q; skipping auto-snapshot", gw.gameID)) return } + if ownOnly { + if err := e.cb.SetLastManifestHash(gw.gameID, currentHash); err != nil { + e.log("warn", fmt.Sprintf("failed to record manifest hash for %q: %v", gw.gameID, err)) + } + return + } for attempt := 1; attempt <= snapshotMaxRetries; attempt++ { err = e.cb.CreateSnapshot(gw.gameID)