Steam's remotecache.vdf is the launcher's file, not a save change

e2e/launcher_files_test.go

@@ -0,0 +1,49 @@
+package e2e
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/opensave/opensave/testutil"
+)
+
+// Steam rewrites remotecache.vdf in a game's userdata folder on every start.
+// That is the launcher's file, not the save: it is not snapshotted as a
+// change, and the other device's copy is neither replaced nor deleted.
+func TestLauncherFile_SteamRewritingRemotecacheIsNoSaveChange(t *testing.T) {
+	a, b, gameID := pairAndTrack(t, "Steam Cloud Game", map[string]string{"save.dat": "progress"})
+	// Each device's Steam keeps its own bookkeeping beside the save.
+	a.WriteSave("remotecache.vdf", "steam bookkeeping on A")
+	b.WriteSave("remotecache.vdf", "steam bookkeeping on B")
+	testutil.SettleSync(t, gameID, a, b)
+	time.Sleep(syncSettleWindow)
+	before := len(snapshotsOn(a, gameID, "main"))
+
+	// Steam starts on A and rewrites its file.
+	a.WriteSave("remotecache.vdf", "steam bookkeeping on A, rewritten")
+	time.Sleep(3 * syncSettleWindow)
+	a.API(http.MethodPost, "/api/games/"+gameID+"/sync", nil, nil)
+	testutil.SettleSync(t, gameID, a, b)
+
+	if after := len(snapshotsOn(a, gameID, "main")); after != before {
+		t.Errorf("Steam rewriting remotecache.vdf was snapshotted as a change: %d -> %d snapshots", before, after)
+	}
+	if got := b.ReadSave("remotecache.vdf"); got != "steam bookkeeping on B" {
+		t.Errorf("B's own remotecache.vdf was replaced: %q", got)
+	}
+	if got := b.ReadSave("save.dat"); got != "progress" {
+		t.Errorf("the save on B changed: %q", got)
+	}
+
+	// A real save change still syncs.
+	a.WriteSave("save.dat", "more progress")
+	time.Sleep(syncSettleWindow)
+	a.API(http.MethodPost, "/api/games/"+gameID+"/sync", nil, nil)
+	if !testutil.WaitFor(45*time.Second, func() bool { return b.ReadSave("save.dat") == "more progress" }) {
+		t.Errorf("a real save change did not reach B: %q", b.ReadSave("save.dat"))
+	}
+	if got := b.ReadSave("remotecache.vdf"); got != "steam bookkeeping on B" {
+		t.Errorf("B's remotecache.vdf was replaced along with the save: %q", got)
+	}
+}

internal/delta/manifest.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -29,6 +30,25 @@ import (
 // peer as a real file (and cascade into name.opensave.tmp.opensave.tmp).
 const TmpSuffix = ".opensave.tmp"
 
+// NeverSynced reports whether a file in a save folder is never part of the
+// save, on any device.
+//
+// remotecache.vdf is Steam's record of what Steam Cloud has uploaded for a
+// game, kept beside the game's files in Steam's userdata folder — which is
+// where a game whose saves live in Steam Cloud is tracked. Steam rewrites it
+// on every start, so it read as a new save: an automatic snapshot of a game
+// nobody played, and every other device taking it. It describes this
+// device's Steam, not the save, and where that folder is under Program Files
+// the copy failed with "access denied" on every pass, from every device.
+//
+// Left out where decisions are made — what to sync, whether the save changed
+// — exactly like an exclusion rule, and NOT from the manifest a device builds
+// and serves: a device on an older build would read the gap as the file
+// deleted, and delete its own copy (syncengine/ignore.go).
+func NeverSynced(rel string) bool {
+	return strings.EqualFold(path.Base(rel), "remotecache.vdf")
+}
+
 // staleTmpAge is how old a leftover TmpSuffix file must be before the
 // manifest walk garbage-collects it. Generous enough that a patch actively
 // writing its temp file is never deleted out from under it.

internal/delta/neversynced_test.go

@@ -0,0 +1,17 @@
+package delta
+
+import "testing"
+
+func TestNeverSynced(t *testing.T) {
+	for p, want := range map[string]bool{
+		"remotecache.vdf":        true,
+		"remote/RemoteCache.VDF": true,
+		"remote/save.sav":        false,
+		"remotecache.vdf.bak":    false,
+		"myremotecache.vdf":      false,
+	} {
+		if got := NeverSynced(p); got != want {
+			t.Errorf("NeverSynced(%q) = %v, want %v", p, got, want)
+		}
+	}
+}

internal/p2p/syncengine/conflict.go

@@ -189,10 +189,9 @@ func (e *Engine) markResolvedLocal(ctx context.Context, gameID string, peer Peer
 	} else {
 		e.Log("info", fmt.Sprintf("could not read %s's save while resolving %q (%v) — nothing is recorded as shared, so nothing it lacks is taken as deleted", peer.Name, gameID, err))
 	}
-	if rules := e.rulesFor(gameID); !rules.Empty() {
-		files = filterPathList(files, rules)
-		dirs = filterPathList(dirs, rules)
-	}
+	rules := e.rulesFor(gameID)
+	files = filterPathList(files, rules)
+	dirs = filterPathList(dirs, rules)
 	if err := e.Store.SetSyncState(gameID, peer.ID, files, dirs); err != nil {
 		e.Log("warn", fmt.Sprintf("persist sync lineage failed: %v", err))
 	}

internal/p2p/syncengine/engine.go

@@ -502,10 +502,10 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re
 	// would propagate a deletion of the very file being protected.
 	ignoreRules := e.rulesFor(gameID)
 	unfilteredLocal, unfilteredRemote := localManifest, remoteData.Manifest
-	if !ignoreRules.Empty() {
-		localManifest = filterManifest(localManifest, ignoreRules)
-		remoteData.Manifest = filterManifest(remoteData.Manifest, ignoreRules)
-	}
+	// Filtered always, not only with rules: launcher files (delta.NeverSynced)
+	// are left out the same way.
+	localManifest = filterManifest(localManifest, ignoreRules)
+	remoteData.Manifest = filterManifest(remoteData.Manifest, ignoreRules)
 
 	// 3. Existing unresolved conflict blocks further syncing.
 	e.mu.Lock()
@@ -589,7 +589,7 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re
 	// side's UNFILTERED state still hashes to the base then that side has not
 	// changed since, whichever view the base was written in, and its filtered
 	// hash is the same fact expressed in today's terms.
-	if !ignoreRules.Empty() && agreedHash != "" {
+	if agreedHash != "" {
 		switch agreedHash {
 		case unfilteredLocal.ManifestHash():
 			agreedHash = localManifest.ManifestHash()
@@ -614,10 +614,8 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re
 	// path is still recorded as shared. Leave it there and the decision reads
 	// "we both had this and now I do not" — and propagates a deletion of the
 	// file the rule exists to protect.
-	if rules := e.rulesFor(gameID); !rules.Empty() {
-		lineageFiles = filterLineage(lineageFiles, rules)
-		lineageDirs = filterLineage(lineageDirs, rules)
-	}
+	lineageFiles = filterLineage(lineageFiles, ignoreRules)
+	lineageDirs = filterLineage(lineageDirs, ignoreRules)
 
 	// A side that is merely behind the other has not diverged from it, whatever
 	// the clocks and the base say (OnlyBehind).
@@ -641,11 +639,9 @@ func (e *Engine) SyncWithPeer(ctx context.Context, gameID string, peer Peer) (Re
 	// Excluded paths are filtered out for the same reason the lineage is: a
 	// rule change must never be able to reach across and remove a peer's file.
 	deleted := e.recordedDeletions(gameID, delta.PrimaryRoot)
-	if rules := e.rulesFor(gameID); !rules.Empty() {
-		for path := range deleted {
-			if rules.Match(path) {
-				delete(deleted, path)
-			}
+	for path := range deleted {
+		if excluded(ignoreRules, path) {
+			delete(deleted, path)
 		}
 	}
 	decision := ComputeWithDeletions(localManifest, remoteData.Manifest, lineageFiles, lineageDirs, agreedHash, deleted)
@@ -920,10 +916,9 @@ func (e *Engine) persistLineage(gameID, peerID string, local, remote delta.Manif
 	// RefreshLineage rebuilds this from unfiltered manifests, so without a
 	// filter here an excluded file would be written back in — and the next
 	// sync would read it as shared, then as deleted, and propagate that.
-	if rules := e.rulesFor(gameID); !rules.Empty() {
-		files = filterPathList(files, rules)
-		dirs = filterPathList(dirs, rules)
-	}
+	rules := e.rulesFor(gameID)
+	files = filterPathList(files, rules)
+	dirs = filterPathList(dirs, rules)
 	if err := e.Store.SetSyncState(gameID, peerID, files, dirs); err != nil {
 		e.Log("warn", fmt.Sprintf("persist sync lineage failed: %v", err))
 	}
@@ -1055,9 +1050,7 @@ func (e *Engine) AddConfirmedLineageForRoot(gameID, peerID, root string, files [
 	// Excluded paths must not enter the record of what both sides hold, for
 	// the same reason persistLineage filters them: the next sync would read an
 	// excluded file as shared, then as deleted, and propagate that.
-	if rules := e.rulesFor(gameID); !rules.Empty() {
-		merged = filterPathList(merged, rules)
-	}
+	merged = filterPathList(merged, e.rulesFor(gameID))
 	sort.Strings(merged)
 	if err := e.Store.SetSyncStateForRoot(gameID, peerID, root, merged, existingDirs); err != nil {
 		e.Log("warn", fmt.Sprintf("recording confirmed lineage failed: %v", err))

internal/p2p/syncengine/hold.go

@@ -85,6 +85,11 @@ type emptiness struct {
 func (x emptiness) back(held map[string][]string) bool {
 	for name, files := range held {
 		for _, f := range files {
+			if delta.NeverSynced(f) {
+				// Recorded by a build that still counted it; never part of
+				// the save, so not something that has to come back.
+				continue
+			}
 			if _, ok := x.present[name][f]; !ok {
 				return false
 			}

internal/p2p/syncengine/ignore.go

@@ -42,10 +42,53 @@ func (e *Engine) rulesFor(gameID string) ignore.Rules {
 	return ignore.Parse(game.SyncIgnore)
 }
 
+// excluded says whether a path takes no part in syncing: a file no save is
+// made of on any device (delta.NeverSynced), or one the game's rules exclude.
+// The first is handled exactly like the second — the manifest served still
+// lists it, the decision never sees it — so a device on an older build, which
+// still counts it, is never told it was deleted.
+func excluded(rules ignore.Rules, p string) bool {
+	return delta.NeverSynced(p) || (!rules.Empty() && rules.Match(p))
+}
+
+func holdsNeverSynced(m delta.Manifest) bool {
+	for p := range m.Files {
+		if delta.NeverSynced(p) {
+			return true
+		}
+	}
+	for _, root := range m.Extra {
+		for p := range root.Files {
+			if delta.NeverSynced(p) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func anyNeverSynced(paths []string) bool {
+	for _, p := range paths {
+		if delta.NeverSynced(p) {
+			return true
+		}
+	}
+	return false
+}
+
+func anyNeverSyncedIn(paths map[string]struct{}) bool {
+	for p := range paths {
+		if delta.NeverSynced(p) {
+			return true
+		}
+	}
+	return false
+}
+
 // filterManifest returns a copy with every excluded path removed. The
 // original is untouched: it is still what gets served to peers.
 func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest {
-	if rules.Empty() {
+	if rules.Empty() && !holdsNeverSynced(m) {
 		return m
 	}
 	out := delta.Manifest{
@@ -61,7 +104,7 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest {
 	// would be told their saves diverged, over a file neither of them is
 	// syncing.
 	for p, entry := range m.Files {
-		if rules.Match(p) {
+		if excluded(rules, p) {
 			continue
 		}
 		out.Files[p] = entry
@@ -70,7 +113,7 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest {
 		}
 	}
 	for _, d := range m.Dirs {
-		if rules.Match(d) {
+		if excluded(rules, d) {
 			continue
 		}
 		out.Dirs = append(out.Dirs, d)
@@ -83,13 +126,13 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest {
 		for name, root := range m.Extra {
 			sub := delta.RootManifest{Files: make(map[string]delta.FileEntry, len(root.Files))}
 			for p, entry := range root.Files {
-				if rules.Match(p) {
+				if excluded(rules, p) {
 					continue
 				}
 				sub.Files[p] = entry
 			}
 			for _, d := range root.Dirs {
-				if rules.Match(d) {
+				if excluded(rules, d) {
 					continue
 				}
 				sub.Dirs = append(sub.Dirs, d)
@@ -107,12 +150,12 @@ func filterManifest(m delta.Manifest, rules ignore.Rules) delta.Manifest {
 // decision reads "we both had this and now I don't" and propagates a deletion
 // to the peer.
 func filterLineage(paths map[string]struct{}, rules ignore.Rules) map[string]struct{} {
-	if rules.Empty() {
+	if rules.Empty() && !anyNeverSyncedIn(paths) {
 		return paths
 	}
 	out := make(map[string]struct{}, len(paths))
 	for p := range paths {
-		if rules.Match(p) {
+		if excluded(rules, p) {
 			continue
 		}
 		out[p] = struct{}{}
@@ -123,12 +166,12 @@ func filterLineage(paths map[string]struct{}, rules ignore.Rules) map[string]str
 // filterPathList drops excluded paths from a plain list, for the lineage that
 // is persisted back after a sync.
 func filterPathList(paths []string, rules ignore.Rules) []string {
-	if rules.Empty() {
+	if rules.Empty() && !anyNeverSynced(paths) {
 		return paths
 	}
 	out := make([]string, 0, len(paths))
 	for _, p := range paths {
-		if rules.Match(p) {
+		if excluded(rules, p) {
 			continue
 		}
 		out = append(out, p)

internal/p2p/syncengine/multiroot.go

@@ -114,24 +114,20 @@ func (e *Engine) syncOneRoot(ctx context.Context, gameID string, game store.Game
 	rules := e.rulesFor(gameID)
 	remote := sr.remote
 	unfilteredLocal, unfilteredRemote := local, remote
-	if !rules.Empty() {
-		local = filterManifest(local, rules)
-		remote = filterManifest(remote, rules)
-	}
+	local = filterManifest(local, rules)
+	remote = filterManifest(remote, rules)
 
 	fileList, dirList, err := e.Store.GetSyncStateForRoot(gameID, peer.ID, sr.root.Name)
 	if err != nil {
 		return err
 	}
 	lineageFiles, lineageDirs := toSet(fileList), toSet(dirList)
-	if !rules.Empty() {
-		// Without this an exclusion added to a location that had already
-		// synced reads as "we both had this file and now I do not", and the
-		// deletion travels to the peer — destroying the copy the rule was
-		// written to protect.
-		lineageFiles = filterLineage(lineageFiles, rules)
-		lineageDirs = filterLineage(lineageDirs, rules)
-	}
+	// Without this an exclusion added to a location that had already
+	// synced reads as "we both had this file and now I do not", and the
+	// deletion travels to the peer — destroying the copy the rule was
+	// written to protect.
+	lineageFiles = filterLineage(lineageFiles, rules)
+	lineageDirs = filterLineage(lineageDirs, rules)
 	decision := Compute(local, remote, lineageFiles, lineageDirs)
 
 	if !decision.HasChanges() {
@@ -172,7 +168,7 @@ func (e *Engine) syncOneRoot(ctx context.Context, gameID string, game store.Game
 	// adds a rule. If a side's UNFILTERED state still hashes to the base then
 	// that side has not changed, and its filtered hash says the same thing in
 	// today's terms. Same translation the main folder does.
-	if !rules.Empty() && base != "" {
+	if base != "" {
 		switch base {
 		case unfilteredLocal.RootHash(delta.PrimaryRoot):
 			base = local.RootHash(delta.PrimaryRoot)