From a1c8bb5aa2b9b1c09d43dee1a2831b7cf9892f71 Mon Sep 17 00:00:00 2001 From: ChakraFusion <112588066+ChakraFusion@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:25:10 +0200 Subject: [PATCH 1/4] Give the database's unused space back to the disk opensave.db had grown to 663 MB on one device, 262 MB of it pages that deleted rows had left: SQLite never shrinks the file on its own, and snapshots - each with a row per file of its save, a quarter of a million for one game - are merged, pruned and replaced all the time. The database is switched to incremental auto-vacuum, which takes one full VACUUM when it is opened (8 seconds for that database, 663 to 380 MB). After that its free pages are handed back a few at a time - when it is opened and hourly - so deleting a snapshot makes the file smaller. Co-Authored-By: Claude Opus 5.5 --- internal/daemon/daemon.go | 15 +++++++++++ internal/store/compact.go | 43 ++++++++++++++++++++++++++++++ internal/store/compact_test.go | 48 ++++++++++++++++++++++++++++++++++ internal/store/db.go | 1 + 4 files changed, 107 insertions(+) create mode 100644 internal/store/compact.go create mode 100644 internal/store/compact_test.go diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index dde8a586..9cbe5b48 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -386,6 +386,21 @@ func (d *Daemon) Start() error { } }) + // The database file gives back the space of what was deleted from it + // (store/compact.go), hourly. + d.P2P.GoSync(func(ctx context.Context) { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + d.Store.Compact() + } + } + }) + // Look for newly installed games: a few minutes after start, then every // hour. See newgames.go. d.P2P.GoSync(func(ctx context.Context) { diff --git a/internal/store/compact.go b/internal/store/compact.go new file mode 100644 index 00000000..069bb895 --- /dev/null +++ b/internal/store/compact.go @@ -0,0 +1,43 @@ +package store + +// Giving the space of deleted rows back to the disk. +// +// SQLite keeps the pages rows were deleted from, for rows to come; the file +// never shrinks on its own. Snapshots are merged, pruned and replaced all the +// time, and each held a row per file of its save — a quarter of a million +// for one game — so the file grew by what every snapshot ever deleted had +// held: 663 MB on one device, 262 MB of it unused. +// +// With auto_vacuum set to INCREMENTAL the free pages can be handed back a +// few at a time (Compact), never holding the database for long. Setting it +// takes one full VACUUM, done once, when the database is opened and nothing +// else is using it: some seconds for a large one. + +// freePagesWorthCompacting is how many free pages are given back at a time, +// and fewer are not worth the bother: 4 MB at the default page size. +const freePagesWorthCompacting = 1024 + +func (s *Store) compactOnOpen() { + var mode int + if err := s.db.Get(&mode, `PRAGMA auto_vacuum`); err != nil { + return + } + if mode != 2 { // 2 = INCREMENTAL + if _, err := s.db.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil { + return + } + _, _ = s.db.Exec(`VACUUM`) + return + } + s.Compact() +} + +// Compact hands the database file's unused pages back to the disk, when +// there are enough of them to matter. +func (s *Store) Compact() { + var free int + if err := s.db.Get(&free, `PRAGMA freelist_count`); err != nil || free < freePagesWorthCompacting { + return + } + _, _ = s.db.Exec(`PRAGMA incremental_vacuum`) +} diff --git a/internal/store/compact_test.go b/internal/store/compact_test.go new file mode 100644 index 00000000..f334211b --- /dev/null +++ b/internal/store/compact_test.go @@ -0,0 +1,48 @@ +package store + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// Deleted rows give their space back to the disk: the file shrinks, rather +// than holding everything it ever held. +func TestCompactGivesSpaceBack(t *testing.T) { + path := filepath.Join(t.TempDir(), "opensave.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + var mode int + _ = s.db.Get(&mode, `PRAGMA auto_vacuum`) + if mode != 2 { + t.Fatalf("auto_vacuum = %d, want incremental (2)", mode) + } + // A table of the test's own, so the test depends on no other. + if _, err := s.db.Exec(`CREATE TABLE padding (k TEXT PRIMARY KEY, v TEXT NOT NULL)`); err != nil { + t.Fatal(err) + } + pad := strings.Repeat("x", 2000) + tx := s.db.MustBegin() + for i := 0; i < 4000; i++ { + tx.MustExec(`INSERT INTO padding (k, v) VALUES (?, ?)`, fmt.Sprint("k", i), pad) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + _, _ = s.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) + full, _ := os.Stat(path) + if _, err := s.db.Exec(`DELETE FROM padding`); err != nil { + t.Fatal(err) + } + s.Compact() + _, _ = s.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) + after, _ := os.Stat(path) + if after.Size() >= full.Size()/2 { + t.Errorf("file %d bytes after deleting everything, %d before: nothing given back", after.Size(), full.Size()) + } + s.Close() +} diff --git a/internal/store/db.go b/internal/store/db.go index bcdbfd2f..a67daca1 100644 --- a/internal/store/db.go +++ b/internal/store/db.go @@ -39,6 +39,7 @@ func Open(path string) (*Store, error) { db.Close() return nil, err } + s.compactOnOpen() // The file holds the device's private key, the Google tokens and the // vault keys. SQLite creates it with the process umask, which is usually // world-readable; the directory around it is private now, but the file From b06defcb00e4cdb1198e93612af8d26215a67240 Mon Sep 17 00:00:00 2001 From: ChakraFusion <112588066+ChakraFusion@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:43:29 +0200 Subject: [PATCH 2/4] Write the database through a WAL; clear deletion records in one go Putting Project Zomboid back after its folder was emptied spent most of an hour in the database, the save untouched meanwhile: every file's deletion record was cleared with a statement of its own, a quarter of a million of them, and with SQLite's default rollback journal every one of them created, flushed and deleted a journal file. - Deletion records are cleared in one transaction per save location, on putting a save back and when files come back in a snapshot (ClearDeletedFiles): 50,000 take a third of a second. - The database runs in WAL mode with synchronous NORMAL: writes are appended to one log instead of each making a journal file of its own. The database stays whole whatever happens; a power cut can cost only the last moments' writes. Co-Authored-By: Claude Opus 5.5 --- internal/p2p/syncengine/hold.go | 2 +- internal/snapshot/snapshot.go | 4 ++- internal/store/compact_test.go | 51 +++++++++++++++++++++++++++++++++ internal/store/db.go | 9 +++++- internal/store/deleted_files.go | 25 ++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/internal/p2p/syncengine/hold.go b/internal/p2p/syncengine/hold.go index 43a63a87..ebcf4ba9 100644 --- a/internal/p2p/syncengine/hold.go +++ b/internal/p2p/syncengine/hold.go @@ -416,8 +416,8 @@ func (e *Engine) PutBack(gameID string) (fetching int, err error) { } for root, files := range hold.Files() { var missing []string + _ = e.Store.ClearDeletedFiles(gameID, root, files) for _, f := range files { - _ = e.Store.ClearDeletedFile(gameID, root, f) if _, ok := x.present[root][f]; !ok { missing = append(missing, f) } diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 694a6ab7..6ae96fdb 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -1243,14 +1243,16 @@ func (m *Manager) recordDeletionsSince(gameID, branch, snapshotID string, captur if err != nil || len(records) == 0 { continue } + var back []string for _, f := range captured { if f.Root != root { continue } if _, remembered := records[f.Path]; remembered { - _ = m.Store.ClearDeletedFile(gameID, root, f.Path) + back = append(back, f.Path) } } + _ = m.Store.ClearDeletedFiles(gameID, root, back) } } diff --git a/internal/store/compact_test.go b/internal/store/compact_test.go index f334211b..043993c3 100644 --- a/internal/store/compact_test.go +++ b/internal/store/compact_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) // Deleted rows give their space back to the disk: the file shrinks, rather @@ -46,3 +47,53 @@ func TestCompactGivesSpaceBack(t *testing.T) { } s.Close() } + +// Clearing a large save's deletion records is one transaction, not one write +// to disk per file. +func TestClearDeletedFilesIsOneTransaction(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "opensave.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err := s.CreateGame(Game{ID: "g", Name: "g", SavePath: t.TempDir(), ActiveBranch: "main"}); err != nil { + t.Fatal(err) + } + var files []DeletedFile + var paths []string + for i := 0; i < 50000; i++ { + p := fmt.Sprintf("map/%d/%d.bin", i/100, i) + files = append(files, DeletedFile{Path: p, Hash: "h"}) + paths = append(paths, p) + } + if err := s.RecordDeletedFiles("g", files); err != nil { + t.Fatal(err) + } + start := time.Now() + if err := s.ClearDeletedFiles("g", "", paths); err != nil { + t.Fatal(err) + } + took := time.Since(start) + left, _ := s.DeletedFiles("g", "") + if len(left) != 0 { + t.Errorf("%d records left", len(left)) + } + if took > 10*time.Second { + t.Errorf("clearing 50,000 records took %v", took) + } + t.Logf("cleared 50,000 deletion records in %v", took) +} + +// The database writes through a write-ahead log, not a rollback journal +// created and deleted for every write. +func TestDatabaseUsesWAL(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "opensave.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var mode string + if err := s.db.Get(&mode, `PRAGMA journal_mode`); err != nil || mode != "wal" { + t.Errorf("journal_mode = %q (%v), want wal", mode, err) + } +} diff --git a/internal/store/db.go b/internal/store/db.go index a67daca1..e6c9a550 100644 --- a/internal/store/db.go +++ b/internal/store/db.go @@ -56,7 +56,14 @@ func openDB(path string) (*sqlx.DB, error) { // _pragma params ensure foreign keys are enforced (SQLite defaults them // off per-connection) and busy_timeout avoids spurious SQLITE_BUSY // errors from the watcher/api/p2p goroutines all touching the DB. - dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)", path) + // + // WAL: a write is appended to one log and the database file is updated + // from it later, rather than every write creating, flushing and deleting a + // rollback journal of its own — which made a quarter of a million small + // writes take most of an hour. synchronous NORMAL is what WAL is meant to + // run with: the database stays whole whatever happens; a power cut can + // only lose the last moments' writes, never corrupt it. + dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)", path) db, err := sqlx.Open("sqlite", dsn) if err != nil { return nil, fmt.Errorf("open sqlite database: %w", err) diff --git a/internal/store/deleted_files.go b/internal/store/deleted_files.go index 1cda3429..d3b11246 100644 --- a/internal/store/deleted_files.go +++ b/internal/store/deleted_files.go @@ -107,6 +107,31 @@ func (s *Store) ClearDeletedFile(gameID, root, path string) error { return nil } +// ClearDeletedFiles forgets the deletions recorded for paths of one save +// location, in one transaction: one at a time, each its own write to disk, +// the quarter of a million of a large save took a quarter of an hour. +func (s *Store) ClearDeletedFiles(gameID, root string, paths []string) error { + if len(paths) == 0 { + return nil + } + tx, err := s.db.Beginx() + if err != nil { + return err + } + defer tx.Rollback() + stmt, err := tx.Preparex(`DELETE FROM deleted_files WHERE game_id = ? AND root = ? AND path = ?`) + if err != nil { + return fmt.Errorf("prepare clear deletion records: %w", err) + } + defer stmt.Close() + for _, p := range paths { + if _, err := stmt.Exec(gameID, root, p); err != nil { + return fmt.Errorf("clear deletion record for %s: %w", p, err) + } + } + return tx.Commit() +} + // ClearDeletedFilesForGame forgets every deletion recorded for a game, for use // when it is untracked — the records describe a folder this device no longer // has an opinion about. From fb7b54fe934d7d50b085e946f8f963b27d465867 Mon Sep 17 00:00:00 2001 From: ChakraFusion <112588066+ChakraFusion@users.noreply.github.com> Date: Sun, 4 Oct 2026 03:10:18 +0200 Subject: [PATCH 3/4] Clear deletion records 500 paths to a statement One statement per path, even inside one transaction, took 10s for 50,000 records under the race detector on CI's runners, and the test's limit failed. 500 paths to a statement: 50,000 records in about 0.3s here. The test's limit is generous now (30s): what it guards against took minutes. Co-Authored-By: Claude Opus 5.5 --- internal/store/compact_test.go | 4 +++- internal/store/deleted_files.go | 28 +++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/internal/store/compact_test.go b/internal/store/compact_test.go index 043993c3..c6537a2a 100644 --- a/internal/store/compact_test.go +++ b/internal/store/compact_test.go @@ -78,7 +78,9 @@ func TestClearDeletedFilesIsOneTransaction(t *testing.T) { if len(left) != 0 { t.Errorf("%d records left", len(left)) } - if took > 10*time.Second { + // Generous: CI runs this under the race detector on shared runners, where + // one statement per file took 10s; what this guards against took minutes. + if took > 30*time.Second { t.Errorf("clearing 50,000 records took %v", took) } t.Logf("cleared 50,000 deletion records in %v", took) diff --git a/internal/store/deleted_files.go b/internal/store/deleted_files.go index d3b11246..0a4a0e0e 100644 --- a/internal/store/deleted_files.go +++ b/internal/store/deleted_files.go @@ -109,7 +109,9 @@ func (s *Store) ClearDeletedFile(gameID, root, path string) error { // ClearDeletedFiles forgets the deletions recorded for paths of one save // location, in one transaction: one at a time, each its own write to disk, -// the quarter of a million of a large save took a quarter of an hour. +// the quarter of a million of a large save took a quarter of an hour. Paths go +// clearDeletedChunk at a time, so that is a few hundred statements, not one +// per file. func (s *Store) ClearDeletedFiles(gameID, root string, paths []string) error { if len(paths) == 0 { return nil @@ -119,19 +121,27 @@ func (s *Store) ClearDeletedFiles(gameID, root string, paths []string) error { return err } defer tx.Rollback() - stmt, err := tx.Preparex(`DELETE FROM deleted_files WHERE game_id = ? AND root = ? AND path = ?`) - if err != nil { - return fmt.Errorf("prepare clear deletion records: %w", err) - } - defer stmt.Close() - for _, p := range paths { - if _, err := stmt.Exec(gameID, root, p); err != nil { - return fmt.Errorf("clear deletion record for %s: %w", p, err) + for len(paths) > 0 { + n := min(len(paths), clearDeletedChunk) + args := make([]any, 0, n+2) + args = append(args, gameID, root) + for _, p := range paths[:n] { + args = append(args, p) } + q := `DELETE FROM deleted_files WHERE game_id = ? AND root = ? AND path IN (?` + + strings.Repeat(",?", n-1) + `)` + if _, err := tx.Exec(q, args...); err != nil { + return fmt.Errorf("clear deletion records: %w", err) + } + paths = paths[n:] } return tx.Commit() } +// clearDeletedChunk is how many paths one statement of ClearDeletedFiles +// names: well under SQLite's limit on bound parameters. +const clearDeletedChunk = 500 + // ClearDeletedFilesForGame forgets every deletion recorded for a game, for use // when it is untracked — the records describe a folder this device no longer // has an opinion about. From c43626f82494e3660a45b6bba8d05c9757163adf Mon Sep 17 00:00:00 2001 From: ChakraFusion <112588066+ChakraFusion@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:00:37 +0200 Subject: [PATCH 4/4] Database: the one-time VACUUM runs after startup, in the background Asked in review: VACUUM inside store.Open made the first start after updating as long as rewriting the whole database, and it needs about the database's size free on its disk. - store.Open no longer rewrites anything. A new database gets incremental auto-vacuum for free: openDB asks for it first in the connection string, while the file is still empty. - A database from an older build is switched over by Store.EnableIncrementalVacuum, which the daemon runs once, two minutes after it has started, then hands free pages back as before (and hourly). It is not attempted without the database's size free next to it; the log says so, and it is tried again at the next start. - The log says when the one-time rewrite ran and how long it took. Test: a database created without auto-vacuum is left as it is by Open, switched over by EnableIncrementalVacuum, and not rewritten a second time. It fails on the previous commit, where Open did the VACUUM. Co-Authored-By: Claude Opus 5.5 --- internal/daemon/compact.go | 3 ++ internal/daemon/daemon.go | 18 +++++++++++- internal/store/compact.go | 54 ++++++++++++++++++++++++++-------- internal/store/compact_test.go | 36 +++++++++++++++++++++++ internal/store/db.go | 12 +++++--- 5 files changed, 106 insertions(+), 17 deletions(-) diff --git a/internal/daemon/compact.go b/internal/daemon/compact.go index d8add997..1f4190d5 100644 --- a/internal/daemon/compact.go +++ b/internal/daemon/compact.go @@ -19,6 +19,9 @@ const ( compactEvery = 3 * time.Hour // compactPace is a pause between snapshots, as for the daily check. compactPace = 50 * time.Millisecond + // dbCompactFirst is when the database file first gives its unused space + // back (see daemon.go), once the start's own work is done. + dbCompactFirst = 2 * time.Minute ) // CompactSnapshots has every snapshot that can share its large files with diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 9cbe5b48..1948c955 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -387,8 +387,24 @@ func (d *Daemon) Start() error { }) // The database file gives back the space of what was deleted from it - // (store/compact.go), hourly. + // (store/compact.go): shortly after start, out of the way of everything a + // start does, and hourly after that. A database from an older build is + // switched over first, once: a rewrite of the whole file, which in + // store.Open would have held up the first start after updating. d.P2P.GoSync(func(ctx context.Context) { + select { + case <-ctx.Done(): + return + case <-time.After(dbCompactFirst): + } + began := time.Now() + if converted, err := d.Store.EnableIncrementalVacuum(); err != nil { + d.Log.Log("warn", fmt.Sprintf("the database could not be set to give unused space back to the disk: %v", err)) + } else if converted { + d.Log.Log("info", fmt.Sprintf("the database now gives unused space back to the disk (a one-time rewrite, %s)", + time.Since(began).Round(100*time.Millisecond))) + } + d.Store.Compact() ticker := time.NewTicker(time.Hour) defer ticker.Stop() for { diff --git a/internal/store/compact.go b/internal/store/compact.go index 069bb895..c5a48cf6 100644 --- a/internal/store/compact.go +++ b/internal/store/compact.go @@ -1,5 +1,13 @@ package store +import ( + "fmt" + "os" + "path/filepath" + + "github.com/opensave/opensave/internal/fsx" +) + // Giving the space of deleted rows back to the disk. // // SQLite keeps the pages rows were deleted from, for rows to come; the file @@ -9,27 +17,49 @@ package store // held: 663 MB on one device, 262 MB of it unused. // // With auto_vacuum set to INCREMENTAL the free pages can be handed back a -// few at a time (Compact), never holding the database for long. Setting it -// takes one full VACUUM, done once, when the database is opened and nothing -// else is using it: some seconds for a large one. +// few at a time (Compact), never holding the database for long. A database +// being created gets the setting for free: it takes effect only on an empty +// file, so openDB asks for it before anything else. One created by an older +// build needs a full VACUUM to take it (EnableIncrementalVacuum), which the +// daemon runs once in the background after it has started, not in Open: on a +// large database that is a rewrite of some seconds, and the first start after +// updating would wait for it. // freePagesWorthCompacting is how many free pages are given back at a time, // and fewer are not worth the bother: 4 MB at the default page size. const freePagesWorthCompacting = 1024 -func (s *Store) compactOnOpen() { +func (s *Store) incrementalVacuum() bool { var mode int - if err := s.db.Get(&mode, `PRAGMA auto_vacuum`); err != nil { - return + return s.db.Get(&mode, `PRAGMA auto_vacuum`) == nil && mode == 2 // 2 = INCREMENTAL +} + +// EnableIncrementalVacuum switches a database created by an older build to +// incremental auto-vacuum, with the one full VACUUM that takes. It reports +// whether it did; on a database that already has it, it does nothing. +// +// The VACUUM rewrites the whole file: it holds the database while it runs and +// needs about the file's size free on the same disk for the copy. Without +// that much free it is not attempted, and the error says so. +func (s *Store) EnableIncrementalVacuum() (bool, error) { + if s.incrementalVacuum() { + return false, nil } - if mode != 2 { // 2 = INCREMENTAL - if _, err := s.db.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil { - return + if s.path != "" { + if info, err := os.Stat(s.path); err == nil { + if free, ok := fsx.FreeBytes(filepath.Dir(s.path)); ok && free < uint64(info.Size()) { + return false, fmt.Errorf("giving unused space back needs about %d MB free next to the database, and %d MB are", + info.Size()>>20, free>>20) + } } - _, _ = s.db.Exec(`VACUUM`) - return } - s.Compact() + if _, err := s.db.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil { + return false, err + } + if _, err := s.db.Exec(`VACUUM`); err != nil { + return false, err + } + return true, nil } // Compact hands the database file's unused pages back to the disk, when diff --git a/internal/store/compact_test.go b/internal/store/compact_test.go index c6537a2a..f334b628 100644 --- a/internal/store/compact_test.go +++ b/internal/store/compact_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" "time" + + "github.com/jmoiron/sqlx" ) // Deleted rows give their space back to the disk: the file shrinks, rather @@ -99,3 +101,37 @@ func TestDatabaseUsesWAL(t *testing.T) { t.Errorf("journal_mode = %q (%v), want wal", mode, err) } } + +// A database created by an older build, without incremental auto-vacuum, is +// not rewritten when it is opened: the start stays fast. It is switched over +// once, by EnableIncrementalVacuum, and after that gives space back as usual. +func TestOlderDatabaseIsSwitchedOverAfterOpenNotDuringIt(t *testing.T) { + path := filepath.Join(t.TempDir(), "opensave.db") + old, err := sqlx.Open("sqlite", "file:"+path) // as an older build left it: no auto-vacuum + if err != nil { + t.Fatal(err) + } + if _, err := old.Exec(`CREATE TABLE older_build (k TEXT)`); err != nil { + t.Fatal(err) + } + old.Close() + + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if s.incrementalVacuum() { + t.Fatal("Open rewrote an older database; that is for after the start") + } + converted, err := s.EnableIncrementalVacuum() + if err != nil || !converted { + t.Fatalf("EnableIncrementalVacuum = %v, %v; want it switched over", converted, err) + } + if !s.incrementalVacuum() { + t.Fatal("auto_vacuum is still not incremental") + } + if converted, err := s.EnableIncrementalVacuum(); err != nil || converted { + t.Errorf("a second EnableIncrementalVacuum = %v, %v; want nothing to do", converted, err) + } +} diff --git a/internal/store/db.go b/internal/store/db.go index e6c9a550..71407db6 100644 --- a/internal/store/db.go +++ b/internal/store/db.go @@ -23,7 +23,8 @@ var migrationsFS embed.FS // methods (see settings.go, games.go, branches.go, snapshots.go, peers.go, // cloudtokens.go). type Store struct { - db *sqlx.DB + db *sqlx.DB + path string // the database file, for EnableIncrementalVacuum's space check } // Open creates (if needed) and opens the SQLite database at path, applying @@ -34,12 +35,11 @@ func Open(path string) (*Store, error) { return nil, err } - s := &Store{db: db} + s := &Store{db: db, path: path} if err := s.migrate(); err != nil { db.Close() return nil, err } - s.compactOnOpen() // The file holds the device's private key, the Google tokens and the // vault keys. SQLite creates it with the process umask, which is usually // world-readable; the directory around it is private now, but the file @@ -63,7 +63,11 @@ func openDB(path string) (*sqlx.DB, error) { // writes take most of an hour. synchronous NORMAL is what WAL is meant to // run with: the database stays whole whatever happens; a power cut can // only lose the last moments' writes, never corrupt it. - dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)", path) + // + // auto_vacuum INCREMENTAL comes first: it takes effect only while the + // file is still empty, before WAL writes its header (see compact.go). On + // an existing database it changes nothing. + dsn := fmt.Sprintf("file:%s?_pragma=auto_vacuum(2)&_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)", path) db, err := sqlx.Open("sqlite", dsn) if err != nil { return nil, fmt.Errorf("open sqlite database: %w", err)