From a1a0ab3b751b9c9713a8cd46a651164836085820 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:53:42 +0200 Subject: [PATCH 01/19] perf(cluster): bound SSH handshakes and poll nodes every second QEMU's port forward accepts connections while the guest still boots and retries towards the guest only every 12s, so a probe without a handshake deadline could stall until then. --- internal/cluster/golden.go | 2 +- internal/cluster/node.go | 2 +- internal/cluster/pvecluster.go | 2 +- internal/cluster/setup.go | 2 +- internal/remote/ssh.go | 19 ++++++++++++ internal/remote/ssh_test.go | 57 ++++++++++++++++++++++++++++++++++ 6 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 internal/remote/ssh_test.go diff --git a/internal/cluster/golden.go b/internal/cluster/golden.go index f809c4d..f3bcb3c 100644 --- a/internal/cluster/golden.go +++ b/internal/cluster/golden.go @@ -99,7 +99,7 @@ func (c *Cluster) personalize(ctx context.Context, name string, keys []string, m } done() // Rebooting: wait for the new boot with the cluster key and the new host key. - err = retry.Do(ctx, 5*time.Minute, 5*time.Second, func() error { + err = retry.Do(ctx, 5*time.Minute, time.Second, func() error { ns, err := c.dial(ctx, name, c.signer, "") if err != nil { return err diff --git a/internal/cluster/node.go b/internal/cluster/node.go index 3855327..14bfcd9 100644 --- a/internal/cluster/node.go +++ b/internal/cluster/node.go @@ -92,7 +92,7 @@ func (c *Cluster) ssh(ctx context.Context, name string) (*remote.SSH, error) { // waitSSH waits until a node accepts SSH logins; a changed host key ends the wait. func (c *Cluster) waitSSH(ctx context.Context, name string, timeout time.Duration) (*remote.SSH, error) { var s *remote.SSH - err := retry.Do(ctx, timeout, 2*time.Second, func() error { + err := retry.Do(ctx, timeout, time.Second, func() error { var err error s, err = c.ssh(ctx, name) if err != nil && strings.Contains(err.Error(), "host key") { diff --git a/internal/cluster/pvecluster.go b/internal/cluster/pvecluster.go index 53b7bbd..6c767b8 100644 --- a/internal/cluster/pvecluster.go +++ b/internal/cluster/pvecluster.go @@ -122,7 +122,7 @@ func (c *Cluster) join(ctx context.Context, api *pve.Client, first, n config.Nod // task is given, its failure ends the wait early. func waitMember(ctx context.Context, api *pve.Client, node string, joiner *pve.Client, upid string, timeout time.Duration) error { var joinErr error - err := retry.Do(ctx, timeout, 2*time.Second, func() error { + err := retry.Do(ctx, timeout, time.Second, func() error { cs, members, err := api.ClusterStatus(ctx) if err == nil && cs != nil && cs.Quorate == 1 && members[node].Online == 1 { return nil diff --git a/internal/cluster/setup.go b/internal/cluster/setup.go index 25735ac..0cde812 100644 --- a/internal/cluster/setup.go +++ b/internal/cluster/setup.go @@ -60,7 +60,7 @@ func (c *Cluster) rebootForKernel(ctx context.Context, name string, s *remote.SS return err } var running string - err = retry.Do(ctx, 5*time.Minute, 5*time.Second, func() error { + err = retry.Do(ctx, 5*time.Minute, time.Second, func() error { ns, err := c.ssh(ctx, name) if err != nil { return err diff --git a/internal/remote/ssh.go b/internal/remote/ssh.go index 594032e..66a320b 100644 --- a/internal/remote/ssh.go +++ b/internal/remote/ssh.go @@ -14,6 +14,9 @@ import ( "golang.org/x/crypto/ssh" ) +// HandshakeTimeout bounds the SSH handshake of one Dial. +const HandshakeTimeout = 3 * time.Second + // SSH runs commands as root on a node through its forwarded port on 127.0.0.1. type SSH struct { Port int @@ -43,11 +46,27 @@ func (s *SSH) Dial(ctx context.Context) (string, error) { if err != nil { return "", err } + // QEMU's port forward accepts at once even while the guest is still booting, + // then retries its own connection to the guest only every 12s. Without a + // deadline the handshake waits for that retry instead of failing fast so + // the caller can dial again. + deadline := time.Now().Add(HandshakeTimeout) + if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { + deadline = ctxDeadline + } + if err := conn.SetDeadline(deadline); err != nil { + conn.Close() + return "", err + } c, chans, reqs, err := ssh.NewClientConn(conn, addr, cfg) if err != nil { conn.Close() return "", err } + if err := conn.SetDeadline(time.Time{}); err != nil { + c.Close() + return "", err + } s.client = ssh.NewClient(c, chans, reqs) return seen, nil } diff --git a/internal/remote/ssh_test.go b/internal/remote/ssh_test.go new file mode 100644 index 0000000..7188b63 --- /dev/null +++ b/internal/remote/ssh_test.go @@ -0,0 +1,57 @@ +package remote + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "net" + "testing" + "time" + + "golang.org/x/crypto/ssh" +) + +// A port forward that accepts but never answers (a booting guest behind QEMU +// user networking) must not block Dial past its deadline. +func TestSSH_DialStalledHandshake(t *testing.T) { + t.Parallel() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + go func() { + // Hold connections open without sending a banner until the listener closes. + var conns []net.Conn + defer func() { + for _, c := range conns { + c.Close() + } + }() + for { + c, err := l.Accept() + if err != nil { + return + } + conns = append(conns, c) + } + }() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatal(err) + } + s := &SSH{Port: l.Addr().(*net.TCPAddr).Port, Signer: signer} + ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond) + defer cancel() + t0 := time.Now() + if _, err := s.Dial(ctx); err == nil { + t.Fatal("Dial succeeded against a server that never answers") + } + if d := time.Since(t0); d > 2*time.Second { + t.Fatalf("Dial took %s; the handshake must honour the context deadline", d) + } +} From ee9447a72402de6f89e9a95e35ea952e59b13b1d Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:54:23 +0200 Subject: [PATCH 02/19] perf(nodes): skip the GRUB menu wait and fsync while provisioning Nodes boot without the 5s GRUB menu. During create, rpool has sync=disabled and dpkg skips its syncs; the last step restores both so ready clusters keep real fsync semantics for fault injection. rpool gets autotrim. --- internal/cluster/create.go | 1 + internal/cluster/nodes.go | 2 +- internal/cluster/setup.go | 35 ++++++++++++++++++++--- internal/nodesetup/nodesetup.go | 35 +++++++++++++++++++++-- internal/nodesetup/nodesetup_test.go | 42 ++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 7 deletions(-) diff --git a/internal/cluster/create.go b/internal/cluster/create.go index b1e6e12..504fd3a 100644 --- a/internal/cluster/create.go +++ b/internal/cluster/create.go @@ -79,6 +79,7 @@ func (c *Cluster) create(ctx context.Context) error { c.setupStorage, c.ensureToken, c.exportCA, + c.endProvisioning, } for _, step := range steps { if err := step(ctx); err != nil { diff --git a/internal/cluster/nodes.go b/internal/cluster/nodes.go index 0ad1c8c..c2fa16f 100644 --- a/internal/cluster/nodes.go +++ b/internal/cluster/nodes.go @@ -149,7 +149,7 @@ func (c *Cluster) RemoveNode(ctx context.Context, name string, force bool) error if err := c.net.Reload(c.Cfg); err != nil { return err } - if err := c.postInstall(ctx); err != nil { // rewrites /etc/hosts + if err := c.configureNodes(ctx, false); err != nil { // rewrites /etc/hosts return err } if err := c.waitQuorum(ctx, 3*time.Minute); err != nil { diff --git a/internal/cluster/setup.go b/internal/cluster/setup.go index 0cde812..3bdf0ff 100644 --- a/internal/cluster/setup.go +++ b/internal/cluster/setup.go @@ -14,9 +14,13 @@ import ( "github.com/virtbase/proxbase/internal/retry" ) -// postInstall configures host names, repositories and bridges on every node and, -// if configured, upgrades them (rebooting into a new kernel). -func (c *Cluster) postInstall(ctx context.Context) error { +// postInstall puts every node into provisioning mode (fast, unsafe writes until +// endProvisioning) and configures it; see configureNodes. +func (c *Cluster) postInstall(ctx context.Context) error { return c.configureNodes(ctx, true) } + +// configureNodes configures host names, repositories and bridges on every node +// and, if configured, upgrades them (rebooting into a new kernel). +func (c *Cluster) configureNodes(ctx context.Context, provisioning bool) error { g, gctx := errgroup.WithContext(ctx) for _, n := range c.Cfg.NodeList() { g.Go(func() error { @@ -25,7 +29,11 @@ func (c *Cluster) postInstall(ctx context.Context) error { return err } defer s.Close() - out, err := s.Run(nodesetup.Script(c.Cfg, n)) + script := nodesetup.Script(c.Cfg, n) + if provisioning { + script = nodesetup.ProvisionStart + script + } + out, err := s.Run(script) if err != nil { return fmt.Errorf("%s: post-install: %w", n.Name, err) } @@ -47,6 +55,25 @@ func (c *Cluster) postInstall(ctx context.Context) error { return g.Wait() } +// endProvisioning restores safe writes on every node (see nodesetup.ProvisionEnd). +func (c *Cluster) endProvisioning(ctx context.Context) error { + g, gctx := errgroup.WithContext(ctx) + for _, n := range c.Cfg.NodeList() { + g.Go(func() error { + s, err := c.waitSSH(gctx, n.Name, time.Minute) + if err != nil { + return err + } + defer s.Close() + if _, err := s.Run(nodesetup.ProvisionEnd); err != nil { + return fmt.Errorf("%s: end provisioning: %w", n.Name, err) + } + return nil + }) + } + return g.Wait() +} + // rebootForKernel reboots a node whose newest installed kernel is not running // and waits until it is back (new boot ID, SSH up). func (c *Cluster) rebootForKernel(ctx context.Context, name string, s *remote.SSH) error { diff --git a/internal/nodesetup/nodesetup.go b/internal/nodesetup/nodesetup.go index 5647b2d..3f97156 100644 --- a/internal/nodesetup/nodesetup.go +++ b/internal/nodesetup/nodesetup.go @@ -63,7 +63,7 @@ block /etc/hosts "$(cat <<'EOF' # Login prompt on the serial console (proxbase console). systemctl enable --now serial-getty@ttyS0.service >/dev/null 2>&1 - +%s # No-subscription repository instead of enterprise. codename=$(. /etc/os-release && echo "$VERSION_CODENAME") for f in /etc/apt/sources.list.d/pve-enterprise.sources /etc/apt/sources.list.d/ceph.sources; do @@ -89,9 +89,40 @@ else ifreload -a echo changed fi -`, hosts.String(), nics.String(), ifaces.String()) +`, hosts.String(), NoGrubTimeout, nics.String(), ifaces.String()) } +// NoGrubTimeout makes the node boot without waiting in the GRUB menu (Debian +// waits 5 seconds), which shortens every reboot and start. +const NoGrubTimeout = ` +if grep -q '^GRUB_TIMEOUT=[1-9]' /etc/default/grub; then + sed -i 's/^GRUB_TIMEOUT=.*/GRUB_TIMEOUT=0/' /etc/default/grub + if proxmox-boot-tool status >/dev/null 2>&1; then + proxmox-boot-tool refresh >/dev/null 2>&1 + else + update-grub >/dev/null 2>&1 + fi +fi +` + +// ProvisionStart speeds up writes while proxbase provisions a node: ZFS treats +// fsync as a no-op and dpkg skips its per-file syncs. ProvisionEnd restores +// both, so a ready cluster has real fsync semantics for fault injection. It also +// turns on autotrim so freed space shrinks the disk images. +const ProvisionStart = ` +if zpool list rpool >/dev/null 2>&1; then + zfs set sync=disabled rpool + zpool set autotrim=on rpool +fi +echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io +` + +// ProvisionEnd undoes ProvisionStart's write shortcuts. +const ProvisionEnd = ` +if zpool list rpool >/dev/null 2>&1; then zfs set sync=standard rpool; fi +rm -f /etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io +` + // AptWait waits for apt runs started by PVE itself (pveupdate right after boot) // and sets $apt to an apt-get that waits for locks and retries downloads. const AptWait = ` diff --git a/internal/nodesetup/nodesetup_test.go b/internal/nodesetup/nodesetup_test.go index e8ee8ae..e5adf1f 100644 --- a/internal/nodesetup/nodesetup_test.go +++ b/internal/nodesetup/nodesetup_test.go @@ -1,6 +1,7 @@ package nodesetup import ( + "os/exec" "strings" "testing" @@ -28,9 +29,50 @@ networks: "auto vmbr3\niface vmbr3 inet manual\n\tbridge-ports $nic2\n\tbridge-stp off\n\tbridge-fd 0\n\tbridge-vlan-aware yes\n\tbridge-vids 100 200-299\n", "Components: pve-no-subscription", "serial-getty@ttyS0", + "GRUB_TIMEOUT=0", } { if !strings.Contains(s, want) { t.Errorf("script lacks %q", want) } } } + +// Provisioning shortcuts must be undone completely. +func TestProvision(t *testing.T) { + for _, pair := range [][2]string{ + {"zfs set sync=disabled rpool", "zfs set sync=standard rpool"}, + {"/etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io", "rm -f /etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io"}, + } { + if !strings.Contains(ProvisionStart, pair[0]) { + t.Errorf("ProvisionStart lacks %q", pair[0]) + } + if !strings.Contains(ProvisionEnd, pair[1]) { + t.Errorf("ProvisionEnd lacks %q", pair[1]) + } + } +} + +// Every script must at least parse. +func TestScriptsSyntax(t *testing.T) { + cfg, err := config.Decode(strings.NewReader("nodes: {count: 2}\n")) + if err != nil { + t.Fatal(err) + } + cfg.SetDefaults() + for name, s := range map[string]string{ + "Script": Script(cfg, cfg.NodeList()[0]), + "NoGrubTimeout": NoGrubTimeout, + "ProvisionStart": ProvisionStart, + "ProvisionEnd": ProvisionEnd, + "Upgrade": Upgrade, + "PendingKernel": PendingKernel, + } { + t.Run(name, func(t *testing.T) { + cmd := exec.CommandContext(t.Context(), "bash", "-n") + cmd.Stdin = strings.NewReader(s) + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("bash -n: %v\n%s", err, out) + } + }) + } +} From cfc7e9580065e0ee9c42d17aa66f0f54d84c22c4 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:54:35 +0200 Subject: [PATCH 03/19] perf(zfs): create pools on all nodes at once, with autotrim --- internal/storage/zfs/zfs.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/internal/storage/zfs/zfs.go b/internal/storage/zfs/zfs.go index 7e151a5..ebc066b 100644 --- a/internal/storage/zfs/zfs.go +++ b/internal/storage/zfs/zfs.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "golang.org/x/sync/errgroup" + "github.com/virtbase/proxbase/internal/config" "github.com/virtbase/proxbase/internal/storage" ) @@ -32,16 +34,18 @@ func New(cfg *config.Cluster) *Backend { return &Backend{Pools: cfg.Storage.ZFS} } -// Setup creates missing pools on every node and the cluster storage entries, -// keeping their node lists current. +// Setup creates missing pools on every node (in parallel) and the cluster +// storage entries, keeping their node lists current. func (b *Backend) Setup(ctx context.Context, h storage.Host) error { nodes := h.Nodes() - var names []string + names := make([]string, 0, len(nodes)) + g, gctx := errgroup.WithContext(ctx) for _, n := range nodes { names = append(names, n.Name) - if err := b.createPools(ctx, h, n.Name); err != nil { - return err - } + g.Go(func() error { return b.createPools(gctx, h, n.Name) }) + } + if err := g.Wait(); err != nil { + return err } return b.registerStorage(ctx, h, nodes[0].Name, names, true) } @@ -75,6 +79,23 @@ func (b *Backend) createPools(ctx context.Context, h storage.Host, node string) return fmt.Errorf("%s: create zfs pool %s: %w", node, z.Name, err) } } + return b.enableAutotrim(ctx, h, node) +} + +// enableAutotrim lets the pools return freed blocks, so the disk images shrink. +func (b *Backend) enableAutotrim(ctx context.Context, h storage.Host, node string) error { + s, err := h.SSH(ctx, node) + if err != nil { + return err + } + defer s.Close() + var script strings.Builder + for _, z := range b.Pools { + fmt.Fprintf(&script, "zpool set autotrim=on %s\n", z.Name) + } + if _, err := s.Run(script.String()); err != nil { + return fmt.Errorf("%s: enable autotrim: %w", node, err) + } return nil } From 95a4a6e84c95cb9214eadc73293117cee8d90307 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:54:39 +0200 Subject: [PATCH 04/19] perf(ceph): create managers and OSDs in parallel, cap daemon memory API logins, managers and OSDs run on all nodes at once; monitors and MDS daemons stay sequential. OSD memory target and MDS cache are capped for small nodes unless already set. --- internal/storage/ceph/ceph.go | 82 +++++++++++++++++++++++++----- internal/storage/ceph/ceph_test.go | 56 ++++++++++++++++++++ 2 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 internal/storage/ceph/ceph_test.go diff --git a/internal/storage/ceph/ceph.go b/internal/storage/ceph/ceph.go index f87f157..773cbe9 100644 --- a/internal/storage/ceph/ceph.go +++ b/internal/storage/ceph/ceph.go @@ -8,6 +8,7 @@ import ( "fmt" "net/url" "strconv" + "strings" "time" "golang.org/x/sync/errgroup" @@ -57,13 +58,9 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error { if err := b.install(ctx, h); err != nil { return err } - apis := map[string]*pve.Client{} - for _, n := range nodes { - api, err := h.API(ctx, n.Name) - if err != nil { - return err - } - apis[n.Name] = api + apis, err := clients(ctx, h) + if err != nil { + return err } first := nodes[0].Name api := apis[first] @@ -71,7 +68,7 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error { return err } - // Monitors (up to three), a manager on every node; one at a time. + // Monitors (up to three) one at a time: each joins the quorum before the next. var mons []named if err := retry.Do(ctx, 2*time.Minute, 2*time.Second, func() error { return api.Get(ctx, "/nodes/"+first+"/ceph/mon", &mons) }); err != nil { return fmt.Errorf("list ceph mon: %w", err) @@ -83,14 +80,20 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error { } mons = append(mons, named{Name: n.Name}) } - if err := daemon(ctx, h, apis[n.Name], api, n.Name, "mgr"); err != nil { - return err - } + } + if err := setMemoryTargets(ctx, h, first); err != nil { + return err } + // Managers and OSDs on all nodes at once; neither edits shared configuration. g, gctx := errgroup.WithContext(ctx) for _, n := range nodes { - g.Go(func() error { return createOSDs(gctx, h, apis[n.Name], n.Name, b.cfg.OSDDisks) }) + g.Go(func() error { + if err := daemon(gctx, h, apis[n.Name], api, n.Name, "mgr"); err != nil { + return err + } + return createOSDs(gctx, h, apis[n.Name], n.Name, b.cfg.OSDDisks) + }) } if err := g.Wait(); err != nil { return err @@ -99,6 +102,7 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error { return err } if *b.cfg.CephFS { + // One at a time: creating an MDS rewrites ceph.conf without a lock. for _, n := range nodes { if err := daemon(ctx, h, apis[n.Name], api, n.Name, "mds"); err != nil { return err @@ -111,6 +115,60 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error { return b.Wait(ctx, h) } +// clients logs in to every node's API in parallel. +func clients(ctx context.Context, h storage.Host) (map[string]*pve.Client, error) { + nodes := h.Nodes() + list := make([]*pve.Client, len(nodes)) + g, gctx := errgroup.WithContext(ctx) + for i, n := range nodes { + g.Go(func() error { + api, err := h.API(gctx, n.Name) + list[i] = api + return err + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + apis := make(map[string]*pve.Client, len(nodes)) + for i, n := range nodes { + apis[n.Name] = list[i] + } + return apis, nil +} + +// Lab memory targets. Ceph's defaults (4 GiB per OSD, 4 GiB MDS cache) are sized +// for dedicated servers; the OSD and MDS grow their caches toward them, which +// would push a node with a few GiB of RAM into the OOM killer. 2 GiB is about +// the lowest OSD target Ceph developers recommend, 1 GiB the minimum MDS cache. +var memoryTargets = []struct{ who, option, value string }{ + {who: "osd", option: "osd_memory_target", value: "2147483648"}, + {who: "mds", option: "mds_cache_memory_limit", value: "1073741824"}, +} + +// setMemoryTargets applies memoryTargets unless the option was set before, so +// values changed by the user survive a later setup (node add). +func setMemoryTargets(ctx context.Context, h storage.Host, node string) error { + s, err := h.SSH(ctx, node) + if err != nil { + return err + } + defer s.Close() + if _, err := s.Run(memoryTargetScript()); err != nil { + return fmt.Errorf("ceph: set memory targets: %w", err) + } + return nil +} + +func memoryTargetScript() string { + var script strings.Builder + script.WriteString("dump=$(ceph config dump --format json)\n") + for _, t := range memoryTargets { + fmt.Fprintf(&script, "echo \"$dump\" | grep -q '\"name\": *\"%[2]s\"' || ceph config set %[1]s %[2]s %[3]s\n", t.who, t.option, t.value) + } + return script.String() +} + // init writes ceph.conf with the public and cluster networks unless it exists. func (b *Backend) init(ctx context.Context, h storage.Host, api *pve.Client, first string) error { s, err := h.SSH(ctx, first) diff --git a/internal/storage/ceph/ceph_test.go b/internal/storage/ceph/ceph_test.go new file mode 100644 index 0000000..bfb5138 --- /dev/null +++ b/internal/storage/ceph/ceph_test.go @@ -0,0 +1,56 @@ +package ceph + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The script sets each memory target once and keeps values the user set. +func TestMemoryTargetScript(t *testing.T) { + t.Parallel() + tests := []struct { + name string + dump string + want string + }{ + { + name: "fresh cluster", + dump: `[]`, + want: "config set osd osd_memory_target 2147483648\nconfig set mds mds_cache_memory_limit 1073741824\n", + }, + { + name: "osd target set by the user", + dump: `[{"section": "osd", "name": "osd_memory_target", "value": "8589934592"}]`, + want: "config set mds mds_cache_memory_limit 1073741824\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + log := filepath.Join(dir, "calls") + fake := "#!/bin/sh\n" + + "if [ \"$1 $2\" = 'config dump' ]; then echo '" + tt.dump + "'; exit 0; fi\n" + + "echo \"$*\" >> " + log + "\n" + if err := os.WriteFile(filepath.Join(dir, "ceph"), []byte(fake), 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(t.Context(), "bash", "-euo", "pipefail", "-s") + cmd.Stdin = strings.NewReader(memoryTargetScript()) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("script failed: %v\n%s", err, out) + } + got, err := os.ReadFile(log) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + if string(got) != tt.want { + t.Errorf("ceph calls:\n%s\nwant:\n%s", got, tt.want) + } + }) + } +} From b8aa11d75b85e78348b57195d3333f666bf9404b Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:54:54 +0200 Subject: [PATCH 05/19] perf(qemu): return freed guest memory and disk space to the host A virtio balloon with free page reporting at a fixed PCI slot, and detect-zeroes=unmap on the disks. --- internal/qemu/qemu.go | 11 +++++++++-- internal/qemu/testdata/install.args | 6 ++++-- internal/qemu/testdata/run.args | 6 ++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/internal/qemu/qemu.go b/internal/qemu/qemu.go index 90bf5a4..3e08961 100644 --- a/internal/qemu/qemu.go +++ b/internal/qemu/qemu.go @@ -21,6 +21,7 @@ const ( slotNAT = 0x04 slotNICBase = 0x05 // up to 8 segment NICs: 0x05..0x0c slotRNG = 0x0e + slotBalloon = 0x0f slotData = 0x10 // up to 8 data disks: 0x10..0x17 ) @@ -41,6 +42,10 @@ func qemuLog(s vm.Spec) string { return filepath.Join(s.LogDir, s.Name+"-qemu // ConsoleSocket is the serial console of a running node (proxbase console). func ConsoleSocket(s vm.Spec) string { return console(s) } +// driveOpts: guest flushes are ignored (nodes are disposable), and discards and +// zero writes free space in the image. +const driveOpts = "cache=unsafe,discard=unmap,detect-zeroes=unmap" + func dev(format string, a ...any) []string { return []string{"-device", fmt.Sprintf(format, a...)} } // args returns the command line; media != nil boots the installer. @@ -59,7 +64,7 @@ func args(s vm.Spec, media *vm.Media) []string { "-display", "none", } a = append(a, dev("VGA,bus=pcie.0,addr=%#x", slotVGA)...) - a = append(a, "-drive", "if=none,id=root,file="+s.RootDisk.Path+",format=qcow2,cache=unsafe,discard=unmap") + a = append(a, "-drive", "if=none,id=root,file="+s.RootDisk.Path+",format=qcow2,"+driveOpts) a = append(a, dev("virtio-blk-pci,drive=root,bus=pcie.0,addr=%#x,bootindex=1", slotRoot)...) a = append(a, dev("qemu-xhci,id=usb,bus=pcie.0,addr=%#x", slotUSB)...) fwd := fmt.Sprintf("hostfwd=tcp:%[1]s:%[2]d-10.0.2.15:22,hostfwd=tcp:%[1]s:%[3]d-10.0.2.15:8006", s.Bind, s.SSHPort, s.UIPort) @@ -72,8 +77,10 @@ func args(s vm.Spec, media *vm.Media) []string { } a = append(a, "-object", "rng-random,id=rng,filename=/dev/urandom") a = append(a, dev("virtio-rng-pci,rng=rng,bus=pcie.0,addr=%#x", slotRNG)...) + // Free page reporting hands memory the guest freed back to the host. + a = append(a, dev("virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=%#x", slotBalloon)...) for i, d := range s.DataDisks { - a = append(a, "-drive", fmt.Sprintf("if=none,id=data%d,file=%s,format=qcow2,cache=unsafe,discard=unmap", i, d.Path)) + a = append(a, "-drive", fmt.Sprintf("if=none,id=data%d,file=%s,format=qcow2,%s", i, d.Path, driveOpts)) a = append(a, dev("virtio-blk-pci,drive=data%d,bus=pcie.0,addr=%#x", i, slotData+i)...) } a = append(a, "-qmp", "unix:"+qmpSocket(s)+",server=on,wait=off", "-pidfile", pidfile(s)) diff --git a/internal/qemu/testdata/install.args b/internal/qemu/testdata/install.args index aa5005a..206f941 100644 --- a/internal/qemu/testdata/install.args +++ b/internal/qemu/testdata/install.args @@ -15,7 +15,7 @@ none -device VGA,bus=pcie.0,addr=0x1 -drive -if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap +if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap -device virtio-blk-pci,drive=root,bus=pcie.0,addr=0x2,bootindex=1 -device @@ -32,8 +32,10 @@ virtio-net-pci,netdev=net0,mac=52:54:00:50:01:01,bus=pcie.0,addr=0x5 rng-random,id=rng,filename=/dev/urandom -device virtio-rng-pci,rng=rng,bus=pcie.0,addr=0xe +-device +virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=0xf -drive -if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap +if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap -device virtio-blk-pci,drive=data0,bus=pcie.0,addr=0x10 -qmp diff --git a/internal/qemu/testdata/run.args b/internal/qemu/testdata/run.args index 2bfd1e7..565c984 100644 --- a/internal/qemu/testdata/run.args +++ b/internal/qemu/testdata/run.args @@ -15,7 +15,7 @@ none -device VGA,bus=pcie.0,addr=0x1 -drive -if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap +if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap -device virtio-blk-pci,drive=root,bus=pcie.0,addr=0x2,bootindex=1 -device @@ -32,8 +32,10 @@ virtio-net-pci,netdev=net0,mac=52:54:00:50:01:01,bus=pcie.0,addr=0x5 rng-random,id=rng,filename=/dev/urandom -device virtio-rng-pci,rng=rng,bus=pcie.0,addr=0xe +-device +virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=0xf -drive -if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap +if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap -device virtio-blk-pci,drive=data0,bus=pcie.0,addr=0x10 -qmp From 163206d750c5a9e53215fe12f038bb9ec750ed28 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:55:06 +0200 Subject: [PATCH 06/19] feat(doctor): report whether KSM is on --- internal/cli/doctor.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index a520469..40fea92 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -28,7 +28,7 @@ func doctorCmd() *cobra.Command { Short: "Check that this host can run proxbase clusters", Args: cobra.NoArgs, RunE: func(*cobra.Command, []string) error { - checks := []check{checkKVM(), checkNested(), checkQEMU(), checkQEMUImg(), checkMemory(), checkDisk(), checkPorts(portBase), checkSocketPath()} + checks := []check{checkKVM(), checkNested(), checkQEMU(), checkQEMUImg(), checkMemory(), checkKSM(), checkDisk(), checkPorts(portBase), checkSocketPath()} failed := false for _, c := range checks { fmt.Printf("%-5s %-12s %s\n", c.status, c.name, c.detail) @@ -119,6 +119,20 @@ func checkMemory() check { return warn("memory", "MemAvailable not found") } +// checkKSM reports whether the kernel merges identical memory pages. Nodes run +// the same Proxmox VE, so with KSM they share much of their memory; QEMU already +// marks guest memory as mergeable, but only root can turn KSM on. +func checkKSM() check { + b, err := os.ReadFile("/sys/kernel/mm/ksm/run") + if err != nil { + return warn("ksm", "not available in this kernel") + } + if strings.TrimSpace(string(b)) != "1" { + return warn("ksm", "off; nodes share identical memory with it: echo 1 | sudo tee /sys/kernel/mm/ksm/run") + } + return ok("ksm", "on: identical memory pages of the nodes are shared") +} + func checkDisk() check { dir := state.DataHome() for { From d0727cd32155b26950d85baabeb0539e26f589b2 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:55:10 +0200 Subject: [PATCH 07/19] perf(image): skip the checksum download for a cached exact ISO version --- internal/image/image.go | 42 ++++++++++++++++------ internal/image/image_test.go | 68 +++++++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 12 deletions(-) diff --git a/internal/image/image.go b/internal/image/image.go index a25015e..1f15bde 100644 --- a/internal/image/image.go +++ b/internal/image/image.go @@ -38,18 +38,23 @@ func Ensure(ctx context.Context, mirror, version, cacheDir string, logf Logf) (* if err := os.MkdirAll(dir, 0o755); err != nil { return nil, err } - mirror = strings.TrimRight(mirror, "/") - sums, err := fetchSums(ctx, mirror, dir) - if err != nil { - return nil, err - } - file, sum, err := resolve(sums, version) - if err != nil { - return nil, err - } + file := "proxmox-ve_" + version + ".iso" iso := filepath.Join(dir, file) - if err := ensureISO(ctx, mirror+"/"+file, iso, sum, logf); err != nil { - return nil, err + // An exact version that was downloaded and verified before needs no network. + if !isoRe.MatchString(file) || !verified(iso) { + mirror = strings.TrimRight(mirror, "/") + sums, err := fetchSums(ctx, mirror, dir) + if err != nil { + return nil, err + } + var sum string + if file, sum, err = resolve(sums, version); err != nil { + return nil, err + } + iso = filepath.Join(dir, file) + if err := ensureISO(ctx, mirror+"/"+file, iso, sum, logf); err != nil { + return nil, err + } } img := &Image{Version: strings.TrimSuffix(strings.TrimPrefix(file, "proxmox-ve_"), ".iso"), ISO: iso} if err := extract(img, strings.TrimSuffix(iso, ".iso")); err != nil { @@ -126,6 +131,21 @@ func resolve(sums map[string]string, version string) (file, sum string, err erro return file, sum, nil } +// verified reports whether path has a stamp from an earlier verification that +// still matches its size. +func verified(path string) bool { + fi, err := os.Stat(path) + if err != nil { + return false + } + b, err := os.ReadFile(path + ".sha256") + if err != nil { + return false + } + f := strings.Fields(string(b)) + return len(f) == 2 && len(f[0]) == 64 && f[1] == strconv.FormatInt(fi.Size(), 10) +} + func ensureISO(ctx context.Context, url, path, sum string, logf Logf) error { stamp := path + ".sha256" if fi, err := os.Stat(path); err == nil { diff --git a/internal/image/image_test.go b/internal/image/image_test.go index 801d8b7..dbdc922 100644 --- a/internal/image/image_test.go +++ b/internal/image/image_test.go @@ -1,6 +1,11 @@ package image -import "testing" +import ( + "os" + "path/filepath" + "strings" + "testing" +) const grub = `menuentry 'Install Proxmox VE (Automated)' --class debian { echo 'Loading Proxmox VE Automatic Installer ...' @@ -34,3 +39,64 @@ dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd proxmox-backup t.Error("expected error for missing version") } } + +// An exact version that is cached and verified must not touch the mirror. +func TestEnsure_CachedExactVersionOffline(t *testing.T) { + t.Parallel() + cache := t.TempDir() + dir := filepath.Join(cache, "iso") + boot := filepath.Join(dir, "proxmox-ve_9.2-1") + if err := os.MkdirAll(boot, 0o755); err != nil { + t.Fatal(err) + } + iso := filepath.Join(dir, "proxmox-ve_9.2-1.iso") + files := map[string]string{ + iso: "iso", + iso + ".sha256": strings.Repeat("a", 64) + " 3\n", + filepath.Join(boot, "linux26"): "k", + filepath.Join(boot, "initrd.img"): "i", + filepath.Join(boot, "grub.cfg"): grub, + } + for p, content := range files { + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + img, err := Ensure(t.Context(), "http://127.0.0.1:1", "9.2-1", cache, t.Logf) + if err != nil { + t.Fatalf("Ensure went to the (unreachable) mirror: %v", err) + } + if img.Version != "9.2-1" || img.ISO != iso { + t.Fatalf("got %+v", img) + } +} + +func TestVerified(t *testing.T) { + t.Parallel() + tests := []struct { + name string + stamp string + want bool + }{ + {name: "matching size", stamp: strings.Repeat("a", 64) + " 3\n", want: true}, + {name: "size changed", stamp: strings.Repeat("a", 64) + " 4\n", want: false}, + {name: "no stamp", stamp: "", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + p := filepath.Join(t.TempDir(), "x.iso") + if err := os.WriteFile(p, []byte("iso"), 0o644); err != nil { + t.Fatal(err) + } + if tt.stamp != "" { + if err := os.WriteFile(p+".sha256", []byte(tt.stamp), 0o644); err != nil { + t.Fatal(err) + } + } + if got := verified(p); got != tt.want { + t.Errorf("verified = %v, want %v", got, tt.want) + } + }) + } +} From 83a3fa462761309d786c3707cd8a52ebb249bdec Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:55:14 +0200 Subject: [PATCH 08/19] fix(pve): log in again when a ticket is rejected, close idle connections Right after a join a node can reject a fresh ticket for a moment. Clients are now safe for concurrent use, and their idle connections time out. --- internal/cluster/access.go | 4 +-- internal/pve/api.go | 61 ++++++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/internal/cluster/access.go b/internal/cluster/access.go index e6d04f3..1621ef3 100644 --- a/internal/cluster/access.go +++ b/internal/cluster/access.go @@ -21,9 +21,7 @@ func (c *Cluster) ensureToken(ctx context.Context) error { return err } if tok, err := c.Token(); err == nil && tok != "" { - probe := *api - probe.SetToken(tok) - if probe.Get(ctx, "/version", &struct{}{}) == nil { + if api.WithToken(tok).Get(ctx, "/version", &struct{}{}) == nil { return nil } } diff --git a/internal/pve/api.go b/internal/pve/api.go index de68db7..1e5d3ce 100644 --- a/internal/pve/api.go +++ b/internal/pve/api.go @@ -15,15 +15,19 @@ import ( "net/http" "net/url" "strings" + "sync" "time" ) +// Client is safe for concurrent use. type Client struct { - base string - http *http.Client - ticket string - csrf string - token string // "user@realm!id=secret" + base string + http *http.Client + token string // "user@realm!id=secret"; set before use + + mu sync.Mutex + ticket, csrf string + user, password string // for logging in again } // Fingerprint returns the colon-separated SHA-256 fingerprint of a PEM certificate, @@ -85,20 +89,26 @@ func newClient(port int, verify func(raw [][]byte) error) *Client { } return &Client{ base: fmt.Sprintf("https://127.0.0.1:%d/api2/json", port), - http: &http.Client{Timeout: 60 * time.Second, Transport: &http.Transport{TLSClientConfig: tlsCfg}}, + http: &http.Client{ + Timeout: 60 * time.Second, + Transport: &http.Transport{TLSClientConfig: tlsCfg, IdleConnTimeout: 30 * time.Second}, + }, } } // Login gets a ticket for user (e.g. root@pam). +// The client logs in again by itself when the ticket is rejected. func (c *Client) Login(ctx context.Context, user, password string) error { var out struct { Ticket string `json:"ticket"` CSRF string `json:"CSRFPreventionToken"` } - if err := c.Do(ctx, http.MethodPost, "/access/ticket", url.Values{"username": {user}, "password": {password}}, &out); err != nil { + if err := c.do(ctx, http.MethodPost, "/access/ticket", url.Values{"username": {user}, "password": {password}}, &out); err != nil { return err } - c.ticket, c.csrf = out.Ticket, out.CSRF + c.mu.Lock() + c.ticket, c.csrf, c.user, c.password = out.Ticket, out.CSRF, user, password + c.mu.Unlock() return nil } @@ -131,7 +141,25 @@ func (c *Client) Delete(ctx context.Context, path string) error { return c.Do(ctx, http.MethodDelete, path, nil, nil) } +// Do sends a request and decodes the data of the response into out. A request +// with a ticket that the node rejects (401) is sent once more after a new login: +// right after a node joined a cluster, tickets can briefly fail to verify while +// the cluster's auth key reaches it. func (c *Client) Do(ctx context.Context, method, path string, form url.Values, out any) error { + err := c.do(ctx, method, path, form, out) + c.mu.Lock() + user, password := c.user, c.password + c.mu.Unlock() + if !IsStatus(err, http.StatusUnauthorized) || c.token != "" || user == "" { + return err + } + if lerr := c.Login(ctx, user, password); lerr != nil { + return errors.Join(err, lerr) + } + return c.do(ctx, method, path, form, out) +} + +func (c *Client) do(ctx context.Context, method, path string, form url.Values, out any) error { var body io.Reader if form != nil { body = strings.NewReader(form.Encode()) @@ -143,13 +171,16 @@ func (c *Client) Do(ctx context.Context, method, path string, form url.Values, o if form != nil { req.Header.Set("Content-Type", "application/x-www-form-urlencoded") } + c.mu.Lock() + ticket, csrf := c.ticket, c.csrf + c.mu.Unlock() switch { case c.token != "": req.Header.Set("Authorization", "PVEAPIToken="+c.token) - case c.ticket != "": - req.AddCookie(&http.Cookie{Name: "PVEAuthCookie", Value: c.ticket}) //nolint:gosec // request cookie, attributes do not apply + case ticket != "": + req.AddCookie(&http.Cookie{Name: "PVEAuthCookie", Value: ticket}) //nolint:gosec // request cookie, attributes do not apply if method != http.MethodGet { - req.Header.Set("CSRFPreventionToken", c.csrf) + req.Header.Set("CSRFPreventionToken", csrf) } } resp, err := c.http.Do(req) @@ -239,5 +270,11 @@ func (c *Client) WaitTask(ctx context.Context, node, upid string, timeout time.D } } -// SetToken switches the client to API token auth ("user@realm!id=secret"). +// SetToken switches the client to API token auth ("user@realm!id=secret"). Call +// it before using the client. func (c *Client) SetToken(token string) { c.token = token } + +// WithToken returns a client for the same node that uses an API token. +func (c *Client) WithToken(token string) *Client { + return &Client{base: c.base, http: c.http, token: token} +} From e72c5c71190615501873a3422b21f397ebd010d6 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:55:18 +0200 Subject: [PATCH 09/19] fix(config): validate proxmox.domain as a DNS name The domain ends up in shell scripts on the nodes. --- internal/config/config_test.go | 2 ++ internal/config/validate.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 87d4c1f..9c7f496 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -71,6 +71,8 @@ func TestValidationErrors(t *testing.T) { "nodes: {defaults: {memory: 1G}}\n": "at least 2G", "nodes: {overrides: {pve9: {cpus: 2}}}\n": "unknown node", "networks: [{name: a, cidr: 10.1.0.0/28}]\nnodes: {count: 5}\n": "too small", + "proxmox: {domain: \"lab/x.internal\"}\n": "proxmox.domain", + "proxmox: {domain: internal}\n": "proxmox.domain", } for in, want := range cases { err := load(t, in).Validate() diff --git a/internal/config/validate.go b/internal/config/validate.go index 37fd06a..dbe0213 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -16,6 +16,7 @@ var ( bridgeRe = regexp.MustCompile(`^vmbr[0-9]{1,4}$`) vlanRe = regexp.MustCompile(`^[0-9]{1,4}(-[0-9]{1,4})?$`) versRe = regexp.MustCompile(`^[0-9]+\.[0-9]+(-[0-9]+)?$`) + domainRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+$`) minDisks = map[string]int{"single": 1, "mirror": 2, "raid10": 4, "raidz": 3, "raidz2": 4, "raidz3": 5} ) @@ -42,6 +43,9 @@ func (c *Cluster) Validate() error { if !nameRe.MatchString(c.Name) { add("name: %q must match %s", c.Name, nameRe) } + if !domainRe.MatchString(c.Proxmox.Domain) || len(c.Proxmox.Domain) > 200 { + add("proxmox.domain: %q must be a DNS domain like lab.internal", c.Proxmox.Domain) + } if !versRe.MatchString(c.Proxmox.Version) { add("proxmox.version: %q must look like 9.2 or 9.2-1", c.Proxmox.Version) } From e2dbd0fa8546a6fdff859f5a6a6e6aa6534d0ff0 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:55:42 +0200 Subject: [PATCH 10/19] fix(cluster): join only once the first node accepts changes Logs in to the joining nodes in parallel, then before each join waits until the first node is quorate with the expected votes and /etc/pve is writable; the API reports quorum earlier. A join refused for lack of quorum is retried. --- internal/cluster/pvecluster.go | 121 ++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/internal/cluster/pvecluster.go b/internal/cluster/pvecluster.go index 6c767b8..5c5cba8 100644 --- a/internal/cluster/pvecluster.go +++ b/internal/cluster/pvecluster.go @@ -8,6 +8,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" "golang.org/x/sync/errgroup" @@ -62,9 +63,18 @@ func (c *Cluster) formCluster(ctx context.Context) error { if err != nil { return err } + joiners, err := c.joinClients(ctx, api, nodes[1:]) + if err != nil { + return err + } + // Joins run one at a time: concurrent joins race for the cluster lock. + members := len(nodes) - len(joiners) for _, n := range nodes[1:] { - if err := c.join(ctx, api, first, n, fp); err != nil { - return err + if napi, ok := joiners[n.Name]; ok { + if err := c.join(ctx, api, napi, first, n, fp, members); err != nil { + return err + } + members++ } } if m, ok := c.Cfg.RoleNetwork(config.RoleMigration); ok { @@ -89,33 +99,57 @@ func (c *Cluster) fingerprint(ctx context.Context, node string) (string, error) return pve.Fingerprint(certPEM) } -// join adds node n unless it is a member already. -func (c *Cluster) join(ctx context.Context, api *pve.Client, first, n config.Node, fp string) error { +// joinClients logs in, in parallel, to the nodes that are not members yet. +func (c *Cluster) joinClients(ctx context.Context, api *pve.Client, nodes []config.Node) (map[string]*pve.Client, error) { _, members, err := api.ClusterStatus(ctx) if err != nil { - return err + return nil, err } - if _, ok := members[n.Name]; ok { - return nil + var mu sync.Mutex + out := map[string]*pve.Client{} + g, gctx := errgroup.WithContext(ctx) + for _, n := range nodes { + if _, ok := members[n.Name]; ok { + continue + } + g.Go(func() error { + napi, err := c.api(gctx, n.Name) + if err != nil { + return err + } + mu.Lock() + out[n.Name] = napi + mu.Unlock() + return nil + }) } + return out, g.Wait() +} + +// join adds node n to the cluster through its own API client napi. members is +// the number of nodes in the cluster before the join. +func (c *Cluster) join(ctx context.Context, api, napi *pve.Client, first, n config.Node, fp string, members int) error { form := c.links(n) c.step("joining %s (%s)", n.Name, describeLinks(form)) - napi, err := c.api(ctx, n.Name) - if err != nil { - return err - } form.Set("hostname", c.corosyncIP(first)) form.Set("password", c.password) form.Set("fingerprint", fp) - // The cluster refuses joins until it is quorate. - if err := waitMember(ctx, api, first.Name, nil, "", 2*time.Minute); err != nil { - return err - } - var upid string - if err := napi.Post(ctx, "/cluster/config/join", form, &upid); err != nil { - return fmt.Errorf("join %s: %w", n.Name, err) + for attempt := 1; ; attempt++ { + // The cluster refuses joins until it is quorate and /etc/pve is writable. + if err := c.waitNodeQuorate(ctx, first.Name, members, 2*time.Minute); err != nil { + return err + } + var upid string + if err := napi.Post(ctx, "/cluster/config/join", form, &upid); err != nil { + return fmt.Errorf("join %s: %w", n.Name, err) + } + err := waitMember(ctx, api, n.Name, napi, upid, 3*time.Minute) + // A join refused for lack of quorum changed nothing; try again. + if err == nil || attempt == 3 || !strings.Contains(err.Error(), "no quorum") { + return err + } + c.step("%s: join refused (the cluster had no quorum yet), retrying", n.Name) } - return waitMember(ctx, api, n.Name, napi, upid, 3*time.Minute) } // waitMember waits until node is listed online in a quorate cluster. If a join @@ -154,28 +188,7 @@ func (c *Cluster) waitQuorum(ctx context.Context, timeout time.Duration) error { want := c.Cfg.Nodes.Count g, gctx := errgroup.WithContext(ctx) for _, n := range c.Cfg.NodeList() { - g.Go(func() error { - var last string - err := retry.Do(gctx, timeout, 2*time.Second, func() error { - s, err := c.waitSSH(gctx, n.Name, time.Minute) - if err != nil { - return retry.Permanent(err) - } - out, err := s.Run(`pvecm status 2>&1 || true -f=/etc/pve/.proxbase-write-test; if echo ok > $f 2>/dev/null; then rm -f $f; echo WRITABLE; fi`) - s.Close() - m := votesRe.FindStringSubmatch(out) - if err == nil && quorateRe.MatchString(out) && m != nil && m[1] == strconv.Itoa(want) && strings.Contains(out, "WRITABLE") { - return nil - } - last = out - return errors.New("no quorum") - }) - if err != nil && gctx.Err() == nil && last != "" { - return fmt.Errorf("%s: no quorum after %s:\n%s", n.Name, timeout, last) - } - return err - }) + g.Go(func() error { return c.waitNodeQuorate(gctx, n.Name, want, timeout) }) } if err := g.Wait(); err != nil { return err @@ -183,3 +196,29 @@ f=/etc/pve/.proxbase-write-test; if echo ok > $f 2>/dev/null; then rm -f $f; ech c.step("cluster quorate: %d/%d votes, /etc/pve writable on every node", want, want) return nil } + +// waitNodeQuorate waits until node is quorate with votes votes and can write +// /etc/pve. The API reports quorum a moment before the cluster file system +// accepts changes such as a join. +func (c *Cluster) waitNodeQuorate(ctx context.Context, node string, votes int, timeout time.Duration) error { + var last string + err := retry.Do(ctx, timeout, time.Second, func() error { + s, err := c.waitSSH(ctx, node, time.Minute) + if err != nil { + return retry.Permanent(err) + } + out, err := s.Run(`pvecm status 2>&1 || true +f=/etc/pve/.proxbase-write-test; if echo ok > $f 2>/dev/null; then rm -f $f; echo WRITABLE; fi`) + s.Close() + m := votesRe.FindStringSubmatch(out) + if err == nil && quorateRe.MatchString(out) && m != nil && m[1] == strconv.Itoa(votes) && strings.Contains(out, "WRITABLE") { + return nil + } + last = out + return errors.New("no quorum") + }) + if err != nil && ctx.Err() == nil && last != "" { + return fmt.Errorf("%s: no quorum after %s:\n%s", node, timeout, last) + } + return err +} From adcfdddc30b784efcc6f4f339158f9a8bb2a0622 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:55:59 +0200 Subject: [PATCH 11/19] fix(cluster): repair joins that lost node files or replaced the CA A joining node finishes its setup before the cluster file system has synced to it. Its key and certificate can be lost, or it writes a new cluster CA, and a failed join leaves pvedaemon and pveproxy rejecting logins. After the joins every node gets a new certificate from the current CA where needed. --- internal/cluster/create.go | 1 + internal/cluster/pvecluster.go | 26 ++++++++++++++++++++++++++ internal/nodesetup/nodesetup.go | 25 +++++++++++++++++++++++++ internal/nodesetup/nodesetup_test.go | 1 + 4 files changed, 53 insertions(+) diff --git a/internal/cluster/create.go b/internal/cluster/create.go index 504fd3a..5381212 100644 --- a/internal/cluster/create.go +++ b/internal/cluster/create.go @@ -76,6 +76,7 @@ func (c *Cluster) create(ctx context.Context) error { c.postInstall, c.formCluster, func(ctx context.Context) error { return c.waitQuorum(ctx, 5*time.Minute) }, + c.repairJoins, c.setupStorage, c.ensureToken, c.exportCA, diff --git a/internal/cluster/pvecluster.go b/internal/cluster/pvecluster.go index 5c5cba8..9f41967 100644 --- a/internal/cluster/pvecluster.go +++ b/internal/cluster/pvecluster.go @@ -14,6 +14,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/virtbase/proxbase/internal/config" + "github.com/virtbase/proxbase/internal/nodesetup" "github.com/virtbase/proxbase/internal/pve" "github.com/virtbase/proxbase/internal/retry" ) @@ -178,6 +179,31 @@ func waitMember(ctx context.Context, api *pve.Client, node string, joiner *pve.C return err } +// repairJoins repairs what racing cluster joins left broken (see +// nodesetup.RepairJoin): without it a node's API may have no certificate, one the +// cluster CA does not vouch for, or reject every login. +func (c *Cluster) repairJoins(ctx context.Context) error { + g, gctx := errgroup.WithContext(ctx) + for _, n := range c.Cfg.NodeList() { + g.Go(func() error { + s, err := c.waitSSH(gctx, n.Name, time.Minute) + if err != nil { + return err + } + defer s.Close() + out, err := s.Run(nodesetup.RepairJoin) + if err != nil { + return fmt.Errorf("%s: repair join: %w", n.Name, err) + } + if strings.Contains(out, "repaired") { + c.step("%s: the cluster join left the node certificate missing or from another CA; issued a new one", n.Name) + } + return nil + }) + } + return g.Wait() +} + var ( votesRe = regexp.MustCompile(`Total votes:\s+(\d+)`) quorateRe = regexp.MustCompile(`Quorate:\s+Yes`) diff --git a/internal/nodesetup/nodesetup.go b/internal/nodesetup/nodesetup.go index 3f97156..3cc9b24 100644 --- a/internal/nodesetup/nodesetup.go +++ b/internal/nodesetup/nodesetup.go @@ -149,5 +149,30 @@ if [ "$new" != "$(uname -r)" ]; then echo "$new $(cat /proc/sys/kernel/random/bo // Reboot reboots two seconds later, so the SSH session can end cleanly. const Reboot = "systemd-run --on-active=2 systemctl reboot >/dev/null" +// RepairJoin repairs what a cluster join can leave broken. The joining node +// finishes its setup before the cluster file system has synced to it: files it +// writes then (its key and certificate) can be lost, and if it believes the +// cluster CA is missing it writes a new one, which leaves the other nodes with +// certificates of the old CA. A failed join also skips restarting pvedaemon and +// pveproxy, which then reject new tickets. RepairJoin issues a new node +// certificate from the current cluster CA and restarts both services if the +// certificate is missing or not signed by that CA, or the node's last join +// failed. It prints "repaired" when it acted. +const RepairJoin = ` +marker=/var/lib/proxbase-join-repaired +join=$(grep ':clusterjoin:' /var/log/pve/tasks/index 2>/dev/null | tail -n 1 || true) +failed= +if [ -n "$join" ] && ! echo "$join" | grep -q ' OK$' && [ "$(cat $marker 2>/dev/null)" != "$join" ]; then + failed=1 +fi +if [ -n "$failed" ] || [ ! -s /etc/pve/local/pve-ssl.key ] || + ! openssl verify -CAfile /etc/pve/pve-root-ca.pem /etc/pve/local/pve-ssl.pem >/dev/null 2>&1; then + pvecm updatecerts --force --silent + systemctl restart pvedaemon pveproxy + if [ -n "$join" ]; then echo "$join" > $marker; fi + echo repaired +fi +` + // BootID prints the boot ID and the running kernel. const BootID = "cat /proc/sys/kernel/random/boot_id; uname -r" diff --git a/internal/nodesetup/nodesetup_test.go b/internal/nodesetup/nodesetup_test.go index e5adf1f..574899b 100644 --- a/internal/nodesetup/nodesetup_test.go +++ b/internal/nodesetup/nodesetup_test.go @@ -66,6 +66,7 @@ func TestScriptsSyntax(t *testing.T) { "ProvisionEnd": ProvisionEnd, "Upgrade": Upgrade, "PendingKernel": PendingKernel, + "RepairJoin": RepairJoin, } { t.Run(name, func(t *testing.T) { cmd := exec.CommandContext(t.Context(), "bash", "-n") From 7daf464765c0b778e76400f324613781e2fc4927 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:56:18 +0200 Subject: [PATCH 12/19] feat(qemu): pass files to guests and boot a node once in the foreground vm.Spec.Files become QEMU fw_cfg files; BootOnce runs an installed node until it powers off and returns its serial console. --- internal/qemu/install.go | 34 ++++++++++++++++++++- internal/qemu/process.go | 2 +- internal/qemu/qemu.go | 28 +++++++++++++++--- internal/qemu/qemu_test.go | 24 ++++++++++----- internal/qemu/testdata/once.args | 51 ++++++++++++++++++++++++++++++++ internal/vm/vm.go | 15 ++++++++++ 6 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 internal/qemu/testdata/once.args diff --git a/internal/qemu/install.go b/internal/qemu/install.go index 4da79f4..bd5a061 100644 --- a/internal/qemu/install.go +++ b/internal/qemu/install.go @@ -30,6 +30,38 @@ func (Runtime) Clone(ctx context.Context, s vm.Spec, base string) error { return createOverlay(ctx, s.RootDisk, base) } +var _ vm.Booter = Runtime{} + +// BootOnce boots the installed node in the foreground and returns its serial +// console output when it powers off. A reboot also ends it (-no-reboot). +func (Runtime) BootOnce(ctx context.Context, s vm.Spec, timeout time.Duration) ([]byte, error) { + if _, ok := running(pidfile(s)); ok { + return nil, fmt.Errorf("a VM is already running (pidfile %s)", pidfile(s)) + } + cleanSockets(s) + qlog, err := os.Create(qemuLog(s)) + if err != nil { + return nil, err + } + defer qlog.Close() + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + cmd := exec.CommandContext(ctx, Binary, args(s, bootOnce, nil)...) + cmd.SysProcAttr = installProcAttr() + cmd.Stdout, cmd.Stderr = qlog, qlog + if err := cmd.Run(); err != nil { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return nil, fmt.Errorf("%s did not power off within %s, see %s", s.Name, timeout, consoleLog(s)) + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + b, _ := os.ReadFile(qemuLog(s)) + return nil, fmt.Errorf("QEMU exited with %w: %s", err, strings.TrimSpace(string(b))) + } + return os.ReadFile(consoleLog(s)) +} + // Install creates fresh disks and runs the unattended installer in the foreground, // answering its debug shell over the serial console. QEMU powers off when done. func (Runtime) Install(ctx context.Context, s vm.Spec, m vm.Media) error { @@ -47,7 +79,7 @@ func (Runtime) Install(ctx context.Context, s vm.Spec, m vm.Media) error { return err } defer qlog.Close() - cmd := exec.Command(Binary, args(s, &m)...) + cmd := exec.Command(Binary, args(s, bootInstall, &m)...) cmd.SysProcAttr = installProcAttr() cmd.Stdout, cmd.Stderr = qlog, qlog if err := cmd.Start(); err != nil { diff --git a/internal/qemu/process.go b/internal/qemu/process.go index 87e150c..70d6e61 100644 --- a/internal/qemu/process.go +++ b/internal/qemu/process.go @@ -18,7 +18,7 @@ func (Runtime) Start(ctx context.Context, s vm.Spec) error { return nil } cleanSockets(s) - out, err := exec.CommandContext(ctx, Binary, args(s, nil)...).CombinedOutput() + out, err := exec.CommandContext(ctx, Binary, args(s, bootDaemon, nil)...).CombinedOutput() if err != nil { return fmt.Errorf("start %s: %w: %s", s.Process, err, strings.TrimSpace(string(out))) } diff --git a/internal/qemu/qemu.go b/internal/qemu/qemu.go index 3e08961..a35f8f1 100644 --- a/internal/qemu/qemu.go +++ b/internal/qemu/qemu.go @@ -48,8 +48,17 @@ const driveOpts = "cache=unsafe,discard=unmap,detect-zeroes=unmap" func dev(format string, a ...any) []string { return []string{"-device", fmt.Sprintf(format, a...)} } -// args returns the command line; media != nil boots the installer. -func args(s vm.Spec, media *vm.Media) []string { +// boot selects how a node is started. +type boot int + +const ( + bootDaemon boot = iota // installed node, daemonized with a console socket + bootInstall // the installer from media, in the foreground + bootOnce // installed node in the foreground until it powers off +) + +// args returns the command line; media is only used by bootInstall. +func args(s vm.Spec, b boot, media *vm.Media) []string { cpu := "host" if !s.Nested { cpu += ",vmx=off,svm=off" @@ -85,7 +94,12 @@ func args(s vm.Spec, media *vm.Media) []string { } a = append(a, "-qmp", "unix:"+qmpSocket(s)+",server=on,wait=off", "-pidfile", pidfile(s)) - if media != nil { + for _, f := range s.Files { + a = append(a, "-fw_cfg", "name="+f.Name+",file="+f.Path) + } + + switch b { + case bootInstall: a = append(a, "-drive", "if=none,id=cd,media=cdrom,readonly=on,file="+media.ISO, "-device", "ide-cd,drive=cd,bus=ide.0", @@ -96,7 +110,13 @@ func args(s vm.Spec, media *vm.Media) []string { "-serial", "chardev:ser0", "-no-reboot", ) - } else { + case bootOnce: + a = append(a, + "-chardev", "file,id=ser0,path="+consoleLog(s), + "-serial", "chardev:ser0", + "-no-reboot", + ) + default: a = append(a, "-chardev", "socket,id=ser0,path="+console(s)+",server=on,wait=off,logfile="+consoleLog(s)+",logappend=on", "-serial", "chardev:ser0", diff --git a/internal/qemu/qemu_test.go b/internal/qemu/qemu_test.go index d0ade50..14c5f81 100644 --- a/internal/qemu/qemu_test.go +++ b/internal/qemu/qemu_test.go @@ -40,21 +40,25 @@ func golden(t *testing.T, name string, args []string) { } } -func TestRunArgs(t *testing.T) { golden(t, "run.args", args(spec(), nil)) } +func TestRunArgs(t *testing.T) { golden(t, "run.args", args(spec(), bootDaemon, nil)) } func TestInstallArgs(t *testing.T) { s := spec() s.Nested = false - golden(t, "install.args", args(s, &vm.Media{ISO: "/c/pve.iso", Kernel: "/c/linux26", Initrd: "/c/initrd.img", Cmdline: "ro quiet console=ttyS0,115200", AnswerDir: "/d/ans"})) + golden(t, "install.args", args(s, bootInstall, &vm.Media{ISO: "/c/pve.iso", Kernel: "/c/linux26", Initrd: "/c/initrd.img", Cmdline: "ro quiet console=ttyS0,115200", AnswerDir: "/d/ans"})) } // Devices shared by install and run must sit at the same PCI addresses. func TestStablePCI(t *testing.T) { - run := devices(args(spec(), nil)) - inst := devices(args(spec(), &vm.Media{})) - for _, d := range run { - if strings.Contains(d, "addr=") && !slices.Contains(inst, d) { - t.Errorf("device %q missing or moved in install mode", d) + run := devices(args(spec(), bootDaemon, nil)) + for mode, other := range map[string][]string{ + "install": devices(args(spec(), bootInstall, &vm.Media{})), + "once": devices(args(spec(), bootOnce, nil)), + } { + for _, d := range run { + if strings.Contains(d, "addr=") && !slices.Contains(other, d) { + t.Errorf("device %q missing or moved in %s mode", d, mode) + } } } } @@ -68,3 +72,9 @@ func devices(args []string) []string { } return out } + +func TestOnceArgs(t *testing.T) { + s := spec() + s.Files = []vm.File{{Name: "opt/proxbase/bake", Path: "/r/bake.sh"}} + golden(t, "once.args", args(s, bootOnce, nil)) +} diff --git a/internal/qemu/testdata/once.args b/internal/qemu/testdata/once.args new file mode 100644 index 0000000..2f1e87e --- /dev/null +++ b/internal/qemu/testdata/once.args @@ -0,0 +1,51 @@ +-name +guest=lab-pve1,process=lab-pve1 +-nodefaults +-no-user-config +-machine +q35,accel=kvm +-cpu +host +-smp +4 +-m +4096 +-display +none +-device +VGA,bus=pcie.0,addr=0x1 +-drive +if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap +-device +virtio-blk-pci,drive=root,bus=pcie.0,addr=0x2,bootindex=1 +-device +qemu-xhci,id=usb,bus=pcie.0,addr=0x3 +-netdev +user,id=nat,dhcpstart=10.0.2.100,hostfwd=tcp:127.0.0.1:18101-10.0.2.15:22,hostfwd=tcp:127.0.0.1:18001-10.0.2.15:8006 +-device +virtio-net-pci,netdev=nat,mac=52:54:00:50:01:00,bus=pcie.0,addr=0x4 +-netdev +dgram,id=net0,local.type=unix,local.path=/r/pve1-cluster.sock,remote.type=unix,remote.path=/r/sw-cluster.sock +-device +virtio-net-pci,netdev=net0,mac=52:54:00:50:01:01,bus=pcie.0,addr=0x5 +-object +rng-random,id=rng,filename=/dev/urandom +-device +virtio-rng-pci,rng=rng,bus=pcie.0,addr=0xe +-device +virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=0xf +-drive +if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap +-device +virtio-blk-pci,drive=data0,bus=pcie.0,addr=0x10 +-qmp +unix:/r/pve1.qmp,server=on,wait=off +-pidfile +/r/pve1.pid +-fw_cfg +name=opt/proxbase/bake,file=/r/bake.sh +-chardev +file,id=ser0,path=/l/pve1-console.log +-serial +chardev:ser0 +-no-reboot diff --git a/internal/vm/vm.go b/internal/vm/vm.go index 04969f6..beef7a3 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -37,6 +37,15 @@ type Spec struct { NICs []NIC RunDir string // pidfile, control and console sockets LogDir string // console, installer and runtime logs + // Files are handed to the guest at boot (QEMU fw_cfg); the guest reads them + // below /sys/firmware/qemu_fw_cfg/by_name/. + Files []File +} + +// File is a host file passed to the guest under Name (e.g. "opt/proxbase/identity"). +type File struct { + Name string + Path string } // Disks returns the root disk followed by the data disks. @@ -56,6 +65,12 @@ type Installer interface { Install(ctx context.Context, s Spec, m Media) error } +// Booter boots an installed node once in the foreground and returns its serial +// console output when it powers off, or fails after timeout. +type Booter interface { + BootOnce(ctx context.Context, s Spec, timeout time.Duration) (console []byte, err error) +} + // Cloner creates a node's disks from an installed base image instead of installing. type Cloner interface { Clone(ctx context.Context, s Spec, base string) error From 0ee6526a0d54cc18f243af87f5b3e4af14cbe627 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:56:44 +0200 Subject: [PATCH 13/19] feat(install): run a proxbase first-boot hook on installed nodes The installer fetches the hook from the host and the node runs it once, before networking: it skips the pveupdate of a first boot and removes the GRUB menu wait. It also applies an identity or a bake script passed as guest files, used by golden images. A ceph.sources that does not point at the enterprise repository is no longer disabled. --- internal/answer/answer.go | 10 ++ internal/answer/answer_test.go | 11 ++ internal/cluster/install.go | 12 ++- internal/firstboot/firstboot.go | 147 +++++++++++++++++++++++++++ internal/firstboot/firstboot_test.go | 107 +++++++++++++++++++ internal/firstboot/hook.sh | 68 +++++++++++++ internal/nodesetup/nodesetup.go | 34 ++++--- internal/nodesetup/nodesetup_test.go | 1 + 8 files changed, 375 insertions(+), 15 deletions(-) create mode 100644 internal/firstboot/firstboot.go create mode 100644 internal/firstboot/firstboot_test.go create mode 100644 internal/firstboot/hook.sh diff --git a/internal/answer/answer.go b/internal/answer/answer.go index 9879f3d..9daa269 100644 --- a/internal/answer/answer.go +++ b/internal/answer/answer.go @@ -18,6 +18,9 @@ type Params struct { SSHKeys []string MAC string // MAC of the NAT NIC; only this NIC gets the static install-time address Filesystem string // zfs, ext4, xfs, btrfs + // FirstBootURL, if set, is a script the installer downloads and the + // installed system runs once on its first boot, before networking starts. + FirstBootURL string } // Render returns answer.toml. The NAT NIC uses QEMU user networking (10.0.2.0/24). @@ -54,6 +57,13 @@ func Render(p Params) string { case "btrfs": w("btrfs.raid = \"raid0\"") } + if p.FirstBootURL != "" { + w("") + w("[first-boot]") + w("source = \"from-url\"") + w("url = %s", quote(p.FirstBootURL)) + w("ordering = \"before-network\"") + } return b.String() } diff --git a/internal/answer/answer_test.go b/internal/answer/answer_test.go index 522ee91..d1dd9fc 100644 --- a/internal/answer/answer_test.go +++ b/internal/answer/answer_test.go @@ -30,6 +30,17 @@ func TestRenderExt4(t *testing.T) { } } +func TestRenderFirstBoot(t *testing.T) { + got := Render(Params{Filesystem: "zfs", FirstBootURL: "http://10.0.2.2:4242/first-boot"}) + want := "\n[first-boot]\nsource = \"from-url\"\nurl = \"http://10.0.2.2:4242/first-boot\"\nordering = \"before-network\"\n" + if !strings.HasSuffix(got, want) { + t.Fatalf("got:\n%s\nwant suffix:\n%s", got, want) + } + if strings.Contains(Render(Params{Filesystem: "zfs"}), "[first-boot]") { + t.Fatal("no first-boot section without a URL") + } +} + func TestPassword(t *testing.T) { if a, b := Password(20), Password(20); len(a) != 20 || a == b { t.Fatalf("bad passwords %q %q", a, b) diff --git a/internal/cluster/install.go b/internal/cluster/install.go index 4af37f7..018c73d 100644 --- a/internal/cluster/install.go +++ b/internal/cluster/install.go @@ -13,6 +13,7 @@ import ( "github.com/virtbase/proxbase/internal/answer" "github.com/virtbase/proxbase/internal/config" + "github.com/virtbase/proxbase/internal/firstboot" "github.com/virtbase/proxbase/internal/image" "github.com/virtbase/proxbase/internal/vm" ) @@ -36,12 +37,17 @@ func (c *Cluster) installAll(ctx context.Context, img *image.Image) error { names[i] = n.Name } c.step("installing Proxmox VE %s on %s (logs: %s)", img.Version, strings.Join(names, ", "), c.Dir.Log("-install.log")) + hook, err := firstboot.Serve() + if err != nil { + return err + } + defer hook.Close() var mu sync.Mutex g, gctx := errgroup.WithContext(ctx) for _, n := range todo { g.Go(func() error { t0 := time.Now() - if err := c.install(gctx, img, n); err != nil { + if err := c.install(gctx, img, n, hook.URL); err != nil { return fmt.Errorf("%s: %w", n.Name, err) } mu.Lock() @@ -55,7 +61,8 @@ func (c *Cluster) installAll(ctx context.Context, img *image.Image) error { } // install writes the node's answer file and runs the unattended installer. -func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node) error { +// The installed system runs the first-boot hook at hookURL. +func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node, hookURL string) error { ansDir := c.Dir.Disk(n.Name + "-answer") if err := os.MkdirAll(ansDir, 0o700); err != nil { return err @@ -76,6 +83,7 @@ func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node) SSHKeys: append([]string{pub}, c.Cfg.Proxmox.SSHKeys...), MAC: spec.NATMAC, Filesystem: n.Spec.RootDisk.Filesystem, + FirstBootURL: hookURL, }) if err := os.WriteFile(filepath.Join(ansDir, "answer.toml"), []byte(ans), 0o600); err != nil { return err diff --git a/internal/firstboot/firstboot.go b/internal/firstboot/firstboot.go new file mode 100644 index 0000000..ff43823 --- /dev/null +++ b/internal/firstboot/firstboot.go @@ -0,0 +1,147 @@ +// Package firstboot prepares nodes on their first boot through the Proxmox +// installer's first-boot hook. The installer downloads the hook from a small +// HTTP server on the host and the installed system runs it once, before +// networking and before any Proxmox VE service starts. Proxbase uses that early +// moment to: +// +// - give nodes cloned from a base image their own identity without a reboot, +// - bake a base image (prepare it once, then power off unbooted by Proxmox VE), +// - skip work Proxmox VE does on a first boot that proxbase does not need. +// +// Per-node data reaches the hook as QEMU fw_cfg files (vm.File). +package firstboot + +import ( + _ "embed" + "fmt" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/virtbase/proxbase/internal/nodesetup" +) + +const ( + // FileBake is the guest file name of the bake script. + FileBake = "opt/proxbase/bake" + // FileIdentity is the guest file name of a node's identity. + FileIdentity = "opt/proxbase/identity" + // IdentityMarker exists on nodes that received their identity; it holds the node name. + IdentityMarker = "/etc/proxbase-identity" + // BakeOK is printed on the serial console when a bake succeeded. + BakeOK = "proxbase-first-boot: bake ok" + + // hostAddr is the host as seen from a guest on QEMU user networking. + hostAddr = "10.0.2.2" + hookPath = "/proxmox-first-boot" +) + +//go:embed hook.sh +var hookTemplate string + +// Hook returns the first-boot hook script. +func Hook() string { + return strings.NewReplacer( + "@NO_GRUB_TIMEOUT@", strings.Trim(nodesetup.NoGrubTimeout, "\n"), + "@IDENTITY_MARKER@", IdentityMarker, + ).Replace(hookTemplate) +} + +// Server serves the hook to installers on QEMU user networking. +type Server struct { + srv *http.Server + // URL is the hook's address as seen from the guest. + URL string +} + +// Serve starts serving the hook on a free port of the host's loopback address, +// which guests on QEMU user networking reach as 10.0.2.2. +func Serve() (*Server, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, fmt.Errorf("first-boot hook server: %w", err) + } + hook := []byte(Hook()) + mux := http.NewServeMux() + mux.HandleFunc("GET "+hookPath, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/x-shellscript") + _, _ = w.Write(hook) + }) + s := &Server{ + srv: &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second}, + URL: "http://" + net.JoinHostPort(hostAddr, strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)) + hookPath, + } + // Serve returns when Close is called. Any other failure surfaces as a failed + // download in the installer, which reports it. + go func() { _ = s.srv.Serve(ln) }() + return s, nil +} + +// Close stops the server. +func (s *Server) Close() error { return s.srv.Close() } + +// Identity renders the identity file of a node cloned from a base image. +func Identity(node, domain, password string, authorizedKeys []string) []byte { + keys := make([]string, len(authorizedKeys)) + for i, k := range authorizedKeys { + keys[i] = strings.TrimSpace(k) + } + return fmt.Appendf(nil, "NODE=%s\nDOMAIN=%s\nPASSWORD=%s\nKEYS=%s\n", + quote(node), quote(domain), quote(password), quote(strings.Join(keys, "\n"))) +} + +// Bake returns the script that prepares a base image on its only boot. With a +// Ceph release (e.g. "squid") it also installs the Ceph packages, so nodes +// cloned from the image skip `pveceph install`; natMAC is the MAC of the NIC on +// QEMU user networking it downloads through. +func Bake(cephRelease, natMAC string) string { + var b strings.Builder + b.WriteString("touch /var/log/pveam.log\n") + b.WriteString(nodesetup.NoGrubTimeout) + if cephRelease != "" { + b.WriteString(nodesetup.Repositories) + fmt.Fprintf(&b, cephBake, cephRelease, natMAC) + } + return b.String() +} + +// cephBake installs what `pveceph install` installs. Networking is not up yet, +// so it configures the NAT NIC itself with ip: ifupdown2 cannot be used here, +// its lock is held by hotplug ifup runs that wait for networking, which waits +// for this hook. policy-rc.d keeps packages from starting services during the +// boot. +const cephBake = ` +cat > /etc/apt/sources.list.d/ceph.sources <&2; exit 1; } +ip link set "$nic" up +ip addr add 10.0.2.15/24 dev "$nic" +ip route add default via 10.0.2.2 +printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d +chmod 755 /usr/sbin/policy-rc.d +trap 'rm -f /usr/sbin/policy-rc.d' EXIT +export DEBIAN_FRONTEND=noninteractive +apt='apt-get -o Acquire::Retries=3 -o Dpkg::Options::=--force-unsafe-io' +retry_apt() { "$@" || "$@" -o Acquire::http::No-Cache=true; } +$apt update +retry_apt $apt install -y --no-install-recommends -o Dpkg::Options::=--force-confnew \ + ceph ceph-common ceph-fuse ceph-mds ceph-volume gdisk nvme-cli +retry_apt $apt install -y --only-upgrade libpve-storage-perl +$apt clean +ip addr flush dev "$nic" +ip link set "$nic" down +` + +// quote returns s single-quoted for the shell. +func quote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } diff --git a/internal/firstboot/firstboot_test.go b/internal/firstboot/firstboot_test.go new file mode 100644 index 0000000..5d26ff8 --- /dev/null +++ b/internal/firstboot/firstboot_test.go @@ -0,0 +1,107 @@ +package firstboot + +import ( + "io" + "net/http" + "os/exec" + "strings" + "testing" +) + +// sh parses (and with run, executes) a script with the guest's /bin/sh. +func sh(t *testing.T, script string, run bool) string { + t.Helper() + args := []string{"-n"} + if run { + args = []string{"-eu"} + } + cmd := exec.CommandContext(t.Context(), "sh", args...) + cmd.Stdin = strings.NewReader(script) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("sh %v: %v\n%s", args, err, out) + } + return string(out) +} + +func TestHook(t *testing.T) { + t.Parallel() + h := Hook() + if strings.Contains(h, "@") { + t.Fatalf("unreplaced placeholder in hook:\n%s", h) + } + for _, want := range []string{"GRUB_TIMEOUT=0", ">" + IdentityMarker, "pending-first-boot-setup", "/var/log/pveam.log"} { + if !strings.Contains(h, want) { + t.Errorf("hook lacks %q", want) + } + } + sh(t, h, false) +} + +func TestServe(t *testing.T) { + t.Parallel() + s, err := Serve() + if err != nil { + t.Fatal(err) + } + defer s.Close() + if !strings.HasPrefix(s.URL, "http://10.0.2.2:") { + t.Fatalf("URL %s is not the host as seen from the guest", s.URL) + } + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, strings.Replace(s.URL, "10.0.2.2", "127.0.0.1", 1), nil) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK || string(body) != Hook() { + t.Fatalf("GET %s: %s, %d bytes", s.URL, resp.Status, len(body)) + } +} + +// The identity file is sourced by the hook; values must arrive unchanged. +func TestIdentity(t *testing.T) { + t.Parallel() + keys := []string{"ssh-ed25519 AAAA proxbase@lab\n", "ssh-rsa BBBB it's mine"} + id := Identity("pve2", "lab.internal", `p'w"$x`, keys) + got := sh(t, string(id)+`printf '%s|%s|%s|%s' "$NODE" "$DOMAIN" "$PASSWORD" "$KEYS"`, true) + want := "pve2|lab.internal|p'w\"$x|ssh-ed25519 AAAA proxbase@lab\nssh-rsa BBBB it's mine" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestBake(t *testing.T) { + t.Parallel() + tests := []struct { + name, ceph string + want, not []string + }{ + {name: "plain", want: []string{"GRUB_TIMEOUT=0", "pveam.log"}, not: []string{"apt-get"}}, + {name: "ceph", ceph: "squid", want: []string{"debian/ceph-squid", "ceph-mds", "policy-rc.d", `"52:54:00:50:00:00"`, "ip route add default via 10.0.2.2"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + b := Bake(tt.ceph, "52:54:00:50:00:00") + for _, w := range tt.want { + if !strings.Contains(b, w) { + t.Errorf("bake lacks %q", w) + } + } + for _, n := range tt.not { + if strings.Contains(b, n) { + t.Errorf("bake must not contain %q", n) + } + } + sh(t, b, false) + }) + } +} diff --git a/internal/firstboot/hook.sh b/internal/firstboot/hook.sh new file mode 100644 index 0000000..51d87e4 --- /dev/null +++ b/internal/firstboot/hook.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# Proxbase first-boot hook. The Proxmox installer downloads it and the installed +# system runs it once, on its first boot, before networking and before any +# Proxmox VE service starts (proxmox-first-boot-network-pre.service). +# +# Proxbase passes data to the guest as QEMU fw_cfg files: +# opt/proxbase/bake script that prepares a base image; the hook runs it, +# stays pending for the clones and powers off +# opt/proxbase/identity shell variables (NODE, DOMAIN, PASSWORD, KEYS) that +# give a node cloned from a base image its own identity +set -eu + +cfg=/sys/firmware/qemu_fw_cfg/by_name/opt/proxbase +modprobe qemu_fw_cfg 2>/dev/null || true + +console() { echo "proxbase-first-boot: $*" >/dev/ttyS0 2>/dev/null || true; } + +if [ -r "$cfg/bake/raw" ]; then + status=ok + if ! sh -eu "$cfg/bake/raw" >/var/log/proxbase-bake.log 2>&1; then + status=failed + tail -n 30 /var/log/proxbase-bake.log >/dev/ttyS0 2>/dev/null || true + fi + # Stay pending: every clone runs this hook again for its identity. + touch /var/lib/proxmox-first-boot/pending-first-boot-setup + console "bake $status" + # Power off before networking and the Proxmox VE services start; the base + # image must never run them (they create per-node keys and certificates). + systemctl poweroff --no-block + exec sleep infinity +fi + +# pveproxy runs pveupdate (apt update, appliance list) on its first start unless +# this log exists; proxbase manages repositories and updates itself. +touch /var/log/pveam.log + +grub_timeout() { +@NO_GRUB_TIMEOUT@ +} +grub_timeout || console "could not remove the GRUB timeout" + +if [ -r "$cfg/identity/raw" ]; then + # shellcheck source=/dev/null + . "$cfg/identity/raw" + fqdn="$NODE.$DOMAIN" + hostname "$NODE" + echo "$NODE" >/etc/hostname + sed -i "/^10\.0\.2\.15[[:space:]]/c\10.0.2.15 $fqdn $NODE" /etc/hosts + # Proxmox VE names the node certificate after the search domain. + sed -i "s/^search .*/search $DOMAIN/" /etc/resolv.conf + echo "$fqdn" >/etc/mailname + postconf -e "myhostname=$fqdn" + rm -f /etc/ssh/ssh_host_* + ssh-keygen -A >/dev/null + rm -f /etc/machine-id /var/lib/dbus/machine-id + systemd-machine-id-setup >/dev/null + # journald files are per machine-id; without a restart this boot's log would + # land under the old one, where journalctl does not look. + systemctl restart systemd-journald + echo "root:$PASSWORD" | chpasswd + # Still a plain file here: pve-cluster moves it into /etc/pve on its first + # start, so the base image's key is never trusted by the node. + mkdir -p /root/.ssh + printf '%s\n' "$KEYS" >/root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys + echo "$NODE" >@IDENTITY_MARKER@ + console "identity $NODE" +fi diff --git a/internal/nodesetup/nodesetup.go b/internal/nodesetup/nodesetup.go index 3cc9b24..71805b6 100644 --- a/internal/nodesetup/nodesetup.go +++ b/internal/nodesetup/nodesetup.go @@ -65,18 +65,7 @@ block /etc/hosts "$(cat <<'EOF' systemctl enable --now serial-getty@ttyS0.service >/dev/null 2>&1 %s # No-subscription repository instead of enterprise. -codename=$(. /etc/os-release && echo "$VERSION_CODENAME") -for f in /etc/apt/sources.list.d/pve-enterprise.sources /etc/apt/sources.list.d/ceph.sources; do - if [ -f "$f" ] && ! grep -q '^Enabled: false' "$f"; then echo 'Enabled: false' >> "$f"; fi -done -cat > /etc/apt/sources.list.d/proxmox.sources <> "$f" + fi +done +cat > /etc/apt/sources.list.d/proxmox.sources < Date: Sun, 11 Oct 2026 20:57:11 +0200 Subject: [PATCH 14/19] perf(golden): give clones their identity on first boot, bake base images Clones get host name, SSH host keys, machine-id, password and keys from the first-boot hook before networking, so personalization needs no reboot. Base images are booted once to bake them (GRUB, pveupdate, and for Ceph clusters the Ceph packages) and power off before Proxmox VE services start. Base images from before the hook are still personalized over SSH. --- internal/cli/image.go | 8 ++- internal/cluster/golden.go | 35 +++++++----- internal/cluster/install.go | 24 +++++++- internal/cluster/node.go | 14 ++++- internal/golden/golden.go | 100 ++++++++++++++++++++++++++++----- internal/golden/golden_test.go | 19 +++++++ internal/golden/personalize.go | 16 ++++-- 7 files changed, 179 insertions(+), 37 deletions(-) diff --git a/internal/cli/image.go b/internal/cli/image.go index ceac206..8f164db 100644 --- a/internal/cli/image.go +++ b/internal/cli/image.go @@ -28,9 +28,13 @@ func imageCmd() *cobra.Command { return err } w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "KEY\tISO\tROOT\tCREATED\tUSED BY") + fmt.Fprintln(w, "KEY\tISO\tROOT\tCEPH\tCREATED\tUSED BY") for _, m := range images { - fmt.Fprintf(w, "%s\t%s\t%s %s\t%s\t%v\n", m.Key, m.ISO, m.Filesystem, m.Size, m.Created.Local().Format(time.DateTime), users[m.Key]) + ceph := m.Ceph + if ceph == "" { + ceph = "-" + } + fmt.Fprintf(w, "%s\t%s\t%s %s\t%s\t%s\t%v\n", m.Key, m.ISO, m.Filesystem, m.Size, ceph, m.Created.Local().Format(time.DateTime), users[m.Key]) } return w.Flush() }, diff --git a/internal/cluster/golden.go b/internal/cluster/golden.go index f3bcb3c..a4b229b 100644 --- a/internal/cluster/golden.go +++ b/internal/cluster/golden.go @@ -21,13 +21,14 @@ import ( // cloneAll creates the nodes' disks from base images instead of installing. func (c *Cluster) cloneAll(ctx context.Context, img *image.Image, todo []config.Node) error { cloner, ok := c.runtime.(vm.Cloner) - if !ok { - return errors.New("the node runtime cannot clone base images") + builder, ok2 := c.runtime.(golden.Builder) + if !ok || !ok2 { + return errors.New("the node runtime cannot build or clone base images") } t0 := time.Now() var names []string for _, n := range todo { - base, err := golden.Ensure(ctx, c.runtime, c.Cfg, n, img, c.step) + base, err := golden.Ensure(ctx, builder, c.Cfg, n, img, c.step) if err != nil { return err } @@ -42,8 +43,10 @@ func (c *Cluster) cloneAll(ctx context.Context, img *image.Image, todo []config. return c.save() } -// personalizeAll gives cloned nodes their own identity (host name, SSH host keys, -// machine-id, password, authorized keys) and reboots them. +// personalizeAll completes the identity of cloned nodes (host name, SSH host +// keys, machine-id, password, authorized keys). Nodes from current base images +// received it on their first boot; nodes from base images built before the +// first-boot hook are personalized over SSH and rebooted. func (c *Cluster) personalizeAll(ctx context.Context) error { pub, err := c.Dir.ReadSecret(secretKey + ".pub") if err != nil { @@ -76,19 +79,23 @@ func (c *Cluster) personalize(ctx context.Context, name string, keys []string, m ns.Pending, ns.HostKey = false, "" mu.Unlock() } - // A previous run may have personalized the node already (golden key gone). - if s, err := c.dial(ctx, name, c.signer, ""); err == nil { - out, rerr := s.Run("cat " + golden.IdentityMarker + " 2>/dev/null || true") - s.Close() - if rerr == nil && strings.TrimSpace(out) == name { - done() - return c.finishIdentity(ctx, name) - } + // The cluster key works once the node has its identity (first-boot hook or a + // previous run); until then only the base image's key does. + s, err := c.waitSSH(ctx, name, 5*time.Minute) + if err != nil { + return err } - s, err := c.waitSSH(ctx, name, 5*time.Minute) // golden key while pending + out, err := s.Run("cat " + golden.IdentityMarker + " 2>/dev/null || true") if err != nil { + s.Close() return err } + if strings.TrimSpace(out) == name { + s.Close() + done() + return c.finishIdentity(ctx, name) + } + // A base image from before the first-boot hook: personalize over SSH. bootID, err := s.Run("cat /proc/sys/kernel/random/boot_id") if err == nil { _, err = s.Run(golden.Personalize(name, c.Cfg.Proxmox.Domain, c.password, keys)) diff --git a/internal/cluster/install.go b/internal/cluster/install.go index 018c73d..5770434 100644 --- a/internal/cluster/install.go +++ b/internal/cluster/install.go @@ -91,11 +91,33 @@ func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node, return c.runtime.Install(ctx, spec, vm.Media{ISO: img.ISO, Kernel: img.Kernel, Initrd: img.Initrd, Cmdline: img.Cmdline, AnswerDir: ansDir}) } +// startNode starts node n. A node cloned from a base image that has no identity yet +// gets it as a guest file, which the first-boot hook applies. +func (c *Cluster) startNode(ctx context.Context, n config.Node) error { + spec := c.spec(n) + if !c.St.Node(n.Name).Pending || c.runtime.Running(spec) { + return c.runtime.Start(ctx, spec) + } + pub, err := c.Dir.ReadSecret(secretKey + ".pub") + if err != nil { + return err + } + keys := append([]string{pub}, c.Cfg.Proxmox.SSHKeys...) + path := filepath.Join(spec.RunDir, n.Name+"-identity") + if err := os.WriteFile(path, firstboot.Identity(n.Name, c.Cfg.Proxmox.Domain, c.password, keys), 0o600); err != nil { + return err + } + // QEMU reads the file at start; it holds the root password, so drop it then. + defer os.Remove(path) + spec.Files = append(spec.Files, vm.File{Name: firstboot.FileIdentity, Path: path}) + return c.runtime.Start(ctx, spec) +} + // bootAll starts every node and waits for SSH. func (c *Cluster) bootAll(ctx context.Context) error { c.step("booting nodes") for _, n := range c.Cfg.NodeList() { - if err := c.runtime.Start(ctx, c.spec(n)); err != nil { + if err := c.startNode(ctx, n); err != nil { return err } } diff --git a/internal/cluster/node.go b/internal/cluster/node.go index 14bfcd9..2edc088 100644 --- a/internal/cluster/node.go +++ b/internal/cluster/node.go @@ -64,10 +64,16 @@ func (c *Cluster) dial(ctx context.Context, name string, signer ssh.Signer, host } // ssh connects to a node, trusting its host key on first use. Nodes cloned from -// a base image are reached with the base image's key until personalized. +// a base image are not pinned until personalized: they get the cluster key on +// their first boot (first-boot hook), or, from base images older than the hook, +// accept only the base image's key until personalized over SSH. func (c *Cluster) ssh(ctx context.Context, name string) (*remote.SSH, error) { ns := c.St.Node(name) if ns.Pending { + s, err := c.dial(ctx, name, c.signer, "") + if err == nil || !isAuthError(err) { + return s, err + } base, err := golden.Open(ns.Base) if err != nil { return nil, retry.Permanent(err) @@ -182,3 +188,9 @@ func (h host) SSH(ctx context.Context, node string) (*remote.SSH, error) { } func (h host) Step(format string, a ...any) { h.c.step(format, a...) } + +// isAuthError reports whether an SSH dial failed because the key was rejected +// (as opposed to the node not being up yet). +func isAuthError(err error) bool { + return strings.Contains(err.Error(), "unable to authenticate") +} diff --git a/internal/golden/golden.go b/internal/golden/golden.go index f63f8d7..f423143 100644 --- a/internal/golden/golden.go +++ b/internal/golden/golden.go @@ -1,10 +1,14 @@ // Package golden caches installed base images so new nodes skip the installer. // -// A base image is an installed but never booted root disk, one per key (ISO, -// root filesystem and size, installer locale). Nodes use qcow2 overlays on top of -// it. Everything Proxmox VE generates on first boot (cluster CA, auth key, node -// certificate, root SSH key) is therefore unique per node; what the installer -// wrote (host name, SSH host keys, machine-id) is replaced by Personalize. +// A base image is an installed root disk, one per key (ISO, root filesystem and +// size, installer locale, baked Ceph release). Nodes use qcow2 overlays on top +// of it. The image is booted once to bake it (package firstboot), but powers off +// before any Proxmox VE service starts, so everything Proxmox VE generates on +// first boot (cluster CA, auth key, node certificate, root SSH key) is unique per +// node. What the installer wrote (host name, SSH host keys, machine-id) is +// replaced on each clone's first boot by the first-boot hook, before networking +// starts. Base images built before the hook existed are personalized over SSH +// instead (Personalize), which costs a reboot. package golden import ( @@ -20,6 +24,7 @@ import ( "net" "os" "path/filepath" + "regexp" "strings" "syscall" "time" @@ -28,6 +33,7 @@ import ( "github.com/virtbase/proxbase/internal/answer" "github.com/virtbase/proxbase/internal/config" + "github.com/virtbase/proxbase/internal/firstboot" "github.com/virtbase/proxbase/internal/image" "github.com/virtbase/proxbase/internal/state" "github.com/virtbase/proxbase/internal/vm" @@ -36,7 +42,15 @@ import ( // Hostname is the name nodes have before personalization. const Hostname = "golden" -const format = "1" // bump when the build or personalization changes +const format = "2" // bump when the build or personalization changes + +const bakeTimeout = 20 * time.Minute + +// Builder installs and bakes base images. +type Builder interface { + vm.Installer + vm.Booter +} // Base is a cached base image. type Base struct { @@ -50,6 +64,7 @@ type Meta struct { ISO string `json:"iso"` Filesystem string `json:"filesystem"` Size string `json:"size"` + Ceph string `json:"ceph,omitempty"` // baked Ceph release Created time.Time `json:"created"` } @@ -70,10 +85,18 @@ func (b Base) Signer() (ssh.Signer, error) { // Key identifies the base image for a node. func Key(cfg *config.Cluster, n config.Node, iso string) string { h := sha256.Sum256([]byte(strings.Join([]string{format, filepath.Base(iso), n.Spec.RootDisk.Filesystem, - n.Spec.RootDisk.Size, cfg.Proxmox.Keyboard, cfg.Proxmox.Country, cfg.Proxmox.Timezone}, "\n"))) + n.Spec.RootDisk.Size, cfg.Proxmox.Keyboard, cfg.Proxmox.Country, cfg.Proxmox.Timezone, cephRelease(cfg)}, "\n"))) return hex.EncodeToString(h[:8]) } +// cephRelease is the Ceph release baked into base images, if Ceph is used. +func cephRelease(cfg *config.Cluster) string { + if !cfg.CephEnabled() { + return "" + } + return cfg.Storage.Ceph.Version +} + // Open returns an existing base image. func Open(key string) (Base, error) { b := Base{Key: key, Dir: filepath.Join(imagesDir(), key)} @@ -84,7 +107,7 @@ func Open(key string) (Base, error) { } // Ensure returns the base image for node n, installing it first if needed. -func Ensure(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config.Node, img *image.Image, logf func(string, ...any)) (Base, error) { +func Ensure(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, logf func(string, ...any)) (Base, error) { key := Key(cfg, n, img.ISO) if b, err := Open(key); err == nil { return b, nil @@ -103,10 +126,14 @@ func Ensure(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n confi if b, err := Open(key); err == nil { // built by another create meanwhile return b, nil } - logf("building base image %s (Proxmox VE %s, %s %s); later creates reuse it", key, img.Version, n.Spec.RootDisk.Filesystem, n.Spec.RootDisk.Size) + what := fmt.Sprintf("Proxmox VE %s, %s %s", img.Version, n.Spec.RootDisk.Filesystem, n.Spec.RootDisk.Size) + if c := cephRelease(cfg); c != "" { + what += ", Ceph " + c + } + logf("building base image %s (%s); later creates reuse it", key, what) tmp := filepath.Join(imagesDir(), key+".tmp") _ = os.RemoveAll(tmp) - if err := build(ctx, inst, cfg, n, img, key, tmp); err != nil { + if err := build(ctx, inst, cfg, n, img, key, tmp, logf); err != nil { _ = os.RemoveAll(tmp) return Base{}, err } @@ -117,7 +144,7 @@ func Ensure(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n confi return Base{Key: key, Dir: dir}, nil } -func build(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config.Node, img *image.Image, key, dir string) error { +func build(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, key, dir string, logf func(string, ...any)) error { for _, sub := range []string{"run", "logs", "answer"} { if err := os.MkdirAll(filepath.Join(dir, sub), 0o700); err != nil { return err @@ -139,11 +166,16 @@ func build(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config if spec.SSHPort, err = freePort(); err != nil { return err } + hook, err := firstboot.Serve() + if err != nil { + return err + } + defer hook.Close() ans := answer.Render(answer.Params{ FQDN: Hostname + ".proxbase.invalid", Mailto: "root@proxbase.invalid", Keyboard: cfg.Proxmox.Keyboard, Country: cfg.Proxmox.Country, Timezone: cfg.Proxmox.Timezone, RootPassword: answer.Password(24), SSHKeys: []string{pub}, MAC: spec.NATMAC, - Filesystem: n.Spec.RootDisk.Filesystem, + Filesystem: n.Spec.RootDisk.Filesystem, FirstBootURL: hook.URL, }) if err := os.WriteFile(filepath.Join(dir, "answer", "answer.toml"), []byte(ans), 0o600); err != nil { return err @@ -153,14 +185,56 @@ func build(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config return fmt.Errorf("base image: %w", err) } _ = os.RemoveAll(filepath.Join(dir, "answer")) + ceph := cephRelease(cfg) + if err := bake(ctx, inst, spec, ceph, logf); err != nil { + return err + } _ = os.RemoveAll(filepath.Join(dir, "run")) - meta, err := json.MarshalIndent(Meta{Key: key, ISO: filepath.Base(img.ISO), Filesystem: n.Spec.RootDisk.Filesystem, Size: n.Spec.RootDisk.Size, Created: time.Now().UTC()}, "", " ") + meta, err := json.MarshalIndent(Meta{Key: key, ISO: filepath.Base(img.ISO), Filesystem: n.Spec.RootDisk.Filesystem, Size: n.Spec.RootDisk.Size, Ceph: ceph, Created: time.Now().UTC()}, "", " ") if err != nil { return err } return os.WriteFile(filepath.Join(dir, "meta.json"), meta, 0o644) } +// bake boots the installed image once with the bake script; the first-boot hook +// runs it and powers off, reporting the result on the serial console. +func bake(ctx context.Context, b vm.Booter, spec vm.Spec, ceph string, logf func(string, ...any)) error { + if ceph != "" { + logf("baking base image: installing Ceph %s", ceph) + } + script := filepath.Join(spec.RunDir, "bake.sh") + if err := os.WriteFile(script, []byte(firstboot.Bake(ceph, spec.NATMAC)), 0o600); err != nil { + return err + } + spec.Files = []vm.File{{Name: firstboot.FileBake, Path: script}} + console, err := b.BootOnce(ctx, spec, bakeTimeout) + if err != nil { + return fmt.Errorf("bake base image: %w", err) + } + if !strings.Contains(string(console), firstboot.BakeOK) { + return fmt.Errorf("bake base image failed; serial console:\n%s", lastLines(string(console), 30)) + } + return nil +} + +// ansiRe matches terminal control sequences (GRUB draws its menu with them). +var ansiRe = regexp.MustCompile(`\x1b(\[[0-9;?]*[ -/]*[@-~]|\][^\x07]*\x07|[()][0-9A-Za-z])`) + +// lastLines returns the last n non-empty lines of console output, indented. +func lastLines(s string, n int) string { + var lines []string + for _, l := range strings.Split(ansiRe.ReplaceAllString(strings.ReplaceAll(s, "\r", ""), ""), "\n") { + if strings.TrimSpace(l) != "" { + lines = append(lines, l) + } + } + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return " " + strings.Join(lines, "\n ") +} + // writeKey creates the login key of nodes before personalization. func writeKey(dir string) (string, error) { pub, priv, err := ed25519.GenerateKey(rand.Reader) diff --git a/internal/golden/golden_test.go b/internal/golden/golden_test.go index a8e7a65..4fc59fe 100644 --- a/internal/golden/golden_test.go +++ b/internal/golden/golden_test.go @@ -21,6 +21,14 @@ func TestKey(t *testing.T) { if Key(cfg, n[0], "/c/proxmox-ve_9.2-2.iso") == k1 { t.Fatal("a different ISO needs a different key") } + ceph, _ := config.Decode(strings.NewReader("nodes: {count: 3}\nstorage: {ceph: {enabled: true}}\n")) + ceph.SetDefaults() + if !ceph.CephEnabled() { + t.Fatal("test config must enable Ceph") + } + if Key(ceph, ceph.NodeList()[0], "/c/proxmox-ve_9.2-1.iso") == k1 { + t.Fatal("a base image with Ceph baked in needs a different key") + } } func TestPersonalize(t *testing.T) { @@ -41,3 +49,14 @@ func TestPersonalize(t *testing.T) { t.Error("Finish must remove the base image's node directory") } } + +func TestLastLines(t *testing.T) { + console := "\x1b[H\x1b[J\x1b[1;1H\x1b[?25l\x1b[m GNU GRUB version 2.12\r\n\x1b[5;78H\r\n" + + " Booting `Proxmox VE GNU/Linux'\r\n\r\n\x1b]104\x07\x1b[?7herror: Another instance of this program is already running.\r\n" + + "proxbase-first-boot: bake failed\r\n" + got := lastLines(console, 2) + want := " error: Another instance of this program is already running.\n proxbase-first-boot: bake failed" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} diff --git a/internal/golden/personalize.go b/internal/golden/personalize.go index 42336a1..8613c0d 100644 --- a/internal/golden/personalize.go +++ b/internal/golden/personalize.go @@ -3,14 +3,17 @@ package golden import ( "fmt" "strings" + + "github.com/virtbase/proxbase/internal/firstboot" ) -// IdentityMarker exists on nodes that were personalized. -const IdentityMarker = "/etc/proxbase-identity" +// IdentityMarker exists on nodes that were personalized; it holds the node name. +const IdentityMarker = firstboot.IdentityMarker -// Personalize returns the script that gives a node cloned from a base image its -// own identity. The node must reboot afterwards (Proxmox services read the host -// name at start); Finish completes it after the reboot. +// Personalize returns the script that gives a node cloned from a base image of +// format 1 (built before the first-boot hook) its own identity. The node must +// reboot afterwards (Proxmox services read the host name at start); Finish +// completes it after the reboot. func Personalize(node, domain, password string, authorizedKeys []string) string { fqdn := node + "." + domain return fmt.Sprintf(` @@ -18,6 +21,7 @@ node=%[1]s fqdn=%[2]s hostname "$node" echo "$node" > /etc/hostname sed -i "/^10\.0\.2\.15[[:space:]]/c\10.0.2.15 $fqdn $node" /etc/hosts +sed -i "s/^search .*/search %[6]s/" /etc/resolv.conf echo "$fqdn" > /etc/mailname postconf -e "myhostname=$fqdn" rm -f /etc/ssh/ssh_host_* @@ -28,7 +32,7 @@ echo "root:%[3]s" | chpasswd printf '%%s\n' %[4]s > /etc/pve/priv/authorized_keys echo "$node" > %[5]s systemd-run --on-active=2 systemctl reboot >/dev/null -`, node, fqdn, password, shellQuote(authorizedKeys), IdentityMarker) +`, node, fqdn, password, shellQuote(authorizedKeys), IdentityMarker, domain) } // Finish removes what is left of the base image's identity after the reboot and From c68ccf7953a6dba7b9285f297e612c593141fca9 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:57:31 +0200 Subject: [PATCH 15/19] feat(aptcache): cache the packages proxbase installs on the host A caching HTTP proxy, reached by the nodes as 10.0.2.2, serves the upgrade, Ceph install and base image bake. It only stores .deb files of the Debian and Proxmox repositories; apt runs retry once with no-cache so a broken package is fetched again. Configurable with proxmox.aptCache (default true). --- internal/aptcache/aptcache.go | 256 +++++++++++++++++++++++++++++ internal/aptcache/aptcache_test.go | 242 +++++++++++++++++++++++++++ internal/cluster/cluster.go | 10 ++ internal/cluster/create.go | 10 ++ internal/cluster/golden.go | 2 +- internal/cluster/node.go | 2 + internal/cluster/setup.go | 2 +- internal/config/config.go | 1 + internal/config/defaults.go | 3 + internal/golden/golden.go | 13 +- internal/nodesetup/nodesetup.go | 7 +- internal/storage/ceph/ceph_test.go | 9 + internal/storage/ceph/install.go | 15 +- internal/storage/storage.go | 3 + schema/v1alpha1/cluster.json | 4 + 15 files changed, 565 insertions(+), 14 deletions(-) create mode 100644 internal/aptcache/aptcache.go create mode 100644 internal/aptcache/aptcache_test.go diff --git a/internal/aptcache/aptcache.go b/internal/aptcache/aptcache.go new file mode 100644 index 0000000..659aa41 --- /dev/null +++ b/internal/aptcache/aptcache.go @@ -0,0 +1,256 @@ +// Package aptcache is a caching HTTP proxy for the nodes' apt. Nodes of a +// cluster download the same packages (Ceph, upgrades), and every create downloads +// them again; through the proxy each package is fetched from the mirror once and +// then served from the host's cache. +// +// Only .deb files are cached: their names are versioned, so the content never +// changes, and apt checks their hashes against the signed repository index. All +// other requests (indexes, Release files) go to the mirror unchanged. Requests +// are limited to the Debian and Proxmox repository hosts. +// +// A package that apt rejects (Hash Sum mismatch: a mirror served a broken file) +// is fetched again when apt asks with Cache-Control: no-cache +// (-o Acquire::http::No-Cache=true), which proxbase's apt runs do on a retry. +package aptcache + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "math/rand/v2" + "net" + "net/http" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "golang.org/x/sync/singleflight" +) + +// Hosts are the repository hosts the proxy serves. +var Hosts = []string{"download.proxmox.com", "deb.debian.org", "security.debian.org"} + +// maxAge is how long a cached package is kept after it was last used. +const maxAge = 30 * 24 * time.Hour + +// hostAddr is the host as seen from a guest on QEMU user networking. +const hostAddr = "10.0.2.2" + +// Proxy is a running apt cache. +type Proxy struct { + root *os.Root // the cache directory; no file access can leave it + srv *http.Server + client *http.Client + inflight singleflight.Group + // URL is the proxy's address as seen from the nodes (for http_proxy). + URL string + // Addr is the address the proxy listens on, on the host. + Addr string +} + +// Start serves the cache in dir on a free port of the host's loopback address, +// which nodes on QEMU user networking reach as 10.0.2.2. It first removes +// packages unused for 30 days. +func Start(dir string) (*Proxy, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + root, err := os.OpenRoot(dir) + if err != nil { + return nil, fmt.Errorf("apt cache: %w", err) + } + prune(root, time.Now().Add(-maxAge)) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + root.Close() + return nil, fmt.Errorf("apt cache: %w", err) + } + p := &Proxy{ + root: root, + client: &http.Client{Transport: &http.Transport{Proxy: http.ProxyFromEnvironment, IdleConnTimeout: 30 * time.Second}}, + URL: "http://" + net.JoinHostPort(hostAddr, strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)), + Addr: ln.Addr().String(), + } + p.srv = &http.Server{Handler: p, ReadHeaderTimeout: 30 * time.Second} + // Serve returns when Close is called; apt reports any other failure itself. + go func() { _ = p.srv.Serve(ln) }() + return p, nil +} + +// Close stops the proxy. +func (p *Proxy) Close() error { return errors.Join(p.srv.Close(), p.root.Close()) } + +// ServeHTTP handles proxy requests ("GET http://host/path"). +func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "only GET and HEAD", http.StatusMethodNotAllowed) + return + } + u := r.URL + if u.Scheme != "http" || !slices.Contains(Hosts, u.Hostname()) || (u.Port() != "" && u.Port() != "80") { + http.Error(w, "not a repository host", http.StatusForbidden) + return + } + clean := path.Clean("/" + u.Path) + if r.Method == http.MethodGet && strings.HasSuffix(clean, ".deb") && u.RawQuery == "" { + p.serveDeb(w, r, u.Hostname(), clean) + return + } + p.pass(w, r) +} + +// pass forwards a request to the mirror and streams the response back. +func (p *Proxy) pass(w http.ResponseWriter, r *http.Request) { + // ServeHTTP admits only plain HTTP to the repository hosts. + req, err := http.NewRequestWithContext(r.Context(), r.Method, r.URL.String(), nil) //nolint:gosec // allow-listed hosts only + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + for _, h := range []string{"If-Modified-Since", "If-None-Match", "Range", "If-Range", "Cache-Control", "User-Agent"} { + if v := r.Header.Get(h); v != "" { + req.Header.Set(h, v) + } + } + resp, err := p.client.Do(req) //nolint:gosec // allow-listed hosts only + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + for _, h := range []string{"Content-Type", "Content-Length", "Last-Modified", "ETag", "Content-Range", "Accept-Ranges"} { + if v := resp.Header.Get(h); v != "" { + w.Header().Set(h, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) +} + +// serveDeb serves a package from the cache, downloading it first if needed. +func (p *Proxy) serveDeb(w http.ResponseWriter, r *http.Request, host, clean string) { + // clean is rooted and cleaned, so the name stays below host/. + name := filepath.Join(host, filepath.FromSlash(strings.TrimPrefix(clean, "/"))) + _, err := p.root.Stat(name) + if err != nil || noCache(r) { + // Concurrent requests for the same package share one download. It must + // not end when the first requester goes away. + ctx := context.WithoutCancel(r.Context()) + _, err, _ := p.inflight.Do(name, func() (any, error) { + return nil, p.download(ctx, "http://"+host+clean, name) + }) + if err != nil { + var se statusError + if errors.As(err, &se) { + http.Error(w, err.Error(), int(se)) + return + } + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + } + f, err := p.root.Open(name) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + now := time.Now() + _ = p.root.Chtimes(name, now, now) // last use, for pruning + w.Header().Set("Content-Type", "application/vnd.debian.binary-package") + http.ServeContent(w, r, "", fi.ModTime(), f) +} + +// statusError is a non-200 status from the mirror. +type statusError int + +func (e statusError) Error() string { return "mirror answered " + http.StatusText(int(e)) } + +// download fetches url into the cache as name, atomically and only if complete. +func (p *Proxy) download(ctx context.Context, url, name string) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) //nolint:gosec // allow-listed hosts only + if err != nil { + return err + } + resp, err := p.client.Do(req) //nolint:gosec // allow-listed hosts only + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return statusError(resp.StatusCode) + } + if err := p.root.MkdirAll(filepath.Dir(name), 0o755); err != nil { + return err + } + tmp := filepath.Join(filepath.Dir(name), fmt.Sprintf(".part-%d", rand.Uint64())) //nolint:gosec // a unique name, not a secret + f, err := p.root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + defer func() { _ = p.root.Remove(tmp) }() // gone after the rename + n, err := io.Copy(f, resp.Body) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return fmt.Errorf("download %s: %w", url, err) + } + if resp.ContentLength >= 0 && n != resp.ContentLength { + return fmt.Errorf("download %s: got %d of %d bytes", url, n, resp.ContentLength) + } + if err := checkDeb(p.root, tmp); err != nil { + return fmt.Errorf("download %s: %w", url, err) + } + return p.root.Rename(tmp, name) +} + +// noCache reports whether the client asked to bypass caches. +func noCache(r *http.Request) bool { + return strings.Contains(r.Header.Get("Cache-Control"), "no-cache") || r.Header.Get("Pragma") == "no-cache" +} + +// debMagic starts every .deb file (an ar archive). +const debMagic = "!\n" + +// checkDeb rejects files that are not .deb packages, such as error pages. +func checkDeb(root *os.Root, name string) error { + f, err := root.Open(name) + if err != nil { + return err + } + defer f.Close() + head := make([]byte, len(debMagic)) + if _, err := io.ReadFull(f, head); err != nil || string(head) != debMagic { + return errors.New("not a Debian package") + } + return nil +} + +// prune removes cached packages last used before cutoff, and leftovers of +// interrupted downloads. +func prune(root *os.Root, cutoff time.Time) { + _ = fs.WalkDir(root.FS(), ".", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + info, err := d.Info() + if err == nil && (info.ModTime().Before(cutoff) || strings.HasPrefix(d.Name(), ".part-")) { + _ = root.Remove(p) + } + return nil + }) +} diff --git a/internal/aptcache/aptcache_test.go b/internal/aptcache/aptcache_test.go new file mode 100644 index 0000000..ecff609 --- /dev/null +++ b/internal/aptcache/aptcache_test.go @@ -0,0 +1,242 @@ +package aptcache + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// mirror is a fake repository that counts requests per path. +type mirror struct { + srv *httptest.Server + mu sync.Mutex + hits map[string]int + gate chan struct{} // if set, .deb responses wait for it +} + +func newMirror(t *testing.T) *mirror { + t.Helper() + m := &mirror{hits: map[string]int{}} + m.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + m.hits[r.URL.Path]++ + gate := m.gate + m.mu.Unlock() + switch r.URL.Path { + case "/missing.deb": + http.NotFound(w, r) + return + case "/html.deb": + _, _ = io.WriteString(w, "captive portal") + return + } + if gate != nil { + <-gate + } + body := "content of " + r.URL.Path + if strings.HasSuffix(r.URL.Path, ".deb") { + body = debMagic + body + } + _, _ = io.WriteString(w, body) //nolint:gosec // test server, plain text + })) + t.Cleanup(m.srv.Close) + return m +} + +func (m *mirror) count(p string) int { + m.mu.Lock() + defer m.mu.Unlock() + return m.hits[p] +} + +// start runs a proxy whose upstream connections all go to the fake mirror. +func start(t *testing.T, m *mirror) (*Proxy, *http.Client) { + t.Helper() + p, err := Start(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { p.Close() }) + mirrorAddr := m.srv.Listener.Addr().String() + p.client = &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, network, mirrorAddr) + }, + }} + proxyURL, err := url.Parse("http://" + p.Addr) + if err != nil { + t.Fatal(err) + } + return p, &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}, Timeout: 10 * time.Second} +} + +func fetch(ctx context.Context, c *http.Client, u string) (int, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return 0, "", err + } + resp, err := c.Do(req) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + return resp.StatusCode, string(b), err +} + +func get(t *testing.T, c *http.Client, u string) (int, string) { + t.Helper() + code, body, err := fetch(t.Context(), c, u) + if err != nil { + t.Fatal(err) + } + return code, body +} + +func TestProxy_CachesPackages(t *testing.T) { + t.Parallel() + m := newMirror(t) + _, c := start(t, m) + const deb = "http://deb.debian.org/debian/pool/main/c/ceph/ceph_19.2.3_amd64.deb" + for range 2 { + code, body := get(t, c, deb) + if code != http.StatusOK || body != debMagic+"content of /debian/pool/main/c/ceph/ceph_19.2.3_amd64.deb" { + t.Fatalf("got %d %q", code, body) + } + } + if n := m.count("/debian/pool/main/c/ceph/ceph_19.2.3_amd64.deb"); n != 1 { + t.Fatalf("mirror asked %d times, want 1", n) + } +} + +// apt asks with no-cache after a Hash Sum mismatch; the package is fetched again. +func TestProxy_NoCacheRefetches(t *testing.T) { + t.Parallel() + m := newMirror(t) + _, c := start(t, m) + const deb = "http://deb.debian.org/debian/pool/main/libn/libnvme/libnvme1t64_1.13-2_amd64.deb" + get(t, c, deb) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, deb, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Cache-Control", "no-cache") + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status %d", resp.StatusCode) + } + if n := m.count("/debian/pool/main/libn/libnvme/libnvme1t64_1.13-2_amd64.deb"); n != 2 { + t.Fatalf("mirror asked %d times, want 2", n) + } +} + +func TestProxy_PassesIndexes(t *testing.T) { + t.Parallel() + m := newMirror(t) + _, c := start(t, m) + for range 2 { + if code, _ := get(t, c, "http://download.proxmox.com/debian/pve/dists/trixie/InRelease"); code != http.StatusOK { + t.Fatalf("status %d", code) + } + } + if n := m.count("/debian/pve/dists/trixie/InRelease"); n != 2 { + t.Fatalf("indexes must not be cached: mirror asked %d times, want 2", n) + } +} + +func TestProxy_SharesConcurrentDownloads(t *testing.T) { + t.Parallel() + m := newMirror(t) + m.gate = make(chan struct{}) + _, c := start(t, m) + const deb = "http://download.proxmox.com/debian/ceph-squid/pool/x.deb" + var wg sync.WaitGroup + var ok atomic.Int32 + for range 5 { + wg.Go(func() { + if code, _, err := fetch(t.Context(), c, deb); err == nil && code == http.StatusOK { + ok.Add(1) + } + }) + } + // Let the requests arrive, then release the single upstream download. + for m.count("/debian/ceph-squid/pool/x.deb") == 0 { + time.Sleep(10 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + close(m.gate) + wg.Wait() + if ok.Load() != 5 { + t.Fatalf("%d of 5 requests succeeded", ok.Load()) + } + if n := m.count("/debian/ceph-squid/pool/x.deb"); n != 1 { + t.Fatalf("mirror asked %d times, want 1", n) + } +} + +func TestProxy_Rejects(t *testing.T) { + t.Parallel() + m := newMirror(t) + _, c := start(t, m) + tests := []struct { + name string + url string + want int + }{ + {name: "other host", url: "http://example.com/x.deb", want: http.StatusForbidden}, + {name: "other port", url: "http://deb.debian.org:8080/x.deb", want: http.StatusForbidden}, + {name: "missing package", url: "http://deb.debian.org/missing.deb", want: http.StatusNotFound}, + {name: "not a package", url: "http://deb.debian.org/html.deb", want: http.StatusBadGateway}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if code, _ := get(t, c, tt.url); code != tt.want { + t.Errorf("status %d, want %d", code, tt.want) + } + }) + } +} + +func TestPrune(t *testing.T) { + t.Parallel() + dir := t.TempDir() + old, recent, part := filepath.Join(dir, "a", "old.deb"), filepath.Join(dir, "a", "new.deb"), filepath.Join(dir, "a", ".part-1") + if err := os.MkdirAll(filepath.Dir(old), 0o755); err != nil { + t.Fatal(err) + } + for _, p := range []string{old, recent, part} { + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + past := time.Now().Add(-2 * maxAge) + if err := os.Chtimes(old, past, past); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + prune(root, time.Now().Add(-maxAge)) + for p, want := range map[string]bool{old: false, recent: true, part: false} { + if _, err := os.Stat(p); (err == nil) != want { + t.Errorf("%s exists=%v, want %v", filepath.Base(p), err == nil, want) + } + } +} diff --git a/internal/cluster/cluster.go b/internal/cluster/cluster.go index 153c949..b8a647f 100644 --- a/internal/cluster/cluster.go +++ b/internal/cluster/cluster.go @@ -17,6 +17,7 @@ import ( "golang.org/x/crypto/ssh" "github.com/virtbase/proxbase/internal/answer" + "github.com/virtbase/proxbase/internal/aptcache" "github.com/virtbase/proxbase/internal/config" "github.com/virtbase/proxbase/internal/network" "github.com/virtbase/proxbase/internal/progress" @@ -50,6 +51,7 @@ type Cluster struct { password string runtime vm.Runtime net network.Switch + apt *aptcache.Proxy // package cache for the nodes while create runs, or nil } func newCluster(d state.Dir, cfg *config.Cluster, st *state.State, p progress.Sink) (*Cluster, error) { @@ -183,6 +185,14 @@ func (c *Cluster) step(format string, a ...any) { func (c *Cluster) save() error { return c.Dir.Save(c.St) } +// aptEnv points apt on the nodes at the package cache, if one runs. +func (c *Cluster) aptEnv() string { + if c.apt == nil { + return "" + } + return "export http_proxy=" + c.apt.URL + "\n" +} + // storages returns the configured storage backends in setup order. func (c *Cluster) storages() []storage.Backend { var out []storage.Backend diff --git a/internal/cluster/create.go b/internal/cluster/create.go index 5381212..bfd9b9f 100644 --- a/internal/cluster/create.go +++ b/internal/cluster/create.go @@ -10,6 +10,7 @@ import ( "strconv" "time" + "github.com/virtbase/proxbase/internal/aptcache" "github.com/virtbase/proxbase/internal/config" "github.com/virtbase/proxbase/internal/image" "github.com/virtbase/proxbase/internal/network" @@ -68,6 +69,15 @@ func (c *Cluster) create(ctx context.Context) error { if err := c.save(); err != nil { return err } + if *c.Cfg.Proxmox.AptCache { + if c.apt, err = aptcache.Start(filepath.Join(state.CacheHome(), "apt")); err != nil { + return err + } + defer func() { + c.apt.Close() + c.apt = nil + }() + } steps := []func(context.Context) error{ func(context.Context) error { return c.net.Start(c.Cfg) }, func(ctx context.Context) error { return c.installAll(ctx, img) }, diff --git a/internal/cluster/golden.go b/internal/cluster/golden.go index a4b229b..49c838a 100644 --- a/internal/cluster/golden.go +++ b/internal/cluster/golden.go @@ -28,7 +28,7 @@ func (c *Cluster) cloneAll(ctx context.Context, img *image.Image, todo []config. t0 := time.Now() var names []string for _, n := range todo { - base, err := golden.Ensure(ctx, builder, c.Cfg, n, img, c.step) + base, err := golden.Ensure(ctx, builder, c.Cfg, n, img, c.aptEnv(), c.step) if err != nil { return err } diff --git a/internal/cluster/node.go b/internal/cluster/node.go index 2edc088..1dbb830 100644 --- a/internal/cluster/node.go +++ b/internal/cluster/node.go @@ -189,6 +189,8 @@ func (h host) SSH(ctx context.Context, node string) (*remote.SSH, error) { func (h host) Step(format string, a ...any) { h.c.step(format, a...) } +func (h host) AptEnv() string { return h.c.aptEnv() } + // isAuthError reports whether an SSH dial failed because the key was rejected // (as opposed to the node not being up yet). func isAuthError(err error) bool { diff --git a/internal/cluster/setup.go b/internal/cluster/setup.go index 3bdf0ff..a2561a7 100644 --- a/internal/cluster/setup.go +++ b/internal/cluster/setup.go @@ -44,7 +44,7 @@ func (c *Cluster) configureNodes(ctx context.Context, provisioning bool) error { return nil } t0 := time.Now() - out, err = s.Run(nodesetup.Upgrade) + out, err = s.Run(c.aptEnv() + nodesetup.Upgrade) if err != nil { return fmt.Errorf("%s: dist-upgrade: %w", n.Name, err) } diff --git a/internal/config/config.go b/internal/config/config.go index eff8fc6..22efdd3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -42,6 +42,7 @@ type Proxmox struct { SSHKeys []string `yaml:"sshKeys,omitempty" json:"sshKeys,omitempty" jsonschema:"description=Extra public keys for root on every node"` Upgrade bool `yaml:"upgrade,omitempty" json:"upgrade,omitempty" jsonschema:"description=Run apt dist-upgrade on every node before clustering"` Golden bool `yaml:"goldenImage,omitempty" json:"goldenImage,omitempty" jsonschema:"description=Clone nodes from a cached installed base image instead of running the installer"` + AptCache *bool `yaml:"aptCache,omitempty" json:"aptCache,omitempty" jsonschema:"description=Download packages that proxbase installs (upgrade and Ceph) once through a cache on the host; default true"` } type Nodes struct { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index f84aa97..cc66705 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -19,6 +19,9 @@ func (c *Cluster) SetDefaults() { def(&p.Keyboard, "en-us") def(&p.Country, "us") def(&p.Domain, "proxbase.internal") + if p.AptCache == nil { + p.AptCache = ptr(true) + } n := &c.Nodes if n.Count == 0 { diff --git a/internal/golden/golden.go b/internal/golden/golden.go index f423143..76ddc9d 100644 --- a/internal/golden/golden.go +++ b/internal/golden/golden.go @@ -107,7 +107,8 @@ func Open(key string) (Base, error) { } // Ensure returns the base image for node n, installing it first if needed. -func Ensure(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, logf func(string, ...any)) (Base, error) { +// aptEnv is put before the bake script's apt commands (see storage.Host.AptEnv). +func Ensure(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, aptEnv string, logf func(string, ...any)) (Base, error) { key := Key(cfg, n, img.ISO) if b, err := Open(key); err == nil { return b, nil @@ -133,7 +134,7 @@ func Ensure(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Nod logf("building base image %s (%s); later creates reuse it", key, what) tmp := filepath.Join(imagesDir(), key+".tmp") _ = os.RemoveAll(tmp) - if err := build(ctx, inst, cfg, n, img, key, tmp, logf); err != nil { + if err := build(ctx, inst, cfg, n, img, key, tmp, aptEnv, logf); err != nil { _ = os.RemoveAll(tmp) return Base{}, err } @@ -144,7 +145,7 @@ func Ensure(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Nod return Base{Key: key, Dir: dir}, nil } -func build(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, key, dir string, logf func(string, ...any)) error { +func build(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, key, dir, aptEnv string, logf func(string, ...any)) error { for _, sub := range []string{"run", "logs", "answer"} { if err := os.MkdirAll(filepath.Join(dir, sub), 0o700); err != nil { return err @@ -186,7 +187,7 @@ func build(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node } _ = os.RemoveAll(filepath.Join(dir, "answer")) ceph := cephRelease(cfg) - if err := bake(ctx, inst, spec, ceph, logf); err != nil { + if err := bake(ctx, inst, spec, ceph, aptEnv, logf); err != nil { return err } _ = os.RemoveAll(filepath.Join(dir, "run")) @@ -199,12 +200,12 @@ func build(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node // bake boots the installed image once with the bake script; the first-boot hook // runs it and powers off, reporting the result on the serial console. -func bake(ctx context.Context, b vm.Booter, spec vm.Spec, ceph string, logf func(string, ...any)) error { +func bake(ctx context.Context, b vm.Booter, spec vm.Spec, ceph, aptEnv string, logf func(string, ...any)) error { if ceph != "" { logf("baking base image: installing Ceph %s", ceph) } script := filepath.Join(spec.RunDir, "bake.sh") - if err := os.WriteFile(script, []byte(firstboot.Bake(ceph, spec.NATMAC)), 0o600); err != nil { + if err := os.WriteFile(script, []byte(aptEnv+firstboot.Bake(ceph, spec.NATMAC)), 0o600); err != nil { return err } spec.Files = []vm.File{{Name: firstboot.FileBake, Path: script}} diff --git a/internal/nodesetup/nodesetup.go b/internal/nodesetup/nodesetup.go index 71805b6..c9dcedf 100644 --- a/internal/nodesetup/nodesetup.go +++ b/internal/nodesetup/nodesetup.go @@ -133,9 +133,12 @@ rm -f /etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io // AptWait waits for apt runs started by PVE itself (pveupdate right after boot) // and sets $apt to an apt-get that waits for locks and retries downloads. +// retry_apt runs an apt command again past any cache if it fails: a broken +// download (Hash Sum mismatch) is then fetched anew, also by the host's cache. const AptWait = ` -for i in $(seq 300); do pgrep -x 'apt-get|apt|dpkg|unattended-upgr' >/dev/null || break; sleep 2; done +for _ in $(seq 300); do pgrep -x 'apt-get|apt|dpkg|unattended-upgr' >/dev/null || break; sleep 2; done apt='apt-get -o DPkg::Lock::Timeout=600 -o Acquire::Retries=3' +retry_apt() { "$@" || "$@" -o Acquire::http::No-Cache=true; } ` // Upgrade brings a node to the newest no-subscription packages and prints the @@ -143,7 +146,7 @@ apt='apt-get -o DPkg::Lock::Timeout=600 -o Acquire::Retries=3' const Upgrade = AptWait + ` export DEBIAN_FRONTEND=noninteractive log=/var/log/proxbase-upgrade.log -if ! { $apt update && $apt -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold dist-upgrade; } >$log 2>&1; then +if ! { $apt update && retry_apt $apt -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold dist-upgrade; } >$log 2>&1; then tail -n 20 $log >&2; exit 1 fi grep -E '^[0-9]+ upgraded' $log || true diff --git a/internal/storage/ceph/ceph_test.go b/internal/storage/ceph/ceph_test.go index bfb5138..3e2716d 100644 --- a/internal/storage/ceph/ceph_test.go +++ b/internal/storage/ceph/ceph_test.go @@ -54,3 +54,12 @@ func TestMemoryTargetScript(t *testing.T) { }) } } + +func TestInstallScriptSyntax(t *testing.T) { + t.Parallel() + cmd := exec.CommandContext(t.Context(), "bash", "-n") + cmd.Stdin = strings.NewReader(installScript("squid")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bash -n: %v\n%s", err, out) + } +} diff --git a/internal/storage/ceph/install.go b/internal/storage/ceph/install.go index 37b132f..76c6f35 100644 --- a/internal/storage/ceph/install.go +++ b/internal/storage/ceph/install.go @@ -19,14 +19,21 @@ func installScript(version string) string { return nodesetup.AptWait + fmt.Sprintf(` export DEBIAN_FRONTEND=noninteractive log=/var/log/proxbase-ceph-install.log +pveceph_install() { { yes || true; } | pveceph install --repository no-subscription --version %[1]s >>$log 2>&1; } if ! [ -x /usr/bin/ceph-mon ] || ! ceph-mon --version 2>/dev/null | grep -q ' %[1]s '; then - if ! { yes || true; } | pveceph install --repository no-subscription --version %[1]s >$log 2>&1; then - tail -n 20 $log >&2; exit 1 + if ! pveceph_install; then + # pveceph takes no apt options: retry past caches through apt's config. + nocache=/etc/apt/apt.conf.d/99proxbase-no-cache + echo 'Acquire::http::No-Cache "true";' > $nocache + if ! pveceph_install; then + rm -f $nocache; tail -n 20 $log >&2; exit 1 + fi + rm -f $nocache fi echo installed fi before=$(dpkg-query -W -f '${Version}' libpve-storage-perl) -if ! $apt install -y --only-upgrade libpve-storage-perl >>$log 2>&1; then +if ! retry_apt $apt install -y --only-upgrade libpve-storage-perl >>$log 2>&1; then tail -n 20 $log >&2; exit 1 fi if [ "$before" != "$(dpkg-query -W -f '${Version}' libpve-storage-perl)" ]; then @@ -38,7 +45,7 @@ fi // install runs the install script on all nodes in parallel. func (b *Backend) install(ctx context.Context, h storage.Host) error { - script := installScript(b.cfg.Version) + script := h.AptEnv() + installScript(b.cfg.Version) g, gctx := errgroup.WithContext(ctx) for _, n := range h.Nodes() { g.Go(func() error { diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 6a7b4b4..00d2df6 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -21,6 +21,9 @@ type Host interface { SSH(ctx context.Context, node string) (*remote.SSH, error) // Step reports progress. Step(format string, a ...any) + // AptEnv is a shell snippet to put before scripts that run apt on the nodes: + // it points apt at the host's package cache, if there is one. + AptEnv() string } // Backend sets up one kind of storage. Setup must be idempotent: it runs on diff --git a/schema/v1alpha1/cluster.json b/schema/v1alpha1/cluster.json index 134037e..06cf624 100644 --- a/schema/v1alpha1/cluster.json +++ b/schema/v1alpha1/cluster.json @@ -279,6 +279,10 @@ "goldenImage": { "type": "boolean", "description": "Clone nodes from a cached installed base image instead of running the installer" + }, + "aptCache": { + "type": "boolean", + "description": "Download packages that proxbase installs (upgrade and Ceph) once through a cache on the host; default true" } }, "additionalProperties": false, From 5b976af7c7da0b891acdc367fd084f6f0e4a3d94 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:57:45 +0200 Subject: [PATCH 16/19] feat(snapshot): live snapshots of running clusters snapshot save --live (MCP: live) pauses all nodes and saves their disks and memory through QMP snapshot-save. Restoring one starts every node paused from it, resumes them together, sets their clocks and waits for quorum. --- internal/cli/snapshot.go | 27 ++++- internal/cluster/snapshot.go | 180 ++++++++++++++++++++++++++++-- internal/cluster/snapshot_test.go | 145 ++++++++++++++++++++++++ internal/mcpserver/tools.go | 18 ++- internal/qemu/fault.go | 8 +- internal/qemu/live.go | 140 +++++++++++++++++++++++ internal/qemu/qemu.go | 3 + internal/qemu/qemu_test.go | 13 +++ internal/state/state.go | 3 + internal/vm/vm.go | 16 +++ 10 files changed, 528 insertions(+), 25 deletions(-) create mode 100644 internal/cluster/snapshot_test.go create mode 100644 internal/qemu/live.go diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index da0c61c..292e62d 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -13,7 +13,16 @@ import ( ) func snapshotCmd() *cobra.Command { - cmd := &cobra.Command{Use: "snapshot", Short: "Save and restore all disks of a stopped cluster"} + cmd := &cobra.Command{ + Use: "snapshot", + Short: "Save and restore a whole cluster", + Long: `snapshot saves and restores all disks of a cluster at once. + +A snapshot of a stopped cluster holds its disks; after a restore, start the +cluster as usual. A live snapshot (save --live) is taken of the running cluster +and also holds the memory of every node; restoring it starts the cluster in that +state within seconds, without booting.`, + } op := func(use, short string, f func(*cluster.Cluster, context.Context, string) error) *cobra.Command { return &cobra.Command{ Use: use + " [cluster] ", @@ -44,16 +53,24 @@ func snapshotCmd() *cobra.Command { return err } w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "NAME\tCREATED\tNODES") + fmt.Fprintln(w, "NAME\tCREATED\tNODES\tLIVE") for _, s := range c.St.Snapshots { - fmt.Fprintf(w, "%s\t%s\t%d\n", s.Name, s.Created.Local().Format(time.DateTime), len(s.Nodes)) + fmt.Fprintf(w, "%s\t%s\t%d\t%v\n", s.Name, s.Created.Local().Format(time.DateTime), len(s.Nodes), s.Live) } return w.Flush() }, } + var live bool + save := op("save", "Snapshot every disk (cluster stopped), or with --live the running cluster", func(c *cluster.Cluster, ctx context.Context, name string) error { + if live { + return c.SnapshotSaveLive(ctx, name) + } + return c.SnapshotSave(ctx, name) + }) + save.Flags().BoolVar(&live, "live", false, "snapshot the running cluster, memory included; restoring it resumes the cluster") cmd.AddCommand( - op("save", "Snapshot every disk (cluster must be stopped)", (*cluster.Cluster).SnapshotSave), - op("restore", "Reset every disk to a snapshot (cluster must be stopped)", (*cluster.Cluster).SnapshotRestore), + save, + op("restore", "Reset every disk to a snapshot (cluster must be stopped); a live snapshot also starts the cluster", (*cluster.Cluster).SnapshotRestore), op("delete", "Delete a snapshot", (*cluster.Cluster).SnapshotDelete), list, ) diff --git a/internal/cluster/snapshot.go b/internal/cluster/snapshot.go index a7a1ea5..bde5340 100644 --- a/internal/cluster/snapshot.go +++ b/internal/cluster/snapshot.go @@ -2,12 +2,17 @@ package cluster import ( "context" + "errors" "fmt" "regexp" "slices" "strings" + "sync" "time" + "golang.org/x/sync/errgroup" + + "github.com/virtbase/proxbase/internal/config" "github.com/virtbase/proxbase/internal/state" "github.com/virtbase/proxbase/internal/vm" ) @@ -41,13 +46,8 @@ func (c *Cluster) requireStopped() error { return nil } -// SnapshotSave takes a snapshot of every disk of the stopped cluster. -func (c *Cluster) SnapshotSave(ctx context.Context, name string) error { - unlock, err := c.Dir.Lock() - if err != nil { - return err - } - defer unlock() +// checkNewSnapshot validates the name of a snapshot about to be saved. +func (c *Cluster) checkNewSnapshot(name string) error { if !snapRe.MatchString(name) { return fmt.Errorf("invalid snapshot name %q", name) } @@ -57,6 +57,19 @@ func (c *Cluster) SnapshotSave(ctx context.Context, name string) error { if c.St.Snapshot(name) != nil { return fmt.Errorf("snapshot %q exists", name) } + return nil +} + +// SnapshotSave takes a snapshot of every disk of the stopped cluster. +func (c *Cluster) SnapshotSave(ctx context.Context, name string) error { + unlock, err := c.Dir.Lock() + if err != nil { + return err + } + defer unlock() + if err := c.checkNewSnapshot(name); err != nil { + return err + } if err := c.requireStopped(); err != nil { return err } @@ -67,6 +80,68 @@ func (c *Cluster) SnapshotSave(ctx context.Context, name string) error { return c.save() } +// SnapshotSaveLive snapshots the running cluster with its memory. Every node is +// paused until all snapshots are taken, so they show one moment of the cluster. +func (c *Cluster) SnapshotSaveLive(ctx context.Context, name string) error { + unlock, err := c.Dir.Lock() + if err != nil { + return err + } + defer unlock() + if err := c.checkNewSnapshot(name); err != nil { + return err + } + live, ok := c.runtime.(vm.LiveSnapshotter) + if !ok { + return errors.New("the node runtime cannot take live snapshots") + } + if len(c.St.Faults) > 0 { + return fmt.Errorf("cluster %s has active faults; clear them first (proxbase fault clear %s)", c.Cfg.Name, c.Cfg.Name) + } + nodes := c.Cfg.NodeList() + for _, n := range nodes { + if !c.running(n) { + return fmt.Errorf("%s is not running; live snapshots need a running cluster (proxbase start %s)", n.Name, c.Cfg.Name) + } + } + var paused []config.Node + defer func() { + for _, n := range paused { + if err := live.Resume(context.WithoutCancel(ctx), c.spec(n)); err != nil { + c.step("%s: resume failed: %v", n.Name, err) + } + } + }() + for _, n := range nodes { + if err := live.Pause(ctx, c.spec(n)); err != nil { + return fmt.Errorf("%s: pause: %w", n.Name, err) + } + paused = append(paused, n) + } + var mu sync.Mutex + var saved []config.Node + g, gctx := errgroup.WithContext(ctx) + for _, n := range nodes { + g.Go(func() error { + if err := live.SaveLive(gctx, c.spec(n), name); err != nil { + return err + } + mu.Lock() + saved = append(saved, n) + mu.Unlock() + return nil + }) + } + if err := g.Wait(); err != nil { + for _, n := range saved { + _ = live.DeleteLive(context.WithoutCancel(ctx), c.spec(n), name) + } + return err + } + c.St.Snapshots = append(c.St.Snapshots, state.Snapshot{Name: name, Created: time.Now().UTC(), Nodes: c.nodeNames(), Live: true}) + return c.save() +} + // SnapshotRestore resets every disk to the snapshot. func (c *Cluster) SnapshotRestore(ctx context.Context, name string) error { unlock, err := c.Dir.Lock() @@ -84,9 +159,73 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, name string) error { if err := c.requireStopped(); err != nil { return err } + if snap.Live { + return c.resume(ctx, name) + } return c.runtime.Snapshot(ctx, vm.SnapshotRestore, name, c.disks()) } +// resume starts every node in the state of live snapshot name. The nodes load +// it paused and are resumed together, so the cluster continues as one; their +// clocks, which stood still in the snapshot, are set to the host's time. +func (c *Cluster) resume(ctx context.Context, name string) error { + live, ok := c.runtime.(vm.LiveSnapshotter) + if !ok { + return errors.New("the node runtime cannot start from live snapshots") + } + if err := c.net.Start(c.Cfg); err != nil { + return err + } + nodes := c.Cfg.NodeList() + for i, n := range nodes { + spec := c.spec(n) + spec.LoadSnapshot = name + if err := c.runtime.Start(ctx, spec); err != nil { + for _, started := range nodes[:i] { + _ = c.runtime.Stop(context.WithoutCancel(ctx), c.spec(started), 0) + } + return fmt.Errorf("%s: start from snapshot %s: %w", n.Name, name, err) + } + } + for _, n := range nodes { + if err := live.Resume(ctx, c.spec(n)); err != nil { + return fmt.Errorf("%s: resume: %w", n.Name, err) + } + } + c.step("%d node(s) resumed from snapshot %s", len(nodes), name) + if err := c.syncClocks(ctx); err != nil { + return err + } + if err := c.waitQuorum(ctx, 5*time.Minute); err != nil { + return err + } + if err := c.waitAPI(ctx, 2*time.Minute); err != nil { + return err + } + return c.waitStorage(ctx) +} + +// syncClocks sets every node's clock to the host's time. +func (c *Cluster) syncClocks(ctx context.Context) error { + g, gctx := errgroup.WithContext(ctx) + for _, n := range c.Cfg.NodeList() { + g.Go(func() error { + s, err := c.waitSSH(gctx, n.Name, time.Minute) + if err != nil { + return err + } + defer s.Close() + now := time.Now() + script := fmt.Sprintf("date -u -s @%d.%03d >/dev/null\nchronyc -a makestep >/dev/null 2>&1 || true\n", now.Unix(), now.Nanosecond()/1e6) + if _, err := s.Run(script); err != nil { + return fmt.Errorf("%s: set clock: %w", n.Name, err) + } + return nil + }) + } + return g.Wait() +} + // SnapshotDelete removes the snapshot from every disk. func (c *Cluster) SnapshotDelete(ctx context.Context, name string) error { unlock, err := c.Dir.Lock() @@ -94,15 +233,32 @@ func (c *Cluster) SnapshotDelete(ctx context.Context, name string) error { return err } defer unlock() - if c.St.Snapshot(name) == nil { + snap := c.St.Snapshot(name) + if snap == nil { return fmt.Errorf("no snapshot %q", name) } - if err := c.requireStopped(); err != nil { - return err - } - if err := c.runtime.Snapshot(ctx, vm.SnapshotDelete, name, c.disks()); err != nil { + if err := c.deleteSnapshot(ctx, snap); err != nil { return err } c.St.Snapshots = slices.DeleteFunc(c.St.Snapshots, func(s state.Snapshot) bool { return s.Name == name }) return c.save() } + +// deleteSnapshot removes a snapshot from the disks: through the running nodes +// for a live snapshot of a running cluster, otherwise from the stopped disks. +func (c *Cluster) deleteSnapshot(ctx context.Context, snap *state.Snapshot) error { + live, ok := c.runtime.(vm.LiveSnapshotter) + nodes := c.Cfg.NodeList() + allRunning := !slices.ContainsFunc(nodes, func(n config.Node) bool { return !c.running(n) }) + if !snap.Live || !ok || !allRunning { + if err := c.requireStopped(); err != nil { + return err + } + return c.runtime.Snapshot(ctx, vm.SnapshotDelete, snap.Name, c.disks()) + } + g, gctx := errgroup.WithContext(ctx) + for _, n := range nodes { + g.Go(func() error { return live.DeleteLive(gctx, c.spec(n), snap.Name) }) + } + return g.Wait() +} diff --git a/internal/cluster/snapshot_test.go b/internal/cluster/snapshot_test.go new file mode 100644 index 0000000..f2fa5ca --- /dev/null +++ b/internal/cluster/snapshot_test.go @@ -0,0 +1,145 @@ +package cluster + +import ( + "context" + "errors" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/virtbase/proxbase/internal/config" + "github.com/virtbase/proxbase/internal/state" + "github.com/virtbase/proxbase/internal/vm" +) + +// fakeRuntime records live snapshot calls; every node runs. +type fakeRuntime struct { + mu sync.Mutex + calls []string // "pause pve1", "save pve2", ... + failOn string // node whose SaveLive fails + deleted []string +} + +var ( + _ vm.Runtime = (*fakeRuntime)(nil) + _ vm.LiveSnapshotter = (*fakeRuntime)(nil) +) + +func (f *fakeRuntime) record(op string, s vm.Spec) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, op+" "+s.Name) +} + +func (f *fakeRuntime) Install(context.Context, vm.Spec, vm.Media) error { return nil } +func (f *fakeRuntime) Start(context.Context, vm.Spec) error { return nil } +func (f *fakeRuntime) Stop(context.Context, vm.Spec, time.Duration) error { return nil } +func (f *fakeRuntime) Running(vm.Spec) bool { return true } +func (f *fakeRuntime) Snapshot(context.Context, vm.SnapshotOp, string, []vm.Disk) error { return nil } + +func (f *fakeRuntime) Pause(_ context.Context, s vm.Spec) error { f.record("pause", s); return nil } +func (f *fakeRuntime) Resume(_ context.Context, s vm.Spec) error { f.record("resume", s); return nil } + +func (f *fakeRuntime) SaveLive(_ context.Context, s vm.Spec, _ string) error { + f.record("save", s) + if s.Name == f.failOn { + return errors.New("disk full") + } + return nil +} + +func (f *fakeRuntime) DeleteLive(_ context.Context, s vm.Spec, _ string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.deleted = append(f.deleted, s.Name) + return nil +} + +// newTestCluster returns a ready 3-node cluster in a temporary data directory. +func newTestCluster(t *testing.T, rt vm.Runtime) *Cluster { + t.Helper() + t.Setenv("XDG_DATA_HOME", t.TempDir()) + cfg, err := config.Decode(strings.NewReader("name: snap\nnodes: {count: 3}\n")) + if err != nil { + t.Fatal(err) + } + cfg.SetDefaults() + c, err := initialize(cfg, nil) + if err != nil { + t.Fatal(err) + } + c.runtime = rt + c.St.Phase = state.PhaseReady + return c +} + +func (f *fakeRuntime) ops(op string) []string { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for _, c := range f.calls { + if name, ok := strings.CutPrefix(c, op+" "); ok { + out = append(out, name) + } + } + slices.Sort(out) + return out +} + +func TestCluster_SnapshotSaveLive(t *testing.T) { + rt := &fakeRuntime{} + c := newTestCluster(t, rt) + if err := c.SnapshotSaveLive(t.Context(), "ready"); err != nil { + t.Fatal(err) + } + all := []string{"pve1", "pve2", "pve3"} + for _, op := range []string{"pause", "save", "resume"} { + if got := rt.ops(op); !slices.Equal(got, all) { + t.Errorf("%s: %v, want %v", op, got, all) + } + } + // Every node is paused before the first snapshot is written. + firstSave := slices.IndexFunc(rt.calls, func(s string) bool { return strings.HasPrefix(s, "save ") }) + if paused := len(rt.ops("pause")); firstSave < paused { + t.Errorf("a snapshot was saved before all nodes were paused: %v", rt.calls) + } + snap := c.St.Snapshot("ready") + if snap == nil || !snap.Live { + t.Fatalf("snapshot not recorded as live: %+v", c.St.Snapshots) + } +} + +func TestCluster_SnapshotSaveLiveRollsBack(t *testing.T) { + rt := &fakeRuntime{failOn: "pve2"} + c := newTestCluster(t, rt) + if err := c.SnapshotSaveLive(t.Context(), "ready"); err == nil { + t.Fatal("expected the failed save to fail the snapshot") + } + if got := rt.ops("resume"); !slices.Equal(got, []string{"pve1", "pve2", "pve3"}) { + t.Errorf("not every node was resumed: %v", got) + } + rt.mu.Lock() + deleted := slices.Sorted(slices.Values(rt.deleted)) + rt.mu.Unlock() + saved := slices.DeleteFunc(rt.ops("save"), func(n string) bool { return n == "pve2" }) + if !slices.Equal(deleted, saved) { + t.Errorf("deleted %v, want the snapshots that were saved: %v", deleted, saved) + } + if c.St.Snapshot("ready") != nil { + t.Error("a failed snapshot must not be recorded") + } +} + +func TestCluster_SnapshotSaveLiveRefusesFaults(t *testing.T) { + rt := &fakeRuntime{} + c := newTestCluster(t, rt) + c.St.Faults = []state.Fault{{Kind: "freeze", Node: "pve1"}} + if err := c.SnapshotSaveLive(t.Context(), "ready"); err == nil || !strings.Contains(err.Error(), "faults") { + t.Fatalf("got %v, want an error about active faults", err) + } + if len(rt.calls) != 0 { + t.Errorf("nodes were touched: %v", rt.calls) + } +} diff --git a/internal/mcpserver/tools.go b/internal/mcpserver/tools.go index b689694..67ea109 100644 --- a/internal/mcpserver/tools.go +++ b/internal/mcpserver/tools.go @@ -83,6 +83,12 @@ type snapshotIn struct { Name string `json:"name" jsonschema:"snapshot name"` } +type snapshotSaveIn struct { + Cluster string `json:"cluster,omitempty" jsonschema:"cluster name; default: the only cluster"` + Name string `json:"name" jsonschema:"snapshot name"` + Live bool `json:"live,omitempty" jsonschema:"snapshot the running cluster with its memory; restoring it resumes the cluster in seconds"` +} + type snapshotsOut struct { Snapshots []state.Snapshot `json:"snapshots"` } @@ -130,8 +136,8 @@ func (h *handlers) register(s *mcp.Server) { mcp.AddTool(s, &mcp.Tool{Name: "cluster_destroy", Description: "Delete a cluster: kill its VMs and remove all its disks, snapshots and credentials. Also stops a create running in the background. Set confirm to the cluster name.", Annotations: local(true, true)}, h.destroy) mcp.AddTool(s, &mcp.Tool{Name: "node_exec", Description: "Run a shell command as root on a running node over SSH and return stdout, stderr and the exit code. Use it for pvesh, qm, pct, pvecm, ceph and the like. Non-interactive; output is capped at 64 KiB per stream.", Annotations: local(true, false)}, h.exec) mcp.AddTool(s, &mcp.Tool{Name: "cluster_env", Description: "Return what API and SSH clients need: API endpoint, API token (proxbase@pve!api, full privileges), the cluster CA certificate (PEM and path), SSH user, key path and ports per node. Root password only if includeRootPassword is set.", Annotations: readOnly}, h.env) - mcp.AddTool(s, &mcp.Tool{Name: "snapshot_save", Description: "Snapshot all disks of a stopped cluster (call cluster_stop first). Restoring is much faster than creating again.", Annotations: local(false, false)}, h.snapshotOp((*cluster.Cluster).SnapshotSave, "saved")) - mcp.AddTool(s, &mcp.Tool{Name: "snapshot_restore", Description: "Reset all disks of a stopped cluster to a snapshot (call cluster_stop first, cluster_start afterwards). Changes since the snapshot are lost.", Annotations: local(true, true)}, h.snapshotOp((*cluster.Cluster).SnapshotRestore, "restored")) + mcp.AddTool(s, &mcp.Tool{Name: "snapshot_save", Description: "Snapshot all disks of a stopped cluster (call cluster_stop first), or with live=true the running cluster including its memory. Restoring is much faster than creating again.", Annotations: local(false, false)}, h.snapshotSave) + mcp.AddTool(s, &mcp.Tool{Name: "snapshot_restore", Description: "Reset all disks of a stopped cluster to a snapshot (call cluster_stop first). Changes since the snapshot are lost. After a disk snapshot call cluster_start; a live snapshot starts the cluster in its saved state by itself.", Annotations: local(true, true)}, h.snapshotOp((*cluster.Cluster).SnapshotRestore, "restored")) mcp.AddTool(s, &mcp.Tool{Name: "snapshot_delete", Description: "Delete a snapshot of a cluster.", Annotations: local(true, true)}, h.snapshotOp((*cluster.Cluster).SnapshotDelete, "deleted")) mcp.AddTool(s, &mcp.Tool{Name: "snapshot_list", Description: "List the snapshots of a cluster.", Annotations: readOnly}, h.snapshots) mcp.AddTool(s, &mcp.Tool{Name: "fault_apply", Description: "Inject a failure into a running cluster to test HA, fencing, Ceph or applications: kill, freeze, thaw, link-down, link-up, partition or degrade. fault_clear undoes them.", Annotations: local(true, false)}, h.fault) @@ -306,6 +312,14 @@ func (h *handlers) snapshotOp(f func(*cluster.Cluster, context.Context, string) } } +func (h *handlers) snapshotSave(ctx context.Context, req *mcp.CallToolRequest, in snapshotSaveIn) (*mcp.CallToolResult, messageOut, error) { + save := (*cluster.Cluster).SnapshotSave + if in.Live { + save = (*cluster.Cluster).SnapshotSaveLive + } + return h.snapshotOp(save, "saved")(ctx, req, snapshotIn{Cluster: in.Cluster, Name: in.Name}) +} + func (h *handlers) snapshots(_ context.Context, _ *mcp.CallToolRequest, in clusterIn) (*mcp.CallToolResult, snapshotsOut, error) { c, err := h.open(in.Cluster, nil) if err != nil { diff --git a/internal/qemu/fault.go b/internal/qemu/fault.go index 974f619..3543dc2 100644 --- a/internal/qemu/fault.go +++ b/internal/qemu/fault.go @@ -20,14 +20,10 @@ func (Runtime) Kill(s vm.Spec) error { } // Freeze stops all vCPUs; the node hangs but keeps its memory. -func (Runtime) Freeze(_ context.Context, s vm.Spec) error { - return qmpExec(qmpSocket(s), "stop", nil) -} +func (r Runtime) Freeze(ctx context.Context, s vm.Spec) error { return r.Pause(ctx, s) } // Thaw resumes a frozen node. -func (Runtime) Thaw(_ context.Context, s vm.Spec) error { - return qmpExec(qmpSocket(s), "cont", nil) -} +func (r Runtime) Thaw(ctx context.Context, s vm.Spec) error { return r.Resume(ctx, s) } // SetLink sets the carrier of internal NIC nic, as if its cable was pulled. func (Runtime) SetLink(_ context.Context, s vm.Spec, nic int, up bool) error { diff --git a/internal/qemu/live.go b/internal/qemu/live.go new file mode 100644 index 0000000..1d9721c --- /dev/null +++ b/internal/qemu/live.go @@ -0,0 +1,140 @@ +package qemu + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/virtbase/proxbase/internal/vm" +) + +var _ vm.LiveSnapshotter = Runtime{} + +const liveSnapshotTimeout = 10 * time.Minute + +// SaveLive saves an internal snapshot of every disk with the node's memory (in +// the root disk) through a QMP snapshot-save job. QEMU stops the vCPUs while it +// saves and resumes them afterwards, unless the node was paused. +func (Runtime) SaveLive(ctx context.Context, s vm.Spec, name string) error { + q, err := dialQMP(qmpSocket(s)) + if err != nil { + return fmt.Errorf("%s: %w", s.Name, err) + } + defer q.Close() + nodes, err := diskNodes(q, s) + if err != nil { + return fmt.Errorf("%s: %w", s.Name, err) + } + job := "snapshot-save-" + name + if _, err := q.Execute("snapshot-save", map[string]any{"job-id": job, "tag": name, "vmstate": nodes[0], "devices": nodes}); err != nil { + return fmt.Errorf("%s: %w", s.Name, err) + } + if err := waitJob(ctx, q, job, liveSnapshotTimeout); err != nil { + return fmt.Errorf("%s: snapshot %s: %w", s.Name, name, err) + } + return nil +} + +// DeleteLive deletes a snapshot from every disk of a running node through a +// QMP snapshot-delete job. Disks without the snapshot are no error. +func (Runtime) DeleteLive(ctx context.Context, s vm.Spec, name string) error { + q, err := dialQMP(qmpSocket(s)) + if err != nil { + return fmt.Errorf("%s: %w", s.Name, err) + } + defer q.Close() + nodes, err := diskNodes(q, s) + if err != nil { + return fmt.Errorf("%s: %w", s.Name, err) + } + job := "snapshot-delete-" + name + if _, err := q.Execute("snapshot-delete", map[string]any{"job-id": job, "tag": name, "devices": nodes}); err != nil { + return fmt.Errorf("%s: %w", s.Name, err) + } + if err := waitJob(ctx, q, job, liveSnapshotTimeout); err != nil { + return fmt.Errorf("%s: delete snapshot %s: %w", s.Name, name, err) + } + return nil +} + +// Pause stops all vCPUs. +func (Runtime) Pause(_ context.Context, s vm.Spec) error { return qmpExec(qmpSocket(s), "stop", nil) } + +// Resume lets the vCPUs run again. +func (Runtime) Resume(_ context.Context, s vm.Spec) error { return qmpExec(qmpSocket(s), "cont", nil) } + +// diskNodes returns the block node names of the root disk and the data disks, +// in that order. +func diskNodes(q *qmpConn, s vm.Spec) ([]string, error) { + raw, err := q.Execute("query-block", nil) + if err != nil { + return nil, err + } + var blocks []struct { + Device string `json:"device"` + Inserted *struct { + NodeName string `json:"node-name"` + } `json:"inserted"` + } + if err := json.Unmarshal(raw, &blocks); err != nil { + return nil, err + } + byDevice := map[string]string{} + for _, b := range blocks { + if b.Inserted != nil { + byDevice[b.Device] = b.Inserted.NodeName + } + } + ids := []string{"root"} + for i := range s.DataDisks { + ids = append(ids, fmt.Sprintf("data%d", i)) + } + nodes := make([]string, 0, len(ids)) + for _, id := range ids { + n, ok := byDevice[id] + if !ok { + return nil, fmt.Errorf("disk %s not found", id) + } + nodes = append(nodes, n) + } + return nodes, nil +} + +// waitJob waits until a QMP job has concluded, returns its error and dismisses it. +func waitJob(ctx context.Context, q *qmpConn, id string, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + tick := time.NewTicker(200 * time.Millisecond) + defer tick.Stop() + for { + raw, err := q.Execute("query-jobs", nil) + if err != nil { + return err + } + var jobs []struct { + ID string `json:"id"` + Status string `json:"status"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &jobs); err != nil { + return err + } + for _, j := range jobs { + if j.ID != id || j.Status != "concluded" { + continue + } + _, derr := q.Execute("job-dismiss", map[string]any{"id": id}) + if j.Error != "" { + return errors.New(j.Error) + } + return derr + } + select { + case <-ctx.Done(): + return fmt.Errorf("job %s: %w", id, ctx.Err()) + case <-tick.C: + } + } +} diff --git a/internal/qemu/qemu.go b/internal/qemu/qemu.go index a35f8f1..c68b793 100644 --- a/internal/qemu/qemu.go +++ b/internal/qemu/qemu.go @@ -122,6 +122,9 @@ func args(s vm.Spec, b boot, media *vm.Media) []string { "-serial", "chardev:ser0", "-daemonize", ) + if s.LoadSnapshot != "" { + a = append(a, "-loadvm", s.LoadSnapshot, "-S") + } } return a } diff --git a/internal/qemu/qemu_test.go b/internal/qemu/qemu_test.go index 14c5f81..ab02867 100644 --- a/internal/qemu/qemu_test.go +++ b/internal/qemu/qemu_test.go @@ -78,3 +78,16 @@ func TestOnceArgs(t *testing.T) { s.Files = []vm.File{{Name: "opt/proxbase/bake", Path: "/r/bake.sh"}} golden(t, "once.args", args(s, bootOnce, nil)) } + +// A node started from a live snapshot loads it and waits paused for Resume. +func TestLoadSnapshotArgs(t *testing.T) { + s := spec() + s.LoadSnapshot = "ready" + a := strings.Join(args(s, bootDaemon, nil), " ") + if !strings.Contains(a, " -loadvm ready -S") { + t.Fatalf("missing -loadvm ready -S in %s", a) + } + if strings.Contains(strings.Join(args(spec(), bootDaemon, nil), " "), "-loadvm") { + t.Fatal("-loadvm without LoadSnapshot") + } +} diff --git a/internal/state/state.go b/internal/state/state.go index c5c5dea..c29cfb5 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -47,6 +47,9 @@ type Snapshot struct { Name string `json:"name"` Created time.Time `json:"created"` Nodes []string `json:"nodes"` + // Live snapshots were taken of the running cluster and include its memory; + // restoring one starts the cluster in that state. + Live bool `json:"live,omitempty"` } func (s *State) Snapshot(name string) *Snapshot { diff --git a/internal/vm/vm.go b/internal/vm/vm.go index beef7a3..38fcdb2 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -40,6 +40,9 @@ type Spec struct { // Files are handed to the guest at boot (QEMU fw_cfg); the guest reads them // below /sys/firmware/qemu_fw_cfg/by_name/. Files []File + // LoadSnapshot starts the node paused in the state of this live snapshot + // (see LiveSnapshotter); Resume lets it run. + LoadSnapshot string } // File is a host file passed to the guest under Name (e.g. "opt/proxbase/identity"). @@ -98,6 +101,19 @@ type Snapshotter interface { Snapshot(ctx context.Context, op SnapshotOp, name string, disks []Disk) error } +// LiveSnapshotter snapshots running nodes with their memory, so they can be +// started again in that state (Spec.LoadSnapshot). Snapshotter deletes them. +type LiveSnapshotter interface { + // SaveLive snapshots every disk and the memory of a running node; a paused + // node stays paused. + SaveLive(ctx context.Context, s Spec, name string) error + // DeleteLive deletes a snapshot from the disks of a running node. + DeleteLive(ctx context.Context, s Spec, name string) error + // Pause stops all vCPUs; Resume lets them run again. + Pause(ctx context.Context, s Spec) error + Resume(ctx context.Context, s Spec) error +} + // Runtime is a complete node runtime. type Runtime interface { Installer From 405e092ac29e0e13e507d3cb48e2d6b5604f9f5f Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:57:45 +0200 Subject: [PATCH 17/19] feat(action): cache a ready cluster as a template With template: true the first run saves a live snapshot of the new cluster in the actions cache; later runs on the same QEMU and CPU model resume it. --- action.yml | 66 ++++++++++++++++++++++++++++++++++ examples/ci/github-actions.yml | 1 + 2 files changed, 67 insertions(+) diff --git a/action.yml b/action.yml index f0f6670..ce77bfd 100644 --- a/action.yml +++ b/action.yml @@ -39,6 +39,11 @@ inputs: cache: description: Cache the ISO and base image with actions/cache default: "true" + template: + description: >- + Cache the ready cluster as a live snapshot (memory included) and resume it in + later runs instead of creating it; needs golden and cache + default: "false" verify-signature: description: Also verify the release signature with cosign (cosign must be installed) default: "false" @@ -131,7 +136,38 @@ runs: restore-keys: | proxbase-${{ runner.os }}-${{ runner.arch }}- + - name: Template cache key + id: template + if: inputs.template == 'true' + shell: bash + env: + NAME: ${{ inputs.name }} + CONFIG: ${{ inputs.config }} + INPUTS: ${{ inputs.nodes }}|${{ inputs.memory }}|${{ inputs.data-disks }}|${{ inputs.storage }}|${{ inputs.pve-version }} + VERSION: ${{ steps.install.outputs.version }} + run: | + set -euo pipefail + # A live snapshot only loads into the same QEMU on the same CPU model. + { + echo "$INPUTS" + [ -n "$CONFIG" ] && sha256sum < "$CONFIG" + qemu-system-x86_64 --version | head -n 1 + grep -m 1 '^model name' /proc/cpuinfo + grep -m 1 '^flags' /proc/cpuinfo + } > "$RUNNER_TEMP/proxbase-template-key" + echo "key=proxbase-template-$NAME-$VERSION-$(sha256sum < "$RUNNER_TEMP/proxbase-template-key" | cut -c1-16)" >> "$GITHUB_OUTPUT" + echo "path=${XDG_DATA_HOME:-$HOME/.local/share}/proxbase/clusters/$NAME" >> "$GITHUB_OUTPUT" + + - name: Restore cluster template + id: template-cache + if: inputs.template == 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.template.outputs.path }} + key: ${{ steps.template.outputs.key }} + - name: Create cluster + id: create shell: bash env: NAME: ${{ inputs.name }} @@ -142,8 +178,17 @@ runs: STORAGE: ${{ inputs.storage }} PVE_VERSION: ${{ inputs.pve-version }} GOLDEN: ${{ inputs.golden }} + TEMPLATE: ${{ inputs.template }} + TEMPLATE_HIT: ${{ steps.template-cache.outputs.cache-hit }} run: | set -euo pipefail + if [ "$TEMPLATE_HIT" = true ]; then + if proxbase snapshot restore "$NAME" template; then + exit 0 + fi + echo "::warning::could not resume the cached cluster template; creating the cluster" + proxbase destroy "$NAME" --yes + fi args=(create "$NAME") [ -n "$CONFIG" ] && args+=(-f "$CONFIG") [ -n "$NODES" ] && args+=(--nodes "$NODES") @@ -162,6 +207,27 @@ runs: done exit 1 fi + if [ "$TEMPLATE" = true ] && [ "$TEMPLATE_HIT" != true ]; then + # Snapshot the ready cluster, stop it so the disk images are not written + # while they are cached, and resume it from the snapshot after saving. + proxbase snapshot save "$NAME" template --live + proxbase stop "$NAME" --timeout 0 + echo "save-template=true" >> "$GITHUB_OUTPUT" + fi + + - name: Save cluster template + if: steps.create.outputs.save-template == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.template.outputs.path }} + key: ${{ steps.template.outputs.key }} + + - name: Resume cluster template + if: steps.create.outputs.save-template == 'true' + shell: bash + env: + NAME: ${{ inputs.name }} + run: proxbase snapshot restore "$NAME" template - name: Export credentials id: env diff --git a/examples/ci/github-actions.yml b/examples/ci/github-actions.yml index b3efba7..26d42ab 100644 --- a/examples/ci/github-actions.yml +++ b/examples/ci/github-actions.yml @@ -23,6 +23,7 @@ jobs: name: ci nodes: "1" # config: test/cluster.yaml # or a cluster file + # template: "true" # cache the ready cluster, resume it in seconds next time - name: Run tests env: From 1467c1c94c477c8f5f5c44eb5b8ee38811cd2c0b Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 20:57:45 +0200 Subject: [PATCH 18/19] docs: first-boot hook, package cache, live snapshots and timings --- README.md | 8 ++++--- docs/architecture.md | 49 ++++++++++++++++++++++++++++++++++++----- docs/ci.md | 29 +++++++++++++++++++++++- docs/cli.md | 6 ++--- docs/configuration.md | 1 + docs/golden-images.md | 44 ++++++++++++++++++++++-------------- docs/lifecycle.md | 25 +++++++++++++++++++++ docs/mcp.md | 6 ++--- docs/storage.md | 15 ++++++++++++- docs/troubleshooting.md | 8 ++++--- 10 files changed, 154 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 4c91ddd..f134fd2 100644 --- a/README.md +++ b/README.md @@ -33,16 +33,18 @@ pve3 running online 10.10.10.13 https://127.0.0.1:18003 127.0.0.1:18103 link0/link1, Ceph public/cluster, migration), VLAN-aware and L2-only bridges, jumbo frames. - **Lifecycle:** start/stop, add and remove nodes (with Ceph draining), consistent - snapshots of the whole cluster, resumable creates. + snapshots of the whole cluster, live snapshots that resume a running cluster in + seconds, resumable creates. - **Fault injection:** power loss, hung nodes, pulled cables, network partitions, latency and loss, to test HA, fencing and Ceph recovery. - **Ready for automation:** API token and CA export, `proxbase env` for shells, JSON and the bpg/proxmox Terraform provider, `-o json` and `--progress json` for scripts, - and a [GitHub Action](docs/ci.md) for CI. + and a [GitHub Action](docs/ci.md) for CI that can cache a ready cluster as a + template. - **For AI agents:** `proxbase mcp` is an [MCP server](docs/mcp.md): agents create clusters, run commands on nodes, snapshot and inject faults through tools. - **Fast:** `--golden` clones nodes from a cached base image: a 3-node cluster in - about 1.5 minutes, with a unique identity per node. + under a minute (Ceph: 1.5 minutes), with a unique identity per node. - **Declarative:** one YAML file with a JSON schema; every flag overrides a field. - **Docker Compose:** `proxbase up` runs a cluster in the foreground with clean shutdown and a health check. diff --git a/docs/architecture.md b/docs/architecture.md index 9f1d2cc..dce897a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -23,6 +23,10 @@ flowchart TD cluster --> nodesetup cluster --> image cluster --> answer + cluster --> firstboot["firstboot
first-boot hook"] + cluster --> aptcache["aptcache
package cache"] + cluster --> golden["golden
base images"] + golden --> firstboot cluster --> state cluster --> pve cluster --> remote @@ -45,8 +49,10 @@ flowchart TD | `state` | On-disk layout of a cluster, `state.json`, secrets, locking | | `image` | Download, verify and cache ISOs; extract the installer kernel and command line | | `answer` | Render the installer's `answer.toml` | +| `firstboot` | The installer's first-boot hook: node identity before networking, base image baking; serves the hook to the installer | +| `aptcache` | Caching HTTP proxy for the packages Proxbase installs on the nodes | | `vm` | Hypervisor-neutral node description and the runtime interfaces | -| `qemu` | QEMU runtime: command lines with fixed PCI slots, install, start/stop via pidfile and QMP, qcow2 snapshots | +| `qemu` | QEMU runtime: command lines with fixed PCI slots, install, start/stop via pidfile and QMP, qcow2 snapshots (live ones through QMP jobs) | | `serial` | Answer the installer's debug shell over the serial console and log it | | `netswitch` | Learning L2 switch over unix datagram sockets | | `network` | The per-cluster switch process, the nodes' NIC sockets and network faults (`run/faults.json`) | @@ -55,7 +61,7 @@ flowchart TD | `pve` | Proxmox VE REST client with certificate pinning and task helpers | | `remote` | SSH to nodes through the forwarded ports | | `retry` | Retry with timeout, interval and permanent errors | -| `golden` | Cached base images and the personalization of cloned nodes | +| `golden` | Cached base images: build, bake, and the personalization of clones from older images | | `cluster` | Orchestration: create, lifecycle, nodes, snapshots, faults, status, exec | | `progress` | Progress events and sinks: text, JSON lines, MCP notifications | | `job` | Runs a create detached from its caller, with a JSON-lines progress log | @@ -78,25 +84,52 @@ sequenceDiagram participant VM as QEMU (per node) participant Node as node (SSH) participant API as Proxmox API - CLI->>Img: ISO + SHA256SUMS (cached, verified) + CLI->>Img: ISO + SHA256SUMS (cached, verified; no network for a cached exact version) CLI->>SW: start proxbase _switch par every node CLI->>VM: install: -kernel/-initrd, answer disk PROXMOX-AIS VM-->>CLI: serial: debug shell CLI->>VM: proxmox-fetch-answer partition PROXMOX-AIS && exit + VM->>CLI: fetch the first-boot hook (HTTP on 10.0.2.2) VM-->>CLI: power off = installed end - CLI->>VM: boot nodes (daemonized) - CLI->>Node: hosts, repositories, bridges (nodesetup) + CLI->>VM: boot nodes (daemonized); clones get their identity as fw_cfg file + Note over VM: first boot: hook runs before networking + CLI->>Node: provisioning mode, hosts, repositories, bridges (nodesetup) CLI->>API: create cluster on node 1, join the others - CLI->>Node: wait for quorum and a writable /etc/pve + CLI->>Node: wait for quorum and a writable /etc/pve, repair lost node certificates CLI->>API: storage backends: ZFS pools / Ceph CLI->>API: API token, export CA + CLI->>Node: end provisioning mode ``` Each step checks first and skips what is done, so a failed create resumes when run again. Progress (`installed` per node, phase, errors) is kept in `state.json`. +### Speed + +- The installer's first-boot hook ([golden images](golden-images.md)) runs on each + node's first boot before networking: it skips the `pveupdate` Proxmox VE starts on + a first boot, removes the 5 s GRUB menu wait and, on clones, sets the identity, + so clones need no extra reboot. +- While provisioning, nodes write without fsync (`zfs set sync=disabled` on `rpool`, + dpkg `force-unsafe-io`); the last step restores both, so a ready cluster has real + fsync semantics for fault injection. +- Per-node work runs in parallel: installs, boots, configuration, ZFS pools, API + logins, Ceph managers and OSDs. Cluster joins, Ceph monitors and MDS daemons stay + sequential (the cluster file system and `ceph.conf` are not safe for concurrent + changes). +- SSH probes time out their handshake after 3 s: QEMU's port forward accepts + connections before the guest is up and otherwise retries towards the guest only + every 12 s. +- NIC 0 stays on QEMU user networking (slirp): host to node it measured 6-7 Gbit/s, + so passt would not speed up provisioning, and slirp also serves the nested guests + on `vmbr0` with DHCP. +- Packages Proxbase installs go through a cache on the host (`aptcache`, reached as + `http://10.0.2.2:PORT` like the hook server), so each is downloaded once. It only + keeps `.deb` files; if apt rejects one (Hash Sum mismatch), Proxbase's apt runs + retry once with `Acquire::http::No-Cache`, which makes the cache fetch it again. + ## Progress and the MCP server Long operations report through a `progress.Sink` (a function taking an `Event`). @@ -130,6 +163,9 @@ runs in the server process, guarded by the same cluster lock as the CLI. for install and run, so interface names never change between them. - NIC 0 is QEMU user networking (`vmbr0`, NAT, port forwards); NIC 1..n are unix datagram sockets to the switch. MACs are derived from node and NIC number. +- A virtio balloon with free page reporting returns memory the guest freed to the + host; disks use `cache=unsafe`, `discard=unmap` and `detect-zeroes=unmap`, and ZFS + pools in the nodes have `autotrim` on, so freed space shrinks the images. - The installer runs in the foreground (killed if proxbase dies); installed nodes run daemonized with a pidfile, a QMP socket and a serial console socket with a log file. - Stop: ACPI powerdown via QMP, `quit` after the timeout, then SIGKILL. @@ -156,6 +192,7 @@ runs in the server process, guarded by the same cluster lock as the CLI. ~/.local/share/proxbase/jobs/ .jsonl, .pid, .yaml of creates started over MCP ~/.cache/proxbase/iso/ ISOs, SHA256SUMS, extracted installer files ~/.cache/proxbase/images/ golden base images (create --golden) +~/.cache/proxbase/apt/ packages served to the nodes (unused for 30 days: removed) ``` Design decisions and measurements are in [research/feasibility.md](research/feasibility.md). diff --git a/docs/ci.md b/docs/ci.md index 12431ca..09c8c96 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,7 +1,8 @@ # CI and automation GitHub-hosted Linux runners expose `/dev/kvm`, so they can run a real cluster. A -single node is ready in 2-3 minutes with a cached base image. +single node is ready in well under a minute with a cached base image (21 s on a +desktop host; runners are slower), and in seconds with a [template](#cluster-templates). ## GitHub Action @@ -39,6 +40,7 @@ and creates the cluster with `--golden`. On failure it prints the node logs. | `nodes`, `memory`, `data-disks`, `storage`, `pve-version` | `1`, file defaults | As the `create` flags | | `golden` | `true` | Clone from a cached base image | | `cache` | `true` | Cache `~/.cache/proxbase` | +| `template` | `false` | Cache the ready cluster as a live snapshot and resume it in later runs (see below) | | `verify-signature` | `false` | Verify with cosign (install it first) | | Output | | @@ -53,6 +55,31 @@ step. A complete workflow is in [examples/ci/github-actions.yml](../examples/ci/ the repository's own [E2E workflow](../.github/workflows/e2e.yml) uses the action from the checkout with a freshly built binary. +## Cluster templates + +With `template: "true"` the first run creates the cluster, takes a +[live snapshot](lifecycle.md#live-snapshots) of it, stops it, saves the cluster +directory with `actions/cache/save` and resumes it from the snapshot. Later runs +restore that cache and resume the cluster in seconds instead of creating it. + +```yaml + - id: pve + uses: virtbase/proxbase@v0 + with: + nodes: "1" + template: "true" +``` + +- The cache key covers the inputs (and the cluster file), the proxbase and QEMU + versions and the CPU model: a live snapshot only loads into the same QEMU on the + same CPU. A runner on another CPU model creates and caches its own template. +- If resuming fails, the action destroys the cluster and creates it as usual. +- The cached cluster includes its secrets (root password, SSH key, API token). They + only reach a cluster bound to the runner's loopback address, but anyone who can + read the repository's caches can read them. +- Each node needs cache space for its disk changes and its memory (about 2 GB for + one idle 4 GiB node), next to the ISO and base image cache. + ## Other CI systems Any Linux runner with KVM works: install QEMU and the binary, then diff --git a/docs/cli.md b/docs/cli.md index f2f83c4..9ff4b2a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -45,8 +45,8 @@ online and Ceph (if configured) is `HEALTH_OK`; use it for health checks. | --- | --- | | `node add [cluster] --count N` | Install, join and configure N new nodes | | `node remove [cluster] ` | Drain and remove a node (`--force` with guests or degraded Ceph pools) | -| `snapshot save [cluster] ` | Snapshot every disk of the stopped cluster | -| `snapshot restore [cluster] ` | Reset every disk to a snapshot | +| `snapshot save [cluster] ` | Snapshot every disk of the stopped cluster; `--live`: the running cluster with its memory | +| `snapshot restore [cluster] ` | Reset every disk to a snapshot; a live snapshot also starts the cluster in its saved state | | `snapshot delete [cluster] ` | Delete a snapshot | | `snapshot list [cluster]` | List snapshots | @@ -78,7 +78,7 @@ See [fault-injection.md](fault-injection.md). | --- | --- | | `image list` | Cached base images and the clusters using them | | `image prune` | Delete base images no cluster uses | -| `doctor` | Check KVM, nested virtualization, QEMU, memory, disk, ports (`--port-base`) | +| `doctor` | Check KVM, nested virtualization, QEMU, memory, KSM, disk, ports (`--port-base`) | | `config init [file]` | Write an annotated example (`--force` to overwrite) | | `config validate ` | Strictly parse and check a cluster file | | `config schema` | Print the JSON schema | diff --git a/docs/configuration.md b/docs/configuration.md index 0d7f443..df5ea5c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,6 +73,7 @@ More in [examples/](../examples). | `sshKeys` | none | Extra public keys for root, next to the generated one | | `upgrade` | `false` | `apt dist-upgrade` on every node before clustering; reboots into a new kernel | | `goldenImage` | `false` | Clone nodes from a cached base image instead of installing each ([golden images](golden-images.md)) | +| `aptCache` | `true` | Packages Proxbase installs (`upgrade`, Ceph) are downloaded once into `~/.cache/proxbase/apt` and served to the nodes from there | ### `nodes` diff --git a/docs/golden-images.md b/docs/golden-images.md index 240b792..e0d81a8 100644 --- a/docs/golden-images.md +++ b/docs/golden-images.md @@ -11,33 +11,43 @@ proxbase image list # base images and the clusters using proxbase image prune # delete base images no cluster uses ``` -Measured on the same host (3 nodes, ISO cached): +Measured on the same host (3 nodes, 4 GiB each, ISO cached): | | Installer | Golden image (cached) | | --- | --- | --- | -| ZFS create | 3m04s | 1m36s | -| Ceph create | 4m43s | 3m03s | -| `node add` | 3m21s | 1m14s | +| ZFS create, 1 node | – | 21s | +| ZFS create | 2m59s | 49s | +| Ceph create | – | 1m30s | -The first golden create builds the base image (about 1m40s) and takes about as long -as a normal create. +The first golden create builds the base image (1m56s; 3m43s with Ceph, which +downloads the Ceph packages) and takes about as long as a normal create. ## How it works -- A base image is the root disk of an installed but **never booted** node, cached in +- A base image is the root disk of an installed node, cached in `~/.cache/proxbase/images//`. The key covers the ISO, the root filesystem and - size, and keyboard, country and time zone; nodes with different root disks use - different base images. + size, keyboard, country and time zone, and the Ceph release if the cluster uses + Ceph; nodes with different root disks use different base images. - Each node's root disk is a qcow2 overlay of the base image; data disks are created fresh. The base image must stay while clusters use it, so `image prune` keeps those. -- Everything Proxmox VE creates on first boot is created per node: the cluster CA, the - auth key, the node certificate and root's SSH key. -- What the installer wrote is replaced before clustering, over SSH with the base image's - own key: host name (`/etc/hostname`, `/etc/hosts`, postfix), SSH host keys, - machine-id, root password and authorized keys. The node reboots, the base image's - node directory in `/etc/pve` is removed, and the new node certificate is checked. -- A create interrupted during personalization resumes correctly: nodes that already - have their identity are only finished. +- The installer installs a first-boot hook (package `firstboot`) that runs on the + system's first boot, before networking and before any Proxmox VE service starts. +- **Baking.** After the install, the base image boots once with a bake script: it + removes the GRUB menu wait, marks the appliance list as fetched (no `pveupdate` on + first boot) and, for Ceph clusters, installs the Ceph packages and the newer + `libpve-storage-perl` they need. Then it powers off before any Proxmox VE service + has started, and the hook stays pending for the clones. +- **Identity.** Each clone gets its identity as a QEMU fw_cfg file. The hook applies + it on the clone's first boot, before networking: host name (`/etc/hostname`, + `/etc/hosts`, search domain, postfix), new SSH host keys, a new machine-id, the root + password and the authorized keys. Everything Proxmox VE creates on its first start + (node certificate, root's SSH key; the cluster CA and auth key on the first node) + is therefore created per node, under the right name. No reboot is needed. +- Base images from Proxbase versions before the first-boot hook have a different + key, so the first golden create after an upgrade builds a new one; the old ones + stay for the clusters that use them (`image prune` removes them afterwards). A + create interrupted under an older version resumes by personalizing the remaining + clones over SSH, with a reboot. Verified per node and across clusters: host keys, machine-ids, node certificates and cluster CAs all differ. diff --git a/docs/lifecycle.md b/docs/lifecycle.md index 863b7b1..4add6c9 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -72,6 +72,31 @@ proxbase snapshot delete lab base A snapshot can only be restored with the same set of nodes it was taken with. Saving takes well under a second; restoring about a second. +### Live snapshots + +A live snapshot is taken of the running cluster and also holds the memory of every +node. Restoring it starts the cluster in that state: no boot, no waiting for +quorum to form or Ceph to settle. + +```bash +proxbase snapshot save lab ready --live # cluster running; nodes pause for the save +proxbase stop lab --timeout 0 # restoring needs a stopped cluster +proxbase snapshot restore lab ready # nodes resume where they were +``` + +- All nodes are paused while their snapshots are written, so the snapshot shows one + moment of the whole cluster; they continue afterwards. Faults must be cleared. +- On restore, every node loads its snapshot paused and all are resumed together. + The node clocks stood still in the snapshot; Proxbase sets them to the host's time, + then waits for quorum, the API and storage health as `start` does. +- Memory is stored in the root disk image, so a live snapshot needs about as much + disk space as the nodes use memory (1.8 GB per idle 4 GiB node). +- On a 3-node ZFS cluster, saving took 3 s and restoring to a quorate, ready + cluster 5 s. +- It loads only into the same QEMU version on the same CPU model (nodes use + `-cpu host`); after a QEMU upgrade, take it again. +- `snapshot delete` works on a running cluster for live snapshots. + ## Access ```bash diff --git a/docs/mcp.md b/docs/mcp.md index 9e333ab..1c555e1 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -65,7 +65,7 @@ Clusters created in the container live in the `proxbase` volume; use | `cluster_destroy` | Delete a cluster; `confirm` must repeat the cluster name | | `node_exec` | Run a command as root on a node; returns `stdout`, `stderr`, `exitCode` | | `cluster_env` | API endpoint, token, CA certificate, SSH key and ports; root password only with `includeRootPassword` | -| `snapshot_save`, `snapshot_restore`, `snapshot_delete`, `snapshot_list` | Snapshots of the stopped cluster | +| `snapshot_save`, `snapshot_restore`, `snapshot_delete`, `snapshot_list` | Snapshots of the stopped cluster, or with `live` of the running cluster including memory (restoring resumes it) | | `fault_apply`, `fault_clear` | kill, freeze, thaw, link-down/up, partition, degrade | | `config_validate` | Check a cluster file; returns errors, warnings and the file with defaults | @@ -75,8 +75,8 @@ clients can ask before `cluster_destroy`, `snapshot_restore` or `fault_apply`. ## Long-running creates -A create takes 2-3 minutes with a cached [golden image](golden-images.md) (the -default for `cluster_create`) and 10-20 minutes the first time, when it downloads the +A create takes under a minute with a cached [golden image](golden-images.md) (the +default for `cluster_create`; Ceph about 1.5 minutes) and 10-20 minutes the first time, when it downloads the ISO and builds the base image. Many clients time out tool calls earlier, so: - `cluster_create` starts the create as a background job on the host diff --git a/docs/storage.md b/docs/storage.md index a79dcb0..7d57e61 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -20,7 +20,9 @@ storage: Every node has its own pool with the same name; live migration of VMs on it copies the disks (`qm migrate --with-local-disks`). Pools are created through the Proxmox -API (`POST /nodes/{node}/disks/zfs`, `ashift=12`). +API (`POST /nodes/{node}/disks/zfs`, `ashift=12`), on all nodes at once, with +`autotrim` on (also for `rpool`), so space freed in a pool shrinks the disk image +on the host. ## Ceph @@ -44,6 +46,17 @@ hyperconverged cluster: `create` and `start` wait until Ceph reports `HEALTH_OK`, every OSD is up and in, all placement groups are `active+clean` and the Ceph storages are active on every node. +Monitors are created one after another (each joins the quorum first) and MDS daemons +too (`pveceph` rewrites `ceph.conf` for each without a lock); managers and OSDs are +created on all nodes at once. With `--golden`, the Ceph packages are baked into the +base image ([golden images](golden-images.md)), so nodes skip `pveceph install`. + +Memory targets for small nodes: Ceph's defaults are sized for dedicated servers +(4 GiB per OSD, 4 GiB MDS cache) and the daemons grow their caches toward them. +Proxbase sets `osd_memory_target` to 2 GiB (about the lowest Ceph developers +recommend) and `mds_cache_memory_limit` to 1 GiB, unless they are set already, so +a value changed with `ceph config set` stays. + Notes: - Ceph needs RAM: the default node memory becomes 6G; less only prints a warning. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 865e61f..df5af70 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -8,7 +8,7 @@ Everything of a cluster lives in `~/.local/share/proxbase/clusters//`: | --- | --- | | `logs/-install.log` | Serial console of the installer | | `logs/-qemu.log` | QEMU errors during the install | -| `logs/-console.log` | Serial console of the running node | +| `logs/-console.log` | Serial console of the running node; the first-boot hook reports `proxbase-first-boot: identity ` there | | `logs/switch.log` | Internal network switch | | `state.json`, `cluster.yaml` | Progress and the resolved configuration | | `secrets/` | Root password, SSH key, API token, CA (mode 0600) | @@ -25,8 +25,10 @@ Everything of a cluster lives in `~/.local/share/proxbase/clusters//`: | `port 18001 for pve1 is in use` | Another cluster or program uses the ports; set `access.portBase` | | `socket path … is too long` | Unix socket paths are limited to 107 bytes; use a shorter `XDG_DATA_HOME` or cluster/network names | | Install times out or `installer reported failure` | Read `logs/-install.log`; the last lines are also in the error | -| `fetch checksums` fails | The ISO mirror is unreachable; set `proxmox.mirror` | -| `Hash Sum mismatch` during apt (nodes or image build) | A broken mirror or path (seen over IPv6). Retry; for image builds pass `--build-arg APT_OPTS="-o Acquire::ForceIPv4=true"` | +| `fetch checksums` fails | The ISO mirror is unreachable; set `proxmox.mirror`. An exact `proxmox.version` (`9.2-1`) that is cached needs no network | +| `the cluster join left the node certificate missing or from another CA; issued a new one` | Not an error: a Proxmox VE join can lose the node's files or replace the cluster CA when the cluster file system syncs late; Proxbase detects it after the joins and issues a new node certificate from the current CA | +| `bake base image failed` | Building a [golden image](golden-images.md) failed while baking; the error shows the end of the serial console (for Ceph usually the package download) | +| `Hash Sum mismatch` during apt (nodes or image build) | A broken mirror or path (seen over IPv6). Proxbase's own apt runs retry once past the package cache; if it persists, retry later. For Docker image builds pass `--build-arg APT_OPTS="-o Acquire::ForceIPv4=true"`. `rm -rf ~/.cache/proxbase/apt` empties the package cache | | `ceph not ready after 10m0s` | The error includes `ceph -s` and `ceph health detail`; usually too little memory per node | | Nested guests are slow or fail to start | Nested virtualization is off on the host (`proxbase doctor`) | | `cluster "x" is busy` | Another proxbase command holds the cluster lock, e.g. a create started over MCP (`cluster_status` shows it) | From 044f80048dc1d4ef71177254c6327e61c1f8c7b3 Mon Sep 17 00:00:00 2001 From: Janic Bellmann Date: Sun, 11 Oct 2026 21:08:34 +0200 Subject: [PATCH 19/19] fix(cluster): restart pveproxy when it serves an outdated certificate After a join, pveproxy can keep serving a certificate that /etc/pve no longer holds, so logins pinned to the stored certificate fail. --- internal/cluster/pvecluster.go | 5 ++++- internal/nodesetup/nodesetup.go | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/cluster/pvecluster.go b/internal/cluster/pvecluster.go index 9f41967..cb6a8a8 100644 --- a/internal/cluster/pvecluster.go +++ b/internal/cluster/pvecluster.go @@ -195,8 +195,11 @@ func (c *Cluster) repairJoins(ctx context.Context) error { if err != nil { return fmt.Errorf("%s: repair join: %w", n.Name, err) } - if strings.Contains(out, "repaired") { + switch { + case strings.Contains(out, "repaired"): c.step("%s: the cluster join left the node certificate missing or from another CA; issued a new one", n.Name) + case strings.Contains(out, "restarted"): + c.step("%s: pveproxy served an outdated certificate after the join; restarted it", n.Name) } return nil }) diff --git a/internal/nodesetup/nodesetup.go b/internal/nodesetup/nodesetup.go index c9dcedf..5bab356 100644 --- a/internal/nodesetup/nodesetup.go +++ b/internal/nodesetup/nodesetup.go @@ -168,7 +168,8 @@ const Reboot = "systemd-run --on-active=2 systemctl reboot >/dev/null" // pveproxy, which then reject new tickets. RepairJoin issues a new node // certificate from the current cluster CA and restarts both services if the // certificate is missing or not signed by that CA, or the node's last join -// failed. It prints "repaired" when it acted. +// failed, and prints "repaired". If pveproxy serves another certificate than +// /etc/pve holds, it restarts pveproxy and prints "restarted". const RepairJoin = ` marker=/var/lib/proxbase-join-repaired join=$(grep ':clusterjoin:' /var/log/pve/tasks/index 2>/dev/null | tail -n 1 || true) @@ -182,6 +183,14 @@ if [ -n "$failed" ] || [ ! -s /etc/pve/local/pve-ssl.key ] || systemctl restart pvedaemon pveproxy if [ -n "$join" ]; then echo "$join" > $marker; fi echo repaired + exit 0 +fi +# pveproxy can also keep serving a certificate that /etc/pve no longer holds. +fp() { openssl x509 -noout -fingerprint -sha256 2>/dev/null || true; } +served=$(openssl s_client -connect 127.0.0.1:8006 /dev/null | fp) +if [ "$served" != "$(fp < /etc/pve/local/pve-ssl.pem)" ]; then + systemctl restart pveproxy + echo restarted fi `