diff --git a/README.md b/README.md
index 4c91ddd..f134fd2 100644
--- a/README.md
+++ b/README.md
@@ -33,16 +33,18 @@ pve3 running online 10.10.10.13 https://127.0.0.1:18003 127.0.0.1:18103
link0/link1, Ceph public/cluster, migration), VLAN-aware and L2-only bridges, jumbo
frames.
- **Lifecycle:** start/stop, add and remove nodes (with Ceph draining), consistent
- snapshots of the whole cluster, resumable creates.
+ snapshots of the whole cluster, live snapshots that resume a running cluster in
+ seconds, resumable creates.
- **Fault injection:** power loss, hung nodes, pulled cables, network partitions,
latency and loss, to test HA, fencing and Ceph recovery.
- **Ready for automation:** API token and CA export, `proxbase env` for shells, JSON
and the bpg/proxmox Terraform provider, `-o json` and `--progress json` for scripts,
- and a [GitHub Action](docs/ci.md) for CI.
+ and a [GitHub Action](docs/ci.md) for CI that can cache a ready cluster as a
+ template.
- **For AI agents:** `proxbase mcp` is an [MCP server](docs/mcp.md): agents create
clusters, run commands on nodes, snapshot and inject faults through tools.
- **Fast:** `--golden` clones nodes from a cached base image: a 3-node cluster in
- about 1.5 minutes, with a unique identity per node.
+ under a minute (Ceph: 1.5 minutes), with a unique identity per node.
- **Declarative:** one YAML file with a JSON schema; every flag overrides a field.
- **Docker Compose:** `proxbase up` runs a cluster in the foreground with clean
shutdown and a health check.
diff --git a/action.yml b/action.yml
index f0f6670..ce77bfd 100644
--- a/action.yml
+++ b/action.yml
@@ -39,6 +39,11 @@ inputs:
cache:
description: Cache the ISO and base image with actions/cache
default: "true"
+ template:
+ description: >-
+ Cache the ready cluster as a live snapshot (memory included) and resume it in
+ later runs instead of creating it; needs golden and cache
+ default: "false"
verify-signature:
description: Also verify the release signature with cosign (cosign must be installed)
default: "false"
@@ -131,7 +136,38 @@ runs:
restore-keys: |
proxbase-${{ runner.os }}-${{ runner.arch }}-
+ - name: Template cache key
+ id: template
+ if: inputs.template == 'true'
+ shell: bash
+ env:
+ NAME: ${{ inputs.name }}
+ CONFIG: ${{ inputs.config }}
+ INPUTS: ${{ inputs.nodes }}|${{ inputs.memory }}|${{ inputs.data-disks }}|${{ inputs.storage }}|${{ inputs.pve-version }}
+ VERSION: ${{ steps.install.outputs.version }}
+ run: |
+ set -euo pipefail
+ # A live snapshot only loads into the same QEMU on the same CPU model.
+ {
+ echo "$INPUTS"
+ [ -n "$CONFIG" ] && sha256sum < "$CONFIG"
+ qemu-system-x86_64 --version | head -n 1
+ grep -m 1 '^model name' /proc/cpuinfo
+ grep -m 1 '^flags' /proc/cpuinfo
+ } > "$RUNNER_TEMP/proxbase-template-key"
+ echo "key=proxbase-template-$NAME-$VERSION-$(sha256sum < "$RUNNER_TEMP/proxbase-template-key" | cut -c1-16)" >> "$GITHUB_OUTPUT"
+ echo "path=${XDG_DATA_HOME:-$HOME/.local/share}/proxbase/clusters/$NAME" >> "$GITHUB_OUTPUT"
+
+ - name: Restore cluster template
+ id: template-cache
+ if: inputs.template == 'true'
+ uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: ${{ steps.template.outputs.path }}
+ key: ${{ steps.template.outputs.key }}
+
- name: Create cluster
+ id: create
shell: bash
env:
NAME: ${{ inputs.name }}
@@ -142,8 +178,17 @@ runs:
STORAGE: ${{ inputs.storage }}
PVE_VERSION: ${{ inputs.pve-version }}
GOLDEN: ${{ inputs.golden }}
+ TEMPLATE: ${{ inputs.template }}
+ TEMPLATE_HIT: ${{ steps.template-cache.outputs.cache-hit }}
run: |
set -euo pipefail
+ if [ "$TEMPLATE_HIT" = true ]; then
+ if proxbase snapshot restore "$NAME" template; then
+ exit 0
+ fi
+ echo "::warning::could not resume the cached cluster template; creating the cluster"
+ proxbase destroy "$NAME" --yes
+ fi
args=(create "$NAME")
[ -n "$CONFIG" ] && args+=(-f "$CONFIG")
[ -n "$NODES" ] && args+=(--nodes "$NODES")
@@ -162,6 +207,27 @@ runs:
done
exit 1
fi
+ if [ "$TEMPLATE" = true ] && [ "$TEMPLATE_HIT" != true ]; then
+ # Snapshot the ready cluster, stop it so the disk images are not written
+ # while they are cached, and resume it from the snapshot after saving.
+ proxbase snapshot save "$NAME" template --live
+ proxbase stop "$NAME" --timeout 0
+ echo "save-template=true" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Save cluster template
+ if: steps.create.outputs.save-template == 'true'
+ uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: ${{ steps.template.outputs.path }}
+ key: ${{ steps.template.outputs.key }}
+
+ - name: Resume cluster template
+ if: steps.create.outputs.save-template == 'true'
+ shell: bash
+ env:
+ NAME: ${{ inputs.name }}
+ run: proxbase snapshot restore "$NAME" template
- name: Export credentials
id: env
diff --git a/docs/architecture.md b/docs/architecture.md
index 9f1d2cc..dce897a 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -23,6 +23,10 @@ flowchart TD
cluster --> nodesetup
cluster --> image
cluster --> answer
+ cluster --> firstboot["firstboot
first-boot hook"]
+ cluster --> aptcache["aptcache
package cache"]
+ cluster --> golden["golden
base images"]
+ golden --> firstboot
cluster --> state
cluster --> pve
cluster --> remote
@@ -45,8 +49,10 @@ flowchart TD
| `state` | On-disk layout of a cluster, `state.json`, secrets, locking |
| `image` | Download, verify and cache ISOs; extract the installer kernel and command line |
| `answer` | Render the installer's `answer.toml` |
+| `firstboot` | The installer's first-boot hook: node identity before networking, base image baking; serves the hook to the installer |
+| `aptcache` | Caching HTTP proxy for the packages Proxbase installs on the nodes |
| `vm` | Hypervisor-neutral node description and the runtime interfaces |
-| `qemu` | QEMU runtime: command lines with fixed PCI slots, install, start/stop via pidfile and QMP, qcow2 snapshots |
+| `qemu` | QEMU runtime: command lines with fixed PCI slots, install, start/stop via pidfile and QMP, qcow2 snapshots (live ones through QMP jobs) |
| `serial` | Answer the installer's debug shell over the serial console and log it |
| `netswitch` | Learning L2 switch over unix datagram sockets |
| `network` | The per-cluster switch process, the nodes' NIC sockets and network faults (`run/faults.json`) |
@@ -55,7 +61,7 @@ flowchart TD
| `pve` | Proxmox VE REST client with certificate pinning and task helpers |
| `remote` | SSH to nodes through the forwarded ports |
| `retry` | Retry with timeout, interval and permanent errors |
-| `golden` | Cached base images and the personalization of cloned nodes |
+| `golden` | Cached base images: build, bake, and the personalization of clones from older images |
| `cluster` | Orchestration: create, lifecycle, nodes, snapshots, faults, status, exec |
| `progress` | Progress events and sinks: text, JSON lines, MCP notifications |
| `job` | Runs a create detached from its caller, with a JSON-lines progress log |
@@ -78,25 +84,52 @@ sequenceDiagram
participant VM as QEMU (per node)
participant Node as node (SSH)
participant API as Proxmox API
- CLI->>Img: ISO + SHA256SUMS (cached, verified)
+ CLI->>Img: ISO + SHA256SUMS (cached, verified; no network for a cached exact version)
CLI->>SW: start proxbase _switch
par every node
CLI->>VM: install: -kernel/-initrd, answer disk PROXMOX-AIS
VM-->>CLI: serial: debug shell
CLI->>VM: proxmox-fetch-answer partition PROXMOX-AIS && exit
+ VM->>CLI: fetch the first-boot hook (HTTP on 10.0.2.2)
VM-->>CLI: power off = installed
end
- CLI->>VM: boot nodes (daemonized)
- CLI->>Node: hosts, repositories, bridges (nodesetup)
+ CLI->>VM: boot nodes (daemonized); clones get their identity as fw_cfg file
+ Note over VM: first boot: hook runs before networking
+ CLI->>Node: provisioning mode, hosts, repositories, bridges (nodesetup)
CLI->>API: create cluster on node 1, join the others
- CLI->>Node: wait for quorum and a writable /etc/pve
+ CLI->>Node: wait for quorum and a writable /etc/pve, repair lost node certificates
CLI->>API: storage backends: ZFS pools / Ceph
CLI->>API: API token, export CA
+ CLI->>Node: end provisioning mode
```
Each step checks first and skips what is done, so a failed create resumes when run
again. Progress (`installed` per node, phase, errors) is kept in `state.json`.
+### Speed
+
+- The installer's first-boot hook ([golden images](golden-images.md)) runs on each
+ node's first boot before networking: it skips the `pveupdate` Proxmox VE starts on
+ a first boot, removes the 5 s GRUB menu wait and, on clones, sets the identity,
+ so clones need no extra reboot.
+- While provisioning, nodes write without fsync (`zfs set sync=disabled` on `rpool`,
+ dpkg `force-unsafe-io`); the last step restores both, so a ready cluster has real
+ fsync semantics for fault injection.
+- Per-node work runs in parallel: installs, boots, configuration, ZFS pools, API
+ logins, Ceph managers and OSDs. Cluster joins, Ceph monitors and MDS daemons stay
+ sequential (the cluster file system and `ceph.conf` are not safe for concurrent
+ changes).
+- SSH probes time out their handshake after 3 s: QEMU's port forward accepts
+ connections before the guest is up and otherwise retries towards the guest only
+ every 12 s.
+- NIC 0 stays on QEMU user networking (slirp): host to node it measured 6-7 Gbit/s,
+ so passt would not speed up provisioning, and slirp also serves the nested guests
+ on `vmbr0` with DHCP.
+- Packages Proxbase installs go through a cache on the host (`aptcache`, reached as
+ `http://10.0.2.2:PORT` like the hook server), so each is downloaded once. It only
+ keeps `.deb` files; if apt rejects one (Hash Sum mismatch), Proxbase's apt runs
+ retry once with `Acquire::http::No-Cache`, which makes the cache fetch it again.
+
## Progress and the MCP server
Long operations report through a `progress.Sink` (a function taking an `Event`).
@@ -130,6 +163,9 @@ runs in the server process, guarded by the same cluster lock as the CLI.
for install and run, so interface names never change between them.
- NIC 0 is QEMU user networking (`vmbr0`, NAT, port forwards); NIC 1..n are unix
datagram sockets to the switch. MACs are derived from node and NIC number.
+- A virtio balloon with free page reporting returns memory the guest freed to the
+ host; disks use `cache=unsafe`, `discard=unmap` and `detect-zeroes=unmap`, and ZFS
+ pools in the nodes have `autotrim` on, so freed space shrinks the images.
- The installer runs in the foreground (killed if proxbase dies); installed nodes run
daemonized with a pidfile, a QMP socket and a serial console socket with a log file.
- Stop: ACPI powerdown via QMP, `quit` after the timeout, then SIGKILL.
@@ -156,6 +192,7 @@ runs in the server process, guarded by the same cluster lock as the CLI.
~/.local/share/proxbase/jobs/ .jsonl, .pid, .yaml of creates started over MCP
~/.cache/proxbase/iso/ ISOs, SHA256SUMS, extracted installer files
~/.cache/proxbase/images/ golden base images (create --golden)
+~/.cache/proxbase/apt/ packages served to the nodes (unused for 30 days: removed)
```
Design decisions and measurements are in [research/feasibility.md](research/feasibility.md).
diff --git a/docs/ci.md b/docs/ci.md
index 12431ca..09c8c96 100644
--- a/docs/ci.md
+++ b/docs/ci.md
@@ -1,7 +1,8 @@
# CI and automation
GitHub-hosted Linux runners expose `/dev/kvm`, so they can run a real cluster. A
-single node is ready in 2-3 minutes with a cached base image.
+single node is ready in well under a minute with a cached base image (21 s on a
+desktop host; runners are slower), and in seconds with a [template](#cluster-templates).
## GitHub Action
@@ -39,6 +40,7 @@ and creates the cluster with `--golden`. On failure it prints the node logs.
| `nodes`, `memory`, `data-disks`, `storage`, `pve-version` | `1`, file defaults | As the `create` flags |
| `golden` | `true` | Clone from a cached base image |
| `cache` | `true` | Cache `~/.cache/proxbase` |
+| `template` | `false` | Cache the ready cluster as a live snapshot and resume it in later runs (see below) |
| `verify-signature` | `false` | Verify with cosign (install it first) |
| Output | |
@@ -53,6 +55,31 @@ step. A complete workflow is in [examples/ci/github-actions.yml](../examples/ci/
the repository's own [E2E workflow](../.github/workflows/e2e.yml) uses the action
from the checkout with a freshly built binary.
+## Cluster templates
+
+With `template: "true"` the first run creates the cluster, takes a
+[live snapshot](lifecycle.md#live-snapshots) of it, stops it, saves the cluster
+directory with `actions/cache/save` and resumes it from the snapshot. Later runs
+restore that cache and resume the cluster in seconds instead of creating it.
+
+```yaml
+ - id: pve
+ uses: virtbase/proxbase@v0
+ with:
+ nodes: "1"
+ template: "true"
+```
+
+- The cache key covers the inputs (and the cluster file), the proxbase and QEMU
+ versions and the CPU model: a live snapshot only loads into the same QEMU on the
+ same CPU. A runner on another CPU model creates and caches its own template.
+- If resuming fails, the action destroys the cluster and creates it as usual.
+- The cached cluster includes its secrets (root password, SSH key, API token). They
+ only reach a cluster bound to the runner's loopback address, but anyone who can
+ read the repository's caches can read them.
+- Each node needs cache space for its disk changes and its memory (about 2 GB for
+ one idle 4 GiB node), next to the ISO and base image cache.
+
## Other CI systems
Any Linux runner with KVM works: install QEMU and the binary, then
diff --git a/docs/cli.md b/docs/cli.md
index f2f83c4..9ff4b2a 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -45,8 +45,8 @@ online and Ceph (if configured) is `HEALTH_OK`; use it for health checks.
| --- | --- |
| `node add [cluster] --count N` | Install, join and configure N new nodes |
| `node remove [cluster] ` | Drain and remove a node (`--force` with guests or degraded Ceph pools) |
-| `snapshot save [cluster] ` | Snapshot every disk of the stopped cluster |
-| `snapshot restore [cluster] ` | Reset every disk to a snapshot |
+| `snapshot save [cluster] ` | Snapshot every disk of the stopped cluster; `--live`: the running cluster with its memory |
+| `snapshot restore [cluster] ` | Reset every disk to a snapshot; a live snapshot also starts the cluster in its saved state |
| `snapshot delete [cluster] ` | Delete a snapshot |
| `snapshot list [cluster]` | List snapshots |
@@ -78,7 +78,7 @@ See [fault-injection.md](fault-injection.md).
| --- | --- |
| `image list` | Cached base images and the clusters using them |
| `image prune` | Delete base images no cluster uses |
-| `doctor` | Check KVM, nested virtualization, QEMU, memory, disk, ports (`--port-base`) |
+| `doctor` | Check KVM, nested virtualization, QEMU, memory, KSM, disk, ports (`--port-base`) |
| `config init [file]` | Write an annotated example (`--force` to overwrite) |
| `config validate ` | Strictly parse and check a cluster file |
| `config schema` | Print the JSON schema |
diff --git a/docs/configuration.md b/docs/configuration.md
index 0d7f443..df5ea5c 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -73,6 +73,7 @@ More in [examples/](../examples).
| `sshKeys` | none | Extra public keys for root, next to the generated one |
| `upgrade` | `false` | `apt dist-upgrade` on every node before clustering; reboots into a new kernel |
| `goldenImage` | `false` | Clone nodes from a cached base image instead of installing each ([golden images](golden-images.md)) |
+| `aptCache` | `true` | Packages Proxbase installs (`upgrade`, Ceph) are downloaded once into `~/.cache/proxbase/apt` and served to the nodes from there |
### `nodes`
diff --git a/docs/golden-images.md b/docs/golden-images.md
index 240b792..e0d81a8 100644
--- a/docs/golden-images.md
+++ b/docs/golden-images.md
@@ -11,33 +11,43 @@ proxbase image list # base images and the clusters using
proxbase image prune # delete base images no cluster uses
```
-Measured on the same host (3 nodes, ISO cached):
+Measured on the same host (3 nodes, 4 GiB each, ISO cached):
| | Installer | Golden image (cached) |
| --- | --- | --- |
-| ZFS create | 3m04s | 1m36s |
-| Ceph create | 4m43s | 3m03s |
-| `node add` | 3m21s | 1m14s |
+| ZFS create, 1 node | – | 21s |
+| ZFS create | 2m59s | 49s |
+| Ceph create | – | 1m30s |
-The first golden create builds the base image (about 1m40s) and takes about as long
-as a normal create.
+The first golden create builds the base image (1m56s; 3m43s with Ceph, which
+downloads the Ceph packages) and takes about as long as a normal create.
## How it works
-- A base image is the root disk of an installed but **never booted** node, cached in
+- A base image is the root disk of an installed node, cached in
`~/.cache/proxbase/images//`. The key covers the ISO, the root filesystem and
- size, and keyboard, country and time zone; nodes with different root disks use
- different base images.
+ size, keyboard, country and time zone, and the Ceph release if the cluster uses
+ Ceph; nodes with different root disks use different base images.
- Each node's root disk is a qcow2 overlay of the base image; data disks are created
fresh. The base image must stay while clusters use it, so `image prune` keeps those.
-- Everything Proxmox VE creates on first boot is created per node: the cluster CA, the
- auth key, the node certificate and root's SSH key.
-- What the installer wrote is replaced before clustering, over SSH with the base image's
- own key: host name (`/etc/hostname`, `/etc/hosts`, postfix), SSH host keys,
- machine-id, root password and authorized keys. The node reboots, the base image's
- node directory in `/etc/pve` is removed, and the new node certificate is checked.
-- A create interrupted during personalization resumes correctly: nodes that already
- have their identity are only finished.
+- The installer installs a first-boot hook (package `firstboot`) that runs on the
+ system's first boot, before networking and before any Proxmox VE service starts.
+- **Baking.** After the install, the base image boots once with a bake script: it
+ removes the GRUB menu wait, marks the appliance list as fetched (no `pveupdate` on
+ first boot) and, for Ceph clusters, installs the Ceph packages and the newer
+ `libpve-storage-perl` they need. Then it powers off before any Proxmox VE service
+ has started, and the hook stays pending for the clones.
+- **Identity.** Each clone gets its identity as a QEMU fw_cfg file. The hook applies
+ it on the clone's first boot, before networking: host name (`/etc/hostname`,
+ `/etc/hosts`, search domain, postfix), new SSH host keys, a new machine-id, the root
+ password and the authorized keys. Everything Proxmox VE creates on its first start
+ (node certificate, root's SSH key; the cluster CA and auth key on the first node)
+ is therefore created per node, under the right name. No reboot is needed.
+- Base images from Proxbase versions before the first-boot hook have a different
+ key, so the first golden create after an upgrade builds a new one; the old ones
+ stay for the clusters that use them (`image prune` removes them afterwards). A
+ create interrupted under an older version resumes by personalizing the remaining
+ clones over SSH, with a reboot.
Verified per node and across clusters: host keys, machine-ids, node certificates and
cluster CAs all differ.
diff --git a/docs/lifecycle.md b/docs/lifecycle.md
index 863b7b1..4add6c9 100644
--- a/docs/lifecycle.md
+++ b/docs/lifecycle.md
@@ -72,6 +72,31 @@ proxbase snapshot delete lab base
A snapshot can only be restored with the same set of nodes it was taken with.
Saving takes well under a second; restoring about a second.
+### Live snapshots
+
+A live snapshot is taken of the running cluster and also holds the memory of every
+node. Restoring it starts the cluster in that state: no boot, no waiting for
+quorum to form or Ceph to settle.
+
+```bash
+proxbase snapshot save lab ready --live # cluster running; nodes pause for the save
+proxbase stop lab --timeout 0 # restoring needs a stopped cluster
+proxbase snapshot restore lab ready # nodes resume where they were
+```
+
+- All nodes are paused while their snapshots are written, so the snapshot shows one
+ moment of the whole cluster; they continue afterwards. Faults must be cleared.
+- On restore, every node loads its snapshot paused and all are resumed together.
+ The node clocks stood still in the snapshot; Proxbase sets them to the host's time,
+ then waits for quorum, the API and storage health as `start` does.
+- Memory is stored in the root disk image, so a live snapshot needs about as much
+ disk space as the nodes use memory (1.8 GB per idle 4 GiB node).
+- On a 3-node ZFS cluster, saving took 3 s and restoring to a quorate, ready
+ cluster 5 s.
+- It loads only into the same QEMU version on the same CPU model (nodes use
+ `-cpu host`); after a QEMU upgrade, take it again.
+- `snapshot delete` works on a running cluster for live snapshots.
+
## Access
```bash
diff --git a/docs/mcp.md b/docs/mcp.md
index 9e333ab..1c555e1 100644
--- a/docs/mcp.md
+++ b/docs/mcp.md
@@ -65,7 +65,7 @@ Clusters created in the container live in the `proxbase` volume; use
| `cluster_destroy` | Delete a cluster; `confirm` must repeat the cluster name |
| `node_exec` | Run a command as root on a node; returns `stdout`, `stderr`, `exitCode` |
| `cluster_env` | API endpoint, token, CA certificate, SSH key and ports; root password only with `includeRootPassword` |
-| `snapshot_save`, `snapshot_restore`, `snapshot_delete`, `snapshot_list` | Snapshots of the stopped cluster |
+| `snapshot_save`, `snapshot_restore`, `snapshot_delete`, `snapshot_list` | Snapshots of the stopped cluster, or with `live` of the running cluster including memory (restoring resumes it) |
| `fault_apply`, `fault_clear` | kill, freeze, thaw, link-down/up, partition, degrade |
| `config_validate` | Check a cluster file; returns errors, warnings and the file with defaults |
@@ -75,8 +75,8 @@ clients can ask before `cluster_destroy`, `snapshot_restore` or `fault_apply`.
## Long-running creates
-A create takes 2-3 minutes with a cached [golden image](golden-images.md) (the
-default for `cluster_create`) and 10-20 minutes the first time, when it downloads the
+A create takes under a minute with a cached [golden image](golden-images.md) (the
+default for `cluster_create`; Ceph about 1.5 minutes) and 10-20 minutes the first time, when it downloads the
ISO and builds the base image. Many clients time out tool calls earlier, so:
- `cluster_create` starts the create as a background job on the host
diff --git a/docs/storage.md b/docs/storage.md
index a79dcb0..7d57e61 100644
--- a/docs/storage.md
+++ b/docs/storage.md
@@ -20,7 +20,9 @@ storage:
Every node has its own pool with the same name; live migration of VMs on it copies
the disks (`qm migrate --with-local-disks`). Pools are created through the Proxmox
-API (`POST /nodes/{node}/disks/zfs`, `ashift=12`).
+API (`POST /nodes/{node}/disks/zfs`, `ashift=12`), on all nodes at once, with
+`autotrim` on (also for `rpool`), so space freed in a pool shrinks the disk image
+on the host.
## Ceph
@@ -44,6 +46,17 @@ hyperconverged cluster:
`create` and `start` wait until Ceph reports `HEALTH_OK`, every OSD is up and in, all
placement groups are `active+clean` and the Ceph storages are active on every node.
+Monitors are created one after another (each joins the quorum first) and MDS daemons
+too (`pveceph` rewrites `ceph.conf` for each without a lock); managers and OSDs are
+created on all nodes at once. With `--golden`, the Ceph packages are baked into the
+base image ([golden images](golden-images.md)), so nodes skip `pveceph install`.
+
+Memory targets for small nodes: Ceph's defaults are sized for dedicated servers
+(4 GiB per OSD, 4 GiB MDS cache) and the daemons grow their caches toward them.
+Proxbase sets `osd_memory_target` to 2 GiB (about the lowest Ceph developers
+recommend) and `mds_cache_memory_limit` to 1 GiB, unless they are set already, so
+a value changed with `ceph config set` stays.
+
Notes:
- Ceph needs RAM: the default node memory becomes 6G; less only prints a warning.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 865e61f..df5af70 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -8,7 +8,7 @@ Everything of a cluster lives in `~/.local/share/proxbase/clusters//`:
| --- | --- |
| `logs/-install.log` | Serial console of the installer |
| `logs/-qemu.log` | QEMU errors during the install |
-| `logs/-console.log` | Serial console of the running node |
+| `logs/-console.log` | Serial console of the running node; the first-boot hook reports `proxbase-first-boot: identity ` there |
| `logs/switch.log` | Internal network switch |
| `state.json`, `cluster.yaml` | Progress and the resolved configuration |
| `secrets/` | Root password, SSH key, API token, CA (mode 0600) |
@@ -25,8 +25,10 @@ Everything of a cluster lives in `~/.local/share/proxbase/clusters//`:
| `port 18001 for pve1 is in use` | Another cluster or program uses the ports; set `access.portBase` |
| `socket path … is too long` | Unix socket paths are limited to 107 bytes; use a shorter `XDG_DATA_HOME` or cluster/network names |
| Install times out or `installer reported failure` | Read `logs/-install.log`; the last lines are also in the error |
-| `fetch checksums` fails | The ISO mirror is unreachable; set `proxmox.mirror` |
-| `Hash Sum mismatch` during apt (nodes or image build) | A broken mirror or path (seen over IPv6). Retry; for image builds pass `--build-arg APT_OPTS="-o Acquire::ForceIPv4=true"` |
+| `fetch checksums` fails | The ISO mirror is unreachable; set `proxmox.mirror`. An exact `proxmox.version` (`9.2-1`) that is cached needs no network |
+| `the cluster join left the node certificate missing or from another CA; issued a new one` | Not an error: a Proxmox VE join can lose the node's files or replace the cluster CA when the cluster file system syncs late; Proxbase detects it after the joins and issues a new node certificate from the current CA |
+| `bake base image failed` | Building a [golden image](golden-images.md) failed while baking; the error shows the end of the serial console (for Ceph usually the package download) |
+| `Hash Sum mismatch` during apt (nodes or image build) | A broken mirror or path (seen over IPv6). Proxbase's own apt runs retry once past the package cache; if it persists, retry later. For Docker image builds pass `--build-arg APT_OPTS="-o Acquire::ForceIPv4=true"`. `rm -rf ~/.cache/proxbase/apt` empties the package cache |
| `ceph not ready after 10m0s` | The error includes `ceph -s` and `ceph health detail`; usually too little memory per node |
| Nested guests are slow or fail to start | Nested virtualization is off on the host (`proxbase doctor`) |
| `cluster "x" is busy` | Another proxbase command holds the cluster lock, e.g. a create started over MCP (`cluster_status` shows it) |
diff --git a/examples/ci/github-actions.yml b/examples/ci/github-actions.yml
index b3efba7..26d42ab 100644
--- a/examples/ci/github-actions.yml
+++ b/examples/ci/github-actions.yml
@@ -23,6 +23,7 @@ jobs:
name: ci
nodes: "1"
# config: test/cluster.yaml # or a cluster file
+ # template: "true" # cache the ready cluster, resume it in seconds next time
- name: Run tests
env:
diff --git a/internal/answer/answer.go b/internal/answer/answer.go
index 9879f3d..9daa269 100644
--- a/internal/answer/answer.go
+++ b/internal/answer/answer.go
@@ -18,6 +18,9 @@ type Params struct {
SSHKeys []string
MAC string // MAC of the NAT NIC; only this NIC gets the static install-time address
Filesystem string // zfs, ext4, xfs, btrfs
+ // FirstBootURL, if set, is a script the installer downloads and the
+ // installed system runs once on its first boot, before networking starts.
+ FirstBootURL string
}
// Render returns answer.toml. The NAT NIC uses QEMU user networking (10.0.2.0/24).
@@ -54,6 +57,13 @@ func Render(p Params) string {
case "btrfs":
w("btrfs.raid = \"raid0\"")
}
+ if p.FirstBootURL != "" {
+ w("")
+ w("[first-boot]")
+ w("source = \"from-url\"")
+ w("url = %s", quote(p.FirstBootURL))
+ w("ordering = \"before-network\"")
+ }
return b.String()
}
diff --git a/internal/answer/answer_test.go b/internal/answer/answer_test.go
index 522ee91..d1dd9fc 100644
--- a/internal/answer/answer_test.go
+++ b/internal/answer/answer_test.go
@@ -30,6 +30,17 @@ func TestRenderExt4(t *testing.T) {
}
}
+func TestRenderFirstBoot(t *testing.T) {
+ got := Render(Params{Filesystem: "zfs", FirstBootURL: "http://10.0.2.2:4242/first-boot"})
+ want := "\n[first-boot]\nsource = \"from-url\"\nurl = \"http://10.0.2.2:4242/first-boot\"\nordering = \"before-network\"\n"
+ if !strings.HasSuffix(got, want) {
+ t.Fatalf("got:\n%s\nwant suffix:\n%s", got, want)
+ }
+ if strings.Contains(Render(Params{Filesystem: "zfs"}), "[first-boot]") {
+ t.Fatal("no first-boot section without a URL")
+ }
+}
+
func TestPassword(t *testing.T) {
if a, b := Password(20), Password(20); len(a) != 20 || a == b {
t.Fatalf("bad passwords %q %q", a, b)
diff --git a/internal/aptcache/aptcache.go b/internal/aptcache/aptcache.go
new file mode 100644
index 0000000..659aa41
--- /dev/null
+++ b/internal/aptcache/aptcache.go
@@ -0,0 +1,256 @@
+// Package aptcache is a caching HTTP proxy for the nodes' apt. Nodes of a
+// cluster download the same packages (Ceph, upgrades), and every create downloads
+// them again; through the proxy each package is fetched from the mirror once and
+// then served from the host's cache.
+//
+// Only .deb files are cached: their names are versioned, so the content never
+// changes, and apt checks their hashes against the signed repository index. All
+// other requests (indexes, Release files) go to the mirror unchanged. Requests
+// are limited to the Debian and Proxmox repository hosts.
+//
+// A package that apt rejects (Hash Sum mismatch: a mirror served a broken file)
+// is fetched again when apt asks with Cache-Control: no-cache
+// (-o Acquire::http::No-Cache=true), which proxbase's apt runs do on a retry.
+package aptcache
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "math/rand/v2"
+ "net"
+ "net/http"
+ "os"
+ "path"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "golang.org/x/sync/singleflight"
+)
+
+// Hosts are the repository hosts the proxy serves.
+var Hosts = []string{"download.proxmox.com", "deb.debian.org", "security.debian.org"}
+
+// maxAge is how long a cached package is kept after it was last used.
+const maxAge = 30 * 24 * time.Hour
+
+// hostAddr is the host as seen from a guest on QEMU user networking.
+const hostAddr = "10.0.2.2"
+
+// Proxy is a running apt cache.
+type Proxy struct {
+ root *os.Root // the cache directory; no file access can leave it
+ srv *http.Server
+ client *http.Client
+ inflight singleflight.Group
+ // URL is the proxy's address as seen from the nodes (for http_proxy).
+ URL string
+ // Addr is the address the proxy listens on, on the host.
+ Addr string
+}
+
+// Start serves the cache in dir on a free port of the host's loopback address,
+// which nodes on QEMU user networking reach as 10.0.2.2. It first removes
+// packages unused for 30 days.
+func Start(dir string) (*Proxy, error) {
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return nil, err
+ }
+ root, err := os.OpenRoot(dir)
+ if err != nil {
+ return nil, fmt.Errorf("apt cache: %w", err)
+ }
+ prune(root, time.Now().Add(-maxAge))
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ root.Close()
+ return nil, fmt.Errorf("apt cache: %w", err)
+ }
+ p := &Proxy{
+ root: root,
+ client: &http.Client{Transport: &http.Transport{Proxy: http.ProxyFromEnvironment, IdleConnTimeout: 30 * time.Second}},
+ URL: "http://" + net.JoinHostPort(hostAddr, strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)),
+ Addr: ln.Addr().String(),
+ }
+ p.srv = &http.Server{Handler: p, ReadHeaderTimeout: 30 * time.Second}
+ // Serve returns when Close is called; apt reports any other failure itself.
+ go func() { _ = p.srv.Serve(ln) }()
+ return p, nil
+}
+
+// Close stops the proxy.
+func (p *Proxy) Close() error { return errors.Join(p.srv.Close(), p.root.Close()) }
+
+// ServeHTTP handles proxy requests ("GET http://host/path").
+func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet && r.Method != http.MethodHead {
+ http.Error(w, "only GET and HEAD", http.StatusMethodNotAllowed)
+ return
+ }
+ u := r.URL
+ if u.Scheme != "http" || !slices.Contains(Hosts, u.Hostname()) || (u.Port() != "" && u.Port() != "80") {
+ http.Error(w, "not a repository host", http.StatusForbidden)
+ return
+ }
+ clean := path.Clean("/" + u.Path)
+ if r.Method == http.MethodGet && strings.HasSuffix(clean, ".deb") && u.RawQuery == "" {
+ p.serveDeb(w, r, u.Hostname(), clean)
+ return
+ }
+ p.pass(w, r)
+}
+
+// pass forwards a request to the mirror and streams the response back.
+func (p *Proxy) pass(w http.ResponseWriter, r *http.Request) {
+ // ServeHTTP admits only plain HTTP to the repository hosts.
+ req, err := http.NewRequestWithContext(r.Context(), r.Method, r.URL.String(), nil) //nolint:gosec // allow-listed hosts only
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ for _, h := range []string{"If-Modified-Since", "If-None-Match", "Range", "If-Range", "Cache-Control", "User-Agent"} {
+ if v := r.Header.Get(h); v != "" {
+ req.Header.Set(h, v)
+ }
+ }
+ resp, err := p.client.Do(req) //nolint:gosec // allow-listed hosts only
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ defer resp.Body.Close()
+ for _, h := range []string{"Content-Type", "Content-Length", "Last-Modified", "ETag", "Content-Range", "Accept-Ranges"} {
+ if v := resp.Header.Get(h); v != "" {
+ w.Header().Set(h, v)
+ }
+ }
+ w.WriteHeader(resp.StatusCode)
+ _, _ = io.Copy(w, resp.Body)
+}
+
+// serveDeb serves a package from the cache, downloading it first if needed.
+func (p *Proxy) serveDeb(w http.ResponseWriter, r *http.Request, host, clean string) {
+ // clean is rooted and cleaned, so the name stays below host/.
+ name := filepath.Join(host, filepath.FromSlash(strings.TrimPrefix(clean, "/")))
+ _, err := p.root.Stat(name)
+ if err != nil || noCache(r) {
+ // Concurrent requests for the same package share one download. It must
+ // not end when the first requester goes away.
+ ctx := context.WithoutCancel(r.Context())
+ _, err, _ := p.inflight.Do(name, func() (any, error) {
+ return nil, p.download(ctx, "http://"+host+clean, name)
+ })
+ if err != nil {
+ var se statusError
+ if errors.As(err, &se) {
+ http.Error(w, err.Error(), int(se))
+ return
+ }
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ }
+ f, err := p.root.Open(name)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ defer f.Close()
+ fi, err := f.Stat()
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ now := time.Now()
+ _ = p.root.Chtimes(name, now, now) // last use, for pruning
+ w.Header().Set("Content-Type", "application/vnd.debian.binary-package")
+ http.ServeContent(w, r, "", fi.ModTime(), f)
+}
+
+// statusError is a non-200 status from the mirror.
+type statusError int
+
+func (e statusError) Error() string { return "mirror answered " + http.StatusText(int(e)) }
+
+// download fetches url into the cache as name, atomically and only if complete.
+func (p *Proxy) download(ctx context.Context, url, name string) error {
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) //nolint:gosec // allow-listed hosts only
+ if err != nil {
+ return err
+ }
+ resp, err := p.client.Do(req) //nolint:gosec // allow-listed hosts only
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return statusError(resp.StatusCode)
+ }
+ if err := p.root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
+ return err
+ }
+ tmp := filepath.Join(filepath.Dir(name), fmt.Sprintf(".part-%d", rand.Uint64())) //nolint:gosec // a unique name, not a secret
+ f, err := p.root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = p.root.Remove(tmp) }() // gone after the rename
+ n, err := io.Copy(f, resp.Body)
+ if cerr := f.Close(); err == nil {
+ err = cerr
+ }
+ if err != nil {
+ return fmt.Errorf("download %s: %w", url, err)
+ }
+ if resp.ContentLength >= 0 && n != resp.ContentLength {
+ return fmt.Errorf("download %s: got %d of %d bytes", url, n, resp.ContentLength)
+ }
+ if err := checkDeb(p.root, tmp); err != nil {
+ return fmt.Errorf("download %s: %w", url, err)
+ }
+ return p.root.Rename(tmp, name)
+}
+
+// noCache reports whether the client asked to bypass caches.
+func noCache(r *http.Request) bool {
+ return strings.Contains(r.Header.Get("Cache-Control"), "no-cache") || r.Header.Get("Pragma") == "no-cache"
+}
+
+// debMagic starts every .deb file (an ar archive).
+const debMagic = "!\n"
+
+// checkDeb rejects files that are not .deb packages, such as error pages.
+func checkDeb(root *os.Root, name string) error {
+ f, err := root.Open(name)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ head := make([]byte, len(debMagic))
+ if _, err := io.ReadFull(f, head); err != nil || string(head) != debMagic {
+ return errors.New("not a Debian package")
+ }
+ return nil
+}
+
+// prune removes cached packages last used before cutoff, and leftovers of
+// interrupted downloads.
+func prune(root *os.Root, cutoff time.Time) {
+ _ = fs.WalkDir(root.FS(), ".", func(p string, d fs.DirEntry, err error) error {
+ if err != nil || d.IsDir() {
+ return nil
+ }
+ info, err := d.Info()
+ if err == nil && (info.ModTime().Before(cutoff) || strings.HasPrefix(d.Name(), ".part-")) {
+ _ = root.Remove(p)
+ }
+ return nil
+ })
+}
diff --git a/internal/aptcache/aptcache_test.go b/internal/aptcache/aptcache_test.go
new file mode 100644
index 0000000..ecff609
--- /dev/null
+++ b/internal/aptcache/aptcache_test.go
@@ -0,0 +1,242 @@
+package aptcache
+
+import (
+ "context"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// mirror is a fake repository that counts requests per path.
+type mirror struct {
+ srv *httptest.Server
+ mu sync.Mutex
+ hits map[string]int
+ gate chan struct{} // if set, .deb responses wait for it
+}
+
+func newMirror(t *testing.T) *mirror {
+ t.Helper()
+ m := &mirror{hits: map[string]int{}}
+ m.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ m.mu.Lock()
+ m.hits[r.URL.Path]++
+ gate := m.gate
+ m.mu.Unlock()
+ switch r.URL.Path {
+ case "/missing.deb":
+ http.NotFound(w, r)
+ return
+ case "/html.deb":
+ _, _ = io.WriteString(w, "captive portal")
+ return
+ }
+ if gate != nil {
+ <-gate
+ }
+ body := "content of " + r.URL.Path
+ if strings.HasSuffix(r.URL.Path, ".deb") {
+ body = debMagic + body
+ }
+ _, _ = io.WriteString(w, body) //nolint:gosec // test server, plain text
+ }))
+ t.Cleanup(m.srv.Close)
+ return m
+}
+
+func (m *mirror) count(p string) int {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return m.hits[p]
+}
+
+// start runs a proxy whose upstream connections all go to the fake mirror.
+func start(t *testing.T, m *mirror) (*Proxy, *http.Client) {
+ t.Helper()
+ p, err := Start(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { p.Close() })
+ mirrorAddr := m.srv.Listener.Addr().String()
+ p.client = &http.Client{Transport: &http.Transport{
+ DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
+ var d net.Dialer
+ return d.DialContext(ctx, network, mirrorAddr)
+ },
+ }}
+ proxyURL, err := url.Parse("http://" + p.Addr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return p, &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}, Timeout: 10 * time.Second}
+}
+
+func fetch(ctx context.Context, c *http.Client, u string) (int, string, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+ if err != nil {
+ return 0, "", err
+ }
+ resp, err := c.Do(req)
+ if err != nil {
+ return 0, "", err
+ }
+ defer resp.Body.Close()
+ b, err := io.ReadAll(resp.Body)
+ return resp.StatusCode, string(b), err
+}
+
+func get(t *testing.T, c *http.Client, u string) (int, string) {
+ t.Helper()
+ code, body, err := fetch(t.Context(), c, u)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return code, body
+}
+
+func TestProxy_CachesPackages(t *testing.T) {
+ t.Parallel()
+ m := newMirror(t)
+ _, c := start(t, m)
+ const deb = "http://deb.debian.org/debian/pool/main/c/ceph/ceph_19.2.3_amd64.deb"
+ for range 2 {
+ code, body := get(t, c, deb)
+ if code != http.StatusOK || body != debMagic+"content of /debian/pool/main/c/ceph/ceph_19.2.3_amd64.deb" {
+ t.Fatalf("got %d %q", code, body)
+ }
+ }
+ if n := m.count("/debian/pool/main/c/ceph/ceph_19.2.3_amd64.deb"); n != 1 {
+ t.Fatalf("mirror asked %d times, want 1", n)
+ }
+}
+
+// apt asks with no-cache after a Hash Sum mismatch; the package is fetched again.
+func TestProxy_NoCacheRefetches(t *testing.T) {
+ t.Parallel()
+ m := newMirror(t)
+ _, c := start(t, m)
+ const deb = "http://deb.debian.org/debian/pool/main/libn/libnvme/libnvme1t64_1.13-2_amd64.deb"
+ get(t, c, deb)
+ req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, deb, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Cache-Control", "no-cache")
+ resp, err := c.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("status %d", resp.StatusCode)
+ }
+ if n := m.count("/debian/pool/main/libn/libnvme/libnvme1t64_1.13-2_amd64.deb"); n != 2 {
+ t.Fatalf("mirror asked %d times, want 2", n)
+ }
+}
+
+func TestProxy_PassesIndexes(t *testing.T) {
+ t.Parallel()
+ m := newMirror(t)
+ _, c := start(t, m)
+ for range 2 {
+ if code, _ := get(t, c, "http://download.proxmox.com/debian/pve/dists/trixie/InRelease"); code != http.StatusOK {
+ t.Fatalf("status %d", code)
+ }
+ }
+ if n := m.count("/debian/pve/dists/trixie/InRelease"); n != 2 {
+ t.Fatalf("indexes must not be cached: mirror asked %d times, want 2", n)
+ }
+}
+
+func TestProxy_SharesConcurrentDownloads(t *testing.T) {
+ t.Parallel()
+ m := newMirror(t)
+ m.gate = make(chan struct{})
+ _, c := start(t, m)
+ const deb = "http://download.proxmox.com/debian/ceph-squid/pool/x.deb"
+ var wg sync.WaitGroup
+ var ok atomic.Int32
+ for range 5 {
+ wg.Go(func() {
+ if code, _, err := fetch(t.Context(), c, deb); err == nil && code == http.StatusOK {
+ ok.Add(1)
+ }
+ })
+ }
+ // Let the requests arrive, then release the single upstream download.
+ for m.count("/debian/ceph-squid/pool/x.deb") == 0 {
+ time.Sleep(10 * time.Millisecond)
+ }
+ time.Sleep(100 * time.Millisecond)
+ close(m.gate)
+ wg.Wait()
+ if ok.Load() != 5 {
+ t.Fatalf("%d of 5 requests succeeded", ok.Load())
+ }
+ if n := m.count("/debian/ceph-squid/pool/x.deb"); n != 1 {
+ t.Fatalf("mirror asked %d times, want 1", n)
+ }
+}
+
+func TestProxy_Rejects(t *testing.T) {
+ t.Parallel()
+ m := newMirror(t)
+ _, c := start(t, m)
+ tests := []struct {
+ name string
+ url string
+ want int
+ }{
+ {name: "other host", url: "http://example.com/x.deb", want: http.StatusForbidden},
+ {name: "other port", url: "http://deb.debian.org:8080/x.deb", want: http.StatusForbidden},
+ {name: "missing package", url: "http://deb.debian.org/missing.deb", want: http.StatusNotFound},
+ {name: "not a package", url: "http://deb.debian.org/html.deb", want: http.StatusBadGateway},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if code, _ := get(t, c, tt.url); code != tt.want {
+ t.Errorf("status %d, want %d", code, tt.want)
+ }
+ })
+ }
+}
+
+func TestPrune(t *testing.T) {
+ t.Parallel()
+ dir := t.TempDir()
+ old, recent, part := filepath.Join(dir, "a", "old.deb"), filepath.Join(dir, "a", "new.deb"), filepath.Join(dir, "a", ".part-1")
+ if err := os.MkdirAll(filepath.Dir(old), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ for _, p := range []string{old, recent, part} {
+ if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ past := time.Now().Add(-2 * maxAge)
+ if err := os.Chtimes(old, past, past); err != nil {
+ t.Fatal(err)
+ }
+ root, err := os.OpenRoot(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer root.Close()
+ prune(root, time.Now().Add(-maxAge))
+ for p, want := range map[string]bool{old: false, recent: true, part: false} {
+ if _, err := os.Stat(p); (err == nil) != want {
+ t.Errorf("%s exists=%v, want %v", filepath.Base(p), err == nil, want)
+ }
+ }
+}
diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go
index a520469..40fea92 100644
--- a/internal/cli/doctor.go
+++ b/internal/cli/doctor.go
@@ -28,7 +28,7 @@ func doctorCmd() *cobra.Command {
Short: "Check that this host can run proxbase clusters",
Args: cobra.NoArgs,
RunE: func(*cobra.Command, []string) error {
- checks := []check{checkKVM(), checkNested(), checkQEMU(), checkQEMUImg(), checkMemory(), checkDisk(), checkPorts(portBase), checkSocketPath()}
+ checks := []check{checkKVM(), checkNested(), checkQEMU(), checkQEMUImg(), checkMemory(), checkKSM(), checkDisk(), checkPorts(portBase), checkSocketPath()}
failed := false
for _, c := range checks {
fmt.Printf("%-5s %-12s %s\n", c.status, c.name, c.detail)
@@ -119,6 +119,20 @@ func checkMemory() check {
return warn("memory", "MemAvailable not found")
}
+// checkKSM reports whether the kernel merges identical memory pages. Nodes run
+// the same Proxmox VE, so with KSM they share much of their memory; QEMU already
+// marks guest memory as mergeable, but only root can turn KSM on.
+func checkKSM() check {
+ b, err := os.ReadFile("/sys/kernel/mm/ksm/run")
+ if err != nil {
+ return warn("ksm", "not available in this kernel")
+ }
+ if strings.TrimSpace(string(b)) != "1" {
+ return warn("ksm", "off; nodes share identical memory with it: echo 1 | sudo tee /sys/kernel/mm/ksm/run")
+ }
+ return ok("ksm", "on: identical memory pages of the nodes are shared")
+}
+
func checkDisk() check {
dir := state.DataHome()
for {
diff --git a/internal/cli/image.go b/internal/cli/image.go
index ceac206..8f164db 100644
--- a/internal/cli/image.go
+++ b/internal/cli/image.go
@@ -28,9 +28,13 @@ func imageCmd() *cobra.Command {
return err
}
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
- fmt.Fprintln(w, "KEY\tISO\tROOT\tCREATED\tUSED BY")
+ fmt.Fprintln(w, "KEY\tISO\tROOT\tCEPH\tCREATED\tUSED BY")
for _, m := range images {
- fmt.Fprintf(w, "%s\t%s\t%s %s\t%s\t%v\n", m.Key, m.ISO, m.Filesystem, m.Size, m.Created.Local().Format(time.DateTime), users[m.Key])
+ ceph := m.Ceph
+ if ceph == "" {
+ ceph = "-"
+ }
+ fmt.Fprintf(w, "%s\t%s\t%s %s\t%s\t%s\t%v\n", m.Key, m.ISO, m.Filesystem, m.Size, ceph, m.Created.Local().Format(time.DateTime), users[m.Key])
}
return w.Flush()
},
diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go
index da0c61c..292e62d 100644
--- a/internal/cli/snapshot.go
+++ b/internal/cli/snapshot.go
@@ -13,7 +13,16 @@ import (
)
func snapshotCmd() *cobra.Command {
- cmd := &cobra.Command{Use: "snapshot", Short: "Save and restore all disks of a stopped cluster"}
+ cmd := &cobra.Command{
+ Use: "snapshot",
+ Short: "Save and restore a whole cluster",
+ Long: `snapshot saves and restores all disks of a cluster at once.
+
+A snapshot of a stopped cluster holds its disks; after a restore, start the
+cluster as usual. A live snapshot (save --live) is taken of the running cluster
+and also holds the memory of every node; restoring it starts the cluster in that
+state within seconds, without booting.`,
+ }
op := func(use, short string, f func(*cluster.Cluster, context.Context, string) error) *cobra.Command {
return &cobra.Command{
Use: use + " [cluster] ",
@@ -44,16 +53,24 @@ func snapshotCmd() *cobra.Command {
return err
}
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
- fmt.Fprintln(w, "NAME\tCREATED\tNODES")
+ fmt.Fprintln(w, "NAME\tCREATED\tNODES\tLIVE")
for _, s := range c.St.Snapshots {
- fmt.Fprintf(w, "%s\t%s\t%d\n", s.Name, s.Created.Local().Format(time.DateTime), len(s.Nodes))
+ fmt.Fprintf(w, "%s\t%s\t%d\t%v\n", s.Name, s.Created.Local().Format(time.DateTime), len(s.Nodes), s.Live)
}
return w.Flush()
},
}
+ var live bool
+ save := op("save", "Snapshot every disk (cluster stopped), or with --live the running cluster", func(c *cluster.Cluster, ctx context.Context, name string) error {
+ if live {
+ return c.SnapshotSaveLive(ctx, name)
+ }
+ return c.SnapshotSave(ctx, name)
+ })
+ save.Flags().BoolVar(&live, "live", false, "snapshot the running cluster, memory included; restoring it resumes the cluster")
cmd.AddCommand(
- op("save", "Snapshot every disk (cluster must be stopped)", (*cluster.Cluster).SnapshotSave),
- op("restore", "Reset every disk to a snapshot (cluster must be stopped)", (*cluster.Cluster).SnapshotRestore),
+ save,
+ op("restore", "Reset every disk to a snapshot (cluster must be stopped); a live snapshot also starts the cluster", (*cluster.Cluster).SnapshotRestore),
op("delete", "Delete a snapshot", (*cluster.Cluster).SnapshotDelete),
list,
)
diff --git a/internal/cluster/access.go b/internal/cluster/access.go
index e6d04f3..1621ef3 100644
--- a/internal/cluster/access.go
+++ b/internal/cluster/access.go
@@ -21,9 +21,7 @@ func (c *Cluster) ensureToken(ctx context.Context) error {
return err
}
if tok, err := c.Token(); err == nil && tok != "" {
- probe := *api
- probe.SetToken(tok)
- if probe.Get(ctx, "/version", &struct{}{}) == nil {
+ if api.WithToken(tok).Get(ctx, "/version", &struct{}{}) == nil {
return nil
}
}
diff --git a/internal/cluster/cluster.go b/internal/cluster/cluster.go
index 153c949..b8a647f 100644
--- a/internal/cluster/cluster.go
+++ b/internal/cluster/cluster.go
@@ -17,6 +17,7 @@ import (
"golang.org/x/crypto/ssh"
"github.com/virtbase/proxbase/internal/answer"
+ "github.com/virtbase/proxbase/internal/aptcache"
"github.com/virtbase/proxbase/internal/config"
"github.com/virtbase/proxbase/internal/network"
"github.com/virtbase/proxbase/internal/progress"
@@ -50,6 +51,7 @@ type Cluster struct {
password string
runtime vm.Runtime
net network.Switch
+ apt *aptcache.Proxy // package cache for the nodes while create runs, or nil
}
func newCluster(d state.Dir, cfg *config.Cluster, st *state.State, p progress.Sink) (*Cluster, error) {
@@ -183,6 +185,14 @@ func (c *Cluster) step(format string, a ...any) {
func (c *Cluster) save() error { return c.Dir.Save(c.St) }
+// aptEnv points apt on the nodes at the package cache, if one runs.
+func (c *Cluster) aptEnv() string {
+ if c.apt == nil {
+ return ""
+ }
+ return "export http_proxy=" + c.apt.URL + "\n"
+}
+
// storages returns the configured storage backends in setup order.
func (c *Cluster) storages() []storage.Backend {
var out []storage.Backend
diff --git a/internal/cluster/create.go b/internal/cluster/create.go
index b1e6e12..bfd9b9f 100644
--- a/internal/cluster/create.go
+++ b/internal/cluster/create.go
@@ -10,6 +10,7 @@ import (
"strconv"
"time"
+ "github.com/virtbase/proxbase/internal/aptcache"
"github.com/virtbase/proxbase/internal/config"
"github.com/virtbase/proxbase/internal/image"
"github.com/virtbase/proxbase/internal/network"
@@ -68,6 +69,15 @@ func (c *Cluster) create(ctx context.Context) error {
if err := c.save(); err != nil {
return err
}
+ if *c.Cfg.Proxmox.AptCache {
+ if c.apt, err = aptcache.Start(filepath.Join(state.CacheHome(), "apt")); err != nil {
+ return err
+ }
+ defer func() {
+ c.apt.Close()
+ c.apt = nil
+ }()
+ }
steps := []func(context.Context) error{
func(context.Context) error { return c.net.Start(c.Cfg) },
func(ctx context.Context) error { return c.installAll(ctx, img) },
@@ -76,9 +86,11 @@ func (c *Cluster) create(ctx context.Context) error {
c.postInstall,
c.formCluster,
func(ctx context.Context) error { return c.waitQuorum(ctx, 5*time.Minute) },
+ c.repairJoins,
c.setupStorage,
c.ensureToken,
c.exportCA,
+ c.endProvisioning,
}
for _, step := range steps {
if err := step(ctx); err != nil {
diff --git a/internal/cluster/golden.go b/internal/cluster/golden.go
index f809c4d..49c838a 100644
--- a/internal/cluster/golden.go
+++ b/internal/cluster/golden.go
@@ -21,13 +21,14 @@ import (
// cloneAll creates the nodes' disks from base images instead of installing.
func (c *Cluster) cloneAll(ctx context.Context, img *image.Image, todo []config.Node) error {
cloner, ok := c.runtime.(vm.Cloner)
- if !ok {
- return errors.New("the node runtime cannot clone base images")
+ builder, ok2 := c.runtime.(golden.Builder)
+ if !ok || !ok2 {
+ return errors.New("the node runtime cannot build or clone base images")
}
t0 := time.Now()
var names []string
for _, n := range todo {
- base, err := golden.Ensure(ctx, c.runtime, c.Cfg, n, img, c.step)
+ base, err := golden.Ensure(ctx, builder, c.Cfg, n, img, c.aptEnv(), c.step)
if err != nil {
return err
}
@@ -42,8 +43,10 @@ func (c *Cluster) cloneAll(ctx context.Context, img *image.Image, todo []config.
return c.save()
}
-// personalizeAll gives cloned nodes their own identity (host name, SSH host keys,
-// machine-id, password, authorized keys) and reboots them.
+// personalizeAll completes the identity of cloned nodes (host name, SSH host
+// keys, machine-id, password, authorized keys). Nodes from current base images
+// received it on their first boot; nodes from base images built before the
+// first-boot hook are personalized over SSH and rebooted.
func (c *Cluster) personalizeAll(ctx context.Context) error {
pub, err := c.Dir.ReadSecret(secretKey + ".pub")
if err != nil {
@@ -76,19 +79,23 @@ func (c *Cluster) personalize(ctx context.Context, name string, keys []string, m
ns.Pending, ns.HostKey = false, ""
mu.Unlock()
}
- // A previous run may have personalized the node already (golden key gone).
- if s, err := c.dial(ctx, name, c.signer, ""); err == nil {
- out, rerr := s.Run("cat " + golden.IdentityMarker + " 2>/dev/null || true")
- s.Close()
- if rerr == nil && strings.TrimSpace(out) == name {
- done()
- return c.finishIdentity(ctx, name)
- }
+ // The cluster key works once the node has its identity (first-boot hook or a
+ // previous run); until then only the base image's key does.
+ s, err := c.waitSSH(ctx, name, 5*time.Minute)
+ if err != nil {
+ return err
}
- s, err := c.waitSSH(ctx, name, 5*time.Minute) // golden key while pending
+ out, err := s.Run("cat " + golden.IdentityMarker + " 2>/dev/null || true")
if err != nil {
+ s.Close()
return err
}
+ if strings.TrimSpace(out) == name {
+ s.Close()
+ done()
+ return c.finishIdentity(ctx, name)
+ }
+ // A base image from before the first-boot hook: personalize over SSH.
bootID, err := s.Run("cat /proc/sys/kernel/random/boot_id")
if err == nil {
_, err = s.Run(golden.Personalize(name, c.Cfg.Proxmox.Domain, c.password, keys))
@@ -99,7 +106,7 @@ func (c *Cluster) personalize(ctx context.Context, name string, keys []string, m
}
done()
// Rebooting: wait for the new boot with the cluster key and the new host key.
- err = retry.Do(ctx, 5*time.Minute, 5*time.Second, func() error {
+ err = retry.Do(ctx, 5*time.Minute, time.Second, func() error {
ns, err := c.dial(ctx, name, c.signer, "")
if err != nil {
return err
diff --git a/internal/cluster/install.go b/internal/cluster/install.go
index 4af37f7..5770434 100644
--- a/internal/cluster/install.go
+++ b/internal/cluster/install.go
@@ -13,6 +13,7 @@ import (
"github.com/virtbase/proxbase/internal/answer"
"github.com/virtbase/proxbase/internal/config"
+ "github.com/virtbase/proxbase/internal/firstboot"
"github.com/virtbase/proxbase/internal/image"
"github.com/virtbase/proxbase/internal/vm"
)
@@ -36,12 +37,17 @@ func (c *Cluster) installAll(ctx context.Context, img *image.Image) error {
names[i] = n.Name
}
c.step("installing Proxmox VE %s on %s (logs: %s)", img.Version, strings.Join(names, ", "), c.Dir.Log("-install.log"))
+ hook, err := firstboot.Serve()
+ if err != nil {
+ return err
+ }
+ defer hook.Close()
var mu sync.Mutex
g, gctx := errgroup.WithContext(ctx)
for _, n := range todo {
g.Go(func() error {
t0 := time.Now()
- if err := c.install(gctx, img, n); err != nil {
+ if err := c.install(gctx, img, n, hook.URL); err != nil {
return fmt.Errorf("%s: %w", n.Name, err)
}
mu.Lock()
@@ -55,7 +61,8 @@ func (c *Cluster) installAll(ctx context.Context, img *image.Image) error {
}
// install writes the node's answer file and runs the unattended installer.
-func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node) error {
+// The installed system runs the first-boot hook at hookURL.
+func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node, hookURL string) error {
ansDir := c.Dir.Disk(n.Name + "-answer")
if err := os.MkdirAll(ansDir, 0o700); err != nil {
return err
@@ -76,6 +83,7 @@ func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node)
SSHKeys: append([]string{pub}, c.Cfg.Proxmox.SSHKeys...),
MAC: spec.NATMAC,
Filesystem: n.Spec.RootDisk.Filesystem,
+ FirstBootURL: hookURL,
})
if err := os.WriteFile(filepath.Join(ansDir, "answer.toml"), []byte(ans), 0o600); err != nil {
return err
@@ -83,11 +91,33 @@ func (c *Cluster) install(ctx context.Context, img *image.Image, n config.Node)
return c.runtime.Install(ctx, spec, vm.Media{ISO: img.ISO, Kernel: img.Kernel, Initrd: img.Initrd, Cmdline: img.Cmdline, AnswerDir: ansDir})
}
+// startNode starts node n. A node cloned from a base image that has no identity yet
+// gets it as a guest file, which the first-boot hook applies.
+func (c *Cluster) startNode(ctx context.Context, n config.Node) error {
+ spec := c.spec(n)
+ if !c.St.Node(n.Name).Pending || c.runtime.Running(spec) {
+ return c.runtime.Start(ctx, spec)
+ }
+ pub, err := c.Dir.ReadSecret(secretKey + ".pub")
+ if err != nil {
+ return err
+ }
+ keys := append([]string{pub}, c.Cfg.Proxmox.SSHKeys...)
+ path := filepath.Join(spec.RunDir, n.Name+"-identity")
+ if err := os.WriteFile(path, firstboot.Identity(n.Name, c.Cfg.Proxmox.Domain, c.password, keys), 0o600); err != nil {
+ return err
+ }
+ // QEMU reads the file at start; it holds the root password, so drop it then.
+ defer os.Remove(path)
+ spec.Files = append(spec.Files, vm.File{Name: firstboot.FileIdentity, Path: path})
+ return c.runtime.Start(ctx, spec)
+}
+
// bootAll starts every node and waits for SSH.
func (c *Cluster) bootAll(ctx context.Context) error {
c.step("booting nodes")
for _, n := range c.Cfg.NodeList() {
- if err := c.runtime.Start(ctx, c.spec(n)); err != nil {
+ if err := c.startNode(ctx, n); err != nil {
return err
}
}
diff --git a/internal/cluster/node.go b/internal/cluster/node.go
index 3855327..1dbb830 100644
--- a/internal/cluster/node.go
+++ b/internal/cluster/node.go
@@ -64,10 +64,16 @@ func (c *Cluster) dial(ctx context.Context, name string, signer ssh.Signer, host
}
// ssh connects to a node, trusting its host key on first use. Nodes cloned from
-// a base image are reached with the base image's key until personalized.
+// a base image are not pinned until personalized: they get the cluster key on
+// their first boot (first-boot hook), or, from base images older than the hook,
+// accept only the base image's key until personalized over SSH.
func (c *Cluster) ssh(ctx context.Context, name string) (*remote.SSH, error) {
ns := c.St.Node(name)
if ns.Pending {
+ s, err := c.dial(ctx, name, c.signer, "")
+ if err == nil || !isAuthError(err) {
+ return s, err
+ }
base, err := golden.Open(ns.Base)
if err != nil {
return nil, retry.Permanent(err)
@@ -92,7 +98,7 @@ func (c *Cluster) ssh(ctx context.Context, name string) (*remote.SSH, error) {
// waitSSH waits until a node accepts SSH logins; a changed host key ends the wait.
func (c *Cluster) waitSSH(ctx context.Context, name string, timeout time.Duration) (*remote.SSH, error) {
var s *remote.SSH
- err := retry.Do(ctx, timeout, 2*time.Second, func() error {
+ err := retry.Do(ctx, timeout, time.Second, func() error {
var err error
s, err = c.ssh(ctx, name)
if err != nil && strings.Contains(err.Error(), "host key") {
@@ -182,3 +188,11 @@ func (h host) SSH(ctx context.Context, node string) (*remote.SSH, error) {
}
func (h host) Step(format string, a ...any) { h.c.step(format, a...) }
+
+func (h host) AptEnv() string { return h.c.aptEnv() }
+
+// isAuthError reports whether an SSH dial failed because the key was rejected
+// (as opposed to the node not being up yet).
+func isAuthError(err error) bool {
+ return strings.Contains(err.Error(), "unable to authenticate")
+}
diff --git a/internal/cluster/nodes.go b/internal/cluster/nodes.go
index 0ad1c8c..c2fa16f 100644
--- a/internal/cluster/nodes.go
+++ b/internal/cluster/nodes.go
@@ -149,7 +149,7 @@ func (c *Cluster) RemoveNode(ctx context.Context, name string, force bool) error
if err := c.net.Reload(c.Cfg); err != nil {
return err
}
- if err := c.postInstall(ctx); err != nil { // rewrites /etc/hosts
+ if err := c.configureNodes(ctx, false); err != nil { // rewrites /etc/hosts
return err
}
if err := c.waitQuorum(ctx, 3*time.Minute); err != nil {
diff --git a/internal/cluster/pvecluster.go b/internal/cluster/pvecluster.go
index 53b7bbd..cb6a8a8 100644
--- a/internal/cluster/pvecluster.go
+++ b/internal/cluster/pvecluster.go
@@ -8,11 +8,13 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
"time"
"golang.org/x/sync/errgroup"
"github.com/virtbase/proxbase/internal/config"
+ "github.com/virtbase/proxbase/internal/nodesetup"
"github.com/virtbase/proxbase/internal/pve"
"github.com/virtbase/proxbase/internal/retry"
)
@@ -62,9 +64,18 @@ func (c *Cluster) formCluster(ctx context.Context) error {
if err != nil {
return err
}
+ joiners, err := c.joinClients(ctx, api, nodes[1:])
+ if err != nil {
+ return err
+ }
+ // Joins run one at a time: concurrent joins race for the cluster lock.
+ members := len(nodes) - len(joiners)
for _, n := range nodes[1:] {
- if err := c.join(ctx, api, first, n, fp); err != nil {
- return err
+ if napi, ok := joiners[n.Name]; ok {
+ if err := c.join(ctx, api, napi, first, n, fp, members); err != nil {
+ return err
+ }
+ members++
}
}
if m, ok := c.Cfg.RoleNetwork(config.RoleMigration); ok {
@@ -89,40 +100,64 @@ func (c *Cluster) fingerprint(ctx context.Context, node string) (string, error)
return pve.Fingerprint(certPEM)
}
-// join adds node n unless it is a member already.
-func (c *Cluster) join(ctx context.Context, api *pve.Client, first, n config.Node, fp string) error {
+// joinClients logs in, in parallel, to the nodes that are not members yet.
+func (c *Cluster) joinClients(ctx context.Context, api *pve.Client, nodes []config.Node) (map[string]*pve.Client, error) {
_, members, err := api.ClusterStatus(ctx)
if err != nil {
- return err
+ return nil, err
}
- if _, ok := members[n.Name]; ok {
- return nil
+ var mu sync.Mutex
+ out := map[string]*pve.Client{}
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range nodes {
+ if _, ok := members[n.Name]; ok {
+ continue
+ }
+ g.Go(func() error {
+ napi, err := c.api(gctx, n.Name)
+ if err != nil {
+ return err
+ }
+ mu.Lock()
+ out[n.Name] = napi
+ mu.Unlock()
+ return nil
+ })
}
+ return out, g.Wait()
+}
+
+// join adds node n to the cluster through its own API client napi. members is
+// the number of nodes in the cluster before the join.
+func (c *Cluster) join(ctx context.Context, api, napi *pve.Client, first, n config.Node, fp string, members int) error {
form := c.links(n)
c.step("joining %s (%s)", n.Name, describeLinks(form))
- napi, err := c.api(ctx, n.Name)
- if err != nil {
- return err
- }
form.Set("hostname", c.corosyncIP(first))
form.Set("password", c.password)
form.Set("fingerprint", fp)
- // The cluster refuses joins until it is quorate.
- if err := waitMember(ctx, api, first.Name, nil, "", 2*time.Minute); err != nil {
- return err
- }
- var upid string
- if err := napi.Post(ctx, "/cluster/config/join", form, &upid); err != nil {
- return fmt.Errorf("join %s: %w", n.Name, err)
+ for attempt := 1; ; attempt++ {
+ // The cluster refuses joins until it is quorate and /etc/pve is writable.
+ if err := c.waitNodeQuorate(ctx, first.Name, members, 2*time.Minute); err != nil {
+ return err
+ }
+ var upid string
+ if err := napi.Post(ctx, "/cluster/config/join", form, &upid); err != nil {
+ return fmt.Errorf("join %s: %w", n.Name, err)
+ }
+ err := waitMember(ctx, api, n.Name, napi, upid, 3*time.Minute)
+ // A join refused for lack of quorum changed nothing; try again.
+ if err == nil || attempt == 3 || !strings.Contains(err.Error(), "no quorum") {
+ return err
+ }
+ c.step("%s: join refused (the cluster had no quorum yet), retrying", n.Name)
}
- return waitMember(ctx, api, n.Name, napi, upid, 3*time.Minute)
}
// waitMember waits until node is listed online in a quorate cluster. If a join
// task is given, its failure ends the wait early.
func waitMember(ctx context.Context, api *pve.Client, node string, joiner *pve.Client, upid string, timeout time.Duration) error {
var joinErr error
- err := retry.Do(ctx, timeout, 2*time.Second, func() error {
+ err := retry.Do(ctx, timeout, time.Second, func() error {
cs, members, err := api.ClusterStatus(ctx)
if err == nil && cs != nil && cs.Quorate == 1 && members[node].Online == 1 {
return nil
@@ -144,6 +179,34 @@ func waitMember(ctx context.Context, api *pve.Client, node string, joiner *pve.C
return err
}
+// repairJoins repairs what racing cluster joins left broken (see
+// nodesetup.RepairJoin): without it a node's API may have no certificate, one the
+// cluster CA does not vouch for, or reject every login.
+func (c *Cluster) repairJoins(ctx context.Context) error {
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range c.Cfg.NodeList() {
+ g.Go(func() error {
+ s, err := c.waitSSH(gctx, n.Name, time.Minute)
+ if err != nil {
+ return err
+ }
+ defer s.Close()
+ out, err := s.Run(nodesetup.RepairJoin)
+ if err != nil {
+ return fmt.Errorf("%s: repair join: %w", n.Name, err)
+ }
+ switch {
+ case strings.Contains(out, "repaired"):
+ c.step("%s: the cluster join left the node certificate missing or from another CA; issued a new one", n.Name)
+ case strings.Contains(out, "restarted"):
+ c.step("%s: pveproxy served an outdated certificate after the join; restarted it", n.Name)
+ }
+ return nil
+ })
+ }
+ return g.Wait()
+}
+
var (
votesRe = regexp.MustCompile(`Total votes:\s+(\d+)`)
quorateRe = regexp.MustCompile(`Quorate:\s+Yes`)
@@ -154,28 +217,7 @@ func (c *Cluster) waitQuorum(ctx context.Context, timeout time.Duration) error {
want := c.Cfg.Nodes.Count
g, gctx := errgroup.WithContext(ctx)
for _, n := range c.Cfg.NodeList() {
- g.Go(func() error {
- var last string
- err := retry.Do(gctx, timeout, 2*time.Second, func() error {
- s, err := c.waitSSH(gctx, n.Name, time.Minute)
- if err != nil {
- return retry.Permanent(err)
- }
- out, err := s.Run(`pvecm status 2>&1 || true
-f=/etc/pve/.proxbase-write-test; if echo ok > $f 2>/dev/null; then rm -f $f; echo WRITABLE; fi`)
- s.Close()
- m := votesRe.FindStringSubmatch(out)
- if err == nil && quorateRe.MatchString(out) && m != nil && m[1] == strconv.Itoa(want) && strings.Contains(out, "WRITABLE") {
- return nil
- }
- last = out
- return errors.New("no quorum")
- })
- if err != nil && gctx.Err() == nil && last != "" {
- return fmt.Errorf("%s: no quorum after %s:\n%s", n.Name, timeout, last)
- }
- return err
- })
+ g.Go(func() error { return c.waitNodeQuorate(gctx, n.Name, want, timeout) })
}
if err := g.Wait(); err != nil {
return err
@@ -183,3 +225,29 @@ f=/etc/pve/.proxbase-write-test; if echo ok > $f 2>/dev/null; then rm -f $f; ech
c.step("cluster quorate: %d/%d votes, /etc/pve writable on every node", want, want)
return nil
}
+
+// waitNodeQuorate waits until node is quorate with votes votes and can write
+// /etc/pve. The API reports quorum a moment before the cluster file system
+// accepts changes such as a join.
+func (c *Cluster) waitNodeQuorate(ctx context.Context, node string, votes int, timeout time.Duration) error {
+ var last string
+ err := retry.Do(ctx, timeout, time.Second, func() error {
+ s, err := c.waitSSH(ctx, node, time.Minute)
+ if err != nil {
+ return retry.Permanent(err)
+ }
+ out, err := s.Run(`pvecm status 2>&1 || true
+f=/etc/pve/.proxbase-write-test; if echo ok > $f 2>/dev/null; then rm -f $f; echo WRITABLE; fi`)
+ s.Close()
+ m := votesRe.FindStringSubmatch(out)
+ if err == nil && quorateRe.MatchString(out) && m != nil && m[1] == strconv.Itoa(votes) && strings.Contains(out, "WRITABLE") {
+ return nil
+ }
+ last = out
+ return errors.New("no quorum")
+ })
+ if err != nil && ctx.Err() == nil && last != "" {
+ return fmt.Errorf("%s: no quorum after %s:\n%s", node, timeout, last)
+ }
+ return err
+}
diff --git a/internal/cluster/setup.go b/internal/cluster/setup.go
index 25735ac..a2561a7 100644
--- a/internal/cluster/setup.go
+++ b/internal/cluster/setup.go
@@ -14,9 +14,13 @@ import (
"github.com/virtbase/proxbase/internal/retry"
)
-// postInstall configures host names, repositories and bridges on every node and,
-// if configured, upgrades them (rebooting into a new kernel).
-func (c *Cluster) postInstall(ctx context.Context) error {
+// postInstall puts every node into provisioning mode (fast, unsafe writes until
+// endProvisioning) and configures it; see configureNodes.
+func (c *Cluster) postInstall(ctx context.Context) error { return c.configureNodes(ctx, true) }
+
+// configureNodes configures host names, repositories and bridges on every node
+// and, if configured, upgrades them (rebooting into a new kernel).
+func (c *Cluster) configureNodes(ctx context.Context, provisioning bool) error {
g, gctx := errgroup.WithContext(ctx)
for _, n := range c.Cfg.NodeList() {
g.Go(func() error {
@@ -25,7 +29,11 @@ func (c *Cluster) postInstall(ctx context.Context) error {
return err
}
defer s.Close()
- out, err := s.Run(nodesetup.Script(c.Cfg, n))
+ script := nodesetup.Script(c.Cfg, n)
+ if provisioning {
+ script = nodesetup.ProvisionStart + script
+ }
+ out, err := s.Run(script)
if err != nil {
return fmt.Errorf("%s: post-install: %w", n.Name, err)
}
@@ -36,7 +44,7 @@ func (c *Cluster) postInstall(ctx context.Context) error {
return nil
}
t0 := time.Now()
- out, err = s.Run(nodesetup.Upgrade)
+ out, err = s.Run(c.aptEnv() + nodesetup.Upgrade)
if err != nil {
return fmt.Errorf("%s: dist-upgrade: %w", n.Name, err)
}
@@ -47,6 +55,25 @@ func (c *Cluster) postInstall(ctx context.Context) error {
return g.Wait()
}
+// endProvisioning restores safe writes on every node (see nodesetup.ProvisionEnd).
+func (c *Cluster) endProvisioning(ctx context.Context) error {
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range c.Cfg.NodeList() {
+ g.Go(func() error {
+ s, err := c.waitSSH(gctx, n.Name, time.Minute)
+ if err != nil {
+ return err
+ }
+ defer s.Close()
+ if _, err := s.Run(nodesetup.ProvisionEnd); err != nil {
+ return fmt.Errorf("%s: end provisioning: %w", n.Name, err)
+ }
+ return nil
+ })
+ }
+ return g.Wait()
+}
+
// rebootForKernel reboots a node whose newest installed kernel is not running
// and waits until it is back (new boot ID, SSH up).
func (c *Cluster) rebootForKernel(ctx context.Context, name string, s *remote.SSH) error {
@@ -60,7 +87,7 @@ func (c *Cluster) rebootForKernel(ctx context.Context, name string, s *remote.SS
return err
}
var running string
- err = retry.Do(ctx, 5*time.Minute, 5*time.Second, func() error {
+ err = retry.Do(ctx, 5*time.Minute, time.Second, func() error {
ns, err := c.ssh(ctx, name)
if err != nil {
return err
diff --git a/internal/cluster/snapshot.go b/internal/cluster/snapshot.go
index a7a1ea5..bde5340 100644
--- a/internal/cluster/snapshot.go
+++ b/internal/cluster/snapshot.go
@@ -2,12 +2,17 @@ package cluster
import (
"context"
+ "errors"
"fmt"
"regexp"
"slices"
"strings"
+ "sync"
"time"
+ "golang.org/x/sync/errgroup"
+
+ "github.com/virtbase/proxbase/internal/config"
"github.com/virtbase/proxbase/internal/state"
"github.com/virtbase/proxbase/internal/vm"
)
@@ -41,13 +46,8 @@ func (c *Cluster) requireStopped() error {
return nil
}
-// SnapshotSave takes a snapshot of every disk of the stopped cluster.
-func (c *Cluster) SnapshotSave(ctx context.Context, name string) error {
- unlock, err := c.Dir.Lock()
- if err != nil {
- return err
- }
- defer unlock()
+// checkNewSnapshot validates the name of a snapshot about to be saved.
+func (c *Cluster) checkNewSnapshot(name string) error {
if !snapRe.MatchString(name) {
return fmt.Errorf("invalid snapshot name %q", name)
}
@@ -57,6 +57,19 @@ func (c *Cluster) SnapshotSave(ctx context.Context, name string) error {
if c.St.Snapshot(name) != nil {
return fmt.Errorf("snapshot %q exists", name)
}
+ return nil
+}
+
+// SnapshotSave takes a snapshot of every disk of the stopped cluster.
+func (c *Cluster) SnapshotSave(ctx context.Context, name string) error {
+ unlock, err := c.Dir.Lock()
+ if err != nil {
+ return err
+ }
+ defer unlock()
+ if err := c.checkNewSnapshot(name); err != nil {
+ return err
+ }
if err := c.requireStopped(); err != nil {
return err
}
@@ -67,6 +80,68 @@ func (c *Cluster) SnapshotSave(ctx context.Context, name string) error {
return c.save()
}
+// SnapshotSaveLive snapshots the running cluster with its memory. Every node is
+// paused until all snapshots are taken, so they show one moment of the cluster.
+func (c *Cluster) SnapshotSaveLive(ctx context.Context, name string) error {
+ unlock, err := c.Dir.Lock()
+ if err != nil {
+ return err
+ }
+ defer unlock()
+ if err := c.checkNewSnapshot(name); err != nil {
+ return err
+ }
+ live, ok := c.runtime.(vm.LiveSnapshotter)
+ if !ok {
+ return errors.New("the node runtime cannot take live snapshots")
+ }
+ if len(c.St.Faults) > 0 {
+ return fmt.Errorf("cluster %s has active faults; clear them first (proxbase fault clear %s)", c.Cfg.Name, c.Cfg.Name)
+ }
+ nodes := c.Cfg.NodeList()
+ for _, n := range nodes {
+ if !c.running(n) {
+ return fmt.Errorf("%s is not running; live snapshots need a running cluster (proxbase start %s)", n.Name, c.Cfg.Name)
+ }
+ }
+ var paused []config.Node
+ defer func() {
+ for _, n := range paused {
+ if err := live.Resume(context.WithoutCancel(ctx), c.spec(n)); err != nil {
+ c.step("%s: resume failed: %v", n.Name, err)
+ }
+ }
+ }()
+ for _, n := range nodes {
+ if err := live.Pause(ctx, c.spec(n)); err != nil {
+ return fmt.Errorf("%s: pause: %w", n.Name, err)
+ }
+ paused = append(paused, n)
+ }
+ var mu sync.Mutex
+ var saved []config.Node
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range nodes {
+ g.Go(func() error {
+ if err := live.SaveLive(gctx, c.spec(n), name); err != nil {
+ return err
+ }
+ mu.Lock()
+ saved = append(saved, n)
+ mu.Unlock()
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ for _, n := range saved {
+ _ = live.DeleteLive(context.WithoutCancel(ctx), c.spec(n), name)
+ }
+ return err
+ }
+ c.St.Snapshots = append(c.St.Snapshots, state.Snapshot{Name: name, Created: time.Now().UTC(), Nodes: c.nodeNames(), Live: true})
+ return c.save()
+}
+
// SnapshotRestore resets every disk to the snapshot.
func (c *Cluster) SnapshotRestore(ctx context.Context, name string) error {
unlock, err := c.Dir.Lock()
@@ -84,9 +159,73 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, name string) error {
if err := c.requireStopped(); err != nil {
return err
}
+ if snap.Live {
+ return c.resume(ctx, name)
+ }
return c.runtime.Snapshot(ctx, vm.SnapshotRestore, name, c.disks())
}
+// resume starts every node in the state of live snapshot name. The nodes load
+// it paused and are resumed together, so the cluster continues as one; their
+// clocks, which stood still in the snapshot, are set to the host's time.
+func (c *Cluster) resume(ctx context.Context, name string) error {
+ live, ok := c.runtime.(vm.LiveSnapshotter)
+ if !ok {
+ return errors.New("the node runtime cannot start from live snapshots")
+ }
+ if err := c.net.Start(c.Cfg); err != nil {
+ return err
+ }
+ nodes := c.Cfg.NodeList()
+ for i, n := range nodes {
+ spec := c.spec(n)
+ spec.LoadSnapshot = name
+ if err := c.runtime.Start(ctx, spec); err != nil {
+ for _, started := range nodes[:i] {
+ _ = c.runtime.Stop(context.WithoutCancel(ctx), c.spec(started), 0)
+ }
+ return fmt.Errorf("%s: start from snapshot %s: %w", n.Name, name, err)
+ }
+ }
+ for _, n := range nodes {
+ if err := live.Resume(ctx, c.spec(n)); err != nil {
+ return fmt.Errorf("%s: resume: %w", n.Name, err)
+ }
+ }
+ c.step("%d node(s) resumed from snapshot %s", len(nodes), name)
+ if err := c.syncClocks(ctx); err != nil {
+ return err
+ }
+ if err := c.waitQuorum(ctx, 5*time.Minute); err != nil {
+ return err
+ }
+ if err := c.waitAPI(ctx, 2*time.Minute); err != nil {
+ return err
+ }
+ return c.waitStorage(ctx)
+}
+
+// syncClocks sets every node's clock to the host's time.
+func (c *Cluster) syncClocks(ctx context.Context) error {
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range c.Cfg.NodeList() {
+ g.Go(func() error {
+ s, err := c.waitSSH(gctx, n.Name, time.Minute)
+ if err != nil {
+ return err
+ }
+ defer s.Close()
+ now := time.Now()
+ script := fmt.Sprintf("date -u -s @%d.%03d >/dev/null\nchronyc -a makestep >/dev/null 2>&1 || true\n", now.Unix(), now.Nanosecond()/1e6)
+ if _, err := s.Run(script); err != nil {
+ return fmt.Errorf("%s: set clock: %w", n.Name, err)
+ }
+ return nil
+ })
+ }
+ return g.Wait()
+}
+
// SnapshotDelete removes the snapshot from every disk.
func (c *Cluster) SnapshotDelete(ctx context.Context, name string) error {
unlock, err := c.Dir.Lock()
@@ -94,15 +233,32 @@ func (c *Cluster) SnapshotDelete(ctx context.Context, name string) error {
return err
}
defer unlock()
- if c.St.Snapshot(name) == nil {
+ snap := c.St.Snapshot(name)
+ if snap == nil {
return fmt.Errorf("no snapshot %q", name)
}
- if err := c.requireStopped(); err != nil {
- return err
- }
- if err := c.runtime.Snapshot(ctx, vm.SnapshotDelete, name, c.disks()); err != nil {
+ if err := c.deleteSnapshot(ctx, snap); err != nil {
return err
}
c.St.Snapshots = slices.DeleteFunc(c.St.Snapshots, func(s state.Snapshot) bool { return s.Name == name })
return c.save()
}
+
+// deleteSnapshot removes a snapshot from the disks: through the running nodes
+// for a live snapshot of a running cluster, otherwise from the stopped disks.
+func (c *Cluster) deleteSnapshot(ctx context.Context, snap *state.Snapshot) error {
+ live, ok := c.runtime.(vm.LiveSnapshotter)
+ nodes := c.Cfg.NodeList()
+ allRunning := !slices.ContainsFunc(nodes, func(n config.Node) bool { return !c.running(n) })
+ if !snap.Live || !ok || !allRunning {
+ if err := c.requireStopped(); err != nil {
+ return err
+ }
+ return c.runtime.Snapshot(ctx, vm.SnapshotDelete, snap.Name, c.disks())
+ }
+ g, gctx := errgroup.WithContext(ctx)
+ for _, n := range nodes {
+ g.Go(func() error { return live.DeleteLive(gctx, c.spec(n), snap.Name) })
+ }
+ return g.Wait()
+}
diff --git a/internal/cluster/snapshot_test.go b/internal/cluster/snapshot_test.go
new file mode 100644
index 0000000..f2fa5ca
--- /dev/null
+++ b/internal/cluster/snapshot_test.go
@@ -0,0 +1,145 @@
+package cluster
+
+import (
+ "context"
+ "errors"
+ "slices"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/virtbase/proxbase/internal/config"
+ "github.com/virtbase/proxbase/internal/state"
+ "github.com/virtbase/proxbase/internal/vm"
+)
+
+// fakeRuntime records live snapshot calls; every node runs.
+type fakeRuntime struct {
+ mu sync.Mutex
+ calls []string // "pause pve1", "save pve2", ...
+ failOn string // node whose SaveLive fails
+ deleted []string
+}
+
+var (
+ _ vm.Runtime = (*fakeRuntime)(nil)
+ _ vm.LiveSnapshotter = (*fakeRuntime)(nil)
+)
+
+func (f *fakeRuntime) record(op string, s vm.Spec) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.calls = append(f.calls, op+" "+s.Name)
+}
+
+func (f *fakeRuntime) Install(context.Context, vm.Spec, vm.Media) error { return nil }
+func (f *fakeRuntime) Start(context.Context, vm.Spec) error { return nil }
+func (f *fakeRuntime) Stop(context.Context, vm.Spec, time.Duration) error { return nil }
+func (f *fakeRuntime) Running(vm.Spec) bool { return true }
+func (f *fakeRuntime) Snapshot(context.Context, vm.SnapshotOp, string, []vm.Disk) error { return nil }
+
+func (f *fakeRuntime) Pause(_ context.Context, s vm.Spec) error { f.record("pause", s); return nil }
+func (f *fakeRuntime) Resume(_ context.Context, s vm.Spec) error { f.record("resume", s); return nil }
+
+func (f *fakeRuntime) SaveLive(_ context.Context, s vm.Spec, _ string) error {
+ f.record("save", s)
+ if s.Name == f.failOn {
+ return errors.New("disk full")
+ }
+ return nil
+}
+
+func (f *fakeRuntime) DeleteLive(_ context.Context, s vm.Spec, _ string) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.deleted = append(f.deleted, s.Name)
+ return nil
+}
+
+// newTestCluster returns a ready 3-node cluster in a temporary data directory.
+func newTestCluster(t *testing.T, rt vm.Runtime) *Cluster {
+ t.Helper()
+ t.Setenv("XDG_DATA_HOME", t.TempDir())
+ cfg, err := config.Decode(strings.NewReader("name: snap\nnodes: {count: 3}\n"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg.SetDefaults()
+ c, err := initialize(cfg, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ c.runtime = rt
+ c.St.Phase = state.PhaseReady
+ return c
+}
+
+func (f *fakeRuntime) ops(op string) []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ var out []string
+ for _, c := range f.calls {
+ if name, ok := strings.CutPrefix(c, op+" "); ok {
+ out = append(out, name)
+ }
+ }
+ slices.Sort(out)
+ return out
+}
+
+func TestCluster_SnapshotSaveLive(t *testing.T) {
+ rt := &fakeRuntime{}
+ c := newTestCluster(t, rt)
+ if err := c.SnapshotSaveLive(t.Context(), "ready"); err != nil {
+ t.Fatal(err)
+ }
+ all := []string{"pve1", "pve2", "pve3"}
+ for _, op := range []string{"pause", "save", "resume"} {
+ if got := rt.ops(op); !slices.Equal(got, all) {
+ t.Errorf("%s: %v, want %v", op, got, all)
+ }
+ }
+ // Every node is paused before the first snapshot is written.
+ firstSave := slices.IndexFunc(rt.calls, func(s string) bool { return strings.HasPrefix(s, "save ") })
+ if paused := len(rt.ops("pause")); firstSave < paused {
+ t.Errorf("a snapshot was saved before all nodes were paused: %v", rt.calls)
+ }
+ snap := c.St.Snapshot("ready")
+ if snap == nil || !snap.Live {
+ t.Fatalf("snapshot not recorded as live: %+v", c.St.Snapshots)
+ }
+}
+
+func TestCluster_SnapshotSaveLiveRollsBack(t *testing.T) {
+ rt := &fakeRuntime{failOn: "pve2"}
+ c := newTestCluster(t, rt)
+ if err := c.SnapshotSaveLive(t.Context(), "ready"); err == nil {
+ t.Fatal("expected the failed save to fail the snapshot")
+ }
+ if got := rt.ops("resume"); !slices.Equal(got, []string{"pve1", "pve2", "pve3"}) {
+ t.Errorf("not every node was resumed: %v", got)
+ }
+ rt.mu.Lock()
+ deleted := slices.Sorted(slices.Values(rt.deleted))
+ rt.mu.Unlock()
+ saved := slices.DeleteFunc(rt.ops("save"), func(n string) bool { return n == "pve2" })
+ if !slices.Equal(deleted, saved) {
+ t.Errorf("deleted %v, want the snapshots that were saved: %v", deleted, saved)
+ }
+ if c.St.Snapshot("ready") != nil {
+ t.Error("a failed snapshot must not be recorded")
+ }
+}
+
+func TestCluster_SnapshotSaveLiveRefusesFaults(t *testing.T) {
+ rt := &fakeRuntime{}
+ c := newTestCluster(t, rt)
+ c.St.Faults = []state.Fault{{Kind: "freeze", Node: "pve1"}}
+ if err := c.SnapshotSaveLive(t.Context(), "ready"); err == nil || !strings.Contains(err.Error(), "faults") {
+ t.Fatalf("got %v, want an error about active faults", err)
+ }
+ if len(rt.calls) != 0 {
+ t.Errorf("nodes were touched: %v", rt.calls)
+ }
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index eff8fc6..22efdd3 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -42,6 +42,7 @@ type Proxmox struct {
SSHKeys []string `yaml:"sshKeys,omitempty" json:"sshKeys,omitempty" jsonschema:"description=Extra public keys for root on every node"`
Upgrade bool `yaml:"upgrade,omitempty" json:"upgrade,omitempty" jsonschema:"description=Run apt dist-upgrade on every node before clustering"`
Golden bool `yaml:"goldenImage,omitempty" json:"goldenImage,omitempty" jsonschema:"description=Clone nodes from a cached installed base image instead of running the installer"`
+ AptCache *bool `yaml:"aptCache,omitempty" json:"aptCache,omitempty" jsonschema:"description=Download packages that proxbase installs (upgrade and Ceph) once through a cache on the host; default true"`
}
type Nodes struct {
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 87d4c1f..9c7f496 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -71,6 +71,8 @@ func TestValidationErrors(t *testing.T) {
"nodes: {defaults: {memory: 1G}}\n": "at least 2G",
"nodes: {overrides: {pve9: {cpus: 2}}}\n": "unknown node",
"networks: [{name: a, cidr: 10.1.0.0/28}]\nnodes: {count: 5}\n": "too small",
+ "proxmox: {domain: \"lab/x.internal\"}\n": "proxmox.domain",
+ "proxmox: {domain: internal}\n": "proxmox.domain",
}
for in, want := range cases {
err := load(t, in).Validate()
diff --git a/internal/config/defaults.go b/internal/config/defaults.go
index f84aa97..cc66705 100644
--- a/internal/config/defaults.go
+++ b/internal/config/defaults.go
@@ -19,6 +19,9 @@ func (c *Cluster) SetDefaults() {
def(&p.Keyboard, "en-us")
def(&p.Country, "us")
def(&p.Domain, "proxbase.internal")
+ if p.AptCache == nil {
+ p.AptCache = ptr(true)
+ }
n := &c.Nodes
if n.Count == 0 {
diff --git a/internal/config/validate.go b/internal/config/validate.go
index 37fd06a..dbe0213 100644
--- a/internal/config/validate.go
+++ b/internal/config/validate.go
@@ -16,6 +16,7 @@ var (
bridgeRe = regexp.MustCompile(`^vmbr[0-9]{1,4}$`)
vlanRe = regexp.MustCompile(`^[0-9]{1,4}(-[0-9]{1,4})?$`)
versRe = regexp.MustCompile(`^[0-9]+\.[0-9]+(-[0-9]+)?$`)
+ domainRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+$`)
minDisks = map[string]int{"single": 1, "mirror": 2, "raid10": 4, "raidz": 3, "raidz2": 4, "raidz3": 5}
)
@@ -42,6 +43,9 @@ func (c *Cluster) Validate() error {
if !nameRe.MatchString(c.Name) {
add("name: %q must match %s", c.Name, nameRe)
}
+ if !domainRe.MatchString(c.Proxmox.Domain) || len(c.Proxmox.Domain) > 200 {
+ add("proxmox.domain: %q must be a DNS domain like lab.internal", c.Proxmox.Domain)
+ }
if !versRe.MatchString(c.Proxmox.Version) {
add("proxmox.version: %q must look like 9.2 or 9.2-1", c.Proxmox.Version)
}
diff --git a/internal/firstboot/firstboot.go b/internal/firstboot/firstboot.go
new file mode 100644
index 0000000..ff43823
--- /dev/null
+++ b/internal/firstboot/firstboot.go
@@ -0,0 +1,147 @@
+// Package firstboot prepares nodes on their first boot through the Proxmox
+// installer's first-boot hook. The installer downloads the hook from a small
+// HTTP server on the host and the installed system runs it once, before
+// networking and before any Proxmox VE service starts. Proxbase uses that early
+// moment to:
+//
+// - give nodes cloned from a base image their own identity without a reboot,
+// - bake a base image (prepare it once, then power off unbooted by Proxmox VE),
+// - skip work Proxmox VE does on a first boot that proxbase does not need.
+//
+// Per-node data reaches the hook as QEMU fw_cfg files (vm.File).
+package firstboot
+
+import (
+ _ "embed"
+ "fmt"
+ "net"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/virtbase/proxbase/internal/nodesetup"
+)
+
+const (
+ // FileBake is the guest file name of the bake script.
+ FileBake = "opt/proxbase/bake"
+ // FileIdentity is the guest file name of a node's identity.
+ FileIdentity = "opt/proxbase/identity"
+ // IdentityMarker exists on nodes that received their identity; it holds the node name.
+ IdentityMarker = "/etc/proxbase-identity"
+ // BakeOK is printed on the serial console when a bake succeeded.
+ BakeOK = "proxbase-first-boot: bake ok"
+
+ // hostAddr is the host as seen from a guest on QEMU user networking.
+ hostAddr = "10.0.2.2"
+ hookPath = "/proxmox-first-boot"
+)
+
+//go:embed hook.sh
+var hookTemplate string
+
+// Hook returns the first-boot hook script.
+func Hook() string {
+ return strings.NewReplacer(
+ "@NO_GRUB_TIMEOUT@", strings.Trim(nodesetup.NoGrubTimeout, "\n"),
+ "@IDENTITY_MARKER@", IdentityMarker,
+ ).Replace(hookTemplate)
+}
+
+// Server serves the hook to installers on QEMU user networking.
+type Server struct {
+ srv *http.Server
+ // URL is the hook's address as seen from the guest.
+ URL string
+}
+
+// Serve starts serving the hook on a free port of the host's loopback address,
+// which guests on QEMU user networking reach as 10.0.2.2.
+func Serve() (*Server, error) {
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ return nil, fmt.Errorf("first-boot hook server: %w", err)
+ }
+ hook := []byte(Hook())
+ mux := http.NewServeMux()
+ mux.HandleFunc("GET "+hookPath, func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "text/x-shellscript")
+ _, _ = w.Write(hook)
+ })
+ s := &Server{
+ srv: &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second},
+ URL: "http://" + net.JoinHostPort(hostAddr, strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)) + hookPath,
+ }
+ // Serve returns when Close is called. Any other failure surfaces as a failed
+ // download in the installer, which reports it.
+ go func() { _ = s.srv.Serve(ln) }()
+ return s, nil
+}
+
+// Close stops the server.
+func (s *Server) Close() error { return s.srv.Close() }
+
+// Identity renders the identity file of a node cloned from a base image.
+func Identity(node, domain, password string, authorizedKeys []string) []byte {
+ keys := make([]string, len(authorizedKeys))
+ for i, k := range authorizedKeys {
+ keys[i] = strings.TrimSpace(k)
+ }
+ return fmt.Appendf(nil, "NODE=%s\nDOMAIN=%s\nPASSWORD=%s\nKEYS=%s\n",
+ quote(node), quote(domain), quote(password), quote(strings.Join(keys, "\n")))
+}
+
+// Bake returns the script that prepares a base image on its only boot. With a
+// Ceph release (e.g. "squid") it also installs the Ceph packages, so nodes
+// cloned from the image skip `pveceph install`; natMAC is the MAC of the NIC on
+// QEMU user networking it downloads through.
+func Bake(cephRelease, natMAC string) string {
+ var b strings.Builder
+ b.WriteString("touch /var/log/pveam.log\n")
+ b.WriteString(nodesetup.NoGrubTimeout)
+ if cephRelease != "" {
+ b.WriteString(nodesetup.Repositories)
+ fmt.Fprintf(&b, cephBake, cephRelease, natMAC)
+ }
+ return b.String()
+}
+
+// cephBake installs what `pveceph install` installs. Networking is not up yet,
+// so it configures the NAT NIC itself with ip: ifupdown2 cannot be used here,
+// its lock is held by hotplug ifup runs that wait for networking, which waits
+// for this hook. policy-rc.d keeps packages from starting services during the
+// boot.
+const cephBake = `
+cat > /etc/apt/sources.list.d/ceph.sources <&2; exit 1; }
+ip link set "$nic" up
+ip addr add 10.0.2.15/24 dev "$nic"
+ip route add default via 10.0.2.2
+printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d
+chmod 755 /usr/sbin/policy-rc.d
+trap 'rm -f /usr/sbin/policy-rc.d' EXIT
+export DEBIAN_FRONTEND=noninteractive
+apt='apt-get -o Acquire::Retries=3 -o Dpkg::Options::=--force-unsafe-io'
+retry_apt() { "$@" || "$@" -o Acquire::http::No-Cache=true; }
+$apt update
+retry_apt $apt install -y --no-install-recommends -o Dpkg::Options::=--force-confnew \
+ ceph ceph-common ceph-fuse ceph-mds ceph-volume gdisk nvme-cli
+retry_apt $apt install -y --only-upgrade libpve-storage-perl
+$apt clean
+ip addr flush dev "$nic"
+ip link set "$nic" down
+`
+
+// quote returns s single-quoted for the shell.
+func quote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }
diff --git a/internal/firstboot/firstboot_test.go b/internal/firstboot/firstboot_test.go
new file mode 100644
index 0000000..5d26ff8
--- /dev/null
+++ b/internal/firstboot/firstboot_test.go
@@ -0,0 +1,107 @@
+package firstboot
+
+import (
+ "io"
+ "net/http"
+ "os/exec"
+ "strings"
+ "testing"
+)
+
+// sh parses (and with run, executes) a script with the guest's /bin/sh.
+func sh(t *testing.T, script string, run bool) string {
+ t.Helper()
+ args := []string{"-n"}
+ if run {
+ args = []string{"-eu"}
+ }
+ cmd := exec.CommandContext(t.Context(), "sh", args...)
+ cmd.Stdin = strings.NewReader(script)
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("sh %v: %v\n%s", args, err, out)
+ }
+ return string(out)
+}
+
+func TestHook(t *testing.T) {
+ t.Parallel()
+ h := Hook()
+ if strings.Contains(h, "@") {
+ t.Fatalf("unreplaced placeholder in hook:\n%s", h)
+ }
+ for _, want := range []string{"GRUB_TIMEOUT=0", ">" + IdentityMarker, "pending-first-boot-setup", "/var/log/pveam.log"} {
+ if !strings.Contains(h, want) {
+ t.Errorf("hook lacks %q", want)
+ }
+ }
+ sh(t, h, false)
+}
+
+func TestServe(t *testing.T) {
+ t.Parallel()
+ s, err := Serve()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer s.Close()
+ if !strings.HasPrefix(s.URL, "http://10.0.2.2:") {
+ t.Fatalf("URL %s is not the host as seen from the guest", s.URL)
+ }
+ req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, strings.Replace(s.URL, "10.0.2.2", "127.0.0.1", 1), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resp.StatusCode != http.StatusOK || string(body) != Hook() {
+ t.Fatalf("GET %s: %s, %d bytes", s.URL, resp.Status, len(body))
+ }
+}
+
+// The identity file is sourced by the hook; values must arrive unchanged.
+func TestIdentity(t *testing.T) {
+ t.Parallel()
+ keys := []string{"ssh-ed25519 AAAA proxbase@lab\n", "ssh-rsa BBBB it's mine"}
+ id := Identity("pve2", "lab.internal", `p'w"$x`, keys)
+ got := sh(t, string(id)+`printf '%s|%s|%s|%s' "$NODE" "$DOMAIN" "$PASSWORD" "$KEYS"`, true)
+ want := "pve2|lab.internal|p'w\"$x|ssh-ed25519 AAAA proxbase@lab\nssh-rsa BBBB it's mine"
+ if got != want {
+ t.Fatalf("got %q, want %q", got, want)
+ }
+}
+
+func TestBake(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ name, ceph string
+ want, not []string
+ }{
+ {name: "plain", want: []string{"GRUB_TIMEOUT=0", "pveam.log"}, not: []string{"apt-get"}},
+ {name: "ceph", ceph: "squid", want: []string{"debian/ceph-squid", "ceph-mds", "policy-rc.d", `"52:54:00:50:00:00"`, "ip route add default via 10.0.2.2"}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ b := Bake(tt.ceph, "52:54:00:50:00:00")
+ for _, w := range tt.want {
+ if !strings.Contains(b, w) {
+ t.Errorf("bake lacks %q", w)
+ }
+ }
+ for _, n := range tt.not {
+ if strings.Contains(b, n) {
+ t.Errorf("bake must not contain %q", n)
+ }
+ }
+ sh(t, b, false)
+ })
+ }
+}
diff --git a/internal/firstboot/hook.sh b/internal/firstboot/hook.sh
new file mode 100644
index 0000000..51d87e4
--- /dev/null
+++ b/internal/firstboot/hook.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+# Proxbase first-boot hook. The Proxmox installer downloads it and the installed
+# system runs it once, on its first boot, before networking and before any
+# Proxmox VE service starts (proxmox-first-boot-network-pre.service).
+#
+# Proxbase passes data to the guest as QEMU fw_cfg files:
+# opt/proxbase/bake script that prepares a base image; the hook runs it,
+# stays pending for the clones and powers off
+# opt/proxbase/identity shell variables (NODE, DOMAIN, PASSWORD, KEYS) that
+# give a node cloned from a base image its own identity
+set -eu
+
+cfg=/sys/firmware/qemu_fw_cfg/by_name/opt/proxbase
+modprobe qemu_fw_cfg 2>/dev/null || true
+
+console() { echo "proxbase-first-boot: $*" >/dev/ttyS0 2>/dev/null || true; }
+
+if [ -r "$cfg/bake/raw" ]; then
+ status=ok
+ if ! sh -eu "$cfg/bake/raw" >/var/log/proxbase-bake.log 2>&1; then
+ status=failed
+ tail -n 30 /var/log/proxbase-bake.log >/dev/ttyS0 2>/dev/null || true
+ fi
+ # Stay pending: every clone runs this hook again for its identity.
+ touch /var/lib/proxmox-first-boot/pending-first-boot-setup
+ console "bake $status"
+ # Power off before networking and the Proxmox VE services start; the base
+ # image must never run them (they create per-node keys and certificates).
+ systemctl poweroff --no-block
+ exec sleep infinity
+fi
+
+# pveproxy runs pveupdate (apt update, appliance list) on its first start unless
+# this log exists; proxbase manages repositories and updates itself.
+touch /var/log/pveam.log
+
+grub_timeout() {
+@NO_GRUB_TIMEOUT@
+}
+grub_timeout || console "could not remove the GRUB timeout"
+
+if [ -r "$cfg/identity/raw" ]; then
+ # shellcheck source=/dev/null
+ . "$cfg/identity/raw"
+ fqdn="$NODE.$DOMAIN"
+ hostname "$NODE"
+ echo "$NODE" >/etc/hostname
+ sed -i "/^10\.0\.2\.15[[:space:]]/c\10.0.2.15 $fqdn $NODE" /etc/hosts
+ # Proxmox VE names the node certificate after the search domain.
+ sed -i "s/^search .*/search $DOMAIN/" /etc/resolv.conf
+ echo "$fqdn" >/etc/mailname
+ postconf -e "myhostname=$fqdn"
+ rm -f /etc/ssh/ssh_host_*
+ ssh-keygen -A >/dev/null
+ rm -f /etc/machine-id /var/lib/dbus/machine-id
+ systemd-machine-id-setup >/dev/null
+ # journald files are per machine-id; without a restart this boot's log would
+ # land under the old one, where journalctl does not look.
+ systemctl restart systemd-journald
+ echo "root:$PASSWORD" | chpasswd
+ # Still a plain file here: pve-cluster moves it into /etc/pve on its first
+ # start, so the base image's key is never trusted by the node.
+ mkdir -p /root/.ssh
+ printf '%s\n' "$KEYS" >/root/.ssh/authorized_keys
+ chmod 600 /root/.ssh/authorized_keys
+ echo "$NODE" >@IDENTITY_MARKER@
+ console "identity $NODE"
+fi
diff --git a/internal/golden/golden.go b/internal/golden/golden.go
index f63f8d7..76ddc9d 100644
--- a/internal/golden/golden.go
+++ b/internal/golden/golden.go
@@ -1,10 +1,14 @@
// Package golden caches installed base images so new nodes skip the installer.
//
-// A base image is an installed but never booted root disk, one per key (ISO,
-// root filesystem and size, installer locale). Nodes use qcow2 overlays on top of
-// it. Everything Proxmox VE generates on first boot (cluster CA, auth key, node
-// certificate, root SSH key) is therefore unique per node; what the installer
-// wrote (host name, SSH host keys, machine-id) is replaced by Personalize.
+// A base image is an installed root disk, one per key (ISO, root filesystem and
+// size, installer locale, baked Ceph release). Nodes use qcow2 overlays on top
+// of it. The image is booted once to bake it (package firstboot), but powers off
+// before any Proxmox VE service starts, so everything Proxmox VE generates on
+// first boot (cluster CA, auth key, node certificate, root SSH key) is unique per
+// node. What the installer wrote (host name, SSH host keys, machine-id) is
+// replaced on each clone's first boot by the first-boot hook, before networking
+// starts. Base images built before the hook existed are personalized over SSH
+// instead (Personalize), which costs a reboot.
package golden
import (
@@ -20,6 +24,7 @@ import (
"net"
"os"
"path/filepath"
+ "regexp"
"strings"
"syscall"
"time"
@@ -28,6 +33,7 @@ import (
"github.com/virtbase/proxbase/internal/answer"
"github.com/virtbase/proxbase/internal/config"
+ "github.com/virtbase/proxbase/internal/firstboot"
"github.com/virtbase/proxbase/internal/image"
"github.com/virtbase/proxbase/internal/state"
"github.com/virtbase/proxbase/internal/vm"
@@ -36,7 +42,15 @@ import (
// Hostname is the name nodes have before personalization.
const Hostname = "golden"
-const format = "1" // bump when the build or personalization changes
+const format = "2" // bump when the build or personalization changes
+
+const bakeTimeout = 20 * time.Minute
+
+// Builder installs and bakes base images.
+type Builder interface {
+ vm.Installer
+ vm.Booter
+}
// Base is a cached base image.
type Base struct {
@@ -50,6 +64,7 @@ type Meta struct {
ISO string `json:"iso"`
Filesystem string `json:"filesystem"`
Size string `json:"size"`
+ Ceph string `json:"ceph,omitempty"` // baked Ceph release
Created time.Time `json:"created"`
}
@@ -70,10 +85,18 @@ func (b Base) Signer() (ssh.Signer, error) {
// Key identifies the base image for a node.
func Key(cfg *config.Cluster, n config.Node, iso string) string {
h := sha256.Sum256([]byte(strings.Join([]string{format, filepath.Base(iso), n.Spec.RootDisk.Filesystem,
- n.Spec.RootDisk.Size, cfg.Proxmox.Keyboard, cfg.Proxmox.Country, cfg.Proxmox.Timezone}, "\n")))
+ n.Spec.RootDisk.Size, cfg.Proxmox.Keyboard, cfg.Proxmox.Country, cfg.Proxmox.Timezone, cephRelease(cfg)}, "\n")))
return hex.EncodeToString(h[:8])
}
+// cephRelease is the Ceph release baked into base images, if Ceph is used.
+func cephRelease(cfg *config.Cluster) string {
+ if !cfg.CephEnabled() {
+ return ""
+ }
+ return cfg.Storage.Ceph.Version
+}
+
// Open returns an existing base image.
func Open(key string) (Base, error) {
b := Base{Key: key, Dir: filepath.Join(imagesDir(), key)}
@@ -84,7 +107,8 @@ func Open(key string) (Base, error) {
}
// Ensure returns the base image for node n, installing it first if needed.
-func Ensure(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config.Node, img *image.Image, logf func(string, ...any)) (Base, error) {
+// aptEnv is put before the bake script's apt commands (see storage.Host.AptEnv).
+func Ensure(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, aptEnv string, logf func(string, ...any)) (Base, error) {
key := Key(cfg, n, img.ISO)
if b, err := Open(key); err == nil {
return b, nil
@@ -103,10 +127,14 @@ func Ensure(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n confi
if b, err := Open(key); err == nil { // built by another create meanwhile
return b, nil
}
- logf("building base image %s (Proxmox VE %s, %s %s); later creates reuse it", key, img.Version, n.Spec.RootDisk.Filesystem, n.Spec.RootDisk.Size)
+ what := fmt.Sprintf("Proxmox VE %s, %s %s", img.Version, n.Spec.RootDisk.Filesystem, n.Spec.RootDisk.Size)
+ if c := cephRelease(cfg); c != "" {
+ what += ", Ceph " + c
+ }
+ logf("building base image %s (%s); later creates reuse it", key, what)
tmp := filepath.Join(imagesDir(), key+".tmp")
_ = os.RemoveAll(tmp)
- if err := build(ctx, inst, cfg, n, img, key, tmp); err != nil {
+ if err := build(ctx, inst, cfg, n, img, key, tmp, aptEnv, logf); err != nil {
_ = os.RemoveAll(tmp)
return Base{}, err
}
@@ -117,7 +145,7 @@ func Ensure(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n confi
return Base{Key: key, Dir: dir}, nil
}
-func build(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config.Node, img *image.Image, key, dir string) error {
+func build(ctx context.Context, inst Builder, cfg *config.Cluster, n config.Node, img *image.Image, key, dir, aptEnv string, logf func(string, ...any)) error {
for _, sub := range []string{"run", "logs", "answer"} {
if err := os.MkdirAll(filepath.Join(dir, sub), 0o700); err != nil {
return err
@@ -139,11 +167,16 @@ func build(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config
if spec.SSHPort, err = freePort(); err != nil {
return err
}
+ hook, err := firstboot.Serve()
+ if err != nil {
+ return err
+ }
+ defer hook.Close()
ans := answer.Render(answer.Params{
FQDN: Hostname + ".proxbase.invalid", Mailto: "root@proxbase.invalid",
Keyboard: cfg.Proxmox.Keyboard, Country: cfg.Proxmox.Country, Timezone: cfg.Proxmox.Timezone,
RootPassword: answer.Password(24), SSHKeys: []string{pub}, MAC: spec.NATMAC,
- Filesystem: n.Spec.RootDisk.Filesystem,
+ Filesystem: n.Spec.RootDisk.Filesystem, FirstBootURL: hook.URL,
})
if err := os.WriteFile(filepath.Join(dir, "answer", "answer.toml"), []byte(ans), 0o600); err != nil {
return err
@@ -153,14 +186,56 @@ func build(ctx context.Context, inst vm.Installer, cfg *config.Cluster, n config
return fmt.Errorf("base image: %w", err)
}
_ = os.RemoveAll(filepath.Join(dir, "answer"))
+ ceph := cephRelease(cfg)
+ if err := bake(ctx, inst, spec, ceph, aptEnv, logf); err != nil {
+ return err
+ }
_ = os.RemoveAll(filepath.Join(dir, "run"))
- meta, err := json.MarshalIndent(Meta{Key: key, ISO: filepath.Base(img.ISO), Filesystem: n.Spec.RootDisk.Filesystem, Size: n.Spec.RootDisk.Size, Created: time.Now().UTC()}, "", " ")
+ meta, err := json.MarshalIndent(Meta{Key: key, ISO: filepath.Base(img.ISO), Filesystem: n.Spec.RootDisk.Filesystem, Size: n.Spec.RootDisk.Size, Ceph: ceph, Created: time.Now().UTC()}, "", " ")
if err != nil {
return err
}
return os.WriteFile(filepath.Join(dir, "meta.json"), meta, 0o644)
}
+// bake boots the installed image once with the bake script; the first-boot hook
+// runs it and powers off, reporting the result on the serial console.
+func bake(ctx context.Context, b vm.Booter, spec vm.Spec, ceph, aptEnv string, logf func(string, ...any)) error {
+ if ceph != "" {
+ logf("baking base image: installing Ceph %s", ceph)
+ }
+ script := filepath.Join(spec.RunDir, "bake.sh")
+ if err := os.WriteFile(script, []byte(aptEnv+firstboot.Bake(ceph, spec.NATMAC)), 0o600); err != nil {
+ return err
+ }
+ spec.Files = []vm.File{{Name: firstboot.FileBake, Path: script}}
+ console, err := b.BootOnce(ctx, spec, bakeTimeout)
+ if err != nil {
+ return fmt.Errorf("bake base image: %w", err)
+ }
+ if !strings.Contains(string(console), firstboot.BakeOK) {
+ return fmt.Errorf("bake base image failed; serial console:\n%s", lastLines(string(console), 30))
+ }
+ return nil
+}
+
+// ansiRe matches terminal control sequences (GRUB draws its menu with them).
+var ansiRe = regexp.MustCompile(`\x1b(\[[0-9;?]*[ -/]*[@-~]|\][^\x07]*\x07|[()][0-9A-Za-z])`)
+
+// lastLines returns the last n non-empty lines of console output, indented.
+func lastLines(s string, n int) string {
+ var lines []string
+ for _, l := range strings.Split(ansiRe.ReplaceAllString(strings.ReplaceAll(s, "\r", ""), ""), "\n") {
+ if strings.TrimSpace(l) != "" {
+ lines = append(lines, l)
+ }
+ }
+ if len(lines) > n {
+ lines = lines[len(lines)-n:]
+ }
+ return " " + strings.Join(lines, "\n ")
+}
+
// writeKey creates the login key of nodes before personalization.
func writeKey(dir string) (string, error) {
pub, priv, err := ed25519.GenerateKey(rand.Reader)
diff --git a/internal/golden/golden_test.go b/internal/golden/golden_test.go
index a8e7a65..4fc59fe 100644
--- a/internal/golden/golden_test.go
+++ b/internal/golden/golden_test.go
@@ -21,6 +21,14 @@ func TestKey(t *testing.T) {
if Key(cfg, n[0], "/c/proxmox-ve_9.2-2.iso") == k1 {
t.Fatal("a different ISO needs a different key")
}
+ ceph, _ := config.Decode(strings.NewReader("nodes: {count: 3}\nstorage: {ceph: {enabled: true}}\n"))
+ ceph.SetDefaults()
+ if !ceph.CephEnabled() {
+ t.Fatal("test config must enable Ceph")
+ }
+ if Key(ceph, ceph.NodeList()[0], "/c/proxmox-ve_9.2-1.iso") == k1 {
+ t.Fatal("a base image with Ceph baked in needs a different key")
+ }
}
func TestPersonalize(t *testing.T) {
@@ -41,3 +49,14 @@ func TestPersonalize(t *testing.T) {
t.Error("Finish must remove the base image's node directory")
}
}
+
+func TestLastLines(t *testing.T) {
+ console := "\x1b[H\x1b[J\x1b[1;1H\x1b[?25l\x1b[m GNU GRUB version 2.12\r\n\x1b[5;78H\r\n" +
+ " Booting `Proxmox VE GNU/Linux'\r\n\r\n\x1b]104\x07\x1b[?7herror: Another instance of this program is already running.\r\n" +
+ "proxbase-first-boot: bake failed\r\n"
+ got := lastLines(console, 2)
+ want := " error: Another instance of this program is already running.\n proxbase-first-boot: bake failed"
+ if got != want {
+ t.Fatalf("got %q, want %q", got, want)
+ }
+}
diff --git a/internal/golden/personalize.go b/internal/golden/personalize.go
index 42336a1..8613c0d 100644
--- a/internal/golden/personalize.go
+++ b/internal/golden/personalize.go
@@ -3,14 +3,17 @@ package golden
import (
"fmt"
"strings"
+
+ "github.com/virtbase/proxbase/internal/firstboot"
)
-// IdentityMarker exists on nodes that were personalized.
-const IdentityMarker = "/etc/proxbase-identity"
+// IdentityMarker exists on nodes that were personalized; it holds the node name.
+const IdentityMarker = firstboot.IdentityMarker
-// Personalize returns the script that gives a node cloned from a base image its
-// own identity. The node must reboot afterwards (Proxmox services read the host
-// name at start); Finish completes it after the reboot.
+// Personalize returns the script that gives a node cloned from a base image of
+// format 1 (built before the first-boot hook) its own identity. The node must
+// reboot afterwards (Proxmox services read the host name at start); Finish
+// completes it after the reboot.
func Personalize(node, domain, password string, authorizedKeys []string) string {
fqdn := node + "." + domain
return fmt.Sprintf(`
@@ -18,6 +21,7 @@ node=%[1]s fqdn=%[2]s
hostname "$node"
echo "$node" > /etc/hostname
sed -i "/^10\.0\.2\.15[[:space:]]/c\10.0.2.15 $fqdn $node" /etc/hosts
+sed -i "s/^search .*/search %[6]s/" /etc/resolv.conf
echo "$fqdn" > /etc/mailname
postconf -e "myhostname=$fqdn"
rm -f /etc/ssh/ssh_host_*
@@ -28,7 +32,7 @@ echo "root:%[3]s" | chpasswd
printf '%%s\n' %[4]s > /etc/pve/priv/authorized_keys
echo "$node" > %[5]s
systemd-run --on-active=2 systemctl reboot >/dev/null
-`, node, fqdn, password, shellQuote(authorizedKeys), IdentityMarker)
+`, node, fqdn, password, shellQuote(authorizedKeys), IdentityMarker, domain)
}
// Finish removes what is left of the base image's identity after the reboot and
diff --git a/internal/image/image.go b/internal/image/image.go
index a25015e..1f15bde 100644
--- a/internal/image/image.go
+++ b/internal/image/image.go
@@ -38,18 +38,23 @@ func Ensure(ctx context.Context, mirror, version, cacheDir string, logf Logf) (*
if err := os.MkdirAll(dir, 0o755); err != nil {
return nil, err
}
- mirror = strings.TrimRight(mirror, "/")
- sums, err := fetchSums(ctx, mirror, dir)
- if err != nil {
- return nil, err
- }
- file, sum, err := resolve(sums, version)
- if err != nil {
- return nil, err
- }
+ file := "proxmox-ve_" + version + ".iso"
iso := filepath.Join(dir, file)
- if err := ensureISO(ctx, mirror+"/"+file, iso, sum, logf); err != nil {
- return nil, err
+ // An exact version that was downloaded and verified before needs no network.
+ if !isoRe.MatchString(file) || !verified(iso) {
+ mirror = strings.TrimRight(mirror, "/")
+ sums, err := fetchSums(ctx, mirror, dir)
+ if err != nil {
+ return nil, err
+ }
+ var sum string
+ if file, sum, err = resolve(sums, version); err != nil {
+ return nil, err
+ }
+ iso = filepath.Join(dir, file)
+ if err := ensureISO(ctx, mirror+"/"+file, iso, sum, logf); err != nil {
+ return nil, err
+ }
}
img := &Image{Version: strings.TrimSuffix(strings.TrimPrefix(file, "proxmox-ve_"), ".iso"), ISO: iso}
if err := extract(img, strings.TrimSuffix(iso, ".iso")); err != nil {
@@ -126,6 +131,21 @@ func resolve(sums map[string]string, version string) (file, sum string, err erro
return file, sum, nil
}
+// verified reports whether path has a stamp from an earlier verification that
+// still matches its size.
+func verified(path string) bool {
+ fi, err := os.Stat(path)
+ if err != nil {
+ return false
+ }
+ b, err := os.ReadFile(path + ".sha256")
+ if err != nil {
+ return false
+ }
+ f := strings.Fields(string(b))
+ return len(f) == 2 && len(f[0]) == 64 && f[1] == strconv.FormatInt(fi.Size(), 10)
+}
+
func ensureISO(ctx context.Context, url, path, sum string, logf Logf) error {
stamp := path + ".sha256"
if fi, err := os.Stat(path); err == nil {
diff --git a/internal/image/image_test.go b/internal/image/image_test.go
index 801d8b7..dbdc922 100644
--- a/internal/image/image_test.go
+++ b/internal/image/image_test.go
@@ -1,6 +1,11 @@
package image
-import "testing"
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
const grub = `menuentry 'Install Proxmox VE (Automated)' --class debian {
echo 'Loading Proxmox VE Automatic Installer ...'
@@ -34,3 +39,64 @@ dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd proxmox-backup
t.Error("expected error for missing version")
}
}
+
+// An exact version that is cached and verified must not touch the mirror.
+func TestEnsure_CachedExactVersionOffline(t *testing.T) {
+ t.Parallel()
+ cache := t.TempDir()
+ dir := filepath.Join(cache, "iso")
+ boot := filepath.Join(dir, "proxmox-ve_9.2-1")
+ if err := os.MkdirAll(boot, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ iso := filepath.Join(dir, "proxmox-ve_9.2-1.iso")
+ files := map[string]string{
+ iso: "iso",
+ iso + ".sha256": strings.Repeat("a", 64) + " 3\n",
+ filepath.Join(boot, "linux26"): "k",
+ filepath.Join(boot, "initrd.img"): "i",
+ filepath.Join(boot, "grub.cfg"): grub,
+ }
+ for p, content := range files {
+ if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ img, err := Ensure(t.Context(), "http://127.0.0.1:1", "9.2-1", cache, t.Logf)
+ if err != nil {
+ t.Fatalf("Ensure went to the (unreachable) mirror: %v", err)
+ }
+ if img.Version != "9.2-1" || img.ISO != iso {
+ t.Fatalf("got %+v", img)
+ }
+}
+
+func TestVerified(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ name string
+ stamp string
+ want bool
+ }{
+ {name: "matching size", stamp: strings.Repeat("a", 64) + " 3\n", want: true},
+ {name: "size changed", stamp: strings.Repeat("a", 64) + " 4\n", want: false},
+ {name: "no stamp", stamp: "", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ p := filepath.Join(t.TempDir(), "x.iso")
+ if err := os.WriteFile(p, []byte("iso"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if tt.stamp != "" {
+ if err := os.WriteFile(p+".sha256", []byte(tt.stamp), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if got := verified(p); got != tt.want {
+ t.Errorf("verified = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/mcpserver/tools.go b/internal/mcpserver/tools.go
index b689694..67ea109 100644
--- a/internal/mcpserver/tools.go
+++ b/internal/mcpserver/tools.go
@@ -83,6 +83,12 @@ type snapshotIn struct {
Name string `json:"name" jsonschema:"snapshot name"`
}
+type snapshotSaveIn struct {
+ Cluster string `json:"cluster,omitempty" jsonschema:"cluster name; default: the only cluster"`
+ Name string `json:"name" jsonschema:"snapshot name"`
+ Live bool `json:"live,omitempty" jsonschema:"snapshot the running cluster with its memory; restoring it resumes the cluster in seconds"`
+}
+
type snapshotsOut struct {
Snapshots []state.Snapshot `json:"snapshots"`
}
@@ -130,8 +136,8 @@ func (h *handlers) register(s *mcp.Server) {
mcp.AddTool(s, &mcp.Tool{Name: "cluster_destroy", Description: "Delete a cluster: kill its VMs and remove all its disks, snapshots and credentials. Also stops a create running in the background. Set confirm to the cluster name.", Annotations: local(true, true)}, h.destroy)
mcp.AddTool(s, &mcp.Tool{Name: "node_exec", Description: "Run a shell command as root on a running node over SSH and return stdout, stderr and the exit code. Use it for pvesh, qm, pct, pvecm, ceph and the like. Non-interactive; output is capped at 64 KiB per stream.", Annotations: local(true, false)}, h.exec)
mcp.AddTool(s, &mcp.Tool{Name: "cluster_env", Description: "Return what API and SSH clients need: API endpoint, API token (proxbase@pve!api, full privileges), the cluster CA certificate (PEM and path), SSH user, key path and ports per node. Root password only if includeRootPassword is set.", Annotations: readOnly}, h.env)
- mcp.AddTool(s, &mcp.Tool{Name: "snapshot_save", Description: "Snapshot all disks of a stopped cluster (call cluster_stop first). Restoring is much faster than creating again.", Annotations: local(false, false)}, h.snapshotOp((*cluster.Cluster).SnapshotSave, "saved"))
- mcp.AddTool(s, &mcp.Tool{Name: "snapshot_restore", Description: "Reset all disks of a stopped cluster to a snapshot (call cluster_stop first, cluster_start afterwards). Changes since the snapshot are lost.", Annotations: local(true, true)}, h.snapshotOp((*cluster.Cluster).SnapshotRestore, "restored"))
+ mcp.AddTool(s, &mcp.Tool{Name: "snapshot_save", Description: "Snapshot all disks of a stopped cluster (call cluster_stop first), or with live=true the running cluster including its memory. Restoring is much faster than creating again.", Annotations: local(false, false)}, h.snapshotSave)
+ mcp.AddTool(s, &mcp.Tool{Name: "snapshot_restore", Description: "Reset all disks of a stopped cluster to a snapshot (call cluster_stop first). Changes since the snapshot are lost. After a disk snapshot call cluster_start; a live snapshot starts the cluster in its saved state by itself.", Annotations: local(true, true)}, h.snapshotOp((*cluster.Cluster).SnapshotRestore, "restored"))
mcp.AddTool(s, &mcp.Tool{Name: "snapshot_delete", Description: "Delete a snapshot of a cluster.", Annotations: local(true, true)}, h.snapshotOp((*cluster.Cluster).SnapshotDelete, "deleted"))
mcp.AddTool(s, &mcp.Tool{Name: "snapshot_list", Description: "List the snapshots of a cluster.", Annotations: readOnly}, h.snapshots)
mcp.AddTool(s, &mcp.Tool{Name: "fault_apply", Description: "Inject a failure into a running cluster to test HA, fencing, Ceph or applications: kill, freeze, thaw, link-down, link-up, partition or degrade. fault_clear undoes them.", Annotations: local(true, false)}, h.fault)
@@ -306,6 +312,14 @@ func (h *handlers) snapshotOp(f func(*cluster.Cluster, context.Context, string)
}
}
+func (h *handlers) snapshotSave(ctx context.Context, req *mcp.CallToolRequest, in snapshotSaveIn) (*mcp.CallToolResult, messageOut, error) {
+ save := (*cluster.Cluster).SnapshotSave
+ if in.Live {
+ save = (*cluster.Cluster).SnapshotSaveLive
+ }
+ return h.snapshotOp(save, "saved")(ctx, req, snapshotIn{Cluster: in.Cluster, Name: in.Name})
+}
+
func (h *handlers) snapshots(_ context.Context, _ *mcp.CallToolRequest, in clusterIn) (*mcp.CallToolResult, snapshotsOut, error) {
c, err := h.open(in.Cluster, nil)
if err != nil {
diff --git a/internal/nodesetup/nodesetup.go b/internal/nodesetup/nodesetup.go
index 5647b2d..5bab356 100644
--- a/internal/nodesetup/nodesetup.go
+++ b/internal/nodesetup/nodesetup.go
@@ -63,20 +63,9 @@ block /etc/hosts "$(cat <<'EOF'
# Login prompt on the serial console (proxbase console).
systemctl enable --now serial-getty@ttyS0.service >/dev/null 2>&1
-
+%s
# No-subscription repository instead of enterprise.
-codename=$(. /etc/os-release && echo "$VERSION_CODENAME")
-for f in /etc/apt/sources.list.d/pve-enterprise.sources /etc/apt/sources.list.d/ceph.sources; do
- if [ -f "$f" ] && ! grep -q '^Enabled: false' "$f"; then echo 'Enabled: false' >> "$f"; fi
-done
-cat > /etc/apt/sources.list.d/proxmox.sources <> "$f"
+ fi
+done
+cat > /etc/apt/sources.list.d/proxmox.sources </dev/null 2>&1; then
+ proxmox-boot-tool refresh >/dev/null 2>&1
+ else
+ update-grub >/dev/null 2>&1
+ fi
+fi
+`
+
+// ProvisionStart speeds up writes while proxbase provisions a node: ZFS treats
+// fsync as a no-op and dpkg skips its per-file syncs. ProvisionEnd restores
+// both, so a ready cluster has real fsync semantics for fault injection. It also
+// turns on autotrim so freed space shrinks the disk images.
+const ProvisionStart = `
+if zpool list rpool >/dev/null 2>&1; then
+ zfs set sync=disabled rpool
+ zpool set autotrim=on rpool
+fi
+echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io
+`
+
+// ProvisionEnd undoes ProvisionStart's write shortcuts.
+const ProvisionEnd = `
+if zpool list rpool >/dev/null 2>&1; then zfs set sync=standard rpool; fi
+rm -f /etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io
+`
+
// AptWait waits for apt runs started by PVE itself (pveupdate right after boot)
// and sets $apt to an apt-get that waits for locks and retries downloads.
+// retry_apt runs an apt command again past any cache if it fails: a broken
+// download (Hash Sum mismatch) is then fetched anew, also by the host's cache.
const AptWait = `
-for i in $(seq 300); do pgrep -x 'apt-get|apt|dpkg|unattended-upgr' >/dev/null || break; sleep 2; done
+for _ in $(seq 300); do pgrep -x 'apt-get|apt|dpkg|unattended-upgr' >/dev/null || break; sleep 2; done
apt='apt-get -o DPkg::Lock::Timeout=600 -o Acquire::Retries=3'
+retry_apt() { "$@" || "$@" -o Acquire::http::No-Cache=true; }
`
// Upgrade brings a node to the newest no-subscription packages and prints the
@@ -104,7 +146,7 @@ apt='apt-get -o DPkg::Lock::Timeout=600 -o Acquire::Retries=3'
const Upgrade = AptWait + `
export DEBIAN_FRONTEND=noninteractive
log=/var/log/proxbase-upgrade.log
-if ! { $apt update && $apt -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold dist-upgrade; } >$log 2>&1; then
+if ! { $apt update && retry_apt $apt -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold dist-upgrade; } >$log 2>&1; then
tail -n 20 $log >&2; exit 1
fi
grep -E '^[0-9]+ upgraded' $log || true
@@ -118,5 +160,39 @@ if [ "$new" != "$(uname -r)" ]; then echo "$new $(cat /proc/sys/kernel/random/bo
// Reboot reboots two seconds later, so the SSH session can end cleanly.
const Reboot = "systemd-run --on-active=2 systemctl reboot >/dev/null"
+// RepairJoin repairs what a cluster join can leave broken. The joining node
+// finishes its setup before the cluster file system has synced to it: files it
+// writes then (its key and certificate) can be lost, and if it believes the
+// cluster CA is missing it writes a new one, which leaves the other nodes with
+// certificates of the old CA. A failed join also skips restarting pvedaemon and
+// pveproxy, which then reject new tickets. RepairJoin issues a new node
+// certificate from the current cluster CA and restarts both services if the
+// certificate is missing or not signed by that CA, or the node's last join
+// failed, and prints "repaired". If pveproxy serves another certificate than
+// /etc/pve holds, it restarts pveproxy and prints "restarted".
+const RepairJoin = `
+marker=/var/lib/proxbase-join-repaired
+join=$(grep ':clusterjoin:' /var/log/pve/tasks/index 2>/dev/null | tail -n 1 || true)
+failed=
+if [ -n "$join" ] && ! echo "$join" | grep -q ' OK$' && [ "$(cat $marker 2>/dev/null)" != "$join" ]; then
+ failed=1
+fi
+if [ -n "$failed" ] || [ ! -s /etc/pve/local/pve-ssl.key ] ||
+ ! openssl verify -CAfile /etc/pve/pve-root-ca.pem /etc/pve/local/pve-ssl.pem >/dev/null 2>&1; then
+ pvecm updatecerts --force --silent
+ systemctl restart pvedaemon pveproxy
+ if [ -n "$join" ]; then echo "$join" > $marker; fi
+ echo repaired
+ exit 0
+fi
+# pveproxy can also keep serving a certificate that /etc/pve no longer holds.
+fp() { openssl x509 -noout -fingerprint -sha256 2>/dev/null || true; }
+served=$(openssl s_client -connect 127.0.0.1:8006 /dev/null | fp)
+if [ "$served" != "$(fp < /etc/pve/local/pve-ssl.pem)" ]; then
+ systemctl restart pveproxy
+ echo restarted
+fi
+`
+
// BootID prints the boot ID and the running kernel.
const BootID = "cat /proc/sys/kernel/random/boot_id; uname -r"
diff --git a/internal/nodesetup/nodesetup_test.go b/internal/nodesetup/nodesetup_test.go
index e8ee8ae..3e368eb 100644
--- a/internal/nodesetup/nodesetup_test.go
+++ b/internal/nodesetup/nodesetup_test.go
@@ -1,6 +1,7 @@
package nodesetup
import (
+ "os/exec"
"strings"
"testing"
@@ -28,9 +29,52 @@ networks:
"auto vmbr3\niface vmbr3 inet manual\n\tbridge-ports $nic2\n\tbridge-stp off\n\tbridge-fd 0\n\tbridge-vlan-aware yes\n\tbridge-vids 100 200-299\n",
"Components: pve-no-subscription",
"serial-getty@ttyS0",
+ "GRUB_TIMEOUT=0",
} {
if !strings.Contains(s, want) {
t.Errorf("script lacks %q", want)
}
}
}
+
+// Provisioning shortcuts must be undone completely.
+func TestProvision(t *testing.T) {
+ for _, pair := range [][2]string{
+ {"zfs set sync=disabled rpool", "zfs set sync=standard rpool"},
+ {"/etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io", "rm -f /etc/dpkg/dpkg.cfg.d/proxbase-unsafe-io"},
+ } {
+ if !strings.Contains(ProvisionStart, pair[0]) {
+ t.Errorf("ProvisionStart lacks %q", pair[0])
+ }
+ if !strings.Contains(ProvisionEnd, pair[1]) {
+ t.Errorf("ProvisionEnd lacks %q", pair[1])
+ }
+ }
+}
+
+// Every script must at least parse.
+func TestScriptsSyntax(t *testing.T) {
+ cfg, err := config.Decode(strings.NewReader("nodes: {count: 2}\n"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg.SetDefaults()
+ for name, s := range map[string]string{
+ "Script": Script(cfg, cfg.NodeList()[0]),
+ "Repositories": Repositories,
+ "NoGrubTimeout": NoGrubTimeout,
+ "ProvisionStart": ProvisionStart,
+ "ProvisionEnd": ProvisionEnd,
+ "Upgrade": Upgrade,
+ "PendingKernel": PendingKernel,
+ "RepairJoin": RepairJoin,
+ } {
+ t.Run(name, func(t *testing.T) {
+ cmd := exec.CommandContext(t.Context(), "bash", "-n")
+ cmd.Stdin = strings.NewReader(s)
+ if out, err := cmd.CombinedOutput(); err != nil {
+ t.Errorf("bash -n: %v\n%s", err, out)
+ }
+ })
+ }
+}
diff --git a/internal/pve/api.go b/internal/pve/api.go
index de68db7..1e5d3ce 100644
--- a/internal/pve/api.go
+++ b/internal/pve/api.go
@@ -15,15 +15,19 @@ import (
"net/http"
"net/url"
"strings"
+ "sync"
"time"
)
+// Client is safe for concurrent use.
type Client struct {
- base string
- http *http.Client
- ticket string
- csrf string
- token string // "user@realm!id=secret"
+ base string
+ http *http.Client
+ token string // "user@realm!id=secret"; set before use
+
+ mu sync.Mutex
+ ticket, csrf string
+ user, password string // for logging in again
}
// Fingerprint returns the colon-separated SHA-256 fingerprint of a PEM certificate,
@@ -85,20 +89,26 @@ func newClient(port int, verify func(raw [][]byte) error) *Client {
}
return &Client{
base: fmt.Sprintf("https://127.0.0.1:%d/api2/json", port),
- http: &http.Client{Timeout: 60 * time.Second, Transport: &http.Transport{TLSClientConfig: tlsCfg}},
+ http: &http.Client{
+ Timeout: 60 * time.Second,
+ Transport: &http.Transport{TLSClientConfig: tlsCfg, IdleConnTimeout: 30 * time.Second},
+ },
}
}
// Login gets a ticket for user (e.g. root@pam).
+// The client logs in again by itself when the ticket is rejected.
func (c *Client) Login(ctx context.Context, user, password string) error {
var out struct {
Ticket string `json:"ticket"`
CSRF string `json:"CSRFPreventionToken"`
}
- if err := c.Do(ctx, http.MethodPost, "/access/ticket", url.Values{"username": {user}, "password": {password}}, &out); err != nil {
+ if err := c.do(ctx, http.MethodPost, "/access/ticket", url.Values{"username": {user}, "password": {password}}, &out); err != nil {
return err
}
- c.ticket, c.csrf = out.Ticket, out.CSRF
+ c.mu.Lock()
+ c.ticket, c.csrf, c.user, c.password = out.Ticket, out.CSRF, user, password
+ c.mu.Unlock()
return nil
}
@@ -131,7 +141,25 @@ func (c *Client) Delete(ctx context.Context, path string) error {
return c.Do(ctx, http.MethodDelete, path, nil, nil)
}
+// Do sends a request and decodes the data of the response into out. A request
+// with a ticket that the node rejects (401) is sent once more after a new login:
+// right after a node joined a cluster, tickets can briefly fail to verify while
+// the cluster's auth key reaches it.
func (c *Client) Do(ctx context.Context, method, path string, form url.Values, out any) error {
+ err := c.do(ctx, method, path, form, out)
+ c.mu.Lock()
+ user, password := c.user, c.password
+ c.mu.Unlock()
+ if !IsStatus(err, http.StatusUnauthorized) || c.token != "" || user == "" {
+ return err
+ }
+ if lerr := c.Login(ctx, user, password); lerr != nil {
+ return errors.Join(err, lerr)
+ }
+ return c.do(ctx, method, path, form, out)
+}
+
+func (c *Client) do(ctx context.Context, method, path string, form url.Values, out any) error {
var body io.Reader
if form != nil {
body = strings.NewReader(form.Encode())
@@ -143,13 +171,16 @@ func (c *Client) Do(ctx context.Context, method, path string, form url.Values, o
if form != nil {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
+ c.mu.Lock()
+ ticket, csrf := c.ticket, c.csrf
+ c.mu.Unlock()
switch {
case c.token != "":
req.Header.Set("Authorization", "PVEAPIToken="+c.token)
- case c.ticket != "":
- req.AddCookie(&http.Cookie{Name: "PVEAuthCookie", Value: c.ticket}) //nolint:gosec // request cookie, attributes do not apply
+ case ticket != "":
+ req.AddCookie(&http.Cookie{Name: "PVEAuthCookie", Value: ticket}) //nolint:gosec // request cookie, attributes do not apply
if method != http.MethodGet {
- req.Header.Set("CSRFPreventionToken", c.csrf)
+ req.Header.Set("CSRFPreventionToken", csrf)
}
}
resp, err := c.http.Do(req)
@@ -239,5 +270,11 @@ func (c *Client) WaitTask(ctx context.Context, node, upid string, timeout time.D
}
}
-// SetToken switches the client to API token auth ("user@realm!id=secret").
+// SetToken switches the client to API token auth ("user@realm!id=secret"). Call
+// it before using the client.
func (c *Client) SetToken(token string) { c.token = token }
+
+// WithToken returns a client for the same node that uses an API token.
+func (c *Client) WithToken(token string) *Client {
+ return &Client{base: c.base, http: c.http, token: token}
+}
diff --git a/internal/qemu/fault.go b/internal/qemu/fault.go
index 974f619..3543dc2 100644
--- a/internal/qemu/fault.go
+++ b/internal/qemu/fault.go
@@ -20,14 +20,10 @@ func (Runtime) Kill(s vm.Spec) error {
}
// Freeze stops all vCPUs; the node hangs but keeps its memory.
-func (Runtime) Freeze(_ context.Context, s vm.Spec) error {
- return qmpExec(qmpSocket(s), "stop", nil)
-}
+func (r Runtime) Freeze(ctx context.Context, s vm.Spec) error { return r.Pause(ctx, s) }
// Thaw resumes a frozen node.
-func (Runtime) Thaw(_ context.Context, s vm.Spec) error {
- return qmpExec(qmpSocket(s), "cont", nil)
-}
+func (r Runtime) Thaw(ctx context.Context, s vm.Spec) error { return r.Resume(ctx, s) }
// SetLink sets the carrier of internal NIC nic, as if its cable was pulled.
func (Runtime) SetLink(_ context.Context, s vm.Spec, nic int, up bool) error {
diff --git a/internal/qemu/install.go b/internal/qemu/install.go
index 4da79f4..bd5a061 100644
--- a/internal/qemu/install.go
+++ b/internal/qemu/install.go
@@ -30,6 +30,38 @@ func (Runtime) Clone(ctx context.Context, s vm.Spec, base string) error {
return createOverlay(ctx, s.RootDisk, base)
}
+var _ vm.Booter = Runtime{}
+
+// BootOnce boots the installed node in the foreground and returns its serial
+// console output when it powers off. A reboot also ends it (-no-reboot).
+func (Runtime) BootOnce(ctx context.Context, s vm.Spec, timeout time.Duration) ([]byte, error) {
+ if _, ok := running(pidfile(s)); ok {
+ return nil, fmt.Errorf("a VM is already running (pidfile %s)", pidfile(s))
+ }
+ cleanSockets(s)
+ qlog, err := os.Create(qemuLog(s))
+ if err != nil {
+ return nil, err
+ }
+ defer qlog.Close()
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, Binary, args(s, bootOnce, nil)...)
+ cmd.SysProcAttr = installProcAttr()
+ cmd.Stdout, cmd.Stderr = qlog, qlog
+ if err := cmd.Run(); err != nil {
+ if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+ return nil, fmt.Errorf("%s did not power off within %s, see %s", s.Name, timeout, consoleLog(s))
+ }
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ b, _ := os.ReadFile(qemuLog(s))
+ return nil, fmt.Errorf("QEMU exited with %w: %s", err, strings.TrimSpace(string(b)))
+ }
+ return os.ReadFile(consoleLog(s))
+}
+
// Install creates fresh disks and runs the unattended installer in the foreground,
// answering its debug shell over the serial console. QEMU powers off when done.
func (Runtime) Install(ctx context.Context, s vm.Spec, m vm.Media) error {
@@ -47,7 +79,7 @@ func (Runtime) Install(ctx context.Context, s vm.Spec, m vm.Media) error {
return err
}
defer qlog.Close()
- cmd := exec.Command(Binary, args(s, &m)...)
+ cmd := exec.Command(Binary, args(s, bootInstall, &m)...)
cmd.SysProcAttr = installProcAttr()
cmd.Stdout, cmd.Stderr = qlog, qlog
if err := cmd.Start(); err != nil {
diff --git a/internal/qemu/live.go b/internal/qemu/live.go
new file mode 100644
index 0000000..1d9721c
--- /dev/null
+++ b/internal/qemu/live.go
@@ -0,0 +1,140 @@
+package qemu
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/virtbase/proxbase/internal/vm"
+)
+
+var _ vm.LiveSnapshotter = Runtime{}
+
+const liveSnapshotTimeout = 10 * time.Minute
+
+// SaveLive saves an internal snapshot of every disk with the node's memory (in
+// the root disk) through a QMP snapshot-save job. QEMU stops the vCPUs while it
+// saves and resumes them afterwards, unless the node was paused.
+func (Runtime) SaveLive(ctx context.Context, s vm.Spec, name string) error {
+ q, err := dialQMP(qmpSocket(s))
+ if err != nil {
+ return fmt.Errorf("%s: %w", s.Name, err)
+ }
+ defer q.Close()
+ nodes, err := diskNodes(q, s)
+ if err != nil {
+ return fmt.Errorf("%s: %w", s.Name, err)
+ }
+ job := "snapshot-save-" + name
+ if _, err := q.Execute("snapshot-save", map[string]any{"job-id": job, "tag": name, "vmstate": nodes[0], "devices": nodes}); err != nil {
+ return fmt.Errorf("%s: %w", s.Name, err)
+ }
+ if err := waitJob(ctx, q, job, liveSnapshotTimeout); err != nil {
+ return fmt.Errorf("%s: snapshot %s: %w", s.Name, name, err)
+ }
+ return nil
+}
+
+// DeleteLive deletes a snapshot from every disk of a running node through a
+// QMP snapshot-delete job. Disks without the snapshot are no error.
+func (Runtime) DeleteLive(ctx context.Context, s vm.Spec, name string) error {
+ q, err := dialQMP(qmpSocket(s))
+ if err != nil {
+ return fmt.Errorf("%s: %w", s.Name, err)
+ }
+ defer q.Close()
+ nodes, err := diskNodes(q, s)
+ if err != nil {
+ return fmt.Errorf("%s: %w", s.Name, err)
+ }
+ job := "snapshot-delete-" + name
+ if _, err := q.Execute("snapshot-delete", map[string]any{"job-id": job, "tag": name, "devices": nodes}); err != nil {
+ return fmt.Errorf("%s: %w", s.Name, err)
+ }
+ if err := waitJob(ctx, q, job, liveSnapshotTimeout); err != nil {
+ return fmt.Errorf("%s: delete snapshot %s: %w", s.Name, name, err)
+ }
+ return nil
+}
+
+// Pause stops all vCPUs.
+func (Runtime) Pause(_ context.Context, s vm.Spec) error { return qmpExec(qmpSocket(s), "stop", nil) }
+
+// Resume lets the vCPUs run again.
+func (Runtime) Resume(_ context.Context, s vm.Spec) error { return qmpExec(qmpSocket(s), "cont", nil) }
+
+// diskNodes returns the block node names of the root disk and the data disks,
+// in that order.
+func diskNodes(q *qmpConn, s vm.Spec) ([]string, error) {
+ raw, err := q.Execute("query-block", nil)
+ if err != nil {
+ return nil, err
+ }
+ var blocks []struct {
+ Device string `json:"device"`
+ Inserted *struct {
+ NodeName string `json:"node-name"`
+ } `json:"inserted"`
+ }
+ if err := json.Unmarshal(raw, &blocks); err != nil {
+ return nil, err
+ }
+ byDevice := map[string]string{}
+ for _, b := range blocks {
+ if b.Inserted != nil {
+ byDevice[b.Device] = b.Inserted.NodeName
+ }
+ }
+ ids := []string{"root"}
+ for i := range s.DataDisks {
+ ids = append(ids, fmt.Sprintf("data%d", i))
+ }
+ nodes := make([]string, 0, len(ids))
+ for _, id := range ids {
+ n, ok := byDevice[id]
+ if !ok {
+ return nil, fmt.Errorf("disk %s not found", id)
+ }
+ nodes = append(nodes, n)
+ }
+ return nodes, nil
+}
+
+// waitJob waits until a QMP job has concluded, returns its error and dismisses it.
+func waitJob(ctx context.Context, q *qmpConn, id string, timeout time.Duration) error {
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ tick := time.NewTicker(200 * time.Millisecond)
+ defer tick.Stop()
+ for {
+ raw, err := q.Execute("query-jobs", nil)
+ if err != nil {
+ return err
+ }
+ var jobs []struct {
+ ID string `json:"id"`
+ Status string `json:"status"`
+ Error string `json:"error"`
+ }
+ if err := json.Unmarshal(raw, &jobs); err != nil {
+ return err
+ }
+ for _, j := range jobs {
+ if j.ID != id || j.Status != "concluded" {
+ continue
+ }
+ _, derr := q.Execute("job-dismiss", map[string]any{"id": id})
+ if j.Error != "" {
+ return errors.New(j.Error)
+ }
+ return derr
+ }
+ select {
+ case <-ctx.Done():
+ return fmt.Errorf("job %s: %w", id, ctx.Err())
+ case <-tick.C:
+ }
+ }
+}
diff --git a/internal/qemu/process.go b/internal/qemu/process.go
index 87e150c..70d6e61 100644
--- a/internal/qemu/process.go
+++ b/internal/qemu/process.go
@@ -18,7 +18,7 @@ func (Runtime) Start(ctx context.Context, s vm.Spec) error {
return nil
}
cleanSockets(s)
- out, err := exec.CommandContext(ctx, Binary, args(s, nil)...).CombinedOutput()
+ out, err := exec.CommandContext(ctx, Binary, args(s, bootDaemon, nil)...).CombinedOutput()
if err != nil {
return fmt.Errorf("start %s: %w: %s", s.Process, err, strings.TrimSpace(string(out)))
}
diff --git a/internal/qemu/qemu.go b/internal/qemu/qemu.go
index 90bf5a4..c68b793 100644
--- a/internal/qemu/qemu.go
+++ b/internal/qemu/qemu.go
@@ -21,6 +21,7 @@ const (
slotNAT = 0x04
slotNICBase = 0x05 // up to 8 segment NICs: 0x05..0x0c
slotRNG = 0x0e
+ slotBalloon = 0x0f
slotData = 0x10 // up to 8 data disks: 0x10..0x17
)
@@ -41,10 +42,23 @@ func qemuLog(s vm.Spec) string { return filepath.Join(s.LogDir, s.Name+"-qemu
// ConsoleSocket is the serial console of a running node (proxbase console).
func ConsoleSocket(s vm.Spec) string { return console(s) }
+// driveOpts: guest flushes are ignored (nodes are disposable), and discards and
+// zero writes free space in the image.
+const driveOpts = "cache=unsafe,discard=unmap,detect-zeroes=unmap"
+
func dev(format string, a ...any) []string { return []string{"-device", fmt.Sprintf(format, a...)} }
-// args returns the command line; media != nil boots the installer.
-func args(s vm.Spec, media *vm.Media) []string {
+// boot selects how a node is started.
+type boot int
+
+const (
+ bootDaemon boot = iota // installed node, daemonized with a console socket
+ bootInstall // the installer from media, in the foreground
+ bootOnce // installed node in the foreground until it powers off
+)
+
+// args returns the command line; media is only used by bootInstall.
+func args(s vm.Spec, b boot, media *vm.Media) []string {
cpu := "host"
if !s.Nested {
cpu += ",vmx=off,svm=off"
@@ -59,7 +73,7 @@ func args(s vm.Spec, media *vm.Media) []string {
"-display", "none",
}
a = append(a, dev("VGA,bus=pcie.0,addr=%#x", slotVGA)...)
- a = append(a, "-drive", "if=none,id=root,file="+s.RootDisk.Path+",format=qcow2,cache=unsafe,discard=unmap")
+ a = append(a, "-drive", "if=none,id=root,file="+s.RootDisk.Path+",format=qcow2,"+driveOpts)
a = append(a, dev("virtio-blk-pci,drive=root,bus=pcie.0,addr=%#x,bootindex=1", slotRoot)...)
a = append(a, dev("qemu-xhci,id=usb,bus=pcie.0,addr=%#x", slotUSB)...)
fwd := fmt.Sprintf("hostfwd=tcp:%[1]s:%[2]d-10.0.2.15:22,hostfwd=tcp:%[1]s:%[3]d-10.0.2.15:8006", s.Bind, s.SSHPort, s.UIPort)
@@ -72,13 +86,20 @@ func args(s vm.Spec, media *vm.Media) []string {
}
a = append(a, "-object", "rng-random,id=rng,filename=/dev/urandom")
a = append(a, dev("virtio-rng-pci,rng=rng,bus=pcie.0,addr=%#x", slotRNG)...)
+ // Free page reporting hands memory the guest freed back to the host.
+ a = append(a, dev("virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=%#x", slotBalloon)...)
for i, d := range s.DataDisks {
- a = append(a, "-drive", fmt.Sprintf("if=none,id=data%d,file=%s,format=qcow2,cache=unsafe,discard=unmap", i, d.Path))
+ a = append(a, "-drive", fmt.Sprintf("if=none,id=data%d,file=%s,format=qcow2,%s", i, d.Path, driveOpts))
a = append(a, dev("virtio-blk-pci,drive=data%d,bus=pcie.0,addr=%#x", i, slotData+i)...)
}
a = append(a, "-qmp", "unix:"+qmpSocket(s)+",server=on,wait=off", "-pidfile", pidfile(s))
- if media != nil {
+ for _, f := range s.Files {
+ a = append(a, "-fw_cfg", "name="+f.Name+",file="+f.Path)
+ }
+
+ switch b {
+ case bootInstall:
a = append(a,
"-drive", "if=none,id=cd,media=cdrom,readonly=on,file="+media.ISO,
"-device", "ide-cd,drive=cd,bus=ide.0",
@@ -89,12 +110,21 @@ func args(s vm.Spec, media *vm.Media) []string {
"-serial", "chardev:ser0",
"-no-reboot",
)
- } else {
+ case bootOnce:
+ a = append(a,
+ "-chardev", "file,id=ser0,path="+consoleLog(s),
+ "-serial", "chardev:ser0",
+ "-no-reboot",
+ )
+ default:
a = append(a,
"-chardev", "socket,id=ser0,path="+console(s)+",server=on,wait=off,logfile="+consoleLog(s)+",logappend=on",
"-serial", "chardev:ser0",
"-daemonize",
)
+ if s.LoadSnapshot != "" {
+ a = append(a, "-loadvm", s.LoadSnapshot, "-S")
+ }
}
return a
}
diff --git a/internal/qemu/qemu_test.go b/internal/qemu/qemu_test.go
index d0ade50..ab02867 100644
--- a/internal/qemu/qemu_test.go
+++ b/internal/qemu/qemu_test.go
@@ -40,21 +40,25 @@ func golden(t *testing.T, name string, args []string) {
}
}
-func TestRunArgs(t *testing.T) { golden(t, "run.args", args(spec(), nil)) }
+func TestRunArgs(t *testing.T) { golden(t, "run.args", args(spec(), bootDaemon, nil)) }
func TestInstallArgs(t *testing.T) {
s := spec()
s.Nested = false
- golden(t, "install.args", args(s, &vm.Media{ISO: "/c/pve.iso", Kernel: "/c/linux26", Initrd: "/c/initrd.img", Cmdline: "ro quiet console=ttyS0,115200", AnswerDir: "/d/ans"}))
+ golden(t, "install.args", args(s, bootInstall, &vm.Media{ISO: "/c/pve.iso", Kernel: "/c/linux26", Initrd: "/c/initrd.img", Cmdline: "ro quiet console=ttyS0,115200", AnswerDir: "/d/ans"}))
}
// Devices shared by install and run must sit at the same PCI addresses.
func TestStablePCI(t *testing.T) {
- run := devices(args(spec(), nil))
- inst := devices(args(spec(), &vm.Media{}))
- for _, d := range run {
- if strings.Contains(d, "addr=") && !slices.Contains(inst, d) {
- t.Errorf("device %q missing or moved in install mode", d)
+ run := devices(args(spec(), bootDaemon, nil))
+ for mode, other := range map[string][]string{
+ "install": devices(args(spec(), bootInstall, &vm.Media{})),
+ "once": devices(args(spec(), bootOnce, nil)),
+ } {
+ for _, d := range run {
+ if strings.Contains(d, "addr=") && !slices.Contains(other, d) {
+ t.Errorf("device %q missing or moved in %s mode", d, mode)
+ }
}
}
}
@@ -68,3 +72,22 @@ func devices(args []string) []string {
}
return out
}
+
+func TestOnceArgs(t *testing.T) {
+ s := spec()
+ s.Files = []vm.File{{Name: "opt/proxbase/bake", Path: "/r/bake.sh"}}
+ golden(t, "once.args", args(s, bootOnce, nil))
+}
+
+// A node started from a live snapshot loads it and waits paused for Resume.
+func TestLoadSnapshotArgs(t *testing.T) {
+ s := spec()
+ s.LoadSnapshot = "ready"
+ a := strings.Join(args(s, bootDaemon, nil), " ")
+ if !strings.Contains(a, " -loadvm ready -S") {
+ t.Fatalf("missing -loadvm ready -S in %s", a)
+ }
+ if strings.Contains(strings.Join(args(spec(), bootDaemon, nil), " "), "-loadvm") {
+ t.Fatal("-loadvm without LoadSnapshot")
+ }
+}
diff --git a/internal/qemu/testdata/install.args b/internal/qemu/testdata/install.args
index aa5005a..206f941 100644
--- a/internal/qemu/testdata/install.args
+++ b/internal/qemu/testdata/install.args
@@ -15,7 +15,7 @@ none
-device
VGA,bus=pcie.0,addr=0x1
-drive
-if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap
+if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap
-device
virtio-blk-pci,drive=root,bus=pcie.0,addr=0x2,bootindex=1
-device
@@ -32,8 +32,10 @@ virtio-net-pci,netdev=net0,mac=52:54:00:50:01:01,bus=pcie.0,addr=0x5
rng-random,id=rng,filename=/dev/urandom
-device
virtio-rng-pci,rng=rng,bus=pcie.0,addr=0xe
+-device
+virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=0xf
-drive
-if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap
+if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap
-device
virtio-blk-pci,drive=data0,bus=pcie.0,addr=0x10
-qmp
diff --git a/internal/qemu/testdata/once.args b/internal/qemu/testdata/once.args
new file mode 100644
index 0000000..2f1e87e
--- /dev/null
+++ b/internal/qemu/testdata/once.args
@@ -0,0 +1,51 @@
+-name
+guest=lab-pve1,process=lab-pve1
+-nodefaults
+-no-user-config
+-machine
+q35,accel=kvm
+-cpu
+host
+-smp
+4
+-m
+4096
+-display
+none
+-device
+VGA,bus=pcie.0,addr=0x1
+-drive
+if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap
+-device
+virtio-blk-pci,drive=root,bus=pcie.0,addr=0x2,bootindex=1
+-device
+qemu-xhci,id=usb,bus=pcie.0,addr=0x3
+-netdev
+user,id=nat,dhcpstart=10.0.2.100,hostfwd=tcp:127.0.0.1:18101-10.0.2.15:22,hostfwd=tcp:127.0.0.1:18001-10.0.2.15:8006
+-device
+virtio-net-pci,netdev=nat,mac=52:54:00:50:01:00,bus=pcie.0,addr=0x4
+-netdev
+dgram,id=net0,local.type=unix,local.path=/r/pve1-cluster.sock,remote.type=unix,remote.path=/r/sw-cluster.sock
+-device
+virtio-net-pci,netdev=net0,mac=52:54:00:50:01:01,bus=pcie.0,addr=0x5
+-object
+rng-random,id=rng,filename=/dev/urandom
+-device
+virtio-rng-pci,rng=rng,bus=pcie.0,addr=0xe
+-device
+virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=0xf
+-drive
+if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap
+-device
+virtio-blk-pci,drive=data0,bus=pcie.0,addr=0x10
+-qmp
+unix:/r/pve1.qmp,server=on,wait=off
+-pidfile
+/r/pve1.pid
+-fw_cfg
+name=opt/proxbase/bake,file=/r/bake.sh
+-chardev
+file,id=ser0,path=/l/pve1-console.log
+-serial
+chardev:ser0
+-no-reboot
diff --git a/internal/qemu/testdata/run.args b/internal/qemu/testdata/run.args
index 2bfd1e7..565c984 100644
--- a/internal/qemu/testdata/run.args
+++ b/internal/qemu/testdata/run.args
@@ -15,7 +15,7 @@ none
-device
VGA,bus=pcie.0,addr=0x1
-drive
-if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap
+if=none,id=root,file=/d/pve1-root.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap
-device
virtio-blk-pci,drive=root,bus=pcie.0,addr=0x2,bootindex=1
-device
@@ -32,8 +32,10 @@ virtio-net-pci,netdev=net0,mac=52:54:00:50:01:01,bus=pcie.0,addr=0x5
rng-random,id=rng,filename=/dev/urandom
-device
virtio-rng-pci,rng=rng,bus=pcie.0,addr=0xe
+-device
+virtio-balloon-pci,id=balloon,free-page-reporting=on,bus=pcie.0,addr=0xf
-drive
-if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap
+if=none,id=data0,file=/d/pve1-data0.qcow2,format=qcow2,cache=unsafe,discard=unmap,detect-zeroes=unmap
-device
virtio-blk-pci,drive=data0,bus=pcie.0,addr=0x10
-qmp
diff --git a/internal/remote/ssh.go b/internal/remote/ssh.go
index 594032e..66a320b 100644
--- a/internal/remote/ssh.go
+++ b/internal/remote/ssh.go
@@ -14,6 +14,9 @@ import (
"golang.org/x/crypto/ssh"
)
+// HandshakeTimeout bounds the SSH handshake of one Dial.
+const HandshakeTimeout = 3 * time.Second
+
// SSH runs commands as root on a node through its forwarded port on 127.0.0.1.
type SSH struct {
Port int
@@ -43,11 +46,27 @@ func (s *SSH) Dial(ctx context.Context) (string, error) {
if err != nil {
return "", err
}
+ // QEMU's port forward accepts at once even while the guest is still booting,
+ // then retries its own connection to the guest only every 12s. Without a
+ // deadline the handshake waits for that retry instead of failing fast so
+ // the caller can dial again.
+ deadline := time.Now().Add(HandshakeTimeout)
+ if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) {
+ deadline = ctxDeadline
+ }
+ if err := conn.SetDeadline(deadline); err != nil {
+ conn.Close()
+ return "", err
+ }
c, chans, reqs, err := ssh.NewClientConn(conn, addr, cfg)
if err != nil {
conn.Close()
return "", err
}
+ if err := conn.SetDeadline(time.Time{}); err != nil {
+ c.Close()
+ return "", err
+ }
s.client = ssh.NewClient(c, chans, reqs)
return seen, nil
}
diff --git a/internal/remote/ssh_test.go b/internal/remote/ssh_test.go
new file mode 100644
index 0000000..7188b63
--- /dev/null
+++ b/internal/remote/ssh_test.go
@@ -0,0 +1,57 @@
+package remote
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "net"
+ "testing"
+ "time"
+
+ "golang.org/x/crypto/ssh"
+)
+
+// A port forward that accepts but never answers (a booting guest behind QEMU
+// user networking) must not block Dial past its deadline.
+func TestSSH_DialStalledHandshake(t *testing.T) {
+ t.Parallel()
+ l, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ go func() {
+ // Hold connections open without sending a banner until the listener closes.
+ var conns []net.Conn
+ defer func() {
+ for _, c := range conns {
+ c.Close()
+ }
+ }()
+ for {
+ c, err := l.Accept()
+ if err != nil {
+ return
+ }
+ conns = append(conns, c)
+ }
+ }()
+ _, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ signer, err := ssh.NewSignerFromKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s := &SSH{Port: l.Addr().(*net.TCPAddr).Port, Signer: signer}
+ ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond)
+ defer cancel()
+ t0 := time.Now()
+ if _, err := s.Dial(ctx); err == nil {
+ t.Fatal("Dial succeeded against a server that never answers")
+ }
+ if d := time.Since(t0); d > 2*time.Second {
+ t.Fatalf("Dial took %s; the handshake must honour the context deadline", d)
+ }
+}
diff --git a/internal/state/state.go b/internal/state/state.go
index c5c5dea..c29cfb5 100644
--- a/internal/state/state.go
+++ b/internal/state/state.go
@@ -47,6 +47,9 @@ type Snapshot struct {
Name string `json:"name"`
Created time.Time `json:"created"`
Nodes []string `json:"nodes"`
+ // Live snapshots were taken of the running cluster and include its memory;
+ // restoring one starts the cluster in that state.
+ Live bool `json:"live,omitempty"`
}
func (s *State) Snapshot(name string) *Snapshot {
diff --git a/internal/storage/ceph/ceph.go b/internal/storage/ceph/ceph.go
index f87f157..773cbe9 100644
--- a/internal/storage/ceph/ceph.go
+++ b/internal/storage/ceph/ceph.go
@@ -8,6 +8,7 @@ import (
"fmt"
"net/url"
"strconv"
+ "strings"
"time"
"golang.org/x/sync/errgroup"
@@ -57,13 +58,9 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error {
if err := b.install(ctx, h); err != nil {
return err
}
- apis := map[string]*pve.Client{}
- for _, n := range nodes {
- api, err := h.API(ctx, n.Name)
- if err != nil {
- return err
- }
- apis[n.Name] = api
+ apis, err := clients(ctx, h)
+ if err != nil {
+ return err
}
first := nodes[0].Name
api := apis[first]
@@ -71,7 +68,7 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error {
return err
}
- // Monitors (up to three), a manager on every node; one at a time.
+ // Monitors (up to three) one at a time: each joins the quorum before the next.
var mons []named
if err := retry.Do(ctx, 2*time.Minute, 2*time.Second, func() error { return api.Get(ctx, "/nodes/"+first+"/ceph/mon", &mons) }); err != nil {
return fmt.Errorf("list ceph mon: %w", err)
@@ -83,14 +80,20 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error {
}
mons = append(mons, named{Name: n.Name})
}
- if err := daemon(ctx, h, apis[n.Name], api, n.Name, "mgr"); err != nil {
- return err
- }
+ }
+ if err := setMemoryTargets(ctx, h, first); err != nil {
+ return err
}
+ // Managers and OSDs on all nodes at once; neither edits shared configuration.
g, gctx := errgroup.WithContext(ctx)
for _, n := range nodes {
- g.Go(func() error { return createOSDs(gctx, h, apis[n.Name], n.Name, b.cfg.OSDDisks) })
+ g.Go(func() error {
+ if err := daemon(gctx, h, apis[n.Name], api, n.Name, "mgr"); err != nil {
+ return err
+ }
+ return createOSDs(gctx, h, apis[n.Name], n.Name, b.cfg.OSDDisks)
+ })
}
if err := g.Wait(); err != nil {
return err
@@ -99,6 +102,7 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error {
return err
}
if *b.cfg.CephFS {
+ // One at a time: creating an MDS rewrites ceph.conf without a lock.
for _, n := range nodes {
if err := daemon(ctx, h, apis[n.Name], api, n.Name, "mds"); err != nil {
return err
@@ -111,6 +115,60 @@ func (b *Backend) Setup(ctx context.Context, h storage.Host) error {
return b.Wait(ctx, h)
}
+// clients logs in to every node's API in parallel.
+func clients(ctx context.Context, h storage.Host) (map[string]*pve.Client, error) {
+ nodes := h.Nodes()
+ list := make([]*pve.Client, len(nodes))
+ g, gctx := errgroup.WithContext(ctx)
+ for i, n := range nodes {
+ g.Go(func() error {
+ api, err := h.API(gctx, n.Name)
+ list[i] = api
+ return err
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return nil, err
+ }
+ apis := make(map[string]*pve.Client, len(nodes))
+ for i, n := range nodes {
+ apis[n.Name] = list[i]
+ }
+ return apis, nil
+}
+
+// Lab memory targets. Ceph's defaults (4 GiB per OSD, 4 GiB MDS cache) are sized
+// for dedicated servers; the OSD and MDS grow their caches toward them, which
+// would push a node with a few GiB of RAM into the OOM killer. 2 GiB is about
+// the lowest OSD target Ceph developers recommend, 1 GiB the minimum MDS cache.
+var memoryTargets = []struct{ who, option, value string }{
+ {who: "osd", option: "osd_memory_target", value: "2147483648"},
+ {who: "mds", option: "mds_cache_memory_limit", value: "1073741824"},
+}
+
+// setMemoryTargets applies memoryTargets unless the option was set before, so
+// values changed by the user survive a later setup (node add).
+func setMemoryTargets(ctx context.Context, h storage.Host, node string) error {
+ s, err := h.SSH(ctx, node)
+ if err != nil {
+ return err
+ }
+ defer s.Close()
+ if _, err := s.Run(memoryTargetScript()); err != nil {
+ return fmt.Errorf("ceph: set memory targets: %w", err)
+ }
+ return nil
+}
+
+func memoryTargetScript() string {
+ var script strings.Builder
+ script.WriteString("dump=$(ceph config dump --format json)\n")
+ for _, t := range memoryTargets {
+ fmt.Fprintf(&script, "echo \"$dump\" | grep -q '\"name\": *\"%[2]s\"' || ceph config set %[1]s %[2]s %[3]s\n", t.who, t.option, t.value)
+ }
+ return script.String()
+}
+
// init writes ceph.conf with the public and cluster networks unless it exists.
func (b *Backend) init(ctx context.Context, h storage.Host, api *pve.Client, first string) error {
s, err := h.SSH(ctx, first)
diff --git a/internal/storage/ceph/ceph_test.go b/internal/storage/ceph/ceph_test.go
new file mode 100644
index 0000000..3e2716d
--- /dev/null
+++ b/internal/storage/ceph/ceph_test.go
@@ -0,0 +1,65 @@
+package ceph
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// The script sets each memory target once and keeps values the user set.
+func TestMemoryTargetScript(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ name string
+ dump string
+ want string
+ }{
+ {
+ name: "fresh cluster",
+ dump: `[]`,
+ want: "config set osd osd_memory_target 2147483648\nconfig set mds mds_cache_memory_limit 1073741824\n",
+ },
+ {
+ name: "osd target set by the user",
+ dump: `[{"section": "osd", "name": "osd_memory_target", "value": "8589934592"}]`,
+ want: "config set mds mds_cache_memory_limit 1073741824\n",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ dir := t.TempDir()
+ log := filepath.Join(dir, "calls")
+ fake := "#!/bin/sh\n" +
+ "if [ \"$1 $2\" = 'config dump' ]; then echo '" + tt.dump + "'; exit 0; fi\n" +
+ "echo \"$*\" >> " + log + "\n"
+ if err := os.WriteFile(filepath.Join(dir, "ceph"), []byte(fake), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ cmd := exec.CommandContext(t.Context(), "bash", "-euo", "pipefail", "-s")
+ cmd.Stdin = strings.NewReader(memoryTargetScript())
+ cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"))
+ if out, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("script failed: %v\n%s", err, out)
+ }
+ got, err := os.ReadFile(log)
+ if err != nil && !os.IsNotExist(err) {
+ t.Fatal(err)
+ }
+ if string(got) != tt.want {
+ t.Errorf("ceph calls:\n%s\nwant:\n%s", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestInstallScriptSyntax(t *testing.T) {
+ t.Parallel()
+ cmd := exec.CommandContext(t.Context(), "bash", "-n")
+ cmd.Stdin = strings.NewReader(installScript("squid"))
+ if out, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("bash -n: %v\n%s", err, out)
+ }
+}
diff --git a/internal/storage/ceph/install.go b/internal/storage/ceph/install.go
index 37b132f..76c6f35 100644
--- a/internal/storage/ceph/install.go
+++ b/internal/storage/ceph/install.go
@@ -19,14 +19,21 @@ func installScript(version string) string {
return nodesetup.AptWait + fmt.Sprintf(`
export DEBIAN_FRONTEND=noninteractive
log=/var/log/proxbase-ceph-install.log
+pveceph_install() { { yes || true; } | pveceph install --repository no-subscription --version %[1]s >>$log 2>&1; }
if ! [ -x /usr/bin/ceph-mon ] || ! ceph-mon --version 2>/dev/null | grep -q ' %[1]s '; then
- if ! { yes || true; } | pveceph install --repository no-subscription --version %[1]s >$log 2>&1; then
- tail -n 20 $log >&2; exit 1
+ if ! pveceph_install; then
+ # pveceph takes no apt options: retry past caches through apt's config.
+ nocache=/etc/apt/apt.conf.d/99proxbase-no-cache
+ echo 'Acquire::http::No-Cache "true";' > $nocache
+ if ! pveceph_install; then
+ rm -f $nocache; tail -n 20 $log >&2; exit 1
+ fi
+ rm -f $nocache
fi
echo installed
fi
before=$(dpkg-query -W -f '${Version}' libpve-storage-perl)
-if ! $apt install -y --only-upgrade libpve-storage-perl >>$log 2>&1; then
+if ! retry_apt $apt install -y --only-upgrade libpve-storage-perl >>$log 2>&1; then
tail -n 20 $log >&2; exit 1
fi
if [ "$before" != "$(dpkg-query -W -f '${Version}' libpve-storage-perl)" ]; then
@@ -38,7 +45,7 @@ fi
// install runs the install script on all nodes in parallel.
func (b *Backend) install(ctx context.Context, h storage.Host) error {
- script := installScript(b.cfg.Version)
+ script := h.AptEnv() + installScript(b.cfg.Version)
g, gctx := errgroup.WithContext(ctx)
for _, n := range h.Nodes() {
g.Go(func() error {
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 6a7b4b4..00d2df6 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -21,6 +21,9 @@ type Host interface {
SSH(ctx context.Context, node string) (*remote.SSH, error)
// Step reports progress.
Step(format string, a ...any)
+ // AptEnv is a shell snippet to put before scripts that run apt on the nodes:
+ // it points apt at the host's package cache, if there is one.
+ AptEnv() string
}
// Backend sets up one kind of storage. Setup must be idempotent: it runs on
diff --git a/internal/storage/zfs/zfs.go b/internal/storage/zfs/zfs.go
index 7e151a5..ebc066b 100644
--- a/internal/storage/zfs/zfs.go
+++ b/internal/storage/zfs/zfs.go
@@ -10,6 +10,8 @@ import (
"strings"
"time"
+ "golang.org/x/sync/errgroup"
+
"github.com/virtbase/proxbase/internal/config"
"github.com/virtbase/proxbase/internal/storage"
)
@@ -32,16 +34,18 @@ func New(cfg *config.Cluster) *Backend {
return &Backend{Pools: cfg.Storage.ZFS}
}
-// Setup creates missing pools on every node and the cluster storage entries,
-// keeping their node lists current.
+// Setup creates missing pools on every node (in parallel) and the cluster
+// storage entries, keeping their node lists current.
func (b *Backend) Setup(ctx context.Context, h storage.Host) error {
nodes := h.Nodes()
- var names []string
+ names := make([]string, 0, len(nodes))
+ g, gctx := errgroup.WithContext(ctx)
for _, n := range nodes {
names = append(names, n.Name)
- if err := b.createPools(ctx, h, n.Name); err != nil {
- return err
- }
+ g.Go(func() error { return b.createPools(gctx, h, n.Name) })
+ }
+ if err := g.Wait(); err != nil {
+ return err
}
return b.registerStorage(ctx, h, nodes[0].Name, names, true)
}
@@ -75,6 +79,23 @@ func (b *Backend) createPools(ctx context.Context, h storage.Host, node string)
return fmt.Errorf("%s: create zfs pool %s: %w", node, z.Name, err)
}
}
+ return b.enableAutotrim(ctx, h, node)
+}
+
+// enableAutotrim lets the pools return freed blocks, so the disk images shrink.
+func (b *Backend) enableAutotrim(ctx context.Context, h storage.Host, node string) error {
+ s, err := h.SSH(ctx, node)
+ if err != nil {
+ return err
+ }
+ defer s.Close()
+ var script strings.Builder
+ for _, z := range b.Pools {
+ fmt.Fprintf(&script, "zpool set autotrim=on %s\n", z.Name)
+ }
+ if _, err := s.Run(script.String()); err != nil {
+ return fmt.Errorf("%s: enable autotrim: %w", node, err)
+ }
return nil
}
diff --git a/internal/vm/vm.go b/internal/vm/vm.go
index 04969f6..38fcdb2 100644
--- a/internal/vm/vm.go
+++ b/internal/vm/vm.go
@@ -37,6 +37,18 @@ type Spec struct {
NICs []NIC
RunDir string // pidfile, control and console sockets
LogDir string // console, installer and runtime logs
+ // Files are handed to the guest at boot (QEMU fw_cfg); the guest reads them
+ // below /sys/firmware/qemu_fw_cfg/by_name/.
+ Files []File
+ // LoadSnapshot starts the node paused in the state of this live snapshot
+ // (see LiveSnapshotter); Resume lets it run.
+ LoadSnapshot string
+}
+
+// File is a host file passed to the guest under Name (e.g. "opt/proxbase/identity").
+type File struct {
+ Name string
+ Path string
}
// Disks returns the root disk followed by the data disks.
@@ -56,6 +68,12 @@ type Installer interface {
Install(ctx context.Context, s Spec, m Media) error
}
+// Booter boots an installed node once in the foreground and returns its serial
+// console output when it powers off, or fails after timeout.
+type Booter interface {
+ BootOnce(ctx context.Context, s Spec, timeout time.Duration) (console []byte, err error)
+}
+
// Cloner creates a node's disks from an installed base image instead of installing.
type Cloner interface {
Clone(ctx context.Context, s Spec, base string) error
@@ -83,6 +101,19 @@ type Snapshotter interface {
Snapshot(ctx context.Context, op SnapshotOp, name string, disks []Disk) error
}
+// LiveSnapshotter snapshots running nodes with their memory, so they can be
+// started again in that state (Spec.LoadSnapshot). Snapshotter deletes them.
+type LiveSnapshotter interface {
+ // SaveLive snapshots every disk and the memory of a running node; a paused
+ // node stays paused.
+ SaveLive(ctx context.Context, s Spec, name string) error
+ // DeleteLive deletes a snapshot from the disks of a running node.
+ DeleteLive(ctx context.Context, s Spec, name string) error
+ // Pause stops all vCPUs; Resume lets them run again.
+ Pause(ctx context.Context, s Spec) error
+ Resume(ctx context.Context, s Spec) error
+}
+
// Runtime is a complete node runtime.
type Runtime interface {
Installer
diff --git a/schema/v1alpha1/cluster.json b/schema/v1alpha1/cluster.json
index 134037e..06cf624 100644
--- a/schema/v1alpha1/cluster.json
+++ b/schema/v1alpha1/cluster.json
@@ -279,6 +279,10 @@
"goldenImage": {
"type": "boolean",
"description": "Clone nodes from a cached installed base image instead of running the installer"
+ },
+ "aptCache": {
+ "type": "boolean",
+ "description": "Download packages that proxbase installs (upgrade and Ceph) once through a cache on the host; default true"
}
},
"additionalProperties": false,