diff --git a/docs/api.md b/docs/api.md index 9c4a6e73..6d6d95f1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -87,7 +87,7 @@ plugin name. See [Resource plugins](resource-plugins.md). | `ExecuteWorkload` ⇅ | `workload_id`, `commands`, `envs`, `workdir`, `open_stdin` | Exec inside a workload. When `open_stdin` is set, further client messages carry stdin in `repl_cmd` | | `RunAndWait` ⇅ | `deploy_options`, `cmd`, `async`, `async_timeout` | Lambda: deploy, attach, wait for exit, then remove. The first messages carry the new workload IDs (`TYPEWORKLOADID`), the last output line is `[exitcode] `. With `async`, core sends the IDs, detaches from the stream, forces `open_stdin` off, and logs the output itself under `async_timeout` seconds (default `global_timeout`) | | `LogStream` ⇊ | `id`, `tail`, `since`, `until`, `follow` | Engine logs for one workload | -| `RawEngine` | `id`, `op`, `params`, `ignore_lock` | Pass an engine-specific operation through to the node's engine. No real engine implements it; containerd, cocoon and process all return `ErrEngineNotImplemented` (the `mock://` engine answers with a canned result) | +| `RawEngine` | `id`, `op`, `params`, `ignore_lock` | Pass an engine-specific operation through to the node's engine. cocoon serves its snapshot ops (see [engines](engines.md#snapshots)); containerd and process return `ErrEngineNotImplemented` (the `mock://` engine answers with a canned result) | ## Files diff --git a/docs/engines.md b/docs/engines.md index d9666439..42fec163 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -254,7 +254,7 @@ the cocoon daemon's events on it. The eru name stays in the meta file and in cor | `engine.API` | cocoon | | --- | --- | -| `VirtualizationCreate` | `vm create --output json --name [--cpu N] [--memory B] [--storage B] [--data-disk …] [--network ] [--windows \| --user U] ` — no boot; then the meta record is written. A failure after the create removes the VM again. cocoon has no way to apply the deploy's `env`, `dns`, `extra_hosts` or the entrypoint's `commands` and `dir` to a guest, so any of them set draws a warning; core's own `APP_NAME`/`ERU_*` env does not | +| `VirtualizationCreate` | `vm create --output json --name [--cpu N] [--memory B] [--storage B] [--data-disk …] [--network ] [--windows \| --user U] ` — no boot; then the meta record is written. A failure after the create removes the VM again. cocoon has no way to apply the deploy's `env`, `dns`, `extra_hosts` or the entrypoint's `commands` and `dir` to a guest, so any of them set draws a warning; core's own `APP_NAME`/`ERU_*` env does not. A `snapshot://` image is a clone instead (see [Snapshots](#snapshots)) | | `VirtualizationStart` | `vm inspect`, `vm start` and `vm inspect` again in one script; the second inspect reports this boot's `console_path`, and both copies of the meta record are rewritten with it and with the VMM pid. An inspect that reports no console keeps the serial socket path. A Windows guest on its first boot gets its address programmed through `vm exec` in the background, after the start has already returned | | `VirtualizationStop` | `vm stop`, `--force` for a forced stop, `--timeout` when a grace period is given; a workload with no record on the node is `ErrWorkloadNotExists`, as for the other verbs. cocoon's stop is idempotent, so stopping a created or already-stopped guest succeeds | | `VirtualizationRemove` | `vm rm [--force]`, then the hibernate snapshot and both copies of the meta record; a running guest is refused unless forced | @@ -266,12 +266,13 @@ the cocoon daemon's events on it. The eru name stays in the meta file and in cor | `Execute` / `ExecExitCode` | `vm exec [-i] [-e K=V …] -- ` through cocoon-agent in pipe mode, stdio on the SSH session, the exit code the guest command's. `ExecResize` is `ErrEngineNotImplemented` (core#660). A `user` and a `working_dir` are applied inside a Linux guest by wrapping the command — `runuser -u U -- env --chdir=D ` for a bare user name, `setpriv --reuid=U --regid=G --clear-groups -- env --chdir=D ` when the id is numeric or a group is named — so the directory is entered as the target user; on a Windows guest both are `ErrEngineNotImplemented` | | `VirtualizationCopyTo` / `CopyFrom` | a one-entry tar through `vm exec … tar -x -P -f -` / `tar -c -P -f -`: the absolute entry name makes tar create the parents, and `tar.exe` ships with Windows 10+. A copy into a guest that is not running is `ErrInvaildWorkloadOps` — the state is checked first, one round trip per file | | `VirtualizationUpdateResource` | a remap (the cpumem binding refresh core runs after every deploy) is a no-op without a round trip; a realloc is `ErrEngineNotImplemented`, CPU and memory hot-plug wait on cocoon (core#661) | -| `ImagePull` | `image pull ` for OCI VM images and cloud-image URLs, registry auth left to cocoon's own config; a split-qcow2 artifact (the Windows images) is `oras pull`ed and `image import`ed under the same ref, once | +| `ImagePull` | `image pull ` for OCI VM images and cloud-image URLs, registry auth left to cocoon's own config; a split-qcow2 artifact (the Windows images) is `oras pull`ed and `image import`ed under the same ref, once. A `snapshot://` ref is only looked up with `snapshot inspect`, so a missing snapshot fails the pull | | `ImageList` / `ImageRemove` | `image list --format json` filtered by name prefix / `image rm`. An empty store answers `No images found.` in prose rather than `[]`, and reads as an empty list, not a failed node | -| `ImageLocalDigests` / `ImageRemoteDigest` | `image inspect` / `oras manifest fetch --descriptor`; a cloud-image URL is its own digest, so it is pulled once. A node without `oras` (probed with `command -v`) reports no remote digest, so every deploy runs `image pull`, which cocoon answers from its cache. Only a node that answered yes is remembered — a probe an ssh failure lost is asked again, instead of pinning the node as oras-less for the engine's life | +| `ImageLocalDigests` / `ImageRemoteDigest` | `image inspect` / `oras manifest fetch --descriptor`; a cloud-image URL is its own digest, so it is pulled once, and so is a `snapshot://` ref while `snapshot inspect` finds the snapshot. A node without `oras` (probed with `command -v`) reports no remote digest, so every deploy runs `image pull`, which cocoon answers from its cache. Only a node that answered yes is remembered — a probe an ssh failure lost is asked again, instead of pinning the node as oras-less for the engine's life | | `ImageBuildFromExist` | `ErrEngineNotImplemented`, and so is `ImagePush`: cocoon has no registry push, so a build from an existing workload can never finish. Saving a snapshot first only left node state behind — core's build always goes on to push the refs and then runs `ImageRemove` over them — so the engine refuses before anything is written | | `NetworkList` | the CNI conf dir (`/etc/cni/net.d`); `NetworkConnect` / `Disconnect` are `ErrEngineNotImplemented` | -| `ImageBuild`, `ImagesPrune`, `RawEngine` | `ErrEngineNotImplemented` | +| `RawEngine` | the snapshot ops, see [Snapshots](#snapshots) | +| `ImageBuild`, `ImagesPrune` | `ErrEngineNotImplemented` | ### Resources and networks @@ -335,6 +336,40 @@ digest, so the two never match and every deploy runs `ImagePull` again. That is the import script exits at once when `image inspect ` already answers — but it does mean a Windows deploy always pays one extra round trip. +### Snapshots + +A deploy whose image is `snapshot://` clones the VM from that cocoon snapshot on the node +instead of booting an image: + +| `image` | cocoon | +| --- | --- | +| `snapshot://` | `vm clone --output json --name [--network ] [--data-disk …] ` — the snapshot name or id, checked against cocoon's grammar (`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`) before any round trip | + +The clone takes its CPU, memory, storage, guest OS and login from the snapshot, so the deploy's +cpumem and storage quotas are not applied (logged at debug) and `user` reaches only the meta record, +where exec reads it. No network in the deploy keeps the snapshot's conflist. The guest resumes with the source VM's NIC files and hostname, so after the +record the engine rewrites them through `vm exec`: one systemd-networkd file per static NIC, matched +by the clone's new MAC, the workload name as hostname, then `systemctl restart systemd-networkd`, +retried for 30 s until cocoon-agent answers. A clone that will not take its address is removed. The meta record is +written from the clone's JSON exactly as after a create, and a failure after the clone removes the +VM again. The clone is already running when `VirtualizationStart` runs: cocoon answers `vm start` +on a running VM with success, so the start only refreshes the record. The volumes become data disks +hot-added to the running guest; cloud-init has already run, so nothing mounts them, and cocoon +refuses to snapshot or hibernate a VM with a hot-added disk. A Windows deploy (`os: windows`) with a +snapshot image is refused with `ErrInvalidEngineArgs`. The image verbs never reach a registry for +such a ref. + +`RawEngine` serves the snapshot ops on the workload's node, `Params` a JSON object, the name checked +as above and refused with `ErrInvalidEngineArgs` before any round trip; any other op is +`ErrEngineNotImplemented`: + +| `Op` | `Params` | cocoon | `Data` | +| --- | --- | --- | --- | +| `snapshot.save` | `{"name": N}` | `snapshot save --name N `, then `snapshot inspect N`, in one round trip; the guest keeps running | the snapshot JSON | +| `snapshot.list` | — | `snapshot list --format json`; the prose empty banner reads as `[]` | the JSON list | +| `snapshot.inspect` | `{"name": N}` | `snapshot inspect N` | the snapshot JSON | +| `snapshot.remove` | `{"name": N}` | `snapshot rm N` | empty | + ### The meta file The same record the process engine writes, at `/.json` and diff --git a/engine/cocoon/cocoon.go b/engine/cocoon/cocoon.go index 48c0cfc4..77875c36 100644 --- a/engine/cocoon/cocoon.go +++ b/engine/cocoon/cocoon.go @@ -92,10 +92,6 @@ func (e *Engine) GetParams() *enginetypes.Params { return e.ep } -func (e *Engine) RawEngine(context.Context, *enginetypes.RawEngineOptions) (*enginetypes.RawEngineResult, error) { - return nil, coretypes.ErrEngineNotImplemented -} - func (e *Engine) call(ctx context.Context, argv ...string) (*sshrunner.Result, error) { return sshrunner.Call(ctx, e.runner, argv...) } diff --git a/engine/cocoon/create.go b/engine/cocoon/create.go index d0d2402a..2ff632c9 100644 --- a/engine/cocoon/create.go +++ b/engine/cocoon/create.go @@ -29,6 +29,28 @@ const ( discardTimeout = 30 * time.Second + // reseedScript rewrites a clone's static NICs by MAC and its hostname; the guest still carries the source's. + reseedScript = `bin=$1; vm=$2; shift 2 +guest='rm -f /etc/systemd/network/10-*.network +hostnamectl set-hostname "$1" 2>/dev/null || hostname "$1" +sed -i "/^127\.0\.1\.1[[:space:]]/d" /etc/hosts +printf "127.0.1.1 %s\n" "$(hostname)" >> /etc/hosts +shift +while [ $# -ge 3 ]; do +f="/etc/systemd/network/10-$(printf %s "$1" | tr -d :).network" +printf "[Match]\nMACAddress=%s\n\n[Network]\nAddress=%s\n" "$1" "$2" > "$f" +if [ -n "$3" ]; then printf "Gateway=%s\n" "$3" >> "$f"; fi +shift 3 +done +systemctl restart systemd-networkd' +tries=0 +until "$bin" vm exec "$vm" -- sh -c "$guest" sh "$@"; do +tries=$((tries+1)) +[ "$tries" -lt 30 ] || exit 1 +sleep 1 +done +` + publishRecord = `mkdir -p "$(dirname "$record")" cp -f "$durable" "$record.tmp" mv "$record.tmp" "$record" @@ -67,7 +89,13 @@ func (e *Engine) VirtualizationCreate(ctx context.Context, opts *enginetypes.Vir return nil, err } ID := utils.RandomID() - argv, err := createArgv(e.cocoon.Binary, ID, opts, resource, rArgs.OS == osWindows, network) + var argv []string + if snapshot, ok := strings.CutPrefix(opts.Image, snapshotScheme); ok { + logger.Debugf(ctx, "vm %s takes its cpu, memory and storage from snapshot %s", opts.Name, snapshot) + argv, err = cloneArgv(e.cocoon.Binary, ID, snapshot, resource.Volumes, rArgs.OS == osWindows, network) + } else { + argv, err = createArgv(e.cocoon.Binary, ID, opts, resource, rArgs.OS == osWindows, network) + } if err != nil { return nil, err } @@ -83,6 +111,9 @@ func (e *Engine) VirtualizationCreate(ctx context.Context, opts *enginetypes.Vir if err == nil { err = e.record(ctx, ID, opts, vm) } + if err == nil && strings.HasPrefix(opts.Image, snapshotScheme) { + _, err = e.run(ctx, reseedArgv(e.cocoon.Binary, ID, opts.Name, vm)...) + } if err != nil { e.discard(ctx, ID) return nil, err @@ -139,7 +170,39 @@ func createArgv(binary, ID string, opts *enginetypes.VirtualizationCreateOptions return append(argv, "--name", ID, opts.Image), nil } +// cloneArgv boots the vm from a snapshot, which fixes its cpu, memory, storage and guest os. +func cloneArgv(binary, ID, snapshot string, volumes []string, windows bool, network string) ([]string, error) { + if windows { + return nil, errors.Wrap(coretypes.ErrInvalidEngineArgs, "a windows guest cannot be cloned from a snapshot") + } + if err := checkSnapshotName(snapshot); err != nil { + return nil, err + } + argv := []string{binary, "vm", "clone", "--output", formatJSON, "--name", ID} + if network != "" { + argv = append(argv, "--network", network) + } + disks, err := dataDisks(volumes, false) + if err != nil { + return nil, err + } + for _, disk := range disks { + argv = append(argv, "--data-disk", disk) + } + return append(argv, snapshot), nil +} + // dataDisks turns the storage plugin's `src:dst:mode:size` volumes into cocoon data disks. +func reseedArgv(binary, ID, hostname string, vm *vmRecord) []string { + args := []string{binary, ID, hostname} + for _, n := range vm.NICs { + if n.MAC != "" && n.Network != nil && n.Network.IP != "" { + args = append(args, n.MAC, n.Network.IP+"/"+strconv.Itoa(n.Network.Prefix), n.Network.Gateway) + } + } + return sshrunner.Shell(reseedScript, args...) +} + func dataDisks(volumes []string, windows bool) ([]string, error) { disks := make([]string, 0, len(volumes)) for _, volume := range volumes { diff --git a/engine/cocoon/create_test.go b/engine/cocoon/create_test.go index 4cc9f235..e877a4cf 100644 --- a/engine/cocoon/create_test.go +++ b/engine/cocoon/create_test.go @@ -121,6 +121,136 @@ func TestVirtualizationCreateKeepsTheConflistOfAnInheritedNetwork(t *testing.T) } } +func TestVirtualizationCreateClonesASnapshotImage(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: clonedFrom} + e := testEngine(t, runner) + + created, err := e.VirtualizationCreate(t.Context(), &enginetypes.VirtualizationCreateOptions{ + Name: "app_web_xyz", + Image: snapshotScheme + testSnap, + User: testUser, + Networks: map[string]string{"eru-cni": ""}, + EngineParams: resourcetypes.Resources{"cpumem": {"cpu": 1.5, "memory": 1 << 30}, "storage": {"storage": 20 << 30, "volumes": []string{"/data:/data:rw:1073741824"}}}, + }) + if err != nil { + t.Fatalf("clone: %v", err) + } + lines := runner.Lines() + if len(lines) != 3 { + t.Fatalf("got %d commands, want the clone, the record and the reseed", len(lines)) + } + if reseed := sshrunner.Quote(reseedArgv(testBinary, created.ID, "app_web_xyz", mustParseVM(t, clonedVM))); lines[2] != reseed { + t.Errorf("got %q, want the reseed %q", lines[2], reseed) + } + for _, arg := range []string{"'app_web_xyz'", "'02:00:00:00:00:07'", "'10.22.0.7/16'", "'10.22.0.1'"} { + if !strings.Contains(lines[2], arg) { + t.Errorf("the reseed does not carry %s", arg) + } + } + want := sshrunner.Quote([]string{ + testBinary, "vm", "clone", "--output", "json", "--name", created.ID, + "--network", "eru-cni", "--data-disk", "size=1073741824,mount=/data", testSnap, + }) + if lines[0] != want { + t.Errorf("got %q, want %q", lines[0], want) + } + for _, field := range []string{ + durablePath(testRoot, created.ID), + `"user":"` + testUser + `"`, + `"networks":{"eru-cni":"10.22.0.7"}`, + `"cgroup":"/sys/fs/cgroup/cocoon.slice/vm-` + testVMID + `.scope"`, + `"iface":"tap01ARZ3ND-0"`, + } { + if !strings.Contains(lines[1], field) { + t.Errorf("the record command does not carry %s", field) + } + } +} + +func TestVirtualizationCreateDiscardsACloneWhoseRecordFailed(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: func(line string) *sshrunner.Result { + switch { + case strings.Contains(line, "'clone'"): + return &sshrunner.Result{Stdout: clonedVM} + case strings.Contains(line, "'rm'"): + return &sshrunner.Result{} + } + return &sshrunner.Result{Code: 1, Stderr: "read-only file system"} + }} + e := testEngine(t, runner) + + if _, err := e.VirtualizationCreate(t.Context(), &enginetypes.VirtualizationCreateOptions{Name: "app_web_xyz", Image: snapshotScheme + testSnap}); err == nil { + t.Fatal("a failed record must fail the clone") + } + lines := runner.Lines() + if len(lines) != 3 || !strings.Contains(lines[2], "'rm' '--force'") { + t.Errorf("got %q, want a forced rm after the failed record", lines) + } +} + +func TestVirtualizationCreateDiscardsACloneThatWouldNotReseed(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: func(line string) *sshrunner.Result { + switch { + case strings.Contains(line, "'clone'"): + return &sshrunner.Result{Stdout: clonedVM} + case strings.Contains(line, "systemd-networkd"): + return &sshrunner.Result{Code: 1, Stderr: "agent did not answer"} + } + return &sshrunner.Result{} + }} + e := testEngine(t, runner) + + if _, err := e.VirtualizationCreate(t.Context(), &enginetypes.VirtualizationCreateOptions{Name: "app_web_xyz", Image: snapshotScheme + testSnap}); err == nil { + t.Fatal("a clone whose network was not rewritten must fail") + } + lines := runner.Lines() + if len(lines) != 4 || !strings.Contains(lines[3], "'rm' '--force'") { + t.Errorf("got %q, want a forced rm after the failed reseed", lines) + } +} + +func TestVirtualizationCreateReportsAMissingSnapshot(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: func(string) *sshrunner.Result { + return &sshrunner.Result{Code: 1, Stderr: "inspect snapshot " + testSnap + ": not found"} + }} + e := testEngine(t, runner) + + _, err := e.VirtualizationCreate(t.Context(), &enginetypes.VirtualizationCreateOptions{Name: "app_web_xyz", Image: snapshotScheme + testSnap}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("got %v, want cocoon's lookup failure", err) + } + if lines := runner.Lines(); len(lines) != 1 { + t.Errorf("got %q, want no rm for a vm cocoon never made", lines) + } +} + +func TestVirtualizationCreateRefusesACloneBeforeTheNode(t *testing.T) { + tests := []struct { + name string + image string + raw string + }{ + {"a windows guest", snapshotScheme + testSnap, `{"os":"windows"}`}, + {"an empty snapshot name", snapshotScheme, ""}, + {"a name cocoon would refuse", snapshotScheme + "-rf", ""}, + {"a name with a space", snapshotScheme + "a b", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: clonedFrom} + e := testEngine(t, runner) + + _, err := e.VirtualizationCreate(t.Context(), &enginetypes.VirtualizationCreateOptions{Name: "app_web_xyz", Image: tt.image, RawArgs: []byte(tt.raw)}) + if !errors.Is(err, coretypes.ErrInvalidEngineArgs) { + t.Errorf("got %v, want ErrInvalidEngineArgs", err) + } + if lines := runner.Lines(); len(lines) != 0 { + t.Errorf("got %q, want no round trip", lines) + } + }) + } +} + func TestCreateArgvForAWindowsGuest(t *testing.T) { opts := &enginetypes.VirtualizationCreateOptions{Image: "win11", User: "eru"} resource := &engine.VirtualizationResource{Quota: 4, Memory: 4 << 30, Volumes: []string{"/scratch:/scratch:rw:2147483648"}} diff --git a/engine/cocoon/fake_test.go b/engine/cocoon/fake_test.go index 636ea752..03370e8c 100644 --- a/engine/cocoon/fake_test.go +++ b/engine/cocoon/fake_test.go @@ -20,6 +20,7 @@ const ( testUser = "eru" testIDLen = 32 testPty = "/dev/pts/3" + testSnap = "offload-v3" storedRecord = `{"id":"w1","kind":"vm","name":"app_web_xyz","user":"` + testUser + `","nodename":"node1"}` @@ -33,6 +34,9 @@ const ( `"network_configs":[{"tap":"tap01ARZ3ND-0","network":{"ip":"10.22.0.5","gateway":"10.22.0.1","prefix":16}}]}` ptyVM = `{"id":"` + testVMID + `","hypervisor":"cloud-hypervisor","state":"running","first_booted":true,"pid":4242,` + `"console_path":"` + testPty + `","config":{"image":"` + testImage + `"}}` + clonedVM = `{"id":"` + testVMID + `","hypervisor":"cloud-hypervisor","state":"running","first_booted":true,"pid":4242,` + + `"config":{"cpu":4,"memory":8589934592,"image":"` + testImage + `","network":"eru-cni"},` + + `"network_configs":[{"tap":"tap01ARZ3ND-0","mac":"02:00:00:00:00:07","network":{"ip":"10.22.0.7","gateway":"10.22.0.1","prefix":16}}],"hints":["x"]}` stoppedVM = `{"id":"` + testVMID + `","state":"stopped","first_booted":true,"config":{"image":"` + testImage + `"}}` bootedWindowsVM = `{"id":"` + testVMID + `","hypervisor":"cloud-hypervisor","state":"running","first_booted":true,"pid":4242,` + `"config":{"image":"win11","windows":true},` + @@ -68,3 +72,19 @@ func createdVMThenCanceled(cancel context.CancelFunc) func(string) *sshrunner.Re return createdVM(line) } } + +func clonedFrom(line string) *sshrunner.Result { + if strings.Contains(line, "'clone'") { + return &sshrunner.Result{Stdout: clonedVM} + } + return &sshrunner.Result{} +} + +func mustParseVM(t *testing.T, out string) *vmRecord { + t.Helper() + vm, err := parseVM(out) + if err != nil { + t.Fatalf("parse vm: %v", err) + } + return vm +} diff --git a/engine/cocoon/image.go b/engine/cocoon/image.go index 1078a356..d58ac20a 100644 --- a/engine/cocoon/image.go +++ b/engine/cocoon/image.go @@ -86,10 +86,17 @@ func (e *Engine) ImagesPrune(context.Context) error { return coretypes.ErrEngineNotImplemented } -// ImagePull hands a registry ref or a cloud-image url to cocoon; a parts artifact goes through oras and import. +// ImagePull hands a registry ref or a cloud-image url to cocoon; a parts artifact goes through oras and import, a snapshot is only looked up. func (e *Engine) ImagePull(ctx context.Context, ref string, _ bool) (io.ReadCloser, error) { argv := []string{e.cocoon.Binary, "image", "pull", ref} - if !enginetypes.IsURL(ref) && e.partsArtifact(ctx, ref) { + name, snapshot := strings.CutPrefix(ref, snapshotScheme) + switch { + case snapshot: + if err := checkSnapshotName(name); err != nil { + return nil, err + } + argv = e.snapshot("inspect", name) + case !enginetypes.IsURL(ref) && e.partsArtifact(ctx, ref): argv = sshrunner.Shell(importScript, e.cocoon.Binary, ref) } res, err := e.run(ctx, argv...) @@ -112,6 +119,9 @@ func (e *Engine) ImageBuildCachePrune(context.Context, bool) (uint64, error) { } func (e *Engine) ImageLocalDigests(ctx context.Context, image string) ([]string, error) { + if name, ok := strings.CutPrefix(image, snapshotScheme); ok { + return e.snapshotDigests(ctx, image, name) + } res, err := e.call(ctx, e.cocoon.Binary, "image", "inspect", image) if err != nil { return nil, err @@ -126,9 +136,9 @@ func (e *Engine) ImageLocalDigests(ctx context.Context, image string) ([]string, return []string{enginetypes.ImageDigest(image, stored.ID)}, nil } -// ImageRemoteDigest asks the registry through oras; a cloud image url is its own digest. +// ImageRemoteDigest asks the registry through oras; a cloud image url and a snapshot are their own digest. func (e *Engine) ImageRemoteDigest(ctx context.Context, image string) (string, error) { - if enginetypes.IsURL(image) { + if enginetypes.IsURL(image) || strings.HasPrefix(image, snapshotScheme) { return image, nil } if !e.orasPresent(ctx) { diff --git a/engine/cocoon/image_test.go b/engine/cocoon/image_test.go index 3e46f339..562dc38d 100644 --- a/engine/cocoon/image_test.go +++ b/engine/cocoon/image_test.go @@ -114,6 +114,80 @@ func TestOrasProbeIsAskedOnlyOnceOnceItAnswered(t *testing.T) { } } +func TestImagePullOnlyLooksUpASnapshot(t *testing.T) { + tests := []struct { + name string + res *sshrunner.Result + wantErr bool + }{ + {"a snapshot on the node", &sshrunner.Result{Stdout: `{"id":"s1","name":"` + testSnap + `"}`}, false}, + {"a snapshot the node lacks", &sshrunner.Result{Code: 1, Stderr: "not found"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: func(string) *sshrunner.Result { return tt.res }} + e := testEngine(t, runner) + + if _, err := e.ImagePull(t.Context(), snapshotScheme+testSnap, false); (err != nil) != tt.wantErr { + t.Fatalf("got error %v, wantErr %v", err, tt.wantErr) + } + want := []string{sshrunner.Quote([]string{testBinary, "snapshot", "inspect", testSnap})} + if !slices.Equal(runner.Lines(), want) { + t.Errorf("got %q, want %q and no registry", runner.Lines(), want) + } + }) + } +} + +func TestImageDigestsOfASnapshot(t *testing.T) { + ref := snapshotScheme + testSnap + tests := []struct { + name string + res *sshrunner.Result + want []string + }{ + {"a snapshot on the node", &sshrunner.Result{Stdout: `{"id":"s1"}`}, []string{ref}}, + {"a snapshot the node lacks", &sshrunner.Result{Code: 1}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: func(string) *sshrunner.Result { return tt.res }} + e := testEngine(t, runner) + + got, err := e.ImageLocalDigests(t.Context(), ref) + if err != nil { + t.Fatalf("digests: %v", err) + } + if !slices.Equal(got, tt.want) { + t.Errorf("got %q, want %q", got, tt.want) + } + remote, err := e.ImageRemoteDigest(t.Context(), ref) + if err != nil || remote != ref { + t.Errorf("got %q %v, want the ref itself", remote, err) + } + if lines := runner.Lines(); len(lines) != 1 { + t.Errorf("got %q, want one lookup and no oras", lines) + } + }) + } +} + +func TestImageVerbsRefuseABadSnapshotName(t *testing.T) { + runner := &sshrunnertest.Fake{} + e := testEngine(t, runner) + ref := snapshotScheme + "a;b" + + if _, err := e.ImagePull(t.Context(), ref, false); !errors.Is(err, coretypes.ErrInvalidEngineArgs) { + t.Errorf("pull: got %v, want ErrInvalidEngineArgs", err) + } + if _, err := e.ImageLocalDigests(t.Context(), ref); !errors.Is(err, coretypes.ErrInvalidEngineArgs) { + t.Errorf("digests: got %v, want ErrInvalidEngineArgs", err) + } + if lines := runner.Lines(); len(lines) != 0 { + t.Errorf("got %q, want no round trip", lines) + } +} + func TestImageListFiltersByName(t *testing.T) { runner := &sshrunnertest.Fake{Respond: func(string) *sshrunner.Result { return &sshrunner.Result{Stdout: `[{"id":"sha256:a","name":"` + testImage + `","type":"oci"},{"id":"sha256:b","name":"win11","type":"cloudimg"}]`} diff --git a/engine/cocoon/lifecycle_test.go b/engine/cocoon/lifecycle_test.go index 7c982850..9f114523 100644 --- a/engine/cocoon/lifecycle_test.go +++ b/engine/cocoon/lifecycle_test.go @@ -81,6 +81,22 @@ func TestVirtualizationStartLeavesABootedWindowsGuestAlone(t *testing.T) { } } +func TestVirtualizationStartRefreshesAClonedGuestThatAlreadyRuns(t *testing.T) { + runner := &sshrunnertest.Fake{ + Respond: func(string) *sshrunner.Result { return &sshrunner.Result{Stdout: clonedVM + "\n" + clonedVM} }, + } + e := testEngine(t, runner) + + if err := e.VirtualizationStart(t.Context(), "w1"); err != nil { + t.Fatalf("cocoon answers a start of a running vm with success: %v", err) + } + want := sshrunner.Quote(sshrunner.Shell(refreshScript, testRoot+"/w1.json", "/run/eru/workloads/w1.json", + testRunDir+"/cloudhypervisor/"+testVMID+"/console.sock", "4242")) + if lines := runner.Lines(); len(lines) != 2 || lines[1] != want { + t.Errorf("got %q, want the record refreshed with the clone's pid", lines) + } +} + func TestVirtualizationStartRecordsTheSerialSocketWhenCocoonReportsNoConsole(t *testing.T) { runner := &sshrunnertest.Fake{ Respond: func(string) *sshrunner.Result { return &sshrunner.Result{Stdout: linuxVM + "\n" + runningVM} }, diff --git a/engine/cocoon/scripts_test.go b/engine/cocoon/scripts_test.go index 0ac8c169..86726e44 100644 --- a/engine/cocoon/scripts_test.go +++ b/engine/cocoon/scripts_test.go @@ -50,6 +50,11 @@ exit "${code:-0}" ;; "vm status") printf '%s\n' "$STUB_EVENTS";; "snapshot rm") exit "${STUB_SNAPSHOT:-0}";; +"snapshot save") +printf 'snapshot saved\n' +exit "${STUB_SAVE:-0}" +;; +"snapshot inspect") printf '%s\n' "$STUB_SNAPSHOT_JSON";; "image inspect") exit "${STUB_IMAGE:-1}";; "image import") exit "${STUB_IMPORT:-0}";; esac @@ -534,6 +539,48 @@ func TestWaitScriptRefusesToWaitOnAVMCocoonDoesNotHave(t *testing.T) { node.assertCalls(t, "cocoon vm inspect "+scriptVM) } +func TestSaveScriptPrintsOnlyTheSavedSnapshot(t *testing.T) { + tests := []struct { + name string + save string + wantCode int + wantStdout string + wantCalls []string + }{ + { + name: "a snapshot cocoon saved", + wantStdout: testSnapshotJSON + "\n", + wantCalls: []string{ + "cocoon snapshot save --name " + testSnap + " " + scriptVM, + "cocoon snapshot inspect " + testSnap, + }, + }, + { + name: "a save cocoon refused", + save: "2", + wantCode: 2, + wantCalls: []string{"cocoon snapshot save --name " + testSnap + " " + scriptVM}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newScriptNode(t) + node.env["STUB_SAVE"] = tt.save + node.env["STUB_SNAPSHOT_JSON"] = testSnapshotJSON + + got := node.run(t, saveScript, node.binary, testSnap, scriptVM) + + if got.code != tt.wantCode { + t.Fatalf("got exit %d, want %d: %s", got.code, tt.wantCode, got.stderr) + } + if got.stdout != tt.wantStdout { + t.Errorf("got %q, want %q", got.stdout, tt.wantStdout) + } + node.assertCalls(t, tt.wantCalls...) + }) + } +} + func TestImportScriptReassemblesAPartsArtifactOnce(t *testing.T) { node := newScriptNode(t) node.env["STUB_ORAS_FILES"] = "disk.0.part disk.1.part" diff --git a/engine/cocoon/snapshot.go b/engine/cocoon/snapshot.go new file mode 100644 index 00000000..82be8bc2 --- /dev/null +++ b/engine/cocoon/snapshot.go @@ -0,0 +1,105 @@ +package cocoon + +import ( + "context" + "encoding/json" + "regexp" + "slices" + "strings" + + "github.com/cockroachdb/errors" + + "github.com/projecteru2/core/engine/sshrunner" + enginetypes "github.com/projecteru2/core/engine/types" + coretypes "github.com/projecteru2/core/types" +) + +const ( + snapshotScheme = "snapshot://" + + opSnapshotSave = "snapshot.save" + opSnapshotList = "snapshot.list" + opSnapshotInspect = "snapshot.inspect" + opSnapshotRemove = "snapshot.remove" + + noSnapshots = "No snapshots" + + saveScript = `set -e +bin=$1; name=$2; vm=$3 +"$bin" snapshot save --name "$name" "$vm" >/dev/null +exec "$bin" snapshot inspect "$name" +` +) + +// validSnapshotName is cocoon's own snapshot-name grammar. +var validSnapshotName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,62}$`) + +type snapshotParams struct { + Name string `json:"name"` +} + +func (e *Engine) RawEngine(ctx context.Context, opts *enginetypes.RawEngineOptions) (*enginetypes.RawEngineResult, error) { + argv, err := e.rawArgv(opts) + if err != nil { + return nil, err + } + res, err := e.run(ctx, argv...) + if err != nil { + return nil, err + } + out := res.Stdout + switch { + case opts.Op == opSnapshotRemove: + out = "" + case opts.Op == opSnapshotList && strings.HasPrefix(strings.TrimSpace(out), noSnapshots): + out = "[]" + } + return &enginetypes.RawEngineResult{ID: opts.ID, Data: []byte(out)}, nil +} + +func (e *Engine) rawArgv(opts *enginetypes.RawEngineOptions) ([]string, error) { + switch opts.Op { + case opSnapshotList: + return e.snapshot("list", "--format", formatJSON), nil + case opSnapshotSave, opSnapshotInspect, opSnapshotRemove: + default: + return nil, errors.Wrapf(coretypes.ErrEngineNotImplemented, "cocoon has no raw op %q", opts.Op) + } + params := &snapshotParams{} + if err := json.Unmarshal(opts.Params, params); err != nil { + return nil, errors.Wrapf(coretypes.ErrInvalidEngineArgs, "%s params: %v", opts.Op, err) + } + if err := checkSnapshotName(params.Name); err != nil { + return nil, err + } + switch opts.Op { + case opSnapshotSave: + return sshrunner.Shell(saveScript, e.cocoon.Binary, params.Name, opts.ID), nil + case opSnapshotRemove: + return e.snapshot("rm", params.Name), nil + } + return e.snapshot("inspect", params.Name), nil +} + +// snapshotDigests reports a snapshot:// image as its own digest while the snapshot is on the node. +func (e *Engine) snapshotDigests(ctx context.Context, image, name string) ([]string, error) { + if err := checkSnapshotName(name); err != nil { + return nil, err + } + res, err := e.call(ctx, e.snapshot("inspect", name)...) + if err != nil || res.Code != 0 { + return nil, err + } + return []string{image}, nil +} + +func (e *Engine) snapshot(args ...string) []string { + return slices.Concat([]string{e.cocoon.Binary, "snapshot"}, args) +} + +func checkSnapshotName(name string) error { + if !validSnapshotName.MatchString(name) { + return errors.Wrapf(coretypes.ErrInvalidEngineArgs, "snapshot name %q must match %s", name, validSnapshotName) + } + return nil +} diff --git a/engine/cocoon/snapshot_test.go b/engine/cocoon/snapshot_test.go new file mode 100644 index 00000000..bf4c4edf --- /dev/null +++ b/engine/cocoon/snapshot_test.go @@ -0,0 +1,134 @@ +package cocoon + +import ( + "strings" + "testing" + + "github.com/cockroachdb/errors" + + "github.com/projecteru2/core/engine/sshrunner" + "github.com/projecteru2/core/engine/sshrunner/sshrunnertest" + enginetypes "github.com/projecteru2/core/engine/types" + coretypes "github.com/projecteru2/core/types" +) + +const testSnapshotJSON = `{"id":"S1","name":"` + testSnap + `","cpu":4}` + +func TestRawEngineRendersEachSnapshotOp(t *testing.T) { + params := []byte(`{"name":"` + testSnap + `"}`) + tests := []struct { + name string + op string + params []byte + stdout string + wantArgv []string + wantData string + }{ + { + name: "save", + op: opSnapshotSave, + params: params, + stdout: testSnapshotJSON, + wantArgv: sshrunner.Shell(saveScript, testBinary, testSnap, "w1"), + wantData: testSnapshotJSON, + }, + { + name: "list", + op: opSnapshotList, + stdout: "[" + testSnapshotJSON + "]", + wantArgv: []string{testBinary, "snapshot", "list", "--format", "json"}, + wantData: "[" + testSnapshotJSON + "]", + }, + { + name: "an empty list in prose", + op: opSnapshotList, + stdout: "No snapshots found.\n", + wantArgv: []string{testBinary, "snapshot", "list", "--format", "json"}, + wantData: "[]", + }, + { + name: "inspect", + op: opSnapshotInspect, + params: params, + stdout: testSnapshotJSON, + wantArgv: []string{testBinary, "snapshot", "inspect", testSnap}, + wantData: testSnapshotJSON, + }, + { + name: "remove", + op: opSnapshotRemove, + params: params, + stdout: "deleted: S1\n", + wantArgv: []string{testBinary, "snapshot", "rm", testSnap}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: func(string) *sshrunner.Result { return &sshrunner.Result{Stdout: tt.stdout} }} + e := testEngine(t, runner) + + got, err := e.RawEngine(t.Context(), &enginetypes.RawEngineOptions{ID: "w1", Op: tt.op, Params: tt.params}) + if err != nil { + t.Fatalf("raw engine: %v", err) + } + if want := sshrunner.Quote(tt.wantArgv); len(runner.Lines()) != 1 || runner.Lines()[0] != want { + t.Errorf("got %q, want %q", runner.Lines(), want) + } + if got.ID != "w1" || string(got.Data) != tt.wantData { + t.Errorf("got %s %q, want w1 %q", got.ID, got.Data, tt.wantData) + } + }) + } +} + +func TestRawEngineRefusesABadSnapshotNameBeforeTheNode(t *testing.T) { + tests := []struct { + name string + params string + }{ + {"no params", ""}, + {"params that are not json", "save"}, + {"no name", `{}`}, + {"a leading dash", `{"name":"-rf"}`}, + {"a shell metacharacter", `{"name":"a;b"}`}, + {"a space", `{"name":"a b"}`}, + {"past cocoon's 63 chars", `{"name":"` + strings.Repeat("a", 64) + `"}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runner := &sshrunnertest.Fake{} + e := testEngine(t, runner) + + for _, op := range []string{opSnapshotSave, opSnapshotInspect, opSnapshotRemove} { + if _, err := e.RawEngine(t.Context(), &enginetypes.RawEngineOptions{ID: "w1", Op: op, Params: []byte(tt.params)}); !errors.Is(err, coretypes.ErrInvalidEngineArgs) { + t.Errorf("%s: got %v, want ErrInvalidEngineArgs", op, err) + } + } + if lines := runner.Lines(); len(lines) != 0 { + t.Errorf("got %q, want no round trip", lines) + } + }) + } +} + +func TestRawEngineRefusesAnUnknownOp(t *testing.T) { + runner := &sshrunnertest.Fake{} + e := testEngine(t, runner) + + if _, err := e.RawEngine(t.Context(), &enginetypes.RawEngineOptions{ID: "w1", Op: "snapshot.export"}); !errors.Is(err, coretypes.ErrEngineNotImplemented) { + t.Errorf("got %v, want ErrEngineNotImplemented", err) + } + if lines := runner.Lines(); len(lines) != 0 { + t.Errorf("got %q, want no round trip", lines) + } +} + +func TestRawEngineReportsACocoonFailure(t *testing.T) { + runner := &sshrunnertest.Fake{Respond: func(string) *sshrunner.Result { return &sshrunner.Result{Code: 1, Stderr: "snapshot name taken"} }} + e := testEngine(t, runner) + + _, err := e.RawEngine(t.Context(), &enginetypes.RawEngineOptions{ID: "w1", Op: opSnapshotSave, Params: []byte(`{"name":"` + testSnap + `"}`)}) + if err == nil || !strings.Contains(err.Error(), "snapshot name taken") { + t.Errorf("got %v, want cocoon's failure", err) + } +} diff --git a/engine/cocoon/vm.go b/engine/cocoon/vm.go index bacf61ce..1ce2118c 100644 --- a/engine/cocoon/vm.go +++ b/engine/cocoon/vm.go @@ -24,6 +24,7 @@ type vmConfig struct { type nic struct { TAP string `json:"tap"` + MAC string `json:"mac"` Network *guestAddress `json:"network"` }