diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b4f7b3..fcef9c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,10 +30,13 @@ jobs: - run: go vet ./... # The non-Linux files must also compile, for development on macOS. - run: GOOS=darwin go vet ./... + # The tests of internal/initbin examine the runplane-init of make init. + - run: make init - run: go test ./... - # Tests of netlink, nftables, loop devices, and mounts skip without root. + # Tests of netlink, nftables, loop devices, mounts, and the debug shell + # of runplane-init skip without root. - name: go test as root - run: sudo --preserve-env=GOCACHE,GOMODCACHE,GOPATH,HOME "$(go env GOROOT)/bin/go" test ./internal/... + run: sudo --preserve-env=GOCACHE,GOMODCACHE,GOPATH,HOME "$(go env GOROOT)/bin/go" test ./init/... ./internal/... env: GOCACHE: /tmp/go-root-cache @@ -53,6 +56,11 @@ jobs: cache: pnpm cache-dependency-path: web/pnpm-lock.yaml - run: pnpm install --frozen-lockfile + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + # The committed spec, types, and limits must match the Go API. + - run: pnpm gen:api && git diff --exit-code -- src/api - run: pnpm typecheck - run: pnpm test - run: pnpm build @@ -97,3 +105,18 @@ jobs: test ! -e /usr/local/bin/runplane test ! -e /etc/systemd/system/runplane.service sudo test ! -e /var/lib/runplane + # The tests that start real sandboxes, on the Docker and Sysbox that + # the uninstall keeps. The service and its firewall rules are gone, so + # the tests can own the runplane network. -p 1 runs one package at a + # time, because the two packages use the same network. The sandboxes + # run the runplane-init that make binary embedded. + - name: Sandbox tests on Docker and Sysbox + run: | + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends nftables + sudo docker build -q -t runplane-test internal/sandbox/sysbox/sysboxtest + sudo --preserve-env=GOCACHE,GOMODCACHE,GOPATH,HOME \ + RUNPLANE_TEST_DOCKER=1 RUNPLANE_TEST_IMAGE=runplane-test \ + "$(go env GOROOT)/bin/go" test -count=1 -p 1 -v ./internal/sandbox/sysbox/ ./internal/sandbox/devdocker/ + env: + GOCACHE: /tmp/go-root-cache diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 8a825d3..57964f1 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -1,17 +1,24 @@ # Builds the base images for linux/amd64 and linux/arm64 and pushes them to -# ghcr.io///runner-ubuntu-24.04 and -26.04. runplane pulls the -# "latest" tag. A weekly build gets the new packages and tool versions. +# ghcr.io///runner-. runplane pulls the "latest" tag. A +# weekly build gets the new packages and tool versions. The two matrices +# must name the images and the build args of images/images.go +# (images_test.go compares them). # # The builds use the runplane runners. QEMU builds the architecture that the # runner does not have, which makes that build much slower. +# +# The build context is the images folder, and images/.dockerignore keeps +# only the Dockerfile in it. A change to images.go, its test, or the README +# does not change an image, so it starts no build. A new image changes this +# file too, because the matrices must name it. name: Images on: push: branches: [main] - paths: ['images/**', '.github/workflows/images.yml'] + paths: ['images/Dockerfile', 'images/.dockerignore', '.github/workflows/images.yml'] pull_request: - paths: ['images/**', '.github/workflows/images.yml'] + paths: ['images/Dockerfile', 'images/.dockerignore', '.github/workflows/images.yml'] schedule: - cron: '23 4 * * 1' workflow_dispatch: @@ -24,7 +31,9 @@ jobs: strategy: fail-fast: false matrix: - image: [ubuntu-24.04, ubuntu-26.04] + image: + - {name: ubuntu-24.04, ubuntu: '24.04', image_os: ubuntu24} + - {name: ubuntu-26.04, ubuntu: '26.04', image_os: ubuntu26} arch: [amd64, arm64] runs-on: runplane permissions: @@ -33,7 +42,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: Image name - run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}/runner-${{ matrix.image }}" >> "$GITHUB_ENV" + run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}/runner-${{ matrix.image.name }}" >> "$GITHUB_ENV" - uses: docker/setup-qemu-action@v4 - uses: docker/setup-buildx-action@v4 - uses: docker/login-action@v4 @@ -46,8 +55,11 @@ jobs: id: build uses: docker/build-push-action@v7 with: - context: images/${{ matrix.image }} + context: images platforms: linux/${{ matrix.arch }} + build-args: | + UBUNTU=${{ matrix.image.ubuntu }} + IMAGE_OS=${{ matrix.image.image_os }} labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.revision=${{ github.sha }} @@ -55,11 +67,12 @@ jobs: # VM mode copies the image into an ext4 root disk, so keep it small. - name: Check the size (3 GB or less) run: | - docker buildx build --platform linux/${{ matrix.arch }} --load -t size-check -q images/${{ matrix.image }} >/dev/null + docker buildx build --platform linux/${{ matrix.arch }} --load -t size-check -q \ + --build-arg UBUNTU=${{ matrix.image.ubuntu }} --build-arg IMAGE_OS=${{ matrix.image.image_os }} images >/dev/null mb=$(docker run --rm size-check du -sxm / | cut -f 1) echo "Image size: $mb MB" if [ "$mb" -gt 3072 ]; then - echo "::error::${{ matrix.image }} uses $mb MB. The limit is 3072 MB." + echo "::error::${{ matrix.image.name }} uses $mb MB. The limit is 3072 MB." exit 1 fi - name: Check that jobs find rustup @@ -73,7 +86,7 @@ jobs: - uses: actions/upload-artifact@v7 if: github.event_name != 'pull_request' with: - name: digests-${{ matrix.image }}-${{ matrix.arch }} + name: digests-${{ matrix.image.name }}-${{ matrix.arch }} path: ${{ runner.temp }}/digests/* if-no-files-found: error diff --git a/Makefile b/Makefile index ace0c7a..1238764 100644 --- a/Makefile +++ b/Makefile @@ -115,11 +115,11 @@ dev: init # runner-smoke downloads the latest actions/runner for this arch and starts # it in the local base image runplane/runner-ubuntu-24.04:local (docker build -# --provenance=false -t runplane/runner-ubuntu-24.04:local images/ubuntu-24.04): -# the .NET libraries must be in the image, and a bad JIT config must end the -# runner. +# --provenance=false -t runplane/runner-ubuntu-24.04:local --build-arg +# UBUNTU=24.04 --build-arg IMAGE_OS=ubuntu24 images): the .NET libraries must +# be in the image, and a bad JIT config must end the runner. runner-smoke: - go test -tags integration -count=1 -timeout 20m -v -run TestRealRunnerInBaseImage ./internal/image/ + go test -tags integration -count=1 -timeout 20m -v -run TestRealRunnerInBaseImage ./internal/runner/ # clean keeps the other files in .dev, such as notes. clean: diff --git a/README.md b/README.md index 0042ddf..f6f2c5d 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ The script does these steps: 3. It downloads runplane and makes sure that the signature of `SHA256SUMS` and the SHA-256 of each file are correct (refer to [Release](#release)). 4. In container mode, it installs Docker Engine (only if the server does not have it) and Sysbox. 5. In VM mode, it installs Firecracker v1.17.0, the jailer, and the guest kernel. It also gives the user `runplane-vm` a range of uids and gids, one for each VM. If the base image of the release repository is not public on GHCR, it installs Docker Engine, because runplane then builds the base image on the server. Docker builds images only, and the jobs operate in VMs. -6. It writes `/etc/runplane/runplane.env` and the systemd units, then it starts `runplane.service`. -7. It runs `runplane doctor` and shows the dashboard URL and a one-time setup code. +6. It writes `/etc/runplane/runplane.env` and the systemd units, and it makes a one-time setup code. Then it starts `runplane.service`. +7. It runs `runplane doctor` and shows the dashboard URL and the setup code. To see each step before the install, add `--dry-run`. The dry run changes nothing on the server. @@ -66,7 +66,7 @@ In VM mode, the script installs Firecracker v1.17.0, which the runplane tests us curl -fsSL https://github.com/runplane/runplane/releases/latest/download/install.sh | sudo RUNPLANE_FIRECRACKER_VERSION=v1.16.1 sh ``` -To update runplane, run the install command again. You can also select the update in **Settings > General**, or run `sudo runplane update` on the server. In VM mode, the update also replaces the guest kernel with the kernel of the release. A restart of runplane does not stop the jobs that operate. A shutdown of the server stops the jobs that operate with the result `cancelled`. +To update runplane, run the install command again. You can also select the update in **Settings > General**, or run `sudo runplane update` on the primary. In VM mode, the update also replaces the guest kernel with the kernel of the release. A restart of runplane does not stop the jobs that operate. A shutdown of the server stops the jobs that operate with the result `cancelled`. The primary updates its agents. When an agent connects with a different version, the primary sends it the update to its own release. After an update of the primary, the agents connect again and get the new release. The primary tries each version one time on each agent. If the update is not successful, the **Servers** page shows **Update failed** after the name of the server. The page of the server shows the reason. You can start the update again with **Update agent** in the menu of the server. A primary that is not a release build does not update its agents. @@ -76,7 +76,7 @@ You can also update an agent by hand. Run this command on the agent server: curl -fsSL http://203.0.113.10:8484/install.sh | sudo sh ``` -The primary serves this script. Thus the agent gets the version of the primary. On an agent, the script keeps the registration in `/etc/runplane/agent.json` and does not install a primary. On the agent server, `sudo runplane update` installs the latest release. +The primary serves this script. Thus the agent gets the version of the primary. On an agent, the script keeps the registration in `/etc/runplane/agent.json` and does not install a primary. An agent must run the release of its primary, so `sudo runplane update` does not operate on an agent. To remove runplane, run this command: @@ -187,7 +187,7 @@ An agent must run the same release as the primary. The primary updates each agen **Warning:** Use runplane only with private repositories. A pull request from a fork of a public repository can run code on your server. -In container mode, runplane always rejects jobs from public repositories. All Sysbox containers share the host kernel and one ID mapping, thus the isolation between jobs is weaker. In VM mode, runplane rejects jobs from public repositories and from fork pull requests, unless you change these settings. You can let jobs from public repositories run only if each server operates in VM mode. +In container mode, runplane never runs jobs from public repositories. All Sysbox containers share the host kernel and one ID mapping, thus the isolation between jobs is weaker. In VM mode, runplane rejects jobs from public repositories and from fork pull requests, unless you change these settings. You can turn on the setting for public repositories only if each server operates in VM mode. While the setting is on, a server that connects in a different mode gets no runners, and the **Servers** page tells the cause. To give runners to that server again, turn the setting off. | Item | Container mode (Sysbox CE) | VM mode (Firecracker) | | --- | --- | --- | @@ -196,7 +196,7 @@ In container mode, runplane always rejects jobs from public repositories. All Sy | Network model | One Docker bridge, with the MAC and IP binding in `bridge runplane` | A TAP with a /30 for each VM | | AppArmor | The Sysbox profile | Not applicable | | Host daemon on the `/proc` path | `sysbox-fs` (Sysbox issue 1018) | None | -| Public repositories | Rejected | Permitted with the setting | +| Public repositories | Never | Permitted with the setting | Sandboxes cannot connect to the host, to other sandboxes, to private networks, to `169.254.169.254`, or to the Azure host endpoint `168.63.129.16`. They can connect only to the public internet and to the runplane cache service on the host. @@ -295,10 +295,10 @@ Other commands: | `sh deploy/test-install.sh` | Dry runs of `install.sh` in Ubuntu, Debian, and Fedora containers | | `kernel/build.sh --config-only` | A check of the guest kernel config, without a kernel build | -The tests of netlink, nftables, loop devices, and mounts must have root access on Linux. To do these tests on macOS, use a Linux container: +The tests of netlink, nftables, loop devices, mounts, and the debug shell of `runplane-init` must have root access on Linux. To do these tests on macOS, use a Linux container: ```sh -docker run --rm --privileged -v "$PWD":/src -v "$HOME/go/pkg/mod":/go/pkg/mod -w /src golang:1.27 go test ./internal/... +docker run --rm --privileged -v "$PWD":/src -v "$HOME/go/pkg/mod":/go/pkg/mod -w /src golang:1.27 go test ./init/... ./internal/... ``` ## Release @@ -321,4 +321,4 @@ A person who can change the release files can also change the `install.sh` of th **Caution:** Do not change the key after the first release. A binary with the earlier key cannot do a self-update to a release with a new key. On these servers, run the install command of the new release again. -The `images.yml` workflow pushes the base images to GHCR each week and after each change in `images/`. The builds use the `runplane` label. QEMU builds the architecture that the runner does not have. After the first push, set the visibility of each image package on GHCR to public. Then servers pull the base images without a login, and VM mode does not need Docker. To do a check, run `docker logout ghcr.io && docker pull ghcr.io///runner-ubuntu-24.04:latest`. +The `images.yml` workflow pushes the base images to GHCR each week and after each change of `images/Dockerfile`. The builds use the `runplane` label. QEMU builds the architecture that the runner does not have. After the first push, set the visibility of each image package on GHCR to public. Then servers pull the base images without a login, and VM mode does not need Docker. To do a check, run `docker logout ghcr.io && docker pull ghcr.io///runner-ubuntu-24.04:latest`. diff --git a/cmd/runplane/agent.go b/cmd/runplane/agent.go index 09d92df..b9f2976 100644 --- a/cmd/runplane/agent.go +++ b/cmd/runplane/agent.go @@ -4,30 +4,38 @@ import ( "context" "errors" "flag" - "net/netip" - "slices" "github.com/runplane/runplane/internal/agent" "github.com/runplane/runplane/internal/config" - "github.com/runplane/runplane/internal/sandbox" + "github.com/runplane/runplane/internal/localnode" + "github.com/runplane/runplane/internal/policy" "github.com/runplane/runplane/internal/version" ) // runAgent operates the sandboxes of this server for the primary in -// agent.json. The primary sends the settings when the agent connects. +// agent.json. The primary sends the settings when the agent connects. The +// agent keeps them, so that after a restart the sandboxes keep their egress +// allowlist before the primary connects. func runAgent(ctx context.Context, args []string) error { - cfg, err := config.Parse(flag.NewFlagSet("agent", flag.ContinueOnError), args) + cfg, err := config.Parse(flag.NewFlagSet("agent", flag.ContinueOnError), args, + config.ServicePort, config.DataDir, config.Mode, config.Dev, config.DevInsecure) if err != nil { return err } log := newLogger(ctx, cfg) - if err := checkMode(cfg.Mode, cfg.DevInsecure); err != nil { + if err := policy.CheckMode(cfg.Mode, cfg.DevInsecure); err != nil { return err } if err := cfg.EnsureDirs(); err != nil { return err } - l, err := newLocal(ctx, cfg, localConfig{mode: cfg.Mode, unit: "runplane-agent", listen: []string{servicePortAddr(cfg)}}, log) + settings, err := localnode.ReadSettings(cfg.AgentSettingsPath()) + if err != nil { + log.Warn("the sandboxes have no egress allowlist until the primary connects", "err", err) + } + l, err := newLocal(ctx, cfg, localConfig{ + mode: cfg.Mode, settings: settings, settingsFile: cfg.AgentSettingsPath(), doctor: doctorOptions(cfg, cfg.Mode, true, ""), + }, log) if err != nil { return err } @@ -38,18 +46,14 @@ func runAgent(ctx context.Context, args []string) error { defer func() { stopNode(); <-nodeDone }() proxy := agent.NewCacheProxy(l.driver, log.With("component", "cache")) - hasSandbox := func(ip netip.Addr) bool { - list, err := l.driver.List(context.Background()) - return err == nil && slices.ContainsFunc(list, func(b sandbox.Sandbox) bool { return b.IP == ip.String() }) - } - mir, stopPort, err := serveServicePort(cfg, cfg.Mode, hasSandbox, proxy, log) + stopPort, err := serveServicePort(cfg, l.node.Mirror(), proxy, log) if err != nil { return err } defer stopPort() log.Info("runplane agent started", "version", version.String(), "mode", cfg.Mode, "config", cfg.AgentConfigPath()) - err = agent.Run(ctx, agent.Config{Path: cfg.AgentConfigPath(), Node: l.node, CacheProxy: proxy, Mirror: mir.Stats, Logger: log}) + err = agent.Run(ctx, agent.Config{Path: cfg.AgentConfigPath(), Node: l.node, CacheProxy: proxy, Version: version.Version, Logger: log}) if errors.Is(err, agent.ErrUpdated) { // systemd starts the new binary. The sandboxes keep operating, and // the driver takes them over after the restart. diff --git a/cmd/runplane/doctor.go b/cmd/runplane/doctor.go new file mode 100644 index 0000000..59383cd --- /dev/null +++ b/cmd/runplane/doctor.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/moby/moby/client" + + "github.com/runplane/runplane/images" + "github.com/runplane/runplane/internal/config" + "github.com/runplane/runplane/internal/doctor" + "github.com/runplane/runplane/internal/image" + "github.com/runplane/runplane/internal/policy" + "github.com/runplane/runplane/internal/sandbox" + "github.com/runplane/runplane/internal/sandbox/devdocker" + "github.com/runplane/runplane/internal/sandbox/firecracker" + "github.com/runplane/runplane/internal/sandbox/sysbox" + "github.com/runplane/runplane/internal/tlsutil" + "github.com/runplane/runplane/internal/version" +) + +// The systemd services of the primary and of an agent, which install.sh +// writes. +const ( + primaryUnit = "runplane" + agentUnit = "runplane-agent" +) + +// doctorOptions returns the doctor options of this server in mode: of an +// agent, or of the primary, whose domain adds the ports of HTTPS. The +// caller adds the internet check, if it can do it, and the +// min_free_disk_percent setting. +func doctorOptions(cfg config.Config, mode sandbox.Mode, agent bool, domain string) doctor.Options { + o := doctor.Options{ + DataDir: cfg.DataDir, EnvFile: cfg.EnvFilePath(), Unit: primaryUnit, Listen: primaryAddrs(cfg, domain), + Requirements: requirements(cfg, mode, agent), Docker: dockerFacts, BaseImage: baseImage, + } + if agent { + o.Unit, o.Listen = agentUnit, []string{servicePortAddr(cfg)} + } + return o +} + +// requirements fails the mode check for a mode that serve and the agent +// refuse. +func requirements(cfg config.Config, mode sandbox.Mode, agent bool) doctor.Requirements { + if err := policy.CheckMode(mode, cfg.DevInsecure); err != nil { + modes := "container, vm, or dev" + if mode == sandbox.ModeDev { + modes = "container or vm" + } + return doctor.Requirements{Mode: doctor.ModeCheck.Fail(err.Error(), modeFix(cfg, agent, modes))} + } + switch mode { + case sandbox.ModeContainer: + return sysbox.Requirements() + case sandbox.ModeVM: + return firecracker.Requirements(cfg.KernelPath(), cfg.JailDir()) + } + return devdocker.Requirements(cfg.ServicePort, modeFix(cfg, agent, "container or vm")) +} + +// modeFix tells how to give this server one of modes. After the first start +// of a primary, which makes its database, the mode setting wins over +// RUNPLANE_MODE (policy.Mode). +func modeFix(cfg config.Config, agent bool, modes string) string { + if !agent && exists(cfg.DBPath()) { + return "Run sudo runplane set-mode " + modes + "." + } + return "Set RUNPLANE_MODE to " + modes + " in " + cfg.EnvFilePath() + "." +} + +// primaryAddrs lists the TCP addresses of the primary. A domain adds the +// ports of HTTPS, which another web server often has. +func primaryAddrs(cfg config.Config, domain string) []string { + addrs := []string{cfg.Listen, cfg.AgentListen, servicePortAddr(cfg), cfg.SSHListen} + if domain != "" { + addrs = append(addrs, tlsutil.HTTPSAddr, tlsutil.HTTPAddr) + } + return addrs +} + +// dockerFacts reads the Docker Engine at DOCKER_HOST or the default socket. +func dockerFacts(ctx context.Context) (doctor.Docker, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + c, err := client.New(client.FromEnv) + if err != nil { + return doctor.Docker{}, fmt.Errorf("make docker client: %w", err) + } + defer c.Close() + res, err := c.Info(ctx, client.InfoOptions{}) + if err != nil { + return doctor.Docker{}, fmt.Errorf("read docker info: %w", err) + } + d := doctor.Docker{Version: res.Info.ServerVersion, RootDir: res.Info.DockerRootDir, Runtimes: map[string]string{}} + if fw := res.Info.FirewallBackend; fw != nil { + d.Firewall = fw.Driver + } + for name, rt := range res.Info.Runtimes { + d.Runtimes[name] = rt.Path + } + return d, nil +} + +func baseImage(ctx context.Context) (string, error) { + ref := images.Ref(version.Repo, images.Default) + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + return ref, image.CanPull(ctx, ref) +} diff --git a/cmd/runplane/doctor_test.go b/cmd/runplane/doctor_test.go new file mode 100644 index 0000000..f211878 --- /dev/null +++ b/cmd/runplane/doctor_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "net/netip" + "os" + "slices" + "testing" + + "github.com/runplane/runplane/internal/config" + "github.com/runplane/runplane/internal/doctor" + "github.com/runplane/runplane/internal/sandbox" +) + +// The doctor must fail where serve and the agent refuse to start. +func TestDoctorModeCheck(t *testing.T) { + cases := []struct { + mode sandbox.Mode + devInsecure bool + want doctor.Status + }{ + {"", false, doctor.StatusFail}, + {sandbox.ModeDev, false, doctor.StatusFail}, + {sandbox.ModeDev, true, doctor.StatusWarn}, + {sandbox.ModeContainer, false, doctor.StatusOK}, + } + for _, c := range cases { + cfg := config.Default() + cfg.ConfigDir, cfg.DataDir, cfg.DevInsecure = t.TempDir(), t.TempDir(), c.devInsecure + if got := requirements(cfg, c.mode, false).Mode; got.Status != c.want { + t.Errorf("mode %q, RUNPLANE_DEV_INSECURE %v: the mode check is %s (%s), want %s", c.mode, c.devInsecure, got.Status, got.Detail, c.want) + } + } +} + +// The firewall of an agent lets the sandboxes send DNS requests only to the +// DNS servers of the settings that the primary sent, so the doctor of an +// agent must probe with these servers. +func TestAgentDoctorUsesTheSettingsOfThePrimary(t *testing.T) { + ctx := context.Background() + cfg := config.Default() + cfg.ConfigDir, cfg.DataDir = t.TempDir(), t.TempDir() + + _, node, err := serviceSettings(ctx, cfg, true) + defaults := []netip.Addr{netip.MustParseAddr("1.1.1.1"), netip.MustParseAddr("8.8.8.8")} + if err != nil || !slices.Equal(node.DNS, defaults) || node.MinFreeDiskPercent != 15 { + t.Errorf("before the first connection: DNS %v, min free %v%%, %v; want the defaults", node.DNS, node.MinFreeDiskPercent, err) + } + + sent := `{"dns":["9.9.9.9"],"egress_allowlist":["github.com"],"min_free_disk_percent":5}` + if err := os.WriteFile(cfg.AgentSettingsPath(), []byte(sent), 0o600); err != nil { + t.Fatal(err) + } + _, node, err = serviceSettings(ctx, cfg, true) + if err != nil || !slices.Equal(node.DNS, []netip.Addr{netip.MustParseAddr("9.9.9.9")}) || node.MinFreeDiskPercent != 5 { + t.Errorf("DNS %v, min free %v%%, %v; want 9.9.9.9 and 5%% from the primary", node.DNS, node.MinFreeDiskPercent, err) + } +} diff --git a/cmd/runplane/install_domain_test.go b/cmd/runplane/install_domain_test.go new file mode 100644 index 0000000..49ed137 --- /dev/null +++ b/cmd/runplane/install_domain_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "strconv" + "strings" + "testing" + + "github.com/runplane/runplane" + "github.com/runplane/runplane/internal/installtest" +) + +// install.sh checks --domain before it stops the primary, and set-domain +// checks the domain after the stop. A domain that only set-domain refuses +// thus leaves runplane stopped. set-domain owns the rule. +func TestInstallDomainAgreesWithSetDomain(t *testing.T) { + dir := t.TempDir() + t.Setenv("RUNPLANE_CONFIG_DIR", dir) + t.Setenv("RUNPLANE_DATA_DIR", dir) + label := strings.Repeat("a", 63) + domains := []string{ + "", + "ci.example.com", + "CI.Example.com", + "a.b", + "xn--bcher-kva.example", + label + ".com", + "a" + label + ".com", + label + "." + label + "." + label + "." + label[:61], // 253 characters + label + "." + label + "." + label + "." + label[:62], // 254 characters + "localhost", + "192.0.2.1", + "0.0.0.0", + "255.255.255.255", + // net.ParseIP reads these as host names, not IP addresses. + "192.0.2.01", + "192.0.2.256", + "192.0.2.1.5", + "ci..example.com", + "ci.example.com.", + "-ci.example.com", + "ci-.example.com", + "bücher.example", + "ci.example.com:443", + "https://ci.example.com", + } + script, err := installtest.ReplaceMain(runplane.InstallScript, + `for d; do if (parse_flags --domain "$d") >/dev/null 2>&1; then echo true; else echo false; fi; done`) + if err != nil { + t.Fatal(err) + } + out, err := installtest.Sh(script, nil, append([]string{"-s", "--"}, domains...)...) + got := strings.Fields(out) + if err != nil || len(got) != len(domains) { + t.Fatalf("install.sh: %v\n%s", err, out) + } + for i, d := range domains { + want := strconv.FormatBool(setDomain(context.Background(), []string{d}) == nil) + if got[i] != want { + t.Errorf("install.sh accepts --domain %q: %s, set-domain accepts it: %s. Change install.sh and internal/settings/clean.go together.", d, got[i], want) + } + } +} diff --git a/cmd/runplane/install_test.go b/cmd/runplane/install_test.go new file mode 100644 index 0000000..523a1c9 --- /dev/null +++ b/cmd/runplane/install_test.go @@ -0,0 +1,535 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "go/ast" + "go/parser" + "go/token" + "maps" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + "testing" + + "github.com/runplane/runplane" + "github.com/runplane/runplane/images" + "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/config" + "github.com/runplane/runplane/internal/fleet" + "github.com/runplane/runplane/internal/initbin" + "github.com/runplane/runplane/internal/installtest" + "github.com/runplane/runplane/internal/sandbox/firecracker" + "github.com/runplane/runplane/internal/sandbox/guest" + "github.com/runplane/runplane/internal/sandbox/netrules" + "github.com/runplane/runplane/internal/sandbox/sysbox" + "github.com/runplane/runplane/internal/update" + "github.com/runplane/runplane/internal/version" +) + +// TestInstallFilesAgreeWithGo compares the copies of Go values in install.sh, +// the Makefile, the workflows, and kernel/build.sh with the Go values. Each +// error names the files that must change together. +func TestInstallFilesAgreeWithGo(t *testing.T) { + script := string(runplane.InstallScript) + def := config.Default() + roles := []struct { + name, flags, unit string + run func(context.Context, []string) error + }{ + {"primary", "", primaryUnit, serve}, + {"agent", "--token rp_srv_TEST --primary 203.0.113.7:8485 --fingerprint sha256:" + strings.Repeat("0", 64), agentUnit, runAgent}, + } + + t.Run("paths and names", func(t *testing.T) { + vars := installVars(t, "CONFIG_DIR", "DATA_DIR", "ENV_FILE", "AGENT_FILE", "KERNEL_FILE", "VM_USER", "VM_ID_COUNT", "SYSBOX_MIN", "RUNPLANE_REPO") + for _, c := range []struct{ name, want, file string }{ + {"CONFIG_DIR", def.ConfigDir, "internal/config/config.go"}, + {"DATA_DIR", def.DataDir, "internal/config/config.go"}, + {"ENV_FILE", def.EnvFilePath(), "internal/config/paths.go"}, + {"AGENT_FILE", def.AgentConfigPath(), "internal/config/paths.go"}, + // runplane update replaces the guest kernel at this path. + {"KERNEL_FILE", def.KernelPath(), "internal/config/paths.go"}, + {"VM_USER", firecracker.JailerUser, "internal/sandbox/firecracker/doctor.go"}, + // The VM of TAP lease n runs with the uid and gid n of the range. + {"VM_ID_COUNT", strconv.Itoa(netrules.TapLeases(netrules.DefaultTapSubnet)), "internal/sandbox/netrules/netrules.go"}, + {"SYSBOX_MIN", sysbox.MinVersion, "internal/sandbox/sysbox/doctor.go"}, + {"RUNPLANE_REPO", version.Repo, "internal/version/version.go"}, + } { + agree(t, c.name+" of install.sh", vars[c.name], c.want, "install.sh", c.file) + } + // store_mode runs set-mode only on a primary that has a database. + db := match(t, script, `-f "\$DATA_DIR/([^"]+)"`) + agree(t, "the database of install.sh", filepath.Join(vars["DATA_DIR"], db), def.DBPath(), "install.sh", "internal/config/paths.go") + }) + + t.Run("runplane.env", func(t *testing.T) { + vars := configVars(t, "-mode", "vm") + for _, role := range roles { + envFile := filepath.Join(t.TempDir(), "runplane.env") + out := installtest.Run(t, "parse_flags --dry-run "+role.flags+"\nMODE=vm\nENV_FILE="+envFile+"\nAGENT_FILE="+filepath.Join(t.TempDir(), "agent.json")+"\nwrite_config") + env, ok := writtenFiles(out)[envFile] + if !ok { + t.Fatalf("install.sh of the %s writes no runplane.env:\n%s", role.name, out) + } + for line := range strings.Lines(env) { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + name, value, _ := strings.Cut(line, "=") + want, ok := vars[name] + if !ok { + t.Errorf("install.sh of the %s writes %s to runplane.env, but runplane does not read it. Change install.sh and internal/config/config.go together.", role.name, name) + continue + } + agree(t, name+" in the runplane.env of the "+role.name, value, want, "install.sh", "internal/config/config.go") + } + } + + listen := installtest.Run(t, "ENV_FILE="+filepath.Join(t.TempDir(), "runplane.env")+"\nlisten_addr") + agree(t, "the dashboard address of install.sh without runplane.env", listen, def.Listen, "install.sh", "internal/config/config.go") + usage := match(t, installtest.Run(t, "usage"), `--listen .*\(default: (\S+)\)`) + agree(t, "the default --listen of install.sh --help", usage, def.Listen, "install.sh", "internal/config/config.go") + + // The script reads and changes runplane.env with sed. Its other + // variables are its own defaults. + own := map[string]bool{} + for _, m := range regexp.MustCompile(`(?m)^(RUNPLANE_\w+)=\$\{`).FindAllStringSubmatch(script, -1) { + own[m[1]] = true + } + names := regexp.MustCompile(`RUNPLANE_\w+`).FindAllString(script, -1) + slices.Sort(names) + for _, name := range slices.Compact(names) { + if _, ok := vars[name]; !ok && !own[name] { + t.Errorf("install.sh uses %s, which runplane does not read. Change install.sh and internal/config/config.go together.", name) + } + } + }) + + t.Run("systemd units", func(t *testing.T) { + unitDir := installVars(t, "UNIT_DIR")["UNIT_DIR"] + for _, role := range roles { + out := installtest.Run(t, "parse_flags --dry-run "+role.flags+"\nMODE=container\nwrite_units") + files := writtenFiles(out) + unit, ok := files[path.Join(unitDir, role.unit+".service")] + if !ok { + t.Errorf("install.sh of the %s writes no %s.service, the unit that the doctor and runplane update use. Change install.sh and cmd/runplane/doctor.go together:\n%s", role.name, role.unit, out) + continue + } + command := match(t, unit, `(?m)^ExecStart=\S+ (\S+)$`) + if cmd, ok := commands[command]; !ok || reflect.ValueOf(cmd.run).Pointer() != reflect.ValueOf(role.run).Pointer() { + t.Errorf("%s.service of install.sh runs the command %q of runplane, which is not the one of its role. Change install.sh and cmd/runplane/main.go together.", role.unit, command) + } + // After an update, serve and agent exit with status 0, and only + // systemd starts the new binary. + if !slices.Contains(strings.Split(unit, "\n"), "Restart=always") { + t.Errorf("%s.service of install.sh has no Restart=always, so an update stops runplane. Change install.sh, cmd/runplane/primary.go, and cmd/runplane/agent.go together.", role.unit) + } + if _, ok := files[path.Join(unitDir, firecracker.Slice)]; !ok { + t.Errorf("install.sh of the %s writes no %s. Change install.sh and internal/sandbox/firecracker/cgroup.go together.", role.name, firecracker.Slice) + } + } + agree(t, "the slice of sysbox", sysbox.Slice, firecracker.Slice, "internal/sandbox/sysbox/driver.go", "internal/sandbox/firecracker/cgroup.go", "install.sh") + for _, slice := range regexp.MustCompile(`[\w-]+\.slice\b`).FindAllString(script, -1) { + agree(t, "a slice of install.sh", slice, firecracker.Slice, "install.sh", "internal/sandbox/firecracker/cgroup.go") + } + + units := []string{primaryUnit + ".service", agentUnit + ".service"} + agree(t, "the units that install.sh --uninstall stops", strings.Fields(match(t, script, `for unit in (.+); do`)), units, "install.sh", "cmd/runplane/doctor.go") + known := append(units, firecracker.Slice) + for _, m := range regexp.MustCompile(`\$UNIT_DIR/([\w.-]+)`).FindAllStringSubmatch(script, -1) { + if !slices.Contains(known, m[1]) { + t.Errorf("install.sh names the unit %s, which runplane does not have. Change install.sh and cmd/runplane/doctor.go together.", m[1]) + } + } + for _, m := range regexp.MustCompile(`\$BIN"? ([a-z][a-z-]*)`).FindAllStringSubmatch(script, -1) { + if _, ok := commands[m[1]]; !ok { + t.Errorf("install.sh runs runplane %s, which is not a command. Change install.sh and cmd/runplane/main.go together.", m[1]) + } + } + }) + + // install.sh --uninstall removes what the sandboxes leave on the host. + t.Run("uninstall", func(t *testing.T) { + agree(t, "the container label of install.sh --uninstall", match(t, script, `docker ps -aq --filter label=(\S+)`), sysbox.LabelSandbox, "install.sh", "internal/sandbox/sysbox/driver.go") + agree(t, "the network label of install.sh --uninstall", match(t, script, `docker network ls -q --filter label=(\S+)`), guest.LabelManaged+"=true", "install.sh", "internal/sandbox/guest/layout.go") + agree(t, "the TAP devices of install.sh --uninstall", match(t, script, `\\\(([^\\[]+)\[0-9\]\*\\\)`), netrules.TapPrefix, "install.sh", "internal/sandbox/netrules/netrules.go") + tables := regexp.MustCompile(`"\w+ (\w+)"`).FindAllStringSubmatch(match(t, script, `for table in (.+); do`), -1) + if len(tables) == 0 { + t.Error("install.sh --uninstall removes no nftables table") + } + for _, m := range tables { + agree(t, "an nftables table of install.sh --uninstall", m[1], netrules.TableName, "install.sh", "internal/sandbox/netrules/netrules.go") + } + agree(t, "the iptables comment of install.sh --uninstall", match(t, script, `--comment (\S+) `), netrules.RuleComment, "install.sh", "internal/sandbox/netrules/netrules.go") + agree(t, "the iptables chains of install.sh --uninstall", strings.Fields(match(t, script, `for chain in (.+); do`)), netrules.RuleChains, "install.sh", "internal/sandbox/netrules/netrules.go") + }) + + t.Run("release files", func(t *testing.T) { + files := []string{"install.sh", "Makefile", ".github/workflows/release.yml", "internal/update/update.go"} + out := installtest.Run(t, "parse_flags --dry-run --version v1.2.3\nARCH=arm64\nTMP=/tmp\nfetch_runplane") + archive := update.ArchiveName("v1.2.3", "arm64") + if !strings.Contains(out, "/releases/download/v1.2.3/"+archive+"\n") { + t.Errorf("install.sh does not download %s. Change %s together:\n%s", archive, list(files), out) + } + makefile := repoFile(t, "Makefile") + dist := strings.NewReplacer("$(VERSION:v%=%)", "1.2.3", "$$arch", "arm64").Replace(match(t, makefile, `-czf dist/(\S+) `)) + agree(t, "the archive of make dist", dist, archive, files...) + release := repoFile(t, ".github/workflows/release.yml") + checked := sh(t, `GITHUB_REF_NAME=v1.2.3; printf '%s' "`+match(t, release, `tar -xzf "dist/([^"]+)"`)+`"`) + agree(t, "the archive that release.yml checks", checked, update.ArchiveName("v1.2.3", "amd64"), files...) + + kernels := map[string]string{"amd64": update.KernelName("amd64"), "arm64": update.KernelName("arm64")} + installed := map[string]string{} + for _, m := range regexp.MustCompile(`(?m)^\t(\w+)\) fc_arch=\S+ kernel_asset=(\S+) ;;$`).FindAllStringSubmatch(script, -1) { + installed[m[1]] = m[2] + } + agree(t, "the guest kernels of install.sh", installed, kernels, "install.sh", "internal/update/update.go") + var built []string + for _, m := range regexp.MustCompile(`(?m)^\t\w+\) .*\bname=(\S+) ;;$`).FindAllStringSubmatch(repoFile(t, "kernel/build.sh"), -1) { + built = append(built, m[1]) + } + slices.Sort(built) + want := slices.Sorted(maps.Values(kernels)) + agree(t, "the guest kernels of kernel/build.sh", built, want, "kernel/build.sh", "internal/update/update.go") + var released []string + for _, f := range strings.Fields(match(t, release, `cp ((?:kernel-out/\S+ )+)dist/`)) { + released = append(released, path.Base(f)) + } + slices.Sort(released) + agree(t, "the guest kernels of release.yml", released, want, ".github/workflows/release.yml", "internal/update/update.go") + }) + + t.Run("release build", func(t *testing.T) { + makefile := repoFile(t, "Makefile") + pkg := match(t, makefile, `(?m)^PKG\s*:=\s*(\S+)$`) + agree(t, "PKG of the Makefile", pkg, match(t, repoFile(t, "go.mod"), `(?m)^module (\S+)$`), "Makefile", "go.mod") + // Go ignores -X for a variable that does not exist. + flags := regexp.MustCompile(`-X \$\(PKG\)/(\S+)\.(\w+)=`).FindAllStringSubmatch(makefile, -1) + if len(flags) == 0 { + t.Error("the Makefile sets no variable with -X") + } + for _, m := range flags { + if !declaresVar(t, filepath.Join("..", "..", m[1]), m[2]) { + t.Errorf("the Makefile sets %s.%s with -X, but %s has no such variable. Change the Makefile and %s together.", m[1], m[2], m[1], m[1]) + } + } + agree(t, "REPO of the Makefile", match(t, makefile, `(?m)^REPO\s*\?=\s*(\S+)$`), version.Repo, "Makefile", "internal/version/version.go") + // A binary without the embedded runplane-init starts no sandbox. + agree(t, "the runplane-init of make init", match(t, makefile, `(?m)^\tgzip .*> (\S+)$`), path.Join("internal/initbin", initbin.File), "Makefile", "internal/initbin/initbin.go") + + // make dist puts the values of the release into these lines of + // install.sh, and release.yml checks them. + var set, checked []string + for _, m := range regexp.MustCompile(`'s\|\^(RUNPLANE_\w+)=\.\*\|`).FindAllStringSubmatch(makefile, -1) { + set = append(set, m[1]) + lines := regexp.MustCompile(`(?m)^`+m[1]+`=.*$`).FindAllString(script, -1) + if len(lines) != 1 || !strings.HasPrefix(lines[0], m[1]+"=${"+m[1]+":-") { + t.Errorf("install.sh has the lines %q, want one line %s=${%s:-...}. Change install.sh and the Makefile together.", lines, m[1], m[1]) + } + } + for _, m := range regexp.MustCompile(`grep -qxF "(RUNPLANE_\w+)=`).FindAllStringSubmatch(repoFile(t, ".github/workflows/release.yml"), -1) { + checked = append(checked, m[1]) + } + agree(t, "the install.sh lines that release.yml checks", checked, set, ".github/workflows/release.yml", "Makefile") + }) + + t.Run("base image", func(t *testing.T) { + ref := images.Ref("Acme/Fork", images.Default) + name, tag, _ := strings.Cut(ref, ":") + registry, repository, _ := strings.Cut(name, "/") + out := installtest.Run(t, `REPO=Acme/Fork +curl() { + for a; do case $a in https://*/manifests/*) printf 'manifest %s\n' "$a" ;; esac; done + printf '{"token":"t"}\n' +} +check_base_image`) + agree(t, "the base image of install.sh", match(t, out, `Base image: (\S+)`), ref, "install.sh", "images/images.go") + agree(t, "the manifest that install.sh reads", match(t, out, `manifest (\S+)`), "https://"+registry+"/v2/"+repository+"/manifests/"+tag, "install.sh", "images/images.go") + + yml := repoFile(t, ".github/workflows/images.yml") + expand := strings.NewReplacer("${GITHUB_REPOSITORY,,}", "acme/fork", "${{ matrix.image.name }}", images.Default, "${{ matrix.image }}", images.Default) + var pushed []string + for line := range strings.Lines(yml) { + if !strings.HasPrefix(strings.TrimSpace(line), "#") { + pushed = append(pushed, regexp.MustCompile(`ghcr\.io/[^"]+`).FindAllString(line, -1)...) + } + } + if len(pushed) == 0 { + t.Error("images.yml pushes no image to ghcr.io") + } + for _, p := range pushed { + agree(t, "an image of images.yml", expand.Replace(p), name, ".github/workflows/images.yml", "images/images.go") + } + if !strings.Contains(yml, `"$image:`+tag+`"`) { + t.Errorf("images.yml does not push the tag %s. Change .github/workflows/images.yml and images/images.go together.", tag) + } + }) + + t.Run("firecracker", func(t *testing.T) { + installed := map[string]string{} + for _, m := range regexp.MustCompile(`install -m 0755 "\$dir/(\w+)-\$fc_tag-\$fc_arch" (\S+)`).FindAllStringSubmatch(script, -1) { + installed[m[1]] = m[2] + } + agree(t, "the programs that install.sh installs", installed, map[string]string{"firecracker": firecracker.Binary, "jailer": firecracker.JailerBinary}, + "install.sh", "internal/sandbox/firecracker/doctor.go") + agree(t, "the firecracker whose version install.sh reads", match(t, script, `\$\((\S*firecracker) --version`), firecracker.Binary, "install.sh", "internal/sandbox/firecracker/doctor.go") + agree(t, "the jailer that install.sh looks for", match(t, script, `\[ -x (/\S+) \]`), firecracker.JailerBinary, "install.sh", "internal/sandbox/firecracker/doctor.go") + // The guest kernel uses the guest config of this Firecracker release. + agree(t, "RUNPLANE_FIRECRACKER_VERSION of install.sh", installVars(t, "RUNPLANE_FIRECRACKER_VERSION")["RUNPLANE_FIRECRACKER_VERSION"], + match(t, repoFile(t, "kernel/build.sh"), `(?m)^FIRECRACKER_REF=\$\{FIRECRACKER_REF:-(\S+)\}$`), "install.sh", "kernel/build.sh") + }) + + // The add-server dialog names the systems of runplane.Systems: the + // systems that select_mode names. install.sh takes them without a + // warning, or with one for a best-effort system. + t.Run("systems", func(t *testing.T) { + system := func(id, version string, bestEffort bool) string { + if bestEffort { + return id + "-" + version + " (best effort)" + } + return id + "-" + version + } + var named, want []string + block := match(t, script, `(?s)\n\tcase \$OS_ID-\$OS_VERSION in\n(.*?)\n\tesac\n`) + for _, m := range regexp.MustCompile(`(?m)^\t([^\t)]+)\)`).FindAllStringSubmatch(block, -1) { + for p := range strings.SplitSeq(m[1], "|") { + p = strings.TrimSpace(p) + if strings.Contains(p, "*") { + continue + } + id, version, _ := strings.Cut(p, "-") + out := installtest.Run(t, "have_kvm() { return 0; }\nMODE=vm\nOS_ID="+id+"\nOS_VERSION="+version+"\nselect_mode") + named = append(named, system(id, version, strings.Contains(out, "best effort"))) + } + } + for _, s := range runplane.Systems { + want = append(want, system(s.ID, s.Version, s.BestEffort)) + } + slices.Sort(named) + slices.Sort(want) + agree(t, "the systems that install.sh names", named, want, "install.sh", "install.go") + }) + + // An install over SSH shows the latest step of install.sh. + t.Run("step lines", func(t *testing.T) { + out := strings.TrimSpace(installtest.Run(t, "step 'Write the configuration'")) + if text, ok := strings.CutPrefix(out, fleet.StepPrefix); !ok || text != "Write the configuration" { + t.Errorf("install.sh shows the step %q, which does not start with %q. Change install.sh and internal/fleet/install.go together.", out, fleet.StepPrefix) + } + }) + + t.Run("dashboard status", func(t *testing.T) { + status := func(s api.Status) string { + b, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + return string(b) + } + out := installtest.Run(t, "ENV_FILE="+filepath.Join(t.TempDir(), "runplane.env")+"\ncurl() { printf '%s\\n' '"+status(api.Status{Mode: "vm"})+"'; }\nrunning_mode") + agree(t, "the mode that install.sh reads from /api/status", strings.TrimSpace(out), "vm", "install.sh", "internal/api/setup.go") + pattern := match(t, script, `grep -q '([^']*needs_setup[^']*)'`) + for _, needs := range []bool{true, false} { + out := sh(t, "if printf '%s\\n' '"+status(api.Status{NeedsSetup: needs})+"' | grep -q '"+pattern+"'; then echo true; else echo false; fi") + agree(t, "needs_setup that install.sh reads from /api/status", strings.TrimSpace(out), strconv.FormatBool(needs), "install.sh", "internal/api/setup.go") + } + }) +} + +// install.sh stops a running primary before the first command that opens +// the database, because the new binary applies the migrations of its +// release, and the old serve must not operate on the new schema. The stop +// also comes before the new unit file, because systemctl warns about a +// changed unit file until the start reloads the units. The start at the end +// starts the primary again, and it makes the setup code before the first +// start. In this test, the steps that need the host or the network do +// nothing. +func TestInstallStopsThePrimaryBeforeTheDatabaseCommands(t *testing.T) { + bin := installVars(t, "BIN")["BIN"] + for _, tt := range []struct { + name string + running bool + want []string + }{ + {"update of a running primary", true, []string{ + "+ systemctl stop runplane.service", + "+ write runplane.service (mode 0644)", + "+ " + bin + " set-mode container", + "+ " + bin + " set-domain ci.example.com", + "+ " + bin + " setup-code", + "+ systemctl restart runplane.service", + }}, + {"new install", false, []string{ + "+ write runplane.service (mode 0644)", + "+ " + bin + " set-mode container", + "+ " + bin + " set-domain ci.example.com", + "+ " + bin + " setup-code", + "+ systemctl restart runplane.service", + }}, + } { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "runplane.db"), nil, 0o600); err != nil { + t.Fatal(err) + } + active := "return 3" + if tt.running { + active = `[ "$*" = "is-active --quiet runplane.service" ]` + } + out := installtest.Run(t, "DATA_DIR="+dir+"\nENV_FILE="+dir+"/runplane.env\nAGENT_FILE="+dir+"/agent.json\nUNIT_DIR="+dir+` +check_host() { :; } +select_mode() { :; } +fetch_runplane() { :; } +setup_container() { :; } +finish() { :; } +systemctl() { `+active+`; } +main --dry-run --mode container --domain ci.example.com`) + var got []string + for line := range strings.Lines(out) { + line = strings.ReplaceAll(strings.TrimSuffix(line, "\n"), dir+"/", "") + if strings.HasPrefix(line, "+ systemctl stop ") || strings.HasPrefix(line, "+ systemctl restart ") || strings.HasPrefix(line, "+ "+bin+" ") || + line == "+ write runplane.service (mode 0644)" { + got = append(got, line) + } + } + if !slices.Equal(got, tt.want) { + t.Errorf("install.sh runs\n%s\nwant\n%s\nfull output:\n%s", strings.Join(got, "\n"), strings.Join(tt.want, "\n"), out) + } + }) + } +} + +// agree reports a copy that differs from the value that it must have. +func agree(t *testing.T, what string, got, want any, files ...string) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: %q, want %q. Change %s together.", what, got, want, list(files)) + } +} + +func list(files []string) string { + if len(files) < 3 { + return strings.Join(files, " and ") + } + return strings.Join(files[:len(files)-1], ", ") + ", and " + files[len(files)-1] +} + +func match(t *testing.T, s, expr string) string { + t.Helper() + m := regexp.MustCompile(expr).FindStringSubmatch(s) + if m == nil { + t.Fatalf("no match of %s", expr) + } + return m[1] +} + +func repoFile(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func sh(t *testing.T, script string) string { + t.Helper() + out, err := installtest.Sh([]byte(script), nil, "-s") + if err != nil { + t.Fatalf("sh: %v\n%s", err, out) + } + return out +} + +func installVars(t *testing.T, names ...string) map[string]string { + t.Helper() + body := "printf '%s\\n'" + for _, n := range names { + body += ` "$` + n + `"` + } + lines := strings.Split(strings.TrimSuffix(installtest.Run(t, body), "\n"), "\n") + if len(lines) != len(names) { + t.Fatalf("values of %v: %q", names, lines) + } + vars := map[string]string{} + for i, n := range names { + vars[n] = lines[i] + } + return vars +} + +// writtenFiles returns the files that a dry run of install.sh shows, by +// path. +func writtenFiles(out string) map[string]string { + files := map[string]string{} + var file string + for line := range strings.Lines(out) { + if p, ok := strings.CutPrefix(line, "+ write "); ok { + file, _, _ = strings.Cut(p, " (mode ") + files[file] = "" + } else if l, ok := strings.CutPrefix(line, " | "); ok && file != "" { + files[file] += l + } else { + file = "" + } + } + return files +} + +// configVars returns the RUNPLANE_* variables of the configuration, as the +// help of each flag names them, with the value of the flag in args or the +// default. +func configVars(t *testing.T, args ...string) map[string]string { + t.Helper() + // Parse reads no runplane.env of this machine. + t.Setenv("RUNPLANE_CONFIG_DIR", t.TempDir()) + fs := flag.NewFlagSet("serve", flag.ContinueOnError) + if _, err := config.Parse(fs, args, config.Listen, config.AgentListen, config.ServicePort, config.DataDir, config.Mode, config.Dev, config.DevInsecure, config.SSHListen); err != nil { + t.Fatal(err) + } + name := regexp.MustCompile(`\((RUNPLANE_\w+)\)$`) + vars := map[string]string{} + fs.VisitAll(func(f *flag.Flag) { + if m := name.FindStringSubmatch(f.Usage); m != nil { + vars[m[1]] = f.Value.String() + } + }) + return vars +} + +// declaresVar reports if the Go package in dir has the package variable +// name. +func declaresVar(t *testing.T, dir, name string) bool { + t.Helper() + files, err := filepath.Glob(filepath.Join(dir, "*.go")) + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + f, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.SkipObjectResolution) + if err != nil { + t.Fatal(err) + } + for _, d := range f.Decls { + if d, ok := d.(*ast.GenDecl); ok && d.Tok == token.VAR { + for _, s := range d.Specs { + if slices.ContainsFunc(s.(*ast.ValueSpec).Names, func(n *ast.Ident) bool { return n.Name == name }) { + return true + } + } + } + } + } + return false +} diff --git a/cmd/runplane/local.go b/cmd/runplane/local.go index a10a758..a182304 100644 --- a/cmd/runplane/local.go +++ b/cmd/runplane/local.go @@ -13,35 +13,37 @@ import ( "github.com/moby/moby/client" - "github.com/runplane/runplane/images" "github.com/runplane/runplane/internal/agent" - "github.com/runplane/runplane/internal/cache" "github.com/runplane/runplane/internal/config" - "github.com/runplane/runplane/internal/host" + "github.com/runplane/runplane/internal/doctor" "github.com/runplane/runplane/internal/image" "github.com/runplane/runplane/internal/initbin" + "github.com/runplane/runplane/internal/localnode" "github.com/runplane/runplane/internal/mirror" + "github.com/runplane/runplane/internal/runner" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/sandbox/devdocker" "github.com/runplane/runplane/internal/sandbox/firecracker" "github.com/runplane/runplane/internal/sandbox/netrules" "github.com/runplane/runplane/internal/sandbox/sysbox" "github.com/runplane/runplane/internal/update" + "github.com/runplane/runplane/internal/version" ) // local is the server that this process operates: its sandbox driver, and // the node that the fleet or the agent protocol uses. type local struct { - node *node + node *localnode.Node driver sandbox.Driver close func() } type localConfig struct { mode sandbox.Mode - settings sandbox.Settings // the first driver settings - unit string // systemd unit of this process, for the doctor - listen []string // TCP addresses of this process, for the doctor + settings agent.Settings // the first settings of the driver and the node + // settingsFile keeps the settings of each change on an agent, or is "". + settingsFile string + doctor doctor.Options // token gives a GitHub token for the runner release reads. Optional. token func(context.Context) (string, error) } @@ -51,7 +53,10 @@ func newLocal(ctx context.Context, cfg config.Config, lc localConfig, log *slog. if err != nil { return nil, err } - l := &local{node: &node{}, close: func() {}} + l := &local{close: func() {}} + // allow takes the rule that tells if an address is a sandbox, and gives + // the rule of the service port in the mode. + allow := func(isSandbox func(netip.Addr) bool) func(netip.Addr) bool { return isSandbox } defer func() { if err != nil { l.close() @@ -61,74 +66,82 @@ func newLocal(ctx context.Context, cfg config.Config, lc localConfig, log *slog. l.onClose(func() { docker.Close() }) } images, err := image.New(image.Config{ - Mode: lc.mode, ImagesDir: cfg.ImagesDir(), Docker: docker, Init: initbin.Bytes, Logger: log.With("component", "image"), + Mode: lc.mode, ImagesDir: cfg.ImagesDir(), Repo: version.Repo, Docker: docker, Init: initbin.Bytes, + Logger: log.With("component", "image"), }) if err != nil { return nil, fmt.Errorf("start the image manager: %w", err) } - bundles := image.NewBundles(image.BundleConfig{ - Mode: lc.mode, Dir: cfg.RunnersDir(), Token: lc.token, Logger: log.With("component", "runners"), + bundles := runner.NewBundles(runner.BundleConfig{ + Mode: lc.mode, Dir: cfg.RunnersDir(), Releases: runner.Releases{Auth: runner.TokenAuth(lc.token)}, + Logger: log.With("component", "runners"), }) - var probe func(context.Context) error + containers := sysbox.Options{ + Docker: docker, DataDir: cfg.DataDir, Sandboxes: cfg.SandboxesDir(), ScratchTemplates: cfg.ScratchTemplatesDir(), + ServicePort: cfg.ServicePort, InitPath: initbin.Path, Settings: lc.settings.Settings, + Runner: func(ctx context.Context, version, arch string) (string, bool, error) { + b, err := bundles.Ensure(ctx, version, arch) + return b.Dir, b.Patched, err + }, + } switch lc.mode { case sandbox.ModeContainer: - rules := netrules.New(log.With("component", "netrules")) + rules := netrules.New(log.With("component", "netrules"), netip.Prefix{}) l.onClose(func() { rules.Close() }) - drv, err := sysbox.New(sysbox.Options{ - Docker: docker, Config: cfg, Mode: sandbox.ModeContainer, Runtime: "sysbox-runc", Scratch: sysbox.ScratchLoop, - Bundle: bundles.Ensure, InitPath: initbin.Path, NetRules: rules, Settings: lc.settings, - Logger: log.With("driver", "sysbox"), - }) + containers.Profile, containers.NetRules, containers.Logger = sysbox.Container, rules, log.With("driver", "sysbox") + drv, err := sysbox.New(containers) if err != nil { return nil, fmt.Errorf("start the container driver: %w", err) } - l.driver, probe = drv, drv.Probe + l.driver = drv case sandbox.ModeDev: - // Docker sends the requests of all sandboxes from one address on - // macOS, so the cache server cannot tell the sandboxes apart. - log.Warn("dev mode: sandboxes get no Actions cache, because the cache server cannot tell them apart by address") - l.node.noCache = true - lc.settings.ActionsCache = false - drv, err := devdocker.New(devdocker.Options{ - Docker: docker, Config: cfg, Bundle: bundles.Ensure, InitPath: initbin.Path, Settings: lc.settings, - Logger: log.With("driver", "devdocker"), - }) + containers.Logger = log.With("driver", "devdocker") + drv, err := devdocker.New(cfg.DevInsecure, containers) if err != nil { return nil, fmt.Errorf("start the dev driver: %w", err) } - l.driver, probe = drv, drv.Probe + l.driver, allow = drv, devdocker.Allow case sandbox.ModeVM: + rules := netrules.New(log.With("component", "netrules"), netrules.DefaultTapSubnet) + l.onClose(func() { rules.Close() }) drv, err := firecracker.New(firecracker.Options{ - DataDir: cfg.DataDir, Kernel: cfg.KernelPath(), ServicePort: cfg.ServicePort, RootDisk: images.RootDisk, - RunnerDrive: func(ctx context.Context, version, arch string) (string, error) { + DataDir: cfg.DataDir, Sandboxes: cfg.SandboxesDir(), ScratchTemplates: cfg.ScratchTemplatesDir(), JailDir: cfg.JailDir(), + Kernel: cfg.KernelPath(), ServicePort: cfg.ServicePort, RootDisk: images.RootDisk, + RunnerDrive: func(ctx context.Context, version, arch string) (string, bool, error) { b, err := bundles.Ensure(ctx, version, arch) - return b.Ext4Path, err + return b.Ext4Path, b.Patched, err }, - CacheEnv: cache.RunnerEnv, Settings: lc.settings, Logger: log.With("driver", "firecracker"), + NetRules: rules, Settings: lc.settings.Settings, Logger: log.With("driver", "firecracker"), }) if err != nil { return nil, fmt.Errorf("start the VM driver: %w", err) } - l.driver, probe = drv, drv.Probe + l.driver = drv default: return nil, fmt.Errorf("sandbox mode %q: set RUNPLANE_MODE to container, vm, or dev", lc.mode) } // The driver closes before the netrules manager and Docker. l.onClose(func() { l.driver.Close() }) - l.node.LocalNode, err = agent.NewLocal(agent.LocalConfig{ + isSandbox := func(ip netip.Addr) bool { + list, err := l.driver.List(context.Background()) + _, ok := sandbox.FindByIP(list, ip) + return err == nil && ok + } + l.node, err = localnode.New(localnode.Config{ Driver: l.driver, Images: images, Bundles: bundles, Updater: update.New(cfg.KernelPath()), - Doctor: host.Options{ - DataDir: cfg.DataDir, GuestKernel: cfg.KernelPath(), Listen: lc.listen, Serving: true, - MinFreeDiskPercent: float64(lc.settings.MinFreeDiskPercent), Unit: lc.unit, Internet: probe, - BaseImage: baseImage, - }, - Logger: log.With("component", "node"), + MinFreeDiskPercent: lc.settings.MinFreeDiskPercent, + SettingsFile: lc.settingsFile, + Mirror: mirror.Config{Dir: cfg.MirrorDir(), Allow: allow(isSandbox), Logger: log.With("component", "mirror")}, + Doctor: lc.doctor, + Logger: log.With("component", "node"), }) if err != nil { return nil, err } + // The registry mirror stops after the service port and before the driver. + l.onClose(func() { l.node.Close() }) return l, nil } @@ -158,46 +171,13 @@ func dockerClient(ctx context.Context, mode sandbox.Mode) (*client.Client, error return docker, nil } -// baseImage tries to pull the default builtin image, for the doctor. -func baseImage(ctx context.Context) (string, error) { - ref := image.BuiltinRef(images.Default) - ctx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - return ref, image.CanPull(ctx, ref) -} - -// node is the LocalNode of this process. In dev mode it keeps the Actions -// cache off for the sandboxes. -type node struct { - *agent.LocalNode - noCache bool -} - -func (n *node) Configure(ctx context.Context, s sandbox.Settings) error { - if n.noCache { - s.ActionsCache = false - } - return n.LocalNode.Configure(ctx, s) -} - -// serveServicePort starts the registry mirror and serves the service port -// of the sandboxes on all addresses: /v2/ goes to the mirror, all other -// paths to the Actions cache. allow tells if an address is a sandbox. The -// returned function stops the listener and the mirror. -func serveServicePort(cfg config.Config, mode sandbox.Mode, allow func(netip.Addr) bool, cache http.Handler, log *slog.Logger) (*mirror.Mirror, func(), error) { - if mode == sandbox.ModeDev { - isSandbox := allow - // Docker sends the requests of dev sandboxes from the loopback address. - allow = func(ip netip.Addr) bool { return ip.IsLoopback() || isSandbox(ip) } - } - mir, err := mirror.New(mirror.Config{Dir: cfg.MirrorDir(), MaxBytes: mirrorLimit(cfg.DataDir), Allow: allow, Logger: log.With("component", "mirror")}) - if err != nil { - return nil, nil, fmt.Errorf("start the registry mirror: %w", err) - } +// serveServicePort serves the service port of the sandboxes on all +// addresses: /v2/ goes to the registry mirror, all other paths to the +// Actions cache. The returned function stops the listener. +func serveServicePort(cfg config.Config, mir, cache http.Handler, log *slog.Logger) (func(), error) { ln, err := net.Listen("tcp", servicePortAddr(cfg)) if err != nil { - mir.Close() - return nil, nil, fmt.Errorf("listen on the service port: %w", err) + return nil, fmt.Errorf("listen on the service port: %w", err) } srv := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -219,25 +199,14 @@ func serveServicePort(cfg config.Config, mode sandbox.Mode, allow func(netip.Add log.Error("service port", "err", err) } }() - return mir, func() { + return func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := srv.Shutdown(ctx); err != nil { srv.Close() } <-done - mir.Close() }, nil } func servicePortAddr(cfg config.Config) string { return fmt.Sprintf(":%d", cfg.ServicePort) } - -// mirrorLimit is the size limit of the registry mirror: a tenth of the data -// disk, from 2 to 50 GiB. -func mirrorLimit(dataDir string) int64 { - res, err := host.ReadResources(dataDir) - if err != nil { - return 10 << 30 - } - return min(max(res.DiskBytes/10, 2<<30), 50<<30) -} diff --git a/cmd/runplane/main.go b/cmd/runplane/main.go index 782256e..fb7143f 100644 --- a/cmd/runplane/main.go +++ b/cmd/runplane/main.go @@ -15,7 +15,6 @@ import ( "syscall" "github.com/runplane/runplane/internal/config" - "github.com/runplane/runplane/internal/sandbox" ) type command struct { @@ -26,7 +25,7 @@ type command struct { var commands = map[string]command{ "serve": {"run the primary: dashboard, API, and sandboxes", serve}, "agent": {"run an agent: operate sandboxes for a primary (agent.json)", runAgent}, - "doctor": {"check this server for runplane", doctor}, + "doctor": {"check this server for runplane", runDoctor}, "update": {"install the latest release (-check: only report it)", updateCmd}, "setup-code": {"print a new one-time setup code for the first admin", setupCode}, "reset-password": {"set a new random password for a user: reset-password ", resetPassword}, @@ -74,20 +73,6 @@ func usage(w io.Writer) { fmt.Fprintln(w, "\nRun runplane -h for the flags of a command.") } -var errDevInsecure = errors.New("mode dev runs jobs in privileged containers; set RUNPLANE_DEV_INSECURE=1 to allow it on a development machine") - -// checkMode refuses to operate sandboxes without a mode, or in dev mode -// without RUNPLANE_DEV_INSECURE. -func checkMode(mode sandbox.Mode, devInsecure bool) error { - switch { - case mode == "": - return errors.New("no sandbox mode: set RUNPLANE_MODE to container, vm, or dev") - case mode == sandbox.ModeDev && !devInsecure: - return errDevInsecure - } - return nil -} - // newLogger returns the logger of a process that stops when ctx ends. func newLogger(ctx context.Context, cfg config.Config) *slog.Logger { level := slog.LevelInfo diff --git a/cmd/runplane/primary.go b/cmd/runplane/primary.go index 7b87b68..7ff607f 100644 --- a/cmd/runplane/primary.go +++ b/cmd/runplane/primary.go @@ -11,20 +11,25 @@ import ( "log/slog" "net" "net/http" - "sync" "sync/atomic" "time" "github.com/runplane/runplane/internal/agent" "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/backup" "github.com/runplane/runplane/internal/cache" "github.com/runplane/runplane/internal/config" + "github.com/runplane/runplane/internal/datadisk" "github.com/runplane/runplane/internal/events" "github.com/runplane/runplane/internal/fleet" "github.com/runplane/runplane/internal/github" "github.com/runplane/runplane/internal/host" + "github.com/runplane/runplane/internal/joblog" "github.com/runplane/runplane/internal/notify" + "github.com/runplane/runplane/internal/pools" + "github.com/runplane/runplane/internal/runner/versions" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/scheduler" "github.com/runplane/runplane/internal/secret" @@ -39,7 +44,8 @@ import ( var errRestart = errors.New("restart") func serve(ctx context.Context, args []string) error { - cfg, err := config.Parse(flag.NewFlagSet("serve", flag.ContinueOnError), args) + cfg, err := config.Parse(flag.NewFlagSet("serve", flag.ContinueOnError), args, config.Listen, config.AgentListen, + config.ServicePort, config.DataDir, config.Mode, config.Dev, config.DevInsecure, config.SSHListen) if err != nil { return err } @@ -64,24 +70,27 @@ func serve(ctx context.Context, args []string) error { if err != nil { return err } - mode, err := initMode(ctx, st, cfg) + bus := &events.Bus{} + settingsService := newSettings(st, bus, cfg, log.With("component", "settings")) + mode, err := settingsService.InitMode(ctx) if err != nil { return err } - if err := checkMode(mode, cfg.DevInsecure); err != nil { - return err - } if cfg.Mode != "" && cfg.Mode != mode { log.Warn("the mode in the dashboard settings overrides RUNPLANE_MODE", "settings", mode, "env", cfg.Mode) } - deps := api.Deps{Store: st, Auth: auth.NewService(st, subkey(key, "runplane device tokens")), Bus: &events.Bus{}, Box: box, Log: log, Config: cfg} + deps := api.Deps{ + Store: st, Auth: auth.NewService(st, subkey(key, "runplane device tokens"), log.With("component", "auth")), + Settings: settingsService, Pools: pools.New(pools.Config{Store: st, Bus: bus, Arch: host.Arch(), Log: log.With("component", "pools")}), + Bus: bus, Box: box, Log: log, Config: cfg, + } p, err := startPrimary(&deps, key, mode, end) if err != nil { return fmt.Errorf("start services: %w", err) } defer p.stop() - if err := logSetupCode(ctx, st, deps.Auth, log); err != nil { + if err := logSetupCode(ctx, deps.Auth, log); err != nil { return err } @@ -122,48 +131,20 @@ func serve(ctx context.Context, args []string) error { return nil } -// initMode returns the sandbox mode. The settings keep the mode after the -// first start, so a change in the dashboard outlives the environment of the -// install. At the first start, the mode comes from the configuration. -func initMode(ctx context.Context, st *store.Store, cfg config.Config) (sandbox.Mode, error) { - settings, err := st.GetSettings(ctx) - if err != nil { - return "", err - } - if settings.Mode != "" || cfg.Mode == "" { - return settings.Mode, nil - } - settings, err = st.UpdateSettings(ctx, func(s *store.Settings) error { - s.Mode = cfg.Mode +// logSetupCode logs a setup code while no admin exists, so the admin can +// find it in the service log. +func logSetupCode(ctx context.Context, a *auth.Service, log *slog.Logger) error { + code, err := a.EnsureSetupCode(ctx) + switch { + case errors.Is(err, auth.ErrSetupDone): return nil - }) - return settings.Mode, err -} - -// logSetupCode logs a new setup code while no admin exists, so the admin -// can find it in the service log. It keeps a code that is still valid, -// because install.sh printed it. -func logSetupCode(ctx context.Context, st *store.Store, a *auth.Service, log *slog.Logger) error { - n, err := st.CountUsers(ctx) - if err != nil || n > 0 { - return err - } - open, err := st.HasOpenSetupCode(ctx, time.Now()) - if err != nil { + case err != nil: return err - } - if open { + case code == "": log.Info("no admin exists: open the dashboard and enter the setup code. Run 'runplane setup-code' for a new setup code.") - return nil - } - code, err := a.NewSetupCode(ctx) - if errors.Is(err, auth.ErrSetupDone) { - return nil - } - if err != nil { - return err + default: + log.Info("no admin exists: open the dashboard and enter the setup code", "code", code, "valid", auth.SetupCodeTTL) } - log.Info("no admin exists: open the dashboard and enter the setup code", "code", code, "valid", auth.SetupCodeTTL) return nil } @@ -180,8 +161,7 @@ type primary struct { acme *tlsutil.ACME handler http.Handler // the dashboard, for HTTPS - wg sync.WaitGroup // work that no stop function ends - stops []func() // stop runs them in reverse order + stops []func() // stop runs them in reverse order } // startPrimary starts the services of the primary and sets them in d. They @@ -201,12 +181,12 @@ func startPrimary(d *api.Deps, key []byte, mode sandbox.Mode, end func(error)) ( return nil, err } p.settings.Store(&settings) + p.followSettings(d.Bus) // Release reads with an installation token have a higher rate limit. token := func(ctx context.Context) (string, error) { return p.gh.Token(ctx) } local, err := newLocal(context.Background(), cfg, localConfig{ - mode: mode, settings: fleet.DriverSettings(settings), unit: "runplane", token: token, - listen: primaryAddrs(cfg, settings.Domain), + mode: mode, settings: fleet.NodeSettings(settings), doctor: doctorOptions(cfg, mode, false, settings.Domain), token: token, }, log) if err != nil { return nil, err @@ -214,26 +194,40 @@ func startPrimary(d *api.Deps, key []byte, mode sandbox.Mode, end func(error)) ( p.stops = append(p.stops, local.close) p.run(func(ctx context.Context) { local.node.Run(ctx) }) - notifier := ¬ify.Dispatcher{Store: st, Box: d.Box, Log: log.With("component", "notify")} - p.run(func(ctx context.Context) { notifier.Run(ctx, d.Bus) }) + channels := notify.NewChannels(st, d.Box, log.With("component", "notify")) + notifier := notify.New(notify.Config{Store: st, Channels: channels, Bus: d.Bus, Log: log.With("component", "notify")}) + p.run(notifier.Run) cert, fingerprint, err := tlsutil.AgentCert(cfg.TLSDir()) if err != nil { return nil, fmt.Errorf("make the agent certificate: %w", err) } + caches, err := p.serveCaches(key, local) + if err != nil { + return nil, err + } + ghRef := &githubRef{} + runners := versions.New(versions.Config{ + Store: st, Bus: d.Bus, GitHub: ghRef, Notify: notifier.Notify, Log: log.With("component", "runners"), + InUse: func() []string { return p.fleet.RunnerVersions() }, + }) // The fleet calls the scheduler only after Run starts. p.fleet = fleet.New(fleet.Config{ Store: st, Bus: d.Bus, Log: log.With("component", "fleet"), Config: cfg, Local: local.node, - AgentFingerprint: fingerprint, OnEvent: p.onEvent, CapacityChanged: func() { p.sched.CapacityChanged() }, - Notifier: notifier, + AgentFingerprint: fingerprint, Version: version.Version, Repo: version.Repo, PublicKey: update.PublicKey, + OnEvent: func(serverID string, ev sandbox.Event) { p.sched.HandleEvent(serverID, ev) }, + CapacityChanged: func() { p.sched.CapacityChanged() }, Runners: runners, Pools: d.Pools, + }) + jobLogs := joblog.New(joblog.Config{ + Dir: cfg.LogsDir(), Store: st, Log: log.With("component", "joblog"), + Driver: func(serverID string) joblog.Driver { return p.fleet.Driver(serverID) }, }) - ghRef := &githubRef{} p.sched = scheduler.New(scheduler.Config{ - Store: st, Bus: d.Bus, Fleet: p.fleet, GitHub: ghRef, Log: log.With("component", "scheduler"), LogsDir: cfg.LogsDir(), + Store: st, Bus: d.Bus, Fleet: schedulerFleet{p.fleet}, GitHub: ghRef, Log: log.With("component", "scheduler"), + JobLogs: jobLogs, }) p.gh = github.New(github.Config{ Store: st, Box: d.Box, Bus: d.Bus, Log: log.With("component", "github"), Scheduler: p.sched, - Bundles: p.fleet, Notifier: notifier, }) ghRef.Service = p.gh // The scheduler stops after the fleet, so that no driver event comes @@ -245,51 +239,48 @@ func startPrimary(d *api.Deps, key []byte, mode sandbox.Mode, end func(error)) ( } }) p.run(p.gh.Run) + p.run(runners.Run) + p.pruneDaily("sessions, setup codes, and invites", d.Auth.Prune) + p.pruneDaily("server tokens and host samples", p.fleet.Prune) - c, err := p.serveCaches(key, mode, local.driver) - if err != nil { + if err := p.listenAgents(cert, caches.ServeAgent); err != nil { return nil, err } - if err := p.listenAgents(cert, c.srv.Proxied()); err != nil { - return nil, err - } - opener := shell.Drivers{Store: st, Driver: p.fleet.Driver} + opener := shell.Drivers{Store: st, Driver: func(serverID string) shell.Driver { return p.fleet.Driver(serverID) }} if err := p.serveSSH(opener); err != nil { // The debug shell in the browser still works. log.Error("no SSH access to debug shells", "err", err) } - notifyBg := func(e notify.Event) { - p.wg.Go(func() { - if err := notifier.Notify(context.Background(), e); err != nil { - log.Warn("send notification", "type", e.Type, "err", err) - } - }) - } - bk := &backups{st: st, box: d.Box, bus: d.Bus, dir: cfg.BackupsDir(), notify: notifyBg, log: log.With("component", "backup")} - p.run(bk.daily) - up := &updater{u: update.New(cfg.KernelPath()), restart: func() { end(errRestart) }, log: log.With("component", "update")} - p.run(func(ctx context.Context) { up.checkDaily(ctx, st, notifyBg) }) + backups := backup.New(backup.Config{ + Store: st, Box: d.Box, Bus: d.Bus, Dir: cfg.BackupsDir(), Log: log.With("component", "backup"), + }) + p.run(backups.Run) + updates := update.NewService(update.Config{ + Updater: update.New(cfg.KernelPath()), Restart: func() { end(errRestart) }, Store: st, Notify: notifier.Notify, + Log: log.With("component", "update"), + }) + p.run(updates.Run) p.stops = append(p.stops, func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() p.acme.Stop(ctx) }) - d.Scheduler = pools{p.sched, p.gh} + d.Scheduler = p.sched d.GitHub = p.gh d.Servers = p.fleet - d.Images = p.fleet.Images() - d.Caches = c - d.Backups = bk - d.Updater = up - d.Notifier = notifier - d.Logs = logs{st: st, cfg: cfg, driver: p.fleet.Driver} + d.Images = p.fleet + d.Caches = caches + d.Mirrors = p.fleet + d.Backups = backups + d.Updater = updates + d.Channels = channels + d.Logs = jobLogs d.Shell = opener d.HostChecker = p.fleet d.InstallScript = p.fleet - d.SettingsHook = p - d.HTTPS = certificate{p.acme} + d.HTTPS = p.acme return p, nil } @@ -304,46 +295,64 @@ func (p *primary) run(fn func(ctx context.Context)) { p.stops = append(p.stops, func() { cancel(); <-done }) } +// pruneDaily runs prune at once and then each day, until stop. Each owner +// of rows that expire prunes them itself. +func (p *primary) pruneDaily(what string, prune func(ctx context.Context, now time.Time) error) { + p.run(func(ctx context.Context) { + tick := time.NewTicker(24 * time.Hour) + defer tick.Stop() + for { + if err := prune(ctx, time.Now()); err != nil && ctx.Err() == nil { + p.log.Warn("delete expired rows", "rows", what, "err", err) + } + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } + }) +} + // stop ends the services in the reverse order of their start. func (p *primary) stop() { for i := len(p.stops) - 1; i >= 0; i-- { p.stops[i]() } - p.wg.Wait() } // serveCaches starts the Actions cache server and the registry mirror, and // serves them on the service port. -func (p *primary) serveCaches(key []byte, mode sandbox.Mode, drv sandbox.Driver) (caches, error) { - resolver := cache.NewSandboxResolver(p.st, drv.List) - stats := cache.NewStats(p.st, p.log.With("component", "cache")) +func (p *primary) serveCaches(key []byte, l *local) (*cache.Server, error) { + resolver := cache.NewSandboxResolver(p.st, l.driver.List) + space := func() datadisk.Space { + st := p.settings.Load() + return datadisk.Of(p.cfg.CacheDir(), datadisk.Settings{CacheMaxGB: st.CacheMaxGB, MinFreePercent: st.MinFreeDiskPercent}) + } srv, err := cache.New(cache.Config{ - Dir: p.cfg.CacheDir(), Store: p.st, Resolver: resolver, Stats: stats, - MaxBytes: func() int64 { return int64(p.settings.Load().CacheMaxGB) << 30 }, - DiskSpace: func() (int64, int64, error) { return host.DiskSpace(p.cfg.CacheDir()) }, - MinFreePercent: func() float64 { return p.settings.Load().MinFreeDiskPercent }, - SigningKey: subkey(key, "runplane cache urls"), Logger: p.log.With("component", "cache"), + Dir: p.cfg.CacheDir(), Store: p.st, Resolver: resolver, Space: space, + SigningKey: subkey(key, "runplane cache urls"), Logger: p.log.With("component", "cache"), }) if err != nil { - return caches{}, fmt.Errorf("start the cache server: %w", err) + return nil, fmt.Errorf("start the cache server: %w", err) } - p.run(stats.Run) - p.run(func(ctx context.Context) { srv.Run(ctx) }) - mir, stopPort, err := serveServicePort(p.cfg, mode, resolver.Allow, srv, p.log) + p.run(srv.Run) + p.pruneDaily("cache stats", srv.Prune) + stopPort, err := serveServicePort(p.cfg, l.node.Mirror(), srv, p.log) if err != nil { - return caches{}, err + return nil, err } p.stops = append(p.stops, stopPort) - return caches{srv: srv, mirror: mir, agents: p.fleet.AgentMirrorStats}, nil + return srv, nil } -func (p *primary) listenAgents(cert tls.Certificate, cacheHandler http.Handler) error { +func (p *primary) listenAgents(cert tls.Certificate, cache func(http.ResponseWriter, *http.Request, string, string)) error { ln, err := net.Listen("tcp", p.cfg.AgentListen) if err != nil { return fmt.Errorf("listen for agents: %w", err) } l := &agent.Listener{ - Cert: cert, Authenticate: p.fleet.Authenticate, OnSession: p.fleet.AddSession, CacheHandler: cacheHandler, + Cert: cert, Authenticate: p.fleet.Authenticate, OnSession: p.fleet.AddSession, Cache: cache, Logger: p.log.With("component", "agents"), } p.run(func(ctx context.Context) { @@ -357,7 +366,7 @@ func (p *primary) listenAgents(cert tls.Certificate, cacheHandler http.Handler) func (p *primary) serveSSH(opener shell.Opener) error { gw, err := shell.New(shell.Config{ Addr: p.cfg.SSHListen, HostKeyPath: p.cfg.SSHHostKeyPath(), Opener: opener, Lookup: shell.LookupStore(p.st), - AdminKeys: shell.AuthorizedKeysFile(p.cfg.AdminKeysPath(), p.log), Audit: p.st.InsertAudit, + AdminKeys: shell.AuthorizedKeysFile(p.cfg.AdminKeysPath(), p.log), Audit: audit.Log{Store: p.st, Logger: p.log}.Record, Log: p.log.With("component", "ssh"), }) if err != nil { @@ -372,55 +381,28 @@ func (p *primary) serveSSH(opener shell.Opener) error { return nil } -// onEvent passes a driver event to the scheduler, and records the address -// of a sandbox when its runner listens. -func (p *primary) onEvent(serverID string, ev sandbox.Event) { - p.sched.HandleEvent(serverID, ev) - if ev.Type != sandbox.EventState || ev.State != sandbox.StateIdle { - return - } - drv := p.fleet.Driver(serverID) - if drv == nil { - return - } - p.wg.Go(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) +// followSettings gives each settings change to the cache server and HTTPS. +// The fleet, the scheduler, and GitHub follow the changes themselves. +func (p *primary) followSettings(bus *events.Bus) { + sub, cancel := context.WithCancel(context.Background()) + changes := events.Subscribe[events.SettingsChanged](sub, bus) + p.run(func(ctx context.Context) { defer cancel() - p.recordIP(ctx, serverID, ev.SandboxID, drv.List) - }) -} - -// recordIP saves the address that the driver of serverID lists for a -// sandbox. An agent sends its own list, so the store takes only a sandbox -// of that server: an agent cannot give a sandbox of the primary its address. -func (p *primary) recordIP(ctx context.Context, serverID, sandboxID string, list func(context.Context) ([]sandbox.Sandbox, error)) { - boxes, err := list(ctx) - if err != nil { - p.log.Debug("list sandboxes for an address", "server_id", serverID, "err", err) - return - } - for _, b := range boxes { - if b.ID == sandboxID && b.IP != "" { - if err := p.st.SetSandboxIP(ctx, serverID, b.ID, b.IP); err != nil { - p.log.Debug("record sandbox address", "server_id", serverID, "sandbox", b.ID, "err", err) + for { + select { + case <-ctx.Done(): + return + case c := <-changes: + p.settings.Store(&c.New) + if c.Old.Domain != c.New.Domain || c.Old.ACMEEmail != c.New.ACMEEmail { + p.serveHTTPS(c.New) + } + if c.Old.Mode != c.New.Mode { + p.log.Warn("the new sandbox mode applies after a restart of runplane", "mode", c.New.Mode) + } } } - } -} - -// SettingsChanged is the api SettingsHook: it gives the new settings to the -// servers, the scale sets, the scheduler, the cache, and HTTPS. -func (p *primary) SettingsChanged(ctx context.Context, prev, next store.Settings) { - p.settings.Store(&next) - p.fleet.SettingsChanged(ctx, prev, next) - p.gh.PoolsChanged(ctx) - p.sched.PoolsChanged(ctx) - if prev.Domain != next.Domain || prev.ACMEEmail != next.ACMEEmail { - p.serveHTTPS(next) - } - if prev.Mode != next.Mode { - p.log.Warn("the new sandbox mode applies after a restart of runplane", "mode", next.Mode) - } + }) } // serveHTTPS serves the dashboard with a Let's Encrypt certificate for the @@ -439,16 +421,6 @@ func (p *primary) serveHTTPS(st store.Settings) { p.log.Info("HTTPS started", "domain", st.Domain) } -// primaryAddrs lists the TCP addresses of the primary for the doctor. A -// domain adds the ports of HTTPS, which another web server often has. -func primaryAddrs(cfg config.Config, domain string) []string { - addrs := []string{cfg.Listen, cfg.AgentListen, servicePortAddr(cfg), cfg.SSHListen} - if domain != "" { - addrs = append(addrs, ":443", ":80") - } - return addrs -} - // subkey derives a key for one use from the secret key. func subkey(key []byte, use string) []byte { m := hmac.New(sha256.New, key) diff --git a/cmd/runplane/primary_test.go b/cmd/runplane/primary_test.go deleted file mode 100644 index 1edd3ee..0000000 --- a/cmd/runplane/primary_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package main - -import ( - "bytes" - "context" - "log/slog" - "regexp" - "testing" - "time" - - "github.com/runplane/runplane/internal/auth" - "github.com/runplane/runplane/internal/sandbox" - "github.com/runplane/runplane/internal/store" -) - -func TestLogSetupCodeKeepsAValidCode(t *testing.T) { - ctx := context.Background() - st := openStore(t) - a := auth.NewService(st, nil) - var out bytes.Buffer - - // install.sh prints a code, then runplane restarts. - printed, err := a.NewSetupCode(ctx) - if err != nil { - t.Fatal(err) - } - if err := logSetupCode(ctx, st, a, slog.New(slog.NewTextHandler(&out, nil))); err != nil { - t.Fatal(err) - } - if loggedCode(out.String()) != "" { - t.Errorf("serve made a new code while one was valid:\n%s", out.String()) - } - if _, _, err := a.Setup(ctx, printed, "Ada", "ada@example.com", "a long password", "127.0.0.1", "test"); err != nil { - t.Fatalf("the printed code stopped working after a restart: %v", err) - } -} - -func TestLogSetupCodeReplacesAnExpiredCode(t *testing.T) { - ctx := context.Background() - st := openStore(t) - a := auth.NewService(st, nil) - old := time.Now().Add(-48 * time.Hour) - if err := st.CreateSetupCode(ctx, store.SetupCode{CodeHash: "expired", CreatedAt: old, ExpiresAt: old.Add(auth.SetupCodeTTL)}); err != nil { - t.Fatal(err) - } - var out bytes.Buffer - if err := logSetupCode(ctx, st, a, slog.New(slog.NewTextHandler(&out, nil))); err != nil { - t.Fatal(err) - } - code := loggedCode(out.String()) - if code == "" { - t.Fatalf("no new code in the log:\n%s", out.String()) - } - if _, _, err := a.Setup(ctx, code, "Ada", "ada@example.com", "a long password", "127.0.0.1", "test"); err != nil { - t.Fatalf("the logged code does not work: %v", err) - } -} - -func loggedCode(log string) string { - if m := regexp.MustCompile(`code=(\S+)`).FindStringSubmatch(log); m != nil { - return m[1] - } - return "" -} - -// The cache and the mirror find a sandbox of the primary by its address, so -// an agent must not give its address to a sandbox of the primary. -func TestRecordIPTakesOnlySandboxesOfTheServer(t *testing.T) { - ctx := context.Background() - st := openStore(t) - for _, b := range []store.Sandbox{ - {ID: "primary-box", ServerID: "local", IP: "10.213.0.5"}, - {ID: "agent-box", ServerID: "agent-1"}, - } { - b.PoolID, b.Mode, b.State, b.VCPU, b.MemoryMB, b.CreatedAt = "pool", sandbox.ModeContainer, sandbox.StateIdle, 2, 4096, time.Now() - if err := st.InsertSandbox(ctx, b); err != nil { - t.Fatal(err) - } - } - p := &primary{st: st, log: slog.New(slog.DiscardHandler)} - // The agent lists the sandbox of the primary with the address of its own. - agentList := func(context.Context) ([]sandbox.Sandbox, error) { - return []sandbox.Sandbox{{ID: "primary-box", IP: "10.214.0.7"}, {ID: "agent-box", IP: "10.214.0.8"}}, nil - } - - p.recordIP(ctx, "agent-1", "primary-box", agentList) - p.recordIP(ctx, "agent-1", "agent-box", agentList) - - for id, want := range map[string]string{"primary-box": "10.213.0.5", "agent-box": "10.214.0.8"} { - if b, err := st.GetSandbox(ctx, id); err != nil || b.IP != want { - t.Errorf("sandbox %s: address %q, %v; want %s", id, b.IP, err, want) - } - } -} diff --git a/cmd/runplane/services.go b/cmd/runplane/services.go index 107c173..c8fc592 100644 --- a/cmd/runplane/services.go +++ b/cmd/runplane/services.go @@ -1,322 +1,26 @@ package main import ( - "context" - "errors" - "fmt" - "log/slog" - "slices" - "strings" - "sync" - "time" - - "github.com/danielgtaylor/huma/v2" - - "github.com/runplane/runplane/internal/api" - "github.com/runplane/runplane/internal/backup" - "github.com/runplane/runplane/internal/cache" - "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/fleet" "github.com/runplane/runplane/internal/github" - "github.com/runplane/runplane/internal/ids" - "github.com/runplane/runplane/internal/mirror" - "github.com/runplane/runplane/internal/notify" "github.com/runplane/runplane/internal/scheduler" - "github.com/runplane/runplane/internal/secret" - "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/tlsutil" - "github.com/runplane/runplane/internal/update" - "github.com/runplane/runplane/internal/version" ) -// pools is the api Scheduler: a pool change also changes the scale sets -// on GitHub. -type pools struct { - *scheduler.Scheduler - gh *github.Service -} - -func (p pools) PoolsChanged(ctx context.Context) { - p.gh.PoolsChanged(ctx) - p.Scheduler.PoolsChanged(ctx) -} - -// githubRef breaks the construction cycle of the scheduler and the github -// service: the scheduler gets it first, and the service fills it. +// githubRef breaks the construction cycle of the github service: the +// scheduler and the runner versions service get it first, and the github +// service fills it. type githubRef struct{ *github.Service } -// caches is the api Caches service. -type caches struct { - srv *cache.Server - mirror *mirror.Mirror - // agents sums the mirror stats of the agents, which have mirrors of - // their own. - agents func(context.Context) mirror.Stats -} - -func (c caches) Delete(ctx context.Context, repository, id string) (int, error) { - if id == "" { - if repository == "" { - return 0, huma.Error422UnprocessableEntity("Give a repository or an entry ID.") - } - return c.srv.DeleteRepository(ctx, repository) - } - if err := c.srv.Delete(ctx, id); errors.Is(err, cache.ErrNotFound) { - return 0, huma.Error404NotFound("No cache entry has this ID.") - } else if err != nil { - return 0, err - } - return 1, nil -} - -func (c caches) MaxBytes() int64 { return c.srv.Limit() } - -// MirrorStats sums the registry mirror stats of all servers. -func (c caches) MirrorStats(ctx context.Context) (api.MirrorStats, error) { - s := c.mirror.Stats() - s.Add(c.agents(ctx)) - return api.MirrorStats{Hits: s.Hits, Misses: s.Misses, HitBytes: s.HitBytes, MissBytes: s.MissBytes, SizeBytes: s.SizeBytes}, nil -} - -// certificate is the api HTTPS service. -type certificate struct{ acme *tlsutil.ACME } - -func (c certificate) CertificateStatus(ctx context.Context) api.HTTPSStatus { - s := c.acme.Status(ctx) - out := api.HTTPSStatus{State: s.State, Error: s.Error} - if !s.NotAfter.IsZero() { - out.NotAfter = &s.NotAfter - } - return out -} - -// backups makes database backups: the api Backups service and the daily backup. -type backups struct { - st *store.Store - box *secret.Box - bus *events.Bus - dir string - notify func(notify.Event) - log *slog.Logger - mu sync.Mutex // one backup at a time -} - -// Run makes a backup, removes the backups over backup_keep, and uploads -// the new one to S3 if a bucket is set. A failed upload keeps the local -// backup and records the error on it. -func (b *backups) Run(ctx context.Context) (string, error) { - b.mu.Lock() - defer b.mu.Unlock() - st, err := b.st.GetSettings(ctx) - if err != nil { - return "", err - } - row := store.Backup{ID: ids.New(), CreatedAt: time.Now(), Status: store.BackupRunning} - if err := b.st.CreateBackup(ctx, row); err != nil { - return "", err - } - b.publish(row) - res, err := backup.Run(ctx, b.st.DB(), b.dir, st.BackupKeep) - if err != nil { - row.Status, row.Error = store.BackupFailed, err.Error() - b.save(ctx, row) - b.failed(st, err) - return "", fmt.Errorf("back up the database: %w", err) - } - row.Path, row.SizeBytes, row.Status = res.Path, res.Size, store.BackupDone - b.save(ctx, row) - b.dropRows(ctx, res.Removed) - if st.BackupS3.Bucket == "" { - return row.ID, nil - } - row.Status = store.BackupUploading - b.save(ctx, row) - row.S3Key, err = b.upload(ctx, st.BackupS3, res.Path) - if err != nil { - row.Status, row.Error = store.BackupDone, err.Error() - b.failed(st, err) - } else { - row.Status = store.BackupUploaded - } - b.save(ctx, row) - return row.ID, nil -} - -func (b *backups) upload(ctx context.Context, s store.BackupS3, path string) (string, error) { - key, err := b.box.Open(s.SecretKeyEnc) - if err != nil { - return "", fmt.Errorf("open the S3 secret key: %w", err) - } - bucket := backup.S3{Endpoint: s.Endpoint, Region: s.Region, Bucket: s.Bucket, Prefix: s.Prefix, AccessKey: s.AccessKey, SecretKey: string(key)} - k, err := bucket.Upload(ctx, path) - if err != nil { - return "", fmt.Errorf("upload to S3: %w", err) - } - return k, nil -} - -func (b *backups) save(ctx context.Context, row store.Backup) { - if err := b.st.UpdateBackup(context.WithoutCancel(ctx), row); err != nil { - b.log.Warn("record backup", "backup", row.ID, "err", err) - return - } - b.publish(row) -} - -func (b *backups) publish(row store.Backup) { - b.bus.Publish(events.Event{Type: events.BackupState, Data: row}) -} +// schedulerFleet gives the fleet to the scheduler, whose drivers have only +// the methods that the scheduler uses. +type schedulerFleet struct{ *fleet.Fleet } -// dropRows deletes the rows of the backup files that the retention removed. -func (b *backups) dropRows(ctx context.Context, removed []string) { - if len(removed) == 0 { - return - } - list, err := b.st.ListBackups(ctx) - if err != nil { - b.log.Warn("list backups", "err", err) - return +func (f schedulerFleet) Online() []scheduler.Host { + var hosts []scheduler.Host + for _, h := range f.Fleet.Online() { + hosts = append(hosts, scheduler.Host{ServerID: h.ServerID, Arch: h.Arch, Driver: h.Driver}) } - for _, row := range list { - if !slices.Contains(removed, row.Path) { - continue - } - if err := b.st.DeleteBackup(ctx, row.ID); err != nil { - b.log.Warn("delete backup row", "backup", row.ID, "err", err) - } - } -} - -func (b *backups) failed(st store.Settings, err error) { - b.log.Error("backup failed", "err", err) - b.notify(notify.BackupFailedEvent(st.DashboardURL(), err)) -} - -// daily makes a backup each day at 03:00 UTC while backup_enabled is on. -func (b *backups) daily(ctx context.Context) { - backup.Daily(ctx, func(ctx context.Context) { - st, err := b.st.GetSettings(ctx) - if err != nil || !st.BackupEnabled { - return - } - if _, err := b.Run(ctx); err != nil { - b.log.Warn("daily backup", "err", err) - } - }) -} - -// updater is the api Updater service. -type updater struct { - u *update.Updater - // restart ends the process after an update, so that systemd starts - // the new binary. - restart func() - log *slog.Logger - - mu sync.Mutex - checked time.Time // of last, which stays valid for checkTTL - last update.Status - state string // of an auto update, for the dashboard + return hosts } -// checkTTL keeps the release reads of the dashboard far below the GitHub -// limit of 60 anonymous requests an hour. -const checkTTL = 10 * time.Minute - -// Check reads the latest release, at most once in checkTTL. A repository -// without releases has no update. -func (u *updater) Check(ctx context.Context) (api.UpdateStatus, error) { - u.mu.Lock() - defer u.mu.Unlock() - if time.Since(u.checked) >= checkTTL { - s, err := u.u.Check(ctx) - if errors.Is(err, update.ErrNoRelease) { - s, err = update.Status{Current: u.u.Current}, nil - } - if err != nil { - return api.UpdateStatus{}, err - } - u.last, u.checked = s, time.Now() - } - s := u.last - out := api.UpdateStatus{Current: s.Current, Latest: s.Latest, Available: s.Available, URL: s.URL, Notes: s.Notes, State: u.state} - if !s.PublishedAt.IsZero() { - out.PublishedAt = &s.PublishedAt - } - return out, nil -} - -func (u *updater) Apply(ctx context.Context) error { - s, err := u.u.Check(ctx) - if err != nil { - return err - } - if !s.Available { - return huma.Error409Conflict("runplane " + s.Current + " is the latest version.") - } - if err := u.u.Apply(ctx, s.Latest); err != nil { - return err - } - u.log.Info("runplane restarts to run the new version", "version", s.Latest) - u.restart() - return nil -} - -func (u *updater) setState(state string) { - u.mu.Lock() - u.state = state - u.mu.Unlock() -} - -// checkDaily checks for a new release each day. It sends one update.available -// notification for each new version, and with auto_update it installs the -// release and restarts runplane. -func (u *updater) checkDaily(ctx context.Context, st *store.Store, send func(notify.Event)) { - if !isRelease(version.Version) || update.PublicKey == "" { - u.log.Debug("no update checks: this binary is not a signed release", "version", version.Version) - return - } - told := "" - wait := time.Minute - for { - select { - case <-ctx.Done(): - return - case <-time.After(wait): - } - wait = 24 * time.Hour - s, err := u.u.Check(ctx) - if err != nil { - u.log.Warn("check for updates", "err", err) - continue - } - if !s.Available { - continue - } - settings, err := st.GetSettings(ctx) - if err != nil { - u.log.Warn("read settings for an update", "err", err) - continue - } - if settings.AutoUpdate { - u.setState("applying") - if err := u.u.Apply(ctx, s.Latest); err != nil { - u.setState("") - u.log.Error("install update", "version", s.Latest, "err", err) - } else { - u.setState("restarting") - u.log.Info("runplane restarts to run the new version", "version", s.Latest) - u.restart() - return - } - } - if told != s.Latest { - told = s.Latest - send(notify.UpdateAvailableEvent(settings.DashboardURL(), s.Current, s.Latest, s.URL)) - } - } -} - -// isRelease reports if v is the version of a release build, like v1.2.0. -func isRelease(v string) bool { - return strings.HasPrefix(v, "v") && update.Compare(v, "v0.0.0") >= 0 -} +func (f schedulerFleet) Driver(serverID string) scheduler.Driver { return f.Fleet.Driver(serverID) } diff --git a/cmd/runplane/services_test.go b/cmd/runplane/services_test.go deleted file mode 100644 index 1dc171c..0000000 --- a/cmd/runplane/services_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -import ( - "context" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "slices" - "sync/atomic" - "testing" - "time" - - "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/notify" - "github.com/runplane/runplane/internal/secret" - "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/update" -) - -// A backup over backup_keep loses its file and its row together, so the -// dashboard lists no backup that cannot download. -func TestBackupsRemoveTheRowsOfPrunedFiles(t *testing.T) { - ctx := context.Background() - st := openStore(t) - dir := t.TempDir() - if _, err := st.UpdateSettings(ctx, func(s *store.Settings) error { s.BackupKeep = 1; return nil }); err != nil { - t.Fatal(err) - } - old := filepath.Join(dir, "runplane-20200101T000000.000Z.db.gz") - if err := os.WriteFile(old, []byte("old"), 0o600); err != nil { - t.Fatal(err) - } - if err := st.CreateBackup(ctx, store.Backup{ID: "old", CreatedAt: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), - Path: old, SizeBytes: 3, Status: store.BackupDone}); err != nil { - t.Fatal(err) - } - b := &backups{st: st, bus: &events.Bus{}, dir: dir, log: slog.New(slog.DiscardHandler)} - - id, err := b.Run(ctx) - if err != nil { - t.Fatal(err) - } - list, err := st.ListBackups(ctx) - if err != nil { - t.Fatal(err) - } - if len(list) != 1 || list[0].ID != id || list[0].Status != store.BackupDone || list[0].SizeBytes == 0 { - t.Fatalf("backups: %+v, want only the new one, done", list) - } - if _, err := os.Stat(list[0].Path); err != nil { - t.Errorf("the new backup file: %v", err) - } - if _, err := os.Stat(old); !os.IsNotExist(err) { - t.Errorf("the old backup file is still there: %v", err) - } -} - -// The settings page follows a backup through backup.state events, also the -// upload to S3 and a failure. Only a failure sends a notification. -func TestBackupStateEvents(t *testing.T) { - s3 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - io.Copy(io.Discard, r.Body) - })) - defer s3.Close() - box, err := secret.NewBox(make([]byte, secret.KeySize)) - if err != nil { - t.Fatal(err) - } - tests := []struct { - name string - bucket bool - dir func(t *testing.T) string - want []string - notified int - }{ - {"local", false, func(t *testing.T) string { return t.TempDir() }, []string{store.BackupRunning, store.BackupDone}, 0}, - {"upload to S3", true, func(t *testing.T) string { return t.TempDir() }, - []string{store.BackupRunning, store.BackupDone, store.BackupUploading, store.BackupUploaded}, 0}, - {"backup fails", false, func(t *testing.T) string { - file := filepath.Join(t.TempDir(), "file") - os.WriteFile(file, nil, 0o600) - return filepath.Join(file, "backups") // a folder in a file cannot exist - }, []string{store.BackupRunning, store.BackupFailed}, 1}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - st := openStore(t) - if tt.bucket { - _, err := st.UpdateSettings(ctx, func(s *store.Settings) error { - s.BackupS3 = store.BackupS3{Endpoint: s3.URL, Region: "us-east-1", Bucket: "ci-backups", AccessKey: "AKIDEXAMPLE", - SecretKeyEnc: box.Seal([]byte("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"))} - return nil - }) - if err != nil { - t.Fatal(err) - } - } - bus := &events.Bus{} - states, stop := bus.Subscribe(16) - notified := 0 - b := &backups{st: st, box: box, bus: bus, dir: tt.dir(t), notify: func(notify.Event) { notified++ }, log: slog.New(slog.DiscardHandler)} - b.Run(ctx) - stop() - var seen []string - for e := range states { - if row, ok := e.Data.(store.Backup); ok && e.Type == events.BackupState { - seen = append(seen, row.Status) - } - } - if !slices.Equal(seen, tt.want) { - t.Errorf("backup.state events %v, want %v", seen, tt.want) - } - if notified != tt.notified { - t.Errorf("%d notifications, want %d", notified, tt.notified) - } - }) - } -} - -// The dashboard reads the update status on each visit. A repository with no -// release has no update, and GitHub sees one read in checkTTL. -func TestUpdaterCheck(t *testing.T) { - var reads, status atomic.Int32 - status.Store(http.StatusNotFound) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - reads.Add(1) - w.WriteHeader(int(status.Load())) - })) - defer srv.Close() - u := &updater{u: &update.Updater{Repo: "runplane/runplane", Current: "v1.0.0", APIURL: srv.URL}, log: slog.New(slog.DiscardHandler)} - ctx := context.Background() - - for range 2 { - s, err := u.Check(ctx) - if err != nil || s.Available || s.Current != "v1.0.0" { - t.Fatalf("Check = %+v, %v; want v1.0.0 without an update", s, err) - } - } - if n := reads.Load(); n != 1 { - t.Errorf("%d release reads for two checks, want 1", n) - } - - // A GitHub fault is not kept, so the next visit reads again. - u.checked = time.Time{} - status.Store(http.StatusBadGateway) - if _, err := u.Check(ctx); err == nil { - t.Fatal("Check with a GitHub fault returned no error") - } - u.Check(ctx) - if n := reads.Load(); n != 3 { - t.Errorf("%d release reads after two failed checks, want 3", n) - } -} diff --git a/cmd/runplane/tools.go b/cmd/runplane/tools.go index d644b1e..ccc39c5 100644 --- a/cmd/runplane/tools.go +++ b/cmd/runplane/tools.go @@ -1,44 +1,51 @@ package main import ( - "cmp" "context" - "crypto/rand" "errors" "flag" "fmt" "io/fs" + "log/slog" "os" "os/exec" "runtime" - "slices" - "strings" "github.com/moby/moby/client" + "github.com/runplane/runplane/internal/agent" "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/auth" "github.com/runplane/runplane/internal/config" + "github.com/runplane/runplane/internal/doctor" + "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/fleet" "github.com/runplane/runplane/internal/host" + "github.com/runplane/runplane/internal/localnode" + "github.com/runplane/runplane/internal/policy" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/sandbox/sysbox" + "github.com/runplane/runplane/internal/settings" "github.com/runplane/runplane/internal/store" "github.com/runplane/runplane/internal/update" "github.com/runplane/runplane/internal/version" ) +// setupCode prints a new setup code for the first admin. It makes the +// database when it does not exist, because install.sh makes the code before +// the first start: serve then keeps that code and logs no different code. func setupCode(ctx context.Context, args []string) error { - config.LoadEnvFile() - cfg, err := config.Parse(flag.NewFlagSet("setup-code", flag.ContinueOnError), args) + cfg, err := config.Parse(flag.NewFlagSet("setup-code", flag.ContinueOnError), args, config.DataDir) if err != nil { return err } - st, err := store.Open(cfg.DBPath()) + st, err := openDB(cfg, true) if err != nil { return err } defer st.Close() - code, err := auth.NewService(st, nil).NewSetupCode(ctx) + code, err := auth.NewService(st, nil, nil).NewSetupCode(ctx) if errors.Is(err, auth.ErrSetupDone) { return errors.New("an admin exists, so the setup is done: log in, or ask an admin for an invite") } @@ -49,79 +56,90 @@ func setupCode(ctx context.Context, args []string) error { return nil } -// doctor runs the checks of the host package for this server: a primary, -// or an agent when agent.json exists. It fails when a check fails. -func doctor(ctx context.Context, args []string) error { - config.LoadEnvFile() - cfg, err := config.Parse(flag.NewFlagSet("doctor", flag.ContinueOnError), args) +// runDoctor runs the doctor on this server: a primary, or an agent when +// agent.json exists. It fails when a check fails. +func runDoctor(ctx context.Context, args []string) error { + cfg, err := config.Parse(flag.NewFlagSet("doctor", flag.ContinueOnError), args, + config.Listen, config.AgentListen, config.ServicePort, config.DataDir, config.Mode, config.SSHListen) if err != nil { return err } - mode := cfg.Mode - var dns []string - opts := host.Options{DataDir: cfg.DataDir, GuestKernel: cfg.KernelPath(), BaseImage: baseImage} - if exists(cfg.AgentConfigPath()) { - opts.Unit, opts.Listen = "runplane-agent", []string{servicePortAddr(cfg)} - } else { - var settings store.Settings - if exists(cfg.DBPath()) { - if settings, err = readSettings(ctx, cfg); err != nil { - return err - } - } - // The dashboard settings win over the environment. - mode = cmp.Or(settings.Mode, mode) - opts.Unit, opts.Listen = "runplane", primaryAddrs(cfg, settings.Domain) - opts.MinFreeDiskPercent = settings.MinFreeDiskPercent - dns = settings.DNSServers - } - // The probe needs the runplane network and the firewall rules, which the - // running service makes. In VM mode only the service can add the rule. - opts.Internet = func(context.Context) error { return host.ErrProbeInService } - if mode == sandbox.ModeContainer || mode == sandbox.ModeDev { - probe, err := internetProbe(cfg, mode, dns) + isAgent := exists(cfg.AgentConfigPath()) + settings, node, err := serviceSettings(ctx, cfg, isAgent) + if err != nil { + return err + } + mode := policy.Mode(settings, cfg.Mode) + opts := doctorOptions(cfg, mode, isAgent, settings.Domain) + opts.MinFreeDiskPercent = node.MinFreeDiskPercent + if profile, ok := probeProfile(mode); ok { + docker, err := client.New(client.FromEnv) if err != nil { - return err + return fmt.Errorf("connect to Docker: %w", err) } - defer probe.Docker.Close() + defer docker.Close() + probe := sysbox.ProbeOptions{Docker: docker, Profile: profile, DataDir: cfg.DataDir, DNS: node.DNS} opts.Internet = func(ctx context.Context) error { return sysbox.Probe(ctx, probe) } } - checks := host.Doctor(ctx, mode, opts) - if err := host.FormatDoctor(os.Stdout, checks); err != nil { + r := doctor.Run(ctx, opts) + if err := doctor.Format(os.Stdout, r.Checks); err != nil { return err } - if slices.ContainsFunc(checks, func(c host.Check) bool { return c.Status == host.StatusFail }) { + if r.Failed() { return errors.New("a check failed") } return nil } -// internetProbe returns the options of the sandbox internet probe for the -// container and dev mode drivers, with the runtimes and the cgroup parent -// of sysbox.New and devdocker.New. -func internetProbe(cfg config.Config, mode sandbox.Mode, dns []string) (sysbox.ProbeOptions, error) { - o := sysbox.ProbeOptions{DataDir: cfg.DataDir, Runtime: "runc"} - if mode == sandbox.ModeContainer { - o.Runtime, o.CgroupParent = "sysbox-runc", "runplane.slice" - } - o.DNS = sandbox.Settings{DNS: dns}.IPv4DNS() - docker, err := client.New(client.FromEnv) - if err != nil { - return o, fmt.Errorf("connect to Docker: %w", err) +// serviceSettings returns the settings of the primary and of the node that +// the service on this server uses. An agent uses the last node settings that +// its primary sent. A primary before its first start, and an agent before +// its first connection, use the default settings. +func serviceSettings(ctx context.Context, cfg config.Config, isAgent bool) (store.Settings, agent.Settings, error) { + settings := store.DefaultSettings() + switch { + case isAgent && exists(cfg.AgentSettingsPath()): + node, err := localnode.ReadSettings(cfg.AgentSettingsPath()) + return settings, node, err + case !isAgent && exists(cfg.DBPath()): + var err error + if settings, err = readSettings(ctx, cfg); err != nil { + return store.Settings{}, agent.Settings{}, err + } } - o.Docker = docker - return o, nil + return settings, fleet.NodeSettings(settings), nil } +// probeProfile returns the profile of the sandbox internet probe of a mode. +// The probe needs the runplane network and the firewall rules, which the +// running service makes. In VM mode only the service can add the rule. +func probeProfile(mode sandbox.Mode) (sysbox.Profile, bool) { + switch mode { + case sandbox.ModeContainer: + return sysbox.Container, true + case sandbox.ModeDev: + return sysbox.Dev, true + } + return sysbox.Profile{}, false +} + +// errAgentUpdate is the error of runplane update on an agent. An agent runs the +// release of its primary, so that the two speak the same protocol, and only +// the primary knows that release. +var errAgentUpdate = errors.New("this server is an agent, so it runs the release of its primary: " + + "select Update agent on the Servers page of the dashboard, or run curl -fsSL /install.sh | sudo sh") + func updateCmd(ctx context.Context, args []string) error { - config.LoadEnvFile() flags := flag.NewFlagSet("update", flag.ContinueOnError) check := flags.Bool("check", false, "only report the latest release") - cfg, err := config.Parse(flags, args) + cfg, err := config.Parse(flags, args, config.DataDir) if err != nil { return err } - u := update.New(cfg.KernelPath()) + if exists(cfg.AgentConfigPath()) { + return errAgentUpdate + } + u := update.NewService(update.Config{Updater: update.New(cfg.KernelPath())}) s, err := u.Check(ctx) if err != nil { return err @@ -134,7 +152,7 @@ func updateCmd(ctx context.Context, args []string) error { if *check { return nil } - if err := u.Apply(ctx, s.Latest); err != nil { + if _, err := u.Apply(ctx); err != nil { return err } fmt.Printf("Installed runplane %s.\n", s.Latest) @@ -142,8 +160,8 @@ func updateCmd(ctx context.Context, args []string) error { fmt.Println("Restart runplane to run the new version.") return nil } - // try-restart restarts only the units that run. - out, err := exec.CommandContext(ctx, "systemctl", "try-restart", "runplane.service", "runplane-agent.service").CombinedOutput() + // try-restart restarts the unit only if it runs. + out, err := exec.CommandContext(ctx, "systemctl", "try-restart", primaryUnit+".service").CombinedOutput() if err != nil { return fmt.Errorf("restart runplane: %w: %s", err, out) } @@ -154,37 +172,26 @@ func updateCmd(ctx context.Context, args []string) error { // resetPassword gives a user a new random password and ends the sessions // of the user. func resetPassword(ctx context.Context, args []string) error { - config.LoadEnvFile() flags := flag.NewFlagSet("reset-password", flag.ContinueOnError) - cfg, err := config.Parse(flags, args) + cfg, err := config.Parse(flags, args, config.DataDir) if err != nil { return err } if flags.NArg() != 1 { return errors.New("give the email address of the user: runplane reset-password ") } - if !exists(cfg.DBPath()) { - return fmt.Errorf("no database at %s", cfg.DBPath()) - } - st, err := store.Open(cfg.DBPath()) + st, err := openDB(cfg, false) if err != nil { return err } defer st.Close() - u, err := st.GetUserByEmail(ctx, flags.Arg(0)) + password, err := auth.NewService(st, nil, nil).ResetPassword(ctx, audit.Actor{}, flags.Arg(0)) if errors.Is(err, store.ErrNotFound) { return fmt.Errorf("no user has the email address %s", flags.Arg(0)) } if err != nil { return err } - password := rand.Text() - if err := st.SetUserPassword(ctx, u.ID, auth.HashPassword(password)); err != nil { - return err - } - if err := st.DeleteUserSessions(ctx, u.ID); err != nil { - return err - } fmt.Println(password) return nil } @@ -193,52 +200,24 @@ func resetPassword(ctx context.Context, args []string) error { // first start, the settings win over RUNPLANE_MODE, so install.sh calls // setMode for --mode. The mode applies when runplane starts again. func setMode(ctx context.Context, args []string) error { - config.LoadEnvFile() flags := flag.NewFlagSet("set-mode", flag.ContinueOnError) - cfg, err := config.Parse(flags, args) + cfg, err := config.Parse(flags, args, config.DataDir, config.DevInsecure) if err != nil { return err } if flags.NArg() != 1 { return errors.New("give the mode: runplane set-mode container|vm|dev") } - mode := sandbox.Mode(flags.Arg(0)) - switch mode { - case sandbox.ModeContainer: - case sandbox.ModeVM: - f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) - if err != nil { - return fmt.Errorf("VM mode needs KVM: %w", err) - } - f.Close() - case sandbox.ModeDev: - if !cfg.DevInsecure { - return errDevInsecure - } - default: - return fmt.Errorf("mode %q: use container, vm, or dev", mode) - } - if !exists(cfg.DBPath()) { - return fmt.Errorf("no database at %s", cfg.DBPath()) - } - st, err := store.Open(cfg.DBPath()) + st, err := openDB(cfg, false) if err != nil { return err } defer st.Close() - _, err = st.UpdateSettings(ctx, func(s *store.Settings) error { - s.Mode = mode - // Container mode always rejects public repositories, and the - // dashboard refuses to save the settings while this one is on. - if mode == sandbox.ModeContainer { - s.AllowPublicRepos = false - } - return nil - }) + next, err := newSettings(st, &events.Bus{}, cfg, nil).SetMode(ctx, audit.Actor{}, sandbox.Mode(flags.Arg(0))) if err != nil { return err } - fmt.Printf("runplane uses %s mode after its next start.\n", mode) + fmt.Printf("runplane uses %s mode after its next start.\n", next.Mode) return nil } @@ -267,44 +246,59 @@ func printVersion(_ context.Context, args []string) error { // install sets the domain before the first start. HTTPS for the domain // starts when runplane starts again. func setDomain(ctx context.Context, args []string) error { - config.LoadEnvFile() flags := flag.NewFlagSet("set-domain", flag.ContinueOnError) - cfg, err := config.Parse(flags, args) + cfg, err := config.Parse(flags, args, config.DataDir) if err != nil { return err } if flags.NArg() != 1 { return errors.New("give the domain: runplane set-domain ci.example.com") } - domain := strings.ToLower(strings.TrimSpace(flags.Arg(0))) - if domain != "" && !api.ValidDomain(domain) { - return fmt.Errorf("%q is not a domain name such as ci.example.com", domain) - } - if exists(cfg.AgentConfigPath()) { - return errors.New("this server is an agent: only the primary serves the dashboard") - } - st, err := store.Open(cfg.DBPath()) + st, err := openDB(cfg, true) if err != nil { return err } defer st.Close() - _, err = st.UpdateSettings(ctx, func(s *store.Settings) error { - s.Domain = domain + next, err := newSettings(st, &events.Bus{}, cfg, nil).Update(ctx, audit.Actor{}, func(s *store.Settings) error { + s.Domain = flags.Arg(0) return nil }) if err != nil { return err } - if domain == "" { + if next.Domain == "" { fmt.Println("runplane stops HTTPS after its next start.") } else { - fmt.Printf("runplane serves https://%s after its next start.\n", domain) + fmt.Printf("runplane serves https://%s after its next start.\n", next.Domain) } return nil } +func newSettings(st *store.Store, bus *events.Bus, cfg config.Config, log *slog.Logger) *settings.Service { + kvm := func() bool { + ok, _ := host.OpenKVM() + return ok + } + return settings.New(settings.Config{Store: st, Bus: bus, Log: log, Mode: cfg.Mode, DevInsecure: cfg.DevInsecure, KVM: kvm}) +} + +var errAgent = errors.New("this server is an agent: run this command on the primary") + +// openDB opens the database of the primary. It rejects an agent, which +// reads only runplane.env and agent.json: a database there changes nothing. +// With create, it makes the database when it does not exist. +func openDB(cfg config.Config, create bool) (*store.Store, error) { + if exists(cfg.AgentConfigPath()) { + return nil, errAgent + } + if !create && !exists(cfg.DBPath()) { + return nil, fmt.Errorf("no database at %s", cfg.DBPath()) + } + return store.Open(cfg.DBPath()) +} + func readSettings(ctx context.Context, cfg config.Config) (store.Settings, error) { - st, err := store.Open(cfg.DBPath()) + st, err := openDB(cfg, false) if err != nil { return store.Settings{}, err } diff --git a/cmd/runplane/tools_test.go b/cmd/runplane/tools_test.go index 1b08e29..bdbfe1a 100644 --- a/cmd/runplane/tools_test.go +++ b/cmd/runplane/tools_test.go @@ -2,8 +2,12 @@ package main import ( "context" + "errors" + "os" + "path/filepath" "testing" + "github.com/runplane/runplane/internal/auth" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" ) @@ -29,7 +33,7 @@ func TestSetModeStoresTheMode(t *testing.T) { st.Close() for _, bad := range []string{"dev", "kvm"} { - if err := setMode(ctx, []string{bad}); err == nil { + if err := run(ctx, []string{"set-mode", bad}); err == nil { t.Errorf("set-mode %s has no error", bad) } } @@ -37,7 +41,7 @@ func TestSetModeStoresTheMode(t *testing.T) { t.Fatalf("a refused set-mode changed the mode to %s", s.Mode) } - if err := setMode(ctx, []string{"container"}); err != nil { + if err := run(ctx, []string{"set-mode", "container"}); err != nil { t.Fatal(err) } // The dashboard refuses to save settings with public repositories on in @@ -55,7 +59,7 @@ func TestSetDomainStoresTheDomain(t *testing.T) { t.Setenv("RUNPLANE_CONFIG_DIR", dir) t.Setenv("RUNPLANE_DATA_DIR", dir) - if err := setDomain(ctx, []string{" CI.Example.com "}); err != nil { + if err := run(ctx, []string{"set-domain", " CI.Example.com "}); err != nil { t.Fatal(err) } if s := settingsIn(t, dir); s.Domain != "ci.example.com" { @@ -63,7 +67,7 @@ func TestSetDomainStoresTheDomain(t *testing.T) { } for _, bad := range []string{"192.0.2.1", "localhost", "https://ci.example.com"} { - if err := setDomain(ctx, []string{bad}); err == nil { + if err := run(ctx, []string{"set-domain", bad}); err == nil { t.Errorf("set-domain %s has no error", bad) } } @@ -71,7 +75,7 @@ func TestSetDomainStoresTheDomain(t *testing.T) { t.Fatalf("a refused set-domain changed the domain to %q", s.Domain) } - if err := setDomain(ctx, []string{""}); err != nil { + if err := run(ctx, []string{"set-domain", ""}); err != nil { t.Fatal(err) } if s := settingsIn(t, dir); s.Domain != "" { @@ -79,6 +83,77 @@ func TestSetDomainStoresTheDomain(t *testing.T) { } } +// install.sh makes the setup code before the first start of runplane, thus +// before the database exists. serve then keeps that code. +func TestSetupCodeBeforeTheFirstStart(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + t.Setenv("RUNPLANE_CONFIG_DIR", dir) + t.Setenv("RUNPLANE_DATA_DIR", dir) + + if err := run(ctx, []string{"setup-code"}); err != nil { + t.Fatal(err) + } + st, err := store.Open(dir + "/runplane.db") + if err != nil { + t.Fatal(err) + } + defer st.Close() + if code, err := auth.NewService(st, nil, nil).EnsureSetupCode(ctx); err != nil || code != "" { + t.Fatalf("serve makes the setup code %q (%v), want none: the code of setup-code is valid", code, err) + } +} + +// An agent must not install the latest release, which can be newer than +// the release of its primary. +func TestUpdateRefusesAnAgent(t *testing.T) { + dir := t.TempDir() + t.Setenv("RUNPLANE_CONFIG_DIR", dir) + t.Setenv("RUNPLANE_DATA_DIR", dir) + if err := os.WriteFile(filepath.Join(dir, "agent.json"), []byte(`{"primary": "203.0.113.5:8485"}`), 0o600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"update"}, {"update", "-check"}} { + if err := run(context.Background(), args); !errors.Is(err, errAgentUpdate) { + t.Errorf("runplane %v on an agent: %v, want the refusal", args, err) + } + } +} + +// An agent reads no database, and the primary makes its database at the +// first start. A command that made a database would report a change that +// runplane does not use. Only set-domain and setup-code make one on a +// primary, because install.sh runs them before the first start. +func TestCommandsMakeNoDatabase(t *testing.T) { + for _, tt := range []struct { + agent bool + args []string + }{ + {true, []string{"setup-code"}}, + {true, []string{"reset-password", "admin@example.com"}}, + {true, []string{"set-mode", "vm"}}, + {true, []string{"set-domain", "ci.example.com"}}, + {false, []string{"reset-password", "admin@example.com"}}, + {false, []string{"set-mode", "vm"}}, + } { + dir := t.TempDir() + t.Setenv("RUNPLANE_CONFIG_DIR", dir) + t.Setenv("RUNPLANE_DATA_DIR", dir) + if tt.agent { + if err := os.WriteFile(filepath.Join(dir, "agent.json"), []byte(`{"primary": "203.0.113.5:8485"}`), 0o600); err != nil { + t.Fatal(err) + } + } + err := run(context.Background(), tt.args) + if err == nil || tt.agent && !errors.Is(err, errAgent) { + t.Errorf("runplane %v (agent %v): %v, want the refusal", tt.args, tt.agent, err) + } + if exists(filepath.Join(dir, "runplane.db")) { + t.Errorf("runplane %v (agent %v) made a database", tt.args, tt.agent) + } + } +} + func settingsIn(t *testing.T, dir string) store.Settings { t.Helper() st, err := store.Open(dir + "/runplane.db") diff --git a/deploy/test-install.sh b/deploy/test-install.sh index fe8f981..77a88ad 100755 --- a/deploy/test-install.sh +++ b/deploy/test-install.sh @@ -24,6 +24,11 @@ main() { "CPUWeight=20" \ "+ systemctl enable runplane.service" refuse "the dry run changes nothing" "$out" "changed:" + # serve keeps a valid setup code, so the code of the install is the only one. + expect "the setup code comes before the start" "$out" \ + "+ /usr/local/bin/runplane setup-code + +==> Start runplane" # The repository has no public base image, so Docker builds it. out=$(install ubuntu:24.04 --dry-run --yes --version v1.2.3 --mode vm --repo runplane/no-such-repo) @@ -53,7 +58,7 @@ main() { "\"primary\": \"203.0.113.5:8485\"" \ "\"fingerprint\": \"$fingerprint\"" \ "\"token\": \"rp_srv_test\"" - refuse "an agent has no dashboard" "$out" "RUNPLANE_LISTEN" "ExecStart=/usr/local/bin/runplane serve" + refuse "an agent has no dashboard" "$out" "RUNPLANE_LISTEN" "ExecStart=/usr/local/bin/runplane serve" "setup-code" out=$(install ubuntu:24.04 --dry-run --yes --version v1.2.3 --listen 127.0.0.1:9000) expect "--listen" "$out" "RUNPLANE_LISTEN=127.0.0.1:9000" "Dashboard: http://127.0.0.1:9000" @@ -61,7 +66,10 @@ main() { # set-domain runs before the start, so that the first start serves HTTPS. out=$(install ubuntu:24.04 --dry-run --yes --version v1.2.3 --mode container --domain CI.Example.com) expect "--domain" "$out" "exit=0" \ - "+ with_env /usr/local/bin/runplane set-domain ci.example.com + "+ /usr/local/bin/runplane set-domain ci.example.com + +==> Make the setup code ++ /usr/local/bin/runplane setup-code ==> Start runplane" \ "Dashboard: https://ci.example.com" diff --git a/images/.dockerignore b/images/.dockerignore new file mode 100644 index 0000000..5d0f124 --- /dev/null +++ b/images/.dockerignore @@ -0,0 +1,2 @@ +* +!Dockerfile diff --git a/images/ubuntu-24.04/Dockerfile b/images/Dockerfile similarity index 92% rename from images/ubuntu-24.04/Dockerfile rename to images/Dockerfile index e80a722..293e141 100644 --- a/images/ubuntu-24.04/Dockerfile +++ b/images/Dockerfile @@ -1,16 +1,23 @@ -# runplane base image for GitHub Actions jobs: Ubuntu 24.04. +# runplane base image for GitHub Actions jobs. The build args select the +# Ubuntu release: UBUNTU is the tag of the ubuntu image, and IMAGE_OS is the +# ImageOS of the GitHub-hosted runners for that release. images/images.go +# names the builtin images and gives their build args. # # runplane starts runplane-init as the first process of each sandbox, so this # image has no ENTRYPOINT. runplane-init starts dockerd and the runner as the # user "runner". See images/README.md for custom images FROM this image. -FROM ubuntu:24.04 +ARG UBUNTU +FROM ubuntu:${UBUNTU} +ARG UBUNTU +ARG IMAGE_OS -LABEL org.opencontainers.image.title="runplane runner, Ubuntu 24.04" \ +LABEL org.opencontainers.image.title="runplane runner, Ubuntu ${UBUNTU}" \ org.opencontainers.image.description="Base image for runplane sandboxes" \ org.opencontainers.image.source="https://github.com/runplane/runplane" \ dev.runplane.managed=true -# Major versions for the tool cache. The build installs the latest release of each. +# Versions for the tool cache: major versions of Node.js, minor versions of +# Python and Go. The build installs the latest release of each. ARG NODE_VERSIONS="22 24 26" ARG NODE_DEFAULT=24 ARG PYTHON_VERSIONS="3.11 3.12 3.13 3.14" @@ -18,7 +25,7 @@ ARG GO_VERSIONS="1.27" ENV LANG=C.UTF-8 \ TZ=Etc/UTC \ - ImageOS=ubuntu24 \ + ImageOS=${IMAGE_OS} \ RUNNER_TOOL_CACHE=/opt/hostedtoolcache \ AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache \ PATH=/home/runner/.cargo/bin:$PATH @@ -44,7 +51,6 @@ RUN export DEBIAN_FRONTEND=noninteractive; \ locale-gen; \ rm -rf /var/lib/apt/lists/* -# Docker Engine with buildx and compose, and the GitHub CLI. RUN export DEBIAN_FRONTEND=noninteractive; \ arch=$(dpkg --print-architecture); \ codename=$(. /etc/os-release && echo "$VERSION_CODENAME"); \ diff --git a/images/README.md b/images/README.md index bcb3c9f..2894607 100644 --- a/images/README.md +++ b/images/README.md @@ -7,7 +7,7 @@ runplane publishes two base images for the sandboxes. Each image is for `linux/a | `ghcr.io/runplane/runplane/runner-ubuntu-24.04:latest` | Ubuntu 24.04 (the default image of a pool) | | `ghcr.io/runplane/runplane/runner-ubuntu-26.04:latest` | Ubuntu 26.04 | -If runplane cannot pull a base image, it builds the image on the server from the Dockerfile in this folder. +One `Dockerfile` in this folder builds the two images. Its build args `UBUNTU` and `IMAGE_OS` select the Ubuntu release. `images.go` names the images and gives their build args. If runplane cannot pull a base image, it builds the image on the server from this `Dockerfile`. ## Contents @@ -71,7 +71,7 @@ runplane pulls the image on each server. When you push a new image with the same To build a base image on your computer, run this command from the repository root: ```sh -docker build -t runner-ubuntu-24.04 images/ubuntu-24.04 +docker build -t runner-ubuntu-24.04 --build-arg UBUNTU=24.04 --build-arg IMAGE_OS=ubuntu24 images ``` The Dockerfile installs the latest release of each tool version in `NODE_VERSIONS`, `PYTHON_VERSIONS`, and `GO_VERSIONS`. To change the versions, use `--build-arg`, for example `--build-arg NODE_VERSIONS="24 26"`. diff --git a/images/images.go b/images/images.go index 259019b..e8afb2b 100644 --- a/images/images.go +++ b/images/images.go @@ -1,11 +1,51 @@ -// Package images embeds the Dockerfile folders of the builtin base images. -// Each top folder of FS is one image name, for example "ubuntu-24.04". +// Package images holds the builtin base images of the sandboxes. One +// Dockerfile builds each of them with its BuildArgs. The images workflow +// (.github/workflows/images.yml) publishes them at Ref, and images/README.md +// lists them. images_test.go compares these files with Names. package images -import "embed" +import ( + "embed" + "slices" + "strings" +) -//go:embed all:ubuntu-24.04 all:ubuntu-26.04 +// FS holds the Dockerfile, which is the build context of each builtin image. +// +//go:embed Dockerfile var FS embed.FS +// Names are the builtin images. Each name is "ubuntu-". +var Names = []string{"ubuntu-24.04", "ubuntu-26.04"} + // Default is the default image of a new install. const Default = "ubuntu-24.04" + +const ( + // Prefix starts the ref of the image row of a builtin image: + // "builtin:". + Prefix = "builtin:" + // Source is the source of the image row of a builtin image. + Source = "builtin" +) + +// Name returns the builtin image that the ref of an image row names. +func Name(ref string) (string, bool) { + name, ok := strings.CutPrefix(ref, Prefix) + return name, ok && slices.Contains(Names, name) +} + +// Ref returns the registry reference of a builtin image that the images +// workflow of the repository repo publishes. +func Ref(repo, name string) string { + return "ghcr.io/" + strings.ToLower(repo) + "/runner-" + name + ":latest" +} + +// BuildArgs returns the build args of the Dockerfile for a builtin image. +// IMAGE_OS is the ImageOS of the GitHub-hosted runners, for example +// "ubuntu24". +func BuildArgs(name string) map[string]string { + release := strings.TrimPrefix(name, "ubuntu-") + major, _, _ := strings.Cut(release, ".") + return map[string]string{"UBUNTU": release, "IMAGE_OS": "ubuntu" + major} +} diff --git a/images/images_test.go b/images/images_test.go new file mode 100644 index 0000000..6ca6f4c --- /dev/null +++ b/images/images_test.go @@ -0,0 +1,73 @@ +package images + +import ( + "maps" + "os" + "regexp" + "slices" + "strings" + "testing" +) + +// The images workflow builds and publishes the builtin images, and the +// README lists them. A new image or a new build arg must change +// images.go, .github/workflows/images.yml, and images/README.md together. +func TestWorkflowBuildsTheBuiltinImages(t *testing.T) { + yml, err := os.ReadFile("../.github/workflows/images.yml") + if err != nil { + t.Fatal(err) + } + entry := regexp.MustCompile(`(?m)^\s+- \{name: (\S+), ubuntu: '([^']+)', image_os: (\S+)\}$`) + var built []string + for _, m := range entry.FindAllStringSubmatch(string(yml), -1) { + built = append(built, m[1]) + want := BuildArgs(m[1]) + if got := map[string]string{"UBUNTU": m[2], "IMAGE_OS": m[3]}; !maps.Equal(got, want) { + t.Errorf("images.yml builds %s with %v, images.go gives %v", m[1], got, want) + } + } + if !slices.Equal(built, Names) { + t.Errorf("the build matrix of images.yml has %v, want %v", built, Names) + } + merged := regexp.MustCompile(`(?m)^\s+image: \[(.*)\]$`).FindStringSubmatch(string(yml)) + if merged == nil || !slices.Equal(strings.Split(merged[1], ", "), Names) { + t.Errorf("the merge matrix of images.yml has %q, want %v", merged, Names) + } + + dockerfile, err := os.ReadFile("Dockerfile") + if err != nil { + t.Fatal(err) + } + // A build arg is in scope only after its ARG in the stage. + _, stage, _ := strings.Cut(string(dockerfile), "\nFROM ") + for arg := range BuildArgs(Default) { + if !strings.Contains(stage, "\nARG "+arg+"\n") { + t.Errorf("the Dockerfile has no ARG %s after FROM", arg) + } + } +} + +func TestReadmeListsTheBuiltinImages(t *testing.T) { + readme, err := os.ReadFile("README.md") + if err != nil { + t.Fatal(err) + } + var refs []string + def := "" + for _, m := range regexp.MustCompile("(?m)^\\| `(ghcr\\.io/[^`]+)` \\| (.*) \\|$").FindAllStringSubmatch(string(readme), -1) { + refs = append(refs, m[1]) + if strings.Contains(m[2], "default") { + def = m[1] + } + } + var want []string + for _, name := range Names { + want = append(want, Ref("runplane/runplane", name)) + } + if !slices.Equal(refs, want) { + t.Errorf("the README table has %v, want %v", refs, want) + } + if want := Ref("runplane/runplane", Default); def != want { + t.Errorf("the README marks %q as the default, want %q", def, want) + } +} diff --git a/images/ubuntu-26.04/Dockerfile b/images/ubuntu-26.04/Dockerfile deleted file mode 100644 index 1dc2f3b..0000000 --- a/images/ubuntu-26.04/Dockerfile +++ /dev/null @@ -1,135 +0,0 @@ -# runplane base image for GitHub Actions jobs: Ubuntu 26.04. -# -# runplane starts runplane-init as the first process of each sandbox, so this -# image has no ENTRYPOINT. runplane-init starts dockerd and the runner as the -# user "runner". See images/README.md for custom images FROM this image. -FROM ubuntu:26.04 - -LABEL org.opencontainers.image.title="runplane runner, Ubuntu 26.04" \ - org.opencontainers.image.description="Base image for runplane sandboxes" \ - org.opencontainers.image.source="https://github.com/runplane/runplane" \ - dev.runplane.managed=true - -# Major versions for the tool cache. The build installs the latest release of each. -ARG NODE_VERSIONS="22 24 26" -ARG NODE_DEFAULT=24 -ARG PYTHON_VERSIONS="3.11 3.12 3.13 3.14" -ARG GO_VERSIONS="1.27" - -ENV LANG=C.UTF-8 \ - TZ=Etc/UTC \ - ImageOS=ubuntu26 \ - RUNNER_TOOL_CACHE=/opt/hostedtoolcache \ - AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache \ - PATH=/home/runner/.cargo/bin:$PATH - -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] - -# Base packages and the libraries that actions/runner needs -# (bin/installdependencies.sh). The first available name of each group wins. -RUN export DEBIAN_FRONTEND=noninteractive; \ - apt-get update -q; \ - first() { for p in "$@"; do if apt-cache show "$p" >/dev/null 2>&1; then echo "$p"; return; fi; done; echo "none of: $*" >&2; return 1; }; \ - apt-get install -y -q --no-install-recommends \ - ca-certificates curl wget gnupg git git-lfs jq zstd unzip zip xz-utils bzip2 \ - build-essential pkg-config cmake \ - python3 python3-pip python3-venv python3-dev \ - sudo locales tzdata lsb-release openssh-client rsync file less vim-tiny \ - iproute2 iptables iputils-ping dnsutils netcat-openbsd procps psmisc time \ - libkrb5-3 zlib1g \ - "$(first liblttng-ust1t64 liblttng-ust1 liblttng-ust0)" \ - "$(first libssl3t64 libssl3 libssl1.1)" \ - "$(first libicu80 libicu79 libicu78 libicu77 libicu76 libicu75 libicu74 libicu72 libicu70)"; \ - sed -i 's/^# *en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen; \ - locale-gen; \ - rm -rf /var/lib/apt/lists/* - -# Docker Engine with buildx and compose, and the GitHub CLI. -RUN export DEBIAN_FRONTEND=noninteractive; \ - arch=$(dpkg --print-architecture); \ - codename=$(. /etc/os-release && echo "$VERSION_CODENAME"); \ - install -m 0755 -d /etc/apt/keyrings; \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc; \ - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /etc/apt/keyrings/githubcli.gpg; \ - echo "deb [arch=$arch signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $codename stable" \ - > /etc/apt/sources.list.d/docker.list; \ - echo "deb [arch=$arch signed-by=/etc/apt/keyrings/githubcli.gpg] https://cli.github.com/packages stable main" \ - > /etc/apt/sources.list.d/github-cli.list; \ - apt-get update -q; \ - apt-get install -y -q --no-install-recommends \ - docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin gh; \ - ln -s /usr/libexec/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose; \ - rm -rf /var/lib/apt/lists/* - -# The runner user, and the folders that runplane-init uses. -RUN useradd --create-home --uid 1001 --shell /bin/bash --groups docker runner; \ - echo 'runner ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/runner; \ - chmod 0440 /etc/sudoers.d/runner; \ - mkdir -p /run/runplane /mnt/scratch /home/runner/actions-runner "$RUNNER_TOOL_CACHE"; \ - chown runner:runner /home/runner/actions-runner; \ - printf '%s\n' "LANG=$LANG" "ImageOS=$ImageOS" "RUNNER_TOOL_CACHE=$RUNNER_TOOL_CACHE" \ - "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" >> /etc/environment - -# The tool cache has the layout of the GitHub-hosted runners, so that -# actions/setup-node, setup-python, and setup-go find these versions and do not -# download them: /// and a //.complete file. -# The Python test suites and static libpython archives use 1 GB, and jobs -# seldom need them, so the build removes them. -RUN case $(dpkg --print-architecture) in \ - amd64) arch=x64 goarch=amd64 ;; \ - arm64) arch=arm64 goarch=arm64 ;; \ - esac; \ - cd /tmp; \ - curl -fsSL https://nodejs.org/dist/index.json -o node.json; \ - for major in $NODE_VERSIONS; do \ - v=$(jq -r --arg m "v$major." 'map(select(.version | startswith($m)))[0].version' node.json); \ - file=node-$v-linux-$arch.tar.xz; \ - curl -fsSL "https://nodejs.org/dist/$v/$file" -o "$file"; \ - curl -fsSL "https://nodejs.org/dist/$v/SHASUMS256.txt" | grep " $file\$" | sha256sum -c -; \ - dir=$RUNNER_TOOL_CACHE/node/${v#v}/$arch; \ - mkdir -p "$dir"; \ - tar -xJf "$file" -C "$dir" --strip-components=1 --no-same-owner; \ - touch "$dir.complete"; \ - rm "$file"; \ - done; \ - default=$(ls -d "$RUNNER_TOOL_CACHE/node/$NODE_DEFAULT".*/"$arch"); \ - ln -s "$default"/bin/* /usr/local/bin/; \ - \ - curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json -o python.json; \ - os=$(. /etc/os-release && echo "$VERSION_ID"); \ - for minor in $PYTHON_VERSIONS; do \ - url=$(jq -r --arg m "$minor." --arg os "$os" --arg arch "$arch" \ - 'first(.[] | select(.stable and (.version | startswith($m))) | .files[] | select(.platform == "linux" and .platform_version == $os and .arch == $arch)) | .download_url' python.json); \ - mkdir python; \ - curl -fsSL "$url" | tar -xz -C python; \ - (cd python && bash ./setup.sh >/dev/null); \ - rm -rf python; \ - done; \ - rm -rf "$RUNNER_TOOL_CACHE"/Python/*/"$arch"/lib/python3.*/test \ - "$RUNNER_TOOL_CACHE"/Python/*/"$arch"/lib/python3.*/config-*/libpython3.*.a; \ - \ - curl -fsSL 'https://go.dev/dl/?mode=json&include=all' -o go.json; \ - for minor in $GO_VERSIONS; do \ - read -r v sum < <(jq -r --arg m "go$minor" --arg arch "$goarch" \ - 'first(.[] | select(.stable and (.version == $m or (.version | startswith($m + "."))))) | .files[] | select(.os == "linux" and .arch == $arch and .kind == "archive") | "\(.version) \(.sha256)"' go.json); \ - curl -fsSL "https://go.dev/dl/$v.linux-$goarch.tar.gz" -o go.tar.gz; \ - echo "$sum go.tar.gz" | sha256sum -c -; \ - dir=$RUNNER_TOOL_CACHE/go/${v#go}/$arch; \ - mkdir -p "$dir"; \ - tar -xzf go.tar.gz -C "$dir" --strip-components=1 --no-same-owner; \ - touch "$dir.complete"; \ - rm go.tar.gz; \ - done; \ - \ - rm -rf /tmp/* /root/.cache; \ - chown -R runner:runner "$RUNNER_TOOL_CACHE" - -# rustup as on the GitHub-hosted runners, but with no toolchain: a stable -# toolchain adds 540 to 640 MB. A job installs the toolchain of its -# rust-toolchain.toml, or runs "rustup default stable". Login shells -# (su -, sudo -i) read PATH from /etc/environment, not from the ENV above. -RUN runuser -u runner -- env HOME=/home/runner bash -o pipefail -c \ - 'curl --proto =https --tlsv1.2 -fsSL https://sh.rustup.rs | sh -s -- -y -q --no-modify-path \ - --profile minimal --default-toolchain none'; \ - sed -i 's|^PATH="|PATH="/home/runner/.cargo/bin:|' /etc/environment; \ - grep -q '^PATH="/home/runner/.cargo/bin:' /etc/environment diff --git a/init/account.go b/init/account.go index 59cf3c1..924f1c4 100644 --- a/init/account.go +++ b/init/account.go @@ -62,8 +62,6 @@ func lookupAccount(name, passwdPath, groupPath string) (account, error) { return a, nil } -// env returns the environment for a process of the account: the init -// environment without TERM, with the account's HOME, USER, LOGNAME, and SHELL. func (a account) env(extra ...string) []string { env := slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "TERM=") }) if os.Getenv("LANG") == "" { diff --git a/init/control.go b/init/control.go index 288b104..0a9ce9f 100644 --- a/init/control.go +++ b/init/control.go @@ -1,11 +1,8 @@ package main import ( - "bytes" "context" - "encoding/json" "errors" - "fmt" "io" "io/fs" "log/slog" @@ -16,10 +13,8 @@ import ( "sync" "time" - "github.com/hashicorp/yamux" - "github.com/runplane/runplane/internal/sandbox" - "github.com/runplane/runplane/internal/sandbox/control" + "github.com/runplane/runplane/internal/sandbox/guest" ) // guestLink is the guest end of the control channel. It dials the host again @@ -32,7 +27,7 @@ type guestLink struct { mux http.Handler // what the guest serves to the host mu sync.Mutex - conn *linkConn // nil while disconnected + conn *guest.Client // nil while disconnected up chan struct{} // closed when a connection opens events []sandbox.RunnerEvent // not sent yet newEvent chan struct{} // closed and replaced on each new event @@ -41,11 +36,6 @@ type guestLink struct { logsOnce sync.Once } -type linkConn struct { - client *http.Client - done <-chan struct{} // closed when the connection drops -} - func newGuestLink(dial func(context.Context) (net.Conn, error), log *slog.Logger, logs *logQueue, mux http.Handler) *guestLink { return &guestLink{ dial: dial, @@ -72,34 +62,20 @@ func (l *guestLink) run(ctx context.Context) { continue } failing = false - cfg := yamux.DefaultConfig() - cfg.LogOutput = nil - cfg.Logger = slog.NewLogLogger(l.log.Handler(), slog.LevelDebug) - sess, err := yamux.Client(conn, cfg) + c, err := guest.NewClient(conn, l.mux, l.log) if err != nil { - conn.Close() - l.log.Warn("start yamux session", "err", err) + l.log.Warn("connect to the host", "err", err) sleep(ctx, time.Second) continue } - l.serve(ctx, sess) + l.serve(ctx, c) sleep(ctx, time.Second) } } -func (l *guestLink) serve(ctx context.Context, sess *yamux.Session) { - srv := &http.Server{Handler: l.mux, ErrorLog: slog.NewLogLogger(l.log.Handler(), slog.LevelDebug)} - go srv.Serve(sess) - c := &linkConn{ - client: &http.Client{Transport: &http.Transport{ - DialContext: func(context.Context, string, string) (net.Conn, error) { return sess.Open() }, - }}, - done: sess.CloseChan(), - } - +func (l *guestLink) serve(ctx context.Context, c *guest.Client) { l.mu.Lock() if l.haveConfig { - l.events = append([]sandbox.RunnerEvent{{Kind: sandbox.RunnerResume}}, l.events...) l.log.Info("connected to the host again") } l.conn = c @@ -109,9 +85,9 @@ func (l *guestLink) serve(ctx context.Context, sess *yamux.Session) { go l.sendEvents(c) go l.streamLogs(c) select { - case <-c.done: + case <-c.Done(): case <-ctx.Done(): - sess.Close() + c.Close() } l.mu.Lock() @@ -120,15 +96,15 @@ func (l *guestLink) serve(ctx context.Context, sess *yamux.Session) { l.mu.Unlock() } -// config gets the GuestConfig. It waits for a connection and asks again on -// the next connection if a request fails. Call it one time only. -func (l *guestLink) config(ctx context.Context) (control.GuestConfig, error) { +// config waits for a connection. When a request fails, it asks again after a +// second or on the next connection. Call it one time only. +func (l *guestLink) config(ctx context.Context) (guest.Config, error) { for { c, err := l.wait(ctx) if err != nil { - return control.GuestConfig{}, err + return guest.Config{}, err } - cfg, err := getConfig(ctx, c.client) + cfg, err := c.Config(ctx) if err == nil { l.mu.Lock() l.haveConfig = true @@ -137,33 +113,15 @@ func (l *guestLink) config(ctx context.Context) (control.GuestConfig, error) { } l.log.Warn("get config", "err", err) select { - case <-c.done: + case <-c.Done(): case <-time.After(time.Second): case <-ctx.Done(): - return control.GuestConfig{}, ctx.Err() + return guest.Config{}, ctx.Err() } } } -func getConfig(ctx context.Context, client *http.Client) (control.GuestConfig, error) { - var cfg control.GuestConfig - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://host/v1/config", nil) - if err != nil { - return cfg, err - } - resp, err := client.Do(req) - if err != nil { - return cfg, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return cfg, fmt.Errorf("host answered %s", resp.Status) - } - err = json.NewDecoder(resp.Body).Decode(&cfg) - return cfg, err -} - -func (l *guestLink) wait(ctx context.Context) (*linkConn, error) { +func (l *guestLink) wait(ctx context.Context) (*guest.Client, error) { for { l.mu.Lock() c, up := l.conn, l.up @@ -187,7 +145,7 @@ func (l *guestLink) event(ev sandbox.RunnerEvent) { l.newEvent = make(chan struct{}) } -func (l *guestLink) sendEvents(c *linkConn) { +func (l *guestLink) sendEvents(c *guest.Client) { for { l.mu.Lock() if len(l.events) == 0 { @@ -196,30 +154,30 @@ func (l *guestLink) sendEvents(c *linkConn) { select { case <-wake: continue - case <-c.done: + case <-c.Done(): return } } ev := l.events[0] l.mu.Unlock() - status, err := postEvent(c.client, ev) + err := postEvent(c, ev) select { - case <-c.done: + case <-c.Done(): // The host may not have the event. Send it again on the next connection. return default: } - if err != nil { + switch { + case errors.Is(err, guest.ErrRejected): + l.log.Warn("the host rejected a runner event", "kind", ev.Kind, "err", err) + case err != nil: l.log.Warn("send runner event", "kind", ev.Kind, "err", err) select { case <-time.After(time.Second): - case <-c.done: + case <-c.Done(): } continue } - if status != http.StatusNoContent { - l.log.Warn("the host rejected a runner event", "kind", ev.Kind, "status", status) - } l.mu.Lock() if len(l.events) > 0 && l.events[0] == ev { l.events = l.events[1:] @@ -228,21 +186,10 @@ func (l *guestLink) sendEvents(c *linkConn) { } } -func postEvent(client *http.Client, ev sandbox.RunnerEvent) (int, error) { - body, _ := json.Marshal(ev) // only strings and an int +func postEvent(c *guest.Client, ev sandbox.RunnerEvent) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://host/v1/events", bytes.NewReader(body)) - if err != nil { - return 0, err - } - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - return 0, err - } - resp.Body.Close() - return resp.StatusCode, nil + return c.PostEvent(ctx, ev) } func (l *guestLink) flushEvents(ctx context.Context) { @@ -266,24 +213,15 @@ func (l *guestLink) flushEvents(ctx context.Context) { // streamLogs sends the log queue as one streamed request until the // connection drops or the queue closes. The host answers 204 only after the // end of the queue. -func (l *guestLink) streamLogs(c *linkConn) { +func (l *guestLink) streamLogs(c *guest.Client) { for { - req, err := http.NewRequest(http.MethodPost, "http://host/v1/logs", l.logs.reader(c.done)) - if err != nil { - return - } - req.Header.Set("Content-Type", "text/plain") - resp, err := c.client.Do(req) + err := c.StreamLogs(context.Background(), l.logs.reader(c.Done())) if err == nil { - resp.Body.Close() - if resp.StatusCode == http.StatusNoContent { - l.logsOnce.Do(func() { close(l.logsDone) }) - return - } - err = fmt.Errorf("host answered %s", resp.Status) + l.logsOnce.Do(func() { close(l.logsDone) }) + return } select { - case <-c.done: + case <-c.Done(): return case <-time.After(time.Second): l.log.Warn("log stream broke, starting it again", "err", err) @@ -300,21 +238,22 @@ func (l *guestLink) flushLogs(ctx context.Context) { } // dialHost connects to the host end of the control channel in container and -// dev mode. In dev mode the ctl folder also holds control.addr: then the -// guest dials TCP and sends the hello line first. It reads the file at each -// dial, because the host writes a new token after a restart. -func dialHost(ctx context.Context, socket string) (net.Conn, error) { +// dev mode: the socket in the ctl folder dir. In dev mode, dir has the +// guest.AddrFile instead: then the guest dials TCP and sends the hello line +// first. It reads the file at each dial, because the host writes a new token +// after a restart. +func dialHost(ctx context.Context, dir string) (net.Conn, error) { // A connect that hangs, for example in the TCP proxy of Docker on // macOS, would otherwise wait for the kernel limit of about 2 minutes. d := net.Dialer{Timeout: 5 * time.Second} - b, err := os.ReadFile(filepath.Join(filepath.Dir(socket), control.AddrFile)) + b, err := os.ReadFile(filepath.Join(dir, guest.AddrFile)) if errors.Is(err, fs.ErrNotExist) { - return d.DialContext(ctx, "unix", socket) + return d.DialContext(ctx, "unix", filepath.Join(dir, guest.SocketFile)) } if err != nil { return nil, err } - addr, hello, err := control.ParseAddr(b) + addr, hello, err := guest.ParseAddr(b) if err != nil { return nil, err } diff --git a/init/control_test.go b/init/control_test.go index 5bb7144..150c1bf 100644 --- a/init/control_test.go +++ b/init/control_test.go @@ -7,42 +7,58 @@ import ( "io" "log/slog" "net" + "net/netip" "os" "path/filepath" "reflect" "slices" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/sandbox/control" + "github.com/runplane/runplane/internal/sandbox/guest" ) func TestControlChannel(t *testing.T) { sock := socketPath(t) log := slog.New(slog.DiscardHandler) - cfg := control.GuestConfig{ + cfg := guest.Config{ SandboxID: "01jbz3x0000000000000000000", Hostname: "ubuntu-24-04-01jbz3x0", JITConfig: "eyJqaXQiOiJjb25maWcifQ==", RunnerDir: "/opt/runplane/runner", - Env: map[string]string{"ACTIONS_RESULTS_URL": "http://10.0.0.1:8486/"}, - DNS: []string{"1.1.1.1", "8.8.8.8"}, + CacheURL: "http://10.0.0.1:8486", + DNS: []netip.Addr{netip.MustParseAddr("1.1.1.1"), netip.MustParseAddr("8.8.8.8")}, RegistryMirror: "http://10.0.0.1:8486", TimeLimitSeconds: 3600, Debug: true, } - rec1 := &recorder{} - host1, err := control.Listen(sock, cfg, rec1, log) + // host1 takes the job.started event, but does not answer. + taking, release := make(chan struct{}, 1), make(chan struct{}) + defer close(release) + rec1 := &recorder{onEvent: func(ev sandbox.RunnerEvent) { + if ev.Kind == "job.started" { + select { + case taking <- struct{}{}: + default: + } + <-release + } + }} + host1, err := control.Listen(sock, &cfg, rec1, log) if err != nil { t.Fatal(err) } sent := &syncBuffer{} + var dials atomic.Int32 dial := func(ctx context.Context) (net.Conn, error) { - c, err := dialHost(ctx, sock) + dials.Add(1) + c, err := dialHost(ctx, filepath.Dir(sock)) if err != nil { return nil, err } @@ -66,7 +82,7 @@ func TestControlChannel(t *testing.T) { link.event(sandbox.RunnerEvent{Kind: "listening"}) logs.push("runner", "√ Connected to GitHub") logs.push("dockerd", "") - long := strings.Repeat("x", control.MaxLine+10) + long := strings.Repeat("x", guest.MaxLine+10) logs.push("diag", long) waitFor(t, "first events and logs", func() bool { return len(rec1.eventList()) == 1 && len(rec1.logList()) == 4 @@ -86,37 +102,41 @@ func TestControlChannel(t *testing.T) { lines[2].Source, lines[3].Source, len(lines[2].Text), len(lines[3].Text)) } - // runplane restarts: the guest keeps events and logs, connects again, - // sends resume first, and does not ask for the config again. + // runplane restarts while it takes an event: the guest keeps that event + // and the events and logs of the outage, connects again, and does not + // ask for the config again. + started := sandbox.RunnerEvent{Kind: "job.started", JobName: "build"} + link.event(started) + select { + case <-taking: + case <-time.After(10 * time.Second): + t.Fatal("the guest did not send the job.started event") + } + before := dials.Load() host1.Close() - waitFor(t, "the guest to see the host leave", func() bool { - link.mu.Lock() - defer link.mu.Unlock() - return link.conn == nil - }) - link.event(sandbox.RunnerEvent{Kind: "job.started", JobName: "build"}) + // The guest dials again only after it dropped the old connection. + waitFor(t, "the guest to dial again", func() bool { return dials.Load() > before }) + completed := sandbox.RunnerEvent{Kind: "job.completed", JobName: "build", Result: "Succeeded"} + link.event(completed) logs.push("runner", "while the host was away") rec2 := &recorder{} - host2, err := control.Listen(sock, control.GuestConfig{}, rec2, log) + host2, err := control.Listen(sock, nil, rec2, log) if err != nil { t.Fatal(err) } defer host2.Close() waitFor(t, "events after the restart", func() bool { return len(rec2.eventList()) == 2 }) - want := []sandbox.RunnerEvent{{Kind: "resume"}, {Kind: "job.started", JobName: "build"}} + want := []sandbox.RunnerEvent{started, completed} if ev := rec2.eventList(); !slices.Equal(ev, want) { t.Errorf("events after the restart = %+v, want %+v", ev, want) } waitFor(t, "the log line from the outage", func() bool { - return slices.ContainsFunc(rec2.logList(), func(l control.LogLine) bool { return l.Text == "while the host was away" }) + return slices.ContainsFunc(rec2.logList(), func(l guest.LogLine) bool { return l.Text == "while the host was away" }) }) // HTTP requests travel as plain text inside yamux frames. if n := strings.Count(sent.String(), "GET /v1/config "); n != 1 { t.Errorf("the guest asked for the config %d times, want 1", n) } - if !host2.Connected() { - t.Error("Connected() = false with a guest on the line") - } // A hold keeps init after the runner exits, until a shutdown. if err := host2.Hold(ctx, time.Now().Add(time.Hour)); err != nil { @@ -159,10 +179,9 @@ func TestControlChannel(t *testing.T) { } cancel() - waitFor(t, "the host to see the guest leave", func() bool { return !host2.Connected() }) - if err := host2.Hold(context.Background(), time.Now()); err != control.ErrNotConnected { - t.Errorf("Hold without a guest = %v, want ErrNotConnected", err) - } + waitFor(t, "Hold to give ErrNotConnected after the guest left", func() bool { + return host2.Hold(context.Background(), time.Now()) == control.ErrNotConnected + }) } // In dev mode the guest dials the TCP address in control.addr. After a host @@ -170,7 +189,7 @@ func TestControlChannel(t *testing.T) { func TestDialHostTCPAfterRestart(t *testing.T) { sock := socketPath(t) log := slog.New(slog.DiscardHandler) - link := newGuestLink(func(ctx context.Context) (net.Conn, error) { return dialHost(ctx, sock) }, + link := newGuestLink(func(ctx context.Context) (net.Conn, error) { return dialHost(ctx, filepath.Dir(sock)) }, log, newLogQueue(1<<20), guestMux(newHostRequests(), newShellServer(log))) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -179,14 +198,15 @@ func TestDialHostTCPAfterRestart(t *testing.T) { if err != nil { t.Fatal(err) } - host, err := tcp.Listen(filepath.Dir(sock), control.GuestConfig{SandboxID: "01jbz3x0000000000000000000"}, &recorder{}, log) + rec := &recorder{} + host, err := tcp.Listen(filepath.Dir(sock), "01jbz3x0000000000000000000", nil, rec, log) if err != nil { t.Fatal(err) } if i == 0 { go link.run(ctx) } - waitFor(t, fmt.Sprintf("connection %d over TCP", i+1), host.Connected) + waitFor(t, fmt.Sprintf("connection %d over TCP", i+1), func() bool { return rec.connects.Load() > 0 }) host.Close() tcp.Close() } @@ -220,14 +240,31 @@ func TestLogQueueDropsOldest(t *testing.T) { func TestWaitHoldInDebugWaitsForALateHold(t *testing.T) { req := newHostRequests() + holds := make(chan time.Time, 1) go func() { time.Sleep(20 * time.Millisecond) - req.setHold(time.Now().Add(300 * time.Millisecond)) + until := time.Now().Add(50 * time.Millisecond) + req.setHold(until) + holds <- until }() - start := time.Now() - req.waitHold(true, 100*time.Millisecond) - if d := time.Since(start); d < 250*time.Millisecond { - t.Errorf("waitHold returned after %v; the hold that came during the grace time is 300ms", d) + returned := make(chan time.Time, 1) + go func() { + req.waitHold(true, time.Hour) + returned <- time.Now() + }() + var end, until time.Time + select { + case end = <-returned: + case <-time.After(10 * time.Second): + t.Fatal("waitHold still waits after the hold ended") + } + select { + case until = <-holds: + case <-time.After(10 * time.Second): + t.Fatal("no hold") + } + if end.Before(until) { + t.Errorf("waitHold returned %v before the end of the hold that came during the grace time", until.Sub(end)) } } @@ -238,7 +275,7 @@ func socketPath(t *testing.T) string { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) - return filepath.Join(dir, "control.sock") + return filepath.Join(dir, guest.SocketFile) } func waitFor(t *testing.T, what string, cond func() bool) { @@ -252,64 +289,21 @@ func waitFor(t *testing.T, what string, cond func() bool) { } } -// After a runplane restart, the host has no JIT config for a guest that did -// not get its config yet, and it stops the sandbox. The guest must then stop -// asking for the config, or the VM never powers off. -func TestShutdownWhileWaitingForConfig(t *testing.T) { - sock := socketPath(t) - log := slog.New(slog.DiscardHandler) - var host *control.Server - shutdown := make(chan error, 1) - rec := &recorder{lost: func() { shutdown <- host.Shutdown(context.Background()) }} - host, err := control.Listen(sock, control.GuestConfig{SandboxID: "s"}, rec, log) - if err != nil { - t.Fatal(err) - } - defer host.Close() - - req := newHostRequests() - link := newGuestLink(func(ctx context.Context) (net.Conn, error) { return dialHost(ctx, sock) }, - log, newLogQueue(1<<20), guestMux(req, newShellServer(log))) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go link.run(ctx) - // As guest.run does: a shutdown ends the wait for the config. - configCtx, stop := context.WithCancel(ctx) - go func() { - <-req.stop - stop() - }() - done := make(chan error, 1) - go func() { - _, err := link.config(configCtx) - done <- err - }() - select { - case err := <-done: - if err == nil { - t.Error("the guest got a config without a JIT config") - } - case <-time.After(10 * time.Second): - t.Fatal("the guest still asks for its config after the host stopped the sandbox") - } - select { - case err := <-shutdown: - if err != nil { - t.Errorf("shutdown: %v", err) - } - case <-time.After(10 * time.Second): - t.Fatal("the host never asked the guest to stop") - } -} - type recorder struct { - mu sync.Mutex - events []sandbox.RunnerEvent - logs []control.LogLine - lost func() // OnLostConfig, if set + connects atomic.Int32 + mu sync.Mutex + events []sandbox.RunnerEvent + logs []guest.LogLine + lost func() // OnLostConfig, if set + onEvent func(sandbox.RunnerEvent) // runs before OnEvent keeps the event, if set } +func (r *recorder) OnConnect() { r.connects.Add(1) } + func (r *recorder) OnEvent(ev sandbox.RunnerEvent) { + if r.onEvent != nil { + r.onEvent(ev) + } r.mu.Lock() defer r.mu.Unlock() r.events = append(r.events, ev) @@ -321,7 +315,7 @@ func (r *recorder) OnLostConfig() { } } -func (r *recorder) OnLog(l control.LogLine) { +func (r *recorder) OnLog(l guest.LogLine) { r.mu.Lock() defer r.mu.Unlock() r.logs = append(r.logs, l) @@ -333,13 +327,12 @@ func (r *recorder) eventList() []sandbox.RunnerEvent { return slices.Clone(r.events) } -func (r *recorder) logList() []control.LogLine { +func (r *recorder) logList() []guest.LogLine { r.mu.Lock() defer r.mu.Unlock() return slices.Clone(r.logs) } -// recordConn copies what the guest writes to sent. type recordConn struct { net.Conn sent io.Writer @@ -350,7 +343,6 @@ func (c *recordConn) Write(p []byte) (int, error) { return c.Conn.Write(p) } -// syncBuffer is a bytes.Buffer that goroutines can share. type syncBuffer struct { mu sync.Mutex buf bytes.Buffer diff --git a/init/logqueue.go b/init/logqueue.go index 8ea0cfb..66b632b 100644 --- a/init/logqueue.go +++ b/init/logqueue.go @@ -4,12 +4,11 @@ import ( "errors" "fmt" "io" - "strconv" "strings" "sync" "time" - "github.com/runplane/runplane/internal/sandbox/control" + "github.com/runplane/runplane/internal/sandbox/guest" ) // logQueueMax is the memory for log lines while the host is away. @@ -17,8 +16,8 @@ const logQueueMax = 8 << 20 var errLinkDown = errors.New("control connection dropped") -// logQueue holds log lines for the host in the wire form -// " \n". When it is full, it drops the oldest lines. +// logQueue holds log lines for the host in the form of the log stream +// (guest.AppendLogLine). When it is full, it drops the oldest lines. type logQueue struct { max int @@ -34,25 +33,15 @@ func newLogQueue(max int) *logQueue { return &logQueue{max: max, wake: make(chan struct{})} } -// push queues a line. It splits a text longer than control.MaxLine, so that -// the host can read each line in one piece. func (q *logQueue) push(source, text string) { - now := time.Now() + b := guest.AppendLogLine(nil, guest.LogLine{Time: time.Now(), Source: source, Text: text}) q.mu.Lock() defer q.mu.Unlock() if q.closed { return } - for { - part := text[:min(len(text), control.MaxLine)] - b := appendLine(nil, now, source, part) - q.lines = append(q.lines, b) - q.size += len(b) - text = text[len(part):] - if text == "" { - break - } - } + q.lines = append(q.lines, b) + q.size += len(b) for q.size > q.max && len(q.lines) > 0 { q.size -= len(q.lines[0]) q.lines[0] = nil @@ -89,7 +78,11 @@ func (q *logQueue) take(done <-chan struct{}) ([]byte, error) { if len(q.lines) > 0 || q.dropped > 0 { var b []byte if q.dropped > 0 { - b = appendLine(b, time.Now(), "init", fmt.Sprintf("dropped %d older log lines: the log buffer was full", q.dropped)) + b = guest.AppendLogLine(b, guest.LogLine{ + Time: time.Now(), + Source: guest.SourceInit, + Text: fmt.Sprintf("dropped %d older log lines: the log buffer was full", q.dropped), + }) q.dropped = 0 } for _, line := range q.lines { @@ -152,12 +145,3 @@ func (w lineWriter) Write(p []byte) (int, error) { } return len(p), nil } - -func appendLine(b []byte, t time.Time, source, text string) []byte { - b = strconv.AppendInt(b, t.UnixMilli(), 10) - b = append(b, ' ') - b = append(b, source...) - b = append(b, ' ') - b = append(b, text...) - return append(b, '\n') -} diff --git a/init/main_linux.go b/init/main_linux.go index b06f5c2..4eac038 100644 --- a/init/main_linux.go +++ b/init/main_linux.go @@ -14,22 +14,15 @@ import ( "net" "os" "os/signal" - "path/filepath" - "time" "github.com/mdlayher/vsock" "golang.org/x/sys/unix" "github.com/runplane/runplane/internal/sandbox" - "github.com/runplane/runplane/internal/sandbox/control" + "github.com/runplane/runplane/internal/sandbox/guest" ) -const ( - controlSocket = "/run/runplane/control.sock" // container mode - vsockPort = 1024 // VM mode, on the host (CID 2) -) - -type guest struct { +type guestInit struct { vm bool log *slog.Logger logs *logQueue @@ -37,7 +30,7 @@ type guest struct { req *hostRequests shells *shellServer link *guestLink - cfg control.GuestConfig + cfg guest.Config } func main() { @@ -45,7 +38,7 @@ func main() { os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") } logs := newLogQueue(logQueueMax) - log := slog.New(slog.NewTextHandler(io.MultiWriter(os.Stderr, logs.writer("init")), &slog.HandlerOptions{ + log := slog.New(slog.NewTextHandler(io.MultiWriter(os.Stderr, logs.writer(guest.SourceInit)), &slog.HandlerOptions{ ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { if a.Key == slog.TimeKey && len(groups) == 0 { return slog.Attr{} // each log line has its own time @@ -53,7 +46,7 @@ func main() { return a }, })) - g := &guest{log: log, logs: logs, reaper: newReaper(), req: newHostRequests(), shells: newShellServer(log)} + g := &guestInit{log: log, logs: logs, reaper: newReaper(), req: newHostRequests(), shells: newShellServer(log)} g.handleSignals() vm, err := detectVM() @@ -67,10 +60,10 @@ func main() { log.Error("set up the VM", "err", err) powerOff(true, 1) } - if err := setImageEnv(sandbox.ImageEnvFile); err != nil { + if err := setImageEnv(guest.ImageEnvFile); err != nil { log.Warn("set the environment of the image", "err", err) } - } else if _, err := os.Stat(filepath.Dir(controlSocket)); err != nil { + } else if _, err := os.Stat(guest.CtlDir); err != nil { log.Error("no VM and no control socket folder: is this a runplane sandbox?", "err", err) os.Exit(1) } @@ -82,9 +75,8 @@ func main() { powerOff(vm, code) } -// run gets the config, prepares the sandbox, runs the runner, and waits for -// a hold. It returns the exit code of init in container mode. -func (g *guest) run() int { +// run returns the exit code of init in container mode. +func (g *guestInit) run() int { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { @@ -111,7 +103,7 @@ func (g *guest) run() int { code, failure = g.runRunner(acct) } g.link.event(sandbox.RunnerEvent{Kind: sandbox.RunnerExited, ExitCode: code, Error: failure}) - g.req.waitHold(cfg.Debug, holdGrace) + g.req.waitHold(cfg.Debug, guest.HoldGrace) if dockerd != nil { dockerd.stop() } @@ -121,7 +113,7 @@ func (g *guest) run() int { return 0 } -func (g *guest) prepare() (account, error) { +func (g *guestInit) prepare() (account, error) { if g.cfg.RunnerDir == "" { return account{}, errors.New("the config has no runner_dir") } @@ -133,7 +125,7 @@ func (g *guest) prepare() (account, error) { return account{}, err } } - acct, err := lookupAccount("runner", "/etc/passwd", "/etc/group") + acct, err := lookupAccount(guest.RunnerUser, "/etc/passwd", "/etc/group") if err != nil { return account{}, err } @@ -147,19 +139,16 @@ func (g *guest) prepare() (account, error) { return acct, nil } -// finish sends the last events and log lines. The host can be away, so it -// waits at most 10 seconds. -func (g *guest) finish() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) +func (g *guestInit) finish() { + ctx, cancel := context.WithTimeout(context.Background(), guest.FlushWait) defer cancel() g.link.flushEvents(ctx) g.log.Info("stopping the sandbox") g.link.flushLogs(ctx) } -// handleSignals turns SIGTERM and SIGINT into a shutdown. Docker stop sends -// SIGTERM; in a VM, Ctrl-Alt-Del sends SIGINT. -func (g *guest) handleSignals() { +// Docker stop sends SIGTERM; in a VM, Ctrl-Alt-Del sends SIGINT. +func (g *guestInit) handleSignals() { stop := make(chan os.Signal, 1) signal.Notify(stop, unix.SIGTERM, unix.SIGINT) // Without a handler, Go exits on SIGHUP, and on SIGPIPE from a write to @@ -181,10 +170,10 @@ func (g *guest) handleSignals() { func dialer(vm bool) func(context.Context) (net.Conn, error) { if vm { return func(ctx context.Context) (net.Conn, error) { - return vsock.DialContext(ctx, vsock.Host, vsockPort, nil) + return vsock.DialContext(ctx, vsock.Host, guest.VsockPort, nil) } } return func(ctx context.Context) (net.Conn, error) { - return dialHost(ctx, controlSocket) + return dialHost(ctx, guest.CtlDir) } } diff --git a/init/main_linux_test.go b/init/main_linux_test.go new file mode 100644 index 0000000..c8d4af7 --- /dev/null +++ b/init/main_linux_test.go @@ -0,0 +1,55 @@ +//go:build linux + +package main + +import ( + "context" + "log/slog" + "net" + "path/filepath" + "testing" + "time" + + "github.com/runplane/runplane/internal/sandbox/control" +) + +// After a runplane restart, the host has no JIT config for a guest that did +// not get its config yet, and it stops the sandbox. init must then stop +// asking for the config and end, or the VM never powers off. +func TestRunEndsWhenTheHostStopsBeforeTheConfig(t *testing.T) { + sock := socketPath(t) + log := slog.New(slog.DiscardHandler) + var host *control.Server + shutdown := make(chan error, 1) + rec := &recorder{lost: func() { shutdown <- host.Shutdown(context.Background()) }} + host, err := control.Listen(sock, nil, rec, log) + if err != nil { + t.Fatal(err) + } + defer host.Close() + + g := &guestInit{log: log, logs: newLogQueue(1 << 20), req: newHostRequests(), shells: newShellServer(log)} + g.link = newGuestLink(func(ctx context.Context) (net.Conn, error) { return dialHost(ctx, filepath.Dir(sock)) }, + log, g.logs, guestMux(g.req, g.shells)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go g.link.run(ctx) + code := make(chan int, 1) + go func() { code <- g.run() }() + select { + case c := <-code: + if c != 0 { + t.Errorf("run = %d, want 0 for a stop before the config", c) + } + case <-time.After(10 * time.Second): + t.Fatal("init still asks for its config after the host stopped the sandbox") + } + select { + case err := <-shutdown: + if err != nil { + t.Errorf("shutdown: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("the host never asked the guest to stop") + } +} diff --git a/init/overlay_linux_test.go b/init/overlay_linux_test.go index db00e57..3a9f9c9 100644 --- a/init/overlay_linux_test.go +++ b/init/overlay_linux_test.go @@ -8,6 +8,8 @@ import ( "testing" "golang.org/x/sys/unix" + + "github.com/runplane/runplane/internal/sandbox/guest" ) // Jobs write caches and tools to the home folders and /opt. In container @@ -20,13 +22,13 @@ func TestOverlayPutsWritesOnScratch(t *testing.T) { if os.Getenv("RUNPLANE_TEST_MOUNTS") != "1" || os.Geteuid() != 0 { t.Skip("set RUNPLANE_TEST_MOUNTS=1 and run as root in a throwaway container") } - if err := mkdirs(scratchDir, "/opt/hostedtoolcache/node"); err != nil { + if err := mkdirs(guest.ScratchDir, "/opt/hostedtoolcache/node"); err != nil { t.Fatal(err) } - if err := unix.Mount("tmpfs", scratchDir, "tmpfs", 0, ""); err != nil { + if err := unix.Mount("tmpfs", guest.ScratchDir, "tmpfs", 0, ""); err != nil { t.Fatal(err) } - t.Cleanup(func() { unix.Unmount("/opt", unix.MNT_DETACH); unix.Unmount(scratchDir, unix.MNT_DETACH) }) + t.Cleanup(func() { unix.Unmount("/opt", unix.MNT_DETACH); unix.Unmount(guest.ScratchDir, unix.MNT_DETACH) }) if err := os.WriteFile("/opt/hostedtoolcache/node/image", []byte("from the image"), 0o644); err != nil { t.Fatal(err) } @@ -43,7 +45,7 @@ func TestOverlayPutsWritesOnScratch(t *testing.T) { if err := os.WriteFile("/opt/hostedtoolcache/node/job", []byte("x"), 0o644); err != nil { t.Fatal(err) } - if _, err := os.Stat(filepath.Join(scratchDir, "root/opt/upper/hostedtoolcache/node/job")); err != nil { + if _, err := os.Stat(filepath.Join(guest.ScratchDir, "root/opt/upper/hostedtoolcache/node/job")); err != nil { t.Errorf("a file that the job wrote is not on scratch: %v", err) } if st, err := os.Stat("/opt"); err != nil || st.Mode().Perm() != 0o751 { diff --git a/init/pages_linux.go b/init/pages_linux.go index a3d26a3..486c4e6 100644 --- a/init/pages_linux.go +++ b/init/pages_linux.go @@ -13,7 +13,7 @@ import ( "golang.org/x/sys/unix" - "github.com/runplane/runplane/internal/sandbox/control" + "github.com/runplane/runplane/internal/sandbox/guest" ) // tailPages streams the lines of the job log pages that the runner writes @@ -138,7 +138,7 @@ func (p *page) read(emit func(string)) { emit(string(bytes.TrimSuffix(p.partial[:i], []byte("\r")))) p.partial = p.partial[i+1:] } - if len(p.partial) >= control.MaxLine { + if len(p.partial) >= guest.MaxLine { emit(string(p.partial)) p.partial = nil } diff --git a/init/reaper_linux.go b/init/reaper_linux.go index b2b2623..19c6e9c 100644 --- a/init/reaper_linux.go +++ b/init/reaper_linux.go @@ -36,11 +36,10 @@ func newReaper() *reaper { return r } -// start starts cmd. It returns the process ID and a channel that gets the -// exit code: the exit status, or 128 plus the signal number. Use only -// *os.File values for the standard streams of cmd, since nothing calls -// cmd.Wait. Do not read cmd.Process.Pid later: the reaper releases the -// process when it exits. +// start returns the process ID and a channel that gets the exit code: the +// exit status, or 128 plus the signal number. Use only *os.File values for +// the standard streams of cmd, since nothing calls cmd.Wait. Do not read +// cmd.Process.Pid later: the reaper releases the process when it exits. func (r *reaper) start(cmd *exec.Cmd) (int, <-chan int, error) { // Hold the lock across the start, so that reap finds the waiter even // when the child exits at once. diff --git a/init/requests.go b/init/requests.go index 10841fc..27a66bf 100644 --- a/init/requests.go +++ b/init/requests.go @@ -6,11 +6,9 @@ import ( "net/http" "sync" "time" -) -// holdGrace is how long a debug sandbox waits for a hold after the runner -// exits. The host sets the hold only after it sees the job complete. -const holdGrace = 2 * time.Minute + "github.com/runplane/runplane/internal/sandbox/guest" +) // hostRequests keeps what the host asks for: a hold time and a shutdown. type hostRequests struct { @@ -68,13 +66,10 @@ func (h *hostRequests) waitHold(debug bool, grace time.Duration) { } } -// guestMux serves the requests that the host sends to the guest. func guestMux(req *hostRequests, shells *shellServer) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("POST /v1/hold", func(w http.ResponseWriter, r *http.Request) { - var body struct { - Until time.Time `json:"until"` - } + mux.HandleFunc("POST "+guest.PathHold, func(w http.ResponseWriter, r *http.Request) { + var body guest.Hold if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body); err != nil { http.Error(w, "bad hold: "+err.Error(), http.StatusBadRequest) return @@ -82,11 +77,11 @@ func guestMux(req *hostRequests, shells *shellServer) *http.ServeMux { req.setHold(body.Until) w.WriteHeader(http.StatusNoContent) }) - mux.HandleFunc("POST /v1/shutdown", func(w http.ResponseWriter, _ *http.Request) { + mux.HandleFunc("POST "+guest.PathShutdown, func(w http.ResponseWriter, _ *http.Request) { req.shutdown() w.WriteHeader(http.StatusNoContent) }) - mux.HandleFunc("POST /v1/shell", shells.handleOpen) - mux.HandleFunc("POST /v1/shell/resize", shells.handleResize) + mux.HandleFunc("POST "+guest.PathShell, shells.handleOpen) + mux.HandleFunc("POST "+guest.PathShellResize, shells.handleResize) return mux } diff --git a/init/runner.go b/init/runner.go index e5f7c39..fbbdce0 100644 --- a/init/runner.go +++ b/init/runner.go @@ -1,14 +1,7 @@ package main import ( - "bytes" - "errors" - "fmt" - "log/slog" - "maps" - "os" "regexp" - "slices" "strings" "github.com/runplane/runplane/internal/sandbox" @@ -20,7 +13,6 @@ var ( completedRe = regexp.MustCompile(`^Job (.+) completed with result: (\S+)$`) ) -// parseRunnerLine finds the runner lifecycle events in a line of the runner stdout. func parseRunnerLine(line string) (sandbox.RunnerEvent, bool) { msg := strings.TrimSpace(line) if loc := stampRe.FindStringIndex(msg); loc != nil { @@ -37,53 +29,3 @@ func parseRunnerLine(line string) (sandbox.RunnerEvent, bool) { } return sandbox.RunnerEvent{}, false } - -// writeEnvFile writes the runner .env file. The runner sets each -// "key=value" line as an environment variable, so it skips an entry that -// cannot fit on one line. -func writeEnvFile(path string, env map[string]string, log *slog.Logger) error { - var b strings.Builder - for _, k := range slices.Sorted(maps.Keys(env)) { - v := env[k] - if k == "" || strings.ContainsAny(k, "=\r\n") || strings.ContainsAny(v, "\r\n") { - log.Warn("skipping a runner environment variable that does not fit in .env", "key", k) - continue - } - fmt.Fprintf(&b, "%s=%s\n", k, v) - } - return os.WriteFile(path, []byte(b.String()), 0o644) -} - -// The runner bundle on the host is patched: in bin/Runner.Worker.dll each -// UTF-16LE "ACTIONS_RESULTS_URL" is "ACTIONS_RESULTS_ORL", so the .env value -// of runplane's cache server reaches the job. A sandbox without that .env -// value needs the original name back, else the job has no results URL and -// artifacts and the GitHub cache stop working. -var ( - resultsURL = utf16le("ACTIONS_RESULTS_URL") - resultsORL = utf16le("ACTIONS_RESULTS_ORL") -) - -// unpatchWorker gives the runner in file its original results variable name. -// An unpatched or missing file stays as it is. -func unpatchWorker(file string) error { - b, err := os.ReadFile(file) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return err - } - if !bytes.Contains(b, resultsORL) { - return nil - } - return os.WriteFile(file, bytes.ReplaceAll(b, resultsORL, resultsURL), 0o644) -} - -func utf16le(s string) []byte { - b := make([]byte, 0, 2*len(s)) - for _, r := range s { - b = append(b, byte(r), 0) - } - return b -} diff --git a/init/runner_linux.go b/init/runner_linux.go index 37234e4..e9f7a3e 100644 --- a/init/runner_linux.go +++ b/init/runner_linux.go @@ -19,28 +19,23 @@ import ( "golang.org/x/sys/unix" "github.com/runplane/runplane/internal/sandbox" - "github.com/runplane/runplane/internal/sandbox/control" -) - -const ( - scratchDir = "/mnt/scratch" // scratch disk: overlay upper layers and Docker data - runnerHome = "/home/runner/actions-runner" - stopGrace = 10 * time.Second // for the runner and dockerd to stop after SIGTERM + "github.com/runplane/runplane/internal/sandbox/guest" ) // prepareRunnerDir makes the writable runner folder: an overlay with the // read-only bundle as the lower layer and the upper layer on scratch. When // the overlay mount fails, for example because scratch is not a separate // disk, it copies the bundle. -func (g *guest) prepareRunnerDir(bundle string, acct account) error { - upper, work := scratchDir+"/runner/upper", scratchDir+"/runner/work" - err := mkdirs(runnerHome, upper, work) +func (g *guestInit) prepareRunnerDir(bundle string, acct account) error { + scratch := filepath.Join(guest.ScratchDir, guest.ScratchRunner) + upper, work := scratch+"/upper", scratch+"/work" + err := mkdirs(guest.RunnerHome, upper, work) if err == nil { - err = mount("overlay", runnerHome, "overlay", 0, "lowerdir="+bundle+",upperdir="+upper+",workdir="+work) + err = mount("overlay", guest.RunnerHome, "overlay", 0, "lowerdir="+bundle+",upperdir="+upper+",workdir="+work) } if err != nil { g.log.Warn("cannot put an overlay on the runner bundle, copying it", "err", err) - cmd := exec.Command("cp", "-a", bundle+"/.", runnerHome) + cmd := exec.Command("cp", "-a", bundle+"/.", guest.RunnerHome) cmd.Stderr = os.Stderr code, err := g.reaper.run(cmd) if err == nil && code != 0 { @@ -51,22 +46,22 @@ func (g *guest) prepareRunnerDir(bundle string, acct account) error { } } // The runner writes its config, _diag, and _work into its folder. - pages := filepath.Join(runnerHome, "_diag", "pages") + pages := filepath.Join(guest.RunnerHome, "_diag", "pages") if err := mkdirs(pages); err != nil { return err } - for _, dir := range []string{runnerHome, filepath.Dir(pages), pages} { + for _, dir := range []string{guest.RunnerHome, filepath.Dir(pages), pages} { if err := os.Lchown(dir, int(acct.uid), int(acct.gid)); err != nil { return err } } - if _, ok := g.cfg.Env["ACTIONS_RESULTS_URL"]; !ok { - if err := unpatchWorker(filepath.Join(runnerHome, "bin", "Runner.Worker.dll")); err != nil { + if g.cfg.CacheURL == "" { + if err := guest.Unpatch(guest.RunnerHome); err != nil { return fmt.Errorf("give the runner the GitHub results URL: %w", err) } } - envFile := filepath.Join(runnerHome, ".env") - if err := writeEnvFile(envFile, g.cfg.Env, g.log); err != nil { + envFile := filepath.Join(guest.RunnerHome, ".env") + if err := os.WriteFile(envFile, guest.RunnerEnv(g.cfg.CacheURL), 0o644); err != nil { return fmt.Errorf("write runner .env: %w", err) } return os.Lchown(envFile, int(acct.uid), int(acct.gid)) @@ -79,13 +74,12 @@ func (g *guest) prepareRunnerDir(bundle string, acct account) error { var writablePaths = []string{"/home", "/root", "/opt", "/usr/local"} // overlayWritablePaths puts the writes to writablePaths on scratch in -// container mode: an overlay with the image folder as the lower layer and -// the upper layer on scratch. Else they go to the container root, which is -// on the host disk with no limit. Without a scratch disk it does nothing. +// container mode. Else they go to the container root, which is on the host +// disk with no limit. Without a scratch disk it does nothing. // The sysbox driver stops a sandbox whose writes outside scratch pass its // disk size, so a failed overlay leaves the host disk safe. -func (g *guest) overlayWritablePaths() { - if !isMountPoint(scratchDir) { +func (g *guestInit) overlayWritablePaths() { + if !isMountPoint(guest.ScratchDir) { return } for _, p := range writablePaths { @@ -95,7 +89,6 @@ func (g *guest) overlayWritablePaths() { } } -// overlayOnScratch mounts an overlay on path, with path as the lower layer. func overlayOnScratch(path string) error { var st unix.Stat_t if err := unix.Stat(path, &st); errors.Is(err, unix.ENOENT) { @@ -103,7 +96,7 @@ func overlayOnScratch(path string) error { } else if err != nil { return err } - base := filepath.Join(scratchDir, "root", strings.ReplaceAll(strings.Trim(path, "/"), "/", "-")) + base := filepath.Join(guest.ScratchDir, guest.ScratchRoot, strings.ReplaceAll(strings.Trim(path, "/"), "/", "-")) upper, work := base+"/upper", base+"/work" if err := mkdirs(upper, work); err != nil { return err @@ -118,7 +111,7 @@ func overlayOnScratch(path string) error { return mount("overlay", path, "overlay", 0, "lowerdir="+path+",upperdir="+upper+",workdir="+work) } -func (g *guest) startDockerd() *child { +func (g *guestInit) startDockerd() *child { path, err := exec.LookPath("dockerd") if err != nil { g.log.Info("no dockerd in the image, so no Docker in this sandbox") @@ -127,14 +120,14 @@ func (g *guest) startDockerd() *child { if err := dockerDataRoot(); err != nil { g.log.Warn("put Docker data on scratch", "err", err) } - args := []string{"--storage-driver", "overlay2", "--data-root", "/var/lib/docker"} + args := []string{"--storage-driver", "overlay2", "--data-root", guest.DockerDir} if m := g.cfg.RegistryMirror; m != "" { args = append(args, "--registry-mirror", m) if u, err := url.Parse(m); err == nil && u.Scheme == "http" { args = append(args, "--insecure-registry", u.Host) } } - c, err := g.startLogged(exec.Command(path, args...), "dockerd", nil) + c, err := g.startLogged(exec.Command(path, args...), guest.SourceDockerd, nil) if err != nil { g.log.Error("start dockerd", "err", err) return nil @@ -142,17 +135,15 @@ func (g *guest) startDockerd() *child { return c } -// dockerDataRoot puts /var/lib/docker on scratch unless something is mounted -// there already. overlay2 does not work on top of an overlay. func dockerDataRoot() error { - if isMountPoint("/var/lib/docker") || !isMountPoint(scratchDir) { + if isMountPoint(guest.DockerDir) || !isMountPoint(guest.ScratchDir) { return nil } - src := scratchDir + "/docker" - if err := mkdirs(src, "/var/lib/docker"); err != nil { + src := filepath.Join(guest.ScratchDir, guest.ScratchDocker) + if err := mkdirs(src, guest.DockerDir); err != nil { return err } - return mount(src, "/var/lib/docker", "", unix.MS_BIND, "") + return mount(src, guest.DockerDir, "", unix.MS_BIND, "") } // Why a runner failed, for the exited event. @@ -163,15 +154,14 @@ const ( // runRunner runs ./run.sh --jitconfig as the runner user until it exits, the // time limit passes, or the host asks for a shutdown. It returns the exit -// code, and why the runner failed or "". The time limit counts from the job -// start; before a job, the runner waits at most the limit plus -// sandbox.IdleGrace. +// code, and why the runner failed or "". The time limit ends at +// sandbox.Deadline from the start of the runner. // // A runner that exits by itself before "Listening for Jobs" failed, also // with exit code 0: the runner exits with 0 after a bad JIT config. -func (g *guest) runRunner(acct account) (int, string) { +func (g *guestInit) runRunner(acct account) (int, string) { cmd := exec.Command("./run.sh", "--jitconfig", g.cfg.JITConfig) - cmd.Dir = runnerHome + cmd.Dir = guest.RunnerHome // With this variable, run.sh passes SIGTERM on to the runner as SIGINT // and exits with the runner's exit code. cmd.Env = acct.env("RUNNER_MANUALLY_TRAP_SIG=1") @@ -179,7 +169,8 @@ func (g *guest) runRunner(acct account) (int, string) { cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Credential: acct.credential()} jobStarted := make(chan struct{}, 1) var listened atomic.Bool - r, err := g.startLogged(cmd, "runner", func(line string) { + start := time.Now() + r, err := g.startLogged(cmd, guest.SourceRunner, func(line string) { if ev, ok := parseRunnerLine(line); ok { g.link.event(ev) switch ev.Kind { @@ -204,7 +195,7 @@ func (g *guest) runRunner(acct account) (int, string) { pagesDone := make(chan struct{}) go func() { defer close(pagesDone) - err := tailPages(pagesCtx, filepath.Join(runnerHome, "_diag", "pages"), func(line string) { g.logs.push("diag", line) }) + err := tailPages(pagesCtx, filepath.Join(guest.RunnerHome, "_diag", "pages"), func(line string) { g.logs.push(guest.SourceDiag, line) }) if err != nil { g.log.Warn("tail runner pages", "err", err) } @@ -214,7 +205,7 @@ func (g *guest) runRunner(acct account) (int, string) { var timer *time.Timer var timeout <-chan time.Time if limit > 0 { - timer = time.NewTimer(limit + sandbox.IdleGrace) + timer = time.NewTimer(time.Until(sandbox.Deadline(start, time.Time{}, limit))) defer timer.Stop() timeout = timer.C } @@ -225,7 +216,7 @@ wait: select { case <-jobStarted: if timer != nil { - timer.Reset(limit) + timer.Reset(time.Until(sandbox.Deadline(start, time.Now(), limit))) } case code = <-r.exit: ended = true @@ -240,7 +231,7 @@ wait: unix.Kill(sid, unix.SIGTERM) select { case code = <-r.exit: - case <-time.After(stopGrace): + case <-time.After(guest.ProcessStop): killSession(sid, unix.SIGKILL) code = <-r.exit } @@ -253,7 +244,7 @@ wait: // Send the last lines before the exited event. select { case <-r.output: - case <-time.After(5 * time.Second): + case <-time.After(guest.OutputWait): } stopPages() <-pagesDone @@ -264,7 +255,6 @@ wait: return code, "" } -// child is a process that init started and reaps. type child struct { pid int exit <-chan int @@ -273,7 +263,7 @@ type child struct { // startLogged starts cmd with its output in the log stream under source. // onLine also gets each output line. -func (g *guest) startLogged(cmd *exec.Cmd, source string, onLine func(string)) (*child, error) { +func (g *guestInit) startLogged(cmd *exec.Cmd, source string, onLine func(string)) (*child, error) { pr, pw, err := os.Pipe() if err != nil { return nil, err @@ -289,7 +279,7 @@ func (g *guest) startLogged(cmd *exec.Cmd, source string, onLine func(string)) ( go func() { defer close(c.output) defer pr.Close() - control.ReadLines(pr, func(line string) { + guest.ReadLines(pr, func(line string) { g.logs.push(source, line) if onLine != nil { onLine(line) @@ -303,7 +293,7 @@ func (c *child) stop() { unix.Kill(c.pid, unix.SIGTERM) select { case <-c.exit: - case <-time.After(stopGrace): + case <-time.After(guest.ProcessStop): unix.Kill(c.pid, unix.SIGKILL) <-c.exit } diff --git a/init/runner_test.go b/init/runner_test.go index 16447bb..a5e9ed1 100644 --- a/init/runner_test.go +++ b/init/runner_test.go @@ -1,12 +1,7 @@ package main import ( - "bytes" - "encoding/binary" - "os" - "path/filepath" "testing" - "unicode/utf16" "github.com/runplane/runplane/internal/sandbox" ) @@ -56,38 +51,3 @@ func TestParseRunnerLine(t *testing.T) { } } } - -// A sandbox without runplane's cache must get GitHub's results URL back, or -// upload-artifact fails with "Unable to get the ACTIONS_RESULTS_URL env variable". -func TestUnpatchWorker(t *testing.T) { - original := dll("ACTIONS_RESULTS_URL") - for _, in := range [][]byte{dll("ACTIONS_RESULTS_ORL"), original} { - file := filepath.Join(t.TempDir(), "Runner.Worker.dll") - if err := os.WriteFile(file, in, 0o644); err != nil { - t.Fatal(err) - } - if err := unpatchWorker(file); err != nil { - t.Fatal(err) - } - if got, _ := os.ReadFile(file); !bytes.Equal(got, original) { - t.Errorf("%q became %q, want %q", in, got, original) - } - } - if err := unpatchWorker(filepath.Join(t.TempDir(), "missing.dll")); err != nil { - t.Errorf("missing file: %v", err) - } -} - -// dll is a fake Runner.Worker.dll with the name two times in UTF-16LE, as -// .NET stores string literals, between other bytes. -func dll(name string) []byte { - var b bytes.Buffer - b.WriteString("MZ head") - for _, s := range []string{name, " and ", name} { - for _, u := range utf16.Encode([]rune(s)) { - binary.Write(&b, binary.LittleEndian, u) - } - } - b.WriteString("tail") - return b.Bytes() -} diff --git a/init/shell.go b/init/shell.go index a0dd6e4..a21634c 100644 --- a/init/shell.go +++ b/init/shell.go @@ -6,13 +6,13 @@ import ( "log/slog" "net/http" "os" - "strconv" "sync" "github.com/creack/pty" "github.com/runplane/runplane/internal/ids" "github.com/runplane/runplane/internal/sandbox" + "github.com/runplane/runplane/internal/sandbox/guest" ) // shellServer serves debug shells. Each shell is a login shell on a PTY. @@ -42,7 +42,7 @@ func (s *shellServer) setStart(start func(sandbox.TermSize) (*shell, error)) { // handleOpen starts a shell and switches the stream to raw PTY bytes. The // stream ends when the shell exits; the shell ends when the host closes the stream. func (s *shellServer) handleOpen(w http.ResponseWriter, r *http.Request) { - size, err := termSize(r) + size, err := guest.ParseSize(r.URL.Query()) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -80,7 +80,7 @@ func (s *shellServer) handleOpen(w http.ResponseWriter, r *http.Request) { }() s.log.Info("debug shell opened", "session", id) - fmt.Fprintf(rw, "HTTP/1.1 101 Switching Protocols\r\nX-Runplane-Session: %s\r\nConnection: Upgrade\r\nUpgrade: runplane-shell\r\n\r\n", id) + fmt.Fprintf(rw, "HTTP/1.1 101 Switching Protocols\r\n%s: %s\r\nConnection: Upgrade\r\nUpgrade: %s\r\n\r\n", guest.SessionHeader, id, guest.ShellProtocol) if err := rw.Flush(); err != nil { return } @@ -93,13 +93,14 @@ func (s *shellServer) handleOpen(w http.ResponseWriter, r *http.Request) { } func (s *shellServer) handleResize(w http.ResponseWriter, r *http.Request) { - size, err := termSize(r) + q := r.URL.Query() + size, err := guest.ParseSize(q) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } s.mu.Lock() - p := s.open[r.URL.Query().Get("session")] + p := s.open[q.Get(guest.QuerySession)] s.mu.Unlock() if p == nil { http.Error(w, "no such shell session", http.StatusNotFound) @@ -111,23 +112,3 @@ func (s *shellServer) handleResize(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(http.StatusNoContent) } - -// termSize reads rows and cols from the query. A missing or zero value -// gives 24 rows and 80 columns. -func termSize(r *http.Request) (sandbox.TermSize, error) { - size := sandbox.TermSize{Rows: 24, Cols: 80} - for name, v := range map[string]*uint16{"rows": &size.Rows, "cols": &size.Cols} { - q := r.URL.Query().Get(name) - if q == "" { - continue - } - n, err := strconv.ParseUint(q, 10, 16) - if err != nil { - return size, fmt.Errorf("bad %s %q", name, q) - } - if n > 0 { - *v = uint16(n) - } - } - return size, nil -} diff --git a/init/shell_linux_test.go b/init/shell_linux_test.go index a825368..8b1d476 100644 --- a/init/shell_linux_test.go +++ b/init/shell_linux_test.go @@ -8,10 +8,15 @@ import ( "log/slog" "net" "os" + "path/filepath" + "regexp" + "strconv" "strings" "testing" "time" + "golang.org/x/sys/unix" + "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/sandbox/control" ) @@ -22,7 +27,8 @@ func TestShell(t *testing.T) { } sock := socketPath(t) log := slog.New(slog.DiscardHandler) - host, err := control.Listen(sock, control.GuestConfig{}, &recorder{}, log) + rec := &recorder{} + host, err := control.Listen(sock, nil, rec, log) if err != nil { t.Fatal(err) } @@ -31,12 +37,12 @@ func TestShell(t *testing.T) { shells := newShellServer(log) acct := account{name: "root", home: t.TempDir(), shell: "/bin/sh"} shells.setStart(shellStarter(acct, newReaper())) - link := newGuestLink(func(ctx context.Context) (net.Conn, error) { return dialHost(ctx, sock) }, + link := newGuestLink(func(ctx context.Context) (net.Conn, error) { return dialHost(ctx, filepath.Dir(sock)) }, log, newLogQueue(1<<20), guestMux(newHostRequests(), shells)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go link.run(ctx) - waitFor(t, "the guest", host.Connected) + waitFor(t, "the guest", func() bool { return rec.connects.Load() > 0 }) term, err := host.Shell(ctx, sandbox.TermSize{Rows: 30, Cols: 90}) if err != nil { @@ -75,12 +81,16 @@ func TestShell(t *testing.T) { if err != nil { t.Fatal(err) } - openSessions := func() int { - shells.mu.Lock() - defer shells.mu.Unlock() - return len(shells.open) - } - waitFor(t, "an open session", func() bool { return openSessions() == 1 }) + out = &syncBuffer{} + go io.Copy(out, term) + io.WriteString(term, "echo pid=$$\n") + pidLine := regexp.MustCompile(`pid=(\d+)`) + waitFor(t, "the shell PID", func() bool { return pidLine.MatchString(out.String()) }) + pid, _ := strconv.Atoi(pidLine.FindStringSubmatch(out.String())[1]) term.Close() - waitFor(t, "the session to end", func() bool { return openSessions() == 0 }) + waitFor(t, "the shell to exit", func() bool { return unix.Kill(pid, 0) == unix.ESRCH }) + waitFor(t, "the session to end", func() bool { + err := term.Resize(sandbox.TermSize{Rows: 10, Cols: 10}) + return err != nil && strings.Contains(err.Error(), "404 Not Found") + }) } diff --git a/init/vm_linux.go b/init/vm_linux.go index 7bca096..cefa943 100644 --- a/init/vm_linux.go +++ b/init/vm_linux.go @@ -13,17 +13,11 @@ import ( "golang.org/x/sys/unix" - "github.com/runplane/runplane/internal/sandbox/control" -) - -// Drives of a microVM: vda is the read-only root disk. -const ( - bundleDev = "/dev/vdb" // read-only runner bundle - scratchDev = "/dev/vdc" // sparse ext4, erased after the job + "github.com/runplane/runplane/internal/sandbox/guest" ) // detectVM reports VM mode: init is PID 1 and the kernel command line has -// runplane.mode=vm. It mounts /proc to read the command line. +// guest.VMFlag. It mounts /proc to read the command line. func detectVM() (bool, error) { if os.Getpid() != 1 { return false, nil @@ -35,7 +29,7 @@ func detectVM() (bool, error) { if err != nil { return false, fmt.Errorf("read kernel command line: %w", err) } - return slices.Contains(strings.Fields(string(cmdline)), "runplane.mode=vm"), nil + return slices.Contains(strings.Fields(string(cmdline)), guest.VMFlag), nil } // setupVM makes the writable root of a microVM. The root disk is read-only @@ -55,11 +49,11 @@ func setupVM() error { } stage := "/run/runplane-init" disk, root := stage+"/scratch", stage+"/root" - upper, work := disk+"/root/upper", disk+"/root/work" + upper, work := disk+"/"+guest.ScratchRoot+"/upper", disk+"/"+guest.ScratchRoot+"/work" if err := mkdirs(disk, root); err != nil { return err } - if err := mount(scratchDev, disk, "ext4", 0, ""); err != nil { + if err := mount(guest.DriveScratch.Device(), disk, "ext4", 0, ""); err != nil { return err } if err := mkdirs(upper, work); err != nil { @@ -68,10 +62,10 @@ func setupVM() error { if err := mount("overlay", root, "overlay", 0, "lowerdir=/,upperdir="+upper+",workdir="+work); err != nil { return err } - if err := mkdirs(root+scratchDir, root+"/.oldroot"); err != nil { + if err := mkdirs(root+guest.ScratchDir, root+"/.oldroot"); err != nil { return err } - if err := unix.Mount(disk, root+scratchDir, "", unix.MS_MOVE, ""); err != nil { + if err := unix.Mount(disk, root+guest.ScratchDir, "", unix.MS_MOVE, ""); err != nil { return fmt.Errorf("move scratch disk: %w", err) } if err := pivot(root); err != nil { @@ -134,17 +128,14 @@ func setImageEnv(file string) error { if err != nil { return err } - for kv := range strings.SplitSeq(string(b), "\x00") { - if k, v, ok := strings.Cut(kv, "="); ok && k != "" { - if err := os.Setenv(k, v); err != nil { - return err - } + for k, v := range guest.DecodeImageEnv(b) { + if err := os.Setenv(k, v); err != nil { + return err } } return nil } -// pivot makes root the new root mount and drops the old root. func pivot(root string) error { if err := unix.Chdir(root); err != nil { return err @@ -163,7 +154,7 @@ func pivot(root string) error { // configureVM applies the parts of the config that Docker does for a // container: host name, /etc/hosts, DNS, and the loopback device. -func configureVM(cfg control.GuestConfig) error { +func configureVM(cfg guest.Config) error { if cfg.Hostname != "" { if err := unix.Sethostname([]byte(cfg.Hostname)); err != nil { return fmt.Errorf("set host name: %w", err) @@ -197,15 +188,16 @@ func mountBundle(dir string) error { if err := mkdirs(dir); err != nil { return err } + dev := guest.DriveRunner.Device() var errs []error for _, fstype := range []string{"ext4", "erofs", "squashfs"} { - err := unix.Mount(bundleDev, dir, fstype, unix.MS_RDONLY, "") + err := unix.Mount(dev, dir, fstype, unix.MS_RDONLY, "") if err == nil { return nil } errs = append(errs, fmt.Errorf("%s: %w", fstype, err)) } - return fmt.Errorf("mount runner bundle %s: %w", bundleDev, errors.Join(errs...)) + return fmt.Errorf("mount runner bundle %s: %w", dev, errors.Join(errs...)) } func loopbackUp() error { @@ -236,7 +228,8 @@ func powerOff(vm bool, code int) { os.Exit(code) } // Firecracker on x86 has no power-off device. It exits when the guest - // resets the CPU, which a reboot does with the kernel option reboot=k. + // resets the CPU, which a reboot does with the kernel option reboot=k + // (guest.KernelArgs). cmd := unix.LINUX_REBOOT_CMD_POWER_OFF if runtime.GOARCH == "amd64" { cmd = unix.LINUX_REBOOT_CMD_RESTART @@ -254,7 +247,6 @@ func mount(src, target, fstype string, flags uintptr, data string) error { return nil } -// mountOnce mounts unless something is mounted at target already. func mountOnce(src, target, fstype string, flags uintptr, data string) error { if isMountPoint(target) { return nil @@ -262,8 +254,7 @@ func mountOnce(src, target, fstype string, flags uintptr, data string) error { return mount(src, target, fstype, flags, data) } -// isMountPoint reports if path is a mount point. Without /proc it reports -// false. +// Without /proc, isMountPoint reports false. func isMountPoint(path string) bool { info, err := os.ReadFile("/proc/self/mountinfo") if err != nil { diff --git a/install.go b/install.go index 81d6b2e..80286e5 100644 --- a/install.go +++ b/install.go @@ -9,3 +9,22 @@ import _ "embed" // //go:embed install.sh var InstallScript []byte + +// System is an operating system of a server, by the ID and the VERSION_ID +// of its /etc/os-release. +type System struct { + ID string // for example "ubuntu" + Name string // for example "Ubuntu" + Version string // for example "24.04" + // BestEffort: install.sh installs runplane with a warning, because + // runplane has no tests on the system. + BestEffort bool +} + +// Systems are the systems that install.sh names for a server. +var Systems = []System{ + {ID: "ubuntu", Name: "Ubuntu", Version: "22.04"}, + {ID: "ubuntu", Name: "Ubuntu", Version: "24.04"}, + {ID: "debian", Name: "Debian", Version: "12", BestEffort: true}, + {ID: "debian", Name: "Debian", Version: "13", BestEffort: true}, +} diff --git a/install.sh b/install.sh index c7a101b..185a6d1 100755 --- a/install.sh +++ b/install.sh @@ -21,6 +21,8 @@ RUNPLANE_PUBLIC_KEY=${RUNPLANE_PUBLIC_KEY:-} # from api.github.com, which has a low rate limit without a token. RUNPLANE_FIRECRACKER_VERSION=${RUNPLANE_FIRECRACKER_VERSION:-v1.17.0} +# The Go code has copies of many values of this script. +# cmd/runplane/install_test.go compares them. BIN=/usr/local/bin/runplane CONFIG_DIR=/etc/runplane DATA_DIR=/var/lib/runplane @@ -29,9 +31,9 @@ AGENT_FILE=$CONFIG_DIR/agent.json KERNEL_FILE=$DATA_DIR/kernel/vmlinux UNIT_DIR=/etc/systemd/system VM_USER=runplane-vm -# The uids and gids of the VMs (reserve_vm_ids): one for each TAP address -# block of 10.214.0.0/16. The range starts after the container ranges of -# systemd (https://systemd.io/UIDS-GIDS/), in IDs that no tool allocates. +# The uids and gids of the VMs (reserve_vm_ids): one for each TAP lease of +# the VM subnet. The range starts after the container ranges of systemd +# (https://systemd.io/UIDS-GIDS/), in IDs that no tool allocates. VM_ID_START=1879048192 VM_ID_COUNT=16384 SYSBOX_MIN=0.7.1 @@ -87,9 +89,11 @@ main() { esac install_runplane write_config + stop_service write_units store_mode store_domain + make_setup_code start_service finish } @@ -168,10 +172,7 @@ parse_flags() { if [ -n "$KERNEL" ] && [ ! -f "$KERNEL" ]; then die "--kernel: $KERNEL does not exist" fi - case $DOMAIN in - '' | *.*) only_chars "$DOMAIN" 'a-z0-9.-' || die "--domain must be a domain name such as ci.example.com" ;; - *) die "--domain must be a domain name such as ci.example.com" ;; - esac + [ -z "$DOMAIN" ] || valid_domain "$DOMAIN" || die "--domain must be a domain name such as ci.example.com" ROLE=primary SERVICE=runplane @@ -550,7 +551,7 @@ setup_vm() { v[0-9]*.[0-9]*.[0-9]*) ;; *) die "RUNPLANE_FIRECRACKER_VERSION=$fc_tag is not a Firecracker release tag such as v1.17.0" ;; esac - have=$(firecracker --version 2>/dev/null | sed -n '1s/^Firecracker \(v[0-9.]*\).*/\1/p' || true) + have=$(/usr/local/bin/firecracker --version 2>/dev/null | sed -n '1s/^Firecracker \(v[0-9.]*\).*/\1/p' || true) if [ "$have" = "$fc_tag" ] && [ -x /usr/local/bin/jailer ]; then say "Firecracker: $fc_tag" else @@ -736,6 +737,19 @@ IOWeight=20 EOF } +# stop_service stops a running primary before the next steps open its +# database with the new binary. The binary applies the migrations of its +# release, and the old serve must not operate on the new schema. The stop +# comes before write_units, because systemctl warns about a changed unit +# file until start_service reloads the units. The sandboxes continue +# (KillMode=process), and start_service starts the primary again. +stop_service() { + [ "$ROLE" = primary ] || return 0 + systemctl is-active --quiet "$SERVICE.service" 2>/dev/null || return 0 + step "Stop $SERVICE" + run systemctl stop "$SERVICE.service" +} + # store_mode writes --mode into the settings of an installed primary, # because the settings win over RUNPLANE_MODE after the first start. store_mode() { @@ -743,7 +757,8 @@ store_mode() { return 0 fi step "Write the mode to the settings" - run with_env "$BIN" set-mode "$MODE" || die "runplane cannot write $MODE mode to its settings" + run "$BIN" set-mode "$MODE" || + die "runplane cannot write $MODE mode to its settings. Correct the problem and run this script again. Until then, runplane does not run." } # store_domain writes --domain into the settings of the primary. On a new @@ -753,7 +768,22 @@ store_domain() { return 0 fi step "Write the domain to the settings" - run with_env "$BIN" set-domain "$DOMAIN" || die "runplane cannot write the domain $DOMAIN to its settings" + run "$BIN" set-domain "$DOMAIN" || + die "runplane cannot write the domain $DOMAIN to its settings. Correct the problem and run this script again. Until then, runplane does not run." +} + +# make_setup_code makes the setup code of the first admin before the start. +# serve then keeps this code and logs no different code. On a primary with +# an admin, setup-code makes no code and fails. +make_setup_code() { + SETUP_CODE= + [ "$ROLE" = primary ] || return 0 + step "Make the setup code" + if [ "$DRY_RUN" = 1 ]; then + say "+ $BIN setup-code" + return 0 + fi + SETUP_CODE=$("$BIN" setup-code 2>/dev/null) || SETUP_CODE= } start_service() { @@ -788,7 +818,7 @@ finish() { step "Run runplane doctor" if [ "$DRY_RUN" = 1 ]; then say "+ $BIN doctor" - elif ! with_env "$BIN" doctor; then + elif ! "$BIN" doctor; then warn "runplane doctor found problems. Jobs can fail until you correct them." fi @@ -822,15 +852,12 @@ finish() { fi if [ "$DRY_RUN" = 1 ]; then say " Setup code: (from runplane setup-code)" - elif curl -fsS "$(local_url)/api/status" 2>/dev/null | grep -q '"needs_setup": *true'; then - code=$(with_env "$BIN" setup-code 2>/dev/null) || code= - if [ -n "$code" ]; then - say " Setup code: $code" - say "" - say "Open the dashboard and type the setup code to make the admin account." - else - warn "cannot get a setup code. Run: sudo runplane setup-code" - fi + elif [ -n "$SETUP_CODE" ]; then + say " Setup code: $SETUP_CODE" + say "" + say "Open the dashboard and type the setup code to make the admin account." + elif needs_setup "$(api_status)"; then + warn "cannot get a setup code. Run: sudo runplane setup-code" fi case $host in 10.* | 192.168.* | 172.1[6-9].* | 172.2[0-9].* | 172.3[01].*) @@ -1044,6 +1071,30 @@ only_chars() { [ -z "$(printf '%s' "$1" | tr -d "$2")" ] } +# valid_domain reports if $1 is a domain that "runplane set-domain" accepts: +# a lowercase host name with at least two labels, not an IP address. The +# rule is validDomain in internal/settings/clean.go. +valid_domain() { + only_chars "$1" 'a-z0-9.-' || return 1 + [ ${#1} -le 253 ] || return 1 + labels=0 + numbers=0 + rest=$1. + while [ -n "$rest" ]; do + label=${rest%%.*} + rest=${rest#*.} + [ ${#label} -le 63 ] || return 1 + case $label in + '' | -* | *-) return 1 ;; + 0 | [1-9] | [1-9][0-9] | 1[0-9][0-9] | 2[0-4][0-9] | 25[0-5]) numbers=$((numbers + 1)) ;; + esac + labels=$((labels + 1)) + done + [ "$labels" -ge 2 ] || return 1 + # net.ParseIP reads four numbers of 0 to 255 as an IPv4 address. + [ "$labels" -ne 4 ] || [ "$numbers" -ne 4 ] +} + os_field() { [ -r /etc/os-release ] || return 0 sed -n "s/^$1=//p" /etc/os-release | tr -d "\"'" @@ -1065,16 +1116,6 @@ confirm() { return 1 } -with_env() { - ( - set -a - # shellcheck disable=SC1090 # this script writes the file - . "$ENV_FILE" - set +a - "$@" - ) -} - listen_addr() { addr=${LISTEN:-$(sed -n 's/^RUNPLANE_LISTEN=//p' "$ENV_FILE" 2>/dev/null | tail -n 1)} printf '%s' "${addr:-:8484}" @@ -1090,10 +1131,22 @@ local_url() { printf 'http://%s:%s' "$host" "${addr##*:}" } +# api_status prints the /api/status answer of the primary that runs on this +# host. It fails when no primary answers. +api_status() { + curl -fsS --max-time 5 "$(local_url)/api/status" 2>/dev/null +} + # running_mode prints the sandbox mode of the primary that runs on this host, # or nothing. running_mode() { - curl -fsS --max-time 5 "$(local_url)/api/status" 2>/dev/null | sed -n 's/.*"mode": *"\([a-z]*\)".*/\1/p' + api_status | sed -n 's/.*"mode": *"\([a-z]*\)".*/\1/p' +} + +# needs_setup reports if the /api/status answer $1 tells that no admin +# exists. +needs_setup() { + printf '%s' "$1" | grep -q '"needs_setup": *true' } # host_ip prints the address of the interface with the default route. diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 53e2905..515e569 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -24,10 +24,8 @@ import ( "github.com/runplane/runplane/internal/atomicfile" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/mirror" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/tlsutil" - "github.com/runplane/runplane/internal/version" ) // ErrUpdated is the result of Run after the primary updated the binary of @@ -40,9 +38,8 @@ type Config struct { // CacheProxy, if not nil, sends its requests through the connection // that Run keeps. CacheProxy *CacheProxy - // Mirror returns the stats of the registry mirror of the agent. - Mirror func() mirror.Stats - Logger *slog.Logger + Version string // of runplane on this server + Logger *slog.Logger } const ( @@ -51,6 +48,10 @@ const ( // maxQueued is the number of events that the agent keeps while it has // no connection. maxQueued = 4096 + // updateWait is how long the agent waits after an update for the + // primary to close the connection, which it does when it has the + // answer. + updateWait = 10 * time.Second ) // Run connects the agent to the primary and serves the agent routes from @@ -69,7 +70,11 @@ func Run(ctx context.Context, cfg Config) error { out: &outbox{ready: make(chan struct{}, 1)}, updated: make(chan struct{}), } - a.routes = a.newRoutes() + mux := http.NewServeMux() + for _, h := range handlers { + mux.HandleFunc(h.pattern, h.serve(a)) + } + a.routes = mux go Forward(ctx, cfg.Node, a.out.addEvent, a.out.setStats) delay := minBackoff @@ -109,7 +114,6 @@ type agent struct { updated chan struct{} } -// connect dials the primary, runs the handshake, and starts yamux. func (a *agent) connect(ctx context.Context) (*yamux.Session, error) { info, err := a.Node.Facts(ctx) if err != nil { @@ -133,7 +137,7 @@ func (a *agent) connect(ctx context.Context) (*yamux.Session, error) { func (a *agent) handshake(conn net.Conn, info host.Info) (*yamux.Session, error) { conn.SetDeadline(time.Now().Add(handshakeTimeout)) - hello := Hello{Version: version.Version, Mode: a.Node.Driver().Mode(), Host: info} + hello := helloLine{Version: a.Version, Mode: a.Node.Driver().Mode(), Host: wireInfo(info)} if a.file.registered() { hello.ServerID, hello.Secret = a.file.ServerID, a.file.Secret } else { @@ -192,7 +196,9 @@ func (a *agent) remember(r reply) error { return nil } -// serve operates one connection until it ends. +// serve operates one connection until it ends. After an update, the +// agent does not close the connection: the answer of the update could be +// lost. It waits for the primary to close it. func (a *agent) serve(ctx context.Context, sess *yamux.Session) error { defer sess.Close() a.Logger.Info("connected to the primary", "primary", a.file.Primary, "server_id", a.file.ServerID) @@ -208,8 +214,13 @@ func (a *agent) serve(ctx context.Context, sess *yamux.Session) error { case <-sess.CloseChan(): return nil case <-a.updated: - return ErrUpdated } + select { + case <-ctx.Done(): + case <-sess.CloseChan(): + case <-time.After(updateWait): + } + return ErrUpdated } // sendEvents streams the outbox to POST /v1/events. When the stream @@ -222,7 +233,7 @@ func (a *agent) sendEvents(sess *yamux.Session) { } pr, pw := io.Pipe() go func() { - req, _ := http.NewRequest(http.MethodPost, "http://primary/v1/events", pr) + req, _ := http.NewRequest(http.MethodPost, "http://primary"+eventsPath, pr) req.Header.Set("Content-Type", "application/x-ndjson") resp, err := streamClient(sess).Do(req) if err == nil { @@ -252,7 +263,7 @@ func (a *agent) sendEvents(sess *yamux.Session) { type outbox struct { mu sync.Mutex events []sandbox.Event - stats *host.Stats + stats *hostStats dropped int ready chan struct{} // has a value after each change } @@ -269,13 +280,13 @@ func (o *outbox) addEvent(ev sandbox.Event) { } func (o *outbox) setStats(st host.Stats) { + ws := wireStats(st) o.mu.Lock() - o.stats = &st + o.stats = &ws o.mu.Unlock() o.signal() } -// putBack returns an event that did not go out to the front of the queue. func (o *outbox) putBack(it streamItem) { if it.Event == nil { return diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 1b10620..024c265 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -9,17 +9,15 @@ import ( "maps" "net/http" "net/http/httptest" + "net/netip" "os" "path/filepath" "reflect" - "regexp" "strings" "testing" "time" - "github.com/runplane/runplane/internal/cache" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/image" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/tlsutil" ) @@ -60,15 +58,13 @@ func TestRegisterThenReconnect(t *testing.T) { if st, _ := os.Stat(a.path); st.Mode().Perm() != 0o600 { t.Errorf("agent.json mode %v, want 0600", st.Mode().Perm()) } - // install.sh keeps a registered agent.json when it finds these lines. - if !regexp.MustCompile(`"server_id": *"[^"]`).Match(raw) || !strings.Contains(string(raw), `"primary": "`+p.addr+`"`) { - t.Errorf("install.sh would not find the registration in:\n%s", raw) - } + held, resume := node.holdConnect() s1.Close() - p.logs.waitFor(t, "connection to the primary ended") + waitClosed(t, held) // An event while the agent has no connection arrives after it connects again. node.events <- sandbox.Event{SandboxID: "sb1", Type: sandbox.EventState, State: sandbox.StateIdle} + resume() s2 := recv(t, p.sessions) second := recv(t, p.hellos) @@ -92,11 +88,10 @@ func TestRefusedAgentKeepsItsConfig(t *testing.T) { name string fingerprint string // empty: the primary's token string - log string hello bool // the primary sees a hello }{ - {name: "wrong fingerprint", fingerprint: otherFP, token: goodToken, log: "does not match pinned"}, - {name: "used token", token: "rp_srv_used", log: "primary refused the agent: the install token is not valid", hello: true}, + {name: "wrong fingerprint", fingerprint: otherFP, token: goodToken}, + {name: "used token", token: "rp_srv_used", hello: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -105,8 +100,14 @@ func TestRefusedAgentKeepsItsConfig(t *testing.T) { if tt.fingerprint != "" { f = strings.Replace(f, p.fp, tt.fingerprint, 1) } - a := p.startAgent(f, newFakeNode(), nil) - p.logs.waitFor(t, tt.log) + node := newFakeNode() + first, resume := node.holdConnect() + a := p.startAgent(f, node, nil) + waitClosed(t, first) + second, _ := node.holdConnect() + resume() + // The agent tries again after the refusal. + waitClosed(t, second) if got := len(p.hellos) > 0; got != tt.hello { t.Errorf("primary saw a hello: %v, want %v", got, tt.hello) @@ -139,7 +140,6 @@ func TestNewSessionClosesOldSessionOfServer(t *testing.T) { func TestCalls(t *testing.T) { a, node, s := connect(t) ctx := t.Context() - // check compares the result of a call and the calls that the node got. check := func(name string, got any, err error, want any, calls ...[]any) { t.Helper() if err != nil || !reflect.DeepEqual(got, want) { @@ -160,46 +160,50 @@ func TestCalls(t *testing.T) { ID: "sb1", PoolID: "pool1", Image: sandbox.ImageRef{Ref: "builtin:ubuntu-24.04", Digest: "sha256:5f1c"}, RunnerVersion: "2.330.0", Arch: "arm64", VCPU: 2, MemoryMB: 4096, DiskGB: 20, - TimeLimit: 6 * time.Hour, JITConfig: "jit-secret", - Env: map[string]string{"ACTIONS_RESULTS_URL": "http://10.0.0.1:8486/"}, - Debug: true, Labels: map[string]string{sandbox.LabelSandbox: "sb1"}, + TimeLimit: 6 * time.Hour, JITConfig: "jit-secret", Debug: true, } - check("mode", s.Mode(), nil, sandbox.ModeContainer) - check("start", nil, s.Start(ctx, spec), nil, []any{"Start", spec}) - checkErr("start again", s.Start(ctx, spec), "sandbox sb1 exists") + d := s.Driver() + check("mode", d.Mode(), nil, sandbox.ModeContainer) + check("start", nil, d.Start(ctx, spec), nil, []any{"Start", spec}) + checkErr("start again", d.Start(ctx, spec), "sandbox sb1 exists") stored := spec stored.JITConfig = "" - list, err := s.List(ctx) + list, err := d.List(ctx) check("list", list, err, []sandbox.Sandbox{{ID: "sb1", PoolID: "pool1", State: sandbox.StateStarting, Spec: stored}}) - c, err := s.Capacity(ctx) + c, err := d.Capacity(ctx) check("capacity", c, err, fakeCapacity) until := time.Date(2026, 11, 4, 12, 30, 0, 0, time.UTC) - check("hold", nil, s.Hold(ctx, "sb1", until), nil, []any{"Hold", "sb1", until}) - check("stop", nil, s.Stop(ctx, "sb1"), nil, []any{"Stop", "sb1"}) - list, err = s.List(ctx) + check("hold", nil, d.Hold(ctx, "sb1", until), nil, []any{"Hold", "sb1", until}) + check("stop", nil, d.Stop(ctx, "sb1"), nil, []any{"Stop", "sb1"}) + list, err = d.List(ctx) check("list after stop", list, err, []sandbox.Sandbox{}) + if err := d.Hold(ctx, "sb1", until); !errors.Is(err, sandbox.ErrNotFound) { + t.Errorf("hold of a stopped sandbox: %v, want sandbox.ErrNotFound", err) + } info, err := s.Facts(ctx) check("facts", info, err, testInfo) - checks, err := s.Doctor(ctx) - check("doctor", checks, err, testChecks) - var steps []image.Progress - ref, err := s.EnsureImage(ctx, "ghcr.io/acme/runner:1", func(p image.Progress) { steps = append(steps, p) }) - check("image", ref, err, sandbox.ImageRef{Ref: "ghcr.io/acme/runner:1", Digest: "sha256:5f1c"}) + report, err := s.Doctor(ctx) + check("doctor", report, err, testReport, []any{"Doctor"}) + var steps []Progress + ref, err := s.EnsureImage(ctx, "ghcr.io/acme/runner:1", func(p Progress) { steps = append(steps, p) }) + check("image", ref, err, Image{ImageRef: sandbox.ImageRef{Ref: "ghcr.io/acme/runner:1", Digest: "sha256:5f1c"}, SizeBytes: 2147483648}) check("image progress", steps, nil, testProgress) _, err = s.EnsureImage(ctx, "missing:latest", nil) checkErr("missing image", err, "pull missing:latest: not found") b, err := s.EnsureRunner(ctx, "2.330.0", "arm64") - check("runner", b, err, image.Bundle{Version: "2.330.0", Arch: "arm64", SHA256: "9e2f", Patched: true, Dir: "/var/lib/runplane/runners/2.330.0-arm64"}) + check("runner", b, err, Bundle{Version: "2.330.0", Arch: "arm64", SHA256: "9e2f", Patched: true}) m, err := s.MirrorStats(ctx) check("mirror stats", m, err, testMirror) keep := []string{"sha256:5f1c", "sha256:9e2f"} check("image GC", nil, s.GCImages(ctx, keep), nil, []any{"GCImages", keep}) check("runner prune", nil, s.PruneRunners(ctx, []string{"2.329.0"}), nil, []any{"PruneRunners", []string{"2.329.0"}}) - settings := sandbox.Settings{ - ReserveVCPU: 1, ReserveMemoryMB: 2048, Overcommit: 1.5, MinFreeDiskPercent: 15, - DNS: []string{"1.1.1.1"}, EgressAllowlist: []string{"github.com"}, BandwidthMbps: 100, - RegistryMirror: true, ActionsCache: true, + settings := Settings{ + Settings: sandbox.Settings{ + DNS: []netip.Addr{netip.MustParseAddr("1.1.1.1")}, EgressAllowlist: []string{"github.com"}, BandwidthMbps: 100, + RegistryMirror: true, ActionsCache: true, + }, + MinFreeDiskPercent: 12.5, } check("settings", nil, s.Configure(ctx, settings), nil, []any{"Configure", settings}) @@ -329,18 +333,19 @@ func TestDisconnectEndsSession(t *testing.T) { func TestCacheProxy(t *testing.T) { type cacheCall struct { - uri, body string - header http.Header + serverID, sandboxID string + uri, body string + header http.Header } calls := make(chan cacheCall, 4) - p := startPrimary(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p := startPrimary(t, func(w http.ResponseWriter, r *http.Request, serverID, sandboxID string) { body, _ := io.ReadAll(r.Body) - calls <- cacheCall{r.URL.RequestURI(), string(body), r.Header} + calls <- cacheCall{serverID, sandboxID, r.URL.RequestURI(), string(body), r.Header} io.WriteString(w, `{"ok":true}`) - })) + }) node := newFakeNode() node.put(sandbox.Sandbox{ID: "sb-here", State: sandbox.StateBusy, IP: "127.0.0.1"}) - proxy := NewCacheProxy(node, p.logs.logger("cache")) + proxy := NewCacheProxy(node.Driver(), p.logs.logger("cache")) p.startAgent(p.agentJSON(goodToken), node, proxy) // The agent attaches the proxy before it answers the first call. if _, err := recv(t, p.sessions).Driver().List(t.Context()); err != nil { @@ -351,8 +356,7 @@ func TestCacheProxy(t *testing.T) { post := func() (int, string) { req, _ := http.NewRequest(http.MethodPost, svc.URL+"/twirp/CacheService/CreateCacheEntry?v=2", strings.NewReader(`{"key":"npm-linux"}`)) // A sandbox must not choose its identity. - req.Header.Set(cache.SandboxHeader, "sb-other") - req.Header.Set(cache.ServerHeader, "other-server") + req.Header.Set(sandboxHeader, "sb-other") resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) @@ -369,9 +373,12 @@ func TestCacheProxy(t *testing.T) { if call.uri != "/twirp/CacheService/CreateCacheEntry?v=2" || call.body != `{"key":"npm-linux"}` { t.Errorf("primary got %q with body %q", call.uri, call.body) } + if call.serverID != testServer || call.sandboxID != "sb-here" { + t.Errorf("primary got the request of server %q and sandbox %q, want %s and sb-here", call.serverID, call.sandboxID, testServer) + } + // The cache server must not send the header of the protocol on to GitHub. wantHeaders := map[string]string{ - cache.SandboxHeader: "sb-here", - cache.ServerHeader: testServer, + sandboxHeader: "", "X-Forwarded-Host": svc.Listener.Addr().String(), "X-Forwarded-Proto": "http", } diff --git a/internal/agent/cacheproxy.go b/internal/agent/cacheproxy.go index 0bec2da..4c0d609 100644 --- a/internal/agent/cacheproxy.go +++ b/internal/agent/cacheproxy.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/yamux" - "github.com/runplane/runplane/internal/cache" "github.com/runplane/runplane/internal/sandbox" ) @@ -23,7 +22,7 @@ import ( // source address of the request and refuses requests from other addresses. // Run connects it to the primary. type CacheProxy struct { - driver sandbox.Driver + driver Driver proxy *httputil.ReverseProxy transport *http.Transport @@ -31,7 +30,7 @@ type CacheProxy struct { sess *yamux.Session // nil while the agent has no connection } -func NewCacheProxy(driver sandbox.Driver, log *slog.Logger) *CacheProxy { +func NewCacheProxy(driver Driver, log *slog.Logger) *CacheProxy { p := &CacheProxy{driver: driver} p.transport = &http.Transport{ DialContext: p.dial, @@ -66,19 +65,19 @@ func (p *CacheProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { p.proxy.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), sandboxKey{}, id))) } -// rewriteCache sends a request to /v1/cache on the primary. The cache +// rewriteCache sends a request to cachePath on the primary. The cache // server makes its signed URLs from X-Forwarded-Host and X-Forwarded-Proto, // so they point back at this service port. func rewriteCache(pr *httputil.ProxyRequest) { in, out := pr.In, pr.Out out.URL.Scheme = "http" out.URL.Host = "primary" - out.URL.Path = "/v1/cache" + in.URL.Path + out.URL.Path = cachePath + in.URL.Path if in.URL.RawPath != "" { - out.URL.RawPath = "/v1/cache" + in.URL.RawPath + out.URL.RawPath = cachePath + in.URL.RawPath } out.Host = "" - out.Header.Set(cache.SandboxHeader, in.Context().Value(sandboxKey{}).(string)) + out.Header.Set(sandboxHeader, in.Context().Value(sandboxKey{}).(string)) out.Header.Set("X-Forwarded-Host", in.Host) proto := "http" if in.TLS != nil { @@ -99,13 +98,8 @@ func (p *CacheProxy) sandboxAt(ctx context.Context, remoteAddr string) (string, if err != nil { return "", err } - for _, sb := range list { - ip, err := netip.ParseAddr(sb.IP) - if err == nil && ip.Unmap() == addrPort.Addr().Unmap() && !sb.State.Done() { - return sb.ID, nil - } - } - return "", nil + sb, _ := sandbox.FindByIP(list, addrPort.Addr()) + return sb.ID, nil } func (p *CacheProxy) dial(context.Context, string, string) (net.Conn, error) { diff --git a/internal/agent/driver.go b/internal/agent/driver.go index 633cb9f..d533785 100644 --- a/internal/agent/driver.go +++ b/internal/agent/driver.go @@ -7,51 +7,43 @@ import ( "fmt" "io" "net/http" - "net/url" - "strconv" "sync" "time" "github.com/runplane/runplane/internal/sandbox" ) -// Mode is the mode that the agent sent in its hello. -func (s *Session) Mode() sandbox.Mode { return s.hello.Mode } +type remoteDriver struct{ s *Session } + +func (d remoteDriver) Mode() sandbox.Mode { return d.s.hello.Mode } // Start is the only call that sends the JIT config of a sandbox to the // agent, and the agent keeps it in memory. -func (s *Session) Start(ctx context.Context, spec sandbox.Spec) error { - return s.call(ctx, http.MethodPost, "/v1/sandboxes", spec, nil) +func (d remoteDriver) Start(ctx context.Context, spec sandbox.Spec) error { + _, err := call(ctx, d.s, opStart, spec) + return err } -func (s *Session) Stop(ctx context.Context, id string) error { - return s.call(ctx, http.MethodDelete, sandboxPath(id, ""), nil, nil) +func (d remoteDriver) Stop(ctx context.Context, id string) error { + _, err := call(ctx, d.s, opStop, sandboxID(id)) + return err } -func (s *Session) List(ctx context.Context) ([]sandbox.Sandbox, error) { - var list []sandbox.Sandbox - err := s.call(ctx, http.MethodGet, "/v1/sandboxes", nil, &list) - return list, err +func (d remoteDriver) List(ctx context.Context) ([]sandbox.Sandbox, error) { + return call(ctx, d.s, opList, none{}) } -func (s *Session) Capacity(ctx context.Context) (sandbox.Capacity, error) { - var c sandbox.Capacity - err := s.call(ctx, http.MethodGet, "/v1/capacity", nil, &c) - return c, err +func (d remoteDriver) Capacity(ctx context.Context) (sandbox.Capacity, error) { + return call(ctx, d.s, opCapacity, none{}) } -// Events delivers the driver events of the agent. It closes when the -// session ends. -func (s *Session) Events() <-chan sandbox.Event { return s.events } +// Events closes its channel when the session ends. +func (d remoteDriver) Events() <-chan sandbox.Event { return d.s.events } -// Logs streams the log of a sandbox. With follow, the stream stays open -// until ctx ends, the reader closes, or the session ends. -func (s *Session) Logs(ctx context.Context, id string, follow bool) (io.ReadCloser, error) { - path := sandboxPath(id, "/logs") - if follow { - path += "?follow=1" - } - resp, err := s.send(ctx, http.MethodGet, path, nil) +// With follow, Logs keeps the stream open until ctx ends, the reader +// closes, or the session ends. +func (d remoteDriver) Logs(ctx context.Context, id string, follow bool) (io.ReadCloser, error) { + resp, err := open(ctx, d.s, opLogs.route, logsRequest{ID: id, Follow: follow}) if err != nil { return nil, err } @@ -60,35 +52,28 @@ func (s *Session) Logs(ctx context.Context, id string, follow bool) (io.ReadClos // Shell opens a debug shell. ctx limits only the open; the terminal stays // open until Close, the end of the shell, or the end of the session. -func (s *Session) Shell(ctx context.Context, id string, size sandbox.TermSize) (sandbox.Terminal, error) { - q := url.Values{ - "rows": {strconv.Itoa(int(size.Rows))}, - "cols": {strconv.Itoa(int(size.Cols))}, - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://agent"+sandboxPath(id, "/shell")+"?"+q.Encode(), nil) +func (d remoteDriver) Shell(ctx context.Context, id string, size sandbox.TermSize) (sandbox.Terminal, error) { + req, err := opShell.request(ctx, shellRequest{ID: id, Size: size}) if err != nil { return nil, err } req.Header.Set("Connection", "Upgrade") req.Header.Set("Upgrade", shellProtocol) - resp, err := s.do(req) + resp, err := d.s.do(req) if err != nil { return nil, err } rwc, ok := resp.Body.(io.ReadWriteCloser) if resp.StatusCode != http.StatusSwitchingProtocols || !ok { resp.Body.Close() - return nil, fmt.Errorf("open shell on server %s: status %s", s.ServerID(), resp.Status) + return nil, fmt.Errorf("open shell on server %s: status %s", d.s.ServerID(), resp.Status) } return &remoteTerminal{rwc: rwc}, nil } -func (s *Session) Hold(ctx context.Context, id string, until time.Time) error { - return s.call(ctx, http.MethodPost, sandboxPath(id, "/hold"), holdRequest{Until: until.UTC()}, nil) -} - -func sandboxPath(id, suffix string) string { - return "/v1/sandboxes/" + url.PathEscape(id) + suffix +func (d remoteDriver) Hold(ctx context.Context, id string, until time.Time) error { + _, err := call(ctx, d.s, opHold, holdRequest{ID: id, Until: until.UTC()}) + return err } // The shell stream after the upgrade: the agent sends the raw terminal @@ -142,7 +127,6 @@ func (t *remoteTerminal) frame(kind byte, payload []byte) error { return err } -// readFrames passes the frames from r to term until r ends. func readFrames(r io.Reader, term sandbox.Terminal) error { var hdr [3]byte buf := make([]byte, maxFrame) diff --git a/internal/agent/helpers_test.go b/internal/agent/helpers_test.go index 5e924f7..b830af8 100644 --- a/internal/agent/helpers_test.go +++ b/internal/agent/helpers_test.go @@ -18,9 +18,8 @@ import ( "testing" "time" + "github.com/runplane/runplane/internal/doctor" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/image" - "github.com/runplane/runplane/internal/mirror" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/tlsutil" ) @@ -41,7 +40,7 @@ type testPrimary struct { sessions chan *Session } -func startPrimary(t *testing.T, cache http.Handler) *testPrimary { +func startPrimary(t *testing.T, cache func(http.ResponseWriter, *http.Request, string, string)) *testPrimary { cert, fp, err := tlsutil.AgentCert(t.TempDir()) if err != nil { t.Fatal(err) @@ -69,9 +68,9 @@ func startPrimary(t *testing.T, cache http.Handler) *testPrimary { } return "", "", errors.New("unknown server") }, - OnSession: func(s *Session) { p.sessions <- s }, - CacheHandler: cache, - Logger: p.logs.logger("primary"), + OnSession: func(s *Session) { p.sessions <- s }, + Cache: cache, + Logger: p.logs.logger("primary"), } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) @@ -105,9 +104,7 @@ func (p *testPrimary) startAgent(agentJSON string, node *fakeNode, proxy *CacheP ctx, cancel := context.WithCancel(context.Background()) a := &testAgent{path: path, cancel: cancel, done: make(chan struct{})} go func() { - a.err = Run(ctx, Config{ - Path: path, Node: node, CacheProxy: proxy, Mirror: func() mirror.Stats { return testMirror }, Logger: p.logs.logger("agent"), - }) + a.err = Run(ctx, Config{Path: path, Node: node, CacheProxy: proxy, Logger: p.logs.logger("agent")}) close(a.done) }() p.t.Cleanup(func() { @@ -124,7 +121,6 @@ func (a *testAgent) result(t *testing.T) error { return a.err } -// connect starts a primary and a registered agent. func connect(t *testing.T) (*testAgent, *fakeNode, *Session) { t.Helper() p := startPrimary(t, nil) @@ -163,7 +159,6 @@ func waitClosed[T any](t *testing.T, ch <-chan T) { } } -// readFull reads n bytes from r. A read error comes back as the text. func readFull(t *testing.T, r io.Reader, n int) string { t.Helper() ch := make(chan string, 1) @@ -210,40 +205,31 @@ func (l *logBuffer) logger(side string) *slog.Logger { return slog.New(slog.NewTextHandler(l, &slog.HandlerOptions{Level: slog.LevelDebug})).With("side", side) } -func (l *logBuffer) waitFor(t *testing.T, text string) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for !strings.Contains(l.String(), text) { - if time.Now().After(deadline) { - t.Fatalf("no log line with %q", text) - } - time.Sleep(10 * time.Millisecond) - } -} - var ( testInfo = host.Info{ - Hostname: "agent-1", OS: "Ubuntu 24.04.3 LTS", OSID: "ubuntu", OSVersion: "24.04", Kernel: "6.8.0-45-generic", Arch: "arm64", + Hostname: "agent-1", OS: "Ubuntu 24.04.3 LTS", Kernel: "6.8.0-45-generic", Arch: "arm64", Resources: host.Resources{VCPU: 16, MemoryMB: 65536, DiskBytes: 500e9, FreeDiskBytes: 320e9}, - DiskKind: "NVMe", CgroupV2: true, NFTables: true, IPForward: true, - DockerVersion: "29.1.0", DockerFirewall: "iptables", SysboxVersion: "0.7.1", SysboxRuntime: true, + DiskKind: "NVMe", KVM: true, DockerVersion: "29.1.0", SysboxVersion: "0.7.1", FirecrackerVersion: "1.13.1", IPv4: "203.0.113.7", Provider: "Hetzner", Region: "fsn1", } - testChecks = []host.Check{ - {ID: "kernel", Name: "Kernel", Status: host.StatusOK, Detail: "6.8.0-45-generic"}, - {ID: "sysbox", Name: "Sysbox", Status: host.StatusFail, Detail: "sysbox-runc is missing", Fix: "Install Sysbox 0.7.1"}, + testReport = doctor.Report{ + Info: host.Info{Hostname: "agent-1", OS: "Ubuntu 24.04.3 LTS", Kernel: "6.8.0-45-generic", Arch: "arm64", Region: "fsn1"}, + Checks: []doctor.Check{ + {ID: "kernel", Name: "Kernel", Status: doctor.StatusOK, Detail: "6.8.0-45-generic"}, + {ID: "sysbox", Name: "Sysbox", Status: doctor.StatusFail, Detail: "sysbox-runc is missing", Fix: "Install Sysbox 0.7.1"}, + }, } - testProgress = []image.Progress{ - {Status: image.StatusPending}, - {Status: image.StatusPulling, Done: 50, Total: 100}, - {Status: image.StatusPulling, Done: 100, Total: 100}, - {Status: image.StatusReady}, + testProgress = []Progress{ + {Status: ImagePending}, + {Status: ImagePulling, Done: 50, Total: 100}, + {Status: ImagePulling, Done: 100, Total: 100}, + {Status: ImageReady}, } - testMirror = mirror.Stats{Hits: 41, Misses: 3, HitBytes: 9 << 30, MissBytes: 700 << 20, SizeBytes: 12 << 30} + testMirror = MirrorStats{Hits: 41, Misses: 3, HitBytes: 9 << 30, MissBytes: 700 << 20, SizeBytes: 12 << 30} fakeCapacity = sandbox.Capacity{ Mode: sandbox.ModeContainer, Arch: "arm64", VCPU: 16, MemoryMB: 65536, - DiskBytes: 500e9, FreeDiskBytes: 320e9, AllocVCPU: 4, AllocMemoryMB: 8192, - ReserveVCPU: 1, ReserveMemoryMB: 2048, Overcommit: 1.5, MaxSandboxes: 10, Sandboxes: 2, + DiskBytes: 500e9, FreeDiskBytes: 320e9, AllocVCPU: 4, AllocMemoryMB: 8192, Sandboxes: 2, + OverheadMB: 64, MaxVCPU: 32, MaxSandboxes: 10, } ) @@ -259,6 +245,22 @@ type fakeNode struct { mu sync.Mutex sandboxes map[string]sandbox.Sandbox calls [][]any + hold *hold // of the next Facts call +} + +// hold makes a Facts call wait: held closes when the call waits, and the +// call goes on when resume closes. +type hold struct{ held, resume chan struct{} } + +// holdConnect makes the next connection of the agent wait, because the +// agent reads the facts before each connection. The agent waits when held +// closes, until the test calls resume. +func (n *fakeNode) holdConnect() (held <-chan struct{}, resume func()) { + h := &hold{held: make(chan struct{}), resume: make(chan struct{})} + n.mu.Lock() + n.hold = h + n.mu.Unlock() + return h.held, func() { close(h.resume) } } // logStream is one Logs call: the test writes the log to w, and done @@ -278,7 +280,6 @@ func newFakeNode() *fakeNode { } } -// record adds a call, and returns nil as the result of the call. func (n *fakeNode) record(call ...any) error { n.mu.Lock() defer n.mu.Unlock() @@ -301,23 +302,42 @@ func (n *fakeNode) put(sb sandbox.Sandbox) { n.sandboxes[sb.ID] = sb } -func (n *fakeNode) Driver() sandbox.Driver { return n } -func (n *fakeNode) Facts(context.Context) (host.Info, error) { return testInfo, nil } -func (n *fakeNode) Doctor(context.Context) ([]host.Check, error) { return testChecks, nil } -func (n *fakeNode) Stats() <-chan host.Stats { return n.stats } +func (n *fakeNode) Local() bool { return true } +func (n *fakeNode) Driver() Driver { return n } -func (n *fakeNode) EnsureImage(_ context.Context, ref string, progress func(image.Progress)) (sandbox.ImageRef, error) { +func (n *fakeNode) Facts(ctx context.Context) (host.Info, error) { + n.mu.Lock() + h := n.hold + n.hold = nil + n.mu.Unlock() + if h != nil { + close(h.held) + select { + case <-h.resume: + case <-ctx.Done(): + return host.Info{}, ctx.Err() + } + } + return testInfo, nil +} + +func (n *fakeNode) Doctor(context.Context) (doctor.Report, error) { + return testReport, n.record("Doctor") +} +func (n *fakeNode) Stats() <-chan host.Stats { return n.stats } + +func (n *fakeNode) EnsureImage(_ context.Context, ref string, progress func(Progress)) (Image, error) { for _, p := range testProgress { progress(p) } if ref == "missing:latest" { - return sandbox.ImageRef{}, errors.New("pull missing:latest: not found") + return Image{}, errors.New("pull missing:latest: not found") } - return sandbox.ImageRef{Ref: ref, Digest: "sha256:5f1c"}, nil + return Image{ImageRef: sandbox.ImageRef{Ref: ref, Digest: "sha256:5f1c"}, SizeBytes: 2147483648}, nil } -func (n *fakeNode) EnsureRunner(_ context.Context, version, arch string) (image.Bundle, error) { - return image.Bundle{Version: version, Arch: arch, SHA256: "9e2f", Patched: true, Dir: "/var/lib/runplane/runners/" + version + "-" + arch}, nil +func (n *fakeNode) EnsureRunner(_ context.Context, version, arch string) (Bundle, error) { + return Bundle{Version: version, Arch: arch, SHA256: "9e2f", Patched: true}, nil } func (n *fakeNode) GCImages(_ context.Context, keep []string) error { @@ -326,9 +346,10 @@ func (n *fakeNode) GCImages(_ context.Context, keep []string) error { func (n *fakeNode) PruneRunners(_ context.Context, keep []string) error { return n.record("PruneRunners", keep) } -func (n *fakeNode) Configure(_ context.Context, s sandbox.Settings) error { +func (n *fakeNode) Configure(_ context.Context, s Settings) error { return n.record("Configure", s) } +func (n *fakeNode) MirrorStats(context.Context) (MirrorStats, error) { return testMirror, nil } func (n *fakeNode) Update(_ context.Context, version string) error { return n.record("Update", version) } @@ -388,6 +409,12 @@ func (n *fakeNode) Shell(_ context.Context, _ string, size sandbox.TermSize) (sa } func (n *fakeNode) Hold(_ context.Context, id string, until time.Time) error { + n.mu.Lock() + _, ok := n.sandboxes[id] + n.mu.Unlock() + if !ok { + return fmt.Errorf("hold sandbox %s: %w", id, sandbox.ErrNotFound) + } return n.record("Hold", id, until) } diff --git a/internal/agent/install_test.go b/internal/agent/install_test.go new file mode 100644 index 0000000..18c0c9b --- /dev/null +++ b/internal/agent/install_test.go @@ -0,0 +1,63 @@ +package agent + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/runplane/runplane/internal/installtest" +) + +// install.sh writes agent.json for a new agent, and on a later run it reads +// the file that the agent wrote after its registration. Change install.sh +// and agentFile together. +func TestInstallScriptAgentFile(t *testing.T) { + for _, primary := range []string{"203.0.113.7:8485", "[2001:db8::5]:8485"} { + t.Run(primary, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "agent.json") + fp := "sha256:" + strings.Repeat("0f", 32) + flags := "--token rp_srv_TEST --primary " + primary + " --fingerprint " + fp + install := func(flags string) string { + t.Helper() + return installtest.Run(t, fmt.Sprintf(`CONFIG_DIR=%[1]s DATA_DIR=%[1]s/data UNIT_DIR=%[1]s AGENT_FILE=%[2]s ENV_FILE=%[3]s +parse_flags %[4]s +check_role +MODE=container +write_config +printf 'primary=%%s\n' "$PRIMARY"`, dir, path, filepath.Join(t.TempDir(), "runplane.env"), flags)) + } + + install(flags) + f, err := readAgentFile(path) + if err != nil { + t.Fatalf("the agent cannot read the agent.json of install.sh: %v", err) + } + if want := (agentFile{Primary: primary, Fingerprint: fp, Token: "rp_srv_TEST"}); f != want { + t.Errorf("the agent.json of install.sh gives %+v, want %+v", f, want) + } + + f.Token, f.ServerID, f.Secret = "", "01jbd5x7k2m4n6p8q9r0s1t2v3", "s3cret" + if err := f.write(path); err != nil { + t.Fatal(err) + } + registered, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + // The same install command again, and an update without a token. + for _, flags := range []string{flags, ""} { + out := install(flags) + if !strings.Contains(out, "primary="+f.Primary+"\n") { + t.Errorf("install.sh %s does not find the primary in the agent.json of the agent:\n%s", flags, out) + } + if got, _ := os.ReadFile(path); !bytes.Equal(got, registered) { + t.Errorf("install.sh %s changed the agent.json of a registered agent:\n%s\nwant\n%s", flags, got, registered) + } + } + }) + } +} diff --git a/internal/agent/node.go b/internal/agent/node.go index 80f7a2c..26ee3a8 100644 --- a/internal/agent/node.go +++ b/internal/agent/node.go @@ -1,46 +1,69 @@ -// Package agent connects the servers of runplane. The primary operates its -// own server through a LocalNode and each agent through a Session: Listener -// accepts the agent connections on the primary, and Run operates the agent -// end on the other servers. +// Package agent connects the servers of runplane. The primary operates each +// server through a Node: its own server through a localnode.Node, and each +// agent through a Session. Listener accepts the agent connections on the +// primary, and Run operates the agent end on the other servers. package agent import ( "context" - "errors" - "log/slog" - "sync" + "io" "time" + "github.com/runplane/runplane/internal/doctor" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/image" "github.com/runplane/runplane/internal/sandbox" - "github.com/runplane/runplane/internal/update" ) // Node is one server as the primary sees it: its sandbox driver, its host, -// its images and runner bundles, and its runplane binary. +// its images and runner bundles, its registry mirror, and its runplane +// binary. type Node interface { - Driver() sandbox.Driver + // Local reports if the node is the server of this process. The other + // nodes are agents. + Local() bool + Driver() Driver + Configure(ctx context.Context, s Settings) error Facts(ctx context.Context) (host.Info, error) - Doctor(ctx context.Context) ([]host.Check, error) + // Doctor runs the doctor on the server, with the min_free_disk_percent + // of its last settings. + Doctor(ctx context.Context) (doctor.Report, error) // EnsureImage makes an image ready on the server. progress, if not nil, // gets each step. - EnsureImage(ctx context.Context, ref string, progress func(image.Progress)) (sandbox.ImageRef, error) - EnsureRunner(ctx context.Context, version, arch string) (image.Bundle, error) + EnsureImage(ctx context.Context, ref string, progress func(Progress)) (Image, error) + EnsureRunner(ctx context.Context, version, arch string) (Bundle, error) // GCImages removes the images whose digest is not in keep. GCImages(ctx context.Context, keep []string) error // PruneRunners removes the runner bundles except the two newest and the // versions in keep. PruneRunners(ctx context.Context, keep []string) error - Configure(ctx context.Context, s sandbox.Settings) error + MirrorStats(ctx context.Context) (MirrorStats, error) // Update replaces the runplane binary with a release. The server runs // the new binary after a restart. Update(ctx context.Context, version string) error // Stats delivers one host sample every 10 seconds. A reader that falls - // behind gets only the latest sample. + // behind gets only the latest sample (SendLatest). Stats() <-chan host.Stats + // Close ends the connection of an agent, whose sandboxes keep + // operating. A local node stops its registry mirror. + Close() error } +// Driver is the part of a sandbox driver that the primary uses on each +// server. The settings and the doctor of a server go through its Node. +type Driver interface { + Mode() sandbox.Mode + Start(ctx context.Context, spec sandbox.Spec) error + Stop(ctx context.Context, id string) error + List(ctx context.Context) ([]sandbox.Sandbox, error) + Capacity(ctx context.Context) (sandbox.Capacity, error) + Events() <-chan sandbox.Event + Logs(ctx context.Context, id string, follow bool) (io.ReadCloser, error) + Shell(ctx context.Context, id string, size sandbox.TermSize) (sandbox.Terminal, error) + Hold(ctx context.Context, id string, until time.Time) error +} + +var _ Node = (*Session)(nil) + // Forward passes the driver events and the host stats of n on until both // channels close. It returns ctx.Err() if ctx ends first. func Forward(ctx context.Context, n Node, onEvent func(sandbox.Event), onStats func(host.Stats)) error { @@ -66,103 +89,9 @@ func Forward(ctx context.Context, n Node, onEvent func(sandbox.Event), onStats f return nil } -type LocalConfig struct { - Driver sandbox.Driver - Images *image.Manager - Bundles *image.Bundles - Updater *update.Updater - // Doctor holds the doctor options of this server. Its DataDir also - // selects the disk for the facts and the stats. Configure sets - // MinFreeDiskPercent. - Doctor host.Options - Logger *slog.Logger -} - -// LocalNode is the server that runplane operates on. -type LocalNode struct { - cfg LocalConfig - mu sync.Mutex // guards cfg.Doctor - stats chan host.Stats -} - -func NewLocal(cfg LocalConfig) (*LocalNode, error) { - if cfg.Driver == nil || cfg.Images == nil || cfg.Bundles == nil || cfg.Updater == nil { - return nil, errors.New("new local node: Driver, Images, Bundles, and Updater are required") - } - return &LocalNode{cfg: cfg, stats: make(chan host.Stats, 1)}, nil -} - -// Run samples the host stats for Stats until ctx ends. -func (n *LocalNode) Run(ctx context.Context) { - sampler := host.Sampler{DataDir: n.cfg.Doctor.DataDir} - for { - err := sampler.Run(ctx, func(s host.Stats) { sendLatest(n.stats, s) }) - if err == nil { - return - } - n.cfg.Logger.Warn("sample host stats", "err", err) - select { - case <-ctx.Done(): - return - case <-time.After(10 * time.Second): - } - } -} - -func (n *LocalNode) Driver() sandbox.Driver { return n.cfg.Driver } - -func (n *LocalNode) Facts(ctx context.Context) (host.Info, error) { - return host.Facts(ctx, n.cfg.Doctor.DataDir) -} - -func (n *LocalNode) Doctor(ctx context.Context) ([]host.Check, error) { - n.mu.Lock() - opts := n.cfg.Doctor - n.mu.Unlock() - return host.Doctor(ctx, n.cfg.Driver.Mode(), opts), nil -} - -func (n *LocalNode) EnsureImage(ctx context.Context, ref string, progress func(image.Progress)) (sandbox.ImageRef, error) { - img, err := n.cfg.Images.Ensure(ctx, ref, progress) - if err != nil { - return sandbox.ImageRef{}, err - } - return sandbox.ImageRef{Ref: img.Ref, Digest: img.Digest}, nil -} - -func (n *LocalNode) EnsureRunner(ctx context.Context, version, arch string) (image.Bundle, error) { - return n.cfg.Bundles.Ensure(ctx, version, arch) -} - -func (n *LocalNode) GCImages(ctx context.Context, keep []string) error { - return n.cfg.Images.GC(ctx, keep) -} - -func (n *LocalNode) PruneRunners(_ context.Context, keep []string) error { - return n.cfg.Bundles.Prune(keep) -} - -// Configure keeps the free disk limit for the doctor and passes the -// settings to the driver if it takes them. -func (n *LocalNode) Configure(_ context.Context, s sandbox.Settings) error { - n.mu.Lock() - n.cfg.Doctor.MinFreeDiskPercent = float64(s.MinFreeDiskPercent) - n.mu.Unlock() - if c, ok := n.cfg.Driver.(sandbox.Configurable); ok { - return c.Configure(s) - } - return nil -} - -func (n *LocalNode) Update(ctx context.Context, version string) error { - return n.cfg.Updater.Apply(ctx, version) -} - -func (n *LocalNode) Stats() <-chan host.Stats { return n.stats } - -// sendLatest puts v in ch, a channel with a buffer of one, and drops an +// SendLatest puts v in ch, a channel with a buffer of one, and drops an // older value that nobody took. -func sendLatest[T any](ch chan T, v T) { +func SendLatest[T any](ch chan T, v T) { for { select { case ch <- v: diff --git a/internal/agent/primary.go b/internal/agent/primary.go index 5569a93..0f699ee 100644 --- a/internal/agent/primary.go +++ b/internal/agent/primary.go @@ -2,7 +2,6 @@ package agent import ( "bufio" - "bytes" "context" "crypto/tls" "encoding/json" @@ -17,10 +16,8 @@ import ( "github.com/hashicorp/yamux" - "github.com/runplane/runplane/internal/cache" + "github.com/runplane/runplane/internal/doctor" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/image" - "github.com/runplane/runplane/internal/mirror" "github.com/runplane/runplane/internal/sandbox" ) @@ -38,11 +35,11 @@ type Listener struct { // OnSession gets each new session. If the server already has a // session, the Listener closes the old one first. OnSession func(*Session) - // CacheHandler serves the Actions cache requests that agents send, with - // /v1/cache cut from the path: the Proxied handler of the cache server. - // Nil answers 503. - CacheHandler http.Handler - Logger *slog.Logger + // Cache serves an Actions cache request that the agent serverID sends + // for its sandbox sandboxID, with /v1/cache cut from the path: the + // ServeAgent method of the cache server. Nil answers 503. + Cache func(w http.ResponseWriter, r *http.Request, serverID, sandboxID string) + Logger *slog.Logger mu sync.Mutex sessions map[string]*Session @@ -99,7 +96,6 @@ func (l *Listener) accept(ctx context.Context, conn net.Conn) { l.OnSession(s) } -// open runs the agent handshake and starts yamux. func (l *Listener) open(ctx context.Context, conn net.Conn) (*Session, error) { ctx, cancel := context.WithTimeout(ctx, handshakeTimeout) defer cancel() @@ -108,10 +104,11 @@ func (l *Listener) open(ctx context.Context, conn net.Conn) (*Session, error) { return nil, fmt.Errorf("tls handshake: %w", err) } br := bufio.NewReaderSize(conn, maxLine) - var hello Hello - if err := readJSONLine(br, &hello); err != nil { + var line helloLine + if err := readJSONLine(br, &line); err != nil { return nil, fmt.Errorf("read hello: %w", err) } + hello := line.hello() hello.Addr = remoteIP(conn.RemoteAddr()) id, secret, err := l.Authenticate(ctx, hello) if err != nil { @@ -128,7 +125,7 @@ func (l *Listener) open(ctx context.Context, conn net.Conn) (*Session, error) { return nil, fmt.Errorf("start yamux: %w", err) } hello.ServerID, hello.Token, hello.Secret = id, "", "" - return newSession(sess, hello, l.CacheHandler, l.Logger.With("server_id", id)), nil + return newSession(sess, hello, l.Cache, l.Logger.With("server_id", id)), nil } // add keeps s as the session of its server and returns the old session. @@ -166,10 +163,9 @@ func (l *Listener) closeAll() { } } -// Session is the connection of one agent. It implements Node and the -// sandbox driver of the agent: each call is an HTTP request on a new yamux -// stream. A Session ends when the connection drops; the agent then connects -// again and the Listener makes a new Session. +// Session is the connection of one agent. It implements Node: each call is +// an HTTP request on a new yamux stream. A Session ends when the connection +// drops; the agent then connects again and the Listener makes a new Session. type Session struct { hello Hello sess *yamux.Session @@ -185,7 +181,7 @@ type Session struct { closed bool } -func newSession(sess *yamux.Session, hello Hello, cache http.Handler, log *slog.Logger) *Session { +func newSession(sess *yamux.Session, hello Hello, cache func(http.ResponseWriter, *http.Request, string, string), log *slog.Logger) *Session { s := &Session{ hello: hello, sess: sess, @@ -196,8 +192,8 @@ func newSession(sess *yamux.Session, hello Hello, cache http.Handler, log *slog. stats: make(chan host.Stats, 1), } mux := http.NewServeMux() - mux.HandleFunc("POST /v1/events", s.handleEvents) - mux.Handle("/v1/cache/", cacheHandler(cache, hello.ServerID)) + mux.HandleFunc("POST "+eventsPath, s.handleEvents) + mux.Handle(cachePath+"/", cacheHandler(cache, hello.ServerID)) go serveStreams(sess, mux, log) go func() { <-sess.CloseChan() @@ -214,6 +210,10 @@ func (s *Session) Hello() Hello { return s.hello } // Done is closed when the session ends. func (s *Session) Done() <-chan struct{} { return s.done } +func (s *Session) Local() bool { return false } + +func (s *Session) Driver() Driver { return remoteDriver{s} } + // Close ends the session. The sandboxes on the agent keep operating. func (s *Session) Close() error { s.sess.Close() @@ -221,28 +221,28 @@ func (s *Session) Close() error { return nil } -func (s *Session) Driver() sandbox.Driver { return s } - -// Stats delivers the host samples of the agent. It closes when the session -// ends. +// Stats closes its channel when the session ends. func (s *Session) Stats() <-chan host.Stats { return s.stats } +func (s *Session) Configure(ctx context.Context, settings Settings) error { + _, err := call(ctx, s, opConfigure, settings) + return err +} + func (s *Session) Facts(ctx context.Context) (host.Info, error) { - var rep hostReport - err := s.call(ctx, http.MethodGet, "/v1/host", nil, &rep) - return rep.Info, err + rep, err := call(ctx, s, opHost, hostQuery{}) + return rep.Info.info(), err } -func (s *Session) Doctor(ctx context.Context) ([]host.Check, error) { - var rep hostReport - err := s.call(ctx, http.MethodGet, "/v1/host?doctor=1", nil, &rep) - return rep.Doctor, err +func (s *Session) Doctor(ctx context.Context) (doctor.Report, error) { + rep, err := call(ctx, s, opHost, hostQuery{Doctor: true}) + return doctor.Report{Info: rep.Info.info(), Checks: checks(rep.Doctor)}, err } -func (s *Session) EnsureImage(ctx context.Context, ref string, progress func(image.Progress)) (sandbox.ImageRef, error) { - resp, err := s.send(ctx, http.MethodPost, "/v1/images/ensure", sandbox.ImageRef{Ref: ref}) +func (s *Session) EnsureImage(ctx context.Context, ref string, progress func(Progress)) (Image, error) { + resp, err := open(ctx, s, opEnsureImage.route, sandbox.ImageRef{Ref: ref}) if err != nil { - return sandbox.ImageRef{}, err + return Image{}, err } defer resp.Body.Close() dec := json.NewDecoder(resp.Body) @@ -252,11 +252,11 @@ func (s *Session) EnsureImage(ctx context.Context, ref string, progress func(ima if errors.Is(err, io.EOF) { err = io.ErrUnexpectedEOF } - return sandbox.ImageRef{}, fmt.Errorf("ensure image %s on server %s: %w", ref, s.ServerID(), s.closedOr(err)) + return Image{}, fmt.Errorf("ensure image %s on server %s: %w", ref, s.ServerID(), s.closedOr(err)) } switch { case m.Error != "": - return sandbox.ImageRef{}, fmt.Errorf("server %s: %s", s.ServerID(), m.Error) + return Image{}, fmt.Errorf("server %s: %s", s.ServerID(), m.Error) case m.Image != nil: return *m.Image, nil case m.Progress != nil && progress != nil: @@ -265,71 +265,31 @@ func (s *Session) EnsureImage(ctx context.Context, ref string, progress func(ima } } -func (s *Session) EnsureRunner(ctx context.Context, version, arch string) (image.Bundle, error) { - var b image.Bundle - err := s.call(ctx, http.MethodPost, "/v1/runners/ensure", runnerRequest{Version: version, Arch: arch}, &b) - return b, err +func (s *Session) EnsureRunner(ctx context.Context, version, arch string) (Bundle, error) { + return call(ctx, s, opEnsureRunner, runnerRequest{Version: version, Arch: arch}) } func (s *Session) GCImages(ctx context.Context, keep []string) error { - return s.call(ctx, http.MethodPost, "/v1/images/gc", gcRequest{Keep: keep}, nil) + _, err := call(ctx, s, opGCImages, keepRequest{Keep: keep}) + return err } func (s *Session) PruneRunners(ctx context.Context, keep []string) error { - return s.call(ctx, http.MethodPost, "/v1/runners/prune", gcRequest{Keep: keep}, nil) -} - -// MirrorStats returns the stats of the registry mirror of the agent. -func (s *Session) MirrorStats(ctx context.Context) (mirror.Stats, error) { - var m mirror.Stats - err := s.call(ctx, http.MethodGet, "/v1/mirror/stats", nil, &m) - return m, err + _, err := call(ctx, s, opPruneRunners, keepRequest{Keep: keep}) + return err } -func (s *Session) Configure(ctx context.Context, settings sandbox.Settings) error { - return s.call(ctx, http.MethodPost, "/v1/settings", settings, nil) +func (s *Session) MirrorStats(ctx context.Context) (MirrorStats, error) { + return call(ctx, s, opMirrorStats, none{}) } -// Update makes the agent replace its binary and restart. The session then -// ends, and the agent connects again with the new version. +// Update makes the agent replace its binary. When the agent answers, the +// session ends: the agent restarts and connects again with the new version. func (s *Session) Update(ctx context.Context, version string) error { - return s.call(ctx, http.MethodPost, "/v1/update", updateRequest{Version: version}, nil) -} - -// call sends a request and decodes the JSON result into out, if out is not nil. -func (s *Session) call(ctx context.Context, method, path string, in, out any) error { - resp, err := s.send(ctx, method, path, in) - if err != nil { + if _, err := call(ctx, s, opUpdate, updateRequest{Version: version}); err != nil { return err } - defer resp.Body.Close() - if out == nil { - return nil - } - if err := json.NewDecoder(resp.Body).Decode(out); err != nil { - return fmt.Errorf("%s %s on server %s: read result: %w", method, path, s.ServerID(), s.closedOr(err)) - } - return nil -} - -// send sends a request with in as the JSON body, if in is not nil. -func (s *Session) send(ctx context.Context, method, path string, in any) (*http.Response, error) { - var body io.Reader - if in != nil { - b, err := json.Marshal(in) - if err != nil { - return nil, err - } - body = bytes.NewReader(b) - } - req, err := http.NewRequestWithContext(ctx, method, "http://agent"+path, body) - if err != nil { - return nil, err - } - if in != nil { - req.Header.Set("Content-Type", "application/json") - } - return s.do(req) + return s.Close() } // do sends req to the agent. It returns an error for a status of 300 or more. @@ -337,7 +297,7 @@ func (s *Session) do(req *http.Request) (*http.Response, error) { fail := func(err error) error { return fmt.Errorf("%s %s on server %s: %w", req.Method, req.URL.Path, s.ServerID(), err) } - if s.isDone() { + if s.sess.IsClosed() { return nil, fail(ErrClosed) } resp, err := s.client.Do(req) @@ -346,29 +306,25 @@ func (s *Session) do(req *http.Request) (*http.Response, error) { } if resp.StatusCode >= 300 { defer resp.Body.Close() - return nil, fail(responseError(resp)) + err := responseError(resp) + if resp.StatusCode == http.StatusNotFound { + err = fmt.Errorf("%w: %w", sandbox.ErrNotFound, err) + } + return nil, fail(err) } return resp, nil } -// closedOr returns ErrClosed if the session has ended, else err. +// closedOr returns ErrClosed if the connection has ended, else err. The +// yamux session closes before Done, so that an error of yamux such as +// "session shutdown" also becomes ErrClosed. func (s *Session) closedOr(err error) error { - if s.isDone() { + if s.sess.IsClosed() { return ErrClosed } return err } -func (s *Session) isDone() bool { - select { - case <-s.done: - return true - default: - return false - } -} - -// handleEvents reads the event stream of the agent. func (s *Session) handleEvents(w http.ResponseWriter, r *http.Request) { dec := json.NewDecoder(r.Body) for { @@ -395,7 +351,7 @@ func (s *Session) deliver(it streamItem) bool { return false } if it.Stats != nil { - sendLatest(s.stats, *it.Stats) + SendLatest(s.stats, it.Stats.stats()) } if it.Event != nil { select { @@ -417,16 +373,18 @@ func (s *Session) finish() { s.client.CloseIdleConnections() } -// cacheHandler serves ANY /v1/cache/* for one agent. -func cacheHandler(h http.Handler, serverID string) http.Handler { - if h == nil { +// cacheHandler gives the cache requests of the agent serverID to serve. It +// removes sandboxHeader, so that the cache server does not send it on to +// GitHub. +func cacheHandler(serve func(http.ResponseWriter, *http.Request, string, string), serverID string) http.Handler { + if serve == nil { return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "the Actions cache is not available", http.StatusServiceUnavailable) }) } - h = http.StripPrefix("/v1/cache", h) - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r.Header.Set(cache.ServerHeader, serverID) - h.ServeHTTP(w, r) - }) + return http.StripPrefix(cachePath, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sandboxID := r.Header.Get(sandboxHeader) + r.Header.Del(sandboxHeader) + serve(w, r, serverID, sandboxID) + })) } diff --git a/internal/agent/protocol.go b/internal/agent/protocol.go new file mode 100644 index 0000000..72fb657 --- /dev/null +++ b/internal/agent/protocol.go @@ -0,0 +1,472 @@ +package agent + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/runplane/runplane/internal/sandbox" +) + +// The agent protocol. The primary calls the operations of a Node on an +// agent, each as one HTTP request on a new yamux stream of the connection. +// The table below gives each operation its method, its path, the types of +// its request and its answer, and the code that serves it on the agent. +// Session calls the operations, and the agent serves each operation of the +// table. The agent sends its events and its cache requests the other way, +// to the primary. +var ( + opStart = newOp(http.MethodPost, "/v1/sandboxes", func(ctx context.Context, a *agent, spec sandbox.Spec) (none, error) { + return none{}, a.Node.Driver().Start(ctx, spec) + }) + + opList = newOp(http.MethodGet, "/v1/sandboxes", func(ctx context.Context, a *agent, _ none) ([]sandbox.Sandbox, error) { + list, err := a.Node.Driver().List(ctx) + if list == nil { + list = []sandbox.Sandbox{} + } + return list, err + }) + + opStop = newOp(http.MethodDelete, "/v1/sandboxes/{id}", func(ctx context.Context, a *agent, id sandboxID) (none, error) { + return none{}, a.Node.Driver().Stop(ctx, string(id)) + }) + + opHold = newOp(http.MethodPost, "/v1/sandboxes/{id}/hold", func(ctx context.Context, a *agent, req holdRequest) (none, error) { + return none{}, a.Node.Driver().Hold(ctx, req.ID, req.Until) + }) + + opLogs = newStream(http.MethodGet, "/v1/sandboxes/{id}/logs", serveLogs) + + opShell = newStream(http.MethodGet, "/v1/sandboxes/{id}/shell", serveShell) + + opCapacity = newOp(http.MethodGet, "/v1/capacity", func(ctx context.Context, a *agent, _ none) (sandbox.Capacity, error) { + return a.Node.Driver().Capacity(ctx) + }) + + opConfigure = newOp(http.MethodPost, "/v1/settings", func(ctx context.Context, a *agent, s Settings) (none, error) { + return none{}, a.Node.Configure(ctx, s) + }) + + // opHost answers the facts, or with Doctor the report of the doctor, + // which has the facts too. + opHost = newOp(http.MethodGet, "/v1/host", func(ctx context.Context, a *agent, q hostQuery) (hostReport, error) { + if !q.Doctor { + info, err := a.Node.Facts(ctx) + return hostReport{Info: wireInfo(info)}, err + } + r, err := a.Node.Doctor(ctx) + return hostReport{Info: wireInfo(r.Info), Doctor: wireChecks(r.Checks)}, err + }) + + opEnsureImage = newStream(http.MethodPost, "/v1/images/ensure", serveEnsureImage) + + opGCImages = newOp(http.MethodPost, "/v1/images/gc", func(ctx context.Context, a *agent, req keepRequest) (none, error) { + return none{}, a.Node.GCImages(ctx, req.Keep) + }) + + opEnsureRunner = newOp(http.MethodPost, "/v1/runners/ensure", func(ctx context.Context, a *agent, req runnerRequest) (Bundle, error) { + return a.Node.EnsureRunner(ctx, req.Version, req.Arch) + }) + + opPruneRunners = newOp(http.MethodPost, "/v1/runners/prune", func(ctx context.Context, a *agent, req keepRequest) (none, error) { + return none{}, a.Node.PruneRunners(ctx, req.Keep) + }) + + opMirrorStats = newOp(http.MethodGet, "/v1/mirror/stats", func(ctx context.Context, a *agent, _ none) (MirrorStats, error) { + return a.Node.MirrorStats(ctx) + }) + + // opUpdate replaces the binary. The primary closes the connection when + // it has the answer, and then Run returns ErrUpdated. + opUpdate = newOp(http.MethodPost, "/v1/update", func(ctx context.Context, a *agent, req updateRequest) (none, error) { + if err := a.Node.Update(ctx, req.Version); err != nil { + return none{}, err + } + a.Logger.Info("binary updated, restarting", "version", req.Version) + a.updateOnce.Do(func() { close(a.updated) }) + return none{}, nil + }) +) + +// The paths of the primary that an agent calls. +const ( + // eventsPath takes the stream of the driver events and the host stats + // of the agent, as JSON lines of streamItem. + eventsPath = "/v1/events" + // cachePath and the paths under it take the Actions cache requests of + // the sandboxes on the agent. + cachePath = "/v1/cache" + // sandboxHeader names the sandbox of a cache request. The agent finds + // the sandbox at the source address of the request. + sandboxHeader = "X-Runplane-Sandbox" +) + +// Settings are the settings that a server gets from the primary: the +// settings of its sandbox driver, and min_free_disk_percent for its +// registry mirror and its doctor. +type Settings struct { + sandbox.Settings + MinFreeDiskPercent float64 `json:"min_free_disk_percent"` +} + +// none is the request or the answer of an operation that has none. The +// answer none is 204 No Content. +type none struct{} + +type ( + holdRequest struct { + ID string `json:"-"` + Until time.Time `json:"until"` + } + logsRequest struct { + ID string + Follow bool + } + shellRequest struct { + ID string + Size sandbox.TermSize + } + hostQuery struct { + Doctor bool + } + hostReport struct { + Info hostInfo `json:"info"` + Doctor []hostCheck `json:"doctor,omitempty"` + } + runnerRequest struct { + Version string `json:"version"` + Arch string `json:"arch"` + } + // keepRequest names the images (by digest) or the runner versions + // that the agent keeps. + keepRequest struct { + Keep []string `json:"keep"` + } + updateRequest struct { + Version string `json:"version"` + } + // imageMessage is one line of the answer of opEnsureImage. The last + // line has Image or Error. + imageMessage struct { + Progress *Progress `json:"progress,omitempty"` + Image *Image `json:"image,omitempty"` + Error string `json:"error,omitempty"` + } + // streamItem is one line of the stream to eventsPath. + streamItem struct { + Event *sandbox.Event `json:"event,omitempty"` + Stats *hostStats `json:"stats,omitempty"` + } +) + +// sandboxID is the request of an operation on one sandbox. +type sandboxID string + +// A request that is a urlArgs goes in the URL: the {id} of the path and the +// query. The agent reads it back with urlParser. A request of a POST also +// goes as the JSON body, without its fields that have no JSON. +type ( + urlArgs interface { + args() (id string, query url.Values) + } + urlParser interface { + parse(id string, query url.Values) error + } +) + +func (id sandboxID) args() (string, url.Values) { return string(id), nil } + +func (id *sandboxID) parse(v string, _ url.Values) error { + *id = sandboxID(v) + return nil +} + +func (r holdRequest) args() (string, url.Values) { return r.ID, nil } + +func (r *holdRequest) parse(id string, _ url.Values) error { + r.ID = id + return nil +} + +func (r logsRequest) args() (string, url.Values) { + if r.Follow { + return r.ID, url.Values{"follow": {"1"}} + } + return r.ID, nil +} + +func (r *logsRequest) parse(id string, q url.Values) error { + r.ID, r.Follow = id, q.Get("follow") == "1" + return nil +} + +func (r shellRequest) args() (string, url.Values) { + return r.ID, url.Values{"rows": {strconv.Itoa(int(r.Size.Rows))}, "cols": {strconv.Itoa(int(r.Size.Cols))}} +} + +func (r *shellRequest) parse(id string, q url.Values) error { + rows, err1 := strconv.ParseUint(q.Get("rows"), 10, 16) + cols, err2 := strconv.ParseUint(q.Get("cols"), 10, 16) + if err := errors.Join(err1, err2); err != nil { + return fmt.Errorf("bad terminal size: %w", err) + } + r.ID, r.Size = id, sandbox.TermSize{Rows: uint16(rows), Cols: uint16(cols)} + return nil +} + +func (q hostQuery) args() (string, url.Values) { + if !q.Doctor { + return "", nil + } + return "", url.Values{"doctor": {"1"}} +} + +func (q *hostQuery) parse(_ string, v url.Values) error { + q.Doctor = v.Get("doctor") == "1" + return nil +} + +type route[In any] struct { + method, path string +} + +// op is an operation whose answer is Out as JSON. +type op[In, Out any] struct { + route[In] + serve func(ctx context.Context, a *agent, in In) (Out, error) +} + +// stream is an operation whose answer the agent writes while it goes: a +// log, a shell, or the progress of an image. +type stream[In any] struct { + route[In] + serve func(w http.ResponseWriter, r *http.Request, a *agent, in In) +} + +// handlers holds the handler of each operation that newOp and newStream +// make. The agent serves all of them. +var handlers []handler + +type handler struct { + pattern string + serve func(a *agent) http.HandlerFunc +} + +func newOp[In, Out any](method, path string, serve func(context.Context, *agent, In) (Out, error)) op[In, Out] { + o := op[In, Out]{route[In]{method, path}, serve} + handlers = append(handlers, handler{method + " " + path, o.handler}) + return o +} + +func newStream[In any](method, path string, serve func(http.ResponseWriter, *http.Request, *agent, In)) stream[In] { + s := stream[In]{route[In]{method, path}, serve} + handlers = append(handlers, handler{method + " " + path, s.handler}) + return s +} + +func (o op[In, Out]) handler(a *agent) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + in, err := o.read(w, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + out, err := o.serve(r.Context(), a, in) + if err != nil { + writeError(w, err) + return + } + if _, ok := any(out).(none); ok { + w.WriteHeader(http.StatusNoContent) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(out) + } +} + +func (s stream[In]) handler(a *agent) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + in, err := s.read(w, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + s.serve(w, r, a, in) + } +} + +func (rt route[In]) request(ctx context.Context, in In) (*http.Request, error) { + path, query := rt.path, url.Values(nil) + if a, ok := any(in).(urlArgs); ok { + var id string + id, query = a.args() + path = strings.Replace(path, "{id}", url.PathEscape(id), 1) + } + if len(query) > 0 { + path += "?" + query.Encode() + } + var body io.Reader + if rt.method == http.MethodPost { + b, err := json.Marshal(in) + if err != nil { + return nil, err + } + body = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, rt.method, "http://agent"+path, body) + if err != nil { + return nil, err + } + // net/http sends a request again on a new stream when the stream + // reports that it wrote nothing, if it can read the body again. A yamux + // stream can report that after the agent got the request, so the body + // cannot be read again, and a POST never goes two times. + req.GetBody = nil + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return req, nil +} + +func (rt route[In]) read(w http.ResponseWriter, r *http.Request) (In, error) { + var in In + if rt.method == http.MethodPost { + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBody)).Decode(&in); err != nil { + return in, fmt.Errorf("bad request body: %w", err) + } + } + if p, ok := any(&in).(urlParser); ok { + if err := p.parse(r.PathValue("id"), r.URL.Query()); err != nil { + return in, err + } + } + return in, nil +} + +func call[In, Out any](ctx context.Context, s *Session, o op[In, Out], in In) (Out, error) { + var out Out + resp, err := open(ctx, s, o.route, in) + if err != nil { + return out, err + } + defer resp.Body.Close() + if _, ok := any(out).(none); ok { + return out, nil + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return out, fmt.Errorf("%s %s on server %s: read the answer: %w", o.method, o.path, s.ServerID(), s.closedOr(err)) + } + return out, nil +} + +func open[In any](ctx context.Context, s *Session, rt route[In], in In) (*http.Response, error) { + req, err := rt.request(ctx, in) + if err != nil { + return nil, err + } + return s.do(req) +} + +// writeError answers 404 for sandbox.ErrNotFound, which Session turns back +// into the error. +func writeError(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + if errors.Is(err, sandbox.ErrNotFound) { + status = http.StatusNotFound + } + http.Error(w, err.Error(), status) +} + +func serveLogs(w http.ResponseWriter, r *http.Request, a *agent, req logsRequest) { + rc, err := a.Node.Driver().Logs(r.Context(), req.ID, req.Follow) + if err != nil { + writeError(w, err) + return + } + defer rc.Close() + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusOK) + // Send the header now: a quiet log can take minutes for its first line. + flush := http.NewResponseController(w).Flush + flush() + buf := make([]byte, 32<<10) + for { + n, err := rc.Read(buf) + if n > 0 { + if _, werr := w.Write(buf[:n]); werr != nil { + return + } + flush() + } + if err != nil { + return + } + } +} + +// serveShell upgrades the stream to a shell. See readFrames for the stream +// format. +func serveShell(w http.ResponseWriter, r *http.Request, a *agent, req shellRequest) { + if r.Header.Get("Upgrade") != shellProtocol { + http.Error(w, "the shell needs Upgrade: "+shellProtocol, http.StatusBadRequest) + return + } + term, err := a.Node.Driver().Shell(r.Context(), req.ID, req.Size) + if err != nil { + writeError(w, err) + return + } + defer term.Close() + conn, brw, err := http.NewResponseController(w).Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer conn.Close() + brw.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: " + shellProtocol + "\r\n\r\n") + if err := brw.Flush(); err != nil { + return + } + go func() { + io.Copy(conn, term) + conn.Close() + }() + if err := readFrames(brw.Reader, term); !errors.Is(err, io.EOF) { + a.Logger.Debug("shell stream ended", "err", err) + } +} + +// serveEnsureImage streams the progress as JSON lines. The last line has +// the image or the error. +func serveEnsureImage(w http.ResponseWriter, r *http.Request, a *agent, ref sandbox.ImageRef) { + if ref.Ref == "" { + http.Error(w, "no image ref", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/x-ndjson") + w.WriteHeader(http.StatusOK) + flush := http.NewResponseController(w).Flush + enc := json.NewEncoder(w) + send := func(m imageMessage) { + enc.Encode(m) + flush() + } + img, err := a.Node.EnsureImage(r.Context(), ref.Ref, func(p Progress) { + send(imageMessage{Progress: &p}) + }) + if err != nil { + send(imageMessage{Error: err.Error()}) + return + } + send(imageMessage{Image: &img}) +} diff --git a/internal/agent/routes.go b/internal/agent/routes.go deleted file mode 100644 index 01dc44d..0000000 --- a/internal/agent/routes.go +++ /dev/null @@ -1,244 +0,0 @@ -package agent - -import ( - "encoding/json" - "errors" - "io" - "net/http" - "strconv" - - "github.com/runplane/runplane/internal/image" - "github.com/runplane/runplane/internal/sandbox" -) - -// newRoutes returns the routes that the agent serves to the primary. -func (a *agent) newRoutes() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("POST /v1/sandboxes", a.startSandbox) - mux.HandleFunc("GET /v1/sandboxes", a.listSandboxes) - mux.HandleFunc("DELETE /v1/sandboxes/{id}", a.stopSandbox) - mux.HandleFunc("GET /v1/sandboxes/{id}/logs", a.sandboxLogs) - mux.HandleFunc("POST /v1/sandboxes/{id}/hold", a.holdSandbox) - mux.HandleFunc("GET /v1/sandboxes/{id}/shell", a.sandboxShell) - mux.HandleFunc("GET /v1/capacity", a.readCapacity) - mux.HandleFunc("GET /v1/host", a.readHost) - mux.HandleFunc("POST /v1/images/ensure", a.ensureImage) - mux.HandleFunc("POST /v1/images/gc", a.gcImages) - mux.HandleFunc("GET /v1/mirror/stats", a.mirrorStats) - mux.HandleFunc("POST /v1/runners/ensure", a.ensureRunner) - mux.HandleFunc("POST /v1/runners/prune", a.pruneRunners) - mux.HandleFunc("POST /v1/settings", a.configure) - mux.HandleFunc("POST /v1/update", a.update) - return mux -} - -func (a *agent) startSandbox(w http.ResponseWriter, r *http.Request) { - var spec sandbox.Spec - if decode(w, r, &spec) { - writeDone(w, a.Node.Driver().Start(r.Context(), spec)) - } -} - -func (a *agent) listSandboxes(w http.ResponseWriter, r *http.Request) { - list, err := a.Node.Driver().List(r.Context()) - if list == nil { - list = []sandbox.Sandbox{} - } - writeJSON(w, list, err) -} - -func (a *agent) stopSandbox(w http.ResponseWriter, r *http.Request) { - writeDone(w, a.Node.Driver().Stop(r.Context(), r.PathValue("id"))) -} - -func (a *agent) sandboxLogs(w http.ResponseWriter, r *http.Request) { - follow := r.URL.Query().Get("follow") == "1" - rc, err := a.Node.Driver().Logs(r.Context(), r.PathValue("id"), follow) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - defer rc.Close() - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.WriteHeader(http.StatusOK) - // Send the header now: a quiet log can take minutes for its first line. - flush := http.NewResponseController(w).Flush - flush() - buf := make([]byte, 32<<10) - for { - n, err := rc.Read(buf) - if n > 0 { - if _, werr := w.Write(buf[:n]); werr != nil { - return - } - flush() - } - if err != nil { - return - } - } -} - -func (a *agent) holdSandbox(w http.ResponseWriter, r *http.Request) { - var req holdRequest - if decode(w, r, &req) { - writeDone(w, a.Node.Driver().Hold(r.Context(), r.PathValue("id"), req.Until)) - } -} - -// sandboxShell upgrades the stream to a shell. See readFrames for the -// stream format. -func (a *agent) sandboxShell(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Upgrade") != shellProtocol { - http.Error(w, "the shell needs Upgrade: "+shellProtocol, http.StatusBadRequest) - return - } - rows, err1 := strconv.ParseUint(r.URL.Query().Get("rows"), 10, 16) - cols, err2 := strconv.ParseUint(r.URL.Query().Get("cols"), 10, 16) - if err := errors.Join(err1, err2); err != nil { - http.Error(w, "bad terminal size: "+err.Error(), http.StatusBadRequest) - return - } - term, err := a.Node.Driver().Shell(r.Context(), r.PathValue("id"), sandbox.TermSize{Rows: uint16(rows), Cols: uint16(cols)}) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - defer term.Close() - conn, brw, err := http.NewResponseController(w).Hijack() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - defer conn.Close() - brw.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: " + shellProtocol + "\r\n\r\n") - if err := brw.Flush(); err != nil { - return - } - go func() { - io.Copy(conn, term) - conn.Close() - }() - if err := readFrames(brw.Reader, term); !errors.Is(err, io.EOF) { - a.Logger.Debug("shell stream ended", "err", err) - } -} - -func (a *agent) readCapacity(w http.ResponseWriter, r *http.Request) { - c, err := a.Node.Driver().Capacity(r.Context()) - writeJSON(w, c, err) -} - -// readHost returns the host facts, and with doctor=1 also the doctor -// checks. -func (a *agent) readHost(w http.ResponseWriter, r *http.Request) { - info, err := a.Node.Facts(r.Context()) - rep := hostReport{Info: info} - if err == nil && r.URL.Query().Get("doctor") == "1" { - rep.Doctor, err = a.Node.Doctor(r.Context()) - } - writeJSON(w, rep, err) -} - -// ensureImage streams the progress as JSON lines. The last line has the -// image or the error. -func (a *agent) ensureImage(w http.ResponseWriter, r *http.Request) { - var ref sandbox.ImageRef - if !decode(w, r, &ref) { - return - } - if ref.Ref == "" { - http.Error(w, "no image ref", http.StatusBadRequest) - return - } - w.Header().Set("Content-Type", "application/x-ndjson") - w.WriteHeader(http.StatusOK) - flush := http.NewResponseController(w).Flush - enc := json.NewEncoder(w) - send := func(m imageMessage) { - enc.Encode(m) - flush() - } - img, err := a.Node.EnsureImage(r.Context(), ref.Ref, func(p image.Progress) { send(imageMessage{Progress: &p}) }) - if err != nil { - send(imageMessage{Error: err.Error()}) - return - } - send(imageMessage{Image: &img}) -} - -func (a *agent) gcImages(w http.ResponseWriter, r *http.Request) { - var req gcRequest - if decode(w, r, &req) { - writeDone(w, a.Node.GCImages(r.Context(), req.Keep)) - } -} - -func (a *agent) pruneRunners(w http.ResponseWriter, r *http.Request) { - var req gcRequest - if decode(w, r, &req) { - writeDone(w, a.Node.PruneRunners(r.Context(), req.Keep)) - } -} - -func (a *agent) mirrorStats(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, a.Mirror(), nil) -} - -func (a *agent) ensureRunner(w http.ResponseWriter, r *http.Request) { - var req runnerRequest - if !decode(w, r, &req) { - return - } - b, err := a.Node.EnsureRunner(r.Context(), req.Version, req.Arch) - writeJSON(w, b, err) -} - -func (a *agent) configure(w http.ResponseWriter, r *http.Request) { - var s sandbox.Settings - if decode(w, r, &s) { - writeDone(w, a.Node.Configure(r.Context(), s)) - } -} - -// update replaces the binary. After the answer is out, Run returns -// ErrUpdated. -func (a *agent) update(w http.ResponseWriter, r *http.Request) { - var req updateRequest - if !decode(w, r, &req) { - return - } - if err := a.Node.Update(r.Context(), req.Version); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusNoContent) - http.NewResponseController(w).Flush() - a.Logger.Info("binary updated, restarting", "version", req.Version) - a.updateOnce.Do(func() { close(a.updated) }) -} - -func decode(w http.ResponseWriter, r *http.Request, v any) bool { - if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBody)).Decode(v); err != nil { - http.Error(w, "bad request body: "+err.Error(), http.StatusBadRequest) - return false - } - return true -} - -func writeDone(w http.ResponseWriter, err error) { - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusNoContent) -} - -func writeJSON(w http.ResponseWriter, v any, err error) { - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(v) -} diff --git a/internal/agent/wire.go b/internal/agent/wire.go index e1ed8b4..58bc015 100644 --- a/internal/agent/wire.go +++ b/internal/agent/wire.go @@ -15,22 +15,41 @@ import ( "github.com/hashicorp/yamux" + "github.com/runplane/runplane/internal/doctor" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/image" "github.com/runplane/runplane/internal/sandbox" ) -// Hello is the first line that an agent sends on a new connection. A new -// agent sends its install token, a known agent its server ID and secret. +// Hello is the first message of an agent on a new connection. A new agent +// sends its install token, a known agent its server ID and secret. type Hello struct { + Token string + ServerID string + Secret string + Version string + Mode sandbox.Mode + Host host.Info + // Addr is the IP address of the agent. The Listener sets it. + Addr string +} + +// The types below are the JSON of the protocol. They are apart from the +// types of the other packages, so that a change in those packages cannot +// change the protocol for agents of an older release. The sandbox types are +// the driver contract, and they go as they are. + +// helloLine is the first line that an agent sends. +type helloLine struct { Token string `json:"token"` ServerID string `json:"server_id"` Secret string `json:"secret"` Version string `json:"version"` Mode sandbox.Mode `json:"mode"` - Host host.Info `json:"host"` - // Addr is the IP address of the agent. The Listener sets it. - Addr string `json:"-"` + Host hostInfo `json:"host"` +} + +func (h helloLine) hello() Hello { + return Hello{Token: h.Token, ServerID: h.ServerID, Secret: h.Secret, Version: h.Version, Mode: h.Mode, Host: h.Host.info()} } // reply is the line that the primary sends back. @@ -41,43 +60,150 @@ type reply struct { Error string `json:"error,omitempty"` } -// hostReport is the body of GET /v1/host. Doctor is set only with doctor=1. -type hostReport struct { - Info host.Info `json:"info"` - Doctor []host.Check `json:"doctor,omitempty"` +// An agent of an earlier release sends more facts in hostInfo, which only its +// doctor read. The primary ignores them. +type hostInfo struct { + Hostname string `json:"hostname"` + OS string `json:"os"` + Kernel string `json:"kernel"` + Arch string `json:"arch"` + VCPU int `json:"vcpu"` + MemoryMB int `json:"memory_mb"` + DiskBytes int64 `json:"disk_bytes"` + FreeDiskBytes int64 `json:"free_disk_bytes"` + DiskKind string `json:"disk_kind"` + KVM bool `json:"kvm"` + DockerVersion string `json:"docker_version"` + SysboxVersion string `json:"sysbox_version"` + FirecrackerVersion string `json:"firecracker_version"` + IPv4 string `json:"ipv4"` + Provider string `json:"provider"` + Region string `json:"region"` } -// imageMessage is one line of the POST /v1/images/ensure stream. The last -// line has Image or Error. -type imageMessage struct { - Progress *image.Progress `json:"progress,omitempty"` - Image *sandbox.ImageRef `json:"image,omitempty"` - Error string `json:"error,omitempty"` +func wireInfo(i host.Info) hostInfo { + return hostInfo{ + Hostname: i.Hostname, OS: i.OS, Kernel: i.Kernel, Arch: i.Arch, + VCPU: i.VCPU, MemoryMB: i.MemoryMB, DiskBytes: i.DiskBytes, FreeDiskBytes: i.FreeDiskBytes, + DiskKind: i.DiskKind, KVM: i.KVM, DockerVersion: i.DockerVersion, SysboxVersion: i.SysboxVersion, + FirecrackerVersion: i.FirecrackerVersion, IPv4: i.IPv4, Provider: i.Provider, Region: i.Region, + } } -// streamItem is one line of the POST /v1/events stream. -type streamItem struct { - Event *sandbox.Event `json:"event,omitempty"` - Stats *host.Stats `json:"stats,omitempty"` +func (i hostInfo) info() host.Info { + return host.Info{ + Hostname: i.Hostname, OS: i.OS, Kernel: i.Kernel, Arch: i.Arch, + Resources: host.Resources{VCPU: i.VCPU, MemoryMB: i.MemoryMB, DiskBytes: i.DiskBytes, FreeDiskBytes: i.FreeDiskBytes}, + DiskKind: i.DiskKind, KVM: i.KVM, DockerVersion: i.DockerVersion, SysboxVersion: i.SysboxVersion, + FirecrackerVersion: i.FirecrackerVersion, IPv4: i.IPv4, Provider: i.Provider, Region: i.Region, + } } -type runnerRequest struct { - Version string `json:"version"` - Arch string `json:"arch"` +type hostCheck struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Detail string `json:"detail"` + Fix string `json:"fix,omitempty"` + Command string `json:"command,omitempty"` +} + +func wireChecks(list []doctor.Check) []hostCheck { + out := make([]hostCheck, len(list)) + for i, c := range list { + out[i] = hostCheck{ID: c.ID, Name: c.Name, Status: string(c.Status), Detail: c.Detail, Fix: c.Fix, Command: c.Command} + } + return out +} + +func checks(list []hostCheck) []doctor.Check { + out := make([]doctor.Check, len(list)) + for i, c := range list { + out[i] = doctor.Check{ID: c.ID, Name: c.Name, Status: doctor.Status(c.Status), Detail: c.Detail, Fix: c.Fix, Command: c.Command} + } + return out } -type holdRequest struct { - Until time.Time `json:"until"` +type hostStats struct { + At time.Time `json:"at"` + CPUPercent float64 `json:"cpu_percent"` + MemoryPercent float64 `json:"memory_percent"` + DiskPercent float64 `json:"disk_percent"` + VCPU int `json:"vcpu"` + MemoryMB int `json:"memory_mb"` + DiskBytes int64 `json:"disk_bytes"` + FreeDiskBytes int64 `json:"free_disk_bytes"` } -type updateRequest struct { +func wireStats(s host.Stats) hostStats { + return hostStats{ + At: s.At, CPUPercent: s.CPUPercent, MemoryPercent: s.MemoryPercent, DiskPercent: s.DiskPercent, + VCPU: s.VCPU, MemoryMB: s.MemoryMB, DiskBytes: s.DiskBytes, FreeDiskBytes: s.FreeDiskBytes, + } +} + +func (s hostStats) stats() host.Stats { + return host.Stats{ + At: s.At, CPUPercent: s.CPUPercent, MemoryPercent: s.MemoryPercent, DiskPercent: s.DiskPercent, + Resources: host.Resources{VCPU: s.VCPU, MemoryMB: s.MemoryMB, DiskBytes: s.DiskBytes, FreeDiskBytes: s.FreeDiskBytes}, + } +} + +// Image is an image that is ready on a server. +type Image struct { + sandbox.ImageRef + // SizeBytes is the space of the image on the server: its root disk in + // VM mode, else its Docker image. An agent of an earlier release sends + // no size. + SizeBytes int64 `json:"size_bytes"` +} + +// Progress is one step of EnsureImage. +type Progress struct { + Status ImageStatus `json:"status"` + Done int64 `json:"done"` // bytes, 0 when unknown + Total int64 `json:"total"` // bytes, 0 when unknown + Error string `json:"error,omitempty"` +} + +// ImageStatus is the step of EnsureImage that a Progress tells about. +type ImageStatus string + +const ( + ImagePending ImageStatus = "pending" + ImagePulling ImageStatus = "pulling" + ImageBuilding ImageStatus = "building" + ImageConverting ImageStatus = "converting" + ImageReady ImageStatus = "ready" + ImageFailed ImageStatus = "failed" +) + +// Bundle is a runner bundle that is ready on a server. +type Bundle struct { Version string `json:"version"` + Arch string `json:"arch"` + SHA256 string `json:"sha256"` // of the release archive + // Patched reports if the runner keeps ACTIONS_RESULTS_URL from its .env + // file, so that jobs use the runplane cache server. + Patched bool `json:"patched"` +} + +// MirrorStats counts the use of the registry mirror of a server since the +// start of runplane on the server. +type MirrorStats struct { + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitBytes int64 `json:"hit_bytes"` // bytes sent to clients from the cache + MissBytes int64 `json:"miss_bytes"` // bytes fetched from upstream + SizeBytes int64 `json:"size_bytes"` // bytes in the cache now } -// gcRequest is the body of POST /v1/images/gc: the digests of the images -// that the server keeps. -type gcRequest struct { - Keep []string `json:"keep"` +func (m *MirrorStats) Add(o MirrorStats) { + m.Hits += o.Hits + m.Misses += o.Misses + m.HitBytes += o.HitBytes + m.MissBytes += o.MissBytes + m.SizeBytes += o.SizeBytes } const ( @@ -99,7 +225,6 @@ func yamuxConfig(log *slog.Logger) *yamux.Config { return cfg } -// streamClient sends each HTTP request on a stream of sess. func streamClient(sess *yamux.Session) *http.Client { return &http.Client{Transport: &http.Transport{ DialContext: func(context.Context, string, string) (net.Conn, error) { return sess.Open() }, @@ -150,7 +275,6 @@ func writeJSONLine(w io.Writer, v any) error { return err } -// responseError reads the error text of a failed response. func responseError(resp *http.Response) error { msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) if text := strings.TrimSpace(string(msg)); text != "" { diff --git a/internal/api/admin_test.go b/internal/api/admin_test.go index 7649f41..13d7f46 100644 --- a/internal/api/admin_test.go +++ b/internal/api/admin_test.go @@ -2,11 +2,16 @@ package api import ( "context" - "errors" + "net/http" + "net/http/httptest" + "slices" "strings" + "sync" "testing" "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/store" + "github.com/runplane/runplane/internal/update" ) func TestNewInviteLinkReplacesTheOldOne(t *testing.T) { @@ -38,31 +43,35 @@ func TestNewInviteLinkReplacesTheOldOne(t *testing.T) { b.do("POST", "/api/invites/"+first.Invite.ID+"/link", nil, nil).want(t, 404) } -// fakeNotifier records the tests and fails for URLs that contain "broken". -type fakeNotifier struct{ sent []string } - -func (f *fakeNotifier) Test(_ context.Context, _, kind, url string) error { - f.sent = append(f.sent, kind+" "+url) - if strings.Contains(url, "broken") { - return errors.New("receiver answered 404 Not Found") - } - return nil -} - +// A test message goes to the receiver of the channel, and the delivery of +// a saved channel shows in the list. func TestNotificationTests(t *testing.T) { - n := &fakeNotifier{} - e := newEnv(t, func(d *Deps) { d.Notifier = n }) + var mu sync.Mutex + var paths []string + receiver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Path) + mu.Unlock() + if strings.Contains(r.URL.Path, "broken") { + http.Error(w, "no such hook", http.StatusNotFound) + } + })) + defer receiver.Close() + e := newEnv(t) b := e.login("ada@example.com", auth.RoleAdmin) b.do("POST", "/api/notifications/test", map[string]string{"kind": "slack", "url": "http://hooks.slack.com/x"}, nil).want(t, 422) - b.do("POST", "/api/notifications/test", map[string]string{"kind": "webhook", "url": "http://10.0.0.2/hook"}, nil).want(t, 204) - if len(n.sent) != 1 || n.sent[0] != "webhook http://10.0.0.2/hook" { - t.Fatalf("sent %q, want one test to the unsaved webhook", n.sent) + b.do("POST", "/api/notifications/test", map[string]string{"kind": "webhook", "url": receiver.URL + "/hook"}, nil).want(t, 204) + mu.Lock() + sent := slices.Clone(paths) + mu.Unlock() + if !slices.Equal(sent, []string{"/hook"}) { + t.Fatalf("receiver got %q, want one test to the unsaved webhook", sent) } var c NotificationChannel b.do("POST", "/api/notifications", map[string]any{ - "name": "ops", "kind": "webhook", "url": "https://example.com/broken", "events": []string{"job.failed"}, + "name": "ops", "kind": "webhook", "url": receiver.URL + "/broken", "events": []string{"job.failed"}, }, &c).want(t, 200) if c.LastSentAt != nil { t.Fatalf("new channel has last_sent_at %v", c.LastSentAt) @@ -74,53 +83,60 @@ func TestNotificationTests(t *testing.T) { t.Fatalf("after a failed test: last_sent_at %v, last_error %q", list[0].LastSentAt, list[0].LastError) } - b.do("PATCH", "/api/notifications/"+c.ID, map[string]any{"url": "https://example.com/fixed"}, nil).want(t, 200) + b.do("PATCH", "/api/notifications/"+c.ID, map[string]any{"url": receiver.URL + "/fixed"}, nil).want(t, 200) b.do("POST", "/api/notifications/"+c.ID+"/test", nil, nil).want(t, 204) b.do("GET", "/api/notifications", nil, &list).want(t, 200) if list[0].LastError != "" { t.Fatalf("after a good test: last_error %q, want none", list[0].LastError) } + b.do("POST", "/api/notifications/nope/test", nil, nil).want(t, 404) } -// fakeUpdater blocks Apply until release closes. +// fakeUpdater is an updater in a state, whose Apply returns err. type fakeUpdater struct { - applying chan struct{} - release chan struct{} + state update.State + err error } -func (f *fakeUpdater) Check(context.Context) (UpdateStatus, error) { - return UpdateStatus{Current: "v1.0.0", Latest: "v1.1.0", Available: true}, nil +func (f fakeUpdater) Check(context.Context) (update.Status, error) { + return update.Status{Current: "v1.0.0", Latest: "v1.1.0", Available: true}, nil } -func (f *fakeUpdater) Apply(context.Context) error { - close(f.applying) - <-f.release - return nil +func (f fakeUpdater) State() update.State { return f.state } + +func (f fakeUpdater) Apply(context.Context) (update.Status, error) { + return update.Status{Current: "v1.0.0", Latest: "v1.1.0", Available: true}, f.err } +// The dashboard shows the state of the updater, also of an auto update. A +// second update while one runs is a conflict, and the audit log has only +// the update that started. func TestUpdateState(t *testing.T) { - u := &fakeUpdater{applying: make(chan struct{}), release: make(chan struct{})} - e := newEnv(t, func(d *Deps) { d.Updater = u }) - b := e.login("ada@example.com", auth.RoleAdmin) - other := e.login("eve@example.com", auth.RoleAdmin) - - var st UpdateStatus - b.do("GET", "/api/update", nil, &st).want(t, 200) - if st.State != "idle" { - t.Fatalf("state before the update = %q, want idle", st.State) - } - done := make(chan response) - go func() { done <- b.do("POST", "/api/update", nil, nil) }() - recv(t, u.applying) - other.do("GET", "/api/update", nil, &st).want(t, 200) - if st.State != "applying" { - t.Fatalf("state during the update = %q, want applying", st.State) - } - other.do("POST", "/api/update", nil, nil).want(t, 409) - close(u.release) - recv(t, done).want(t, 204) - other.do("GET", "/api/update", nil, &st).want(t, 200) - if st.State != "restarting" { - t.Fatalf("state after the update = %q, want restarting", st.State) + for _, tt := range []struct { + state update.State + err error + status int + }{ + {update.Idle, nil, 204}, + {update.Applying, update.ErrBusy, 409}, + {update.Restarting, update.ErrBusy, 409}, + {update.Idle, update.ErrNoUpdate, 409}, + } { + e := newEnv(t, func(d *Deps) { d.Updater = fakeUpdater{state: tt.state, err: tt.err} }) + b := e.login("ada@example.com", auth.RoleAdmin) + var st UpdateStatus + b.do("GET", "/api/update", nil, &st).want(t, 200) + if st.State != string(tt.state) || st.Latest != "v1.1.0" { + t.Errorf("state %q, latest %q; want %s and v1.1.0", st.State, st.Latest, tt.state) + } + if res := b.do("POST", "/api/update", nil, nil); res.StatusCode != tt.status { + t.Errorf("update with %v: status %d, want %d", tt.err, res.StatusCode, tt.status) + } + entries, _, err := e.st.ListAudit(context.Background(), "", 10) + must(t, err) + recorded := slices.ContainsFunc(entries, func(a store.AuditEntry) bool { return a.Action == "update.apply" }) + if recorded != (tt.err == nil) { + t.Errorf("update with %v: update.apply entry %v, want %v", tt.err, recorded, tt.err == nil) + } } } diff --git a/internal/api/advice.go b/internal/api/advice.go index 61a08a1..6daed30 100644 --- a/internal/api/advice.go +++ b/internal/api/advice.go @@ -17,6 +17,12 @@ type SizeAdvice struct { Reason string `json:"reason"` } +const ( + adviceLarger = "larger" + adviceSmaller = "smaller" + adviceNone = "none" +) + const ( adviceRuns = 10 // the last completed runs that count adviceMinRuns = 3 // fewer runs give no advice @@ -37,11 +43,11 @@ const ( // than pool and keeps 30% headroom on the two peaks. func sizeAdvice(runs []store.RunUsage, pool *store.Pool, pools []store.Pool) SizeAdvice { if pool == nil { - return SizeAdvice{Kind: "none", Reason: "The pool of this job no longer exists."} + return SizeAdvice{Kind: adviceNone, Reason: "The pool of this job no longer exists."} } runs = slices.DeleteFunc(slices.Clone(runs), func(r store.RunUsage) bool { return r.VCPU <= 0 || r.MemoryMB <= 0 }) if len(runs) < adviceMinRuns { - return SizeAdvice{Kind: "none", Reason: fmt.Sprintf( + return SizeAdvice{Kind: adviceNone, Reason: fmt.Sprintf( "Size advice needs %d completed runs with CPU and memory samples. This job has %d.", adviceMinRuns, len(runs))} } var cpuPeaks, cpuAvgs, mems []float64 @@ -67,7 +73,7 @@ func sizeAdvice(runs []store.RunUsage, pool *store.Pool, pools []store.Pool) Siz return p.VCPU >= pool.VCPU && p.MemoryMB >= pool.MemoryMB && (!memShort || mem <= headroom*float64(p.MemoryMB)) && (!cpuShort || cpuAvg <= headroom*float64(p.VCPU)) }) - return SizeAdvice{Kind: "larger", Label: label, Reason: strings.Join(why, " and ") + of + noPool(label, pool)} + return SizeAdvice{Kind: adviceLarger, Label: label, Reason: strings.Join(why, " and ") + of + noPool(label, pool)} } if cpuPeak < smallerShare*vcpu && mem < smallerShare*memory { label := smallest(pools, pool, func(p store.Pool) bool { @@ -75,9 +81,9 @@ func sizeAdvice(runs []store.RunUsage, pool *store.Pool, pools []store.Pool) Siz cpuPeak <= headroom*float64(p.VCPU) && mem <= headroom*float64(p.MemoryMB) }) why := fmt.Sprintf("CPU use peaked at %.0f%% of %d vCPUs and memory at %s of %s", 100*cpuPeak/vcpu, pool.VCPU, gb(mem), gb(memory)) - return SizeAdvice{Kind: "smaller", Label: label, Reason: why + of + noPool(label, pool)} + return SizeAdvice{Kind: adviceSmaller, Label: label, Reason: why + of + noPool(label, pool)} } - return SizeAdvice{Kind: "none", Reason: "The CPU and memory use fit the pool" + of} + return SizeAdvice{Kind: adviceNone, Reason: "The CPU and memory use fit the pool" + of} } // p90 is the 90th percentile with the nearest-rank method. diff --git a/internal/api/api.go b/internal/api/api.go index 3ccada2..166fdc2 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -10,19 +10,30 @@ import ( "io" "log/slog" "net/http" + "os" + "reflect" "strings" "sync" - "time" "github.com/danielgtaylor/huma/v2" "github.com/danielgtaylor/huma/v2/adapters/humago" + "github.com/runplane/runplane/internal/agent" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/auth" "github.com/runplane/runplane/internal/config" "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/field" + "github.com/runplane/runplane/internal/fleet" + "github.com/runplane/runplane/internal/ghapp" + "github.com/runplane/runplane/internal/notify" + "github.com/runplane/runplane/internal/pools" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/secret" + "github.com/runplane/runplane/internal/settings" "github.com/runplane/runplane/internal/store" + "github.com/runplane/runplane/internal/tlsutil" + "github.com/runplane/runplane/internal/update" "github.com/runplane/runplane/internal/version" "github.com/runplane/runplane/internal/webui" ) @@ -30,6 +41,7 @@ import ( // Handler serves the HTTP requests of the primary. type Handler struct { d Deps + audit audit.Log api huma.API handler http.Handler @@ -37,39 +49,39 @@ type Handler struct { done chan struct{} closeOnce sync.Once - started time.Time // the start of this process, near enough - - updateMu sync.Mutex - updating string // state of an update that POST /api/update applies; empty when idle + // streams are the bus events that the browser gets, by their Go type. + streams map[reflect.Type]stream } -// Deps are the parts that the API uses. Store, Auth, Bus, Box, and Config are -// required. Each service is optional: without it, its routes answer 503. +// Deps are the parts that the API uses. All are required, except Log. // -// A service error is shown to the user as the detail of a 500 problem, so -// services should return errors that make sense to an admin. A service can -// also return a huma.StatusError to choose the status. +// A service error that a handler gives to serviceErr is shown to the user as +// the detail of a 500 problem, so services should return errors that make +// sense to an admin. The problems of a change (field.Problems) are a 422 +// with one error at each field. type Deps struct { - Store *store.Store - Auth *auth.Service - Bus *events.Bus - Box *secret.Box - Log *slog.Logger - Config config.Config + Store *store.Store + Auth *auth.Service + Settings *settings.Service + Pools *pools.Service + Channels *notify.Channels + Bus *events.Bus + Box *secret.Box + Log *slog.Logger + Config config.Config Scheduler Scheduler GitHub GitHub Servers Servers Images Images Caches Caches + Mirrors Mirrors Backups Backups Updater Updater - Notifier Notifier Logs Logs Shell Shell HostChecker HostChecker InstallScript InstallScript - SettingsHook SettingsHook HTTPS HTTPS } @@ -79,7 +91,7 @@ func New(d Deps) *Handler { if d.Log == nil { d.Log = slog.New(slog.DiscardHandler) } - h := &Handler{d: d, done: make(chan struct{}), started: time.Now().UTC()} + h := &Handler{d: d, audit: audit.Log{Store: d.Store, Logger: d.Log}, done: make(chan struct{})} mux := http.NewServeMux() h.api = humago.New(mux, humaConfig()) h.api.UseMiddleware(h.authorize) @@ -110,104 +122,129 @@ func OpenAPI() ([]byte, error) { } type Scheduler interface { - // PoolsChanged tells the scheduler that a pool was made, changed, or deleted. - PoolsChanged(ctx context.Context) StopSandbox(ctx context.Context, id string) error // RerunJob asks GitHub to run the job again. With debug, the new attempt // keeps its sandbox after the job for a debug shell. RerunJob(ctx context.Context, jobID string, debug bool) error } -// GitHub has the GitHub App flows. base is the browser URL of runplane, for -// example "https://ci.example.com", without a trailing slash. +// GitHub has the GitHub App flows. type GitHub interface { // Manifest returns the GitHub URL that the browser posts the form to and - // the manifest JSON for the form field "manifest". - Manifest(ctx context.Context, base, accountType, organization string) (actionURL, manifest string, err error) + // the manifest JSON for the form field "manifest". base is the browser + // URL of runplane, for example "https://ci.example.com", without a + // trailing slash. + Manifest(ctx context.Context, base string, owner ghapp.Account) (actionURL, manifest string, err error) // Callback converts the manifest code into the app and returns the URL // for the browser, normally the app installation page on GitHub. - Callback(ctx context.Context, base, code, state string) (redirect string, err error) + Callback(ctx context.Context, code, state string) (redirect string, err error) // Setup records an installation, syncs its repositories, and makes the // scale sets. It returns the URL for the browser. installationID is 0 // when setupAction is "request". - Setup(ctx context.Context, base string, installationID int64, setupAction string) (redirect string, err error) + Setup(ctx context.Context, installationID int64, setupAction string) (redirect string, err error) // Sync reads the installations, repositories, and suspensions now. Sync(ctx context.Context) error // Disconnect stops the listeners, deletes the scale sets, and removes the app. Disconnect(ctx context.Context) error // OpenPullRequest opens a pull request in the repository (owner/name) - // that sets runs-on to label in each job on a GitHub-hosted Ubuntu - // runner. While the pull request of an earlier call is open, it returns - // that one. + // that sets runs-on to label in each job on one of ghapp.HostedLabels. + // While the pull request of an earlier call is open, it returns that + // one. It returns ghapp.ErrNoPullPermissions when the account has not + // accepted the permissions, and field.Problems at "repository" or + // "label" when the jobs of the pull request would not start. OpenPullRequest(ctx context.Context, base, repository, label string) (number int, url string, err error) } +// Servers is the fleet of servers: *fleet.Fleet. type Servers interface { // NewToken makes a one-time install token for a new agent and the - // command that installs the agent with it. An empty name means the host - // name of the agent. - NewToken(ctx context.Context, base, name, createdBy string) (ServerToken, error) + // command that installs the agent with it. + NewToken(ctx context.Context, by audit.Actor, base, name string) (fleet.Token, error) // InstallSSH starts an install over SSH and returns the ID of the new // server. The install continues in the background; the server status - // and status detail show its progress. - InstallSSH(ctx context.Context, base string, req SSHInstall) (serverID string, err error) - // SSHPublicKey returns the runplane SSH key in authorized_keys format. + // and status detail show its progress. A name or a host that is not + // valid gives field.Problems. + InstallSSH(ctx context.Context, by audit.Actor, base string, req fleet.SSHInstall) (serverID string, err error) SSHPublicKey(ctx context.Context) (string, error) - Doctor(ctx context.Context, serverID string) ([]Check, error) - // Remove stops the sandboxes of an agent and disconnects it. The API - // then deletes the server row. - Remove(ctx context.Context, serverID string) error - // UpdateState tells whether the server runs the release of the - // primary: one of the Update* values. - UpdateState(v store.Server) string + // Doctor runs the doctor on a server and keeps the result. It returns + // fleet.ErrNotConnected for a server without a connection. + Doctor(ctx context.Context, serverID string) (store.DoctorResult, error) + // Limits returns the limits of the driver of a connected server. + Limits(serverID string) (sandbox.Limits, bool) + // UpdateServer renames a server or changes its status. It returns + // fleet.ErrInstalling or fleet.ErrNotPaused for a status that the + // server cannot get now. + UpdateServer(ctx context.Context, by audit.Actor, serverID string, c fleet.ServerChange) (store.Server, error) + // RemoveServer stops the sandboxes of an agent, disconnects it, and + // deletes its row. It returns fleet.ErrLocalServer for the primary. + RemoveServer(ctx context.Context, by audit.Actor, serverID string) error + UpdateState(v store.Server) fleet.UpdateState + Actions(v store.Server) fleet.Actions // UpdateAgent makes a connected agent install the release of the // primary. The update runs in the background, and UpdateState shows it. - UpdateAgent(ctx context.Context, serverID string) error + UpdateAgent(ctx context.Context, by audit.Actor, serverID string) error } type Images interface { - // Refresh pulls the image again on all servers. - Refresh(ctx context.Context, imageID string) error - // Remove deletes the files of the image on all servers. The API then - // deletes the image row. - Remove(ctx context.Context, imageID string) error + CreateImage(ctx context.Context, by audit.Actor, name, ref string) (store.Image, error) + UpdateImage(ctx context.Context, by audit.Actor, imageID string, c fleet.ImageChange) (store.Image, error) + // RemoveImage stops the use of an image and deletes its row. The pools + // with the image change to the default image. The connected servers + // remove the files of the image in the background, after the call + // returns. It returns fleet.ErrDefaultImage for the default image. + RemoveImage(ctx context.Context, by audit.Actor, imageID string) error + // RefreshImage pulls the image again on all servers. + RefreshImage(ctx context.Context, by audit.Actor, imageID string) error } type Caches interface { - // Delete removes the cache entry id, or all entries of repository if id - // is empty, with their blobs. It returns the count of removed entries. - Delete(ctx context.Context, repository, id string) (int, error) - // MirrorStats returns the counts of the mirror. The API sets StartedAt. - MirrorStats(ctx context.Context) (MirrorStats, error) - // MaxBytes returns the limit that the cache server uses now: the - // cache_max_gb setting, at most a quarter of the disk of the cache. + // Delete removes the cache entry id with its blob. It returns + // store.ErrNotFound if no entry has the ID. + Delete(ctx context.Context, id string) error + // DeleteRepository removes the entries of a repository with their blobs, + // and returns the count. + DeleteRepository(ctx context.Context, repository string) (int, error) + // MaxBytes returns the byte limit that the cache server uses now. MaxBytes() int64 } -type Backups interface { - Run(ctx context.Context) (backupID string, err error) -} - -type Updater interface { - // Check reads the latest release. It sets State while an update that - // did not come through the API (auto update) runs. - Check(ctx context.Context) (UpdateStatus, error) - // Apply installs the latest release. The process restarts afterwards. - Apply(ctx context.Context) error +type Mirrors interface { + // MirrorStats sums the counts of the registry mirrors of all servers. + MirrorStats(ctx context.Context) agent.MirrorStats } -type Notifier interface { - Test(ctx context.Context, base, kind, url string) error +type Backups interface { + // Create makes a backup now. + Create(ctx context.Context) (backupID string, err error) + // Available reports if the file of b is on this server. + Available(b store.Backup) bool + // Open opens the file of the backup id. It returns store.ErrNotFound + // for an unknown ID, and backup.ErrNoFile when the file is not on this + // server. + Open(ctx context.Context, id string) (store.Backup, *os.File, error) } +type Updater interface { + // Check returns the latest release. + Check(ctx context.Context) (update.Status, error) + // State tells if an update of this process runs, also an auto update. + State() update.State + // Apply installs the latest release, and the process restarts. It + // returns update.ErrNoUpdate when no newer release exists, and + // update.ErrBusy while an update runs. + Apply(ctx context.Context) (update.Status, error) +} + +// Logs gives the logs of the sandboxes and jobs. Its errors are the errors +// of joblog. type Logs interface { - // SandboxLog returns the console and runner log of a sandbox. With - // follow, the reader stays open until the sandbox stops or ctx ends. - SandboxLog(ctx context.Context, id string, follow bool) (io.ReadCloser, error) - // JobLog returns the log of a job: the stored GitHub log after the job, + // Sandbox returns the console and runner log of a sandbox. With follow, + // the reader stays open until the sandbox stops or ctx ends. + Sandbox(ctx context.Context, id string, follow bool) (io.ReadCloser, error) + // Job returns the log of a job: the stored GitHub log after the job, // the sandbox log while it runs. step 0 means all steps. The sandbox log - // has no step marks, so it always has all steps: live is true for it. - JobLog(ctx context.Context, jobID string, step int, follow bool) (log io.ReadCloser, live bool, err error) + // has no step marks, so it always has all steps. + Job(ctx context.Context, jobID string, step int, follow bool) (io.ReadCloser, error) } type Shell interface { @@ -216,25 +253,20 @@ type Shell interface { } type HostChecker interface { - // Check reads this server and runs the doctor checks for the setup page. - Check(ctx context.Context) (HostCheck, error) + // Check runs the doctor on this server for the setup page, and keeps + // the result. + Check(ctx context.Context) (fleet.HostCheck, error) } type InstallScript interface { - // Script returns install.sh with the primary URL and the agent - // certificate fingerprint filled in for the request. - Script(r *http.Request) []byte -} - -type SettingsHook interface { - // SettingsChanged runs after an admin changes the settings. - SettingsChanged(ctx context.Context, old, new store.Settings) + // Script returns install.sh with the values of the primary, for the + // host[:port] that the request named. + Script(host string) []byte } -// HTTPS reports the ACME certificate of the domain setting. Without it, the -// settings have no https field. +// HTTPS reports the ACME certificate of the domain setting. type HTTPS interface { - CertificateStatus(ctx context.Context) HTTPSStatus + Status(ctx context.Context) tlsutil.CertStatus } // Handlers send [] for empty lists, never null. @@ -279,7 +311,7 @@ func (h *Handler) routes() { handle(h, public, "GET /api/github/setup", "github-setup", "Record an installation (browser redirect from GitHub)", h.githubSetup, redirectResponse) handle(h, auth.ActionAdmin, "POST /api/github/sync", "sync-github", "Sync the installations now", h.syncGitHub) handle(h, auth.ActionAdmin, "POST /api/github/pull-requests", "create-github-pull-request", "Open a pull request that moves jobs to a runplane label", h.createPullRequest, - describe("The pull request sets runs-on to the label for each job on ubuntu-latest or ubuntu-24.04. While an earlier pull request of runplane is open in the repository, the response gives that one.")) + describe("The pull request sets runs-on to the label for each job on "+ghapp.HostedLabelsText("")+". While an earlier pull request of runplane is open in the repository, the response gives that one.")) handle(h, auth.ActionAdmin, "DELETE /api/github", "disconnect-github", "Disconnect GitHub", h.disconnectGitHub) handle(h, auth.ActionRead, "GET /api/pools", "list-pools", "List the pools with live sandbox counts", h.listPools) @@ -298,7 +330,7 @@ func (h *Handler) routes() { handle(h, auth.ActionRead, "GET /api/jobs", "list-jobs", "List jobs, newest first", h.listJobs) handle(h, auth.ActionRead, "GET /api/jobs/facets", "get-job-facets", "List the values for the job filters", h.getJobFacets) handle(h, auth.ActionRead, "GET /api/jobs/{id}", "get-job", "Read a job", h.getJob) - handle(h, auth.ActionRead, "GET /api/jobs/{id}/log", "get-job-log", "Read the log of a job", h.jobLog, jobLogResponse) + handle(h, auth.ActionRead, "GET /api/jobs/{id}/log", "get-job-log", "Read the log of a job", h.jobLog, textResponse) handle(h, auth.ActionRead, "GET /api/jobs/{id}/metrics", "get-job-metrics", "Read the CPU and memory samples of a job", h.getJobMetrics) handle(h, auth.ActionRerunJob, "POST /api/jobs/{id}/rerun", "rerun-job", "Run a job again", h.rerunJob) @@ -360,21 +392,44 @@ const public = auth.Action(-1) const actionKey = "runplane.action" -// handle registers a Huma operation. pattern is "METHOD /path". An error that -// is not a huma.StatusError goes to the log, and the client gets a 500 -// problem without its text. +// handle registers a Huma operation. pattern is "METHOD /path". problem +// changes the errors of fn into problems. func handle[I, O any](h *Handler, a auth.Action, pattern, id, summary string, fn func(context.Context, *I) (*O, error), opts ...func(*huma.Operation)) { huma.Register(h.api, operation(a, pattern, id, summary, opts...), func(ctx context.Context, in *I) (*O, error) { out, err := fn(ctx, in) - var se huma.StatusError - if err != nil && !errors.As(err, &se) { - h.d.Log.Error("request failed", "err", err) - return nil, huma.Error500InternalServerError("Something went wrong. The server log has the details.") + if err != nil { + return nil, h.problem(err) } - return out, err + return out, nil }) } +// problem changes the error of a handler into a problem: field.Problems +// into a 422 with one error at each field, and store.ErrNotFound into a 404. +// An error from serviceErr is a 500 with its text. Each other error goes to +// the log, and the client gets a 500 without its text, so that SQL and +// paths do not go into responses. +func (h *Handler) problem(err error) error { + var se huma.StatusError + var p field.Problems + var shown shownError + switch { + case errors.As(err, &se): + return se + case errors.As(err, &p): + return unprocessable(p) + case errors.Is(err, store.ErrNotFound): + return huma.Error404NotFound(field.Sentence(err)) + case errors.Is(err, context.Canceled): + return huma.Error503ServiceUnavailable("The request was cancelled.") + case errors.As(err, &shown): + h.d.Log.Error("service failed", "err", err) + return huma.Error500InternalServerError(err.Error()) + } + h.d.Log.Error("request failed", "err", err) + return huma.Error500InternalServerError("Something went wrong. The server log has the details.") +} + func operation(a auth.Action, pattern, id, summary string, opts ...func(*huma.Operation)) huma.Operation { method, path, _ := strings.Cut(pattern, " ") tag, _, _ := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(path, "/api"), "/"), "/") @@ -406,14 +461,6 @@ func textResponse(op *huma.Operation) { }} } -func jobLogResponse(op *huma.Operation) { - textResponse(op) - op.Responses["200"].Headers = map[string]*huma.Param{stepFilterHeader: { - Description: "ignored: the log has all steps, because the job runs and its live log has no step marks.", - Schema: &huma.Schema{Type: huma.TypeString, Enum: []any{"ignored"}}, - }} -} - func binaryResponse(contentType string) func(*huma.Operation) { return func(op *huma.Operation) { op.Responses = map[string]*huma.Response{"200": { @@ -443,13 +490,29 @@ type PathID struct { ID string `path:"id" maxLength:"100"` } +// patchOf is the TransformSchema of a patch request: each key gets the +// schema of the key of the create request C with the same name, so that the +// two requests have the same limits. The patch keeps its own docs. +func patchOf[C any](r huma.Registry, s *huma.Schema) *huma.Schema { + create := r.Schema(reflect.TypeFor[C](), false, "") + for key, own := range s.Properties { + c, ok := create.Properties[key] + if !ok { + continue + } + p := *c + p.Description, p.Nullable = own.Description, own.Nullable + s.Properties[key] = &p + } + return s +} + // body is the input or output of an operation with a JSON body. type body[T any] struct{ Body T } // empty is the output of operations without a response body (204). type empty struct{} -// reply returns v as the response body, or err. func reply[T any](v T, err error) (*body[T], error) { if err != nil { return nil, err @@ -474,7 +537,6 @@ func orEmpty[T any](list []T) []T { return list } -// set copies *v to dst if v is not nil. func set[T any](dst *T, v *T) { if v != nil { *dst = *v @@ -515,23 +577,28 @@ func notFound(err error, msg string) error { return err } -// serviceErr shows the error of a service to the user. -func (h *Handler) serviceErr(err error) error { - var se huma.StatusError - switch { - case errors.As(err, &se): - return se - case errors.Is(err, store.ErrNotFound): - return huma.Error404NotFound(err.Error()) - case errors.Is(err, context.Canceled): - return huma.Error503ServiceUnavailable("The request was cancelled.") +// serviceErr marks the error of a service, so that the user sees its text. +func serviceErr(err error) error { + if err == nil { + return nil } - h.d.Log.Error("service failed", "err", err) - return huma.Error500InternalServerError(err.Error()) + return shownError{err} } -func unavailable(what string) error { - return huma.Error503ServiceUnavailable(what + " is not available in this runplane process.") +type shownError struct{ error } + +func (e shownError) Unwrap() error { return e.error } + +func unprocessable(p field.Problems) error { + details := make([]error, len(p)) + for i, x := range p { + details[i] = &huma.ErrorDetail{Location: "body." + x.Key, Message: x.Message, Value: x.Value} + } + msg := "Some values are not valid." + if len(p) == 1 { + msg = p[0].Message + } + return huma.Error422UnprocessableEntity(msg, details...) } // invalid returns a 422 problem with one field error. loc is for example "body.label". @@ -539,24 +606,6 @@ func invalid(loc, msg string, value any) error { return huma.Error422UnprocessableEntity(msg, &huma.ErrorDetail{Location: loc, Message: msg, Value: value}) } -// conflict returns a 409 problem with one field error. func conflict(loc, msg string) error { return huma.Error409Conflict(msg, &huma.ErrorDetail{Location: loc, Message: msg}) } - -// problems collects field errors for one 422 problem. -type problems []error - -func (p *problems) add(loc, msg string, value any) { - *p = append(*p, &huma.ErrorDetail{Location: loc, Message: msg, Value: value}) -} - -func (p problems) err() error { - if len(p) == 0 { - return nil - } - if len(p) == 1 { - return huma.Error422UnprocessableEntity(p[0].(*huma.ErrorDetail).Message, p...) - } - return huma.Error422UnprocessableEntity("Some values are not valid.", p...) -} diff --git a/internal/api/api_test.go b/internal/api/api_test.go index a0e6132..6f94147 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -18,12 +18,18 @@ import ( "time" "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/backup" "github.com/runplane/runplane/internal/config" "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/fleet" "github.com/runplane/runplane/internal/ids" + "github.com/runplane/runplane/internal/notify" + "github.com/runplane/runplane/internal/pools" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/secret" + "github.com/runplane/runplane/internal/settings" "github.com/runplane/runplane/internal/store" + "github.com/runplane/runplane/internal/tlsutil" ) type env struct { @@ -33,7 +39,9 @@ type env struct { d Deps } -// newEnv starts the API on a new database. with sets services in Deps. +// newEnv starts the API on a new database. with sets services in Deps. The +// other services are fakes that do nothing, or that panic in a method that +// the test does not expect. func newEnv(t *testing.T, with ...func(*Deps)) *env { t.Helper() dir := t.TempDir() @@ -42,13 +50,24 @@ func newEnv(t *testing.T, with ...func(*Deps)) *env { t.Cleanup(func() { st.Close() }) box, err := secret.NewBox(bytes.Repeat([]byte{7}, secret.KeySize)) must(t, err) + cfg := config.Default() + cfg.DataDir, cfg.ConfigDir, cfg.Mode = dir, dir, sandbox.ModeContainer + bus := &events.Bus{} d := Deps{ - Store: st, Auth: auth.NewService(st, []byte("device key")), Bus: &events.Bus{}, Box: box, - Config: config.Config{DataDir: dir, ConfigDir: dir, Mode: sandbox.ModeContainer}, + Store: st, Auth: auth.NewService(st, []byte("device key"), nil), Channels: notify.NewChannels(st, box, nil), + Bus: bus, Box: box, Config: cfg, + Scheduler: okScheduler{}, GitHub: struct{ GitHub }{}, Servers: &fakeServers{}, Images: struct{ Images }{}, + Caches: fakeCaches{}, Mirrors: fakeMirrors{}, Backups: backup.New(backup.Config{Store: st, Box: box, Bus: bus, Dir: cfg.BackupsDir()}), + Updater: fakeUpdater{}, Logs: fakeLogs{}, Shell: struct{ Shell }{}, HostChecker: fakeHostChecker{}, + InstallScript: struct{ InstallScript }{}, HTTPS: fakeHTTPS{}, } for _, f := range with { f(&d) } + d.Settings = settings.New(settings.Config{ + Store: st, Bus: d.Bus, Mode: d.Config.Mode, DevInsecure: d.Config.DevInsecure, KVM: func() bool { return false }, + }) + d.Pools = pools.New(pools.Config{Store: st, Bus: d.Bus, Arch: "x64"}) h := New(d) srv := httptest.NewServer(h) t.Cleanup(func() { @@ -58,6 +77,16 @@ func newEnv(t *testing.T, with ...func(*Deps)) *env { return &env{t: t, srv: srv, st: st, d: d} } +type fakeHostChecker struct{} + +func (fakeHostChecker) Check(context.Context) (fleet.HostCheck, error) { + return fleet.HostCheck{Mode: sandbox.ModeContainer}, nil +} + +type fakeHTTPS struct{} + +func (fakeHTTPS) Status(context.Context) tlsutil.CertStatus { return tlsutil.CertStatus{State: "none"} } + func must(t *testing.T, err error) { t.Helper() if err != nil { @@ -219,13 +248,12 @@ func (e *env) login(email, role string) *browser { return b } -// addJob records a queued job of repo. func (e *env) addJob(requestID int64, repo string, queued time.Time) store.Job { e.t.Helper() j, err := e.st.UpsertJob(context.Background(), store.JobMessage{ RunnerRequestID: requestID, State: store.JobQueued, Repository: repo, JobName: fmt.Sprint("job ", requestID), WorkflowRef: repo + "/.github/workflows/ci.yml@refs/heads/main", QueuedAt: queued, - }) + }, time.Now()) must(e.t, err) return j } @@ -269,7 +297,9 @@ func TestSetupMakesAdminWithSession(t *testing.T) { t.Fatalf("setup user = %+v, want admin ada@example.com", u) } cookie := res.Header.Get("Set-Cookie") - for _, part := range []string{"runplane_session=", "Path=/", "Max-Age=2592000", "HttpOnly", "SameSite=Lax"} { + // The cookie lives as long as the session that it holds. + maxAge := fmt.Sprintf("Max-Age=%d", int(auth.SessionTTL.Seconds())) + for _, part := range []string{"runplane_session=", "Path=/", maxAge, "HttpOnly", "SameSite=Lax"} { if !strings.Contains(cookie, part) { t.Errorf("Set-Cookie %q has no %q", cookie, part) } @@ -316,8 +346,7 @@ func TestSetupHostChecksTheCode(t *testing.T) { b := e.browser() b.do("GET", "/api/setup/host?code=AAAA-BBBB-CCCC", nil, nil).want(t, 422) // The code passes without dashes and in lowercase, as it does for setup. - // No host checker runs in the test, so a valid code gets 503. - b.do("GET", "/api/setup/host?code="+strings.ToLower(strings.ReplaceAll(code, "-", "")), nil, nil).want(t, 503) + b.do("GET", "/api/setup/host?code="+strings.ToLower(strings.ReplaceAll(code, "-", "")), nil, nil).want(t, 200) } func TestMutationNeedsCSRFHeader(t *testing.T) { @@ -358,10 +387,10 @@ func TestRoles(t *testing.T) { {"GET", "/api/users", nil, 403, 401}, {"GET", "/api/audit", nil, 403, 401}, {"POST", "/api/invites", map[string]any{"email": "eve@example.com", "role": "admin"}, 403, 401}, - // A member may stop sandboxes and re-run jobs. Without a scheduler - // the request passes the role check and gets 503. - {"POST", "/api/sandboxes/x/stop", nil, 503, 401}, - {"POST", "/api/jobs/x/rerun", map[string]any{}, 503, 401}, + // A member may stop sandboxes and re-run jobs. The request passes the + // role check and gets 404 for the unknown ID. + {"POST", "/api/sandboxes/x/stop", nil, 404, 401}, + {"POST", "/api/jobs/x/rerun", map[string]any{}, 404, 401}, } { if got := member.do(tc.method, tc.path, tc.body, nil).StatusCode; got != tc.member { t.Errorf("member %s %s: status %d, want %d", tc.method, tc.path, got, tc.member) @@ -456,6 +485,9 @@ func TestSettingsValidation(t *testing.T) { {map[string]any{"public_url": "ci.example.com"}, "body.public_url"}, {map[string]any{"domain": "192.0.2.1"}, "body.domain"}, {map[string]any{"egress_allowlist": []string{"github.com", "not a host"}}, "body.egress_allowlist[1]"}, + // The firewall of the sandboxes cannot apply a wildcard or an IPv6 entry. + {map[string]any{"egress_allowlist": []string{"*.example.com"}}, "body.egress_allowlist[0]"}, + {map[string]any{"egress_allowlist": []string{"10.0.0.0/8", "2001:db8::/32"}}, "body.egress_allowlist[1]"}, {map[string]any{"cpu_overcommit": 20}, "body.cpu_overcommit"}, {map[string]any{"default_time_limit_minutes": 0}, "body.default_time_limit_minutes"}, {map[string]any{"allow_public_repos": true}, "body.allow_public_repos"}, @@ -596,15 +628,28 @@ func TestEventsStreamConvertsStoreModels(t *testing.T) { pool := store.Pool{ID: ids.New(), Name: "Default", Label: "runplane", Arch: "x64", VCPU: 2, MemoryMB: 4096, DiskGB: 20, MaxRunners: 1, Enabled: true, CreatedAt: now, UpdatedAt: now} must(t, e.st.CreatePool(context.Background(), pool)) sb := store.Sandbox{ID: "sb1", PoolID: pool.ID, ServerID: "srv", State: sandbox.StateBusy, VCPU: 2, MemoryMB: 4096, CreatedAt: now} - e.publish(events.Event{Type: events.SettingsChanged, Data: store.Settings{}}, events.Event{Type: events.SandboxState, Data: &sb}) - - ev := recv(t, ch) - if ev.name != "sandbox.state" { - t.Fatalf("got event %q with %s; settings.changed must not reach the browser", ev.name, ev.data) - } - var got Sandbox - must(t, json.Unmarshal([]byte(ev.data), &got)) - if got.ID != "sb1" || got.PoolLabel != "runplane" || got.State != "busy" { - t.Fatalf("sandbox.state data = %+v, want sb1 busy in pool runplane", got) + renamed := store.DefaultSettings() + renamed.InstanceName = "Acme CI" + e.publish(events.SettingsChanged{New: renamed}, events.SandboxState(sb)) + + for seen := map[string]bool{}; len(seen) < 2; { + ev := recv(t, ch) + switch ev.name { + case "sandbox.state": + var got Sandbox + must(t, json.Unmarshal([]byte(ev.data), &got)) + if got.ID != "sb1" || got.PoolLabel != "runplane" || got.State != "busy" { + t.Fatalf("sandbox.state data = %+v, want sb1 busy in pool runplane", got) + } + case "settings.changed": + var got Settings + must(t, json.Unmarshal([]byte(ev.data), &got)) + if got.InstanceName != "Acme CI" { + t.Fatalf("settings.changed data has the instance name %q, want the new name Acme CI", got.InstanceName) + } + default: + t.Fatalf("got event %q with %s", ev.name, ev.data) + } + seen[ev.name] = true } } diff --git a/internal/api/audit.go b/internal/api/audit.go index f05faa3..a905145 100644 --- a/internal/api/audit.go +++ b/internal/api/audit.go @@ -1,58 +1,26 @@ package api import ( - "bytes" "context" "encoding/json" "errors" - "slices" "time" - "github.com/runplane/runplane/internal/ids" + "github.com/danielgtaylor/huma/v2" + + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/store" ) -// audit records a change by the current user. action is ".", -// for example "pool.create". A failure is logged and does not fail the -// request, because the change is already done. -func (h *Handler) audit(ctx context.Context, action, target string, detail map[string]any) { - var raw json.RawMessage - if detail != nil { - b, err := json.Marshal(detail) - if err != nil { - h.d.Log.Error("encode audit detail", "action", action, "err", err) - } - raw = b - } - err := h.d.Store.InsertAudit(context.WithoutCancel(ctx), store.AuditEntry{ - ID: ids.New(), At: time.Now(), UserID: currentUser(ctx).ID, Action: action, Target: target, - Detail: raw, IP: clientOf(ctx).ip, - }) - if err != nil { - h.d.Log.Error("write audit entry", "action", action, "target", target, "err", err) - } +// actor is the user of the request, who makes its changes. +func actor(ctx context.Context) audit.Actor { + return audit.Actor{UserID: currentUser(ctx).ID, IP: clientOf(ctx).ip} } -// fieldChanges gives the old and new JSON value of each field that differs -// between a and b, two values of one struct type, except the fields in skip. -func fieldChanges(a, b any, skip ...string) map[string][2]any { - fa, fb := jsonFields(a), jsonFields(b) - changes := map[string][2]any{} - for _, fields := range []map[string]json.RawMessage{fa, fb} { - for k := range fields { - if !slices.Contains(skip, k) && !bytes.Equal(fa[k], fb[k]) { - changes[k] = [2]any{fa[k], fb[k]} - } - } - } - return changes -} - -func jsonFields(v any) map[string]json.RawMessage { - var fields map[string]json.RawMessage - b, _ := json.Marshal(v) // the store models always encode - json.Unmarshal(b, &fields) - return fields +// record writes an audit entry for a change that the handler makes itself. +// The services record their own changes. +func (h *Handler) record(ctx context.Context, action audit.Action, target string, detail map[string]any) { + h.audit.Record(ctx, actor(ctx), action, target, detail) } // jobDetail adds the names of a job to the detail of an audit entry. @@ -76,6 +44,16 @@ type AuditEntry struct { IP string `json:"ip"` } +// TransformSchema gives action the enum of audit.Actions. Entries of an +// older runplane can have other actions. +func (AuditEntry) TransformSchema(_ huma.Registry, s *huma.Schema) *huma.Schema { + action := s.Properties["action"] + for _, a := range audit.Actions { + action.Enum = append(action.Enum, string(a)) + } + return s +} + type AuditPage struct { Items []AuditEntry `json:"items"` NextCursor string `json:"next_cursor,omitempty" doc:"Cursor of the next page. Absent on the last page."` diff --git a/internal/api/audit_test.go b/internal/api/audit_test.go index 9c63fec..e63a42b 100644 --- a/internal/api/audit_test.go +++ b/internal/api/audit_test.go @@ -15,7 +15,6 @@ import ( type okScheduler struct{} -func (okScheduler) PoolsChanged(context.Context) {} func (okScheduler) StopSandbox(context.Context, string) error { return nil } func (okScheduler) RerunJob(context.Context, string, bool) error { return nil } @@ -45,7 +44,6 @@ func TestAuditDetails(t *testing.T) { b.do("PATCH", "/api/settings", map[string]any{ "cpu_overcommit": 1.5, "backup_s3": map[string]any{"bucket": "backups", "access_key": "AKIA1", "secret_key": "s3-secret-value"}, }, nil).want(t, 200) - b.do("PATCH", "/api/servers/s1", map[string]any{"name": "fsn1-c", "status": "draining"}, nil).want(t, 200) var invite CreatedInvite b.do("POST", "/api/invites", map[string]any{"email": "eve@example.com", "role": "member"}, &invite).want(t, 200) b.do("DELETE", "/api/invites/"+invite.Invite.ID, nil, nil).want(t, 204) @@ -53,9 +51,9 @@ func TestAuditDetails(t *testing.T) { job, err := e.st.UpsertJob(ctx, store.JobMessage{ RunnerRequestID: 7, State: store.JobCompleted, Result: store.ResultFailed, Repository: "acme/api", JobName: "build", WorkflowRef: "acme/api/.github/workflows/ci.yml@refs/heads/main", QueuedAt: now, - }) + }, now) must(t, err) - must(t, e.st.SetJobEnrichment(ctx, job.ID, store.JobEnrichment{RunNumber: 2290})) + must(t, e.st.SetJobEnrichment(ctx, job.ID, store.JobEnrichment{RunNumber: 2290}, now)) b.do("POST", "/api/jobs/"+job.ID+"/rerun", map[string]any{"debug": true}, nil).want(t, 204) must(t, e.st.InsertSandbox(ctx, store.Sandbox{ ID: "sb1", PoolID: pool.ID, ServerID: "s2", RunnerName: "runplane-8cpu-7q2m", Mode: sandbox.ModeContainer, @@ -85,7 +83,6 @@ func TestAuditDetails(t *testing.T) { "servers":[["fsn1-a"],["fsn1-a","fsn1-b"]]}}`, "settings.update": `{"keys":["backup_s3","cpu_overcommit"],"changes":{"cpu_overcommit":[1,1.5], "backup_s3.bucket":["","backups"],"backup_s3.access_key":["","AKIA1"],"backup_s3.secret_key_set":[false,true]}}`, - "server.update": `{"name":"fsn1-c","status":"draining","changes":{"name":["fsn1-a","fsn1-c"],"status":["online","draining"]}}`, "invite.delete": `{"email":"eve@example.com","role":"member"}`, "job.rerun": `{"debug":true,"repository":"acme/api","workflow_file":".github/workflows/ci.yml","job_name":"build", "run_number":2290}`, diff --git a/internal/api/caches.go b/internal/api/caches.go index 1897a7a..ed15557 100644 --- a/internal/api/caches.go +++ b/internal/api/caches.go @@ -7,41 +7,43 @@ import ( "strings" "time" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/store" ) type CacheOverview struct { Enabled bool `json:"enabled" doc:"The actions_cache setting."` - SizeBytes int64 `json:"size_bytes" doc:"Size of all entries, also the unfinished uploads."` - MaxBytes int64 `json:"max_bytes" doc:"The limit of the cache: the cache_max_gb setting in bytes, at most a quarter of the disk of the cache. runplane removes the least recently used entries above it."` - HitRate float64 `json:"hit_rate" doc:"Hits / (hits + misses) of the last 7 days, 0 to 1."` + SizeBytes int64 `json:"size_bytes" doc:"Size of all entries and of the files of the unfinished uploads."` + MaxBytes int64 `json:"max_bytes" doc:"The byte limit that the cache server uses now. It comes from the cache_max_gb setting and the size of the disk. runplane removes the least recently used entries above it."` + StatsDays int `json:"stats_days" doc:"The hit rates, hits, and misses count the cache lookups of this many last days."` + HitRate *float64 `json:"hit_rate" doc:"Hits / (hits + misses) of the last stats_days days, 0 to 1. Null without hits and misses."` Repositories []CacheRepository `json:"repositories" doc:"Largest first."` MirrorEnabled bool `json:"mirror_enabled" doc:"The registry_mirror setting."` - Mirror *MirrorStats `json:"mirror,omitempty" doc:"Absent when the mirror stats are not available."` + Mirror MirrorStats `json:"mirror"` } type CacheRepository struct { Repository string `json:"repository"` Entries int `json:"entries"` SizeBytes int64 `json:"size_bytes"` - HitRate float64 `json:"hit_rate" doc:"Hits / (hits + misses) of the last 7 days, 0 to 1."` - Hits int64 `json:"hits" doc:"Of the last 7 days."` - Misses int64 `json:"misses" doc:"Of the last 7 days."` + HitRate *float64 `json:"hit_rate" doc:"Hits / (hits + misses) of the last stats_days days, 0 to 1. Null without hits and misses."` + Hits int64 `json:"hits" doc:"Of the last stats_days days."` + Misses int64 `json:"misses" doc:"Of the last stats_days days."` LastUsedAt *time.Time `json:"last_used_at"` } -// MirrorStats are the counts of the registry mirrors of all servers since the -// start of runplane. +// MirrorStats are the sums of the counts of the registry mirrors of the +// connected servers. Each server counts from its own start. type MirrorStats struct { - Hits int64 `json:"hits"` - Misses int64 `json:"misses"` - HitBytes int64 `json:"hit_bytes" doc:"Bytes sent from the mirror."` - MissBytes int64 `json:"miss_bytes" doc:"Bytes fetched from the upstream registry."` - SizeBytes int64 `json:"size_bytes" doc:"Bytes in the mirror now."` - StartedAt time.Time `json:"started_at" doc:"Start of runplane, where the counts start."` + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitBytes int64 `json:"hit_bytes" doc:"Bytes sent from the mirror."` + MissBytes int64 `json:"miss_bytes" doc:"Bytes fetched from the upstream registry."` + HitRate *float64 `json:"hit_rate" doc:"Hits / (hits + misses), 0 to 1. Null without hits and misses."` + SizeBytes int64 `json:"size_bytes" doc:"Bytes in the mirror now."` } -const cacheStatsWindow = 7 * 24 * time.Hour +const cacheStatsDays = 7 func (h *Handler) getCaches(ctx context.Context, _ *struct{}) (*body[CacheOverview], error) { st := h.d.Store @@ -53,7 +55,7 @@ func (h *Handler) getCaches(ctx context.Context, _ *struct{}) (*body[CacheOvervi if err != nil { return nil, err } - stats, err := st.CacheStatsByRepository(ctx, time.Now().Add(-cacheStatsWindow)) + stats, err := st.CacheStatsByRepository(ctx, time.Now().AddDate(0, 0, -cacheStatsDays)) if err != nil { return nil, err } @@ -62,8 +64,9 @@ func (h *Handler) getCaches(ctx context.Context, _ *struct{}) (*body[CacheOvervi return nil, err } out := CacheOverview{ - Enabled: settings.ActionsCache, SizeBytes: size, MaxBytes: int64(settings.CacheMaxGB) << 30, - MirrorEnabled: settings.RegistryMirror, Repositories: []CacheRepository{}, + Enabled: settings.ActionsCache, SizeBytes: size.Entries + size.Uploads, MaxBytes: h.d.Caches.MaxBytes(), + StatsDays: cacheStatsDays, MirrorEnabled: settings.RegistryMirror, + Repositories: []CacheRepository{}, } // A repository with hits in the window but no entries now also shows. repos := map[string]*CacheRepository{} @@ -87,24 +90,21 @@ func (h *Handler) getCaches(ctx context.Context, _ *struct{}) (*body[CacheOvervi slices.SortFunc(out.Repositories, func(a, b CacheRepository) int { return cmp.Or(cmp.Compare(b.SizeBytes, a.SizeBytes), strings.Compare(a.Repository, b.Repository)) }) - if h.d.Caches != nil { - out.MaxBytes = h.d.Caches.MaxBytes() - m, err := h.d.Caches.MirrorStats(ctx) - if err != nil { - h.d.Log.Warn("read mirror stats", "err", err) - } else { - m.StartedAt = h.started - out.Mirror = &m - } + m := h.d.Mirrors.MirrorStats(ctx) + out.Mirror = MirrorStats{ + Hits: m.Hits, Misses: m.Misses, HitBytes: m.HitBytes, MissBytes: m.MissBytes, HitRate: rate(m.Hits, m.Misses), + SizeBytes: m.SizeBytes, } return &body[CacheOverview]{out}, nil } -func rate(hits, misses int64) float64 { +// rate is hits / (hits + misses), or nil without either. +func rate(hits, misses int64) *float64 { if hits+misses == 0 { - return 0 + return nil } - return float64(hits) / float64(hits+misses) + r := float64(hits) / float64(hits+misses) + return &r } type CacheEntry struct { @@ -154,14 +154,17 @@ func (h *Handler) deleteCacheEntries(ctx context.Context, in *deleteCachesInput) if in.Repository == "" && in.ID == "" { return nil, invalid("query.repository", "Give a repository or an entry ID.", nil) } - if h.d.Caches == nil { - return nil, unavailable("The cache server") + n := 1 + var err error + if in.ID != "" { + err = h.d.Caches.Delete(ctx, in.ID) + } else { + n, err = h.d.Caches.DeleteRepository(ctx, in.Repository) } - n, err := h.d.Caches.Delete(ctx, in.Repository, in.ID) if err != nil { - return nil, h.serviceErr(err) + return nil, notFound(serviceErr(err), "No cache entry has this ID.") } - h.audit(ctx, "cache.delete", cmp.Or(in.ID, in.Repository), map[string]any{"repository": in.Repository, "id": in.ID, "deleted": n}) + h.record(ctx, audit.CacheDelete, cmp.Or(in.ID, in.Repository), map[string]any{"repository": in.Repository, "id": in.ID, "deleted": n}) return &body[DeletedCaches]{DeletedCaches{Deleted: n}}, nil } @@ -190,12 +193,10 @@ type RunnerCurrent struct { ARM64 string `json:"arm64"` } -const endsSoon = 14 * 24 * time.Hour - func runnerVersion(b store.RunnerBundle, now time.Time) RunnerVersion { return RunnerVersion{ Version: b.Version, Arch: b.Arch, Status: b.Status, Patched: b.Patched, SHA256: b.SHA256, Error: b.Error, - EndDate: b.EndDate, EndsSoon: b.EndDate != nil && b.EndDate.Sub(now) < endsSoon, + EndDate: b.EndDate, EndsSoon: b.EndsSoon(now), ReleasedAt: b.ReleasedAt, CreatedAt: b.CreatedAt, } } diff --git a/internal/api/caches_test.go b/internal/api/caches_test.go index b4dc2eb..80278d6 100644 --- a/internal/api/caches_test.go +++ b/internal/api/caches_test.go @@ -2,8 +2,11 @@ package api import ( "context" + "fmt" "testing" + "time" + "github.com/runplane/runplane/internal/agent" "github.com/runplane/runplane/internal/auth" "github.com/runplane/runplane/internal/store" ) @@ -16,7 +19,25 @@ type fakeCaches struct { func (f fakeCaches) MaxBytes() int64 { return f.limit } -func (f fakeCaches) MirrorStats(context.Context) (MirrorStats, error) { return MirrorStats{}, nil } +type fakeMirrors agent.MirrorStats + +func (f fakeMirrors) MirrorStats(context.Context) agent.MirrorStats { return agent.MirrorStats(f) } + +func TestCachesReportTheMirrorsOfAllServers(t *testing.T) { + e := newEnv(t, func(d *Deps) { + d.Mirrors = fakeMirrors{Hits: 6, Misses: 5, HitBytes: 600, MissBytes: 500, SizeBytes: 1300} + }) + var out CacheOverview + e.login("ada@example.com", auth.RoleMember).do("GET", "/api/caches", nil, &out).want(t, 200) + want := MirrorStats{Hits: 6, Misses: 5, HitBytes: 600, MissBytes: 500, SizeBytes: 1300} + if r := out.Mirror.HitRate; r == nil || *r != 6.0/11 { + t.Errorf("mirror hit rate %v, want 6/11", r) + } + want.HitRate = out.Mirror.HitRate + if out.Mirror != want { + t.Errorf("mirror %+v, want %+v", out.Mirror, want) + } +} func TestCachesReportTheLimitOfTheCacheServer(t *testing.T) { // cache_max_gb is 50 by default, but a 40 GB disk allows 10 GB. @@ -29,6 +50,48 @@ func TestCachesReportTheLimitOfTheCacheServer(t *testing.T) { } } +// deletingCaches is a cache server with the entries of one repository. +type deletingCaches struct { + Caches + ids map[string]bool + repo map[string]int +} + +func (f deletingCaches) Delete(_ context.Context, id string) error { + if !f.ids[id] { + return fmt.Errorf("get entry: %w", store.ErrNotFound) + } + return nil +} + +func (f deletingCaches) DeleteRepository(_ context.Context, repository string) (int, error) { + return f.repo[repository], nil +} + +func TestDeleteCaches(t *testing.T) { + caches := deletingCaches{ids: map[string]bool{"e1": true}, repo: map[string]int{"acme/app": 3}} + e := newEnv(t, func(d *Deps) { d.Caches = caches }) + b := e.login("ada@example.com", auth.RoleAdmin) + tests := []struct { + query string + status int + want int + }{ + {"?id=e1", 200, 1}, + {"?id=e1&repository=other/lib", 200, 1}, // the ID wins + {"?id=gone", 404, 0}, + {"?repository=acme/app", 200, 3}, + } + for _, tt := range tests { + var out DeletedCaches + b.do("DELETE", "/api/caches"+tt.query, nil, &out).want(t, tt.status) + if out.Deleted != tt.want { + t.Errorf("DELETE %s: deleted %d, want %d", tt.query, out.Deleted, tt.want) + } + } + b.do("DELETE", "/api/caches", nil, nil).wantField(t, 422, "query.repository") +} + func TestRunnerVersionsCurrentIsTheNewestReady(t *testing.T) { e := newEnv(t) ctx := context.Background() @@ -38,7 +101,7 @@ func TestRunnerVersionsCurrentIsTheNewestReady(t *testing.T) { {Version: "2.99.0", Arch: "x64", Status: store.BundleReady}, {Version: "2.331.0", Arch: "arm64", Status: store.BundlePending}, } { - must(t, e.st.SetRunnerBundleBuild(ctx, b)) + must(t, e.st.SetRunnerBundleBuild(ctx, b, time.Now())) } var got RunnerVersions e.login("ada@example.com", auth.RoleMember).do("GET", "/api/runner-versions", nil, &got).want(t, 200) diff --git a/internal/api/events.go b/internal/api/events.go index a5bad0f..38e501b 100644 --- a/internal/api/events.go +++ b/internal/api/events.go @@ -2,8 +2,9 @@ package api import ( "context" - "encoding/json" "errors" + "fmt" + "reflect" "time" "github.com/danielgtaylor/huma/v2/sse" @@ -13,45 +14,94 @@ import ( "github.com/runplane/runplane/internal/store" ) -type PoolStats struct { - PoolID string `json:"pool_id"` - Idle int `json:"idle"` - Busy int `json:"busy"` - Starting int `json:"starting"` - Queued int `json:"queued"` - Assigned int `json:"assigned"` +const pingEvery = 15 * time.Second + +// stream is a bus event that the browser gets: the SSE event name, the API +// object of its data, and the conversion of the bus event into the data. +type stream struct { + event reflect.Type + name string + data any + convert func(context.Context, events.Event) (any, error) } -type HostStats struct { - ServerID string `json:"server_id"` - At time.Time `json:"at"` - CPUPercent float64 `json:"cpu_percent"` - MemoryPercent float64 `json:"memory_percent"` - DiskPercent float64 `json:"disk_percent"` +// errSkip is the error of a conversion when the browser does not get the +// event. +var errSkip = errors.New("the browser does not get this event") + +func streamOf[E events.Event, D any](name string, convert func(context.Context, E) (D, error)) stream { + var data D + return stream{event: reflect.TypeFor[E](), name: name, data: data, convert: func(ctx context.Context, e events.Event) (any, error) { + return convert(ctx, e.(E)) + }} } -const pingEvery = 15 * time.Second +func same[E any](_ context.Context, e E) (E, error) { return e, nil } + +func (h *Handler) streamList() []stream { + return []stream{ + streamOf("sandbox.state", func(ctx context.Context, e events.SandboxState) (Sandbox, error) { + return first(h.sandboxes(ctx, []store.Sandbox{store.Sandbox(e)})) + }), + streamOf("job.state", func(ctx context.Context, e events.JobState) (JobSummary, error) { + return first(h.jobSummaries(ctx, []store.Job{store.Job(e)})) + }), + streamOf("pool.stats", same[events.PoolStats]), + streamOf("pools.changed", same[events.PoolsChanged]), + streamOf("server.state", func(ctx context.Context, e events.ServerState) (Server, error) { + return first(h.servers(ctx, []store.Server{store.Server(e)})) + }), + streamOf("server.removed", same[events.ServerRemoved]), + streamOf("host.stats", same[events.HostStats]), + streamOf("runner.version", func(_ context.Context, e events.RunnerVersion) (RunnerVersion, error) { + return runnerVersion(store.RunnerBundle(e), time.Now()), nil + }), + streamOf("image.state", func(ctx context.Context, e events.ImageState) (Image, error) { + return h.imageEvent(ctx, e.ID) + }), + streamOf("image.state", func(ctx context.Context, e events.ImageBuildState) (Image, error) { + return h.imageEvent(ctx, e.ImageID) + }), + streamOf("image.removed", same[events.ImageRemoved]), + streamOf("github.state", func(ctx context.Context, _ events.GitHubState) (GitHubStatus, error) { + return h.gitHubStatus(ctx) + }), + streamOf("settings.changed", func(ctx context.Context, e events.SettingsChanged) (Settings, error) { + return h.settingsOut(ctx, e.New) + }), + // One stream route serves all roles. + streamOf("backup.state", func(ctx context.Context, e events.BackupState) (Backup, error) { + if !auth.CanDo(currentUser(ctx).Role, auth.ActionAdmin) { + return Backup{}, errSkip + } + return h.backupOut(store.Backup(e)), nil + }), + } +} +// registerEvents registers the SSE route. sse.Sender finds the name of an +// event from the type of its data, so each name has its own data type. func (h *Handler) registerEvents() { op := operation(auth.ActionRead, "GET /api/events", "stream-events", "Stream live events (SSE)", describe( "Server-Sent Events. Each event has the API object of its name as data. A comment line \": ping\" comes each 15 seconds. "+ "Only admins get backup.state.")) - sse.Register(h.api, op, map[string]any{ - events.SandboxState: Sandbox{}, - events.JobState: JobSummary{}, - events.PoolStats: PoolStats{}, - events.ServerState: Server{}, - events.HostStats: HostStats{}, - events.RunnerVersion: RunnerVersion{}, - events.ImageState: Image{}, - events.GitHubState: GitHubStatus{}, - events.BackupState: Backup{}, - }, h.streamEvents) + names := map[string]any{} + byData := map[reflect.Type]string{} + h.streams = map[reflect.Type]stream{} + for _, s := range h.streamList() { + data := reflect.TypeOf(s.data) + if name, ok := byData[data]; ok && name != s.name { + panic(fmt.Sprintf("the SSE events %s and %s have the same data type %s", name, s.name, data)) + } + byData[data], names[s.name], h.streams[s.event] = s.name, s.data, s + } + sse.Register(h.api, op, names, h.streamEvents) } +// streamEvents ends when the browser reads the events too slowly. The +// browser then connects again and reads all its queries again. func (h *Handler) streamEvents(ctx context.Context, _ *struct{}, send sse.Sender) { - ch, cancel := h.d.Bus.Subscribe(256) - defer cancel() + ch := h.d.Bus.Watch(ctx, 256) ping := time.NewTicker(pingEvery) defer ping.Stop() for { @@ -66,94 +116,37 @@ func (h *Handler) streamEvents(ctx context.Context, _ *struct{}, send sse.Sender } case e, ok := <-ch: if !ok { + h.d.Log.Info("stop a slow event stream") return } - data, err := h.eventData(ctx, e) + s, ok := h.streams[reflect.TypeOf(e)] + if !ok { + continue + } + data, err := s.convert(ctx, e) + if errors.Is(err, errSkip) { + continue + } if err != nil { - h.d.Log.Warn("convert event", "type", e.Type, "err", err) + h.d.Log.Warn("convert event", "type", fmt.Sprintf("%T", e), "err", err) continue } - if data != nil && send.Data(data) != nil { + if send.Data(data) != nil { return } } } } -// eventData converts the data of a bus event into the API object of the -// event. Publishers can send the API object, the store model, or a value -// with the same JSON fields as the API object. It returns nil for events -// that the browser does not get. -func (h *Handler) eventData(ctx context.Context, e events.Event) (any, error) { - switch e.Type { - case events.SandboxState: - if b, ok := events.As[store.Sandbox](e.Data); ok { - return first(h.sandboxes(ctx, []store.Sandbox{b})) - } - return convert[Sandbox](e.Data) - case events.JobState: - if j, ok := events.As[store.Job](e.Data); ok { - return first(h.jobSummaries(ctx, []store.Job{j})) - } - return convert[JobSummary](e.Data) - case events.PoolStats: - return convert[PoolStats](e.Data) - case events.ServerState: - if v, ok := events.As[store.Server](e.Data); ok { - return first(h.servers(ctx, []store.Server{v})) - } - return convert[Server](e.Data) - case events.HostStats: - if st, ok := events.As[store.ServerStat](e.Data); ok { - return HostStats(st), nil - } - return convert[HostStats](e.Data) - case events.RunnerVersion: - if b, ok := events.As[store.RunnerBundle](e.Data); ok { - return runnerVersion(b, time.Now()), nil - } - return convert[RunnerVersion](e.Data) - case events.ImageState: - id := "" - if img, ok := events.As[store.Image](e.Data); ok { - id = img.ID - } else if b, ok := events.As[store.ImageBuild](e.Data); ok { - id = b.ImageID - } else { - return convert[Image](e.Data) - } - // Read the image again for its current builds. - img, err := h.d.Store.GetImage(ctx, id) - if errors.Is(err, store.ErrNotFound) { - return nil, nil - } - if err != nil { - return nil, err - } - return first(h.images(ctx, []store.Image{img})) - case events.GitHubState: - return h.gitHubStatus(ctx) - case events.BackupState: - if !auth.CanDo(currentUser(ctx).Role, auth.ActionAdmin) { - return nil, nil - } - if b, ok := events.As[store.Backup](e.Data); ok { - return h.backupOut(b), nil - } - return convert[Backup](e.Data) - } - return nil, nil -} - -// convert returns v as a T, or through JSON for another type with the same fields. -func convert[T any](v any) (T, error) { - if x, ok := events.As[T](v); ok { - return x, nil +// imageEvent reads the image again for its current builds. A removed image +// has no event. +func (h *Handler) imageEvent(ctx context.Context, id string) (Image, error) { + img, err := h.d.Store.GetImage(ctx, id) + if errors.Is(err, store.ErrNotFound) { + return Image{}, errSkip } - var out T - b, err := json.Marshal(v) - if err == nil { - err = json.Unmarshal(b, &out) + if err != nil { + return Image{}, err } - return out, err + return first(h.images(ctx, []store.Image{img})) } diff --git a/internal/api/github.go b/internal/api/github.go index f1a6b32..2dd73ff 100644 --- a/internal/api/github.go +++ b/internal/api/github.go @@ -5,32 +5,53 @@ import ( "errors" "net/http" "net/url" - "slices" "strconv" "time" + "github.com/danielgtaylor/huma/v2" + + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/field" + "github.com/runplane/runplane/internal/ghapp" + "github.com/runplane/runplane/internal/ghrest" + "github.com/runplane/runplane/internal/pages" + "github.com/runplane/runplane/internal/policy" "github.com/runplane/runplane/internal/store" ) type GitHubStatus struct { - Connected bool `json:"connected" doc:"An app with at least one installation exists."` - App *GitHubApp `json:"app,omitempty" doc:"Absent before the manifest flow."` - Installations []Installation `json:"installations"` - ScaleSets []ScaleSetStatus `json:"scale_sets" doc:"One for each pool and installation, or for each pool and repository of a personal account."` + App *GitHubApp `json:"app,omitempty" doc:"Absent before the manifest flow."` + Installations []Installation `json:"installations"` + ScaleSets []ScaleSetStatus `json:"scale_sets" doc:"One for each pool and installation, or for each pool and repository of a personal account."` + AppPermissions AppPermissions `json:"app_permissions" doc:"The permissions that the manifest of a new app asks for."` + HostedLabels []string `json:"hosted_labels" doc:"The runs-on labels of GitHub-hosted runners that a pull request of runplane moves to a pool."` +} + +// AppPermissions has the permissions of the app for each account_type of +// ManifestRequest. +type AppPermissions struct { + Organization []AppPermission `json:"organization"` + User []AppPermission `json:"user"` +} + +type AppPermission struct { + Name string `json:"name" enum:"organization_self_hosted_runners,administration,actions,contents,pull_requests,workflows,metadata"` + Access string `json:"access" enum:"read,write"` } type GitHubApp struct { - AppID int64 `json:"app_id"` - Slug string `json:"slug"` - Name string `json:"name"` - ClientID string `json:"client_id"` - OwnerLogin string `json:"owner_login"` - OwnerType string `json:"owner_type" doc:"Organization or User."` - HTMLURL string `json:"html_url" doc:"Public page of the app on GitHub."` - SettingsURL string `json:"settings_url" doc:"Settings page of the app on GitHub: name, permissions, and keys."` - InstallURL string `json:"install_url" doc:"GitHub page to install the app on more accounts or repositories."` - CreatedAt time.Time `json:"created_at"` + AppID int64 `json:"app_id"` + Slug string `json:"slug"` + Name string `json:"name"` + ClientID string `json:"client_id"` + OwnerLogin string `json:"owner_login"` + OwnerType string `json:"owner_type" doc:"Organization or User."` + HTMLURL string `json:"html_url" doc:"Public page of the app on GitHub."` + SettingsURL string `json:"settings_url" doc:"Settings page of the app on GitHub: name, permissions, and keys."` + PermissionsURL string `json:"permissions_url" doc:"Permissions page of the app on GitHub. An app from an earlier runplane release gets the pull request permissions there."` + InstallURL string `json:"install_url" doc:"GitHub page to install the app on more accounts or repositories."` + CreatedAt time.Time `json:"created_at"` } type Installation struct { @@ -47,10 +68,11 @@ type Installation struct { } type Repository struct { - ID int64 `json:"id"` - FullName string `json:"full_name" doc:"owner/name"` - Private bool `json:"private" doc:"runplane rejects jobs of public repositories unless allow_public_repos is on in VM mode."` - Archived bool `json:"archived"` + ID int64 `json:"id"` + FullName string `json:"full_name" doc:"owner/name"` + Private bool `json:"private"` + Archived bool `json:"archived"` + TakesJobs bool `json:"takes_jobs" doc:"runplane takes the jobs of the repository. The scheduler still checks each job."` } type ScaleSetStatus struct { @@ -79,18 +101,20 @@ type ScaleSetStats struct { func (h *Handler) gitHubStatus(ctx context.Context) (GitHubStatus, error) { st := h.d.Store - out := GitHubStatus{Installations: []Installation{}, ScaleSets: []ScaleSetStatus{}} - web := "https://github.com" + out := GitHubStatus{Installations: []Installation{}, ScaleSets: []ScaleSetStatus{}, AppPermissions: AppPermissions{ + Organization: appPermissions(ghapp.Account{Type: ghapp.Organization}), + User: appPermissions(ghapp.Account{Type: ghapp.User}), + }, HostedLabels: ghapp.HostedLabels} app, err := st.GetGitHubApp(ctx) switch { case err == nil: - web = gitHubWeb(app.HTMLURL) - slug := url.PathEscape(app.Slug) + owner := ghapp.Account{Type: app.OwnerType, Login: app.OwnerLogin} out.App = &GitHubApp{ AppID: app.AppID, Slug: app.Slug, Name: app.Name, ClientID: app.ClientID, OwnerLogin: app.OwnerLogin, OwnerType: app.OwnerType, HTMLURL: app.HTMLURL, CreatedAt: app.CreatedAt, - SettingsURL: accountSettings(web, app.OwnerType, app.OwnerLogin) + "/apps/" + slug, - InstallURL: web + "/apps/" + slug + "/installations/new", + SettingsURL: owner.AppURL(ghrest.Web, app.Slug), + PermissionsURL: owner.AppPermissionsURL(ghrest.Web, app.Slug), + InstallURL: ghapp.AppInstallURL(ghrest.Web, app.Slug), } case !errors.Is(err, store.ErrNotFound): return GitHubStatus{}, err @@ -99,6 +123,10 @@ func (h *Handler) gitHubStatus(ctx context.Context) (GitHubStatus, error) { if err != nil { return GitHubStatus{}, err } + settings, err := st.GetSettings(ctx) + if err != nil { + return GitHubStatus{}, err + } accounts := map[int64]string{} repoNames := map[int64]string{} for _, inst := range installs { @@ -110,16 +138,16 @@ func (h *Handler) gitHubStatus(ctx context.Context) (GitHubStatus, error) { ID: inst.ID, AccountLogin: inst.AccountLogin, AccountType: inst.AccountType, AvatarURL: inst.AvatarURL, RepositorySelection: inst.RepositorySelection, RepositoryCount: len(repos), Repositories: make([]Repository, len(repos)), Suspended: inst.SuspendedAt != nil, SuspendedAt: inst.SuspendedAt, - HTMLURL: accountSettings(web, inst.AccountType, inst.AccountLogin) + "/installations/" + strconv.FormatInt(inst.ID, 10), + HTMLURL: ghapp.Account{Type: inst.AccountType, Login: inst.AccountLogin}.InstallationURL(ghrest.Web, inst.ID), }) list := out.Installations[len(out.Installations)-1].Repositories for i, r := range repos { - list[i] = Repository{ID: r.ID, FullName: r.FullName, Private: r.Private, Archived: r.Archived} + list[i] = Repository{ID: r.ID, FullName: r.FullName, Private: r.Private, Archived: r.Archived, + TakesJobs: inst.SuspendedAt == nil && policy.TakesJobs(settings, r)} repoNames[r.ID] = r.FullName } accounts[inst.ID] = inst.AccountLogin } - out.Connected = out.App != nil && len(installs) > 0 sets, err := st.ListScaleSets(ctx) if err != nil { @@ -144,45 +172,26 @@ func (h *Handler) gitHubStatus(ctx context.Context) (GitHubStatus, error) { return out, nil } -// gitHubWeb returns the scheme and host of the app URL, which differ from -// https://github.com on GitHub Enterprise Server. -func gitHubWeb(appURL string) string { - if u, err := url.Parse(appURL); err == nil && u.Host != "" { - return u.Scheme + "://" + u.Host - } - return "https://github.com" -} - -// accountSettings is the settings page of a GitHub account. Its apps and -// installations pages are below it. -func accountSettings(web, accountType, login string) string { - if accountType == "Organization" { - return web + "/organizations/" + url.PathEscape(login) + "/settings" - } - return web + "/settings" -} - // runnerGroup names the runner group of a scale set. runplane makes its -// scale sets in the group Default, which has the ID 1. +// scale sets in the group Default. func runnerGroup(id int64) string { - if id <= 1 { + if id <= ghapp.DefaultRunnerGroup { return "Default" } return "Group " + strconv.FormatInt(id, 10) } -func (h *Handler) getGitHub(ctx context.Context, _ *struct{}) (*body[GitHubStatus], error) { - return reply(h.gitHubStatus(ctx)) +func appPermissions(owner ghapp.Account) []AppPermission { + list := owner.AppPermissions() + out := make([]AppPermission, len(list)) + for i, p := range list { + out[i] = AppPermission(p) + } + return out } -func (h *Handler) githubConnected(ctx context.Context) (bool, error) { - if _, err := h.d.Store.GetGitHubApp(ctx); errors.Is(err, store.ErrNotFound) { - return false, nil - } else if err != nil { - return false, err - } - list, err := h.d.Store.ListInstallations(ctx) - return len(list) > 0, err +func (h *Handler) getGitHub(ctx context.Context, _ *struct{}) (*body[GitHubStatus], error) { + return reply(h.gitHubStatus(ctx)) } type ManifestRequest struct { @@ -197,24 +206,22 @@ type ManifestForm struct { func (h *Handler) createManifest(ctx context.Context, in *body[ManifestRequest]) (*body[ManifestForm], error) { b := in.Body - if b.AccountType == "organization" && b.Organization == "" { - return nil, invalid("body.organization", "Enter the organization.", nil) - } - if b.AccountType == "user" { - b.Organization = "" - } - if h.d.GitHub == nil { - return nil, unavailable("The GitHub connection") + owner := ghapp.Account{Type: ghapp.User} + if b.AccountType == "organization" { + if b.Organization == "" { + return nil, invalid("body.organization", "Enter the organization.", nil) + } + owner = ghapp.Account{Type: ghapp.Organization, Login: b.Organization} } base, err := h.baseURL(ctx) if err != nil { return nil, err } - action, manifest, err := h.d.GitHub.Manifest(ctx, base, b.AccountType, b.Organization) + action, manifest, err := h.d.GitHub.Manifest(ctx, base, owner) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - h.audit(ctx, "github.manifest", "", map[string]any{"account_type": b.AccountType, "organization": b.Organization}) + h.record(ctx, audit.GitHubManifest, "", map[string]any{"account_type": b.AccountType, "organization": owner.Login}) return &body[ManifestForm]{ManifestForm{ActionURL: action, Manifest: manifest}}, nil } @@ -238,25 +245,25 @@ func (h *Handler) githubCallback(ctx context.Context, in *githubCallbackInput) ( if to != "" { return redirect(to), nil } - if h.d.GitHub == nil { - return redirect(h.githubErrorPage(ctx, "The GitHub connection is not available in this runplane process.")), nil - } - base, err := h.baseURL(ctx) - if err != nil { - return nil, err - } - to, err = h.d.GitHub.Callback(ctx, base, in.Code, in.State) + to, err := h.d.GitHub.Callback(ctx, in.Code, in.State) if err != nil { h.d.Log.Warn("github app manifest callback", "err", err) return redirect(h.githubErrorPage(ctx, err.Error())), nil } - h.audit(ctx, "github.app.create", "", nil) + h.record(ctx, audit.GitHubAppCreate, "", nil) return redirect(to), nil } type githubSetupInput struct { InstallationID int64 `query:"installation_id" minimum:"0"` - SetupAction string `query:"setup_action" maxLength:"20" doc:"install, update, or request."` + SetupAction string `query:"setup_action" required:"true" enum:"install,update,request"` +} + +// installActions are the audit actions of the setup_action values of GitHub. +var installActions = map[string]audit.Action{ + "install": audit.GitHubInstall, + "update": audit.GitHubInstallUpdate, + "request": audit.GitHubInstallRequest, } func (h *Handler) githubSetup(ctx context.Context, in *githubSetupInput) (*redirectOutput, error) { @@ -268,19 +275,12 @@ func (h *Handler) githubSetup(ctx context.Context, in *githubSetupInput) (*redir if to != "" { return redirect(to), nil } - if h.d.GitHub == nil { - return redirect(h.githubErrorPage(ctx, "The GitHub connection is not available in this runplane process.")), nil - } - base, err := h.baseURL(ctx) - if err != nil { - return nil, err - } - to, err = h.d.GitHub.Setup(ctx, base, in.InstallationID, in.SetupAction) + to, err := h.d.GitHub.Setup(ctx, in.InstallationID, in.SetupAction) if err != nil { h.d.Log.Warn("github installation setup", "installation", in.InstallationID, "err", err) return redirect(h.githubErrorPage(ctx, err.Error())), nil } - h.audit(ctx, "github.installation."+in.SetupAction, strconv.FormatInt(in.InstallationID, 10), nil) + h.record(ctx, installActions[in.SetupAction], strconv.FormatInt(in.InstallationID, 10), nil) return redirect(to), nil } @@ -290,7 +290,7 @@ func (h *Handler) githubSetup(ctx context.Context, in *githubSetupInput) (*redir func (h *Handler) browserAdmin(ctx context.Context, self string) (context.Context, string) { u, err := h.d.Auth.UserForToken(ctx, clientOf(ctx).token) if err != nil { - return ctx, "/login?" + url.Values{"redirect": {self}}.Encode() + return ctx, pages.Login(self) } ctx = context.WithValue(ctx, userKey{}, u) if !auth.CanDo(u.Role, auth.ActionAdmin) { @@ -299,30 +299,22 @@ func (h *Handler) browserAdmin(ctx context.Context, self string) (context.Contex return ctx, "" } -// githubErrorPage is the setup page before GitHub is connected, and the -// GitHub settings page after. func (h *Handler) githubErrorPage(ctx context.Context, msg string) string { - page := "/setup/github" - if ok, err := h.githubConnected(ctx); err == nil && ok { - page = "/settings/github" - } - return page + "?" + url.Values{"error": {msg}}.Encode() + ok, err := h.d.Store.GitHubConnected(ctx) + return pages.GitHubError(err == nil && ok, msg) } func (h *Handler) syncGitHub(ctx context.Context, _ *struct{}) (*body[GitHubStatus], error) { - if h.d.GitHub == nil { - return nil, unavailable("The GitHub connection") - } if err := h.d.GitHub.Sync(ctx); err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - h.audit(ctx, "github.sync", "", nil) + h.record(ctx, audit.GitHubSync, "", nil) return reply(h.gitHubStatus(ctx)) } type CreatePullRequestRequest struct { - Repository string `json:"repository" maxLength:"140" doc:"owner/name of a repository that the GitHub App can access."` - Label string `json:"label" pattern:"^[a-z0-9][a-z0-9-]{0,62}$" patternDescription:"lowercase letters, digits, and dashes" doc:"The label of an enabled pool."` + Repository string `json:"repository" maxLength:"140" doc:"owner/name of a repository whose takes_jobs is true."` + Label string `json:"label" doc:"The label of an enabled pool."` } type PullRequest struct { @@ -332,35 +324,25 @@ type PullRequest struct { func (h *Handler) createPullRequest(ctx context.Context, in *body[CreatePullRequestRequest]) (*body[PullRequest], error) { b := in.Body - if h.d.GitHub == nil { - return nil, unavailable("The GitHub connection") - } - pools, err := h.d.Store.ListPools(ctx) - if err != nil { - return nil, err - } - if !slices.ContainsFunc(pools, func(p store.Pool) bool { return p.Enabled && p.Label == b.Label }) { - return nil, invalid("body.label", "No enabled pool has this label.", b.Label) - } base, err := h.baseURL(ctx) if err != nil { return nil, err } number, htmlURL, err := h.d.GitHub.OpenPullRequest(ctx, base, b.Repository, b.Label) - if err != nil { - return nil, h.serviceErr(err) + switch { + case errors.Is(err, ghapp.ErrNoPullPermissions): + return nil, huma.Error403Forbidden(field.Sentence(err)) + case err != nil: + return nil, serviceErr(err) } - h.audit(ctx, "github.pull_request.create", b.Repository, map[string]any{"label": b.Label, "number": number}) + h.record(ctx, audit.GitHubPullRequest, b.Repository, map[string]any{"repository": b.Repository, "label": b.Label, "number": number}) return &body[PullRequest]{PullRequest{Number: number, HTMLURL: htmlURL}}, nil } func (h *Handler) disconnectGitHub(ctx context.Context, _ *struct{}) (*empty, error) { - if h.d.GitHub == nil { - return nil, unavailable("The GitHub connection") - } if err := h.d.GitHub.Disconnect(ctx); err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - h.audit(ctx, "github.disconnect", "", nil) + h.record(ctx, audit.GitHubDisconnect, "", nil) return nil, nil } diff --git a/internal/api/github_test.go b/internal/api/github_test.go index 1991881..d440783 100644 --- a/internal/api/github_test.go +++ b/internal/api/github_test.go @@ -3,22 +3,24 @@ package api import ( "context" "encoding/json" + "maps" "slices" "testing" "time" "github.com/runplane/runplane/internal/auth" "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/field" "github.com/runplane/runplane/internal/store" ) -func TestGitHubLinksFollowTheAppHost(t *testing.T) { +func TestGitHubLinks(t *testing.T) { e := newEnv(t) ctx := context.Background() now := time.Now() must(t, e.st.SaveGitHubApp(ctx, store.GitHubApp{ ID: "a", AppID: 5, Slug: "runplane-acme", Name: "runplane acme", OwnerLogin: "acme", OwnerType: "Organization", - HTMLURL: "https://ghe.example.com/apps/runplane-acme", PrivateKeyEnc: []byte{1}, CreatedAt: now, + HTMLURL: "https://github.com/apps/runplane-acme", PrivateKeyEnc: []byte{1}, CreatedAt: now, })) for _, inst := range []store.Installation{ {ID: 77, AppID: 5, AccountLogin: "acme", AccountType: "Organization", CreatedAt: now, UpdatedAt: now}, @@ -32,10 +34,11 @@ func TestGitHubLinksFollowTheAppHost(t *testing.T) { var gh GitHubStatus e.login("ada@example.com", auth.RoleMember).do("GET", "/api/github", nil, &gh).want(t, 200) for _, c := range []struct{ name, got, want string }{ - {"app settings", gh.App.SettingsURL, "https://ghe.example.com/organizations/acme/settings/apps/runplane-acme"}, - {"app install", gh.App.InstallURL, "https://ghe.example.com/apps/runplane-acme/installations/new"}, - {"organization installation", gh.Installations[0].HTMLURL, "https://ghe.example.com/organizations/acme/settings/installations/77"}, - {"personal installation", gh.Installations[1].HTMLURL, "https://ghe.example.com/settings/installations/78"}, + {"app settings", gh.App.SettingsURL, "https://github.com/organizations/acme/settings/apps/runplane-acme"}, + {"app permissions", gh.App.PermissionsURL, "https://github.com/organizations/acme/settings/apps/runplane-acme/permissions"}, + {"app install", gh.App.InstallURL, "https://github.com/apps/runplane-acme/installations/new"}, + {"organization installation", gh.Installations[0].HTMLURL, "https://github.com/organizations/acme/settings/installations/77"}, + {"personal installation", gh.Installations[1].HTMLURL, "https://github.com/settings/installations/78"}, {"runner group", gh.ScaleSets[0].RunnerGroup, "Default"}, } { if c.got != c.want { @@ -44,35 +47,65 @@ func TestGitHubLinksFollowTheAppHost(t *testing.T) { } } -// fakeGitHub opens pull requests. The tests use no other GitHub method. +// The pull request form lists the repositories whose jobs runplane takes, +// so that the jobs of a pull request start. +func TestRepositoriesTakeJobs(t *testing.T) { + e := newEnv(t) + ctx := context.Background() + now := time.Now() + must(t, e.st.SaveGitHubApp(ctx, store.GitHubApp{ID: "a", AppID: 5, Slug: "runplane-acme", Name: "runplane acme", + OwnerLogin: "acme", OwnerType: "Organization", PrivateKeyEnc: []byte{1}, CreatedAt: now})) + for _, inst := range []store.Installation{ + {ID: 77, AppID: 5, AccountLogin: "acme", AccountType: "Organization", CreatedAt: now, UpdatedAt: now}, + {ID: 78, AppID: 5, AccountLogin: "octo", AccountType: "User", SuspendedAt: &now, CreatedAt: now, UpdatedAt: now}, + } { + must(t, e.st.UpsertInstallation(ctx, inst)) + } + must(t, e.st.ReplaceRepositories(ctx, 77, []store.Repository{ + {ID: 1, FullName: "acme/api", Private: true, UpdatedAt: now}, + {ID: 2, FullName: "acme/site", UpdatedAt: now}, + {ID: 3, FullName: "acme/old", Private: true, Archived: true, UpdatedAt: now}, + })) + must(t, e.st.ReplaceRepositories(ctx, 78, []store.Repository{{ID: 4, FullName: "octo/app", Private: true, UpdatedAt: now}})) + + var gh GitHubStatus + e.login("ada@example.com", auth.RoleMember).do("GET", "/api/github", nil, &gh).want(t, 200) + takes := map[string]bool{} + for _, inst := range gh.Installations { + for _, r := range inst.Repositories { + takes[r.FullName] = r.TakesJobs + } + } + want := map[string]bool{"acme/api": true, "acme/site": false, "acme/old": false, "octo/app": false} + if !maps.Equal(takes, want) { + t.Errorf("takes_jobs = %v, want %v", takes, want) + } +} + +// fakeGitHub opens pull requests to the label big. The tests use no other +// GitHub method. type fakeGitHub struct { GitHub opened []string // "repository label" of each call } func (f *fakeGitHub) OpenPullRequest(_ context.Context, _, repository, label string) (int, string, error) { + if label != "big" { + return 0, "", field.Problems{{Key: "label", Message: "No enabled pool has this label.", Value: label}} + } f.opened = append(f.opened, repository+" "+label) return 12, "https://github.com/" + repository + "/pull/12", nil } -// A pull request must move jobs to a label that an enabled pool serves, or -// the jobs never start. Only an admin can write to repositories. +// The form shows a refused pull request at its field. Only an admin can +// write to repositories. func TestCreatePullRequest(t *testing.T) { gh := &fakeGitHub{} e := newEnv(t, func(d *Deps) { d.GitHub = gh }) - now := time.Now() - for _, p := range []store.Pool{ - {ID: "p1", Name: "big", Label: "big", Arch: "x64", VCPU: 4, MemoryMB: 8192, DiskGB: 64, MaxRunners: 2, Enabled: true, CreatedAt: now, UpdatedAt: now}, - {ID: "p2", Name: "off", Label: "off", Arch: "x64", VCPU: 2, MemoryMB: 4096, DiskGB: 64, MaxRunners: 2, CreatedAt: now, UpdatedAt: now}, - } { - must(t, e.st.CreatePool(context.Background(), p)) - } const path = "/api/github/pull-requests" e.login("bob@example.com", auth.RoleMember).do("POST", path, map[string]any{"repository": "acme/web", "label": "big"}, nil).want(t, 403) admin := e.login("ada@example.com", auth.RoleAdmin) - for _, label := range []string{"off", "gone"} { - admin.do("POST", path, map[string]any{"repository": "acme/web", "label": label}, nil).wantField(t, 422, "body.label") - } + admin.do("POST", path, map[string]any{"repository": "acme/web", "label": "off"}, nil).wantField(t, 422, "body.label") var pr PullRequest admin.do("POST", path, map[string]any{"repository": "acme/web", "label": "big"}, &pr).want(t, 200) @@ -85,8 +118,8 @@ func TestGitHubAndBackupEvents(t *testing.T) { e := newEnv(t) member := e.login("bob@example.com", auth.RoleMember).events(t) admin := e.login("ada@example.com", auth.RoleAdmin).events(t) - backup := store.Backup{ID: "bk1", CreatedAt: time.Now(), Status: "running"} - e.publish(events.Event{Type: events.BackupState, Data: backup}, events.Event{Type: events.GitHubState}) + backup := events.BackupState{ID: "bk1", CreatedAt: time.Now(), Status: "running"} + e.publish(backup, events.GitHubState{}) for seen := 0; seen < 2; { ev := recv(t, member) @@ -95,8 +128,8 @@ func TestGitHubAndBackupEvents(t *testing.T) { t.Fatalf("a member got backup.state: %s", ev.data) case "github.state": var gh GitHubStatus - if err := json.Unmarshal([]byte(ev.data), &gh); err != nil || gh.Connected || gh.Installations == nil { - t.Fatalf("github.state data %s: want the GitHub status without a connection", ev.data) + if err := json.Unmarshal([]byte(ev.data), &gh); err != nil || gh.App != nil || gh.Installations == nil { + t.Fatalf("github.state data %s: want the GitHub status without an app", ev.data) } seen++ } diff --git a/internal/api/http.go b/internal/api/http.go index 2cd5df0..b85bafe 100644 --- a/internal/api/http.go +++ b/internal/api/http.go @@ -102,7 +102,6 @@ func devCORS(w http.ResponseWriter, r *http.Request) bool { h.Set("Access-Control-Allow-Credentials", "true") h.Add("Vary", "Origin") if r.Method != http.MethodOptions { - h.Set("Access-Control-Expose-Headers", stepFilterHeader) return false } h.Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE") diff --git a/internal/api/images.go b/internal/api/images.go index 5fa428f..0e01158 100644 --- a/internal/api/images.go +++ b/internal/api/images.go @@ -3,14 +3,11 @@ package api import ( "context" "errors" - "strings" "time" "github.com/danielgtaylor/huma/v2" - "github.com/google/go-containerregistry/pkg/name" - "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/ids" + "github.com/runplane/runplane/internal/fleet" "github.com/runplane/runplane/internal/store" ) @@ -29,7 +26,7 @@ type Image struct { type ImageBuild struct { ServerID string `json:"server_id"` ServerName string `json:"server_name"` - Arch string `json:"arch" doc:"Arch of the server: x64 or arm64. Empty until the server reports its facts."` + Arch string `json:"arch" enum:"x64,arm64," doc:"Arch of the server. Empty until the server reports its facts."` Digest string `json:"digest"` Status string `json:"status" enum:"pending,pulling,building,converting,ready,failed"` Error string `json:"error"` @@ -95,130 +92,54 @@ type CreateImageRequest struct { } func (h *Handler) createImage(ctx context.Context, in *body[CreateImageRequest]) (*body[Image], error) { - imgName, ref, err := checkImage(in.Body.Name, in.Body.Ref) + img, err := h.d.Images.CreateImage(ctx, actor(ctx), in.Body.Name, in.Body.Ref) if err != nil { - return nil, err - } - now := time.Now() - img := store.Image{ID: ids.New(), Name: imgName, Ref: ref, Source: "registry", CreatedAt: now, UpdatedAt: now} - if err := h.d.Store.CreateImage(ctx, img); err != nil { return nil, imageErr(err) } - h.audit(ctx, "image.create", img.ID, map[string]any{"name": img.Name, "ref": img.Ref}) - h.d.Bus.Publish(events.Event{Type: events.ImageState, Data: img}) - h.refresh(ctx, img.ID) return h.imageOutput(ctx, img.ID) } +// UpdateImageRequest has the limits of CreateImageRequest. type UpdateImageRequest struct { - Name *string `json:"name,omitempty" minLength:"1" maxLength:"64"` - Ref *string `json:"ref,omitempty" minLength:"1" maxLength:"512" doc:"A new ref pulls the image again."` + Name *string `json:"name,omitempty"` + Ref *string `json:"ref,omitempty" doc:"A new ref pulls the image again."` IsDefault *bool `json:"is_default,omitempty" doc:"true makes this image the default. To change the default, make another image the default."` } +func (UpdateImageRequest) TransformSchema(r huma.Registry, s *huma.Schema) *huma.Schema { + return patchOf[CreateImageRequest](r, s) +} + func (h *Handler) updateImage(ctx context.Context, in *struct { PathID Body UpdateImageRequest }) (*body[Image], error) { - img, err := h.d.Store.GetImage(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No image has this ID.") - } b := in.Body - if b.IsDefault != nil && !*b.IsDefault && img.IsDefault { - return nil, invalid("body.is_default", "Make another image the default instead.", false) - } - if b.Ref != nil && *b.Ref != img.Ref && img.Source == "builtin" { - return nil, invalid("body.ref", "The ref of a built-in image cannot change. Add a new image instead.", *b.Ref) - } - newName, ref := img.Name, img.Ref - set(&newName, b.Name) - set(&ref, b.Ref) - if newName, ref, err = checkImage(newName, ref); err != nil { - return nil, err - } - refChanged := ref != img.Ref - if newName != img.Name || refChanged { - img.Name, img.Ref = newName, ref - if err := h.d.Store.UpdateImage(ctx, img); err != nil { - return nil, imageErr(err) - } - } - if b.IsDefault != nil && *b.IsDefault && !img.IsDefault { - if err := h.d.Store.SetDefaultImage(ctx, img.ID); err != nil { - return nil, imageErr(err) - } - h.poolsChanged(ctx) - } - h.audit(ctx, "image.update", img.ID, map[string]any{"name": img.Name, "ref": img.Ref, "is_default": b.IsDefault}) - h.d.Bus.Publish(events.Event{Type: events.ImageState, Data: img}) - if refChanged { - h.refresh(ctx, img.ID) + img, err := h.d.Images.UpdateImage(ctx, actor(ctx), in.ID, fleet.ImageChange{Name: b.Name, Ref: b.Ref, Default: b.IsDefault}) + if err != nil { + return nil, imageErr(err) } return h.imageOutput(ctx, img.ID) } func (h *Handler) deleteImage(ctx context.Context, in *PathID) (*empty, error) { - img, err := h.d.Store.GetImage(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No image has this ID.") - } - if img.IsDefault { + err := h.d.Images.RemoveImage(ctx, actor(ctx), in.ID) + if errors.Is(err, fleet.ErrDefaultImage) { return nil, huma.Error409Conflict("The default image cannot be removed. Make another image the default first.") } - if h.d.Images == nil { - return nil, unavailable("Removing images") - } - if err := h.d.Images.Remove(ctx, img.ID); err != nil { - return nil, h.serviceErr(err) - } - // Pools with this image change to the default image. - if err := h.d.Store.DeleteImage(ctx, img.ID); err != nil { - return nil, notFound(err, "No image has this ID.") + if err != nil { + return nil, imageErr(err) } - h.poolsChanged(ctx) - h.audit(ctx, "image.delete", img.ID, map[string]any{"name": img.Name, "ref": img.Ref}) return nil, nil } func (h *Handler) refreshImage(ctx context.Context, in *PathID) (*empty, error) { - if h.d.Images == nil { - return nil, unavailable("Pulling images") - } - img, err := h.d.Store.GetImage(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No image has this ID.") - } - if err := h.d.Images.Refresh(ctx, img.ID); err != nil { - return nil, h.serviceErr(err) + if err := h.d.Images.RefreshImage(ctx, actor(ctx), in.ID); err != nil { + return nil, imageErr(err) } - h.audit(ctx, "image.refresh", img.ID, map[string]any{"ref": img.Ref}) return nil, nil } -// refresh starts the pulls of a new or changed image. The builds show a -// failure, so a failure here does not fail the request. -func (h *Handler) refresh(ctx context.Context, id string) { - if h.d.Images == nil { - return - } - if err := h.d.Images.Refresh(context.WithoutCancel(ctx), id); err != nil { - h.d.Log.Warn("pull image", "image", id, "err", err) - } -} - -func checkImage(imgName, ref string) (string, string, error) { - imgName, ref = strings.TrimSpace(imgName), strings.TrimSpace(ref) - var errs problems - if imgName == "" { - errs.add("body.name", "Enter a name.", imgName) - } - if _, err := name.ParseReference(ref); err != nil { - errs.add("body.ref", "The ref is not a valid image reference, for example ghcr.io/owner/image:tag.", ref) - } - return imgName, ref, errs.err() -} - func imageErr(err error) error { if errors.Is(err, store.ErrConflict) { return conflict("body.ref", "An image with this ref exists.") diff --git a/internal/api/jobs.go b/internal/api/jobs.go index 602ea15..b88975b 100644 --- a/internal/api/jobs.go +++ b/internal/api/jobs.go @@ -10,6 +10,8 @@ import ( "github.com/danielgtaylor/huma/v2" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/ghrest" "github.com/runplane/runplane/internal/store" ) @@ -28,7 +30,7 @@ type JobSummary struct { ServerID string `json:"server_id" doc:"Empty until the job has a sandbox."` ServerName string `json:"server_name"` State string `json:"state" enum:"queued,assigned,running,completed"` - Result *string `json:"result" enum:"succeeded,failed,cancelled,skipped" doc:"Null until the job completes."` + Status jobStatus `json:"status" doc:"queued until a runner starts the job, also when GitHub assigned it, then running, and then the result."` Debug bool `json:"debug" doc:"The sandbox stays after the job for a debug shell."` RejectReason string `json:"reject_reason" doc:"Why runplane does not take this queued job. Empty if it takes it."` QueueMS int64 `json:"queue_ms" doc:"Time from queued to assigned. For a job that still waits, the time until now."` @@ -56,7 +58,8 @@ type Job struct { SandboxState string `json:"sandbox_state" enum:"starting,idle,busy,stopping,gone,failed," doc:"Empty until the job has a sandbox."` HoldUntil *time.Time `json:"hold_until" doc:"A debug sandbox stays until this time after the job."` SSHCommand string `json:"ssh_command" doc:"Command that opens the debug shell over SSH, for example \"ssh -p 2222 @ci.example.com\". Empty unless the sandbox is a debug sandbox that has not stopped."` - LogState string `json:"log_state" enum:"none,live,stored,expired"` + LogState string `json:"log_state" enum:"none,live,stored,expired" doc:"live while the job runs and its sandbox streams the log."` + FinishPending bool `json:"finish_pending" doc:"The job completed, and runplane still gets its run data and its log from GitHub."` CPUPeak *float64 `json:"cpu_peak" doc:"Percent of the vCPU allocation."` CPUAvg *float64 `json:"cpu_avg" doc:"Percent of the vCPU allocation."` MemPeakMB *int `json:"mem_peak_mb"` @@ -66,12 +69,22 @@ type Job struct { SizeAdvice SizeAdvice `json:"size_advice" doc:"Whether the pool fits the CPU and memory use of the last 10 completed runs of this job (same repository, workflow file, and job name)."` } -// JobStep is one step from the GitHub jobs API. +// jobStatus is a store.Job status, with the enum store.JobStatuses. +type jobStatus string + +func (jobStatus) Schema(huma.Registry) *huma.Schema { + s := &huma.Schema{Type: huma.TypeString} + for _, v := range store.JobStatuses { + s.Enum = append(s.Enum, v) + } + return s +} + +// JobStep is one step of the job on GitHub. type JobStep struct { Number int `json:"number"` Name string `json:"name"` - Status string `json:"status" doc:"queued, in_progress, or completed."` - Conclusion string `json:"conclusion" doc:"For example success, failure, cancelled, or skipped. Empty until the step completes."` + Status jobStatus `json:"status" doc:"queued until the step starts, then running, and then the result."` StartedAt *time.Time `json:"started_at"` CompletedAt *time.Time `json:"completed_at"` } @@ -83,15 +96,11 @@ type JobPage struct { } func jobSummary(j store.Job, r refs, now time.Time) JobSummary { - var result *string - if j.Result != "" { - result = &j.Result - } return JobSummary{ ID: j.ID, JobName: j.JobName, WorkflowName: j.WorkflowName, WorkflowFile: j.WorkflowFile, Repository: j.Repository, RunNumber: j.RunNumber, HeadBranch: j.HeadBranch, HeadSHA: j.HeadSHA, PoolID: j.PoolID, PoolLabel: r.pools[j.PoolID].Label, ServerID: j.ServerID, ServerName: r.servers[j.ServerID].Name, - State: j.State, Result: result, Debug: j.Debug, RejectReason: j.RejectReason, + State: j.State, Status: jobStatus(j.Status()), Debug: j.Debug, RejectReason: j.RejectReason, QueueMS: queueMS(j, now), DurationMS: durationMS(j, now), QueuedAt: j.QueuedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, } @@ -151,14 +160,14 @@ func (h *Handler) jobSummaries(ctx context.Context, jobs []store.Job) ([]JobSumm } type listJobsInput struct { - Status string `query:"status" enum:"running,queued,succeeded,failed,cancelled" doc:"queued also has the assigned jobs."` - Repository string `query:"repository" maxLength:"200"` - PoolID string `query:"pool_id" maxLength:"100"` - ServerID string `query:"server_id" maxLength:"100"` - Q string `query:"q" maxLength:"200" doc:"Text in the job, workflow, repository, or branch name, or the start of a commit SHA."` - Range string `query:"range" enum:"24h,7d,30d" doc:"Only the jobs queued in this time. All jobs if absent."` - Cursor string `query:"cursor" maxLength:"200" doc:"next_cursor of the previous page."` - Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"` + Status jobStatus `query:"status" doc:"The status of the jobs."` + Repository string `query:"repository" maxLength:"200"` + PoolID string `query:"pool_id" maxLength:"100"` + ServerID string `query:"server_id" maxLength:"100"` + Q string `query:"q" maxLength:"200" doc:"Text in the job, workflow, repository, or branch name, or the start of a commit SHA."` + Range string `query:"range" enum:"24h,7d,30d" doc:"Only the jobs queued in this time. All jobs if absent."` + Cursor string `query:"cursor" maxLength:"200" doc:"next_cursor of the previous page."` + Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"` } // ranges are the time ranges of the job list and the overview, in buckets @@ -174,7 +183,7 @@ var ranges = map[string]struct { func (h *Handler) listJobs(ctx context.Context, in *listJobsInput) (*body[JobPage], error) { f := store.JobFilter{ - Status: in.Status, Repository: in.Repository, PoolID: in.PoolID, ServerID: in.ServerID, + Status: string(in.Status), Repository: in.Repository, PoolID: in.PoolID, ServerID: in.ServerID, Q: in.Q, Cursor: in.Cursor, Limit: in.Limit, } if rg, ok := ranges[in.Range]; ok { @@ -240,10 +249,10 @@ func (h *Handler) getJob(ctx context.Context, in *PathID) (*body[Job], error) { WorkflowRef: j.WorkflowRef, RunID: j.RunID, RunAttempt: j.RunAttempt, CommitMessage: j.CommitMessage, Event: j.Event, Actor: j.Actor, PRNumber: j.PRNumber, HTMLURL: jobURL(j), Steps: make([]JobStep, len(j.Steps)), Labels: orEmpty(j.Labels), SandboxID: j.SandboxID, - LogState: j.LogState, CPUPeak: j.CPUPeak, CPUAvg: j.CPUAvg, MemPeakMB: j.MemPeakMB, AssignedAt: j.AssignedAt, + LogState: j.LogState, FinishPending: j.FinishPending, CPUPeak: j.CPUPeak, CPUAvg: j.CPUAvg, MemPeakMB: j.MemPeakMB, AssignedAt: j.AssignedAt, } for i, st := range j.Steps { - out.Steps[i] = JobStep(st) + out.Steps[i] = JobStep{Number: st.Number, Name: st.Name, Status: jobStatus(st.Status()), StartedAt: st.StartedAt, CompletedAt: st.CompletedAt} } if out.ExpectedDurationMS, err = h.d.Store.ExpectedDurationMS(ctx, j.Repository, j.WorkflowFile, j.JobName); err != nil { return nil, err @@ -279,7 +288,7 @@ func jobURL(j store.Job) string { if j.HTMLURL != "" || j.RunID == 0 || j.Repository == "" { return j.HTMLURL } - u := fmt.Sprintf("https://github.com/%s/actions/runs/%d", j.Repository, j.RunID) + u := fmt.Sprintf("%s/%s/actions/runs/%d", ghrest.Web, j.Repository, j.RunID) if j.RunAttempt > 1 { u += fmt.Sprintf("/attempts/%d", j.RunAttempt) } @@ -354,9 +363,6 @@ func (h *Handler) rerunJob(ctx context.Context, in *struct { PathID Body RerunRequest }) (*empty, error) { - if h.d.Scheduler == nil { - return nil, unavailable("The scheduler") - } j, err := h.d.Store.GetJob(ctx, in.ID) if err != nil { return nil, notFound(err, "No job has this ID.") @@ -365,60 +371,46 @@ func (h *Handler) rerunJob(ctx context.Context, in *struct { return nil, huma.Error409Conflict("Only a completed job can run again.") } if err := h.d.Scheduler.RerunJob(ctx, j.ID, in.Body.Debug); err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - h.audit(ctx, "job.rerun", j.ID, jobDetail(map[string]any{"debug": in.Body.Debug}, j)) + h.record(ctx, audit.JobRerun, j.ID, jobDetail(map[string]any{"debug": in.Body.Debug}, j)) return nil, nil } type logInput struct { PathID - Step int `query:"step" minimum:"0" doc:"Only this step of a stored log. 0 or absent: all steps. The log of a running job always has all steps; then the response has the header X-Runplane-Step-Filter: ignored."` + Step int `query:"step" minimum:"0" doc:"Only this step of a stored log. 0 or absent: all steps. The log of a running job always has all steps."` Follow bool `query:"follow" doc:"Stream new lines until the log ends."` } func (h *Handler) jobLog(ctx context.Context, in *logInput) (*huma.StreamResponse, error) { - if h.d.Logs == nil { - return nil, unavailable("The job log") - } if _, err := h.d.Store.GetJob(ctx, in.ID); err != nil { return nil, notFound(err, "No job has this ID.") } - rc, live, err := h.d.Logs.JobLog(ctx, in.ID, in.Step, in.Follow) + rc, err := h.d.Logs.Job(ctx, in.ID, in.Step, in.Follow) if err != nil { - return nil, h.serviceErr(err) - } - resp := h.stream(rc, "text/plain; charset=utf-8", in.Follow) - if live && in.Step > 0 { - send := resp.Body - resp.Body = func(hctx huma.Context) { - hctx.SetHeader(stepFilterHeader, "ignored") - send(hctx) - } + return nil, h.logErr(err) } - return resp, nil + return h.stream(rc, "text/plain; charset=utf-8", in.Follow), nil } -// stepFilterHeader tells that the job log has all steps although the -// request asked for one: the live log of a running job has no step marks. -const stepFilterHeader = "X-Runplane-Step-Filter" - type Overview struct { - Range string `json:"range" enum:"24h,7d,30d"` - Stats OverviewStats `json:"stats"` - Running []JobSummary `json:"running" doc:"All running jobs, newest first."` - Servers []ServerSummary `json:"servers" doc:"The primary first."` - Buckets []OverviewBucket `json:"buckets" doc:"Completed jobs by queue time, oldest first, in UTC: 24 buckets of 1 hour for 24h, 28 of 6 hours for 7d, 30 of 1 day for 30d. The last bucket ends after now."` - Recent []JobSummary `json:"recent" doc:"The last 10 completed jobs, newest first."` + Range string `json:"range" enum:"24h,7d,30d"` + BucketMS int64 `json:"bucket_ms" doc:"Length of each bucket: 1 hour for 24h, 6 hours for 7d, 1 day for 30d."` + Stats OverviewStats `json:"stats"` + Running []JobSummary `json:"running" doc:"All running jobs, newest first."` + Servers []ServerSummary `json:"servers" doc:"The primary first."` + Buckets []OverviewBucket `json:"buckets" doc:"Completed jobs by queue time, oldest first, in UTC: 24 buckets of 1 hour for 24h, 28 of 6 hours for 7d, 30 of 1 day for 30d. The last bucket ends after now."` + Recent []JobSummary `json:"recent" doc:"The last 10 completed jobs, newest first."` } // OverviewStats cover the jobs queued in the time of the buckets. type OverviewStats struct { Jobs int `json:"jobs"` - VCPUInUse int `json:"vcpu_in_use" doc:"vCPUs of the live sandboxes on the online and draining servers."` - VCPUTotal int `json:"vcpu_total" doc:"vCPUs of the online and draining servers."` + VCPUInUse int `json:"vcpu_in_use" doc:"vCPUs of the live sandboxes on the online, draining, and disabled servers."` + VCPUTotal int `json:"vcpu_total" doc:"vCPUs of the online, draining, and disabled servers."` MedianQueueMS *int64 `json:"median_queue_ms" doc:"Median time from queued to assigned. Null when no job of the range has a runner yet."` - FailureRate float64 `json:"failure_rate" doc:"failed / (succeeded + failed), 0 to 1."` + FailureRate *float64 `json:"failure_rate" doc:"failed / (succeeded + failed), 0 to 1. Null when no job of the range succeeded or failed."` CacheHitRate *float64 `json:"cache_hit_rate" doc:"Actions cache hits / (hits + misses), 0 to 1. Null when the range has no cache lookups."` DurationMS int64 `json:"duration_ms" doc:"Sum of the run times of the completed jobs."` } @@ -444,21 +436,14 @@ func (h *Handler) getOverview(ctx context.Context, in *overviewInput) (*body[Ove if err != nil { return nil, err } - queueTimes, cacheLookups, err := h.d.Store.OverviewSamples(ctx, since, until) - if err != nil { - return nil, err - } out := Overview{ - Range: in.Range, - Stats: OverviewStats{Jobs: o.Jobs, FailureRate: o.FailureRate, DurationMS: o.DurationMS}, + Range: in.Range, BucketMS: rg.bucket.Milliseconds(), + Stats: OverviewStats{ + Jobs: o.Jobs, MedianQueueMS: o.MedianQueueMS, FailureRate: o.FailureRate, CacheHitRate: o.CacheHitRate, + DurationMS: o.DurationMS, + }, Buckets: make([]OverviewBucket, len(o.Buckets)), } - if queueTimes > 0 { - out.Stats.MedianQueueMS = &o.MedianQueueMS - } - if cacheLookups > 0 { - out.Stats.CacheHitRate = &o.CacheHitRate - } for i, x := range o.Buckets { out.Buckets[i] = OverviewBucket(x) } @@ -480,8 +465,10 @@ func (h *Handler) getOverview(ctx context.Context, in *overviewInput) (*body[Ove for i, v := range servers { out.Servers[i] = v.ServerSummary // An unreachable server keeps its sandbox rows until it connects - // again, so its sandboxes count only with its vCPUs. - if v.Status == store.ServerOnline || v.Status == store.ServerDraining { + // again, so its sandboxes count only with its vCPUs. A draining or + // disabled server stays connected and finishes its jobs. + switch v.Status { + case store.ServerOnline, store.ServerDraining, store.ServerDisabled: out.Stats.VCPUInUse += v.VCPUInUse out.Stats.VCPUTotal += v.VCPU } diff --git a/internal/api/jobs_test.go b/internal/api/jobs_test.go index 3f28c70..50c904f 100644 --- a/internal/api/jobs_test.go +++ b/internal/api/jobs_test.go @@ -3,7 +3,6 @@ package api import ( "context" "fmt" - "io" "slices" "strings" "testing" @@ -38,8 +37,8 @@ func TestJobPages(t *testing.T) { sizes = append(sizes, len(page.Items)) for _, j := range page.Items { got = append(got, j.ID) - if j.WorkflowFile != ".github/workflows/ci.yml" || j.State != "queued" || j.Result != nil { - t.Fatalf("job %+v: wrong workflow file, state, or result", j) + if j.WorkflowFile != ".github/workflows/ci.yml" || j.State != "queued" || j.Status != store.StatusQueued { + t.Fatalf("job %+v: wrong workflow file, state, or status", j) } } if page.NextCursor == "" { @@ -70,8 +69,8 @@ func TestJobShowsItsDebugSandbox(t *testing.T) { j := e.addJob(1, "acme/api", time.Now()) const id = "01jbz00000000000000000debg" e.addSandbox(id, sandbox.StateBusy) - must(t, e.st.SetJobSandbox(ctx, j.ID, id, "s")) - must(t, e.st.SetSandboxJob(ctx, id, j.ID)) + _, _, err = e.st.LinkJobSandbox(ctx, j.ID, id) + must(t, err) var got Job b.do("GET", "/api/jobs/"+j.ID, nil, &got).want(t, 200) @@ -107,12 +106,23 @@ func TestJobShowsItsDebugSandbox(t *testing.T) { func TestSSHPortComesFromTheListenAddress(t *testing.T) { e := newEnv(t, func(d *Deps) { d.Config.SSHListen = "0.0.0.0:2022" }) - var st Status - e.browser().do("GET", "/api/status", nil, &st).want(t, 200) - var settings Settings - e.login("ada@example.com", auth.RoleMember).do("GET", "/api/settings", nil, &settings).want(t, 200) - if st.SSHPort != 2022 || settings.SSHPort != 2022 { - t.Fatalf("ssh_port in status %d, in settings %d; want 2022", st.SSHPort, settings.SSHPort) + ctx := context.Background() + _, err := e.st.UpdateSettings(ctx, func(s *store.Settings) error { + s.PublicURL = "https://ci.example.com" + return nil + }) + must(t, err) + j := e.addJob(1, "acme/api", time.Now()) + const id = "01jbz00000000000000000debg" + e.addSandbox(id, sandbox.StateBusy) + _, _, err = e.st.LinkJobSandbox(ctx, j.ID, id) + must(t, err) + must(t, e.st.SetSandboxDebug(ctx, id)) + + var got Job + e.login("ada@example.com", auth.RoleMember).do("GET", "/api/jobs/"+j.ID, nil, &got).want(t, 200) + if want := "ssh -p 2022 " + id + "@ci.example.com"; got.SSHCommand != want { + t.Fatalf("ssh command %q, want %q", got.SSHCommand, want) } } @@ -122,10 +132,10 @@ func TestJobLinksToItsRunBeforeGitHubGivesTheJob(t *testing.T) { ctx := context.Background() job := func(id, runID int64, enrich *store.JobEnrichment) string { j, err := e.st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: id, State: store.JobQueued, Repository: "acme/api", JobName: "test", - WorkflowRef: "acme/api/.github/workflows/ci.yml@refs/heads/main", RunID: runID, QueuedAt: time.Now()}) + WorkflowRef: "acme/api/.github/workflows/ci.yml@refs/heads/main", RunID: runID, QueuedAt: time.Now()}, time.Now()) must(t, err) if enrich != nil { - must(t, e.st.SetJobEnrichment(ctx, j.ID, *enrich)) + must(t, e.st.SetJobEnrichment(ctx, j.ID, *enrich, time.Now())) } return j.ID } @@ -146,33 +156,19 @@ func TestJobLinksToItsRunBeforeGitHubGivesTheJob(t *testing.T) { } } -// The live log of a running job has no step marks. A request for one step -// gets the whole log and a header that says so. -func TestLiveJobLogSaysThatTheStepFilterIsOff(t *testing.T) { - for _, tc := range []struct { - name string - live bool - query string - header string - }{ - {"live log, one step", true, "?step=2", "ignored"}, - {"live log, all steps", true, "", ""}, - {"stored log, one step", false, "?step=2", ""}, - } { - t.Run(tc.name, func(t *testing.T) { - e := newEnv(t, func(d *Deps) { - d.Logs = fakeLogs{log: io.NopCloser(strings.NewReader("line 1\n")), live: tc.live} - }) - j := e.addJob(1, "acme/api", time.Now()) - res := e.login("bob@example.com", auth.RoleMember).do("GET", "/api/jobs/"+j.ID+"/log"+tc.query, nil, nil) - res.want(t, 200) - if got := res.Header.Get("X-Runplane-Step-Filter"); got != tc.header { - t.Errorf("X-Runplane-Step-Filter = %q, want %q", got, tc.header) - } - if string(res.body) != "line 1\n" { - t.Errorf("body %q, want the log", res.body) - } - }) +// The web shows the status of the API, so a job that GitHub assigned must +// have the status of the filter that lists it. +func TestAssignedJobHasTheQueuedStatus(t *testing.T) { + e := newEnv(t) + now := time.Now() + _, err := e.st.UpsertJob(context.Background(), store.JobMessage{ + RunnerRequestID: 1, State: store.JobAssigned, Repository: "acme/api", QueuedAt: now, AssignedAt: now, + }, now) + must(t, err) + var page JobPage + e.login("ada@example.com", auth.RoleMember).do("GET", "/api/jobs?status=queued", nil, &page).want(t, 200) + if len(page.Items) != 1 || page.Items[0].Status != store.StatusQueued { + t.Errorf("queued jobs %+v, want the assigned job with the status queued", page.Items) } } @@ -185,13 +181,13 @@ func TestOverviewHasNoRatesWithoutData(t *testing.T) { res := b.do("GET", "/api/overview", nil, nil) res.want(t, 200) - for _, want := range []string{`"median_queue_ms":null`, `"cache_hit_rate":null`} { + for _, want := range []string{`"median_queue_ms":null`, `"failure_rate":null`, `"cache_hit_rate":null`} { if !strings.Contains(string(res.body), want) { - t.Errorf("overview without assigned jobs and cache lookups has no %s: %s", want, res.body) + t.Errorf("overview without assigned, finished, and cached jobs has no %s: %s", want, res.body) } } - _, err := e.st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: 1, State: store.JobAssigned, AssignedAt: now}) + _, err := e.st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: 1, State: store.JobAssigned, AssignedAt: now}, now) must(t, err) must(t, e.st.AddCacheStats(ctx, now, "acme/api", store.CacheCounts{Misses: 4})) var o Overview @@ -203,15 +199,17 @@ func TestOverviewHasNoRatesWithoutData(t *testing.T) { // An unreachable agent keeps its sandbox rows until it connects again. Its // sandboxes must not count against the vCPUs of the servers that are left. +// A disabled server still runs its jobs. func TestOverviewVCPUsSkipUnreachableServers(t *testing.T) { e := newEnv(t) b := e.login("ada@example.com", auth.RoleMember) e.addServer("online", "x64", store.ServerOnline, 8, 16384) e.addServer("lost", "x64", store.ServerUnreachable, 16, 32768) + e.addServer("off", "x64", store.ServerDisabled, 4, 8192) for i, x := range []struct { server string vcpu int - }{{"online", 2}, {"lost", 4}, {"lost", 4}} { + }{{"online", 2}, {"lost", 4}, {"lost", 4}, {"off", 1}} { must(t, e.st.InsertSandbox(context.Background(), store.Sandbox{ ID: fmt.Sprint("box", i), PoolID: "p", ServerID: x.server, Mode: sandbox.ModeContainer, State: sandbox.StateBusy, VCPU: x.vcpu, MemoryMB: 4096, CreatedAt: time.Now(), @@ -219,7 +217,7 @@ func TestOverviewVCPUsSkipUnreachableServers(t *testing.T) { } var o Overview b.do("GET", "/api/overview", nil, &o).want(t, 200) - if o.Stats.VCPUInUse != 2 || o.Stats.VCPUTotal != 8 { - t.Fatalf("vCPUs %d / %d, want 2 / 8", o.Stats.VCPUInUse, o.Stats.VCPUTotal) + if o.Stats.VCPUInUse != 3 || o.Stats.VCPUTotal != 12 { + t.Fatalf("vCPUs %d / %d, want 3 / 12", o.Stats.VCPUInUse, o.Stats.VCPUTotal) } } diff --git a/internal/api/maintenance.go b/internal/api/maintenance.go index 6092435..8e1927b 100644 --- a/internal/api/maintenance.go +++ b/internal/api/maintenance.go @@ -1,19 +1,19 @@ package api import ( - "cmp" "context" + "errors" "net/http" - "os" "path/filepath" - "strings" "time" "github.com/danielgtaylor/huma/v2" "github.com/danielgtaylor/huma/v2/adapters/humago" - "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/backup" "github.com/runplane/runplane/internal/store" + "github.com/runplane/runplane/internal/update" ) type Backup struct { @@ -21,16 +21,15 @@ type Backup struct { CreatedAt time.Time `json:"created_at"` SizeBytes int64 `json:"size_bytes"` S3Key string `json:"s3_key" doc:"Object key in the S3 bucket. Empty if not uploaded."` - Status string `json:"status" enum:"running,done,uploading,uploaded,failed"` + Status string `json:"status" enum:"running,done,uploading,uploaded,failed" doc:"failed: the backup or its upload to S3 failed, and error tells why. After a failed upload, the file stays on this server."` Error string `json:"error"` Available bool `json:"available" doc:"The file is on this server, so it can be downloaded."` } func (h *Handler) backupOut(b store.Backup) Backup { - _, err := os.Stat(b.Path) return Backup{ ID: b.ID, CreatedAt: b.CreatedAt, SizeBytes: b.SizeBytes, S3Key: b.S3Key, Status: b.Status, Error: b.Error, - Available: err == nil && h.inBackupsDir(b.Path), + Available: h.d.Backups.Available(b), } } @@ -47,54 +46,37 @@ func (h *Handler) listBackups(ctx context.Context, _ *struct{}) (*body[[]Backup] } func (h *Handler) createBackup(ctx context.Context, _ *struct{}) (*body[Backup], error) { - if h.d.Backups == nil { - return nil, unavailable("Backups") - } - id, err := h.d.Backups.Run(ctx) + id, err := h.d.Backups.Create(ctx) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } b, err := h.d.Store.GetBackup(ctx, id) if err != nil { return nil, notFound(err, "The new backup is gone.") } - h.audit(ctx, "backup.create", b.ID, nil) - h.d.Bus.Publish(events.Event{Type: events.BackupState, Data: b}) + h.record(ctx, audit.BackupCreate, b.ID, nil) return &body[Backup]{h.backupOut(b)}, nil } func (h *Handler) downloadBackup(ctx context.Context, in *PathID) (*huma.StreamResponse, error) { - b, err := h.d.Store.GetBackup(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No backup has this ID.") - } - if !h.inBackupsDir(b.Path) { - return nil, huma.Error404NotFound("The backup file is not on this server.") - } - f, err := os.Open(b.Path) - if os.IsNotExist(err) { + b, f, err := h.d.Backups.Open(ctx, in.ID) + switch { + case errors.Is(err, backup.ErrNoFile): return nil, huma.Error404NotFound("The backup file is not on this server.") + case err != nil: + return nil, notFound(err, "No backup has this ID.") } - if err != nil { - return nil, err - } - h.audit(ctx, "backup.download", b.ID, nil) + h.record(ctx, audit.BackupDownload, b.ID, nil) return &huma.StreamResponse{Body: func(hctx huma.Context) { defer f.Close() r, w := humago.Unwrap(hctx) - name := filepath.Base(b.Path) + name := filepath.Base(f.Name()) w.Header().Set("Content-Type", "application/gzip") w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`) http.ServeContent(w, r, name, b.CreatedAt, f) }}, nil } -// inBackupsDir guards the download against a path outside the backups folder. -func (h *Handler) inBackupsDir(path string) bool { - rel, err := filepath.Rel(h.d.Config.BackupsDir(), path) - return err == nil && rel != "." && !strings.HasPrefix(rel, "..") && !filepath.IsAbs(rel) -} - type UpdateStatus struct { Current string `json:"current" doc:"Version of this runplane."` Latest string `json:"latest" doc:"Tag of the latest release, for example v0.5.0."` @@ -102,64 +84,47 @@ type UpdateStatus struct { URL string `json:"url" doc:"Release page."` Notes string `json:"notes" doc:"Release notes in Markdown."` PublishedAt *time.Time `json:"published_at"` - AutoUpdate bool `json:"auto_update" doc:"The auto_update setting."` State string `json:"state" enum:"idle,applying,restarting" doc:"applying: runplane downloads and installs the update. restarting: the new version starts."` } func (h *Handler) getUpdate(ctx context.Context, _ *struct{}) (*body[UpdateStatus], error) { - if h.d.Updater == nil { - return nil, unavailable("The updater") - } - st, err := h.d.Store.GetSettings(ctx) - if err != nil { - return nil, err - } u, err := h.d.Updater.Check(ctx) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) + } + out := UpdateStatus{ + Current: u.Current, Latest: u.Latest, Available: u.Available, URL: u.URL, Notes: u.Notes, + State: string(h.d.Updater.State()), + } + if !u.PublishedAt.IsZero() { + out.PublishedAt = &u.PublishedAt } - u.AutoUpdate = st.AutoUpdate - h.updateMu.Lock() - u.State = cmp.Or(h.updating, u.State, "idle") - h.updateMu.Unlock() - return &body[UpdateStatus]{u}, nil + return &body[UpdateStatus]{out}, nil } func (h *Handler) applyUpdate(ctx context.Context, _ *struct{}) (*empty, error) { - if h.d.Updater == nil { - return nil, unavailable("The updater") - } - h.updateMu.Lock() - if h.updating != "" { - h.updateMu.Unlock() - return nil, huma.Error409Conflict("An update is being installed.") - } - h.updating = "applying" - h.updateMu.Unlock() - h.audit(ctx, "update.apply", "", nil) // The update goes on when the admin closes the page. - err := h.d.Updater.Apply(context.WithoutCancel(ctx)) - next := "restarting" - if err != nil { - next = "" + u, err := h.d.Updater.Apply(context.WithoutCancel(ctx)) + switch { + case errors.Is(err, update.ErrBusy): + return nil, huma.Error409Conflict("An update is being installed.") + case errors.Is(err, update.ErrNoUpdate): + return nil, huma.Error409Conflict("runplane " + u.Current + " is the latest version.") } - h.updateMu.Lock() - h.updating = next - h.updateMu.Unlock() + // serve stops after the update, but the HTTP server waits for this + // handler before the store closes. + h.record(ctx, audit.UpdateApply, "", nil) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } return nil, nil } func (h *Handler) installScript(ctx context.Context, _ *struct{}) (*huma.StreamResponse, error) { - if h.d.InstallScript == nil { - return nil, unavailable("The install script") - } return &huma.StreamResponse{Body: func(hctx huma.Context) { r, w := humago.Unwrap(hctx) w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") w.Header().Set("Cache-Control", "no-store") - w.Write(h.d.InstallScript.Script(r)) + w.Write(h.d.InstallScript.Script(r.Host)) }}, nil } diff --git a/internal/api/notifications.go b/internal/api/notifications.go index 0e3ad48..641f41c 100644 --- a/internal/api/notifications.go +++ b/internal/api/notifications.go @@ -2,13 +2,11 @@ package api import ( "context" - "net/url" - "slices" - "strings" "time" - "github.com/runplane/runplane/internal/ids" - "github.com/runplane/runplane/internal/store" + "github.com/danielgtaylor/huma/v2" + + "github.com/runplane/runplane/internal/notify" ) type NotificationChannel struct { @@ -25,49 +23,23 @@ type NotificationChannel struct { LastError string `json:"last_error" doc:"Why the last delivery failed. Empty if it worked."` } -// channels converts store channels and adds their last delivery. -func (h *Handler) channels(ctx context.Context, list []store.NotificationChannel) ([]NotificationChannel, error) { - deliveries, err := h.d.Store.NotificationDeliveries(ctx) - if err != nil { - return nil, err +func channelOut(c notify.Channel) NotificationChannel { + return NotificationChannel{ + ID: c.ID, Name: c.Name, Kind: c.Kind, URL: c.URL, Events: orEmpty(c.Events), Enabled: c.Enabled, + CreatedAt: c.CreatedAt, UpdatedAt: c.UpdatedAt, LastSentAt: c.Last.SentAt, LastError: c.Last.Error, } - out := make([]NotificationChannel, len(list)) - for i, c := range list { - masked := "" - if plain, err := h.d.Box.Open(c.URLEnc); err == nil { - masked = maskURL(string(plain)) - } else { - h.d.Log.Warn("open notification URL", "channel", c.ID, "err", err) - } - d := deliveries[c.ID] - out[i] = NotificationChannel{ - ID: c.ID, Name: c.Name, Kind: c.Kind, URL: masked, Events: orEmpty(c.Events), Enabled: c.Enabled, - CreatedAt: c.CreatedAt, UpdatedAt: c.UpdatedAt, LastSentAt: d.SentAt, LastError: d.Error, - } - } - return out, nil -} - -// maskURL keeps the scheme, the host, and the last 4 characters. The path of -// a webhook URL is its secret. -func maskURL(raw string) string { - u, err := url.Parse(raw) - if err != nil || u.Host == "" { - return "…" - } - tail := strings.TrimRight(u.EscapedPath(), "/") - if len(tail) > 4 { - tail = tail[len(tail)-4:] - } - return u.Scheme + "://" + u.Host + "/…" + tail } func (h *Handler) listNotifications(ctx context.Context, _ *struct{}) (*body[[]NotificationChannel], error) { - list, err := h.d.Store.ListNotificationChannels(ctx) + list, err := h.d.Channels.List(ctx) if err != nil { return nil, err } - return reply(h.channels(ctx, list)) + out := make([]NotificationChannel, len(list)) + for i, c := range list { + out[i] = channelOut(c) + } + return &body[[]NotificationChannel]{out}, nil } type CreateNotificationRequest struct { @@ -80,111 +52,53 @@ type CreateNotificationRequest struct { func (h *Handler) createNotification(ctx context.Context, in *body[CreateNotificationRequest]) (*body[NotificationChannel], error) { b := in.Body - name, rawURL := strings.TrimSpace(b.Name), strings.TrimSpace(b.URL) - var errs problems - if name == "" { - errs.add("body.name", "Enter a name.", b.Name) - } - checkChannelURL(&errs, b.Kind, rawURL) - if err := errs.err(); err != nil { - return nil, err - } - now := time.Now() - c := store.NotificationChannel{ - ID: ids.New(), Name: name, Kind: b.Kind, URLEnc: h.d.Box.Seal([]byte(rawURL)), Events: dedupe(b.Events), - Enabled: b.Enabled == nil || *b.Enabled, CreatedAt: now, UpdatedAt: now, - } - if err := h.d.Store.CreateNotificationChannel(ctx, c); err != nil { + c, err := h.d.Channels.Create(ctx, actor(ctx), notify.ChannelChange{ + Name: &b.Name, Kind: &b.Kind, URL: &b.URL, Events: &b.Events, Enabled: b.Enabled, + }) + if err != nil { return nil, err } - h.audit(ctx, "notification.create", c.ID, map[string]any{"name": c.Name, "kind": c.Kind}) - return reply(first(h.channels(ctx, []store.NotificationChannel{c}))) + return &body[NotificationChannel]{channelOut(c)}, nil } +// UpdateNotificationRequest has the limits of CreateNotificationRequest. type UpdateNotificationRequest struct { - Name *string `json:"name,omitempty" minLength:"1" maxLength:"64"` - Kind *string `json:"kind,omitempty" enum:"slack,discord,webhook"` - URL *string `json:"url,omitempty" minLength:"1" maxLength:"2048" doc:"Absent: keep the URL."` - Events *[]string `json:"events,omitempty" maxItems:"16" enum:"job.failed,server.unreachable,disk.low,runner.deprecation,update.available,backup.failed"` + Name *string `json:"name,omitempty"` + Kind *string `json:"kind,omitempty"` + URL *string `json:"url,omitempty" doc:"Absent: keep the URL."` + Events *[]string `json:"events,omitempty"` Enabled *bool `json:"enabled,omitempty"` } +func (UpdateNotificationRequest) TransformSchema(r huma.Registry, s *huma.Schema) *huma.Schema { + return patchOf[CreateNotificationRequest](r, s) +} + func (h *Handler) updateNotification(ctx context.Context, in *struct { PathID Body UpdateNotificationRequest }) (*body[NotificationChannel], error) { - c, err := h.d.Store.GetNotificationChannel(ctx, in.ID) + c, err := h.d.Channels.Update(ctx, actor(ctx), in.ID, notify.ChannelChange(in.Body)) if err != nil { return nil, notFound(err, "No notification channel has this ID.") } - b := in.Body - var errs problems - if b.Name != nil { - if c.Name = strings.TrimSpace(*b.Name); c.Name == "" { - errs.add("body.name", "Enter a name.", *b.Name) - } - } - set(&c.Kind, b.Kind) - set(&c.Enabled, b.Enabled) - if b.Events != nil { - c.Events = dedupe(*b.Events) - } - // The URL rules depend on the kind, so check the URL again when the - // kind changes. - rawURL := "" - if b.URL != nil { - rawURL = strings.TrimSpace(*b.URL) - } else if plain, err := h.d.Box.Open(c.URLEnc); err == nil { - rawURL = string(plain) - } - checkChannelURL(&errs, c.Kind, rawURL) - if err := errs.err(); err != nil { - return nil, err - } - if b.URL != nil { - c.URLEnc = h.d.Box.Seal([]byte(rawURL)) - } - if err := h.d.Store.UpdateNotificationChannel(ctx, c); err != nil { - return nil, notFound(err, "No notification channel has this ID.") - } - h.audit(ctx, "notification.update", c.ID, map[string]any{"name": c.Name, "kind": c.Kind, "url_changed": b.URL != nil}) - return reply(first(h.channels(ctx, []store.NotificationChannel{c}))) + return &body[NotificationChannel]{channelOut(c)}, nil } func (h *Handler) deleteNotification(ctx context.Context, in *PathID) (*empty, error) { - if err := h.d.Store.DeleteNotificationChannel(ctx, in.ID); err != nil { + if err := h.d.Channels.Delete(ctx, actor(ctx), in.ID); err != nil { return nil, notFound(err, "No notification channel has this ID.") } - h.audit(ctx, "notification.delete", in.ID, nil) return nil, nil } func (h *Handler) testNotification(ctx context.Context, in *PathID) (*empty, error) { - if h.d.Notifier == nil { - return nil, unavailable("Notifications") - } - c, err := h.d.Store.GetNotificationChannel(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No notification channel has this ID.") - } - plain, err := h.d.Box.Open(c.URLEnc) - if err != nil { - return nil, err - } base, err := h.baseURL(ctx) if err != nil { return nil, err } - sendErr := h.d.Notifier.Test(ctx, base, c.Kind, string(plain)) - msg := "" - if sendErr != nil { - msg = sendErr.Error() - } - if err := h.d.Store.RecordNotificationDelivery(ctx, c.ID, time.Now(), msg); err != nil { - h.d.Log.Warn("record notification delivery", "channel", c.ID, "err", err) - } - if sendErr != nil { - return nil, h.serviceErr(sendErr) + if err := h.d.Channels.Test(ctx, in.ID, base); err != nil { + return nil, notFound(serviceErr(err), "No notification channel has this ID.") } return nil, nil } @@ -195,41 +109,9 @@ type TestChannelRequest struct { } func (h *Handler) testChannel(ctx context.Context, in *body[TestChannelRequest]) (*empty, error) { - rawURL := strings.TrimSpace(in.Body.URL) - var errs problems - checkChannelURL(&errs, in.Body.Kind, rawURL) - if err := errs.err(); err != nil { - return nil, err - } - if h.d.Notifier == nil { - return nil, unavailable("Notifications") - } base, err := h.baseURL(ctx) if err != nil { return nil, err } - if err := h.d.Notifier.Test(ctx, base, in.Body.Kind, rawURL); err != nil { - return nil, h.serviceErr(err) - } - return nil, nil -} - -func checkChannelURL(errs *problems, kind, raw string) { - u, err := url.Parse(raw) - switch { - case err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https"): - errs.add("body.url", "Enter an http or https URL.", nil) - case kind != "webhook" && u.Scheme != "https": - errs.add("body.url", "A "+kind+" webhook URL starts with https://.", nil) - } -} - -func dedupe(list []string) []string { - out := []string{} - for _, v := range list { - if !slices.Contains(out, v) { - out = append(out, v) - } - } - return out + return nil, serviceErr(h.d.Channels.TestURL(ctx, in.Body.Kind, in.Body.URL, base)) } diff --git a/internal/api/pools.go b/internal/api/pools.go index d930e06..e9eb72c 100644 --- a/internal/api/pools.go +++ b/internal/api/pools.go @@ -4,13 +4,10 @@ import ( "cmp" "context" "errors" - "fmt" - "runtime" - "slices" - "strings" "time" - "github.com/runplane/runplane/internal/ids" + "github.com/danielgtaylor/huma/v2" + "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" ) @@ -33,11 +30,7 @@ type Pool struct { CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` - Idle int `json:"idle" doc:"Live count of idle sandboxes."` - Busy int `json:"busy" doc:"Live count of sandboxes that run a job."` - Starting int `json:"starting" doc:"Live count of sandboxes that start."` - Queued int `json:"queued" doc:"Jobs of the last 24 hours that wait for a runner."` - + store.PoolCounts PoolPlacement } @@ -46,7 +39,8 @@ type Pool struct { // resources now. type PoolPlacement struct { OnlineServers int `json:"online_servers" doc:"Online servers with the arch, limited to server_ids if the pool has them."` - PlaceableServers int `json:"placeable_servers" doc:"Of online_servers, the servers whose vCPUs and memory, after the reserve and the CPU overcommit, fit one sandbox. 0 means that no sandbox can start."` + WaitingServers int `json:"waiting_servers" doc:"Of online_servers, the servers that get no runners now. wait_reason of each server tells why."` + PlaceableServers int `json:"placeable_servers" doc:"Of online_servers, the servers that get runners and whose vCPUs and memory, after the reserve, the CPU overcommit, and the limits of the driver, fit one sandbox. 0 means that no sandbox can start."` PlaceableServerIDs []string `json:"placeable_server_ids"` } @@ -68,43 +62,18 @@ func (h *Handler) pools(ctx context.Context, list []store.Pool) ([]Pool, error) defaultImage = img.Name } } - counts, err := h.d.Store.CountLiveSandboxes(ctx) - if err != nil { - return nil, err - } - queued, _, err := h.d.Store.ListJobs(ctx, store.JobFilter{Status: "queued", Since: time.Now().Add(-24 * time.Hour), Limit: 200}) + counts, err := h.d.Store.PoolCounts(ctx) if err != nil { return nil, err } out := make([]Pool, len(list)) - byID := map[string]*Pool{} for i, p := range list { out[i] = Pool{ ID: p.ID, Name: p.Name, Label: p.Label, Arch: p.Arch, VCPU: p.VCPU, MemoryMB: p.MemoryMB, DiskGB: p.DiskGB, ImageID: p.ImageID, ImageName: cmp.Or(imageNames[p.ImageID], defaultImage), MinIdle: p.MinIdle, MaxRunners: p.MaxRunners, TimeLimitMinutes: p.TimeLimitMinutes, ServerIDs: orEmpty(p.ServerIDs), Enabled: p.Enabled, CreatedAt: p.CreatedAt, UpdatedAt: p.UpdatedAt, - PoolPlacement: placer.place(p.Arch, p.VCPU, p.MemoryMB, p.ServerIDs), - } - byID[p.ID] = &out[i] - } - for _, c := range counts { - p := byID[c.PoolID] - if p == nil { - continue - } - switch c.State { - case sandbox.StateIdle: - p.Idle += c.Count - case sandbox.StateBusy: - p.Busy += c.Count - case sandbox.StateStarting: - p.Starting += c.Count - } - } - for _, j := range queued { - if p := byID[j.PoolID]; p != nil && j.RejectReason == "" { - p.Queued++ + PoolCounts: counts[p.ID], PoolPlacement: placer.place(p), } } return out, nil @@ -126,12 +95,18 @@ func (h *Handler) getPool(ctx context.Context, in *PathID) (*body[Pool], error) return reply(first(h.pools(ctx, []store.Pool{p}))) } +// PoolSize is the size of the sandboxes of a pool. The pool requests and the +// placement query have it, with the same limits. +type PoolSize struct { + VCPU int `json:"vcpu" query:"vcpu" required:"true" minimum:"1" maximum:"64"` + MemoryMB int `json:"memory_mb" query:"memory_mb" required:"true" minimum:"512" maximum:"262144"` +} + type CreatePoolRequest struct { - Name string `json:"name,omitempty" maxLength:"64" doc:"Defaults to the label."` - Label string `json:"label" pattern:"^[a-z0-9][a-z0-9-]{0,62}$" patternDescription:"lowercase letters, digits, and dashes" doc:"The runs-on label. Unique."` - Arch string `json:"arch,omitempty" enum:"x64,arm64" doc:"Defaults to the arch of the primary server."` - VCPU int `json:"vcpu" minimum:"1" maximum:"64"` - MemoryMB int `json:"memory_mb" minimum:"512" maximum:"262144"` + Name string `json:"name,omitempty" maxLength:"64" doc:"Defaults to the label."` + Label string `json:"label" maxLength:"63" pattern:"^[a-z0-9][a-z0-9-]{0,62}$" patternDescription:"lowercase letters, digits, and dashes" doc:"The runs-on label. Unique."` + Arch string `json:"arch,omitempty" enum:"x64,arm64" doc:"Defaults to the arch of the primary server."` + PoolSize DiskGB int `json:"disk_gb" minimum:"8" maximum:"1024"` ImageID string `json:"image_id,omitempty" maxLength:"100" doc:"Empty: the default image."` MinIdle int `json:"min_idle,omitempty" minimum:"0" maximum:"1000"` @@ -143,169 +118,72 @@ type CreatePoolRequest struct { func (h *Handler) createPool(ctx context.Context, in *body[CreatePoolRequest]) (*body[Pool], error) { b := in.Body - now := time.Now() - p := store.Pool{ - ID: ids.New(), Name: cmp.Or(strings.TrimSpace(b.Name), b.Label), Label: b.Label, Arch: b.Arch, - VCPU: b.VCPU, MemoryMB: b.MemoryMB, DiskGB: b.DiskGB, ImageID: b.ImageID, MinIdle: b.MinIdle, - MaxRunners: b.MaxRunners, TimeLimitMinutes: b.TimeLimitMinutes, ServerIDs: orEmpty(b.ServerIDs), - Enabled: b.Enabled == nil || *b.Enabled, CreatedAt: now, UpdatedAt: now, - } - if p.Arch == "" { - arch, err := h.defaultArch(ctx) - if err != nil { - return nil, err - } - p.Arch = arch - } - if err := h.checkPool(ctx, p, nil); err != nil { - return nil, err - } - if err := h.d.Store.CreatePool(ctx, p); err != nil { + p, err := h.d.Pools.Create(ctx, actor(ctx), store.Pool{ + Name: b.Name, Label: b.Label, Arch: b.Arch, VCPU: b.VCPU, MemoryMB: b.MemoryMB, DiskGB: b.DiskGB, + ImageID: b.ImageID, MinIdle: b.MinIdle, MaxRunners: b.MaxRunners, TimeLimitMinutes: b.TimeLimitMinutes, + ServerIDs: b.ServerIDs, Enabled: b.Enabled == nil || *b.Enabled, + }) + if err != nil { return nil, poolErr(err) } - h.poolsChanged(ctx) - h.audit(ctx, "pool.create", p.ID, map[string]any{"label": p.Label}) return reply(first(h.pools(ctx, []store.Pool{p}))) } +// UpdatePoolRequest has the limits of CreatePoolRequest. type UpdatePoolRequest struct { - Name *string `json:"name,omitempty" minLength:"1" maxLength:"64"` - Label *string `json:"label,omitempty" pattern:"^[a-z0-9][a-z0-9-]{0,62}$" patternDescription:"lowercase letters, digits, and dashes"` - Arch *string `json:"arch,omitempty" enum:"x64,arm64"` - VCPU *int `json:"vcpu,omitempty" minimum:"1" maximum:"64"` - MemoryMB *int `json:"memory_mb,omitempty" minimum:"512" maximum:"262144"` - DiskGB *int `json:"disk_gb,omitempty" minimum:"8" maximum:"1024"` - ImageID *string `json:"image_id,omitempty" maxLength:"100" doc:"Empty: the default image."` - MinIdle *int `json:"min_idle,omitempty" minimum:"0" maximum:"1000"` - MaxRunners *int `json:"max_runners,omitempty" minimum:"1" maximum:"1000"` - TimeLimitMinutes *int `json:"time_limit_minutes,omitempty" minimum:"0" maximum:"7200"` - ServerIDs *[]string `json:"server_ids,omitempty" maxItems:"100"` + Name *string `json:"name,omitempty"` + Label *string `json:"label,omitempty"` + Arch *string `json:"arch,omitempty"` + VCPU *int `json:"vcpu,omitempty"` + MemoryMB *int `json:"memory_mb,omitempty"` + DiskGB *int `json:"disk_gb,omitempty"` + ImageID *string `json:"image_id,omitempty" doc:"Empty: the default image."` + MinIdle *int `json:"min_idle,omitempty"` + MaxRunners *int `json:"max_runners,omitempty"` + TimeLimitMinutes *int `json:"time_limit_minutes,omitempty"` + ServerIDs *[]string `json:"server_ids,omitempty"` Enabled *bool `json:"enabled,omitempty"` } +func (UpdatePoolRequest) TransformSchema(r huma.Registry, s *huma.Schema) *huma.Schema { + return patchOf[CreatePoolRequest](r, s) +} + func (h *Handler) updatePool(ctx context.Context, in *struct { PathID Body UpdatePoolRequest }) (*body[Pool], error) { - p, err := h.d.Store.GetPool(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No pool has this ID.") - } - old := p b := in.Body - set(&p.Name, b.Name) - set(&p.Label, b.Label) - set(&p.Arch, b.Arch) - set(&p.VCPU, b.VCPU) - set(&p.MemoryMB, b.MemoryMB) - set(&p.DiskGB, b.DiskGB) - set(&p.ImageID, b.ImageID) - set(&p.MinIdle, b.MinIdle) - set(&p.MaxRunners, b.MaxRunners) - set(&p.TimeLimitMinutes, b.TimeLimitMinutes) - set(&p.ServerIDs, b.ServerIDs) - set(&p.Enabled, b.Enabled) - p.Name = strings.TrimSpace(p.Name) - if err := h.checkPool(ctx, p, old.ServerIDs); err != nil { - return nil, err - } - if err := h.d.Store.UpdatePool(ctx, p); err != nil { + p, err := h.d.Pools.Update(ctx, actor(ctx), in.ID, func(p *store.Pool) { + set(&p.Name, b.Name) + set(&p.Label, b.Label) + set(&p.Arch, b.Arch) + set(&p.VCPU, b.VCPU) + set(&p.MemoryMB, b.MemoryMB) + set(&p.DiskGB, b.DiskGB) + set(&p.ImageID, b.ImageID) + set(&p.MinIdle, b.MinIdle) + set(&p.MaxRunners, b.MaxRunners) + set(&p.TimeLimitMinutes, b.TimeLimitMinutes) + set(&p.ServerIDs, b.ServerIDs) + set(&p.Enabled, b.Enabled) + }) + if err != nil { return nil, poolErr(err) } - h.poolsChanged(ctx) - if changes := h.poolChanges(ctx, old, p); len(changes) > 0 { - h.audit(ctx, "pool.update", p.ID, map[string]any{"label": p.Label, "changes": changes}) - } return reply(first(h.pools(ctx, []store.Pool{p}))) } -// poolChanges gives the old and new value of each changed field of a pool, -// with the names of the image and the servers in place of their IDs. An -// image or server that runplane cannot name keeps its ID. An empty image is -// the default image. -func (h *Handler) poolChanges(ctx context.Context, old, p store.Pool) map[string][2]any { - old.ServerIDs, p.ServerIDs = orEmpty(old.ServerIDs), orEmpty(p.ServerIDs) - changes := fieldChanges(old, p, "updated_at") - if _, ok := changes["image_id"]; ok { - names := map[string]string{} - images, err := h.d.Store.ListImages(ctx) - if err != nil { - h.d.Log.Warn("read the image names for the audit log", "err", err) - } - for _, img := range images { - names[img.ID] = img.Name - } - delete(changes, "image_id") - changes["image"] = [2]any{cmp.Or(names[old.ImageID], old.ImageID), cmp.Or(names[p.ImageID], p.ImageID)} - } - if _, ok := changes["server_ids"]; ok { - r, err := h.loadRefs(ctx) - if err != nil { - h.d.Log.Warn("read the server names for the audit log", "err", err) - } - names := func(ids []string) []string { - out := make([]string, len(ids)) - for i, id := range ids { - out[i] = cmp.Or(r.servers[id].Name, id) - } - return out - } - delete(changes, "server_ids") - changes["servers"] = [2]any{names(old.ServerIDs), names(p.ServerIDs)} - } - return changes -} - func (h *Handler) deletePool(ctx context.Context, in *PathID) (*empty, error) { - p, err := h.d.Store.GetPool(ctx, in.ID) - if err != nil { + if err := h.d.Pools.Delete(ctx, actor(ctx), in.ID); err != nil { return nil, notFound(err, "No pool has this ID.") } - // The scheduler stops the sandboxes of the pool and the GitHub code - // deletes its scale sets when they see that the pool is gone. - if err := h.d.Store.DeletePool(ctx, p.ID); err != nil { - return nil, notFound(err, "No pool has this ID.") - } - h.poolsChanged(ctx) - h.audit(ctx, "pool.delete", p.ID, map[string]any{"label": p.Label}) return nil, nil } -// checkPool checks the rules that need more than one field or the database. -// saved holds the server IDs of the stored pool: an unknown ID among them is -// from a server that no longer exists, and it does not block the change. -func (h *Handler) checkPool(ctx context.Context, p store.Pool, saved []string) error { - var errs problems - if p.Name == "" { - errs.add("body.name", "Enter a name.", p.Name) - } - if p.MinIdle > p.MaxRunners { - errs.add("body.min_idle", "Warm runners cannot be more than max runners.", p.MinIdle) - } - if p.ImageID != "" { - if _, err := h.d.Store.GetImage(ctx, p.ImageID); errors.Is(err, store.ErrNotFound) { - errs.add("body.image_id", "No image has this ID.", p.ImageID) - } else if err != nil { - return err - } - } - for i, id := range p.ServerIDs { - if slices.Contains(saved, id) { - continue - } - if _, err := h.d.Store.GetServer(ctx, id); errors.Is(err, store.ErrNotFound) { - errs.add(fmt.Sprintf("body.server_ids[%d]", i), "No server has this ID.", id) - } else if err != nil { - return err - } - } - return errs.err() -} - type placementInput struct { - Arch string `query:"arch" required:"true" enum:"x64,arm64"` - VCPU int `query:"vcpu" required:"true" minimum:"1" maximum:"64"` - MemoryMB int `query:"memory_mb" required:"true" minimum:"512" maximum:"262144"` + Arch string `query:"arch" required:"true" enum:"x64,arm64"` + PoolSize ServerIDs []string `query:"server_ids" maxItems:"100" doc:"Comma-separated. Empty: all servers."` } @@ -316,12 +194,15 @@ func (h *Handler) getPlacement(ctx context.Context, in *placementInput) (*body[P if err != nil { return nil, err } - return &body[PoolPlacement]{placer.place(in.Arch, in.VCPU, in.MemoryMB, in.ServerIDs)}, nil + pool := store.Pool{Arch: in.Arch, VCPU: in.VCPU, MemoryMB: in.MemoryMB, ServerIDs: in.ServerIDs} + return &body[PoolPlacement]{placer.place(pool)}, nil } type placer struct { servers []store.Server settings store.Settings + // limits are the limits of the drivers of the connected servers. + limits map[string]sandbox.Limits } func (h *Handler) newPlacer(ctx context.Context) (placer, error) { @@ -333,27 +214,32 @@ func (h *Handler) newPlacer(ctx context.Context) (placer, error) { if err != nil { return placer{}, err } - p := placer{settings: settings} + p := placer{settings: settings, limits: map[string]sandbox.Limits{}} for _, v := range list { - if v.Status == store.ServerOnline { - p.servers = append(p.servers, v) + if v.Status != store.ServerOnline { + continue + } + p.servers = append(p.servers, v) + if l, ok := h.d.Servers.Limits(v.ID); ok { + p.limits[v.ID] = l } } return p, nil } -func (p placer) place(arch string, vcpu, memoryMB int, serverIDs []string) PoolPlacement { +func (p placer) place(pool store.Pool) PoolPlacement { out := PoolPlacement{PlaceableServerIDs: []string{}} for _, v := range p.servers { - if v.Arch != arch || len(serverIDs) > 0 && !slices.Contains(serverIDs, v.ID) { + if !pool.CanUse(v.ID, v.Arch) { continue } out.OnlineServers++ - c := sandbox.Capacity{ - VCPU: v.VCPU, MemoryMB: v.MemoryMB, ReserveVCPU: p.settings.ReserveVCPU, - ReserveMemoryMB: p.settings.ReserveMemoryMB, Overcommit: p.settings.CPUOvercommit, + if waitReason(p.settings, v) != "" { + out.WaitingServers++ + continue } - if c.Fits(vcpu, memoryMB) { + empty := sandbox.Capacity{VCPU: v.VCPU, MemoryMB: v.MemoryMB, Limits: p.limits[v.ID]} + if empty.Room(pool.VCPU, pool.MemoryMB, p.settings.Reserve()) > 0 { out.PlaceableServerIDs = append(out.PlaceableServerIDs, v.ID) } } @@ -367,25 +253,3 @@ func poolErr(err error) error { } return notFound(err, "No pool has this ID.") } - -func (h *Handler) poolsChanged(ctx context.Context) { - if h.d.Scheduler != nil { - h.d.Scheduler.PoolsChanged(context.WithoutCancel(ctx)) - } -} - -// defaultArch is the arch of the primary server, or of this binary before -// the primary has reported its facts. -func (h *Handler) defaultArch(ctx context.Context) (string, error) { - local, err := h.d.Store.GetLocalServer(ctx) - if err != nil && !errors.Is(err, store.ErrNotFound) { - return "", err - } - if local.Arch != "" { - return local.Arch, nil - } - if runtime.GOARCH == "arm64" { - return "arm64", nil - } - return "x64", nil -} diff --git a/internal/api/sandboxes.go b/internal/api/sandboxes.go index da703d0..dc84981 100644 --- a/internal/api/sandboxes.go +++ b/internal/api/sandboxes.go @@ -15,7 +15,10 @@ import ( "github.com/danielgtaylor/huma/v2" "github.com/danielgtaylor/huma/v2/adapters/humago" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/joblog" "github.com/runplane/runplane/internal/sandbox" + "github.com/runplane/runplane/internal/shell" "github.com/runplane/runplane/internal/store" ) @@ -27,7 +30,6 @@ type Sandbox struct { ServerName string `json:"server_name"` State string `json:"state" enum:"starting,idle,busy,stopping,gone,failed"` RunnerName string `json:"runner_name"` - IP string `json:"ip"` JobID string `json:"job_id" doc:"Empty until the runner gets a job."` JobName string `json:"job_name"` JobRepository string `json:"job_repository" doc:"owner/name of the job."` @@ -60,7 +62,7 @@ func (h *Handler) sandboxes(ctx context.Context, list []store.Sandbox) ([]Sandbo } out[i] = Sandbox{ ID: b.ID, PoolID: b.PoolID, PoolLabel: r.pools[b.PoolID].Label, ServerID: b.ServerID, - ServerName: r.servers[b.ServerID].Name, State: string(b.State), RunnerName: b.RunnerName, IP: b.IP, + ServerName: r.servers[b.ServerID].Name, State: string(b.State), RunnerName: b.RunnerName, JobID: b.JobID, JobName: j.JobName, JobRepository: j.Repository, JobWorkflowFile: j.WorkflowFile, Debug: b.Debug, HoldUntil: b.HoldUntil, VCPU: b.VCPU, MemoryMB: b.MemoryMB, Error: b.Error, CreatedAt: b.CreatedAt, IdleAt: b.IdleAt, BusyAt: b.BusyAt, StoppedAt: b.StoppedAt, @@ -84,7 +86,7 @@ func (h *Handler) listSandboxes(ctx context.Context, in *listSandboxesInput) (*b if in.State != "" { f.States = []sandbox.State{sandbox.State(in.State)} } else { - f.States = []sandbox.State{sandbox.StateStarting, sandbox.StateIdle, sandbox.StateBusy, sandbox.StateStopping} + f.States = sandbox.LiveStates() } list, err := h.d.Store.ListSandboxes(ctx, f) if err != nil { @@ -102,9 +104,6 @@ func (h *Handler) getSandbox(ctx context.Context, in *PathID) (*body[Sandbox], e } func (h *Handler) stopSandbox(ctx context.Context, in *PathID) (*empty, error) { - if h.d.Scheduler == nil { - return nil, unavailable("The scheduler") - } b, err := h.d.Store.GetSandbox(ctx, in.ID) if err != nil { return nil, notFound(err, "No sandbox has this ID.") @@ -113,9 +112,9 @@ func (h *Handler) stopSandbox(ctx context.Context, in *PathID) (*empty, error) { return nil, nil } if err := h.d.Scheduler.StopSandbox(ctx, b.ID); err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - h.audit(ctx, "sandbox.stop", b.ID, h.sandboxDetail(ctx, b)) + h.record(ctx, audit.SandboxStop, b.ID, h.sandboxDetail(ctx, b)) return nil, nil } @@ -140,19 +139,35 @@ type sandboxLogInput struct { } func (h *Handler) sandboxLog(ctx context.Context, in *sandboxLogInput) (*huma.StreamResponse, error) { - if h.d.Logs == nil { - return nil, unavailable("The sandbox log") - } if _, err := h.d.Store.GetSandbox(ctx, in.ID); err != nil { return nil, notFound(err, "No sandbox has this ID.") } - rc, err := h.d.Logs.SandboxLog(ctx, in.ID, in.Follow) + rc, err := h.d.Logs.Sandbox(ctx, in.ID, in.Follow) if err != nil { - return nil, h.serviceErr(err) + return nil, h.logErr(err) } return h.stream(rc, "text/plain; charset=utf-8", in.Follow), nil } +// logErr changes the errors of a log read into problems. +func (h *Handler) logErr(err error) error { + switch { + case errors.Is(err, joblog.ErrMissing): + return huma.Error404NotFound("The log file of this job is missing.") + case errors.Is(err, joblog.ErrExpired): + return huma.Error404NotFound("The log of this job expired.") + case errors.Is(err, joblog.ErrNoGitHubLog): + return huma.Error404NotFound("GitHub has no log for this job.") + case errors.Is(err, joblog.ErrNotYet): + return huma.Error404NotFound("This job has no log yet.") + case errors.Is(err, joblog.ErrSandboxStopped): + return huma.Error404NotFound("The sandbox stopped, and its log went with it.") + case errors.Is(err, joblog.ErrNotConnected): + return huma.Error503ServiceUnavailable("The server of this sandbox is not connected.") + } + return serviceErr(err) +} + type shellInput struct { PathID Rows uint16 `query:"rows" minimum:"1" maximum:"500" default:"24"` @@ -167,9 +182,6 @@ type shellMessage struct { } func (h *Handler) openShell(ctx context.Context, in *shellInput) (*huma.StreamResponse, error) { - if h.d.Shell == nil { - return nil, unavailable("The debug shell") - } b, err := h.d.Store.GetSandbox(ctx, in.ID) if err != nil { return nil, notFound(err, "No sandbox has this ID.") @@ -209,35 +221,31 @@ func (h *Handler) openShell(ctx context.Context, in *shellInput) (*huma.StreamRe conn.Close(websocket.StatusInternalError, closeReason(err.Error())) return } - h.audit(ctx, "sandbox.shell", b.ID, h.sandboxDetail(ctx, b)) + h.record(ctx, audit.SandboxShell, b.ID, h.sandboxDetail(ctx, b)) bridge(ctx, conn, term) }}, nil } -// shellAllowed applies the rule of the SSH gateway: only a debug sandbox -// has a shell, while its job runs or during its hold after the job. +// shellAllowed applies the rule of the debug shells to a sandbox. func shellAllowed(b store.Sandbox, now time.Time) error { - switch { - case !b.Debug: + switch (shell.Target{State: b.State, Debug: b.Debug, HoldUntil: b.HoldUntil}).Access(now) { + case shell.NotDebug: return huma.Error403Forbidden("Only a debug sandbox has a shell. Re-run the job with debug to get one.") - case b.State.Done(): + case shell.Stopped: return huma.Error409Conflict("The sandbox has stopped.") - case b.State == sandbox.StateBusy || b.HoldUntil != nil && now.Before(*b.HoldUntil): - return nil - case b.HoldUntil != nil: + case shell.HoldEnded: return huma.Error409Conflict("The debug hold of the sandbox has ended.") + case shell.NoJobYet: + return huma.Error409Conflict("The sandbox has no job yet. Open the shell when its job runs.") } - return huma.Error409Conflict("The sandbox has no job yet. Open the shell when its job runs.") + return nil } -// sshPort is the port of the SSH gateway from RUNPLANE_SSH_LISTEN, or the -// default 2222 when the config has no address. +// sshPort is the port of the SSH gateway from RUNPLANE_SSH_LISTEN. func (h *Handler) sshPort() int { _, port, _ := net.SplitHostPort(h.d.Config.SSHListen) - if n, err := strconv.Atoi(port); err == nil { - return n - } - return 2222 + n, _ := strconv.Atoi(port) + return n } func (h *Handler) sshCommand(ctx context.Context, sandboxID string) (string, error) { diff --git a/internal/api/sandboxes_test.go b/internal/api/sandboxes_test.go index cf42ef4..434338b 100644 --- a/internal/api/sandboxes_test.go +++ b/internal/api/sandboxes_test.go @@ -3,6 +3,7 @@ package api import ( "bufio" "context" + "errors" "io" "net/http" "net/url" @@ -13,21 +14,42 @@ import ( "github.com/coder/websocket" "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/ids" + "github.com/runplane/runplane/internal/joblog" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" ) // fakeShell stands in for the sandbox driver: the test plays the terminal. +// With err, no shell opens. type fakeShell struct { opened chan sandbox.TermSize term *fakeTerm + err error } func (s *fakeShell) Open(_ context.Context, _ string, size sandbox.TermSize) (sandbox.Terminal, error) { + if s.err != nil { + return nil, s.err + } s.opened <- size return s.term, nil } +// dialShell opens a shell as the dashboard does: path over WebSocket, with +// the cookies of b and the headers h. +func (b *browser) dialShell(ctx context.Context, path string, h http.Header) (*websocket.Conn, *http.Response, error) { + u, _ := url.Parse(b.e.srv.URL) + h = h.Clone() + if h == nil { + h = http.Header{} + } + for _, c := range b.client.Jar.Cookies(u) { + h.Add("Cookie", c.String()) + } + return websocket.Dial(ctx, "ws"+strings.TrimPrefix(b.e.srv.URL, "http")+path, &websocket.DialOptions{HTTPHeader: h}) +} + type fakeTerm struct { output *io.PipeReader // what the shell prints input chan string @@ -55,24 +77,17 @@ func TestShellBridgesTerminal(t *testing.T) { e.addSandbox("plain", sandbox.StateBusy) must(t, e.st.SetSandboxDebug(context.Background(), "sb1")) b := e.login("bob@example.com", auth.RoleMember) - u, _ := url.Parse(e.srv.URL) - cookie := b.client.Jar.Cookies(u)[0].String() - shellURL := "ws" + strings.TrimPrefix(e.srv.URL, "http") + "/api/sandboxes/sb1/shell" ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() // A page on another site cannot open the shell with the cookie. - _, res, err := websocket.Dial(ctx, shellURL, &websocket.DialOptions{ - HTTPHeader: http.Header{"Cookie": {cookie}, "Origin": {"https://evil.example"}}, - }) + _, res, err := b.dialShell(ctx, "/api/sandboxes/sb1/shell", http.Header{"Origin": {"https://evil.example"}}) if err == nil || res == nil || res.StatusCode != http.StatusForbidden { t.Fatalf("cross-site dial: err %v, response %v; want 403", err, res) } b.do("GET", "/api/sandboxes/plain/shell", nil, nil).want(t, http.StatusForbidden) - conn, _, err := websocket.Dial(ctx, shellURL+"?rows=30&cols=100", &websocket.DialOptions{ - HTTPHeader: http.Header{"Cookie": {cookie}}, - }) + conn, _, err := b.dialShell(ctx, "/api/sandboxes/sb1/shell?rows=30&cols=100", nil) must(t, err) defer conn.CloseNow() if got := recv(t, shell.opened); got != (sandbox.TermSize{Rows: 30, Cols: 100}) { @@ -107,48 +122,87 @@ func TestShellBridgesTerminal(t *testing.T) { } } -func TestShellAllowedOnlyForLiveDebugJobs(t *testing.T) { - now := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) - later, earlier := now.Add(time.Minute), now.Add(-time.Minute) +// The API answers the shell rule of package shell, which has its own test, +// with a status: 403 for a sandbox without debug, and 409 for a debug +// sandbox without a job that runs or holds. A plain GET of a shell that the +// rule lets open gets 426, because it is no WebSocket. +func TestShellNeedsALiveDebugJob(t *testing.T) { + e := newEnv(t) + b := e.login("bob@example.com", auth.RoleMember) + later, earlier := time.Now().Add(time.Hour), time.Now().Add(-time.Hour) for _, tc := range []struct { name string b store.Sandbox - want int // 0: allowed + want int }{ - {"busy debug sandbox", store.Sandbox{Debug: true, State: sandbox.StateBusy}, 0}, - {"held after the job", store.Sandbox{Debug: true, State: sandbox.StateStopping, HoldUntil: &later}, 0}, + {"busy debug sandbox", store.Sandbox{Debug: true, State: sandbox.StateBusy}, http.StatusUpgradeRequired}, + {"held after the job", store.Sandbox{Debug: true, State: sandbox.StateStopping, HoldUntil: &later}, http.StatusUpgradeRequired}, {"busy without debug", store.Sandbox{State: sandbox.StateBusy}, http.StatusForbidden}, {"hold ended", store.Sandbox{Debug: true, State: sandbox.StateStopping, HoldUntil: &earlier}, http.StatusConflict}, {"debug sandbox before its job", store.Sandbox{Debug: true, State: sandbox.StateIdle}, http.StatusConflict}, {"gone during its hold", store.Sandbox{Debug: true, State: sandbox.StateGone, HoldUntil: &later}, http.StatusConflict}, } { - err := shellAllowed(tc.b, now) - got := 0 - if err != nil { - got = err.(interface{ GetStatus() int }).GetStatus() - } - if got != tc.want { - t.Errorf("%s: status %d (%v), want %d", tc.name, got, err, tc.want) + sb := tc.b + sb.ID, sb.PoolID, sb.ServerID, sb.Mode = ids.New(), "p", "s", sandbox.ModeContainer + sb.VCPU, sb.MemoryMB, sb.CreatedAt = 2, 4096, time.Now() + must(t, e.st.InsertSandbox(context.Background(), sb)) + if got := b.do("GET", "/api/sandboxes/"+sb.ID+"/shell", nil, nil).StatusCode; got != tc.want { + t.Errorf("%s: status %d, want %d", tc.name, got, tc.want) } } } -func TestCloseReasonKeepsWholeCharacters(t *testing.T) { +// The browser shows why a shell did not open: the reason of the close. A +// close reason has at most 123 bytes of valid UTF-8. +func TestShellErrorKeepsWholeCharacters(t *testing.T) { // Byte 120 falls inside the 60th "é", so a cut there leaves half of it. - got := closeReason("a" + strings.Repeat("é", 100)) - if want := "a" + strings.Repeat("é", 59) + "..."; got != want { - t.Errorf("closeReason = %q, want %q", got, want) + e := newEnv(t, func(d *Deps) { d.Shell = &fakeShell{err: errors.New("a" + strings.Repeat("é", 100))} }) + e.addSandbox("sb1", sandbox.StateBusy) + must(t, e.st.SetSandboxDebug(context.Background(), "sb1")) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, _, err := e.login("bob@example.com", auth.RoleMember).dialShell(ctx, "/api/sandboxes/sb1/shell", nil) + must(t, err) + defer conn.CloseNow() + _, _, err = conn.Read(ctx) + var ce websocket.CloseError + if want := "a" + strings.Repeat("é", 59) + "..."; !errors.As(err, &ce) || ce.Code != websocket.StatusInternalError || ce.Reason != want { + t.Errorf("close %v, want status 1011 with the reason %q", err, want) } } type fakeLogs struct { - log io.ReadCloser - live bool + log io.ReadCloser + err error } -func (f fakeLogs) SandboxLog(context.Context, string, bool) (io.ReadCloser, error) { return f.log, nil } -func (f fakeLogs) JobLog(context.Context, string, int, bool) (io.ReadCloser, bool, error) { - return f.log, f.live, nil +func (f fakeLogs) Sandbox(context.Context, string, bool) (io.ReadCloser, error) { return f.log, f.err } +func (f fakeLogs) Job(context.Context, string, int, bool) (io.ReadCloser, error) { return f.log, f.err } + +// A log that is not there is a 404, and a server that is not connected a +// 503, so that the page tells why it shows no log. +func TestMissingLogs(t *testing.T) { + for _, tc := range []struct { + err error + status int + }{ + {joblog.ErrMissing, 404}, + {joblog.ErrExpired, 404}, + {joblog.ErrNoGitHubLog, 404}, + {joblog.ErrNotYet, 404}, + {joblog.ErrSandboxStopped, 404}, + {joblog.ErrNotConnected, 503}, + } { + e := newEnv(t, func(d *Deps) { d.Logs = fakeLogs{err: tc.err} }) + e.addSandbox("sb1", sandbox.StateBusy) + j := e.addJob(1, "acme/api", time.Now()) + b := e.login("bob@example.com", auth.RoleMember) + for _, path := range []string{"/api/sandboxes/sb1/log", "/api/jobs/" + j.ID + "/log"} { + if res := b.do("GET", path, nil, nil); res.StatusCode != tc.status { + t.Errorf("%s with %v: status %d, want %d", path, tc.err, res.StatusCode, tc.status) + } + } + } } func TestFollowedLogStreamsEachLine(t *testing.T) { diff --git a/internal/api/servers.go b/internal/api/servers.go index 2263db4..b544f24 100644 --- a/internal/api/servers.go +++ b/internal/api/servers.go @@ -3,15 +3,16 @@ package api import ( "context" "errors" - "strings" "time" "github.com/danielgtaylor/huma/v2" - "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/doctor" + "github.com/runplane/runplane/internal/field" + "github.com/runplane/runplane/internal/fleet" + "github.com/runplane/runplane/internal/policy" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/version" ) // ServerSummary is the server card of the overview. @@ -21,14 +22,15 @@ type ServerSummary struct { Kind string `json:"kind" enum:"local,agent" doc:"local is the primary."` Status string `json:"status" enum:"online,installing,unreachable,draining,disabled"` StatusDetail string `json:"status_detail" doc:"For example the install progress or the last error."` - Arch string `json:"arch" doc:"x64 or arm64. Empty until the server reports its facts."` - Mode string `json:"mode" doc:"container, vm, or dev. Empty until the server reports its facts."` + Arch string `json:"arch" enum:"x64,arm64," doc:"Empty until the server reports its facts."` + Mode string `json:"mode" enum:"container,vm,dev," doc:"Empty until the server reports its facts."` VCPU int `json:"vcpu"` MemoryMB int `json:"memory_mb"` DiskBytes int64 `json:"disk_bytes"` CPUPercent *float64 `json:"cpu_percent" doc:"Latest host sample. Null without samples."` MemoryPercent *float64 `json:"memory_percent"` DiskPercent *float64 `json:"disk_percent"` + StatsAt *time.Time `json:"stats_at" doc:"Time of the latest host sample."` RunningJobs int `json:"running_jobs" doc:"Sandboxes that run a job."` Sandboxes int `json:"sandboxes" doc:"Sandboxes that are not gone or failed."` VCPUInUse int `json:"vcpu_in_use" doc:"vCPUs of the live sandboxes."` @@ -38,34 +40,35 @@ type ServerSummary struct { type Server struct { ServerSummary - Address string `json:"address"` - Provider string `json:"provider" doc:"Cloud provider, if runplane knows it."` - Region string `json:"region" doc:"Region or location of the cloud provider, for example fsn1. Empty if unknown."` - OS string `json:"os"` - Kernel string `json:"kernel"` - DiskKind string `json:"disk_kind" doc:"NVMe, SSD, HDD, or empty if unknown."` - KVM bool `json:"kvm"` - DockerVersion string `json:"docker_version"` - SysboxVersion string `json:"sysbox_version"` - FirecrackerVersion string `json:"firecracker_version"` - Version string `json:"version" doc:"runplane version on the server."` - UpdateState string `json:"update_state" enum:"current,available,updating,failed,none" doc:"current: the server runs the version of the primary. available: POST /api/servers/{id}/update installs the release of the primary on the agent. updating: an update runs. failed: the last update failed, and status_detail tells why; POST again tries again. none: runplane cannot update the server, because the primary is not a release build or the server has not reported its version."` - StatsAt *time.Time `json:"stats_at" doc:"Time of the latest host sample."` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + Address string `json:"address"` + Provider string `json:"provider" doc:"Cloud provider, if runplane knows it."` + Region string `json:"region" doc:"Region or location of the cloud provider, for example fsn1. Empty if unknown."` + OS string `json:"os"` + Kernel string `json:"kernel"` + DiskKind string `json:"disk_kind" doc:"NVMe, SSD, HDD, or empty if unknown."` + KVM bool `json:"kvm"` + DockerVersion string `json:"docker_version"` + SysboxVersion string `json:"sysbox_version"` + FirecrackerVersion string `json:"firecracker_version"` + Version string `json:"version" doc:"runplane version on the server."` + UpdateState string `json:"update_state" enum:"current,available,updating,failed,none" doc:"current: the server runs the version of the primary. available: POST /api/servers/{id}/update installs the release of the primary on the agent. updating: an update runs. failed: the last update failed, and update_error tells why; POST again tries again. none: runplane cannot update the server, because the primary is not a release build or the server has not reported its version."` + UpdateError string `json:"update_error" doc:"Why the last update of the agent failed. Empty when it did not fail."` + WaitReason string `json:"wait_reason" doc:"Why the server gets no runners in its mode, for example while allow_public_repos is on and the server is not in VM mode. Empty when it gets runners. status tells if the server is draining or disabled."` + Actions ServerActions `json:"actions" doc:"The changes that an admin can make to the server now."` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } -// Values of Server.UpdateState. -const ( - UpdateCurrent = "current" - UpdateAvailable = "available" - UpdateUpdating = "updating" - UpdateFailed = "failed" - UpdateNone = "none" -) +type ServerActions struct { + CanDrain bool `json:"can_drain" doc:"PATCH status draining."` + CanResume bool `json:"can_resume" doc:"PATCH status online, after a drain or a disable."` + CanRemove bool `json:"can_remove" doc:"DELETE. The primary cannot be removed."` + CanUpdate bool `json:"can_update" doc:"POST /api/servers/{id}/update: the agent is connected, and update_state is available or failed."` + CanRunDoctor bool `json:"can_run_doctor" doc:"POST /api/servers/{id}/doctor: the server is connected."` +} // servers converts store servers and adds the latest host sample, the -// sandbox use, and the update state. +// sandbox use, the update state, and why a server waits. func (h *Handler) servers(ctx context.Context, list []store.Server) ([]Server, error) { stats, err := h.d.Store.LatestServerStats(ctx) if err != nil { @@ -75,25 +78,27 @@ func (h *Handler) servers(ctx context.Context, list []store.Server) ([]Server, e if err != nil { return nil, err } - regions, err := h.d.Store.ServerRegions(ctx) + settings, err := h.d.Store.GetSettings(ctx) if err != nil { return nil, err } out := make([]Server, len(list)) byID := map[string]*Server{} for i, v := range list { - f := v.ServerFacts + f, a := v.ServerFacts, h.d.Servers.Actions(v) out[i] = Server{ ServerSummary: ServerSummary{ ID: v.ID, Name: v.Name, Kind: v.Kind, Status: v.Status, StatusDetail: v.StatusDetail, Arch: f.Arch, Mode: string(f.Mode), VCPU: f.VCPU, MemoryMB: f.MemoryMB, DiskBytes: f.DiskBytes, LastSeenAt: v.LastSeenAt, }, - Address: f.Address, Provider: f.Provider, Region: regions[v.ID], OS: f.OS, Kernel: f.Kernel, DiskKind: f.DiskKind, KVM: f.KVM, + Address: f.Address, Provider: f.Provider, Region: f.Region, OS: f.OS, Kernel: f.Kernel, DiskKind: f.DiskKind, KVM: f.KVM, DockerVersion: f.DockerVersion, SysboxVersion: f.SysboxVersion, FirecrackerVersion: f.FirecrackerVersion, - Version: f.Version, UpdateState: UpdateNone, CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, - } - if h.d.Servers != nil { - out[i].UpdateState = h.d.Servers.UpdateState(v) + Version: f.Version, UpdateState: string(h.d.Servers.UpdateState(v)), UpdateError: v.UpdateError, + WaitReason: waitReason(settings, v), + Actions: ServerActions{ + CanDrain: a.Drain, CanResume: a.Resume, CanRemove: a.Remove, CanUpdate: a.Update, CanRunDoctor: a.RunDoctor, + }, + CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, } if st, ok := stats[v.ID]; ok { out[i].CPUPercent, out[i].MemoryPercent, out[i].DiskPercent = &st.CPUPercent, &st.MemoryPercent, &st.DiskPercent @@ -116,6 +121,17 @@ func (h *Handler) servers(ctx context.Context, list []store.Server) ([]Server, e return out, nil } +// waitReason tells why the scheduler starts no runners on the server, or +// returns "". A server without facts runs no driver yet. A public job that +// waits keeps a server out of its own scale set only, so it is no reason +// here. +func waitReason(st store.Settings, v store.Server) string { + if v.Mode == "" { + return "" + } + return policy.RunnerRefusal(st, v.Mode, false) +} + func (h *Handler) listServers(ctx context.Context, _ *struct{}) (*body[[]Server], error) { list, err := h.d.Store.ListServers(ctx) if err != nil { @@ -144,23 +160,19 @@ type ServerToken struct { } func (h *Handler) createServerToken(ctx context.Context, in *body[*ServerTokenRequest]) (*body[ServerToken], error) { - if h.d.Servers == nil { - return nil, unavailable("Adding servers") - } var name string if in.Body != nil { - name = strings.TrimSpace(in.Body.Name) + name = in.Body.Name } base, err := h.baseURL(ctx) if err != nil { return nil, err } - t, err := h.d.Servers.NewToken(ctx, base, name, currentUser(ctx).ID) + t, err := h.d.Servers.NewToken(ctx, actor(ctx), base, name) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - h.audit(ctx, "server.token", t.ID, map[string]any{"name": name, "expires_at": t.ExpiresAt}) - return &body[ServerToken]{t}, nil + return &body[ServerToken]{ServerToken{ID: t.ID, Token: t.Token, Command: t.Command, ExpiresAt: t.ExpiresAt}}, nil } type ServerTokenStatus struct { @@ -189,30 +201,15 @@ type SSHInstall struct { } func (h *Handler) installServerSSH(ctx context.Context, in *body[SSHInstall]) (*body[Server], error) { - if h.d.Servers == nil { - return nil, unavailable("Adding servers") - } req := in.Body - req.Name, req.Host = strings.TrimSpace(req.Name), strings.TrimSpace(req.Host) - var errs problems - if req.Name == "" { - errs.add("body.name", "Enter a name.", in.Body.Name) - } - if req.Host == "" || strings.ContainsAny(req.Host, " /@") { - errs.add("body.host", "Enter a host name or an IP address.", in.Body.Host) - } - if err := errs.err(); err != nil { - return nil, err - } base, err := h.baseURL(ctx) if err != nil { return nil, err } - id, err := h.d.Servers.InstallSSH(ctx, base, req) + id, err := h.d.Servers.InstallSSH(ctx, actor(ctx), base, fleet.SSHInstall{Name: req.Name, Host: req.Host, Port: req.Port, User: req.User}) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - h.audit(ctx, "server.install", id, map[string]any{"name": req.Name, "host": req.Host, "port": req.Port, "user": req.User}) v, err := h.d.Store.GetServer(ctx, id) if err != nil { return nil, notFound(err, "The new server is gone.") @@ -225,116 +222,67 @@ type SSHKey struct { } func (h *Handler) getSSHKey(ctx context.Context, _ *struct{}) (*body[SSHKey], error) { - if h.d.Servers == nil { - return nil, unavailable("Adding servers") - } key, err := h.d.Servers.SSHPublicKey(ctx) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } return &body[SSHKey]{SSHKey{PublicKey: key}}, nil } +// UpdateServerRequest has the name limits of SSHInstall. type UpdateServerRequest struct { - Name *string `json:"name,omitempty" minLength:"1" maxLength:"64"` + Name *string `json:"name,omitempty"` Status *string `json:"status,omitempty" enum:"online,draining,disabled" doc:"draining and disabled servers get no new sandboxes. online makes a draining or disabled server take sandboxes again."` } +func (UpdateServerRequest) TransformSchema(r huma.Registry, s *huma.Schema) *huma.Schema { + return patchOf[SSHInstall](r, s) +} + func (h *Handler) updateServer(ctx context.Context, in *struct { PathID Body UpdateServerRequest }) (*body[Server], error) { - v, err := h.d.Store.GetServer(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No server has this ID.") - } - b := in.Body - var name string - if b.Name != nil { - if name = strings.TrimSpace(*b.Name); name == "" { - return nil, invalid("body.name", "Enter a name.", *b.Name) - } - } - changeStatus := b.Status != nil && *b.Status != v.Status - if changeStatus { - if v.Status == store.ServerInstalling { - return nil, huma.Error409Conflict("The server is still installing.") - } - if *b.Status == store.ServerOnline && v.Status != store.ServerDraining && v.Status != store.ServerDisabled { - return nil, huma.Error409Conflict("Only a draining or disabled server can go online. The server is " + v.Status + ".") - } - } - if name != "" { - if err := h.d.Store.SetServerName(ctx, v.ID, name); err != nil { - return nil, notFound(err, "No server has this ID.") - } + v, err := h.d.Servers.UpdateServer(ctx, actor(ctx), in.ID, fleet.ServerChange{Name: in.Body.Name, Status: in.Body.Status}) + if errors.Is(err, fleet.ErrInstalling) || errors.Is(err, fleet.ErrNotPaused) { + return nil, huma.Error409Conflict(field.Sentence(err)) } - if changeStatus { - if err := h.d.Store.SetServerStatus(ctx, v.ID, *b.Status, ""); err != nil { - return nil, notFound(err, "No server has this ID.") - } - } - old := v - if v, err = h.d.Store.GetServer(ctx, v.ID); err != nil { + if err != nil { return nil, notFound(err, "No server has this ID.") } - h.d.Bus.Publish(events.Event{Type: events.ServerState, Data: v}) - changes := map[string][2]any{} - if v.Name != old.Name { - changes["name"] = [2]any{old.Name, v.Name} - } - if v.Status != old.Status { - changes["status"] = [2]any{old.Status, v.Status} - } - if len(changes) > 0 { - h.audit(ctx, "server.update", v.ID, map[string]any{"name": v.Name, "status": v.Status, "changes": changes}) - } return reply(first(h.servers(ctx, []store.Server{v}))) } func (h *Handler) deleteServer(ctx context.Context, in *PathID) (*empty, error) { - v, err := h.d.Store.GetServer(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No server has this ID.") - } - if v.Kind == store.ServerLocal { + err := h.d.Servers.RemoveServer(ctx, actor(ctx), in.ID) + switch { + case errors.Is(err, fleet.ErrLocalServer): return nil, huma.Error409Conflict("The primary server cannot be removed.") + case errors.Is(err, store.ErrNotFound): + return nil, huma.Error404NotFound("No server has this ID.") + case err != nil: + return nil, serviceErr(err) } - if h.d.Servers == nil { - return nil, unavailable("Removing servers") - } - if err := h.d.Servers.Remove(ctx, v.ID); err != nil { - return nil, h.serviceErr(err) - } - if err := h.d.Store.DeleteServer(ctx, v.ID); err != nil { - return nil, notFound(err, "No server has this ID.") - } - // The delete can change the servers of pools, or disable them. - h.poolsChanged(ctx) - h.audit(ctx, "server.delete", v.ID, map[string]any{"name": v.Name}) return nil, nil } func (h *Handler) updateAgent(ctx context.Context, in *PathID) (*body[Server], error) { - if h.d.Servers == nil { - return nil, unavailable("Agent updates") - } v, err := h.d.Store.GetServer(ctx, in.ID) if err != nil { return nil, notFound(err, "No server has this ID.") } - switch h.d.Servers.UpdateState(v) { - case UpdateCurrent: + switch err := h.d.Servers.UpdateAgent(ctx, actor(ctx), v.ID); { + case errors.Is(err, fleet.ErrUpToDate): return nil, huma.Error409Conflict("The server runs runplane " + v.Version + ", the version of the primary.") - case UpdateUpdating: + case errors.Is(err, fleet.ErrUpdateInProgress): return nil, huma.Error409Conflict("An update of the server runs.") - case UpdateNone: + case errors.Is(err, fleet.ErrCannotUpdate): return nil, huma.Error409Conflict("runplane cannot update the server: the primary is not a release build, or the server has not reported its version.") + case errors.Is(err, fleet.ErrNotConnected): + return nil, huma.Error409Conflict("The server is not connected, so it cannot update.") + case err != nil: + return nil, serviceErr(err) } - if err := h.d.Servers.UpdateAgent(ctx, v.ID); err != nil { - return nil, h.serviceErr(err) - } - h.audit(ctx, "server.update_agent", v.ID, map[string]any{"name": v.Name, "from": v.Version, "to": version.Version}) return reply(first(h.servers(ctx, []store.Server{v}))) } @@ -349,43 +297,43 @@ func (h *Handler) getDoctor(ctx context.Context, in *PathID) (*body[DoctorResult if err != nil { return nil, notFound(err, "No server has this ID.") } - out := DoctorResult{ServerID: v.ID, Checks: []Check{}} r, err := h.d.Store.GetDoctorResult(ctx, v.ID) switch { case errors.Is(err, store.ErrNotFound): - return &body[DoctorResult]{out}, nil + return &body[DoctorResult]{DoctorResult{ServerID: v.ID, Checks: []Check{}}}, nil case err != nil: return nil, err } - out.CheckedAt = &r.CheckedAt + return &body[DoctorResult]{doctorResult(r)}, nil +} + +func doctorResult(r store.DoctorResult) DoctorResult { + out := DoctorResult{ServerID: r.ServerID, CheckedAt: &r.CheckedAt, Checks: []Check{}} for _, c := range r.Checks { out.Checks = append(out.Checks, Check(c)) } - return &body[DoctorResult]{out}, nil + return out } func (h *Handler) runDoctor(ctx context.Context, in *PathID) (*body[DoctorResult], error) { - if h.d.Servers == nil { - return nil, unavailable("The doctor") - } v, err := h.d.Store.GetServer(ctx, in.ID) if err != nil { return nil, notFound(err, "No server has this ID.") } - checks, err := h.d.Servers.Doctor(ctx, v.ID) + r, err := h.d.Servers.Doctor(ctx, v.ID) + if errors.Is(err, fleet.ErrNotConnected) { + return nil, huma.Error409Conflict("The server is not connected, so its checks cannot run.") + } if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - return reply(h.saveDoctor(ctx, v.ID, checks)) + return &body[DoctorResult]{doctorResult(r)}, nil } -func (h *Handler) saveDoctor(ctx context.Context, serverID string, checks []Check) (DoctorResult, error) { - r := store.DoctorResult{ServerID: serverID, CheckedAt: time.Now().UTC().Truncate(time.Millisecond)} - for _, c := range checks { - r.Checks = append(r.Checks, store.DoctorCheck(c)) - } - if err := h.d.Store.SaveDoctorResult(ctx, r); err != nil { - return DoctorResult{}, err +func checks(list []doctor.Check) []Check { + out := make([]Check, len(list)) + for i, c := range list { + out[i] = Check{ID: c.ID, Name: c.Name, Status: string(c.Status), Detail: c.Detail, Fix: c.Fix, Command: c.Command} } - return DoctorResult{ServerID: serverID, CheckedAt: &r.CheckedAt, Checks: orEmpty(checks)}, nil + return out } diff --git a/internal/api/servers_test.go b/internal/api/servers_test.go index d13afae..2723f14 100644 --- a/internal/api/servers_test.go +++ b/internal/api/servers_test.go @@ -8,36 +8,59 @@ import ( "testing" "time" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/auth" "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/fleet" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" ) // fakeServers records the token names and the agent updates, and returns -// fixed doctor checks and update states. +// a fixed doctor result and update states. type fakeServers struct { Servers names []string - checks []Check - states map[string]string // update state by server ID - updated []string // server IDs of the UpdateAgent calls + doctor store.DoctorResult + states map[string]fleet.UpdateState // update state by server ID + updated []string // server IDs of the UpdateAgent calls + limits map[string]sandbox.Limits // the connected servers } -func (f *fakeServers) UpdateState(v store.Server) string { return f.states[v.ID] } +func (f *fakeServers) Limits(id string) (sandbox.Limits, bool) { + l, ok := f.limits[id] + return l, ok +} + +func (f *fakeServers) UpdateState(v store.Server) fleet.UpdateState { return f.states[v.ID] } + +func (f *fakeServers) Actions(v store.Server) fleet.Actions { + return fleet.Actions{Update: f.states[v.ID] == fleet.UpdateAvailable} +} -func (f *fakeServers) UpdateAgent(_ context.Context, id string) error { +// UpdateAgent refuses an update as the fleet does for each state. +func (f *fakeServers) UpdateAgent(_ context.Context, _ audit.Actor, id string) error { + switch f.states[id] { + case fleet.UpdateCurrent: + return fleet.ErrUpToDate + case fleet.UpdateUpdating: + return fleet.ErrUpdateInProgress + case fleet.UpdateNone: + return fleet.ErrCannotUpdate + } f.updated = append(f.updated, id) - f.states[id] = UpdateUpdating + f.states[id] = fleet.UpdateUpdating return nil } -func (f *fakeServers) NewToken(_ context.Context, _, name, _ string) (ServerToken, error) { +func (f *fakeServers) NewToken(_ context.Context, _ audit.Actor, _, name string) (fleet.Token, error) { f.names = append(f.names, name) - return ServerToken{ID: "tok1", Token: "rp_srv_x", Command: "curl", ExpiresAt: time.Now().Add(time.Hour)}, nil + return fleet.Token{ID: "tok1", Token: "rp_srv_x", Command: "curl", ExpiresAt: time.Now().Add(time.Hour)}, nil } -func (f *fakeServers) Doctor(context.Context, string) ([]Check, error) { return f.checks, nil } +func (f *fakeServers) Doctor(context.Context, string) (store.DoctorResult, error) { + return f.doctor, nil +} func TestServerTokenWithoutNameAndItsStatus(t *testing.T) { servers := &fakeServers{} @@ -72,15 +95,20 @@ func TestServerTokenWithoutNameAndItsStatus(t *testing.T) { e.login("bob@example.com", auth.RoleMember).do("GET", "/api/servers/tokens/t1/status", nil, nil).want(t, 403) } -func TestDoctorResultIsKept(t *testing.T) { - servers := &fakeServers{checks: []Check{ +// The fleet runs the doctor and keeps the result. The API shows the result +// of a run, and later the kept result. +func TestDoctorResult(t *testing.T) { + checkedAt := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + result := store.DoctorResult{ServerID: "srv1", CheckedAt: checkedAt, Checks: []store.DoctorCheck{ {ID: "clock", Name: "Time synchronization", Status: "warn", Detail: "Not synchronized", Fix: "Turn on time sync.", Command: "sudo timedatectl set-ntp true"}, }} - e := newEnv(t, func(d *Deps) { d.Servers = servers }) + e := newEnv(t, func(d *Deps) { d.Servers = &fakeServers{doctor: result} }) e.addServer("srv1", "x64", store.ServerOnline, 4, 8192) admin := e.login("ada@example.com", auth.RoleAdmin) member := e.login("bob@example.com", auth.RoleMember) + want := []Check{{ID: "clock", Name: "Time synchronization", Status: "warn", Detail: "Not synchronized", + Fix: "Turn on time sync.", Command: "sudo timedatectl set-ntp true"}} var before DoctorResult member.do("GET", "/api/servers/srv1/doctor", nil, &before).want(t, 200) @@ -89,14 +117,15 @@ func TestDoctorResultIsKept(t *testing.T) { } var ran DoctorResult admin.do("POST", "/api/servers/srv1/doctor", nil, &ran).want(t, 200) - - var after DoctorResult - member.do("GET", "/api/servers/srv1/doctor", nil, &after).want(t, 200) - if after.CheckedAt == nil || !after.CheckedAt.Equal(*ran.CheckedAt) { - t.Fatalf("checked_at %v, want the time of the run %v", after.CheckedAt, ran.CheckedAt) + if ran.CheckedAt == nil || !ran.CheckedAt.Equal(checkedAt) || !slices.Equal(ran.Checks, want) { + t.Fatalf("doctor run = %+v, want the checks of %v", ran, checkedAt) } - if !slices.Equal(after.Checks, servers.checks) { - t.Fatalf("kept checks = %+v, want %+v", after.Checks, servers.checks) + + must(t, e.st.SaveDoctorResult(context.Background(), result)) + var kept DoctorResult + member.do("GET", "/api/servers/srv1/doctor", nil, &kept).want(t, 200) + if kept.CheckedAt == nil || !kept.CheckedAt.Equal(checkedAt) || !slices.Equal(kept.Checks, want) { + t.Fatalf("kept doctor result = %+v, want the checks of %v", kept, checkedAt) } member.do("GET", "/api/servers/nope/doctor", nil, nil).want(t, 404) } @@ -104,12 +133,7 @@ func TestDoctorResultIsKept(t *testing.T) { func TestServerRegion(t *testing.T) { e := newEnv(t) e.addServer("srv1", "x64", store.ServerOnline, 4, 8192) - if changed, err := e.st.SetServerRegion(context.Background(), "srv1", "fsn1"); err != nil || !changed { - t.Fatalf("set region: changed %v, %v", changed, err) - } - if changed, _ := e.st.SetServerRegion(context.Background(), "srv1", "fsn1"); changed { - t.Fatal("the same region again reports a change") - } + must(t, e.st.UpdateServerFacts(context.Background(), "srv1", store.ServerFacts{Arch: "x64", Region: "fsn1"}, time.Now())) var v Server e.login("ada@example.com", auth.RoleMember).do("GET", "/api/servers/srv1", nil, &v).want(t, 200) if v.Region != "fsn1" { @@ -152,6 +176,26 @@ func TestPoolPlacement(t *testing.T) { } } +// The placement knows the limits of the driver of each connected server: +// a VM has at most 32 vCPUs, and each VM takes memory for its process. +func TestPoolPlacementKnowsTheDriverLimits(t *testing.T) { + servers := &fakeServers{limits: map[string]sandbox.Limits{"vm": {OverheadMB: 64, MaxVCPU: 32}}} + e := newEnv(t, func(d *Deps) { d.Servers = servers }) + b := e.login("ada@example.com", auth.RoleAdmin) + e.addServer("vm", "x64", store.ServerOnline, 64, 34816) + e.addServer("other", "x64", store.ServerOnline, 64, 34816) + var got PoolPlacement + b.do("GET", "/api/pools/placement?arch=x64&vcpu=48&memory_mb=4096", nil, &got).want(t, 200) + if !slices.Equal(got.PlaceableServerIDs, []string{"other"}) { + t.Errorf("placement of 48 vCPUs = %+v, want only the server without a vCPU limit", got) + } + // The default reserve leaves 32768 MB. + b.do("GET", "/api/pools/placement?arch=x64&vcpu=2&memory_mb=32768", nil, &got).want(t, 200) + if !slices.Equal(got.PlaceableServerIDs, []string{"other"}) { + t.Errorf("placement of 32768 MB = %+v, want only the server without overhead", got) + } +} + // The scheduler takes a public job only on VM servers, so the setting must // not turn on while a server runs another mode. func TestPublicReposNeedVMOnEachServer(t *testing.T) { @@ -175,7 +219,7 @@ func TestPublicReposNeedVMOnEachServer(t *testing.T) { t.Fatalf("problem %s does not name hel1-agent", res.body) } - must(t, e.st.DeleteServer(ctx, "agent")) + must(t, e.st.DeleteServer(ctx, "agent", time.Now())) var out Settings b.do("PATCH", "/api/settings", on, &out).want(t, 200) if !out.AllowPublicRepos { @@ -185,9 +229,51 @@ func TestPublicReposNeedVMOnEachServer(t *testing.T) { b.do("PATCH", "/api/settings", map[string]any{"mode": "container"}, nil).want(t, 422) } +// An agent can connect in container mode while allow_public_repos is on. +// The setting stays on, the agent waits, and the API tells why. The +// placement does not count the agent, as the scheduler does not. +func TestServerWaitsWhilePublicReposAreOn(t *testing.T) { + e := newEnv(t, func(d *Deps) { d.Config.Mode = sandbox.ModeVM }) + b := e.login("ada@example.com", auth.RoleAdmin) + ctx := context.Background() + now := time.Now() + addServer := func(id string, mode sandbox.Mode) { + must(t, e.st.CreateServer(ctx, store.Server{ID: id, Name: id, Kind: store.ServerAgent, Status: store.ServerOnline, + ServerFacts: store.ServerFacts{Arch: "x64", Mode: mode, VCPU: 8, MemoryMB: 16384}, CreatedAt: now, UpdatedAt: now})) + } + addServer("primary", sandbox.ModeVM) + b.do("PATCH", "/api/settings", map[string]any{"allow_public_repos": true}, nil).want(t, 200) + addServer("cheap-vps", sandbox.ModeContainer) + const placement = "/api/pools/placement?arch=x64&vcpu=2&memory_mb=4096" + + var st Settings + b.do("GET", "/api/settings", nil, &st).want(t, 200) + if !st.AllowPublicRepos || !strings.Contains(st.PublicReposRefusal, "cheap-vps") { + t.Errorf("settings: allow_public_repos %v, public_repos_refusal %q; want on, with the name cheap-vps", st.AllowPublicRepos, st.PublicReposRefusal) + } + var agent, primary Server + b.do("GET", "/api/servers/cheap-vps", nil, &agent).want(t, 200) + b.do("GET", "/api/servers/primary", nil, &primary).want(t, 200) + if agent.WaitReason == "" || primary.WaitReason != "" { + t.Errorf("wait_reason: cheap-vps %q, primary %q; want a reason only for cheap-vps", agent.WaitReason, primary.WaitReason) + } + var got PoolPlacement + b.do("GET", placement, nil, &got).want(t, 200) + if got.OnlineServers != 2 || got.WaitingServers != 1 || !slices.Equal(got.PlaceableServerIDs, []string{"primary"}) { + t.Errorf("placement = %+v, want 2 online servers, 1 that waits, and primary", got) + } + + b.do("PATCH", "/api/settings", map[string]any{"allow_public_repos": false}, nil).want(t, 200) + b.do("GET", "/api/servers/cheap-vps", nil, &agent).want(t, 200) + b.do("GET", placement, nil, &got).want(t, 200) + if agent.WaitReason != "" || got.WaitingServers != 0 || len(got.PlaceableServerIDs) != 2 { + t.Errorf("after the setting went off: wait_reason %q, placement %+v; want no wait", agent.WaitReason, got) + } +} + func TestUpdateAgentOnlyWhenAnUpdateApplies(t *testing.T) { - servers := &fakeServers{states: map[string]string{ - "same": UpdateCurrent, "old": UpdateAvailable, "broken": UpdateFailed, "busy": UpdateUpdating, "dev": UpdateNone, + servers := &fakeServers{states: map[string]fleet.UpdateState{ + "same": fleet.UpdateCurrent, "old": fleet.UpdateAvailable, "broken": fleet.UpdateFailed, "busy": fleet.UpdateUpdating, "dev": fleet.UpdateNone, }} e := newEnv(t, func(d *Deps) { d.Servers = servers }) for id := range servers.states { @@ -203,7 +289,7 @@ func TestUpdateAgentOnlyWhenAnUpdateApplies(t *testing.T) { for _, id := range []string{"old", "broken"} { var out Server admin.do("POST", "/api/servers/"+id+"/update", nil, &out).want(t, 200) - if out.UpdateState != UpdateUpdating { + if out.UpdateState != string(fleet.UpdateUpdating) { t.Errorf("server %s: update state %q after the call, want updating", id, out.UpdateState) } } @@ -232,7 +318,7 @@ func TestPoolWithAGoneServerStillChanges(t *testing.T) { // Server API object, because the frontend puts the event data in place of // GET /api/servers/{id}. func TestServerStateEventIsTheServerObject(t *testing.T) { - servers := &fakeServers{states: map[string]string{"s1": UpdateAvailable}} + servers := &fakeServers{states: map[string]fleet.UpdateState{"s1": fleet.UpdateAvailable}} e := newEnv(t, func(d *Deps) { d.Servers = servers }) ch := e.login("bob@example.com", auth.RoleMember).events(t) ctx := context.Background() @@ -248,12 +334,14 @@ func TestServerStateEventIsTheServerObject(t *testing.T) { v, err := e.st.GetServer(ctx, "s1") must(t, err) - e.publish(events.Event{Type: events.ServerState, Data: v}) + e.publish(events.ServerState(v)) ev := recv(t, ch) var got Server must(t, json.Unmarshal([]byte(ev.data), &got)) - if ev.name != "server.state" || got.ID != "s1" || got.UpdateState != UpdateAvailable || got.Sandboxes != 2 || - got.RunningJobs != 1 || got.VCPUInUse != 6 || got.MemoryInUseMB != 12288 || got.CPUPercent == nil || *got.CPUPercent != 42 { - t.Errorf("%s data %s: want s1 with update_state available, 2 sandboxes, 1 running job, 6 vCPUs and 12288 MB in use, and 42%% CPU", ev.name, ev.data) + if ev.name != "server.state" || got.ID != "s1" || got.UpdateState != string(fleet.UpdateAvailable) || !got.Actions.CanUpdate || + got.Sandboxes != 2 || got.RunningJobs != 1 || got.VCPUInUse != 6 || got.MemoryInUseMB != 12288 || + got.CPUPercent == nil || *got.CPUPercent != 42 || got.StatsAt == nil || !got.StatsAt.Equal(now.Truncate(time.Millisecond)) { + t.Errorf("%s data %s: want s1 with update_state available and the update action, 2 sandboxes, 1 running job, "+ + "6 vCPUs and 12288 MB in use, and 42%% CPU at the sample time", ev.name, ev.data) } } diff --git a/internal/api/session.go b/internal/api/session.go index 83134dc..388d9ad 100644 --- a/internal/api/session.go +++ b/internal/api/session.go @@ -4,13 +4,12 @@ import ( "context" "errors" "net/http" - "strings" "time" - "unicode" "github.com/danielgtaylor/huma/v2" "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/field" "github.com/runplane/runplane/internal/store" ) @@ -61,16 +60,44 @@ func userOut(u store.User) User { return User{ID: u.ID, Email: u.Email, Name: u.Name, Role: u.Role, CreatedAt: u.CreatedAt, LastLoginAt: u.LastLoginAt} } +// Session is the user of the session, with what the user can do. +type Session struct { + User + Capabilities Capabilities `json:"capabilities"` +} + +// Capabilities are the actions that a role allows. All roles can read the +// dashboard data. +type Capabilities struct { + RerunJobs bool `json:"rerun_jobs" doc:"Run a job again, also with a debug sandbox."` + StopSandboxes bool `json:"stop_sandboxes"` + OpenShells bool `json:"open_shells" doc:"Open the debug shell of a sandbox."` + Admin bool `json:"admin" doc:"All other changes, and the members, the notification channels, the audit log, and the backups."` +} + +func capabilities(role string) Capabilities { + return Capabilities{ + RerunJobs: auth.CanDo(role, auth.ActionRerunJob), + StopSandboxes: auth.CanDo(role, auth.ActionStopSandbox), + OpenShells: auth.CanDo(role, auth.ActionOpenShell), + Admin: auth.CanDo(role, auth.ActionAdmin), + } +} + +func sessionOut(u store.User) Session { + return Session{User: userOut(u), Capabilities: capabilities(u.Role)} +} + type sessionOutput struct { SetCookie []string `header:"Set-Cookie"` - Body User + Body Session } func (h *Handler) getSession(ctx context.Context, _ *struct{}) (*sessionOutput, error) { // Send the cookie again, so it lives as long as the sliding session. return &sessionOutput{ SetCookie: []string{sessionCookieValue(ctx, clientOf(ctx).token)}, - Body: userOut(currentUser(ctx)), + Body: sessionOut(currentUser(ctx)), }, nil } @@ -81,7 +108,7 @@ func (h *Handler) newSession(ctx context.Context, u store.User, token string) *s device := h.d.Auth.Device(u.ID, clientOf(ctx).device) return &sessionOutput{ SetCookie: []string{sessionCookieValue(ctx, token), deviceCookieValue(ctx, device)}, - Body: userOut(u), + Body: sessionOut(u), } } @@ -121,47 +148,28 @@ func (h *Handler) logout(ctx context.Context, _ *struct{}) (*cookieOutput, error type UpdateMeRequest struct { Name *string `json:"name,omitempty" maxLength:"100"` CurrentPassword *string `json:"current_password,omitempty" maxLength:"1024" doc:"Required with new_password."` - NewPassword *string `json:"new_password,omitempty" maxLength:"1024" doc:"Logs out the other sessions of the user."` + NewPassword *string `json:"new_password,omitempty" minLength:"10" maxLength:"1024" doc:"Logs out the other sessions of the user."` } func (h *Handler) updateMe(ctx context.Context, in *body[UpdateMeRequest]) (*sessionOutput, error) { - u := currentUser(ctx) b := in.Body - out := &sessionOutput{} - name := u.Name - if b.Name != nil { - if name = strings.TrimSpace(*b.Name); name == "" { - return nil, invalid("body.name", sentence(auth.ErrBadName), *b.Name) - } - } - if b.NewPassword != nil { - if b.CurrentPassword == nil || *b.CurrentPassword == "" { - return nil, invalid("body.current_password", "Enter your current password.", nil) - } - // The change ends the sessions on other devices. This device gets - // a new session. - c := clientOf(ctx) - token, err := h.d.Auth.ChangePassword(ctx, u.ID, *b.CurrentPassword, *b.NewPassword, c.ip, c.userAgent) - if err != nil { - return nil, authErr(err, "body.new_password") - } - out = h.newSession(ctx, u, token) - h.audit(ctx, "user.password", u.ID, nil) + c := clientOf(ctx) + u, token, err := h.d.Auth.UpdateMe(ctx, currentUser(ctx).ID, b.Name, b.CurrentPassword, b.NewPassword, c.ip, c.userAgent) + if err != nil { + return nil, authErr(err, "body.new_password") } - if name != u.Name { - u.Name = name - if err := h.d.Store.UpdateUser(ctx, u); err != nil { - return nil, err - } + // A new password ends the sessions on other devices. This device gets + // a new session. + if token != "" { + return h.newSession(ctx, u, token), nil } - out.Body = userOut(u) - return out, nil + return &sessionOutput{Body: sessionOut(u)}, nil } // authErr maps the errors of auth to problems. passwordLoc is the field // that ErrShortPassword belongs to. func authErr(err error, passwordLoc string) error { - msg := sentence(err) + msg := field.Sentence(err) switch { case errors.Is(err, auth.ErrRateLimited): return huma.Error429TooManyRequests(msg) @@ -169,6 +177,10 @@ func authErr(err error, passwordLoc string) error { return huma.Error401Unauthorized(msg) case errors.Is(err, auth.ErrSetupDone): return huma.Error409Conflict("The setup is done. Log in instead.") + case errors.Is(err, auth.ErrLastAdmin): + return huma.Error409Conflict("runplane needs at least one admin.") + case errors.Is(err, auth.ErrOwnAccount): + return huma.Error409Conflict(msg) case errors.Is(err, auth.ErrBadInvite): return huma.Error404NotFound(msg) case errors.Is(err, auth.ErrEmailTaken): @@ -183,26 +195,12 @@ func authErr(err error, passwordLoc string) error { return invalid("body.role", msg, nil) case errors.Is(err, auth.ErrShortPassword): return invalid(passwordLoc, msg, nil) - case errors.Is(err, auth.ErrBadPassword): + case errors.Is(err, auth.ErrBadPassword), errors.Is(err, auth.ErrNoCurrentPassword): return invalid("body.current_password", msg, nil) } return err } -// sentence makes an error message into a sentence for the UI. -func sentence(err error) string { - msg := err.Error() - if msg == "" { - return msg - } - r := []rune(msg) - r[0] = unicode.ToUpper(r[0]) - if !strings.HasSuffix(msg, ".") { - r = append(r, '.') - } - return string(r) -} - // deviceCookieValue returns the Set-Cookie value for a device token. The // cookie outlives logouts and sessions: it marks a browser that logged in. // The browser sends it only to /api/session, where the login reads it. diff --git a/internal/api/settings.go b/internal/api/settings.go index 87762ac..ddc447b 100644 --- a/internal/api/settings.go +++ b/internal/api/settings.go @@ -1,53 +1,31 @@ package api import ( - "cmp" "context" - "fmt" - "maps" - "net" - "net/mail" - "net/netip" - "net/url" - "regexp" - "slices" + "encoding/json" + "reflect" "strings" "time" - "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/sandbox" + "github.com/danielgtaylor/huma/v2" + + "github.com/runplane/runplane/internal/backup" + "github.com/runplane/runplane/internal/policy" + "github.com/runplane/runplane/internal/secret" "github.com/runplane/runplane/internal/store" ) +// Settings has the keys of store.Settings, with their docs and limits, and +// the facts that come from them. type Settings struct { - InstanceName string `json:"instance_name" doc:"Title in the sidebar."` - PublicURL string `json:"public_url" doc:"Browser URL of the dashboard, for example https://ci.example.com. Empty: the URL of each request."` - Domain string `json:"domain" doc:"With a domain, runplane gets a certificate with ACME and serves HTTPS on ports 443 and 80. The DNS record of the domain must point to the server, and the two ports must be free and open in the firewall."` - ACMEEmail string `json:"acme_email" doc:"Contact for the certificate authority. Optional."` - Mode string `json:"mode" enum:"container,vm,dev," doc:"Sandbox mode. Empty until the install sets it."` - ReserveVCPU int `json:"reserve_vcpu" doc:"vCPUs that sandboxes do not get on each server."` - ReserveMemoryMB int `json:"reserve_memory_mb" doc:"Memory that sandboxes do not get on each server."` - CPUOvercommit float64 `json:"cpu_overcommit" doc:"Factor for the vCPUs that sandboxes can get. 1 means no overcommit."` - MinFreeDiskPercent float64 `json:"min_free_disk_percent" doc:"No new sandboxes start on a server with less free disk."` - DefaultTimeLimitMinutes int `json:"default_time_limit_minutes" doc:"Time limit of a job in a pool without its own limit."` - AllowPublicRepos bool `json:"allow_public_repos" doc:"Take jobs of public repositories. Only when the mode setting and each server are in VM mode."` - AllowForkPRs bool `json:"allow_fork_prs" doc:"Take jobs of pull requests from forks."` - EgressAllowlist []string `json:"egress_allowlist" doc:"Host names, *.domains, IP addresses, or CIDRs that sandboxes can reach. Empty: all of the public internet."` - BandwidthMbps int `json:"bandwidth_mbps" doc:"Bandwidth limit of each VM. 0: no limit."` - DNSServers []string `json:"dns_servers" doc:"IPv4 addresses of the DNS servers of the sandboxes."` - RegistryMirror bool `json:"registry_mirror" doc:"Docker Hub pull-through cache for the sandboxes."` - ActionsCache bool `json:"actions_cache" doc:"Local Actions cache server."` - CacheMaxGB int `json:"cache_max_gb"` - JobRetentionDays int `json:"job_retention_days"` - LogRetentionDays int `json:"log_retention_days"` - DebugHoldMinutes int `json:"debug_hold_minutes" doc:"Time that a debug sandbox stays after its job."` - AutoUpdate bool `json:"auto_update"` - BackupEnabled bool `json:"backup_enabled" doc:"Daily database backup."` - BackupKeep int `json:"backup_keep" doc:"Backups to keep."` - BackupS3 BackupS3 `json:"backup_s3" doc:"Optional upload of each backup to S3."` + store.Settings + BackupS3 BackupS3 `json:"backup_s3" doc:"Optional upload of each backup to S3."` - SSHPort int `json:"ssh_port" doc:"Port of the SSH gateway for debug shells. Read-only."` - HTTPS *HTTPSStatus `json:"https,omitempty" doc:"Certificate of the domain. Read-only. Absent when this runplane process cannot report it."` + BackupTime string `json:"backup_time" doc:"Time of day in UTC of the daily backup, for example 03:00. Read-only."` + SecretKeyPath string `json:"secret_key_path" doc:"File of the key that seals the secrets in the database. A restore of a backup needs it, and backups do not hold it. Read-only."` + HTTPS HTTPSStatus `json:"https" doc:"Certificate of the domain. Read-only."` + PublicReposRefusal string `json:"public_repos_refusal" doc:"Why an admin cannot turn allow_public_repos on: the mode setting or a server is not in VM mode. It names each server that is not in VM mode, and while the setting is on, these servers get no runners. Empty when each server is in VM mode. Read-only."` + BandwidthApplies bool `json:"bandwidth_applies" doc:"bandwidth_mbps limits sandboxes: only VMs have the limit, so the mode setting or a server is in VM mode. Read-only."` } type HTTPSStatus struct { @@ -65,23 +43,25 @@ type BackupS3 struct { SecretKeySet bool `json:"secret_key_set" doc:"The secret key is write-only."` } -func (h *Handler) settingsOut(ctx context.Context, st store.Settings) Settings { +func (h *Handler) settingsOut(ctx context.Context, st store.Settings) (Settings, error) { + servers, err := h.d.Store.ListServers(ctx) + if err != nil { + return Settings{}, err + } + mode := h.d.Settings.Mode(st) + st.Mode = mode + st.EgressAllowlist, st.DNSServers = orEmpty(st.EgressAllowlist), orEmpty(st.DNSServers) out := Settings{ - InstanceName: st.InstanceName, PublicURL: st.PublicURL, Domain: st.Domain, ACMEEmail: st.ACMEEmail, - Mode: string(st.Mode), ReserveVCPU: st.ReserveVCPU, ReserveMemoryMB: st.ReserveMemoryMB, - CPUOvercommit: st.CPUOvercommit, MinFreeDiskPercent: st.MinFreeDiskPercent, - DefaultTimeLimitMinutes: st.DefaultTimeLimitMinutes, AllowPublicRepos: st.AllowPublicRepos, - AllowForkPRs: st.AllowForkPRs, EgressAllowlist: orEmpty(st.EgressAllowlist), BandwidthMbps: st.BandwidthMbps, - DNSServers: orEmpty(st.DNSServers), RegistryMirror: st.RegistryMirror, ActionsCache: st.ActionsCache, - CacheMaxGB: st.CacheMaxGB, JobRetentionDays: st.JobRetentionDays, LogRetentionDays: st.LogRetentionDays, - DebugHoldMinutes: st.DebugHoldMinutes, AutoUpdate: st.AutoUpdate, BackupEnabled: st.BackupEnabled, - BackupKeep: st.BackupKeep, BackupS3: backupS3Out(st.BackupS3), SSHPort: h.sshPort(), + Settings: st, BackupS3: backupS3Out(st.BackupS3), + BackupTime: time.Time{}.Add(backup.DailyAt).Format("15:04"), SecretKeyPath: h.d.Config.SecretKeyPath(), + PublicReposRefusal: policy.PublicRepos(mode, servers), BandwidthApplies: policy.BandwidthApplies(mode, servers), } - if h.d.HTTPS != nil { - s := h.d.HTTPS.CertificateStatus(ctx) - out.HTTPS = &s + s := h.d.HTTPS.Status(ctx) + out.HTTPS = HTTPSStatus{State: s.State, Error: s.Error} + if !s.NotAfter.IsZero() { + out.HTTPS.NotAfter = &s.NotAfter } - return out + return out, nil } func backupS3Out(b store.BackupS3) BackupS3 { @@ -96,35 +76,27 @@ func (h *Handler) getSettings(ctx context.Context, _ *struct{}) (*body[Settings] if err != nil { return nil, err } - return &body[Settings]{h.settingsOut(ctx, st)}, nil + return reply(h.settingsOut(ctx, st)) } -type SettingsPatch struct { - InstanceName *string `json:"instance_name,omitempty" minLength:"1" maxLength:"64"` - PublicURL *string `json:"public_url,omitempty" maxLength:"2048"` - Domain *string `json:"domain,omitempty" maxLength:"253"` - ACMEEmail *string `json:"acme_email,omitempty" maxLength:"254"` - Mode *string `json:"mode,omitempty" enum:"container,vm,dev" doc:"vm needs KVM; dev needs RUNPLANE_DEV_INSECURE=1. A change takes effect after a restart."` - ReserveVCPU *int `json:"reserve_vcpu,omitempty" minimum:"0" maximum:"256"` - ReserveMemoryMB *int `json:"reserve_memory_mb,omitempty" minimum:"0" maximum:"1048576"` - CPUOvercommit *float64 `json:"cpu_overcommit,omitempty" minimum:"0.5" maximum:"8"` - MinFreeDiskPercent *float64 `json:"min_free_disk_percent,omitempty" minimum:"0" maximum:"90"` - DefaultTimeLimitMinutes *int `json:"default_time_limit_minutes,omitempty" minimum:"1" maximum:"7200"` - AllowPublicRepos *bool `json:"allow_public_repos,omitempty"` - AllowForkPRs *bool `json:"allow_fork_prs,omitempty"` - EgressAllowlist *[]string `json:"egress_allowlist,omitempty" maxItems:"256"` - BandwidthMbps *int `json:"bandwidth_mbps,omitempty" minimum:"0" maximum:"100000"` - DNSServers *[]string `json:"dns_servers,omitempty" minItems:"1" maxItems:"4"` - RegistryMirror *bool `json:"registry_mirror,omitempty"` - ActionsCache *bool `json:"actions_cache,omitempty"` - CacheMaxGB *int `json:"cache_max_gb,omitempty" minimum:"1" maximum:"100000"` - JobRetentionDays *int `json:"job_retention_days,omitempty" minimum:"1" maximum:"3650"` - LogRetentionDays *int `json:"log_retention_days,omitempty" minimum:"1" maximum:"3650"` - DebugHoldMinutes *int `json:"debug_hold_minutes,omitempty" minimum:"1" maximum:"1440"` - AutoUpdate *bool `json:"auto_update,omitempty"` - BackupEnabled *bool `json:"backup_enabled,omitempty"` - BackupKeep *int `json:"backup_keep,omitempty" minimum:"1" maximum:"365"` - BackupS3 *BackupS3Patch `json:"backup_s3,omitempty"` +// SettingsPatch changes some settings. Its keys are the keys of +// store.Settings, with their limits, except that backup_s3 is a +// BackupS3Patch. The handler reads the keys from the raw body. +type SettingsPatch struct{} + +func (SettingsPatch) TransformSchema(r huma.Registry, s *huma.Schema) *huma.Schema { + t := reflect.TypeFor[store.Settings]() + s.Properties = map[string]*huma.Schema{} + for i := range t.NumField() { + f := t.Field(i) + key, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if key == "backup_s3" { + s.Properties[key] = r.Schema(reflect.TypeFor[BackupS3Patch](), true, "") + continue + } + s.Properties[key] = huma.SchemaFromField(r, f, "") + } + return s } type BackupS3Patch struct { @@ -136,242 +108,38 @@ type BackupS3Patch struct { SecretKey *string `json:"secret_key,omitempty" maxLength:"256" doc:"Write-only. An empty string removes the key."` } -func (h *Handler) updateSettings(ctx context.Context, in *body[SettingsPatch]) (*body[Settings], error) { - p := in.Body - if err := p.check(); err != nil { - return nil, err - } - servers, err := h.d.Store.ListServers(ctx) - if err != nil { - return nil, err - } - var old store.Settings - next, err := h.d.Store.UpdateSettings(ctx, func(st *store.Settings) error { - old = *st - p.apply(st, h) - return h.checkSettings(*st, old, servers) +func (h *Handler) updateSettings(ctx context.Context, in *struct { + Body SettingsPatch + RawBody []byte +}) (*body[Settings], error) { + next, err := h.d.Settings.Update(ctx, actor(ctx), func(st *store.Settings) error { + return applyPatch(st, in.RawBody, h.d.Box) }) if err != nil { return nil, err } - if changes := fieldChanges(old, next); len(changes) > 0 { - h.audit(ctx, "settings.update", "", settingsDetail(old, next, changes)) - h.d.Bus.Publish(events.Event{Type: events.SettingsChanged, Data: next}) - if h.d.SettingsHook != nil { - h.d.SettingsHook.SettingsChanged(context.WithoutCancel(ctx), old, next) - } - } - return &body[Settings]{h.settingsOut(ctx, next)}, nil -} - -// check validates the values that do not depend on other settings, and -// trims them. -func (p *SettingsPatch) check() error { - var errs problems - trim := func(list ...*string) { - for _, v := range list { - if v != nil { - *v = strings.TrimSpace(*v) - } - } - } - trim(p.InstanceName, p.PublicURL, p.Domain, p.ACMEEmail) - if p.InstanceName != nil && *p.InstanceName == "" { - errs.add("body.instance_name", "Enter a name.", *p.InstanceName) - } - if p.PublicURL != nil && *p.PublicURL != "" { - u, err := url.Parse(*p.PublicURL) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || - (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { - errs.add("body.public_url", "Enter a URL such as https://ci.example.com, without a path.", *p.PublicURL) - } else { - *p.PublicURL = u.Scheme + "://" + u.Host - } - } - if p.Domain != nil { - *p.Domain = strings.ToLower(*p.Domain) - if *p.Domain != "" && !ValidDomain(*p.Domain) { - errs.add("body.domain", "Enter a domain name such as ci.example.com.", *p.Domain) - } - } - if p.ACMEEmail != nil && *p.ACMEEmail != "" { - if a, err := mail.ParseAddress(*p.ACMEEmail); err != nil || a.Address != *p.ACMEEmail { - errs.add("body.acme_email", "Enter an email address.", *p.ACMEEmail) - } - } - if p.EgressAllowlist != nil { - for i, v := range *p.EgressAllowlist { - v = strings.ToLower(strings.TrimSpace(v)) - (*p.EgressAllowlist)[i] = v - if !validEgress(v) { - errs.add(fmt.Sprintf("body.egress_allowlist[%d]", i), "Enter a host name, *.domain, IP address, or CIDR.", v) - } - } - } - if p.DNSServers != nil { - for i, v := range *p.DNSServers { - v = strings.TrimSpace(v) - (*p.DNSServers)[i] = v - if a, err := netip.ParseAddr(v); err != nil || !a.Is4() { - errs.add(fmt.Sprintf("body.dns_servers[%d]", i), "Enter an IPv4 address.", v) - } - } - } - if b := p.BackupS3; b != nil { - trim(b.Endpoint, b.Region, b.Bucket, b.Prefix, b.AccessKey) - if b.Endpoint != nil && *b.Endpoint != "" { - u, err := url.Parse(*b.Endpoint) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { - errs.add("body.backup_s3.endpoint", "Enter a URL such as https://s3.example.com.", *b.Endpoint) - } - } - if b.Region != nil && !s3Region.MatchString(*b.Region) { - errs.add("body.backup_s3.region", "Enter a region such as us-east-1.", *b.Region) - } - if b.Bucket != nil && *b.Bucket != "" && !s3Bucket.MatchString(*b.Bucket) { - errs.add("body.backup_s3.bucket", "Use 3 to 63 lowercase letters, digits, dots, and dashes.", *b.Bucket) - } - if b.Prefix != nil { - *b.Prefix = strings.TrimPrefix(*b.Prefix, "/") - } - } - return errs.err() -} - -var ( - s3Region = regexp.MustCompile(`^[a-z0-9-]*$`) - s3Bucket = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`) - dnsLabel = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`) -) - -// ValidDomain accepts a lowercase host name with at least two labels, not an -// IP address. -func ValidDomain(d string) bool { - labels := strings.Split(d, ".") - if len(d) > 253 || len(labels) < 2 || net.ParseIP(d) != nil { - return false - } - for _, l := range labels { - if !dnsLabel.MatchString(l) { - return false - } - } - return true -} - -func validEgress(v string) bool { - if _, err := netip.ParseAddr(v); err == nil { - return true - } - if _, err := netip.ParsePrefix(v); err == nil { - return true - } - return ValidDomain(strings.TrimPrefix(v, "*.")) -} - -func (p *SettingsPatch) apply(st *store.Settings, h *Handler) { - set(&st.InstanceName, p.InstanceName) - set(&st.PublicURL, p.PublicURL) - set(&st.Domain, p.Domain) - set(&st.ACMEEmail, p.ACMEEmail) - if p.Mode != nil { - st.Mode = sandbox.Mode(*p.Mode) - } - set(&st.ReserveVCPU, p.ReserveVCPU) - set(&st.ReserveMemoryMB, p.ReserveMemoryMB) - set(&st.CPUOvercommit, p.CPUOvercommit) - set(&st.MinFreeDiskPercent, p.MinFreeDiskPercent) - set(&st.DefaultTimeLimitMinutes, p.DefaultTimeLimitMinutes) - set(&st.AllowPublicRepos, p.AllowPublicRepos) - set(&st.AllowForkPRs, p.AllowForkPRs) - set(&st.EgressAllowlist, p.EgressAllowlist) - set(&st.BandwidthMbps, p.BandwidthMbps) - set(&st.DNSServers, p.DNSServers) - set(&st.RegistryMirror, p.RegistryMirror) - set(&st.ActionsCache, p.ActionsCache) - set(&st.CacheMaxGB, p.CacheMaxGB) - set(&st.JobRetentionDays, p.JobRetentionDays) - set(&st.LogRetentionDays, p.LogRetentionDays) - set(&st.DebugHoldMinutes, p.DebugHoldMinutes) - set(&st.AutoUpdate, p.AutoUpdate) - set(&st.BackupEnabled, p.BackupEnabled) - set(&st.BackupKeep, p.BackupKeep) - if b := p.BackupS3; b != nil { - set(&st.BackupS3.Endpoint, b.Endpoint) - set(&st.BackupS3.Region, b.Region) - set(&st.BackupS3.Bucket, b.Bucket) - set(&st.BackupS3.Prefix, b.Prefix) - set(&st.BackupS3.AccessKey, b.AccessKey) - if b.SecretKey != nil { - st.BackupS3.SecretKeyEnc = nil - if *b.SecretKey != "" { - st.BackupS3.SecretKeyEnc = h.d.Box.Seal([]byte(*b.SecretKey)) - } - } - } + return reply(h.settingsOut(ctx, next)) } -// checkSettings validates the rules between settings and the servers. -func (h *Handler) checkSettings(st, old store.Settings, servers []store.Server) error { - var errs problems - var local store.Server - if i := slices.IndexFunc(servers, func(v store.Server) bool { return v.Kind == store.ServerLocal }); i >= 0 { - local = servers[i] - } - if st.Mode != old.Mode { - switch { - case st.Mode == sandbox.ModeDev && !h.d.Config.DevInsecure: - errs.add("body.mode", "Mode dev is for development only and needs RUNPLANE_DEV_INSECURE=1.", st.Mode) - case st.Mode == sandbox.ModeVM && local.ID != "" && local.VCPU > 0 && !local.KVM: - errs.add("body.mode", "VM mode needs KVM, and the primary server has no /dev/kvm.", st.Mode) - } - } - // A server reports the mode of its running driver, which can differ - // from the mode setting until a restart. The scheduler checks each - // runner again, but the setting must not claim what the servers cannot do. - if st.AllowPublicRepos && (!old.AllowPublicRepos || st.Mode != old.Mode) { - if detail := notVM(h.mode(st), servers); detail != "" { - errs.add("body.allow_public_repos", detail, true) - } +// applyPatch sets the keys of a SettingsPatch on st. The secret key of S3 +// replaces the sealed key; an empty key removes it. +func applyPatch(st *store.Settings, patch []byte, box *secret.Box) error { + var s3 struct { + BackupS3 struct { + SecretKey *string `json:"secret_key"` + } `json:"backup_s3"` } - if b := st.BackupS3; b.Bucket != "" { - if b.AccessKey == "" { - errs.add("body.backup_s3.access_key", "Enter the access key for the bucket.", nil) - } - if len(b.SecretKeyEnc) == 0 { - errs.add("body.backup_s3.secret_key", "Enter the secret key for the bucket.", nil) - } + if err := json.Unmarshal(patch, &s3); err != nil { + return err } - return errs.err() -} - -// notVM tells why public repositories are not safe: the mode setting or a -// server that is not in VM mode. It returns "" when all are in VM mode. -func notVM(mode sandbox.Mode, servers []store.Server) string { - if mode != sandbox.ModeVM { - return fmt.Sprintf("Jobs of public repositories need VM mode, and the mode setting is %s.", cmp.Or(string(mode), "not set")) + if err := json.Unmarshal(patch, st); err != nil { + return err } - for _, v := range servers { - if v.Mode != "" && v.Mode != sandbox.ModeVM { - return fmt.Sprintf("Jobs of public repositories need VM mode on each server, and the server %s runs %s mode.", v.Name, v.Mode) + if k := s3.BackupS3.SecretKey; k != nil { + st.BackupS3.SecretKeyEnc = nil + if *k != "" { + st.BackupS3.SecretKeyEnc = box.Seal([]byte(*k)) } } - return "" -} - -// settingsDetail is the audit detail of a settings change: the changed keys, -// and the old and new value of each. The values of backup_s3 come one field -// at a time, as backup_s3.bucket for example, and without the secret key. -func settingsDetail(old, next store.Settings, changes map[string][2]any) map[string]any { - keys := slices.Sorted(maps.Keys(changes)) - delete(changes, "backup_s3") - for k, v := range fieldChanges(backupS3Out(old.BackupS3), backupS3Out(next.BackupS3)) { - changes["backup_s3."+k] = v - } - return map[string]any{"keys": keys, "changes": changes} -} - -// mode is the sandbox mode from the settings, or from the install. -func (h *Handler) mode(st store.Settings) sandbox.Mode { - return cmp.Or(st.Mode, h.d.Config.Mode) + return nil } diff --git a/internal/api/setup.go b/internal/api/setup.go index bc91822..162232b 100644 --- a/internal/api/setup.go +++ b/internal/api/setup.go @@ -5,19 +5,26 @@ import ( "github.com/danielgtaylor/huma/v2" + "github.com/runplane/runplane" "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/field" "github.com/runplane/runplane/internal/store" "github.com/runplane/runplane/internal/version" ) type Status struct { - NeedsSetup bool `json:"needs_setup" doc:"True until the first admin exists."` - Version string `json:"version" doc:"Version of runplane, for example \"v1.2.0 (3f2a9c1)\"."` - Mode string `json:"mode" enum:"container,vm,dev," doc:"Sandbox mode. Empty until the install sets it."` - InstanceName string `json:"instance_name"` - GitHubConnected bool `json:"github_connected" doc:"True when a GitHub App with at least one installation exists."` - HasJobs bool `json:"has_jobs" doc:"True when runplane has recorded at least one job."` - SSHPort int `json:"ssh_port" doc:"Port of the SSH gateway for debug shells."` + NeedsSetup bool `json:"needs_setup" doc:"True until the first admin exists."` + Version string `json:"version" doc:"Version of runplane, for example \"v1.2.0 (3f2a9c1)\"."` + Mode string `json:"mode" enum:"container,vm,dev" doc:"Sandbox mode."` + InstanceName string `json:"instance_name"` + HasJobs bool `json:"has_jobs" doc:"True when runplane has recorded at least one job."` + Systems []System `json:"systems" doc:"The operating systems of a server that install.sh names."` +} + +type System struct { + Name string `json:"name" doc:"For example Ubuntu."` + Version string `json:"version" doc:"For example 24.04."` + BestEffort bool `json:"best_effort" doc:"install.sh installs runplane with a warning, because runplane has no tests on the system."` } func (h *Handler) getStatus(ctx context.Context, _ *struct{}) (*body[Status], error) { @@ -30,30 +37,29 @@ func (h *Handler) getStatus(ctx context.Context, _ *struct{}) (*body[Status], er if err != nil { return nil, err } - connected, err := h.githubConnected(ctx) - if err != nil { - return nil, err - } jobs, _, err := st.ListJobs(ctx, store.JobFilter{Limit: 1}) if err != nil { return nil, err } + systems := make([]System, len(runplane.Systems)) + for i, s := range runplane.Systems { + systems[i] = System{Name: s.Name, Version: s.Version, BestEffort: s.BestEffort} + } return &body[Status]{Status{ - NeedsSetup: users == 0, - Version: version.String(), - Mode: string(h.mode(settings)), - InstanceName: settings.InstanceName, - GitHubConnected: connected, - HasJobs: len(jobs) > 0, - SSHPort: h.sshPort(), + NeedsSetup: users == 0, + Version: version.String(), + Mode: string(h.d.Settings.Mode(settings)), + InstanceName: settings.InstanceName, + HasJobs: len(jobs) > 0, + Systems: systems, }}, nil } type SetupRequest struct { - Code string `json:"code" doc:"Setup code from the install script, for example ABCD-EFGH-JKLM."` + Code string `json:"code" pattern:"^[A-HJ-NP-Za-hj-np-z2-9]{4}[- ]?[A-HJ-NP-Za-hj-np-z2-9]{4}[- ]?[A-HJ-NP-Za-hj-np-z2-9]{4}$" patternDescription:"a setup code such as ABCD-EFGH-JKLM" doc:"Setup code from the install script, for example ABCD-EFGH-JKLM."` Name string `json:"name" maxLength:"100"` Email string `json:"email" maxLength:"254"` - Password string `json:"password" maxLength:"1024"` + Password string `json:"password" minLength:"10" maxLength:"1024"` } func (h *Handler) setup(ctx context.Context, in *body[SetupRequest]) (*sessionOutput, error) { @@ -63,7 +69,6 @@ func (h *Handler) setup(ctx context.Context, in *body[SetupRequest]) (*sessionOu if err != nil { return nil, authErr(err, "body.password") } - h.audit(context.WithValue(ctx, userKey{}, u), "setup.done", u.ID, map[string]any{"email": u.Email}) return h.newSession(ctx, u, token), nil } @@ -78,7 +83,7 @@ type HostCheck struct { FreeDiskBytes int64 `json:"free_disk_bytes" doc:"Free space on the disk of the data folder."` DiskKind string `json:"disk_kind" doc:"NVMe, SSD, or HDD. Empty if unknown."` OS string `json:"os" doc:"For example \"Ubuntu 24.04.1 LTS\"."` - Checks []Check `json:"checks"` + Checks []Check `json:"checks" doc:"The doctor checks without the mode check: mode has its result."` } type Check struct { @@ -91,7 +96,7 @@ type Check struct { } type setupHostInput struct { - Code string `query:"code" required:"true" doc:"The setup code."` + Code string `query:"code" required:"true" pattern:"^[A-HJ-NP-Za-hj-np-z2-9]{4}[- ]?[A-HJ-NP-Za-hj-np-z2-9]{4}[- ]?[A-HJ-NP-Za-hj-np-z2-9]{4}$" patternDescription:"a setup code such as ABCD-EFGH-JKLM" doc:"The setup code."` } func (h *Handler) getSetupHost(ctx context.Context, in *setupHostInput) (*body[HostCheck], error) { @@ -107,21 +112,15 @@ func (h *Handler) getSetupHost(ctx context.Context, in *setupHostInput) (*body[H return nil, err } if !ok { - return nil, invalid("query.code", sentence(auth.ErrBadSetupCode), nil) - } - if h.d.HostChecker == nil { - return nil, unavailable("The host check") + return nil, invalid("query.code", field.Sentence(auth.ErrBadSetupCode), nil) } - hc, err := h.d.HostChecker.Check(ctx) + c, err := h.d.HostChecker.Check(ctx) if err != nil { - return nil, h.serviceErr(err) + return nil, serviceErr(err) } - hc.Checks = orEmpty(hc.Checks) - // Server detail shows these checks as the last doctor result of the primary. - if local, err := h.d.Store.GetLocalServer(ctx); err == nil { - if _, err := h.saveDoctor(ctx, local.ID, hc.Checks); err != nil { - h.d.Log.Warn("save doctor result", "server", local.ID, "err", err) - } - } - return &body[HostCheck]{hc}, nil + i := c.Info + return &body[HostCheck]{HostCheck{ + Hostname: i.Hostname, Mode: string(c.Mode), KVM: i.KVM, Arch: i.Arch, VCPU: i.VCPU, MemoryMB: i.MemoryMB, + DiskBytes: i.DiskBytes, FreeDiskBytes: i.FreeDiskBytes, DiskKind: i.DiskKind, OS: i.OS, Checks: checks(c.Checks), + }}, nil } diff --git a/internal/api/spec_test.go b/internal/api/spec_test.go new file mode 100644 index 0000000..f8f68db --- /dev/null +++ b/internal/api/spec_test.go @@ -0,0 +1,249 @@ +package api + +import ( + "context" + "encoding/json" + "maps" + "regexp" + "slices" + "strings" + "testing" + + "github.com/runplane/runplane/images" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/doctor" + "github.com/runplane/runplane/internal/fleet" + "github.com/runplane/runplane/internal/ghapp" + "github.com/runplane/runplane/internal/notify" + "github.com/runplane/runplane/internal/runner" + "github.com/runplane/runplane/internal/sandbox" + "github.com/runplane/runplane/internal/store" + "github.com/runplane/runplane/internal/tlsutil" + "github.com/runplane/runplane/internal/update" +) + +// specSchema is the part of a schema of the spec that the tests read. +type specSchema struct { + Enum []string `json:"enum"` + Pattern string `json:"pattern"` + MinLength *int `json:"minLength"` + Items *specSchema `json:"items"` + Properties map[string]specSchema `json:"properties"` +} + +type spec struct { + Components struct { + Schemas map[string]specSchema `json:"schemas"` + } `json:"components"` + Paths map[string]map[string]struct { + Parameters []struct { + Name string `json:"name"` + In string `json:"in"` + Schema specSchema `json:"schema"` + } `json:"parameters"` + Responses map[string]struct { + Headers map[string]struct { + Schema specSchema `json:"schema"` + } `json:"headers"` + } `json:"responses"` + } `json:"paths"` +} + +func readSpec(t *testing.T) spec { + t.Helper() + raw, err := OpenAPI() + must(t, err) + var s spec + must(t, json.Unmarshal(raw, &s)) + return s +} + +// enums returns each enum of the spec: "Schema.key" for the keys of the +// schemas, and "method path name" for the parameters and headers. +func (s spec) enums() map[string][]string { + out := map[string][]string{} + for name, schema := range s.Components.Schemas { + for key, p := range schema.Properties { + if p.Items != nil { + p = *p.Items + } + if len(p.Enum) > 0 { + out[name+"."+key] = p.Enum + } + } + } + for path, ops := range s.Paths { + for method, op := range ops { + for _, p := range op.Parameters { + if len(p.Schema.Enum) > 0 { + out[method+" "+path+" "+p.Name] = p.Schema.Enum + } + } + for _, r := range op.Responses { + for name, h := range r.Headers { + if len(h.Schema.Enum) > 0 { + out[method+" "+path+" "+name] = h.Schema.Enum + } + } + } + } + } + return out +} + +func strs[T ~string](list ...T) []string { + out := make([]string, len(list)) + for i, v := range list { + out[i] = string(v) + } + return out +} + +// The web takes the enums of the spec as the values that the API sends and +// accepts. Each enum must have the values of the Go code. +func TestSpecEnumsHaveTheGoValues(t *testing.T) { + var ( + roles = auth.Roles + arches = runner.Arches + modes = strs(sandbox.ModeContainer, sandbox.ModeVM, sandbox.ModeDev) + sandboxStates = strs(append(sandbox.LiveStates(), sandbox.DoneStates()...)...) + jobStates = []string{store.JobQueued, store.JobAssigned, store.JobRunning, store.JobCompleted} + results = []string{store.ResultSucceeded, store.ResultFailed, store.ResultCancelled, store.ResultSkipped} + jobStatuses = append([]string{store.StatusQueued, store.StatusRunning}, results...) + rangeKeys = slices.Collect(maps.Keys(ranges)) + kinds = notify.Kinds + types = strs(notify.Types...) + serverKinds = []string{store.ServerLocal, store.ServerAgent} + serverStates = []string{store.ServerOnline, store.ServerInstalling, store.ServerUnreachable, store.ServerDraining, store.ServerDisabled} + serverArches = append(slices.Clip(arches), "") // empty until the server reports its facts + serverModes = append(slices.Clip(modes), "") + permissions = map[string]bool{} + accesses = map[string]bool{} + ) + for _, owner := range []ghapp.Account{{Type: ghapp.Organization}, {Type: ghapp.User}} { + for _, p := range owner.AppPermissions() { + permissions[p.Name], accesses[p.Access] = true, true + } + } + want := map[string][]string{ + "AppPermission.access": slices.Collect(maps.Keys(accesses)), + "AppPermission.name": slices.Collect(maps.Keys(permissions)), + "AuditEntry.action": strs(audit.Actions...), + "Backup.status": {store.BackupRunning, store.BackupDone, store.BackupUploading, store.BackupUploaded, store.BackupFailed}, + "Check.status": strs(doctor.StatusOK, doctor.StatusWarn, doctor.StatusFail), + "CreateInviteRequest.role": roles, + "CreateNotificationRequest.events": types, + "CreateNotificationRequest.kind": kinds, + "CreatePoolRequest.arch": arches, + "DomainDNS.state": {dnsNone, dnsOK, dnsMissing, dnsOther, dnsUnknown, dnsError}, + "HTTPSStatus.state": {tlsutil.CertNone, tlsutil.CertRequesting, tlsutil.CertValid, tlsutil.CertError}, + "HostCheck.arch": arches, + "HostCheck.mode": modes, + "Image.source": {"registry", images.Source}, + "ImageBuild.arch": serverArches, + "ImageBuild.status": {store.BuildPending, store.BuildPulling, store.BuildBuilding, store.BuildConverting, store.BuildReady, store.BuildFailed}, + "Installation.account_type": {ghapp.Organization, ghapp.User}, + "Installation.repository_selection": {"all", "selected"}, // GitHub sets them + "Invite.role": roles, + "InviteInfo.role": roles, + "Job.log_state": {store.LogNone, store.LogLive, store.LogStored, store.LogExpired}, + "Job.sandbox_state": append(sandboxStates, ""), + "Job.state": jobStates, + "Job.status": jobStatuses, + "JobStep.status": jobStatuses, + "JobSummary.state": jobStates, + "JobSummary.status": jobStatuses, + "ManifestRequest.account_type": {"organization", "user"}, // github.Service.Manifest takes them + "NotificationChannel.kind": kinds, + "Overview.range": rangeKeys, + "Pool.arch": arches, + "RunnerVersion.arch": arches, + "RunnerVersion.status": {store.BundlePending, store.BundleReady, store.BundleFailed}, + "Sandbox.state": sandboxStates, + "ScaleSetStatus.status": {store.ScaleSetCreating, store.ScaleSetListening, store.ScaleSetError, store.ScaleSetDeleting}, + "Server.arch": serverArches, + "Server.kind": serverKinds, + "Server.mode": serverModes, + "Server.status": serverStates, + "Server.update_state": strs(fleet.UpdateCurrent, fleet.UpdateAvailable, fleet.UpdateUpdating, fleet.UpdateFailed, fleet.UpdateNone), + "ServerSummary.arch": serverArches, + "ServerSummary.kind": serverKinds, + "ServerSummary.mode": serverModes, + "ServerSummary.status": serverStates, + "Session.role": roles, + "Settings.mode": modes, + "SettingsPatch.mode": modes, + "SizeAdvice.kind": {adviceLarger, adviceSmaller, adviceNone}, + "Status.mode": modes, + "TestChannelRequest.kind": kinds, + "UpdateNotificationRequest.events": types, + "UpdateNotificationRequest.kind": kinds, + "UpdatePoolRequest.arch": arches, + "UpdateServerRequest.status": {store.ServerOnline, store.ServerDraining, store.ServerDisabled}, // the statuses that an admin sets + "UpdateStatus.state": strs(update.Idle, update.Applying, update.Restarting), + "UpdateUserRequest.role": roles, + "User.role": roles, + "get /api/github/setup setup_action": slices.Collect(maps.Keys(installActions)), + "get /api/jobs status": jobStatuses, + "get /api/jobs range": rangeKeys, + "get /api/overview range": rangeKeys, + "get /api/pools/placement arch": arches, + "get /api/sandboxes state": sandboxStates, + } + got := readSpec(t).enums() + for key, values := range got { + w, ok := want[key] + if !ok { + t.Errorf("the enum %s %q has no Go values in this test", key, values) + continue + } + if !sameSet(values, w) { + t.Errorf("the enum %s is %q, but the Go values are %q", key, values, w) + } + } + for key := range want { + if _, ok := got[key]; !ok { + t.Errorf("the spec has no enum %s", key) + } + } +} + +func sameSet(a, b []string) bool { + return slices.Equal(slices.Sorted(slices.Values(a)), slices.Sorted(slices.Values(b))) +} + +// The forms read the password minimum and the setup code pattern from the +// spec, so the spec must have the rules of auth. +func TestSpecHasTheRulesOfAuth(t *testing.T) { + s := readSpec(t) + for _, f := range []struct{ schema, key string }{ + {"SetupRequest", "password"}, {"AcceptInviteRequest", "password"}, {"UpdateMeRequest", "new_password"}, + } { + if min := s.Components.Schemas[f.schema].Properties[f.key].MinLength; min == nil || *min != auth.MinPasswordLen { + t.Errorf("minLength of %s.%s is %v, want %d", f.schema, f.key, min, auth.MinPasswordLen) + } + } + + patterns := map[string]string{"SetupRequest.code": s.Components.Schemas["SetupRequest"].Properties["code"].Pattern} + for _, p := range s.Paths["/api/setup/host"]["get"].Parameters { + if p.Name == "code" { + patterns["get /api/setup/host code"] = p.Schema.Pattern + } + } + code, err := newEnv(t).d.Auth.NewSetupCode(context.Background()) + must(t, err) + // Setup takes a code without dashes, with spaces, and in lowercase too. + codes := []string{code, strings.ReplaceAll(code, "-", ""), strings.ReplaceAll(code, "-", " "), strings.ToLower(code)} + for name, pattern := range patterns { + re := regexp.MustCompile(pattern) + for _, c := range codes { + if !re.MatchString(c) { + t.Errorf("the pattern %s %q refuses the setup code %q", name, pattern, c) + } + } + } + if len(patterns) != 2 { + t.Errorf("found the setup code patterns %v, want 2", patterns) + } +} diff --git a/internal/api/users.go b/internal/api/users.go index eed641e..1d0a0e6 100644 --- a/internal/api/users.go +++ b/internal/api/users.go @@ -3,12 +3,10 @@ package api import ( "context" "errors" - "slices" "time" - "github.com/danielgtaylor/huma/v2" - "github.com/runplane/runplane/internal/auth" + "github.com/runplane/runplane/internal/pages" "github.com/runplane/runplane/internal/store" ) @@ -22,8 +20,15 @@ type Invite struct { } type Members struct { - Users []User `json:"users"` - Invites []Invite `json:"invites" doc:"Invites that are not accepted and not expired."` + Users []User `json:"users"` + Invites []Invite `json:"invites" doc:"Invites that are not accepted and not expired."` + LastAdmin string `json:"last_admin" doc:"ID of the only admin, which cannot lose the role or be deleted, because runplane needs an admin. Empty when two or more users are admins."` + Roles Roles `json:"roles" doc:"What each role can do."` +} + +type Roles struct { + Admin Capabilities `json:"admin"` + Member Capabilities `json:"member"` } func (h *Handler) listUsers(ctx context.Context, _ *struct{}) (*body[Members], error) { @@ -35,7 +40,10 @@ func (h *Handler) listUsers(ctx context.Context, _ *struct{}) (*body[Members], e if err != nil { return nil, err } - out := Members{Users: make([]User, len(users)), Invites: make([]Invite, len(invites))} + out := Members{ + Users: make([]User, len(users)), Invites: make([]Invite, len(invites)), LastAdmin: auth.LastAdmin(users), + Roles: Roles{Admin: capabilities(auth.RoleAdmin), Member: capabilities(auth.RoleMember)}, + } for i, u := range users { out.Users[i] = userOut(u) } @@ -57,65 +65,20 @@ func (h *Handler) updateUser(ctx context.Context, in *struct { PathID Body UpdateUserRequest }) (*body[User], error) { - u, err := h.d.Store.GetUser(ctx, in.ID) + u, err := h.d.Auth.SetRole(ctx, actor(ctx), in.ID, in.Body.Role) if err != nil { - return nil, notFound(err, "No user has this ID.") - } - if u.Role == in.Body.Role { - return &body[User]{userOut(u)}, nil + return nil, notFound(authErr(err, ""), "No user has this ID.") } - if err := h.keepAnAdmin(ctx, u); err != nil { - return nil, err - } - u.Role = in.Body.Role - if err := h.d.Store.UpdateUser(ctx, u); err != nil { - return nil, notFound(err, "No user has this ID.") - } - h.audit(ctx, "user.update", u.ID, map[string]any{"email": u.Email, "role": u.Role}) return &body[User]{userOut(u)}, nil } func (h *Handler) deleteUser(ctx context.Context, in *PathID) (*empty, error) { - if in.ID == currentUser(ctx).ID { - return nil, huma.Error409Conflict("You cannot delete your own account.") - } - u, err := h.d.Store.GetUser(ctx, in.ID) - if err != nil { - return nil, notFound(err, "No user has this ID.") - } - if err := h.keepAnAdmin(ctx, u); err != nil { - return nil, err + if err := h.d.Auth.DeleteUser(ctx, actor(ctx), in.ID); err != nil { + return nil, notFound(authErr(err, ""), "No user has this ID.") } - // The sessions of the user go with it. - if err := h.d.Store.DeleteUser(ctx, u.ID); err != nil { - return nil, notFound(err, "No user has this ID.") - } - h.audit(ctx, "user.delete", u.ID, map[string]any{"email": u.Email}) return nil, nil } -// keepAnAdmin returns a conflict when u is the only admin, before a change -// that takes the admin role from u. -func (h *Handler) keepAnAdmin(ctx context.Context, u store.User) error { - if u.Role != auth.RoleAdmin { - return nil - } - users, err := h.d.Store.ListUsers(ctx) - if err != nil { - return err - } - admins := 0 - for _, v := range users { - if v.Role == auth.RoleAdmin { - admins++ - } - } - if admins <= 1 { - return huma.Error409Conflict("runplane needs at least one admin.") - } - return nil -} - type CreateInviteRequest struct { Email string `json:"email" maxLength:"254"` Role string `json:"role" enum:"admin,member"` @@ -127,57 +90,33 @@ type CreatedInvite struct { } func (h *Handler) createInvite(ctx context.Context, in *body[CreateInviteRequest]) (*body[CreatedInvite], error) { - out, err := h.invite(ctx, in.Body.Email, in.Body.Role) + inv, token, err := h.d.Auth.CreateInvite(ctx, actor(ctx), in.Body.Email, in.Body.Role) if err != nil { - return nil, err + return nil, authErr(err, "") } - h.audit(ctx, "invite.create", out.Invite.ID, map[string]any{"email": out.Invite.Email, "role": out.Invite.Role}) - return &body[CreatedInvite]{out}, nil + return reply(h.createdInvite(ctx, inv, token)) } -// newInviteLink replaces an open invite with a new one for the same email -// and role, because only the hash of the old token is stored. func (h *Handler) newInviteLink(ctx context.Context, in *PathID) (*body[CreatedInvite], error) { - invites, err := h.d.Store.ListOpenInvites(ctx, time.Now()) - if err != nil { - return nil, err - } - i := slices.IndexFunc(invites, func(inv store.Invite) bool { return inv.ID == in.ID }) - if i < 0 { - return nil, huma.Error404NotFound("No open invite has this ID.") - } - out, err := h.invite(ctx, invites[i].Email, invites[i].Role) + inv, token, err := h.d.Auth.NewInviteLink(ctx, actor(ctx), in.ID) if err != nil { - return nil, err + return nil, notFound(authErr(err, ""), "No open invite has this ID.") } - h.audit(ctx, "invite.link", out.Invite.ID, map[string]any{"email": out.Invite.Email, "old_id": in.ID}) - return &body[CreatedInvite]{out}, nil + return reply(h.createdInvite(ctx, inv, token)) } -// invite makes an invite from the current user. Open invites for the email -// stop working. -func (h *Handler) invite(ctx context.Context, email, role string) (CreatedInvite, error) { - token, err := h.d.Auth.CreateInvite(ctx, currentUser(ctx).ID, email, role) - if err != nil { - return CreatedInvite{}, authErr(err, "") - } - inv, err := h.d.Auth.GetInvite(ctx, token) - if err != nil { - return CreatedInvite{}, err - } +func (h *Handler) createdInvite(ctx context.Context, inv store.Invite, token string) (CreatedInvite, error) { base, err := h.baseURL(ctx) if err != nil { return CreatedInvite{}, err } - return CreatedInvite{URL: base + "/invite/" + token, Invite: inviteOut(inv)}, nil + return CreatedInvite{URL: base + pages.Invite(token), Invite: inviteOut(inv)}, nil } func (h *Handler) deleteInvite(ctx context.Context, in *PathID) (*empty, error) { - inv, err := h.d.Store.DeleteInvite(ctx, in.ID) - if err != nil { + if err := h.d.Auth.DeleteInvite(ctx, actor(ctx), in.ID); err != nil { return nil, notFound(err, "No invite has this ID.") } - h.audit(ctx, "invite.delete", inv.ID, map[string]any{"email": inv.Email, "role": inv.Role}) return nil, nil } @@ -212,7 +151,7 @@ func (h *Handler) getInvite(ctx context.Context, in *InviteToken) (*body[InviteI type AcceptInviteRequest struct { Name string `json:"name" maxLength:"100"` - Password string `json:"password" maxLength:"1024"` + Password string `json:"password" minLength:"10" maxLength:"1024"` } func (h *Handler) acceptInvite(ctx context.Context, in *struct { diff --git a/internal/atomicfile/atomicfile.go b/internal/atomicfile/atomicfile.go index c9f70e0..1f4dc64 100644 --- a/internal/atomicfile/atomicfile.go +++ b/internal/atomicfile/atomicfile.go @@ -3,7 +3,6 @@ package atomicfile import ( - "errors" "fmt" "io" "os" @@ -13,17 +12,85 @@ import ( // Write copies r to a temporary file in the folder of path, syncs it, and // renames it to path. func Write(path string, r io.Reader, mode os.FileMode) error { - tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+"-*") + f, err := Create(path, mode) if err != nil { - return fmt.Errorf("write %s: %w", path, err) + return err } - defer os.Remove(tmp.Name()) - _, err = io.Copy(tmp, r) - if err = errors.Join(err, tmp.Chmod(mode), tmp.Sync(), tmp.Close()); err == nil { - err = os.Rename(tmp.Name(), path) + defer f.Close() + if _, err := io.Copy(f, r); err != nil { + return fmt.Errorf("write %s: %w", path, err) } + return f.Commit() +} + +// CreateOnce writes data to a new file at path with mode, and syncs it. If a +// file is at path, for example because another process made it first, it +// returns an error that wraps fs.ErrExist and keeps that file. +func CreateOnce(path string, data []byte, mode os.FileMode) error { + f, err := Create(path, mode) if err != nil { + return err + } + defer f.Close() + if _, err := f.Write(data); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + if err := f.Sync(); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + // A hard link puts the complete file in place. The link is not + // successful if a file is there. Close removes the temporary name. + if err := os.Link(f.Name(), path); err != nil { return fmt.Errorf("write %s: %w", path, err) } return nil } + +// File is a temporary file that replaces the file at its path on Commit. +type File struct { + *os.File + path string + committed bool +} + +// Create makes the temporary file for path in the folder of path. +func Create(path string, mode os.FileMode) (*File, error) { + return CreateIn(filepath.Dir(path), path, mode) +} + +// CreateIn makes the temporary file for path in dir, which must be on the +// file system of path. +func CreateIn(dir, path string, mode os.FileMode) (*File, error) { + f, err := os.CreateTemp(dir, "."+filepath.Base(path)+"-*") + if err != nil { + return nil, fmt.Errorf("write %s: %w", path, err) + } + if err := f.Chmod(mode); err != nil { + f.Close() + os.Remove(f.Name()) + return nil, fmt.Errorf("write %s: %w", path, err) + } + return &File{File: f, path: path}, nil +} + +// Commit syncs the file and renames it to its path. The file stays open. +func (f *File) Commit() error { + err := f.Sync() + if err == nil { + err = os.Rename(f.Name(), f.path) + } + if err != nil { + return fmt.Errorf("write %s: %w", f.path, err) + } + f.committed = true + return nil +} + +// Before a Commit, Close also removes the file. +func (f *File) Close() error { + err := f.File.Close() + if !f.committed { + os.Remove(f.Name()) + } + return err +} diff --git a/internal/audit/audit.go b/internal/audit/audit.go new file mode 100644 index 0000000..3558edf --- /dev/null +++ b/internal/audit/audit.go @@ -0,0 +1,135 @@ +// Package audit records who changed what: the users of the dashboard, and +// runplane itself for the commands that install.sh runs. +package audit + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "slices" + "time" + + "github.com/runplane/runplane/internal/ids" + "github.com/runplane/runplane/internal/store" +) + +// Action is the kind of change of an entry, as ".". +type Action string + +const ( + SetupDone Action = "setup.done" + UserUpdate Action = "user.update" + UserDelete Action = "user.delete" + UserPassword Action = "user.password" + UserPasswordReset Action = "user.password_reset" + InviteCreate Action = "invite.create" + InviteLink Action = "invite.link" + InviteDelete Action = "invite.delete" + PoolCreate Action = "pool.create" + PoolUpdate Action = "pool.update" + PoolDelete Action = "pool.delete" + ServerToken Action = "server.token" + ServerInstall Action = "server.install" + ServerUpdate Action = "server.update" + ServerDelete Action = "server.delete" + ServerUpdateAgent Action = "server.update_agent" + SandboxStop Action = "sandbox.stop" + SandboxShell Action = "sandbox.shell" + SandboxSSH Action = "sandbox.ssh" + CacheDelete Action = "cache.delete" + ImageCreate Action = "image.create" + ImageUpdate Action = "image.update" + ImageDelete Action = "image.delete" + ImageRefresh Action = "image.refresh" + SettingsUpdate Action = "settings.update" + JobRerun Action = "job.rerun" + BackupCreate Action = "backup.create" + BackupDownload Action = "backup.download" + UpdateApply Action = "update.apply" + GitHubManifest Action = "github.manifest" + GitHubAppCreate Action = "github.app.create" + // The returns from the GitHub page of the app installation, one for each + // setup_action of GitHub. + GitHubInstall Action = "github.installation.install" + GitHubInstallUpdate Action = "github.installation.update" + GitHubInstallRequest Action = "github.installation.request" + GitHubSync Action = "github.sync" + GitHubPullRequest Action = "github.pull_request.create" + GitHubDisconnect Action = "github.disconnect" + ChannelCreate Action = "notification.create" + ChannelUpdate Action = "notification.update" + ChannelDelete Action = "notification.delete" +) + +var Actions = []Action{ + SetupDone, UserUpdate, UserDelete, UserPassword, UserPasswordReset, InviteCreate, InviteLink, InviteDelete, + PoolCreate, PoolUpdate, PoolDelete, + ServerToken, ServerInstall, ServerUpdate, ServerDelete, ServerUpdateAgent, + SandboxStop, SandboxShell, SandboxSSH, CacheDelete, + ImageCreate, ImageUpdate, ImageDelete, ImageRefresh, + SettingsUpdate, JobRerun, BackupCreate, BackupDownload, UpdateApply, + GitHubManifest, GitHubAppCreate, GitHubInstall, GitHubInstallUpdate, GitHubInstallRequest, + GitHubSync, GitHubPullRequest, GitHubDisconnect, + ChannelCreate, ChannelUpdate, ChannelDelete, +} + +// Actor makes a change. The zero Actor is runplane itself. +type Actor struct { + UserID string + IP string // of the request +} + +// Log writes the entries of the audit log. +type Log struct { + Store *store.Store + Logger *slog.Logger // nil: no log +} + +// Record writes an entry. A failure goes to the log and does not fail the +// change, because the change is done already. +func (l Log) Record(ctx context.Context, by Actor, action Action, target string, detail map[string]any) { + var raw json.RawMessage + if detail != nil { + b, err := json.Marshal(detail) + if err != nil { + l.logger().Error("encode audit detail", "action", action, "err", err) + } + raw = b + } + err := l.Store.InsertAudit(context.WithoutCancel(ctx), store.AuditEntry{ + ID: ids.New(), At: time.Now(), UserID: by.UserID, Action: string(action), Target: target, Detail: raw, IP: by.IP, + }) + if err != nil { + l.logger().Error("write audit entry", "action", action, "target", target, "err", err) + } +} + +func (l Log) logger() *slog.Logger { + if l.Logger == nil { + return slog.New(slog.DiscardHandler) + } + return l.Logger +} + +// Changes gives the old and new JSON value of each field that differs +// between a and b, two values of one type, except the fields in skip. +func Changes(a, b any, skip ...string) map[string][2]any { + fa, fb := jsonFields(a), jsonFields(b) + changes := map[string][2]any{} + for _, fields := range []map[string]json.RawMessage{fa, fb} { + for k := range fields { + if !slices.Contains(skip, k) && !bytes.Equal(fa[k], fb[k]) { + changes[k] = [2]any{fa[k], fb[k]} + } + } + } + return changes +} + +func jsonFields(v any) map[string]json.RawMessage { + var fields map[string]json.RawMessage + b, _ := json.Marshal(v) // the store models always encode + json.Unmarshal(b, &fields) + return fields +} diff --git a/internal/audit/audit_test.go b/internal/audit/audit_test.go new file mode 100644 index 0000000..de246bb --- /dev/null +++ b/internal/audit/audit_test.go @@ -0,0 +1,40 @@ +package audit + +import ( + "go/ast" + "go/parser" + "go/token" + "slices" + "strconv" + "testing" +) + +// The API gives Actions to the web as an enum, and the web shows a text for +// each one. An action that is not in Actions shows as "did ". +func TestActionsHasEachAction(t *testing.T) { + f, err := parser.ParseFile(token.NewFileSet(), "audit.go", nil, 0) + if err != nil { + t.Fatal(err) + } + found := 0 + for _, d := range f.Decls { + g, ok := d.(*ast.GenDecl) + if !ok || g.Tok != token.CONST { + continue + } + for _, s := range g.Specs { + spec := s.(*ast.ValueSpec) + value, err := strconv.Unquote(spec.Values[0].(*ast.BasicLit).Value) + if err != nil { + t.Fatal(err) + } + found++ + if !slices.Contains(Actions, Action(value)) { + t.Errorf("Actions does not have %s (%q)", spec.Names[0].Name, value) + } + } + } + if found != len(Actions) { + t.Errorf("audit.go has %d actions, and Actions has %d", found, len(Actions)) + } +} diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 7524f61..2955dc3 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -11,11 +11,14 @@ import ( "encoding/hex" "errors" "fmt" + "log/slog" "net/mail" + "slices" "strings" "time" "unicode/utf8" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/ids" "github.com/runplane/runplane/internal/store" ) @@ -37,6 +40,8 @@ const ( RoleMember = store.RoleMember ) +var Roles = []string{RoleAdmin, RoleMember} + type Action int const ( @@ -48,22 +53,26 @@ const ( ) var ( - ErrSetupDone = store.ErrSetupDone - ErrBadSetupCode = errors.New("the setup code is wrong, used, or expired") - ErrBadLogin = errors.New("the email or password is wrong") - ErrBadPassword = errors.New("the current password is wrong") - ErrRateLimited = errors.New("too many failed attempts, try again later") - ErrNoSession = errors.New("no valid session") - ErrShortPassword = fmt.Errorf("the password must have at least %d characters", MinPasswordLen) - ErrBadEmail = errors.New("the email address is not valid") - ErrBadName = errors.New("enter a name") - ErrBadRole = errors.New("the role must be admin or member") - ErrBadInvite = errors.New("the invite is wrong, used, or expired") - ErrEmailTaken = errors.New("a user with this email exists") + ErrSetupDone = store.ErrSetupDone + ErrLastAdmin = store.ErrLastAdmin + ErrOwnAccount = errors.New("you cannot delete your own account") + ErrBadSetupCode = errors.New("the setup code is wrong, used, or expired") + ErrBadLogin = errors.New("the email or password is wrong") + ErrBadPassword = errors.New("the current password is wrong") + ErrNoCurrentPassword = errors.New("enter your current password") + ErrRateLimited = errors.New("too many failed attempts, try again later") + ErrNoSession = errors.New("no valid session") + ErrShortPassword = fmt.Errorf("the password must have at least %d characters", MinPasswordLen) + ErrBadEmail = errors.New("the email address is not valid") + ErrBadName = errors.New("enter a name") + ErrBadRole = errors.New("the role must be admin or member") + ErrBadInvite = errors.New("the invite is wrong, used, or expired") + ErrEmailTaken = errors.New("a user with this email exists") ) type Service struct { store *store.Store + audit audit.Log limiter *limiter deviceKey []byte // signs device tokens now func() time.Time @@ -71,8 +80,12 @@ type Service struct { // NewService returns the accounts of st. deviceKey signs the device tokens // of browsers that logged in; a service without one makes no valid tokens. -func NewService(st *store.Store, deviceKey []byte) *Service { - return &Service{store: st, limiter: &limiter{fails: map[string][]time.Time{}}, deviceKey: deviceKey, now: time.Now} +// log gets the failed writes of the audit log, and can be nil. +func NewService(st *store.Store, deviceKey []byte, log *slog.Logger) *Service { + return &Service{ + store: st, audit: audit.Log{Store: st, Logger: log}, limiter: &limiter{fails: map[string][]time.Time{}}, + deviceKey: deviceKey, now: time.Now, + } } // CanDo reports if a user with role can do the action. An unknown role can do nothing. @@ -92,6 +105,17 @@ func CanDo(role string, a Action) bool { // NewSetupCode makes a new setup code in the format XXXX-XXXX-XXXX. It is // valid for 24 hours, and the earlier codes stop working. func (s *Service) NewSetupCode(ctx context.Context) (string, error) { + return s.setupCode(ctx, false) +} + +// EnsureSetupCode makes a setup code as NewSetupCode does, but only when no +// code is valid. It returns "" while a code is valid, because install.sh +// printed that code and only its hash is stored. +func (s *Service) EnsureSetupCode(ctx context.Context) (string, error) { + return s.setupCode(ctx, true) +} + +func (s *Service) setupCode(ctx context.Context, keepValid bool) (string, error) { n, err := s.store.CountUsers(ctx) if err != nil { return "", err @@ -99,8 +123,13 @@ func (s *Service) NewSetupCode(ctx context.Context) (string, error) { if n > 0 { return "", ErrSetupDone } - code := newSetupCode() now := s.now() + if keepValid { + if open, err := s.store.HasOpenSetupCode(ctx, now); err != nil || open { + return "", err + } + } + code := newSetupCode() err = s.store.CreateSetupCode(ctx, store.SetupCode{ CodeHash: hashCode(code), CreatedAt: now, ExpiresAt: now.Add(SetupCodeTTL), }) @@ -136,20 +165,14 @@ func (s *Service) Setup(ctx context.Context, code, name, email, password, ip, us case err != nil: return store.User{}, "", err } + s.audit.Record(ctx, audit.Actor{UserID: u.ID, IP: ip}, audit.SetupDone, u.ID, map[string]any{"email": u.Email}) return u, token, nil } // SetupCodeValid reports whether code is an unused setup code that has not // expired. Unlike Setup, it does not use the code. func (s *Service) SetupCodeValid(ctx context.Context, code string) (bool, error) { - var n int - err := s.store.DB().QueryRowContext(ctx, - "SELECT count(*) FROM setup_codes WHERE code_hash = ? AND used_at IS NULL AND expires_at > ?", - hashCode(code), s.now().UnixMilli()).Scan(&n) - if err != nil { - return false, fmt.Errorf("check setup code: %w", err) - } - return n > 0, nil + return s.store.SetupCodeOpen(ctx, hashCode(code), s.now()) } // Login checks the password and returns the user and a new session token. @@ -265,46 +288,184 @@ func (s *Service) ChangePassword(ctx context.Context, userID, current, next, ip, if utf8.RuneCountInString(next) < MinPasswordLen { return "", ErrShortPassword } - if err := s.store.SetUserPassword(ctx, u.ID, HashPassword(next)); err != nil { - return "", err - } - if err := s.store.DeleteUserSessions(ctx, u.ID); err != nil { + if err := s.setPassword(ctx, u.ID, next); err != nil { return "", err } token, sess := newSession(u.ID, ip, userAgent, now) if err := s.store.CreateSession(ctx, sess); err != nil { return "", err } + s.audit.Record(ctx, audit.Actor{UserID: u.ID, IP: ip}, audit.UserPassword, u.ID, nil) return token, nil } -// CreateInvite returns the token for the invite URL. The invite is valid -// for 7 days, and earlier open invites for the email stop working. -func (s *Service) CreateInvite(ctx context.Context, by, email, role string) (string, error) { +// ResetPassword gives the user with the email a new random password, ends +// all sessions of the user, and returns the password. It returns +// store.ErrNotFound when no user has the email. +func (s *Service) ResetPassword(ctx context.Context, by audit.Actor, email string) (string, error) { + u, err := s.store.GetUserByEmail(ctx, cleanEmail(email)) + if err != nil { + return "", err + } + password := rand.Text() + if err := s.setPassword(ctx, u.ID, password); err != nil { + return "", err + } + s.audit.Record(ctx, by, audit.UserPasswordReset, u.ID, map[string]any{"email": u.Email}) + return password, nil +} + +// setPassword changes the password of a user. It ends all sessions of the +// user, because each browser must log in with the new password. +func (s *Service) setPassword(ctx context.Context, userID, password string) error { + if err := s.store.SetUserPassword(ctx, userID, HashPassword(password)); err != nil { + return err + } + return s.store.DeleteUserSessions(ctx, userID) +} + +// UpdateMe changes the name or the password of a user, for the user, and +// returns the user. A nil name or next keeps it. A new password needs the +// current one (ErrNoCurrentPassword), works as in ChangePassword, and the +// token is "" without one. A refused name changes nothing. +func (s *Service) UpdateMe(ctx context.Context, userID string, name, current, next *string, ip, userAgent string) (store.User, string, error) { + var newName string + if name != nil { + var err error + if newName, err = userName(*name); err != nil { + return store.User{}, "", err + } + } + var token string + if next != nil { + if current == nil || *current == "" { + return store.User{}, "", ErrNoCurrentPassword + } + var err error + if token, err = s.ChangePassword(ctx, userID, *current, *next, ip, userAgent); err != nil { + return store.User{}, "", err + } + } + if name == nil { + u, err := s.store.GetUser(ctx, userID) + return u, token, err + } + u, err := s.store.SetUserName(ctx, userID, newName) + return u, token, err +} + +// SetRole gives a user a role, and returns the user. It returns ErrLastAdmin +// for the only admin, because runplane needs one. +func (s *Service) SetRole(ctx context.Context, by audit.Actor, userID, role string) (store.User, error) { + if !slices.Contains(Roles, role) { + return store.User{}, ErrBadRole + } + u, err := s.store.SetUserRole(ctx, userID, role) + if err != nil { + return store.User{}, err + } + if u.Role != role { + u.Role = role + s.audit.Record(ctx, by, audit.UserUpdate, u.ID, map[string]any{"email": u.Email, "role": u.Role}) + } + return u, nil +} + +// DeleteUser deletes a user with its sessions. A user cannot delete its own +// account, and runplane keeps its last admin (ErrLastAdmin). +func (s *Service) DeleteUser(ctx context.Context, by audit.Actor, userID string) error { + if userID == by.UserID { + return ErrOwnAccount + } + u, err := s.store.DeleteUser(ctx, userID) + if err != nil { + return err + } + s.audit.Record(ctx, by, audit.UserDelete, u.ID, map[string]any{"email": u.Email}) + return nil +} + +// LastAdmin returns the ID of the only admin of users, or "" when two or more +// users are admins. That user cannot lose the role or be deleted. +func LastAdmin(users []store.User) string { + var admins []string + for _, u := range users { + if u.Role == RoleAdmin { + admins = append(admins, u.ID) + } + } + if len(admins) != 1 { + return "" + } + return admins[0] +} + +// CreateInvite makes an invite, and returns it with the token for the invite +// URL. The invite is valid for 7 days, and earlier open invites for the email +// stop working. +func (s *Service) CreateInvite(ctx context.Context, by audit.Actor, email, role string) (store.Invite, string, error) { + inv, token, err := s.invite(ctx, by, email, role) + if err != nil { + return store.Invite{}, "", err + } + s.audit.Record(ctx, by, audit.InviteCreate, inv.ID, map[string]any{"email": inv.Email, "role": inv.Role}) + return inv, token, nil +} + +// NewInviteLink replaces the open invite id with a new one for the same +// email and role, because only the hash of the old token is stored. It +// returns store.ErrNotFound when no open invite has the ID. +func (s *Service) NewInviteLink(ctx context.Context, by audit.Actor, id string) (store.Invite, string, error) { + invites, err := s.store.ListOpenInvites(ctx, s.now()) + if err != nil { + return store.Invite{}, "", err + } + i := slices.IndexFunc(invites, func(inv store.Invite) bool { return inv.ID == id }) + if i < 0 { + return store.Invite{}, "", fmt.Errorf("new invite link: %w", store.ErrNotFound) + } + inv, token, err := s.invite(ctx, by, invites[i].Email, invites[i].Role) + if err != nil { + return store.Invite{}, "", err + } + s.audit.Record(ctx, by, audit.InviteLink, inv.ID, map[string]any{"email": inv.Email, "old_id": id}) + return inv, token, nil +} + +func (s *Service) DeleteInvite(ctx context.Context, by audit.Actor, id string) error { + inv, err := s.store.DeleteInvite(ctx, id) + if err != nil { + return err + } + s.audit.Record(ctx, by, audit.InviteDelete, inv.ID, map[string]any{"email": inv.Email, "role": inv.Role}) + return nil +} + +func (s *Service) invite(ctx context.Context, by audit.Actor, email, role string) (store.Invite, string, error) { email = cleanEmail(email) if !validEmail(email) { - return "", ErrBadEmail + return store.Invite{}, "", ErrBadEmail } - if role != RoleAdmin && role != RoleMember { - return "", ErrBadRole + if !slices.Contains(Roles, role) { + return store.Invite{}, "", ErrBadRole } _, err := s.store.GetUserByEmail(ctx, email) if err == nil { - return "", ErrEmailTaken + return store.Invite{}, "", ErrEmailTaken } if !errors.Is(err, store.ErrNotFound) { - return "", err + return store.Invite{}, "", err } token, hash := newToken() now := s.now() - err = s.store.CreateInvite(ctx, store.Invite{ - ID: ids.New(), Email: email, Role: role, TokenHash: hash, CreatedBy: by, + inv := store.Invite{ + ID: ids.New(), Email: email, Role: role, TokenHash: hash, CreatedBy: by.UserID, CreatedAt: now, ExpiresAt: now.Add(InviteTTL), - }) - if err != nil { - return "", err } - return token, nil + if err := s.store.CreateInvite(ctx, inv); err != nil { + return store.Invite{}, "", err + } + return inv, token, nil } // GetInvite returns an open invite, or ErrBadInvite. @@ -353,9 +514,9 @@ func (s *Service) deviceMAC(userID, id string) string { // newUser checks the name and the password, and returns a user with no email and role. func newUser(name, password string, now time.Time) (store.User, error) { - name = strings.TrimSpace(name) - if name == "" { - return store.User{}, ErrBadName + name, err := userName(name) + if err != nil { + return store.User{}, err } if utf8.RuneCountInString(password) < MinPasswordLen { return store.User{}, ErrShortPassword @@ -365,6 +526,14 @@ func newUser(name, password string, now time.Time) (store.User, error) { }, nil } +func userName(name string) (string, error) { + name = strings.TrimSpace(name) + if name == "" { + return "", ErrBadName + } + return name, nil +} + func newSession(userID, ip, userAgent string, now time.Time) (string, store.Session) { token, hash := newToken() return token, store.Session{ diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 4b2549c..7e29163 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -6,10 +6,12 @@ import ( "fmt" "path/filepath" "regexp" + "slices" "strings" "testing" "time" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/store" ) @@ -27,7 +29,7 @@ func newEnv(t *testing.T) *testEnv { } t.Cleanup(func() { st.Close() }) e := &testEnv{st: st, now: time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)} - e.svc = NewService(st, []byte("device key")) + e.svc = NewService(st, []byte("device key"), nil) e.svc.now = func() time.Time { return e.now } return e } @@ -58,9 +60,6 @@ func TestPasswordHashFormat(t *testing.T) { // Made with the reference implementation (libargon2): // echo -n "correct horse battery" | argon2 saltsaltsaltsalt -id -t 2 -k 19456 -p 1 -l 32 const ref = "$argon2id$v=19$m=19456,t=2,p=1$c2FsdHNhbHRzYWx0c2FsdA$iPOQ5f2O21FsjnBvo1AiFcDuSXciCnKUrXFO+yfgNoM" - if got := hashWithSalt("correct horse battery", []byte("saltsaltsaltsalt")); got != ref { - t.Errorf("hash:\n got %s\nwant %s", got, ref) - } if ok, err := CheckPassword(ref, "correct horse battery"); !ok || err != nil { t.Errorf("right password: %v, %v", ok, err) } @@ -72,6 +71,24 @@ func TestPasswordHashFormat(t *testing.T) { t.Errorf("CheckPassword(%q) has no error", bad) } } + + // CheckPassword reads the parameters from the hash, so HashPassword must + // write the ones that it used, at least the OWASP minimum of 19 MiB and + // 2 passes, with a new salt for each hash. + hash := HashPassword("correct horse battery") + if ok, err := CheckPassword(hash, "correct horse battery"); !ok || err != nil { + t.Errorf("right password against %s: %v, %v", hash, ok, err) + } + if ok, err := CheckPassword(hash, "correct horse batterY"); ok || err != nil { + t.Errorf("wrong password against %s: %v, %v", hash, ok, err) + } + var memory, passes, threads int + if _, err := fmt.Sscanf(hash, "$argon2id$v=19$m=%d,t=%d,p=%d$", &memory, &passes, &threads); err != nil || memory < 19456 || passes < 2 { + t.Errorf("hash %s, want argon2id with m >= 19456 and t >= 2", hash) + } + if again := HashPassword("correct horse battery"); again == hash { + t.Error("two hashes of one password are equal: the salt is not random") + } } func TestLoginWrongPassword(t *testing.T) { @@ -150,7 +167,7 @@ func TestLoginFromAKnownDevice(t *testing.T) { "no device": "", "device of another user": e.svc.Device("someone else", ""), "device with a changed ID": "x" + id + "." + mac, - "device of a key that is wrong": NewService(e.st, []byte("other key")).Device(admin.ID, ""), + "device of a key that is wrong": NewService(e.st, []byte("other key"), nil).Device(admin.ID, ""), } { if err := e.login("admin@example.com", "first password", token, "10.9.9.9"); !errors.Is(err, ErrRateLimited) { t.Errorf("%s with the email locked: err = %v, want ErrRateLimited", name, err) @@ -231,28 +248,23 @@ func TestSetupCode(t *testing.T) { t.Fatalf("short password: err = %v", err) } - // Lowercase without dashes also works. - typed := code[0:4] + code[5:9] + code[10:14] - u, err := setup(" " + strings.ToLower(typed) + " ") + expiring, err := e.svc.NewSetupCode(ctx) if err != nil { - t.Fatalf("setup: %v", err) - } - - // The code is used: it fails even when no user exists again. - if err := e.st.DeleteUser(ctx, u.ID); err != nil { t.Fatal(err) } - if _, err := setup(code); !errors.Is(err, ErrBadSetupCode) { - t.Fatalf("used code: err = %v, want ErrBadSetupCode", err) + e.wait(24 * time.Hour) + if _, err := setup(expiring); !errors.Is(err, ErrBadSetupCode) { + t.Fatalf("expired code: err = %v, want ErrBadSetupCode", err) } - expiring, err := e.svc.NewSetupCode(ctx) + // Lowercase without dashes also works. + code, err = e.svc.NewSetupCode(ctx) if err != nil { t.Fatal(err) } - e.wait(24 * time.Hour) - if _, err := setup(expiring); !errors.Is(err, ErrBadSetupCode) { - t.Fatalf("expired code: err = %v, want ErrBadSetupCode", err) + typed := code[0:4] + code[5:9] + code[10:14] + if _, err := setup(" " + strings.ToLower(typed) + " "); err != nil { + t.Fatalf("setup: %v", err) } } @@ -274,23 +286,60 @@ func TestSetupRefusedAfterFirstAdmin(t *testing.T) { if _, err := e.svc.NewSetupCode(ctx); !errors.Is(err, ErrSetupDone) { t.Fatalf("new code after the first user: err = %v, want ErrSetupDone", err) } + // The valid code does not make serve tell that no admin exists. + if _, err := e.svc.EnsureSetupCode(ctx); !errors.Is(err, ErrSetupDone) { + t.Fatalf("ensure a code after the first user: err = %v, want ErrSetupDone", err) + } if n, _ := e.st.CountUsers(ctx); n != 1 { t.Fatalf("users = %d, want 1", n) } } +// install.sh prints a code, then runplane restarts. serve keeps the code. +func TestEnsureSetupCodeKeepsAValidCode(t *testing.T) { + e := newEnv(t) + ctx := context.Background() + printed, err := e.svc.NewSetupCode(ctx) + if err != nil { + t.Fatal(err) + } + if code, err := e.svc.EnsureSetupCode(ctx); err != nil || code != "" { + t.Fatalf("EnsureSetupCode = %q, %v; want no new code while one is valid", code, err) + } + if _, _, err := e.svc.Setup(ctx, printed, "Ada", "ada@example.com", "a long password", "127.0.0.1", "test"); err != nil { + t.Fatalf("the printed code stopped working after a restart: %v", err) + } +} + +func TestEnsureSetupCodeReplacesAnExpiredCode(t *testing.T) { + e := newEnv(t) + ctx := context.Background() + if _, err := e.svc.NewSetupCode(ctx); err != nil { + t.Fatal(err) + } + e.wait(SetupCodeTTL) + code, err := e.svc.EnsureSetupCode(ctx) + if err != nil || code == "" { + t.Fatalf("EnsureSetupCode = %q, %v; want a new code", code, err) + } + if _, _, err := e.svc.Setup(ctx, code, "Ada", "ada@example.com", "a long password", "127.0.0.1", "test"); err != nil { + t.Fatalf("the new code does not work: %v", err) + } +} + func TestInviteSingleUse(t *testing.T) { e := newEnv(t) ctx := context.Background() admin, _ := e.setup(t) - if _, err := e.svc.CreateInvite(ctx, admin.ID, "admin@example.com", RoleMember); !errors.Is(err, ErrEmailTaken) { + by := audit.Actor{UserID: admin.ID} + if _, _, err := e.svc.CreateInvite(ctx, by, "admin@example.com", RoleMember); !errors.Is(err, ErrEmailTaken) { t.Fatalf("invite for a user: err = %v", err) } - if _, err := e.svc.CreateInvite(ctx, admin.ID, "bob@example.com", "owner"); !errors.Is(err, ErrBadRole) { + if _, _, err := e.svc.CreateInvite(ctx, by, "bob@example.com", "owner"); !errors.Is(err, ErrBadRole) { t.Fatalf("bad role: err = %v", err) } - token, err := e.svc.CreateInvite(ctx, admin.ID, " Bob@Example.com", RoleMember) + _, token, err := e.svc.CreateInvite(ctx, by, " Bob@Example.com", RoleMember) if err != nil { t.Fatal(err) } @@ -316,7 +365,7 @@ func TestInviteSingleUse(t *testing.T) { t.Fatalf("second accept: err = %v, want ErrBadInvite", err) } - late, err := e.svc.CreateInvite(ctx, admin.ID, "carol@example.com", RoleAdmin) + _, late, err := e.svc.CreateInvite(ctx, by, "carol@example.com", RoleAdmin) if err != nil { t.Fatal(err) } @@ -355,24 +404,100 @@ func TestChangePassword(t *testing.T) { } } -func TestCanDo(t *testing.T) { - tests := []struct { - role string - a Action - want bool - }{ - {RoleAdmin, ActionAdmin, true}, - {RoleMember, ActionRead, true}, - {RoleMember, ActionRerunJob, true}, - {RoleMember, ActionStopSandbox, true}, - {RoleMember, ActionOpenShell, true}, - {RoleMember, ActionAdmin, false}, - {"", ActionRead, false}, - {"owner", ActionRead, false}, - } - for _, tt := range tests { - if got := CanDo(tt.role, tt.a); got != tt.want { - t.Errorf("CanDo(%q, %d) = %v, want %v", tt.role, tt.a, got, tt.want) - } +// The operator resets a password from the shell. The Activity page shows it. +func TestResetPassword(t *testing.T) { + e := newEnv(t) + ctx := context.Background() + admin, oldToken := e.setup(t) + if _, err := e.svc.ResetPassword(ctx, audit.Actor{}, "eve@example.com"); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("unknown email: err = %v, want store.ErrNotFound", err) + } + password, err := e.svc.ResetPassword(ctx, audit.Actor{}, " Admin@Example.com ") + if err != nil { + t.Fatal(err) + } + if _, err := e.svc.UserForToken(ctx, oldToken); !errors.Is(err, ErrNoSession) { + t.Errorf("session after the reset: err = %v, want ErrNoSession", err) + } + if err := e.login("admin@example.com", "first password", "", ""); !errors.Is(err, ErrBadLogin) { + t.Errorf("old password: err = %v, want ErrBadLogin", err) + } + if err := e.login("admin@example.com", password, "", ""); err != nil { + t.Errorf("new password: %v", err) + } + entries, _, err := e.st.ListAudit(ctx, "", 10) + if err != nil { + t.Fatal(err) + } + resets := slices.DeleteFunc(entries, func(a store.AuditEntry) bool { return a.Action != "user.password_reset" }) + if len(resets) != 1 || resets[0].Target != admin.ID || resets[0].UserID != "" { + t.Errorf("reset entries %+v, want one of %s by runplane", resets, admin.ID) + } +} + +// A refused name must not leave a changed password behind. +func TestUpdateMeChangesNothingForABadName(t *testing.T) { + e := newEnv(t) + ctx := context.Background() + admin, _ := e.setup(t) + blank, current, next := " ", "first password", "second password" + if _, _, err := e.svc.UpdateMe(ctx, admin.ID, &blank, ¤t, &next, "", ""); !errors.Is(err, ErrBadName) { + t.Fatalf("blank name: err = %v, want ErrBadName", err) + } + if err := e.login("admin@example.com", "first password", "", ""); err != nil { + t.Fatalf("old password after the refused change: %v", err) + } + name := " Ada L " + u, token, err := e.svc.UpdateMe(ctx, admin.ID, &name, nil, nil, "", "") + if err != nil || u.Name != "Ada L" || token != "" { + t.Fatalf("rename: %+v, token %q, %v; want Ada L and no new session", u, token, err) + } +} + +func TestUpdateMeNeedsTheCurrentPassword(t *testing.T) { + e := newEnv(t) + admin, _ := e.setup(t) + next := "second password" + if _, _, err := e.svc.UpdateMe(context.Background(), admin.ID, nil, nil, &next, "", ""); !errors.Is(err, ErrNoCurrentPassword) { + t.Fatalf("new password without the current one: err = %v, want ErrNoCurrentPassword", err) + } +} + +// runplane keeps an admin: the last admin cannot lose the role or be +// deleted, and nobody deletes the own account. +func TestLastAdminStays(t *testing.T) { + e := newEnv(t) + ctx := context.Background() + ada, _ := e.setup(t) + bob := store.User{ID: "bob", Email: "bob@example.com", Name: "Bob", Role: RoleAdmin, CreatedAt: e.now} + if err := e.st.CreateUser(ctx, bob); err != nil { + t.Fatal(err) + } + by := audit.Actor{UserID: ada.ID} + + if err := e.svc.DeleteUser(ctx, by, ada.ID); !errors.Is(err, ErrOwnAccount) { + t.Errorf("delete of the own account: err = %v, want ErrOwnAccount", err) + } + if u, err := e.svc.SetRole(ctx, by, ada.ID, RoleMember); err != nil || u.Role != RoleMember { + t.Fatalf("one of two admins becomes a member: %+v, %v", u, err) + } + users, err := e.st.ListUsers(ctx) + if err != nil { + t.Fatal(err) + } + if got := LastAdmin(users); got != "bob" { + t.Errorf("last admin %q, want bob", got) + } + if _, err := e.svc.SetRole(ctx, by, "bob", RoleMember); !errors.Is(err, ErrLastAdmin) { + t.Errorf("the last admin becomes a member: err = %v, want ErrLastAdmin", err) + } + if err := e.svc.DeleteUser(ctx, by, "bob"); !errors.Is(err, ErrLastAdmin) { + t.Errorf("delete of the last admin: err = %v, want ErrLastAdmin", err) + } + if u, err := e.st.GetUser(ctx, "bob"); err != nil || u.Role != RoleAdmin { + t.Errorf("bob after the refused changes: %+v, %v; want an admin", u, err) + } + if err := e.svc.DeleteUser(ctx, audit.Actor{UserID: "bob"}, ada.ID); err != nil { + t.Errorf("delete of a member: %v", err) } } diff --git a/internal/auth/prune.go b/internal/auth/prune.go new file mode 100644 index 0000000..42dc98f --- /dev/null +++ b/internal/auth/prune.go @@ -0,0 +1,11 @@ +package auth + +import ( + "context" + "time" +) + +// Prune deletes the sessions, setup codes, and invites that expired at now. +func (s *Service) Prune(ctx context.Context, now time.Time) error { + return s.store.DeleteExpiredAuth(ctx, now) +} diff --git a/internal/backup/backup.go b/internal/backup/backup.go index e68215b..15c345a 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -1,11 +1,11 @@ -// Package backup copies the SQLite database to gzip files, keeps the newest -// ones, and uploads them to S3-compatible storage. +// Package backup makes the backups of the database: it copies the SQLite +// database to gzip files, keeps the newest ones, and uploads them to +// S3-compatible storage. package backup import ( "compress/gzip" "context" - "database/sql" "errors" "fmt" "io" @@ -15,13 +15,14 @@ import ( "slices" "strings" "time" + + "github.com/runplane/runplane/internal/atomicfile" ) -// Result describes one backup. -type Result struct { - Path string // the new backup file - Size int64 // its size in bytes - Removed []string // older backup files that Run removed +type result struct { + path string // the new backup file + size int64 // its size in bytes + removed []string // older backup files that write removed } const ( @@ -29,31 +30,30 @@ const ( filePattern = "runplane-????????T??????.???Z.db.gz" ) -// Run copies the database with VACUUM INTO, writes it to -// dir/runplane-.db.gz, and removes the oldest backups so that -// keep files stay. If keep is 0 or less, Run removes no backups. -func Run(ctx context.Context, db *sql.DB, dir string, keep int) (Result, error) { +// write copies the database with vacuumInto to dir/runplane-.db.gz +// and removes the oldest backups so that keep files stay. If keep is 0 or +// less, write removes no backups. +func write(ctx context.Context, vacuumInto func(ctx context.Context, path string) error, dir string, keep int) (result, error) { if err := os.MkdirAll(dir, 0o700); err != nil { - return Result{}, fmt.Errorf("make backup dir: %w", err) + return result{}, fmt.Errorf("make backup dir: %w", err) } name, path := freeName(dir, time.Now().UTC()) - // VACUUM INTO writes a consistent copy that includes the WAL, and it - // does not block writers. It fails if the target file exists. + // VACUUM INTO fails if the target file exists. snapshot := filepath.Join(dir, "."+name+".db") os.Remove(snapshot) defer os.Remove(snapshot) - if _, err := db.ExecContext(ctx, "VACUUM INTO ?", snapshot); err != nil { - return Result{}, fmt.Errorf("vacuum into snapshot: %w", err) + if err := vacuumInto(ctx, snapshot); err != nil { + return result{}, err } size, err := gzipFile(snapshot, path) if err != nil { - return Result{}, err + return result{}, err } - res := Result{Path: path, Size: size} + res := result{path: path, size: size} if keep > 0 { - if res.Removed, err = prune(dir, keep); err != nil { + if res.removed, err = prune(dir, keep); err != nil { return res, err } } @@ -73,8 +73,7 @@ func freeName(dir string, t time.Time) (string, string) { } } -// gzipFile compresses src to dst through a temporary file and returns the -// size of dst. +// gzipFile compresses src to dst and returns the size of dst. func gzipFile(src, dst string) (int64, error) { in, err := os.Open(src) if err != nil { @@ -82,13 +81,11 @@ func gzipFile(src, dst string) (int64, error) { } defer in.Close() - tmp := dst + ".part" - out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + out, err := atomicfile.Create(dst, 0o600) if err != nil { return 0, fmt.Errorf("create backup: %w", err) } - defer os.Remove(tmp) - + defer out.Close() zw := gzip.NewWriter(out) zw.Name = strings.TrimSuffix(filepath.Base(dst), ".gz") _, err = io.Copy(zw, in) @@ -96,18 +93,12 @@ func gzipFile(src, dst string) (int64, error) { err = zw.Close() } if err == nil { - err = out.Sync() - } - if cerr := out.Close(); err == nil { - err = cerr + err = out.Commit() } if err != nil { return 0, fmt.Errorf("write backup: %w", err) } - if err := os.Rename(tmp, dst); err != nil { - return 0, fmt.Errorf("write backup: %w", err) - } - info, err := os.Stat(dst) + info, err := out.Stat() if err != nil { return 0, fmt.Errorf("stat backup: %w", err) } @@ -135,8 +126,11 @@ func prune(dir string, keep int) ([]string, error) { return removed, nil } -// Daily calls job each day at 03:00 UTC until ctx ends. -func Daily(ctx context.Context, job func(context.Context)) { +// DailyAt is the time of day in UTC of the daily backup. +const DailyAt = 3 * time.Hour + +// daily calls job each day at DailyAt until ctx ends. +func daily(ctx context.Context, job func(context.Context)) { for { t := time.NewTimer(time.Until(nextRun(time.Now()))) select { @@ -149,10 +143,10 @@ func Daily(ctx context.Context, job func(context.Context)) { } } -// nextRun returns the first 03:00 UTC after now. +// nextRun returns the first DailyAt after now. func nextRun(now time.Time) time.Time { now = now.UTC() - next := time.Date(now.Year(), now.Month(), now.Day(), 3, 0, 0, 0, time.UTC) + next := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).Add(DailyAt) if !next.After(now) { next = next.AddDate(0, 0, 1) } diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 3c20dae..2cc74aa 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -1,22 +1,17 @@ package backup import ( - "compress/gzip" "context" "crypto/sha256" - "database/sql" "encoding/hex" "io" "net/http" "net/http/httptest" "os" "path/filepath" - "slices" "strings" "testing" "time" - - _ "modernc.org/sqlite" ) // The values come from the AWS documentation, "Signature Calculations for @@ -82,7 +77,7 @@ func TestObjectURL(t *testing.T) { {"AWS default endpoint", S3{Region: "eu-central-1", Bucket: "rp-backups"}, "db/a.gz", "https://rp-backups.s3.eu-central-1.amazonaws.com/db/a.gz"}, {"AWS bucket with dots uses path style", S3{Bucket: "backups.example.com"}, "a.gz", "https://s3.us-east-1.amazonaws.com/backups.example.com/a.gz"}, {"R2", S3{Endpoint: "https://acc123.r2.cloudflarestorage.com", Region: "auto", Bucket: "rp"}, "a.gz", "https://acc123.r2.cloudflarestorage.com/rp/a.gz"}, - {"MinIO without scheme", S3{Endpoint: "minio.lan:9000/", Bucket: "rp"}, "a.gz", "https://minio.lan:9000/rp/a.gz"}, + {"MinIO with a final slash", S3{Endpoint: "https://minio.lan:9000/", Bucket: "rp"}, "a.gz", "https://minio.lan:9000/rp/a.gz"}, {"key with special characters", S3{Endpoint: "http://127.0.0.1:9000", Bucket: "rp"}, "my backups/a+b$.gz", "http://127.0.0.1:9000/rp/my%20backups/a%2Bb%24.gz"}, } for _, tt := range tests { @@ -104,21 +99,24 @@ func TestUpload(t *testing.T) { os.WriteFile(file, content, 0o600) type request struct { - method, path, host, contentHash, auth string - body []byte + method, path, host, contentHash, date, auth string + body []byte } got := make(chan request, 1) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - got <- request{r.Method, r.URL.EscapedPath(), r.Host, r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), body} + got <- request{r.Method, r.URL.EscapedPath(), r.Host, r.Header.Get("X-Amz-Content-Sha256"), + r.Header.Get("X-Amz-Date"), r.Header.Get("Authorization"), body} })) defer srv.Close() s3 := S3{Endpoint: srv.URL, Region: "auto", Bucket: "rp", Prefix: "/nightly/", AccessKey: "AK", SecretKey: "SK"} + before := time.Now().UTC().Truncate(time.Second) key, err := s3.Upload(context.Background(), file) if err != nil { t.Fatal(err) } + after := time.Now().UTC() if key != "nightly/runplane-20261104T030000Z.db.gz" { t.Fatalf("key %q", key) } @@ -133,8 +131,12 @@ func TestUpload(t *testing.T) { if r.contentHash != hex.EncodeToString(sum[:]) { t.Fatalf("x-amz-content-sha256 %s does not match the body", r.contentHash) } - day := time.Now().UTC().Format("20060102") - if !strings.HasPrefix(r.auth, "AWS4-HMAC-SHA256 Credential=AK/"+day+"/auto/s3/aws4_request,SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date,") { + // The scope of the signature has the day of X-Amz-Date, also at midnight. + at, err := time.Parse("20060102T150405Z", r.date) + if err != nil || at.Before(before) || at.After(after) { + t.Fatalf("X-Amz-Date %q, want a time between %s and %s", r.date, before, after) + } + if !strings.HasPrefix(r.auth, "AWS4-HMAC-SHA256 Credential=AK/"+at.Format("20060102")+"/auto/s3/aws4_request,SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date,") { t.Fatalf("Authorization %q", r.auth) } } @@ -154,108 +156,18 @@ func TestUploadError(t *testing.T) { } } -func TestRun(t *testing.T) { - dir := t.TempDir() - // No automatic checkpoint: the rows stay in the WAL file, so a plain - // copy of runplane.db would not have them. - db, err := sql.Open("sqlite", "file:"+filepath.Join(dir, "runplane.db")+"?_pragma=journal_mode(WAL)&_pragma=wal_autocheckpoint(0)") - if err != nil { - t.Fatal(err) - } - defer db.Close() - if _, err := db.Exec(`CREATE TABLE jobs(id TEXT PRIMARY KEY, repository TEXT)`); err != nil { - t.Fatal(err) - } - for _, id := range []string{"j1", "j2", "j3"} { - if _, err := db.Exec(`INSERT INTO jobs VALUES (?, 'acme/api')`, id); err != nil { - t.Fatal(err) - } - } - - backups := filepath.Join(dir, "backups") - res, err := Run(context.Background(), db, backups, 7) - if err != nil { - t.Fatal(err) - } - info, err := os.Stat(res.Path) - if err != nil { - t.Fatal(err) - } - if info.Size() != res.Size { - t.Fatalf("size %d, file has %d", res.Size, info.Size()) - } - if mode := info.Mode().Perm(); mode != 0o600 { - t.Fatalf("backup mode %o, want 600: it holds password hashes", mode) - } - if !strings.HasPrefix(filepath.Base(res.Path), "runplane-") || !strings.HasSuffix(res.Path, "Z.db.gz") { - t.Fatalf("backup name %s", res.Path) - } - entries, _ := os.ReadDir(backups) - if len(entries) != 1 { - t.Fatalf("backup dir has %d files, want only the backup", len(entries)) - } - - rdb, err := sql.Open("sqlite", "file:"+gunzip(t, res.Path)+"?mode=ro") - if err != nil { - t.Fatal(err) - } - defer rdb.Close() - var ids string - if err := rdb.QueryRow(`SELECT group_concat(id) FROM (SELECT id FROM jobs ORDER BY id)`).Scan(&ids); err != nil || ids != "j1,j2,j3" { - t.Fatalf("restored jobs %q, %v; want j1,j2,j3", ids, err) - } -} - -func TestRunKeepsNewest(t *testing.T) { - dir := t.TempDir() - old := []string{ - "runplane-20250101T030000.000Z.db.gz", - "runplane-20250102T030000.000Z.db.gz", - "runplane-20250103T030000.000Z.db.gz", - "runplane-20250104T030000.000Z.db.gz", - } - others := []string{"runplane-before-upgrade.db.gz", "notes.txt"} - for _, name := range append(slices.Clone(old), others...) { - os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o600) - } - db, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "runplane.db")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - res, err := Run(context.Background(), db, dir, 3) - if err != nil { - t.Fatal(err) - } - wantRemoved := []string{filepath.Join(dir, old[0]), filepath.Join(dir, old[1])} - if !slices.Equal(res.Removed, wantRemoved) { - t.Fatalf("removed %v, want %v", res.Removed, wantRemoved) - } - var left []string - entries, _ := os.ReadDir(dir) - for _, e := range entries { - left = append(left, e.Name()) - } - want := []string{"notes.txt", old[2], old[3], "runplane-before-upgrade.db.gz", filepath.Base(res.Path)} - slices.Sort(want) - if !slices.Equal(left, want) { - t.Fatalf("files %v, want %v", left, want) - } -} - // Two backups in one millisecond, for example two clicks, must not write -// the same file. +// the same file. The second name sorts after the first, because the +// pruning keeps the names that sort last. func TestFreeName(t *testing.T) { dir := t.TempDir() at := time.Date(2026, 11, 4, 3, 0, 0, 0, time.UTC) - first, _ := freeName(dir, at) - if first != "runplane-20261104T030000.000Z.db.gz" { - t.Fatalf("name %q", first) + first, path := freeName(dir, at) + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatal(err) } - os.WriteFile(filepath.Join(dir, first), []byte("x"), 0o600) - if next, _ := freeName(dir, at); next != "runplane-20261104T030000.001Z.db.gz" { - t.Fatalf("name after %s: %q, want the next millisecond", first, next) + if next, _ := freeName(dir, at); next <= first { + t.Fatalf("name after %s: %q, want a name that sorts after it", first, next) } } @@ -276,25 +188,3 @@ func TestNextRun(t *testing.T) { } } } - -func gunzip(t *testing.T, path string) string { - t.Helper() - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - zr, err := gzip.NewReader(f) - if err != nil { - t.Fatal(err) - } - data, err := io.ReadAll(zr) - if err != nil { - t.Fatal(err) - } - out := filepath.Join(t.TempDir(), "restored.db") - if err := os.WriteFile(out, data, 0o600); err != nil { - t.Fatal(err) - } - return out -} diff --git a/internal/backup/s3.go b/internal/backup/s3.go index c43b040..e0b724f 100644 --- a/internal/backup/s3.go +++ b/internal/backup/s3.go @@ -80,14 +80,10 @@ func (s S3) Upload(ctx context.Context, file string) (string, error) { func (s S3) region() string { return cmp.Or(s.Region, "us-east-1") } -// objectURL returns the URL of the object key. Path and RawPath are both -// set, so the request sends the same escaped path that sign uses. +// objectURL sets both Path and RawPath, so the request sends the same +// escaped path that sign uses. func (s S3) objectURL(key string) (*url.URL, error) { - endpoint := cmp.Or(s.Endpoint, "https://s3."+s.region()+".amazonaws.com") - if !strings.Contains(endpoint, "://") { - endpoint = "https://" + endpoint - } - u, err := url.Parse(endpoint) + u, err := url.Parse(cmp.Or(s.Endpoint, "https://s3."+s.region()+".amazonaws.com")) if err != nil || u.Host == "" { return nil, fmt.Errorf("parse S3 endpoint %q: invalid URL", s.Endpoint) } diff --git a/internal/backup/service.go b/internal/backup/service.go new file mode 100644 index 0000000..925b6fc --- /dev/null +++ b/internal/backup/service.go @@ -0,0 +1,180 @@ +package backup + +import ( + "cmp" + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/ids" + "github.com/runplane/runplane/internal/secret" + "github.com/runplane/runplane/internal/store" +) + +// ErrNoFile is the error of Open when the file of a backup is not on this +// server. +var ErrNoFile = errors.New("the backup file is not on this server") + +type Config struct { + Store *store.Store + Box *secret.Box // opens the S3 secret key + Bus *events.Bus + Dir string // the backups folder + Log *slog.Logger // nil: no log +} + +// Service makes the backups of the database: when an admin asks, and each +// day while backup_enabled is on. Each backup has a row, and each change of +// the row goes out as a backup.state event. +type Service struct { + cfg Config + log *slog.Logger + mu sync.Mutex // one backup at a time +} + +func New(cfg Config) *Service { + return &Service{cfg: cfg, log: cmp.Or(cfg.Log, slog.New(slog.DiscardHandler))} +} + +// Run makes a backup each day at DailyAt while backup_enabled is on, until +// ctx ends. +func (s *Service) Run(ctx context.Context) { + daily(ctx, func(ctx context.Context) { + st, err := s.cfg.Store.GetSettings(ctx) + if err != nil || !st.BackupEnabled { + return + } + if _, err := s.Create(ctx); err != nil { + s.log.Warn("daily backup", "err", err) + } + }) +} + +// Create makes a backup, removes the backups over backup_keep, and uploads +// the new one to S3 if a bucket is set. A failed upload fails the backup +// but keeps its file. +func (s *Service) Create(ctx context.Context) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + st, err := s.cfg.Store.GetSettings(ctx) + if err != nil { + return "", err + } + row := store.Backup{ID: ids.New(), CreatedAt: time.Now(), Status: store.BackupRunning} + if err := s.cfg.Store.CreateBackup(ctx, row); err != nil { + return "", err + } + s.publish(row) + res, err := write(ctx, s.cfg.Store.VacuumInto, s.cfg.Dir, st.BackupKeep) + if err != nil { + s.fail(ctx, row, err) + return "", fmt.Errorf("back up the database: %w", err) + } + row.Path, row.SizeBytes, row.Status = res.path, res.size, store.BackupDone + s.save(ctx, row) + s.dropRows(ctx, res.removed) + if st.BackupS3.Bucket == "" { + return row.ID, nil + } + row.Status = store.BackupUploading + s.save(ctx, row) + row.S3Key, err = s.upload(ctx, st.BackupS3, res.path) + if err != nil { + s.fail(ctx, row, err) + return row.ID, nil + } + row.Status = store.BackupUploaded + s.save(ctx, row) + return row.ID, nil +} + +// Available reports if the file of b is on this server. +func (s *Service) Available(b store.Backup) bool { + _, err := os.Stat(b.Path) + return err == nil && s.inDir(b.Path) +} + +// Open opens the file of the backup id for a download. It returns +// store.ErrNotFound for an unknown ID, and ErrNoFile when the file is not +// on this server. +func (s *Service) Open(ctx context.Context, id string) (store.Backup, *os.File, error) { + b, err := s.cfg.Store.GetBackup(ctx, id) + if err != nil { + return store.Backup{}, nil, err + } + if !s.inDir(b.Path) { + return b, nil, ErrNoFile + } + f, err := os.Open(b.Path) + if errors.Is(err, fs.ErrNotExist) { + return b, nil, ErrNoFile + } + return b, f, err +} + +// inDir guards the download against a path outside the backups folder. +func (s *Service) inDir(path string) bool { + rel, err := filepath.Rel(s.cfg.Dir, path) + return err == nil && rel != "." && !strings.HasPrefix(rel, "..") && !filepath.IsAbs(rel) +} + +func (s *Service) upload(ctx context.Context, b store.BackupS3, path string) (string, error) { + key, err := s.cfg.Box.Open(b.SecretKeyEnc) + if err != nil { + return "", fmt.Errorf("open the S3 secret key: %w", err) + } + bucket := S3{Endpoint: b.Endpoint, Region: b.Region, Bucket: b.Bucket, Prefix: b.Prefix, AccessKey: b.AccessKey, SecretKey: string(key)} + k, err := bucket.Upload(ctx, path) + if err != nil { + return "", fmt.Errorf("upload to S3: %w", err) + } + return k, nil +} + +// fail records a failed backup. The notifier sends the backup.failed alert +// for the row. +func (s *Service) fail(ctx context.Context, row store.Backup, err error) { + s.log.Error("backup failed", "err", err) + row.Status, row.Error = store.BackupFailed, err.Error() + s.save(ctx, row) +} + +func (s *Service) save(ctx context.Context, row store.Backup) { + if err := s.cfg.Store.UpdateBackup(context.WithoutCancel(ctx), row); err != nil { + s.log.Warn("record backup", "backup", row.ID, "err", err) + return + } + s.publish(row) +} + +func (s *Service) publish(row store.Backup) { + s.cfg.Bus.Publish(events.BackupState(row)) +} + +func (s *Service) dropRows(ctx context.Context, removed []string) { + if len(removed) == 0 { + return + } + list, err := s.cfg.Store.ListBackups(ctx) + if err != nil { + s.log.Warn("list backups", "err", err) + return + } + for _, row := range list { + if !slices.Contains(removed, row.Path) { + continue + } + if err := s.cfg.Store.DeleteBackup(ctx, row.ID); err != nil { + s.log.Warn("delete backup row", "backup", row.ID, "err", err) + } + } +} diff --git a/internal/backup/service_test.go b/internal/backup/service_test.go new file mode 100644 index 0000000..8ea2017 --- /dev/null +++ b/internal/backup/service_test.go @@ -0,0 +1,260 @@ +package backup + +import ( + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/secret" + "github.com/runplane/runplane/internal/store" +) + +// A backup is the database in a gzip file, as the restore steps of the +// README expect. Only root can read it, because it holds password hashes. +func TestCreateWritesTheDatabase(t *testing.T) { + ctx := context.Background() + st := openStore(t) + dir := t.TempDir() + if err := st.CreateUser(ctx, store.User{ID: "u1", Email: "ada@example.com", Name: "Ada", PasswordHash: "hash", + Role: store.RoleAdmin, CreatedAt: time.Now()}); err != nil { + t.Fatal(err) + } + id, err := New(Config{Store: st, Bus: &events.Bus{}, Dir: dir}).Create(ctx) + if err != nil { + t.Fatal(err) + } + row, err := st.GetBackup(ctx, id) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(row.Path) + if err != nil { + t.Fatal(err) + } + if row.Status != store.BackupDone || row.SizeBytes != info.Size() { + t.Errorf("backup row %+v, want done with the size %d of the file", row, info.Size()) + } + if mode := info.Mode().Perm(); mode != 0o600 { + t.Errorf("backup mode %o, want 600", mode) + } + if entries, _ := os.ReadDir(dir); len(entries) != 1 { + t.Errorf("the backups folder has %d files, want only the backup", len(entries)) + } + restored, err := store.Open(gunzip(t, row.Path)) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if u, err := restored.GetUserByEmail(ctx, "ada@example.com"); err != nil || u.ID != "u1" { + t.Errorf("user of the restored database: %+v, %v; want u1", u, err) + } +} + +// Create keeps the newest backup_keep backups. An older backup loses its +// file and its row together, so the dashboard lists no backup that cannot +// download. Other files in the folder stay. +func TestCreateKeepsTheNewest(t *testing.T) { + ctx := context.Background() + st := openStore(t) + dir := t.TempDir() + if _, err := st.UpdateSettings(ctx, func(s *store.Settings) error { s.BackupKeep = 2; return nil }); err != nil { + t.Fatal(err) + } + others := []string{"notes.txt", "runplane-before-upgrade.db.gz"} + for _, name := range others { + if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + b := New(Config{Store: st, Bus: &events.Bus{}, Dir: dir}) + var ids []string + for range 3 { + id, err := b.Create(ctx) + if err != nil { + t.Fatal(err) + } + ids = append(ids, id) + } + + list, err := st.ListBackups(ctx) + if err != nil { + t.Fatal(err) + } + var kept []string + files := slices.Clone(others) + for _, row := range list { + kept = append(kept, row.ID) + files = append(files, filepath.Base(row.Path)) + } + if !sameSet(kept, ids[1:]) { + t.Errorf("backup rows %v, want the two newest %v", kept, ids[1:]) + } + var left []string + entries, _ := os.ReadDir(dir) + for _, e := range entries { + left = append(left, e.Name()) + } + if !sameSet(left, files) { + t.Errorf("files %v, want %v", left, files) + } +} + +func sameSet(a, b []string) bool { + return slices.Equal(slices.Sorted(slices.Values(a)), slices.Sorted(slices.Values(b))) +} + +// The settings page follows a backup through backup.state events, also the +// upload to S3 and a failure. A failed upload fails the backup, and the +// file stays. +func TestBackupStateEvents(t *testing.T) { + s3 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(io.Discard, r.Body) + if strings.HasPrefix(r.URL.Path, "/denied/") { + http.Error(w, "AccessDeniedAccess Denied", http.StatusForbidden) + } + })) + defer s3.Close() + box, err := secret.NewBox(make([]byte, secret.KeySize)) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + bucket string + dir func(t *testing.T) string + want []string + }{ + {"local", "", func(t *testing.T) string { return t.TempDir() }, []string{store.BackupRunning, store.BackupDone}}, + {"upload to S3", "ci-backups", func(t *testing.T) string { return t.TempDir() }, + []string{store.BackupRunning, store.BackupDone, store.BackupUploading, store.BackupUploaded}}, + {"upload fails", "denied", func(t *testing.T) string { return t.TempDir() }, + []string{store.BackupRunning, store.BackupDone, store.BackupUploading, store.BackupFailed}}, + {"backup fails", "", func(t *testing.T) string { + file := filepath.Join(t.TempDir(), "file") + os.WriteFile(file, nil, 0o600) + return filepath.Join(file, "backups") // a folder in a file cannot exist + }, []string{store.BackupRunning, store.BackupFailed}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + st := openStore(t) + if tt.bucket != "" { + _, err := st.UpdateSettings(ctx, func(s *store.Settings) error { + s.BackupS3 = store.BackupS3{Endpoint: s3.URL, Region: "us-east-1", Bucket: tt.bucket, AccessKey: "AKIDEXAMPLE", + SecretKeyEnc: box.Seal([]byte("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"))} + return nil + }) + if err != nil { + t.Fatal(err) + } + } + bus := &events.Bus{} + states := events.Subscribe[events.BackupState](t.Context(), bus) + b := New(Config{Store: st, Box: box, Bus: bus, Dir: tt.dir(t)}) + b.Create(ctx) + var seen []string + var last events.BackupState + for len(states) > 0 { + last = <-states + seen = append(seen, last.Status) + } + if !slices.Equal(seen, tt.want) { + t.Errorf("backup.state events %v, want %v", seen, tt.want) + } + if tt.bucket == "denied" { + if _, err := os.Stat(last.Path); err != nil || last.Error == "" { + t.Errorf("failed upload: file %v, error %q; want the file and the error", err, last.Error) + } + } + }) + } +} + +// A download opens only a backup file in the backups folder. +func TestOpen(t *testing.T) { + ctx := context.Background() + st := openStore(t) + dir := t.TempDir() + outside := filepath.Join(t.TempDir(), "secret.key") + for _, path := range []string{filepath.Join(dir, "runplane-a.db.gz"), outside} { + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + for id, path := range map[string]string{ + "here": filepath.Join(dir, "runplane-a.db.gz"), + "gone": filepath.Join(dir, "runplane-b.db.gz"), + "outside": outside, + } { + if err := st.CreateBackup(ctx, store.Backup{ID: id, CreatedAt: time.Now(), Path: path, Status: store.BackupDone}); err != nil { + t.Fatal(err) + } + } + b := New(Config{Store: st, Bus: &events.Bus{}, Dir: dir}) + + for _, tt := range []struct { + id string + err error + available bool + }{ + {"here", nil, true}, + {"gone", ErrNoFile, false}, + {"outside", ErrNoFile, false}, + {"unknown", store.ErrNotFound, false}, + } { + row, f, err := b.Open(ctx, tt.id) + if f != nil { + f.Close() + } + if !errors.Is(err, tt.err) || (err == nil) != (f != nil) { + t.Errorf("Open(%s): file %v, err %v; want err %v", tt.id, f != nil, err, tt.err) + } + if got := b.Available(row); got != tt.available { + t.Errorf("Available(%s) = %v, want %v", tt.id, got, tt.available) + } + } +} + +func openStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "runplane.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + return st +} + +// gunzip writes the database of a backup to a new file, and returns its path. +func gunzip(t *testing.T, path string) string { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + zr, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(zr) + if err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "runplane.db") + if err := os.WriteFile(out, data, 0o600); err != nil { + t.Fatal(err) + } + return out +} diff --git a/internal/blob/blob.go b/internal/blob/blob.go new file mode 100644 index 0000000..7224fd8 --- /dev/null +++ b/internal/blob/blob.go @@ -0,0 +1,59 @@ +// Package blob holds the sha256 digest form and a file serve that counts +// the bytes. The Actions cache, the registry mirror, and the images use +// them for the content that they keep on disk. +package blob + +import ( + "encoding/hex" + "io" + "net/http" + "strings" + "time" +) + +const digestPrefix = "sha256:" + +// Digest returns the digest of a SHA-256 sum: "sha256:". +func Digest(sum []byte) string { + return digestPrefix + hex.EncodeToString(sum) +} + +// ValidDigest reports if d is "sha256:" and 64 hex digits in lower case. +func ValidDigest(d string) bool { + h, ok := strings.CutPrefix(d, digestPrefix) + if !ok || len(h) != 64 { + return false + } + for _, c := range h { + if !strings.ContainsRune("0123456789abcdef", c) { + return false + } + } + return true +} + +// Serve answers r with content as http.ServeContent does, and returns the +// count of body bytes that it sent. +func Serve(w http.ResponseWriter, r *http.Request, modtime time.Time, content io.ReadSeeker) int64 { + cw := &countingWriter{ResponseWriter: w} + http.ServeContent(cw, r, "", modtime, content) + return cw.n +} + +type countingWriter struct { + http.ResponseWriter + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.ResponseWriter.Write(p) + c.n += int64(n) + return n, err +} + +// ReadFrom keeps the sendfile path of the ResponseWriter. +func (c *countingWriter) ReadFrom(r io.Reader) (int64, error) { + n, err := io.Copy(c.ResponseWriter, r) + c.n += n + return n, err +} diff --git a/internal/blob/blob_test.go b/internal/blob/blob_test.go new file mode 100644 index 0000000..e832c5b --- /dev/null +++ b/internal/blob/blob_test.go @@ -0,0 +1,28 @@ +package blob + +import ( + "strings" + "testing" +) + +func TestValidDigest(t *testing.T) { + hex := strings.Repeat("0123456789abcdef", 4) + tests := []struct { + d string + want bool + }{ + {"sha256:" + hex, true}, + // The mirror names the cache files by the hex part. + {"sha256:" + strings.ToUpper(hex), false}, + {"sha256:" + hex[:63], false}, + {"sha256:" + hex + "0", false}, + {"sha512:" + hex, false}, + {hex, false}, + {"sha256:../../etc/passwd", false}, + } + for _, tt := range tests { + if got := ValidDigest(tt.d); got != tt.want { + t.Errorf("ValidDigest(%q) = %v, want %v", tt.d, got, tt.want) + } + } +} diff --git a/internal/cache/blob.go b/internal/cache/blob.go index f98f122..bad4fff 100644 --- a/internal/cache/blob.go +++ b/internal/cache/blob.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/runplane/runplane/internal/blob" "github.com/runplane/runplane/internal/ids" "github.com/runplane/runplane/internal/store" ) @@ -58,7 +59,7 @@ func (s *Server) mac(perm, id, se string) string { // serveBlob emulates the Azure Blob calls that the cache clients make on // signed URLs: Put Blob, Put Block, Put Block List, and Get Blob with a range. -func (s *Server) serveBlob(w http.ResponseWriter, r *http.Request, trusted bool) { +func (s *Server) serveBlob(w http.ResponseWriter, r *http.Request, src source) { id := strings.TrimPrefix(r.URL.Path, blobPrefix) perm := "r" switch r.Method { @@ -74,7 +75,7 @@ func (s *Server) serveBlob(w http.ResponseWriter, r *http.Request, trusted bool) return } e, err := s.cfg.Store.GetCacheEntry(r.Context(), id) - if errors.Is(err, ErrNotFound) { + if errors.Is(err, store.ErrNotFound) { blobError(w, http.StatusNotFound, "BlobNotFound", "the cache entry does not exist") return } @@ -83,7 +84,7 @@ func (s *Server) serveBlob(w http.ResponseWriter, r *http.Request, trusted bool) blobError(w, http.StatusInternalServerError, "InternalError", "internal error") return } - ident, err := s.identify(r, trusted) + ident, err := s.identify(r, src) if err != nil && !errors.Is(err, ErrNoSandbox) { s.cfg.Logger.Error("cache: identify sandbox", "err", err) blobError(w, http.StatusInternalServerError, "InternalError", "internal error") @@ -122,7 +123,7 @@ func (s *Server) upload(w http.ResponseWriter, r *http.Request, e store.CacheEnt blobError(w, http.StatusInternalServerError, "InternalError", "internal error") return } - defer s.release(e.ID, reserved) + defer s.release(reserved) switch q.Get("comp") { case "": err = s.disk.putBlob(e.ID, r.Body, reserved) @@ -137,7 +138,7 @@ func (s *Server) upload(w http.ResponseWriter, r *http.Request, e store.CacheEnt blobError(w, http.StatusConflict, "BlockCountExceedsLimit", "the upload has too many blocks") return } - err = s.disk.putBlock(e.ID, blockID, r.Body, min(reserved, s.Limit()-size)) + err = s.disk.putBlock(e.ID, blockID, r.Body, min(reserved, s.MaxBytes()-size)) case "blocklist": var blocks []string blocks, err = parseBlockList(io.LimitReader(r.Body, maxBlockListBytes)) @@ -162,8 +163,8 @@ func (s *Server) upload(w http.ResponseWriter, r *http.Request, e store.CacheEnt blobError(w, http.StatusInternalServerError, "InternalError", "internal error") return } - if err := s.cfg.Store.TouchCacheEntry(r.Context(), e.ID, s.now()); err != nil { - s.cfg.Logger.Warn("cache: touch entry", "id", e.ID, "err", err) + if err := s.recordUpload(r.Context(), e.ID, s.now()); err != nil { + s.cfg.Logger.Warn("cache: record upload", "id", e.ID, "err", err) } h := w.Header() h.Set("ETag", `"`+ids.New()+`"`) @@ -194,10 +195,8 @@ func (s *Server) download(w http.ResponseWriter, r *http.Request, e store.CacheE h.Set("ETag", `"`+e.ID+`"`) h.Set("x-ms-blob-type", "BlockBlob") setAzureHeaders(h) - cw := &countingWriter{ResponseWriter: w} - http.ServeContent(cw, r, "", e.CreatedAt, f) - if cw.n > 0 { - s.cfg.Stats.Add(e.Repository, store.CacheCounts{BytesRead: cw.n}) + if n := blob.Serve(w, r, e.CreatedAt, f); n > 0 { + s.stats.add(e.Repository, store.CacheCounts{BytesRead: n}) } } @@ -237,22 +236,3 @@ func blobError(w http.ResponseWriter, status int, code, msg string) { xml.EscapeText(&esc, []byte(msg)) fmt.Fprintf(w, `%s%s`, code, esc.String()) } - -// countingWriter counts the body bytes of a response. ReadFrom keeps the -// sendfile path of http.ServeContent. -type countingWriter struct { - http.ResponseWriter - n int64 -} - -func (c *countingWriter) Write(p []byte) (int, error) { - n, err := c.ResponseWriter.Write(p) - c.n += int64(n) - return n, err -} - -func (c *countingWriter) ReadFrom(r io.Reader) (int64, error) { - n, err := io.Copy(c.ResponseWriter, r) - c.n += n - return n, err -} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 1621008..2fa02c1 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -3,7 +3,7 @@ // emulates the Azure Blob calls on the signed URLs that the service hands // out, and sends all other results traffic, for example artifacts, to GitHub. // A patched runner keeps the ACTIONS_RESULTS_URL of its .env file (see -// RunnerEnv), so jobs send cache calls here. +// guest.NewConfig), so jobs send cache calls here. package cache import ( @@ -12,7 +12,6 @@ import ( "errors" "fmt" "log/slog" - "net" "net/http" "net/http/httputil" "net/netip" @@ -22,11 +21,10 @@ import ( "sync" "time" + "github.com/runplane/runplane/internal/datadisk" "github.com/runplane/runplane/internal/store" ) -var resultsURL = &url.URL{Scheme: "https", Host: "results-receiver.actions.githubusercontent.com"} - const ( // staleUpload is the age at which the cleanup removes an unfinished upload. staleUpload = time.Hour @@ -37,32 +35,49 @@ const ( jobWait = time.Minute ) +// Store keeps the cache entries and the hourly counts. *store.Store +// implements it. +type Store interface { + CreateCacheEntry(ctx context.Context, e store.CacheEntry) error + GetCacheEntry(ctx context.Context, id string) (store.CacheEntry, error) + GetCacheEntryByKey(ctx context.Context, repository, ref, key, version string) (store.CacheEntry, error) + MatchCacheEntry(ctx context.Context, repository, version, ref, key string, prefix bool) (store.CacheEntry, error) + TouchCacheEntry(ctx context.Context, id string, used time.Time) error + SetCacheUpload(ctx context.Context, id string, size int64, used time.Time) error + FinishCacheEntry(ctx context.Context, id string, size int64, at time.Time) error + ListCacheEntries(ctx context.Context, repository string) ([]store.CacheEntry, error) + LeastRecentCacheEntries(ctx context.Context, limit int) ([]store.CacheEntry, error) + ListUnfinishedCacheEntries(ctx context.Context, before time.Time) ([]store.CacheEntry, error) + DeleteCacheEntry(ctx context.Context, id string) error + CacheSize(ctx context.Context) (store.CacheBytes, error) + AddCacheStats(ctx context.Context, hour time.Time, repository string, c store.CacheCounts) error + PruneCacheStats(ctx context.Context, before time.Time) error +} + type Config struct { - Dir string // blob folder, for example /var/lib/runplane/cache - Store *store.Store // the cache_entries table; the blobs stay in Dir + Dir string // blob folder, for example /var/lib/runplane/cache + Store Store // the cache_entries table; the blobs stay in Dir Resolver Resolver - Stats *Stats - // MaxBytes returns the byte limit of all entries; a value <= 0 means no - // limit. The server reads it at each check, so a settings change applies - // at once. - MaxBytes func() int64 - // DiskSpace returns the size and the free bytes of the disk of Dir. The - // limit is at most a quarter of the disk. - DiskSpace func() (total, free int64, err error) - // MinFreePercent returns the part of the disk, 0 to 100, that the cache - // leaves free: below it, the server takes no upload and removes the - // least recently used entries. - MinFreePercent func() float64 - SigningKey []byte // signs blob URLs - Logger *slog.Logger + // Space returns the room of the cache on the data disk now + // (datadisk.Limits). The server reads it at each check, so a settings + // change applies at once. Below min_free_disk_percent, the server takes + // no upload and removes the least recently used entries. + Space func() datadisk.Space + SigningKey []byte // signs blob URLs + Logger *slog.Logger + Now func() time.Time // nil: time.Now + // Results is the GitHub results service, which gets all requests that + // are not for the cache. nil: results-receiver.actions.githubusercontent.com. + Results *url.URL } -// Server is the cache server. As an http.Handler it serves requests that -// come from sandboxes directly; Proxied serves requests that a trusted proxy -// forwards. +// As an http.Handler, Server serves the requests of the sandboxes of this +// server. ServeAgent serves the requests that an agent sends for its +// sandboxes. type Server struct { cfg Config disk disk + stats *stats jobWait time.Duration proxy *httputil.ReverseProxy now func() time.Time @@ -74,13 +89,20 @@ type Server struct { func New(cfg Config) (*Server, error) { cfg.Logger = cmp.Or(cfg.Logger, slog.New(slog.DiscardHandler)) + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.Results == nil { + cfg.Results = &url.URL{Scheme: "https", Host: "results-receiver.actions.githubusercontent.com"} + } s := &Server{ cfg: cfg, disk: disk{root: cfg.Dir}, + stats: newStats(cfg.Store, cfg.Logger, cfg.Now), jobWait: jobWait, - proxy: newProxy(resultsURL, cfg.Logger), - now: time.Now, - uploads: uploads{onDisk: map[string]int64{}, owner: map[string]string{}}, + proxy: newProxy(cfg.Results, cfg.Logger), + now: cfg.Now, + uploads: uploads{owner: map[string]string{}}, } if err := s.disk.init(); err != nil { return nil, err @@ -88,49 +110,46 @@ func New(cfg Config) (*Server, error) { return s, nil } -// RunnerEnv returns the runner .env values that send the cache calls of a -// patched runner to the cache server at serviceURL, for example -// http://10.0.0.1:8486. The cache server must be at the root path: -// @actions/cache drops the path of ACTIONS_RESULTS_URL. -func RunnerEnv(serviceURL string) map[string]string { - return map[string]string{ - "ACTIONS_RESULTS_URL": strings.TrimRight(serviceURL, "/") + "/", - "ACTIONS_CACHE_SERVICE_V2": "true", - } +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.serve(w, r, source{}) } -func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.serve(w, r, false) +// ServeAgent serves a request that the agent serverID sends through its +// connection for its sandbox sandboxID. The agent finds the sandbox at the +// source address of the request, and it gives the host for the signed URLs +// in X-Forwarded-Host and X-Forwarded-Proto. +func (s *Server) ServeAgent(w http.ResponseWriter, r *http.Request, serverID, sandboxID string) { + s.serve(w, r, source{serverID: serverID, sandboxID: sandboxID}) } -// Proxied returns the handler for requests from a trusted proxy, such as the -// agent cache endpoint. It reads the sandbox from SandboxHeader, and the -// host for signed URLs from X-Forwarded-Host and X-Forwarded-Proto. -func (s *Server) Proxied() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.serve(w, r, true) - }) +// source tells where a request comes from: a sandbox of this server, which +// the server finds by the source address, or a sandbox that an agent names. +// The zero source is this server. +type source struct { + serverID string // the agent + sandboxID string // the sandbox on the agent } -func (s *Server) serve(w http.ResponseWriter, r *http.Request, trusted bool) { +func (src source) agent() bool { return src.serverID != "" } + +func (s *Server) serve(w http.ResponseWriter, r *http.Request, src source) { switch { case strings.HasPrefix(r.URL.Path, twirpPrefix): - s.serveTwirp(w, r, trusted) + s.serveTwirp(w, r, src) case strings.HasPrefix(r.URL.Path, blobPrefix): - s.serveBlob(w, r, trusted) + s.serveBlob(w, r, src) default: - s.serveResults(w, r, trusted) + s.serveResults(w, r, src) } } // serveResults sends a request to the GitHub results service. The service // port listens on all addresses, so without a check anyone could send -// requests to GitHub through this server. A sandbox, with or without a job, -// and this host may; a trusted proxy checks its own sandboxes. In dev mode, -// Docker sends the requests of sandboxes from the loopback address. -func (s *Server) serveResults(w http.ResponseWriter, r *http.Request, trusted bool) { - if !trusted && !fromLoopback(r) { - _, err := s.sandbox(r, false) +// requests to GitHub through this server. Only a sandbox, with or without a +// job, may. An agent examines the address of its own sandboxes. +func (s *Server) serveResults(w http.ResponseWriter, r *http.Request, src source) { + if !src.agent() { + _, err := s.sandbox(r, src) if errors.Is(err, ErrNoSandbox) { http.Error(w, "the request comes from no known sandbox", http.StatusForbidden) return @@ -144,22 +163,6 @@ func (s *Server) serveResults(w http.ResponseWriter, r *http.Request, trusted bo s.proxy.ServeHTTP(w, r) } -func fromLoopback(r *http.Request) bool { - ip, _, _ := net.SplitHostPort(r.RemoteAddr) - addr, err := netip.ParseAddr(ip) - return err == nil && addr.IsLoopback() -} - -const ( - // SandboxHeader names the sandbox of a request that a trusted proxy (the - // agent cache endpoint) forwards. The server ignores it on other requests. - SandboxHeader = "X-Runplane-Sandbox" - // ServerHeader names the server of a cache request that an agent sends. - // The primary sets it, so that the resolver can check that the sandbox of - // the request belongs to that server. - ServerHeader = "X-Runplane-Server" -) - // ErrNoSandbox means that no sandbox with a job matches a request. var ErrNoSandbox = errors.New("no sandbox for this request") @@ -177,12 +180,13 @@ type Identity struct { BaseRef string } -// Resolver finds the sandbox behind a request. header contains SandboxHeader -// only when the request came through a trusted proxy; then the sandbox ID -// identifies the sandbox, and remoteIP is the address of the proxy. -// Sandbox returns ErrNoSandbox if no sandbox matches. +// Resolver finds the sandbox behind a request. Its methods return +// ErrNoSandbox if no sandbox matches. type Resolver interface { - Sandbox(ctx context.Context, remoteIP string, header http.Header) (Identity, error) + // SandboxAt finds the sandbox of this server at the address addr. + SandboxAt(ctx context.Context, addr netip.Addr) (Identity, error) + // SandboxOn finds the sandbox id on the agent serverID. + SandboxOn(ctx context.Context, serverID, id string) (Identity, error) } // readRefs returns the refs that the job may restore from, in search order: @@ -198,10 +202,10 @@ func (id Identity) readRefs() []string { // identify returns the Identity of a request, or an error that wraps // ErrNoSandbox when the request has none. -func (s *Server) identify(r *http.Request, trusted bool) (Identity, error) { +func (s *Server) identify(r *http.Request, src source) (Identity, error) { deadline := time.Now().Add(s.jobWait) for { - ident, err := s.sandbox(r, trusted) + ident, err := s.sandbox(r, src) if err != nil { return Identity{}, err } @@ -220,27 +224,25 @@ func (s *Server) identify(r *http.Request, trusted bool) (Identity, error) { } // sandbox returns the sandbox of a request, which can have no job yet. -func (s *Server) sandbox(r *http.Request, trusted bool) (Identity, error) { - header := r.Header - if !trusted && header.Get(SandboxHeader) != "" { - header = header.Clone() - header.Del(SandboxHeader) +func (s *Server) sandbox(r *http.Request, src source) (Identity, error) { + if src.agent() { + return s.cfg.Resolver.SandboxOn(r.Context(), src.serverID, src.sandboxID) } - ip, _, err := net.SplitHostPort(r.RemoteAddr) + addr, err := netip.ParseAddrPort(r.RemoteAddr) if err != nil { - ip = r.RemoteAddr + return Identity{}, fmt.Errorf("source address %q: %w", r.RemoteAddr, ErrNoSandbox) } - return s.cfg.Resolver.Sandbox(r.Context(), ip, header) + return s.cfg.Resolver.SandboxAt(r.Context(), addr.Addr().Unmap()) } // baseURL returns the scheme and host that the client used, for signed URLs -// that point back to this server. -func baseURL(r *http.Request, trusted bool) string { +// that point back to this server or to the service port of the agent. +func baseURL(r *http.Request, src source) string { scheme, host := "http", r.Host if r.TLS != nil { scheme = "https" } - if trusted { + if src.agent() { if h, _, _ := strings.Cut(r.Header.Get("X-Forwarded-Host"), ","); h != "" { host = strings.TrimSpace(h) } @@ -251,18 +253,22 @@ func baseURL(r *http.Request, trusted bool) string { return scheme + "://" + host } -// Run removes stale uploads and files that belong to no entry, and keeps the -// cache under its byte limit. It returns when ctx ends. -func (s *Server) Run(ctx context.Context) error { +// Run removes stale uploads and files that belong to no entry, keeps the +// cache under its byte limit, and writes the hourly counts. It returns when +// ctx ends. +func (s *Server) Run(ctx context.Context) { + var wg sync.WaitGroup + defer wg.Wait() + wg.Go(func() { s.stats.run(ctx) }) s.removeOrphans(ctx, "blobs") - s.countUploads(ctx) + s.syncUploads(ctx) tick := time.NewTicker(time.Minute) defer tick.Stop() for { s.cleanup(ctx) select { case <-ctx.Done(): - return nil + return case <-tick.C: } } @@ -289,7 +295,7 @@ func (s *Server) cleanup(ctx context.Context) { // entry is gone, for example after a crash between two steps of a removal. func (s *Server) removeOrphans(ctx context.Context, sub string) { for _, id := range s.disk.ids(sub) { - if _, err := s.cfg.Store.GetCacheEntry(ctx, id); !errors.Is(err, ErrNotFound) { + if _, err := s.cfg.Store.GetCacheEntry(ctx, id); !errors.Is(err, store.ErrNotFound) { continue } if err := s.disk.remove(id); err != nil { @@ -298,7 +304,7 @@ func (s *Server) removeOrphans(ctx context.Context, sub string) { } } -// Delete removes one entry and its blob. +// Delete returns store.ErrNotFound if no entry has the ID. func (s *Server) Delete(ctx context.Context, id string) error { e, err := s.cfg.Store.GetCacheEntry(ctx, id) if err != nil { @@ -307,12 +313,9 @@ func (s *Server) Delete(ctx context.Context, id string) error { return s.remove(ctx, e) } -// DeleteRepository removes all entries of a repository and returns the count. +// DeleteRepository returns the count of removed entries. An empty +// repository means all repositories. func (s *Server) DeleteRepository(ctx context.Context, repository string) (int, error) { - // The store lists all repositories for "". - if repository == "" { - return 0, nil - } list, err := s.cfg.Store.ListCacheEntries(ctx, repository) if err != nil { return 0, fmt.Errorf("list entries: %w", err) @@ -328,7 +331,7 @@ func (s *Server) DeleteRepository(ctx context.Context, repository string) (int, // remove deletes the index row first: if the file removal fails, the cleanup // finds an orphan file, and no entry points to a missing blob. func (s *Server) remove(ctx context.Context, e store.CacheEntry) error { - if _, err := s.cfg.Store.DeleteCacheEntries(ctx, "", e.ID); err != nil { + if err := s.cfg.Store.DeleteCacheEntry(ctx, e.ID); err != nil { return fmt.Errorf("delete entry: %w", err) } if err := s.disk.remove(e.ID); err != nil { @@ -342,10 +345,7 @@ func (s *Server) remove(ctx context.Context, e store.CacheEntry) error { // artifact uploads, to GitHub with the job's own token. func newProxy(upstream *url.URL, log *slog.Logger) *httputil.ReverseProxy { return &httputil.ReverseProxy{ - Rewrite: func(r *httputil.ProxyRequest) { - r.SetURL(upstream) - r.Out.Header.Del(SandboxHeader) - }, + Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(upstream) }, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { log.Warn("cache: results proxy", "path", r.URL.Path, "err", err) w.WriteHeader(http.StatusBadGateway) diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index cc3439e..44b911c 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -7,10 +7,11 @@ import ( "errors" "fmt" "io" - "log/slog" + "io/fs" "maps" "net/http" "net/http/httptest" + "net/netip" "net/url" "os" "path/filepath" @@ -68,8 +69,9 @@ func TestBlockUploadAndRestore(t *testing.T) { if got := f.counts(t, "acme/app"); got != (store.CacheCounts{Hits: 2, Misses: 1, BytesRead: 36, BytesWritten: 18}) { t.Errorf("stats = %+v", got) } - if staged, _ := os.ReadDir(filepath.Join(f.dir, "uploads")); len(staged) != 0 { - t.Errorf("staged blocks left after commit: %v", staged) + // Only the blob of 18 bytes is left, not the staged blocks. + if n := dirBytes(t, f.dir); n != 18 { + t.Errorf("%d bytes in the cache folder after the commit, want 18", n) } } @@ -135,8 +137,11 @@ func TestBranchRules(t *testing.T) { main.save("k", version, []byte("from main")) pr1.save("k", version, []byte("from pr 1")) pr1.save("pr1-only", version, []byte("pr 1 data")) + pr1.save("x-old", version, []byte("old on pr 1")) feature.save("feature-only", version, []byte("feature data")) develop.save("develop-only", version, []byte("develop data")) + f.clock.advance(time.Minute) + main.save("x-new", version, []byte("new on main")) tests := []struct { name string @@ -145,6 +150,7 @@ func TestBranchRules(t *testing.T) { want string }{ {"a pull request reads its own ref first", pr1, "k", "from pr 1"}, + {"a pull request reads its own ref before a newer entry of the default branch", pr1, "x-", "old on pr 1"}, {"a pull request reads the default branch", pr2, "k", "from main"}, {"a pull request reads its base branch", pr3, "develop-only", "develop data"}, {"a pull request cannot read a different pull request", pr2, "pr1-only", ""}, @@ -189,22 +195,10 @@ func TestRepositoryIsolation(t *testing.T) { if matched, _ := other.lookup(version, "k"); matched != "" { t.Errorf("other/lib restored the acme/app entry %q", matched) } - // A signed URL of acme/app is useless in a sandbox of other/lib. if status, _ := other.get(strings.Replace(acmeURL, f.url, f.proxied, 1), nil); status != http.StatusForbidden { t.Errorf("download from another repository: %d, want 403", status) } - t.Run("only a trusted proxy can name the sandbox", func(t *testing.T) { - spoof := client{t: t, url: f.url, header: http.Header{SandboxHeader: {"sb-other"}}} - spoof.save("spoofed", version, []byte("x")) - e, err := f.st.GetCacheEntryByKey(context.Background(), "acme/app", "refs/heads/main", "spoofed", version) - if err != nil { - t.Fatalf("entry not saved for the sandbox of the source IP: %v", err) - } - if e.Repository != "acme/app" { - t.Errorf("repository = %q", e.Repository) - } - }) } func TestRequestsWithoutIdentity(t *testing.T) { @@ -368,8 +362,8 @@ func TestEviction(t *testing.T) { t.Errorf("entry %q kept = %v, want %v", key, matched != "", want) } } - if blobs, _ := os.ReadDir(filepath.Join(f.dir, "blobs")); len(blobs) != 2 { - t.Errorf("%d blobs on disk, want 2", len(blobs)) + if n := dirBytes(t, f.dir); n != 200 { + t.Errorf("%d bytes in the cache folder, want the 200 bytes of a and c", n) } } @@ -401,40 +395,54 @@ func TestUploadOverLimit(t *testing.T) { }) } +// Run removes the uploads that stay unfinished for an hour, and the files +// of no entry. func TestCleanup(t *testing.T) { f := newFixture(t, 0) c := client{t: t, url: f.url} - ctx := context.Background() + ctx := t.Context() + abandoned := func() error { + _, err := f.st.GetCacheEntryByKey(ctx, "acme/app", "refs/heads/main", "abandoned", version) + return err + } c.save("kept", version, []byte("x")) u := c.reserve("abandoned", version) c.put(u+"&comp=block&blockid="+url.QueryEscape(blockID(0)), []byte("partial"), nil) - orphan := filepath.Join(f.dir, "blobs", "01jorphan0000000000000000") - os.WriteFile(orphan, []byte("x"), 0o600) - // A block that arrived while its upload was removed. - os.MkdirAll(filepath.Join(f.dir, "uploads", "01jorphan0000000000000001"), 0o700) - os.WriteFile(filepath.Join(f.dir, "uploads", "01jorphan0000000000000001", "block"), []byte("x"), 0o600) - + // The files of removed entries, as a crash during a removal leaves them, + // and a block that arrived while its upload was removed. + orphanBlob := filepath.Join(f.dir, "blobs", "01jorphan0000000000000000") + orphanUpload := filepath.Join(f.dir, "uploads", "01jorphan0000000000000001") + if err := errors.Join( + os.WriteFile(orphanBlob, []byte("x"), 0o600), + os.MkdirAll(orphanUpload, 0o700), + os.WriteFile(filepath.Join(orphanUpload, "block"), []byte("x"), 0o600), + ); err != nil { + t.Fatal(err) + } + gone := func(path string) bool { + _, err := os.Stat(path) + return errors.Is(err, os.ErrNotExist) + } + + // The cleanup removes the orphan uploads after the stale uploads. f.clock.advance(staleUpload - time.Minute) - f.srv.cleanup(ctx) - if _, err := f.st.GetCacheEntryByKey(ctx, "acme/app", "refs/heads/main", "abandoned", version); err != nil { + stop := f.start(t) + waitFor(t, "the removal of the orphan files", func() bool { return gone(orphanBlob) && gone(orphanUpload) }) + stop() + if err := abandoned(); err != nil { t.Fatalf("upload removed before one hour: %v", err) } - if staged, _ := os.ReadDir(filepath.Join(f.dir, "uploads")); len(staged) != 1 { - t.Errorf("staging folders = %v, want only the active upload", staged) + if n := dirBytes(t, f.dir); n != int64(len("x")+len("partial")) { + t.Errorf("%d bytes in the cache folder, want the blob of kept and the block of the active upload", n) } f.clock.advance(2 * time.Minute) - f.srv.removeOrphans(ctx, "blobs") - f.srv.cleanup(ctx) - if _, err := f.st.GetCacheEntryByKey(ctx, "acme/app", "refs/heads/main", "abandoned", version); !errors.Is(err, ErrNotFound) { - t.Errorf("abandoned upload still in the index: %v", err) - } - if staged, _ := os.ReadDir(filepath.Join(f.dir, "uploads")); len(staged) != 0 { - t.Errorf("staged blocks left: %v", staged) - } - if _, err := os.Stat(orphan); !os.IsNotExist(err) { - t.Errorf("orphan blob still on disk: %v", err) + stop = f.start(t) + waitFor(t, "the removal of the abandoned upload", func() bool { return errors.Is(abandoned(), store.ErrNotFound) }) + stop() + if n := dirBytes(t, f.dir); n != int64(len("x")) { + t.Errorf("%d bytes in the cache folder, want only the blob of kept", n) } if matched, _ := c.lookup(version, "kept"); matched != "kept" { t.Error("cleanup removed a finalized entry") @@ -447,8 +455,7 @@ func TestDamagedBlobIsAMiss(t *testing.T) { c.save("deps-old", version, []byte("old")) f.clock.advance(time.Minute) c.save("deps-new", version, []byte("new")) - e, _ := f.st.GetCacheEntryByKey(context.Background(), "acme/app", "refs/heads/main", "deps-new", version) - os.Truncate(filepath.Join(f.dir, e.Path), 1) + damage(t, f.dir, "new") // A download URL is a promise: BuildKit fails the build on a 404. if matched, data := c.restore(version, "deps-"); matched != "deps-old" || string(data) != "old" { @@ -470,60 +477,73 @@ func TestDelete(t *testing.T) { if matched, _ := c.lookup(version, "a"); matched != "" { t.Error("entry restorable after delete") } - if blobs, _ := os.ReadDir(filepath.Join(f.dir, "blobs")); len(blobs) != 0 { - t.Errorf("blobs left after delete: %v", blobs) + if n := dirBytes(t, f.dir); n != 0 { + t.Errorf("%d bytes in the cache folder after delete, want 0", n) + } +} + +// damage cuts each file under dir whose content is data to one byte, as a +// disk error can. +func damage(t *testing.T, dir, data string) { + t.Helper() + found := false + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + if b, err := os.ReadFile(path); err != nil || string(b) != data { + return err + } + found = true + return os.Truncate(path, 1) + }) + if err != nil || !found { + t.Fatalf("damage the file with %q: found %v, %v", data, found, err) } } func TestResultsProxy(t *testing.T) { - type call struct{ path, body, auth, sandbox string } + type call struct{ path, body, auth string } calls := make(chan call, 10) upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { b, _ := io.ReadAll(r.Body) - calls <- call{r.URL.Path, string(b), r.Header.Get("Authorization"), r.Header.Get(SandboxHeader)} + calls <- call{r.URL.Path, string(b), r.Header.Get("Authorization")} w.WriteHeader(http.StatusTeapot) io.WriteString(w, `{"ok":true}`) })) defer upstream.Close() - f := newFixture(t, 0) u, _ := url.Parse(upstream.URL) - f.srv.proxy = newProxy(u, f.srv.cfg.Logger) + f := newFixture(t, 0, func(c *Config) { c.Results = u }) const path = "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact" req, _ := http.NewRequest(http.MethodPost, f.proxied+path, strings.NewReader(`{"name":"dist"}`)) req.Header.Set("Authorization", "Bearer job-token") - req.Header.Set(SandboxHeader, "sb-1") + req.Header.Set(agentSandbox, "sb-1") resp := client{t: t}.do(req) body, _ := io.ReadAll(resp.Body) resp.Body.Close() if resp.StatusCode != http.StatusTeapot || string(body) != `{"ok":true}` { t.Errorf("response %d %q", resp.StatusCode, body) } - if got, want := <-calls, (call{path, `{"name":"dist"}`, "Bearer job-token", ""}); got != want { + if got, want := <-calls, (call{path, `{"name":"dist"}`, "Bearer job-token"}); got != want { t.Errorf("upstream got %+v, want %+v", got, want) } - // The service port listens on all addresses. Only sandboxes and the host - // itself may send requests to GitHub through it, else anyone could hide - // behind the server. + // The service port listens on all addresses. Only sandboxes may send + // requests to GitHub through it, else anyone could hide behind the + // server. tests := []struct { name string remote string - header http.Header want int }{ - {"sandbox without a job", "10.213.0.5:40000", nil, http.StatusTeapot}, - {"this host", "127.0.0.1:40000", nil, http.StatusTeapot}, - {"internet host", "203.0.113.9:40000", nil, http.StatusForbidden}, - {"internet host that names a sandbox", "203.0.113.9:40000", http.Header{SandboxHeader: {"10.213.0.5"}}, http.StatusForbidden}, + {"sandbox without a job", "10.213.0.5:40000", http.StatusTeapot}, + {"internet host", "203.0.113.9:40000", http.StatusForbidden}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodPost, path, strings.NewReader("{}")) req.RemoteAddr = tt.remote - for k, v := range tt.header { - req.Header[k] = v - } rec := httptest.NewRecorder() f.srv.ServeHTTP(rec, req) if rec.Code != tt.want { @@ -539,76 +559,76 @@ func TestResultsProxy(t *testing.T) { } } -// lateJob is a sandbox whose job runplane learns after linked is set, as -// with GitHub, which sends the job message after the job starts. -type lateJob struct{ linked atomic.Bool } +// lateJob is a sandbox whose job runplane learns after the first lookup of +// the sandbox, as with GitHub, which sends the job message after the job +// starts. A job that never comes stays unknown. +type lateJob struct { + never bool + lookups atomic.Int32 +} -func (l *lateJob) Sandbox(context.Context, string, http.Header) (Identity, error) { - if !l.linked.Load() { +func (l *lateJob) SandboxAt(context.Context, netip.Addr) (Identity, error) { + if l.lookups.Add(1) == 1 || l.never { return Identity{SandboxID: "sb-1"}, nil } return acme("sb-1", "refs/heads/main", ""), nil } +func (l *lateJob) SandboxOn(ctx context.Context, _, _ string) (Identity, error) { + return l.SandboxAt(ctx, netip.Addr{}) +} + // The first cache step of a job runs before runplane knows the job of the // sandbox. The request must wait for the job, not fail with 403. func TestCacheRequestWaitsForTheJob(t *testing.T) { tests := []struct { - name string - linkIn time.Duration // 0 = never - wantErr error + name string + never bool + status int }{ - {"job known after 200 ms", 200 * time.Millisecond, nil}, - {"job never known", 0, ErrNoSandbox}, + {"job known after the first lookup", false, http.StatusOK}, + {"job never known", true, http.StatusForbidden}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - r := &lateJob{} - s := &Server{cfg: Config{Resolver: r}, jobWait: time.Second} - if tt.linkIn > 0 { - time.AfterFunc(tt.linkIn, func() { r.linked.Store(true) }) - } - ident, err := s.identify(httptest.NewRequest("POST", "/twirp/x", nil), false) - if !errors.Is(err, tt.wantErr) { - t.Fatalf("err %v, want %v", err, tt.wantErr) - } - if tt.wantErr == nil && ident.Repository != "acme/app" { - t.Errorf("identity %+v, want the repository of the job", ident) + f := newFixture(t, 0, func(c *Config) { c.Resolver = &lateJob{never: tt.never} }) + status, out := client{t: t, url: f.url}.call("CreateCacheEntry", map[string]any{"key": "k", "version": version}) + if status != tt.status { + t.Errorf("CreateCacheEntry: %d %v, want %d", status, out, tt.status) } }) } } -func TestStatsAddPerHour(t *testing.T) { - st := openStore(t) - s := NewStats(st, slog.New(slog.DiscardHandler)) - t0 := time.Date(2026, 11, 4, 10, 15, 0, 0, time.UTC) - now := t0 - s.now = func() time.Time { return now } - s.Add("acme/api", store.CacheCounts{Hits: 1, BytesRead: 100}) - now = t0.Add(30 * time.Minute) - s.Add("acme/api", store.CacheCounts{Misses: 1}) - s.Add("acme/web", store.CacheCounts{BytesWritten: 7}) - now = t0.Add(time.Hour) - s.Add("acme/api", store.CacheCounts{Hits: 2}) - - // Run writes the counts when it ends. - ctx, cancel := context.WithCancel(context.Background()) - cancel() - s.Run(ctx) - // A later flush adds to the rows that exist. - s.Add("acme/api", store.CacheCounts{Hits: 1}) - s.flush(context.Background()) +// Each request counts in the hour of its time, and a later write of the +// counts adds to the rows of the hour. +func TestStatsPerHour(t *testing.T) { + f := newFixture(t, 0) + t0 := f.clock.now() + c, other := client{t: t, url: f.url}, f.via(t, "sb-other") + c.save("k", version, []byte("data")) + f.clock.advance(30 * time.Minute) + c.restore(version, "k") + c.lookup(version, "missing") + other.save("o", version, []byte("12345")) + f.clock.advance(30 * time.Minute) + c.restore(version, "k") + f.start(t)() + c.restore(version, "k") + f.start(t)() tests := []struct { since time.Time want map[string]store.CacheCounts }{ - {t0, map[string]store.CacheCounts{"acme/api": {Hits: 4, Misses: 1, BytesRead: 100}, "acme/web": {BytesWritten: 7}}}, - {t0.Add(time.Hour), map[string]store.CacheCounts{"acme/api": {Hits: 3}}}, + {t0, map[string]store.CacheCounts{ + "acme/app": {Hits: 3, Misses: 1, BytesRead: 12, BytesWritten: 4}, + "other/lib": {BytesWritten: 5}, + }}, + {t0.Add(time.Hour), map[string]store.CacheCounts{"acme/app": {Hits: 2, BytesRead: 8}}}, } for _, tt := range tests { - got, err := st.CacheStatsByRepository(context.Background(), tt.since) + got, err := f.st.CacheStatsByRepository(t.Context(), tt.since) if err != nil || !maps.Equal(got, tt.want) { t.Errorf("counts since %s = %v, %v; want %v", tt.since, got, err, tt.want) } diff --git a/internal/cache/clients_test.go b/internal/cache/clients_test.go index 001d2dc..1e3b56c 100644 --- a/internal/cache/clients_test.go +++ b/internal/cache/clients_test.go @@ -21,7 +21,7 @@ import ( // server. It needs Node, npm, and network access for npm install. func TestActionsCacheClient(t *testing.T) { if os.Getenv("RUNPLANE_CACHE_NODE_TEST") == "" { - t.Skip("set RUNPLANE_CACHE_NODE_TEST=1 to run @actions/cache against the server") + t.Skip("set RUNPLANE_CACHE_NODE_TEST=1 to run @actions/cache against the server; the test needs node, npm, and network access") } modules := t.TempDir() run(t, modules, "npm", "init", "-y") @@ -32,12 +32,10 @@ func TestActionsCacheClient(t *testing.T) { t.Fatal(err) } - f := newFixture(t, 0) // The clock of the fixture stays still; the client needs real expiry times. - f.srv.now = time.Now + f := newFixture(t, 0, func(c *Config) { c.Now = time.Now }) var blocks, blockLists atomic.Int64 var sandboxID atomic.Value // of the next client run - proxied := f.srv.Proxied() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Query().Get("comp") { case "block": @@ -45,8 +43,7 @@ func TestActionsCacheClient(t *testing.T) { case "blocklist": blockLists.Add(1) } - r.Header.Set(SandboxHeader, sandboxID.Load().(string)) - proxied.ServeHTTP(w, r) + f.srv.ServeAgent(w, r, "agent-1", sandboxID.Load().(string)) })) defer server.Close() @@ -143,7 +140,6 @@ func TestActionsCacheClient(t *testing.T) { t.Errorf("restore by restore key: %v", res) } - // The feature branch entry is not visible from a pull request. if res := node("sb-pr1", "restore", "npm-linux-abc"); res["matched"] != nil { t.Errorf("pull request restored a feature branch entry: %v", res) } @@ -165,7 +161,7 @@ func TestActionsCacheClient(t *testing.T) { // network access for the module download. func TestBuildKitClient(t *testing.T) { if os.Getenv("RUNPLANE_CACHE_BUILDKIT_TEST") == "" { - t.Skip("set RUNPLANE_CACHE_BUILDKIT_TEST=1 to run BuildKit's cache client against the server") + t.Skip("set RUNPLANE_CACHE_BUILDKIT_TEST=1 to run BuildKit's cache client against the server; the test needs network access for the Go module download") } mod := t.TempDir() src, err := os.ReadFile(filepath.Join("testdata", "buildkit", "main.go")) @@ -177,8 +173,7 @@ func TestBuildKitClient(t *testing.T) { run(t, mod, "go", "get", "github.com/tonistiigi/go-actions-cache@latest") run(t, mod, "go", "build", "-o", "client", ".") - f := newFixture(t, 0) - f.srv.now = time.Now + f := newFixture(t, 0, func(c *Config) { c.Now = time.Now }) server := httptest.NewServer(f.srv) defer server.Close() diff --git a/internal/cache/disk.go b/internal/cache/disk.go index 6e0e74f..be39108 100644 --- a/internal/cache/disk.go +++ b/internal/cache/disk.go @@ -8,6 +8,8 @@ import ( "io" "os" "path/filepath" + + "github.com/runplane/runplane/internal/atomicfile" ) var ( @@ -25,10 +27,7 @@ type disk struct { root string } -// blobPath is the path of the blob of an entry, relative to the cache folder. -func blobPath(id string) string { return filepath.Join("blobs", id) } - -func (d disk) blob(id string) string { return filepath.Join(d.root, blobPath(id)) } +func (d disk) blob(id string) string { return filepath.Join(d.root, "blobs", id) } func (d disk) staging(id string) string { return filepath.Join(d.root, "uploads", id) } func (d disk) init() error { @@ -69,22 +68,18 @@ func (d disk) commitBlocks(id string, blockIDs []string) error { return errEmptyCommit } dir := d.staging(id) - tmp, err := os.CreateTemp(dir, "tmp-") + f, err := atomicfile.CreateIn(dir, d.blob(id), 0o600) if err != nil { - return fmt.Errorf("create blob: %w", err) + return err } - defer os.Remove(tmp.Name()) - defer tmp.Close() + defer f.Close() for _, blockID := range blockIDs { - if err := appendFile(tmp, filepath.Join(dir, blockName(blockID))); err != nil { + if err := appendFile(f, filepath.Join(dir, blockName(blockID))); err != nil { return err } } - if err := tmp.Close(); err != nil { - return fmt.Errorf("write blob: %w", err) - } - if err := os.Rename(tmp.Name(), d.blob(id)); err != nil { - return fmt.Errorf("commit blob: %w", err) + if err := f.Commit(); err != nil { + return err } return os.RemoveAll(dir) } @@ -102,7 +97,6 @@ func (d disk) putBlob(id string, body io.Reader, limit int64) error { return os.RemoveAll(dir) } -// size returns the size of the committed blob of an entry. func (d disk) size(id string) (int64, error) { info, err := os.Stat(d.blob(id)) if errors.Is(err, os.ErrNotExist) { @@ -118,7 +112,6 @@ func (d disk) open(id string) (*os.File, error) { return os.Open(d.blob(id)) } -// remove deletes the blob and the staged blocks of an entry. func (d disk) remove(id string) error { err := os.Remove(d.blob(id)) if errors.Is(err, os.ErrNotExist) { @@ -143,32 +136,25 @@ func blockName(blockID string) string { return hex.EncodeToString(sum[:]) } -// writeFile writes body to path through a temporary file in dir, so that a -// failed request leaves no partial file at path. +// writeFile writes body to path through a temporary file in the staging +// folder dir, so that a failed request leaves no partial file at path. func writeFile(dir, path string, body io.Reader, limit int64) error { - tmp, err := os.CreateTemp(dir, "tmp-") + f, err := atomicfile.CreateIn(dir, path, 0o600) if err != nil { - return fmt.Errorf("create file: %w", err) + return err } - defer os.Remove(tmp.Name()) - defer tmp.Close() - n, err := io.Copy(tmp, io.LimitReader(body, limit+1)) + defer f.Close() + n, err := io.Copy(f, io.LimitReader(body, limit+1)) if err != nil { return fmt.Errorf("read upload: %w", err) } if n > limit { return errTooLarge } - if err := tmp.Close(); err != nil { - return fmt.Errorf("write file: %w", err) - } - if err := os.Rename(tmp.Name(), path); err != nil { - return fmt.Errorf("move file: %w", err) - } - return nil + return f.Commit() } -func appendFile(dst *os.File, path string) error { +func appendFile(dst io.Writer, path string) error { src, err := os.Open(path) if errors.Is(err, os.ErrNotExist) { return errNoBlock diff --git a/internal/cache/helpers_test.go b/internal/cache/helpers_test.go index a457f5f..103cc77 100644 --- a/internal/cache/helpers_test.go +++ b/internal/cache/helpers_test.go @@ -7,27 +7,33 @@ import ( "encoding/json" "fmt" "io" - "log/slog" + "io/fs" "net/http" "net/http/httptest" + "net/netip" "path/filepath" "strconv" "strings" + "sync" "sync/atomic" "testing" "time" + "github.com/runplane/runplane/internal/datadisk" "github.com/runplane/runplane/internal/store" ) -// sandboxes resolves by sandbox ID when the header has one, else by IP. +// sandboxes resolves a sandbox of this server by its address, and a sandbox +// of an agent by its ID. type sandboxes map[string]Identity -func (s sandboxes) Sandbox(_ context.Context, ip string, h http.Header) (Identity, error) { - key := ip - if id := h.Get(SandboxHeader); id != "" { - key = id - } +func (s sandboxes) SandboxAt(_ context.Context, addr netip.Addr) (Identity, error) { + return s.find(addr.String()) +} + +func (s sandboxes) SandboxOn(_ context.Context, _, id string) (Identity, error) { return s.find(id) } + +func (s sandboxes) find(key string) (Identity, error) { ident, ok := s[key] if !ok { return Identity{}, ErrNoSandbox @@ -35,6 +41,10 @@ func (s sandboxes) Sandbox(_ context.Context, ip string, h http.Header) (Identit return ident, nil } +// agentSandbox names the sandbox of a request to the proxied URL of the +// fixture, which serves the requests as the requests of an agent. +const agentSandbox = "X-Test-Sandbox" + type clock struct{ ns atomic.Int64 } func (c *clock) now() time.Time { return time.Unix(0, c.ns.Load()).UTC() } @@ -42,23 +52,24 @@ func (c *clock) advance(d time.Duration) { c.ns.Add(int64(d)) } type fixture struct { limit atomic.Int64 + disk func() datadisk.Space // if set, the room of the cache instead of limit srv *Server st *store.Store clock clock dir string url string // direct sandbox traffic - proxied string // traffic from a trusted proxy + proxied string // traffic of an agent } // newFixture starts a server whose requests from 127.0.0.1 belong to // sandbox "sb-1" with a push job on main of acme/app. The other sandboxes -// send through the trusted proxy (see via). -func newFixture(t *testing.T, maxBytes int64) *fixture { +// are on an agent (see via). opts change the config. +func newFixture(t *testing.T, maxBytes int64, opts ...func(*Config)) *fixture { t.Helper() f := &fixture{st: openStore(t), dir: t.TempDir()} f.limit.Store(maxBytes) f.clock.ns.Store(time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC).UnixNano()) - srv, err := New(Config{ + cfg := Config{ Dir: f.dir, Store: f.st, Resolver: sandboxes{ @@ -75,28 +86,37 @@ func newFixture(t *testing.T, maxBytes int64) *fixture { "sb-idle": {SandboxID: "sb-idle"}, "10.213.0.5": {SandboxID: "sb-idle"}, }, - Stats: NewStats(f.st, slog.New(slog.DiscardHandler)), - MaxBytes: f.limit.Load, - DiskSpace: func() (int64, int64, error) { return 0, 0, nil }, - MinFreePercent: func() float64 { return 0 }, - }) + Space: func() datadisk.Space { + if f.disk != nil { + return f.disk() + } + return datadisk.Space{Cache: f.limit.Load()} + }, + Now: f.clock.now, + } + for _, opt := range opts { + opt(&cfg) + } + srv, err := New(cfg) if err != nil { t.Fatal(err) } - srv.now = f.clock.now srv.jobWait = 100 * time.Millisecond f.srv = srv direct := httptest.NewServer(srv) - proxied := httptest.NewServer(srv.Proxied()) + proxied := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv.ServeAgent(w, r, "agent-1", r.Header.Get(agentSandbox)) + })) t.Cleanup(direct.Close) t.Cleanup(proxied.Close) f.url, f.proxied = direct.URL, proxied.URL return f } -// counts returns the cache statistics of a repository. +// counts returns the counts of a repository in the store. Run writes the +// counts that it has in memory when it stops. func (f *fixture) counts(t *testing.T, repository string) store.CacheCounts { - f.srv.cfg.Stats.flush(t.Context()) + f.start(t)() counts, err := f.st.CacheStatsByRepository(t.Context(), time.Time{}) if err != nil { t.Fatal(err) @@ -104,6 +124,56 @@ func (f *fixture) counts(t *testing.T, repository string) store.CacheCounts { return counts[repository] } +// start runs Run of the server until stop, which waits until Run returns. +func (f *fixture) start(t *testing.T) (stop func()) { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + f.srv.Run(ctx) + close(done) + }() + var once sync.Once + stop = func() { + once.Do(func() { + cancel() + <-done + }) + } + t.Cleanup(stop) + return stop +} + +// dirBytes returns the bytes of the files under dir. +func dirBytes(t *testing.T, dir string) int64 { + t.Helper() + var n int64 + err := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + info, err := d.Info() + if err == nil { + n += info.Size() + } + return err + }) + if err != nil { + t.Fatal(err) + } + return n +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(10 * time.Millisecond) + } +} + func openStore(t *testing.T) *store.Store { t.Helper() st, err := store.Open(filepath.Join(t.TempDir(), "runplane.db")) @@ -118,12 +188,12 @@ func acme(id, ref, baseRef string) Identity { return Identity{SandboxID: id, Repository: "acme/app", DefaultBranch: "main", Ref: ref, BaseRef: baseRef} } -// via returns a client of a sandbox whose requests come through the trusted -// proxy. A job can write any cache scopes into its runtime token, because -// runplane cannot check the signature. So each of these tokens claims main, -// and the branch rules must hold anyway. +// via returns a client of a sandbox on an agent, whose requests come +// through the agent. A job can write any cache scopes into its runtime +// token, because runplane cannot check the signature. So each of these +// tokens claims main, and the branch rules must hold anyway. func (f *fixture) via(t *testing.T, sandboxID string) client { - return client{t: t, url: f.proxied, token: token(scope{"refs/heads/main", 3}), header: http.Header{SandboxHeader: {sandboxID}}} + return client{t: t, url: f.proxied, token: token(scope{"refs/heads/main", 3}), header: http.Header{agentSandbox: {sandboxID}}} } type scope struct { diff --git a/internal/cache/prune.go b/internal/cache/prune.go new file mode 100644 index 0000000..beacd89 --- /dev/null +++ b/internal/cache/prune.go @@ -0,0 +1,13 @@ +package cache + +import ( + "context" + "time" + + "github.com/runplane/runplane/internal/store" +) + +// Prune deletes the hourly counts older than store.SampleRetention. +func (s *Server) Prune(ctx context.Context, now time.Time) error { + return s.cfg.Store.PruneCacheStats(ctx, now.Add(-store.SampleRetention)) +} diff --git a/internal/cache/resolver.go b/internal/cache/resolver.go index 098d4aa..b417dd1 100644 --- a/internal/cache/resolver.go +++ b/internal/cache/resolver.go @@ -4,11 +4,8 @@ import ( "context" "errors" "fmt" - "net/http" "net/netip" "strings" - "sync" - "time" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" @@ -18,33 +15,42 @@ import ( // the primary, and the repository and the refs of its job, in the store. type SandboxResolver struct { st *store.Store - // local lists the sandboxes of the primary's own driver. It finds a - // sandbox whose address the sandboxes table does not have yet. + // local lists the sandboxes of the primary's own driver. local func(ctx context.Context) ([]sandbox.Sandbox, error) - - mu sync.Mutex - localID string } func NewSandboxResolver(st *store.Store, local func(ctx context.Context) ([]sandbox.Sandbox, error)) *SandboxResolver { return &SandboxResolver{st: st, local: local} } -// Sandbox finds a sandbox of the primary by the source address of the -// request. A request through the agent tunnel names its sandbox in -// SandboxHeader instead; the agent can name only a sandbox of its own -// server, which the primary sets in ServerHeader. -func (r *SandboxResolver) Sandbox(ctx context.Context, remoteIP string, header http.Header) (Identity, error) { - var b store.SandboxJob - var err error - if id := header.Get(SandboxHeader); id != "" { - b, err = r.st.LiveSandboxJob(ctx, id) - if err == nil && b.ServerID != header.Get(ServerHeader) { - err = fmt.Errorf("sandbox %s is on another server: %w", id, store.ErrNotFound) - } - } else { - b, err = r.byIP(ctx, remoteIP) +// SandboxAt finds the sandbox of the primary at addr in the list of its +// driver. It reads the list each time, because an address can belong to a +// new sandbox a short time after a different sandbox stops. +func (r *SandboxResolver) SandboxAt(ctx context.Context, addr netip.Addr) (Identity, error) { + list, err := r.local(ctx) + if err != nil { + return Identity{}, fmt.Errorf("list sandboxes: %w", err) + } + s, ok := sandbox.FindByIP(list, addr) + if !ok { + return Identity{}, fmt.Errorf("no sandbox has address %s: %w", addr, ErrNoSandbox) + } + return found(r.st.LiveSandboxJob(ctx, s.ID)) +} + +// SandboxOn finds a sandbox that an agent names. The agent can name only a +// sandbox of its own server. +func (r *SandboxResolver) SandboxOn(ctx context.Context, serverID, id string) (Identity, error) { + b, err := r.st.LiveSandboxJob(ctx, id) + if err == nil && b.ServerID != serverID { + err = fmt.Errorf("sandbox %s is on another server: %w", id, store.ErrNotFound) } + return found(b, err) +} + +// found gives the Identity of a live sandbox row, and ErrNoSandbox when no +// live sandbox has the row. +func found(b store.SandboxJob, err error) (Identity, error) { if errors.Is(err, store.ErrNotFound) { return Identity{}, fmt.Errorf("%w: %w", ErrNoSandbox, err) } @@ -87,56 +93,3 @@ func identity(b store.SandboxJob) Identity { } return id } - -// Allow reports if a sandbox of the primary has the address: the Allow -// hook of the registry mirror. -func (r *SandboxResolver) Allow(ip netip.Addr) bool { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := r.byIP(ctx, ip.String()) - return err == nil -} - -func (r *SandboxResolver) byIP(ctx context.Context, remoteIP string) (store.SandboxJob, error) { - addr, err := netip.ParseAddr(remoteIP) - if err != nil { - return store.SandboxJob{}, fmt.Errorf("source address %q: %w", remoteIP, store.ErrNotFound) - } - ip := addr.Unmap().String() - serverID, err := r.localServer(ctx) - if err != nil { - return store.SandboxJob{}, err - } - b, err := r.st.LiveSandboxJobByIP(ctx, serverID, ip) - if !errors.Is(err, store.ErrNotFound) { - return b, err - } - list, err := r.local(ctx) - if err != nil { - return store.SandboxJob{}, fmt.Errorf("list sandboxes: %w", err) - } - for _, s := range list { - if s.IP != ip { - continue - } - if err := r.st.SetSandboxIP(ctx, serverID, s.ID, ip); err != nil { - return store.SandboxJob{}, err - } - return r.st.LiveSandboxJob(ctx, s.ID) - } - return store.SandboxJob{}, fmt.Errorf("no sandbox has address %s: %w", ip, store.ErrNotFound) -} - -func (r *SandboxResolver) localServer(ctx context.Context) (string, error) { - r.mu.Lock() - defer r.mu.Unlock() - if r.localID != "" { - return r.localID, nil - } - v, err := r.st.GetLocalServer(ctx) - if err != nil { - return "", err - } - r.localID = v.ID - return v.ID, nil -} diff --git a/internal/cache/resolver_test.go b/internal/cache/resolver_test.go index a9edfa3..9a78216 100644 --- a/internal/cache/resolver_test.go +++ b/internal/cache/resolver_test.go @@ -3,7 +3,6 @@ package cache import ( "context" "errors" - "net/http" "net/netip" "testing" "time" @@ -21,7 +20,7 @@ func TestResolver(t *testing.T) { Status: store.ServerOnline, CreatedAt: t0, UpdatedAt: t0}); err != nil { t.Fatal(err) } - if err := st.SaveGitHubApp(ctx, store.GitHubApp{ID: "app", AppID: 9, Slug: "rp", PrivateKeyEnc: []byte("k"), ClientSecretEnc: []byte("s"), WebhookSecretEnc: []byte("w"), CreatedAt: t0}); err != nil { + if err := st.SaveGitHubApp(ctx, store.GitHubApp{ID: "app", AppID: 9, Slug: "rp", PrivateKeyEnc: []byte("k"), CreatedAt: t0}); err != nil { t.Fatal(err) } if err := st.UpsertInstallation(ctx, store.Installation{ID: 1, AppID: 9, AccountLogin: "acme", @@ -35,16 +34,16 @@ func TestResolver(t *testing.T) { } var jobs []string for i, repo := range []string{"acme/api", "acme/web"} { - j, err := st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: int64(i + 1), State: store.JobRunning, Repository: repo}) + j, err := st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: int64(i + 1), State: store.JobRunning, Repository: repo}, t0) if err != nil { t.Fatal(err) } jobs = append(jobs, j.ID) } - if err := st.SetJobRunRef(ctx, jobs[0], store.RunRef{Event: "push", HeadBranch: "trunk"}); err != nil { + if err := st.SetJobRun(ctx, jobs[0], false, store.RunRef{Event: "push", HeadBranch: "trunk"}); err != nil { t.Fatal(err) } - if err := st.SetJobRunRef(ctx, jobs[1], store.RunRef{Event: "pull_request", HeadBranch: "main", PRNumber: 5, BaseRef: "trunk"}); err != nil { + if err := st.SetJobRun(ctx, jobs[1], false, store.RunRef{Event: "pull_request", HeadBranch: "main", PRNumber: 5, BaseRef: "trunk"}); err != nil { t.Fatal(err) } api := Identity{SandboxID: "busy", Repository: "acme/api", DefaultBranch: "trunk", Ref: "refs/heads/trunk"} @@ -52,11 +51,10 @@ func TestResolver(t *testing.T) { return Identity{SandboxID: id, Repository: "acme/web", Ref: "refs/pull/5/merge", BaseRef: "refs/heads/trunk"} } sandboxes := []store.Sandbox{ - {ID: "busy", ServerID: "local", State: sandbox.StateBusy, IP: "10.213.0.5", JobID: jobs[0]}, - {ID: "idle", ServerID: "local", State: sandbox.StateIdle, IP: "10.213.0.6"}, - {ID: "old", ServerID: "local", State: sandbox.StateGone, IP: "10.213.0.7", JobID: jobs[0]}, - {ID: "agent", ServerID: "remote", State: sandbox.StateBusy, IP: "10.213.0.8", JobID: jobs[1]}, - {ID: "new", ServerID: "local", State: sandbox.StateBusy, JobID: jobs[1]}, + {ID: "busy", ServerID: "local", State: sandbox.StateBusy, JobID: jobs[0]}, + {ID: "idle", ServerID: "local", State: sandbox.StateIdle}, + {ID: "old", ServerID: "local", State: sandbox.StateGone, JobID: jobs[0]}, + {ID: "agent", ServerID: "remote", State: sandbox.StateBusy, JobID: jobs[1]}, } for _, b := range sandboxes { b.PoolID, b.Mode, b.VCPU, b.MemoryMB, b.CreatedAt = "pool", sandbox.ModeContainer, 2, 4096, t0 @@ -64,39 +62,38 @@ func TestResolver(t *testing.T) { t.Fatal(err) } } - // The driver knows the address of the newest sandbox before the table does. + // The driver of the primary knows the addresses. It still lists old, + // which the scheduler has recorded as gone. r := NewSandboxResolver(st, func(context.Context) ([]sandbox.Sandbox, error) { - return []sandbox.Sandbox{{ID: "busy", IP: "10.213.0.5"}, {ID: "new", IP: "10.213.0.9"}}, nil + return []sandbox.Sandbox{ + {ID: "busy", State: sandbox.StateBusy, IP: "10.213.0.5"}, + {ID: "idle", State: sandbox.StateIdle, IP: "10.213.0.6"}, + {ID: "old", State: sandbox.StateBusy, IP: "10.213.0.7"}, + }, nil }) - tunnel := func(sandboxID, serverID string) http.Header { - h := http.Header{} - h.Set(SandboxHeader, sandboxID) - if serverID != "" { - h.Set(ServerHeader, serverID) - } - return h + at := func(ip string) func() (Identity, error) { + return func() (Identity, error) { return r.SandboxAt(ctx, netip.MustParseAddr(ip)) } + } + on := func(serverID, id string) func() (Identity, error) { + return func() (Identity, error) { return r.SandboxOn(ctx, serverID, id) } } tests := []struct { - name string - ip string - header http.Header - want Identity // zero: ErrNoSandbox + name string + find func() (Identity, error) + want Identity // zero: ErrNoSandbox }{ - {"sandbox address", "10.213.0.5", nil, api}, - {"IPv4 in IPv6 form", "::ffff:10.213.0.5", nil, api}, - {"sandbox without a job", "10.213.0.6", nil, Identity{SandboxID: "idle"}}, - {"address from the driver", "10.213.0.9", nil, web("new")}, - {"stopped sandbox", "10.213.0.7", nil, Identity{}}, - {"address of an agent sandbox", "10.213.0.8", nil, Identity{}}, - {"unknown address", "192.0.2.1", nil, Identity{}}, - {"agent names its sandbox", "203.0.113.4", tunnel("agent", "remote"), web("agent")}, - {"agent names a sandbox of the primary", "203.0.113.4", tunnel("busy", "remote"), Identity{}}, - {"sandbox header without a server", "10.213.0.5", tunnel("agent", ""), Identity{}}, + {"sandbox address", at("10.213.0.5"), api}, + {"IPv4 in IPv6 form", at("::ffff:10.213.0.5"), api}, + {"sandbox without a job", at("10.213.0.6"), Identity{SandboxID: "idle"}}, + {"stopped sandbox", at("10.213.0.7"), Identity{}}, + {"unknown address", at("192.0.2.1"), Identity{}}, + {"agent names its sandbox", on("remote", "agent"), web("agent")}, + {"agent names a sandbox of the primary", on("remote", "busy"), Identity{}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := r.Sandbox(ctx, tt.ip, tt.header) + got, err := tt.find() if tt.want == (Identity{}) { if !errors.Is(err, ErrNoSandbox) { t.Fatalf("got %+v, %v; want ErrNoSandbox", got, err) @@ -108,13 +105,6 @@ func TestResolver(t *testing.T) { } }) } - - if b, err := st.GetSandbox(ctx, "new"); err != nil || b.IP != "10.213.0.9" { - t.Errorf("the table did not get the address from the driver: %+v, %v", b, err) - } - if !r.Allow(netip.MustParseAddr("10.213.0.6")) || r.Allow(netip.MustParseAddr("10.213.0.8")) { - t.Error("the mirror must allow the sandboxes of the primary and nothing else") - } } // The cache server lets a job save only to Ref, so Ref must never name a diff --git a/internal/cache/space.go b/internal/cache/space.go index c200fb1..ce93d06 100644 --- a/internal/cache/space.go +++ b/internal/cache/space.go @@ -5,36 +5,28 @@ import ( "fmt" "math" "sync" + "time" ) -const ( - // evictMargin is how far above MinFreePercent the eviction frees the - // disk, so that it does not run again at each upload. - evictMargin = 5 - // maxOpenUploads is the count of unfinished uploads that one sandbox may - // have. Each one is an index row until it is finalized or stale. - maxOpenUploads = 64 -) +// maxOpenUploads is the count of unfinished uploads that one sandbox may +// have. Each one is an index row until it is finalized or stale. +const maxOpenUploads = 64 -// uploads counts the bytes of the unfinished uploads, which count against -// the limit like the entries: their files on disk and the writes in -// progress. Without the count, many unfinished uploads could each take the -// whole limit. +// uploads holds what only memory knows of the unfinished uploads: the bytes +// that the writes in progress may still add, and the sandbox of each upload. +// The rows of the uploads keep the bytes of their files. The uploads count +// against the limit like the entries. Without the count, many unfinished +// uploads could each take the whole limit. type uploads struct { mu sync.Mutex - onDisk map[string]int64 // bytes of the files of each unfinished entry, by entry ID - writing int64 // bytes that the writes in progress may still add + writing int64 owner map[string]string // sandbox ID by unfinished entry ID } -func (u *uploads) bytes() int64 { +func (u *uploads) inFlight() int64 { u.mu.Lock() defer u.mu.Unlock() - n := u.writing - for _, b := range u.onDisk { - n += b - } - return n + return u.writing } // open adds an upload of the sandbox. It returns false if the sandbox has @@ -55,35 +47,20 @@ func (u *uploads) open(id, sandboxID string) bool { return true } -// drop stops the count of an upload that is finalized or removed. func (u *uploads) drop(id string) { u.mu.Lock() defer u.mu.Unlock() - delete(u.onDisk, id) delete(u.owner, id) } -// Limit returns the byte limit of all entries and unfinished uploads, and -// so of one entry: MaxBytes, and at most a quarter of the disk. -func (s *Server) Limit() int64 { - limit := int64(math.MaxInt64) - if n := s.cfg.MaxBytes(); n > 0 { - limit = n - } - if total, _, err := s.cfg.DiskSpace(); err == nil && total > 0 { - limit = min(limit, total/4) - } - return limit -} - -// diskShort returns the bytes that the disk lacks to have MinFreePercent -// plus margin free, or 0. -func (s *Server) diskShort(margin float64) int64 { - total, free, err := s.cfg.DiskSpace() - if err != nil || total <= 0 { - return 0 - } - return max(0, int64(float64(total)*(s.cfg.MinFreePercent()+margin)/100)-free) +// MaxBytes returns the byte limit of all entries and unfinished uploads, +// and so of one entry, that the server uses now. It is math.MaxInt64 when +// the cache has no limit. +func (s *Server) MaxBytes() int64 { + if limit := s.cfg.Space().Cache; limit > 0 { + return limit + } + return math.MaxInt64 } // reserve takes up to want bytes of the limit for one write to an upload. @@ -98,27 +75,28 @@ func (s *Server) reserve(ctx context.Context, want int64) (int64, error) { } s.mu.Lock() defer s.mu.Unlock() + limit := s.MaxBytes() + size, err := s.cfg.Store.CacheSize(ctx) + if err != nil { + return 0, err + } // The entries can make room only for what fits next to the other uploads. - if s.uploads.bytes()+extra > s.Limit() { + if size.Uploads+s.uploads.inFlight()+extra > limit { return 0, errTooLarge } if err := s.evict(ctx, extra); err != nil { return 0, err } - if s.diskShort(0) > 0 { + if s.cfg.Space().Low() { return 0, errTooLarge } - entries, err := s.cfg.Store.CacheSize(ctx) - if err != nil { + if size, err = s.cfg.Store.CacheSize(ctx); err != nil { return 0, err } u := &s.uploads u.mu.Lock() defer u.mu.Unlock() - room := s.Limit() - entries - u.writing - for _, n := range u.onDisk { - room -= n - } + room := limit - size.Entries - size.Uploads - u.writing if room <= 0 { return 0, errTooLarge } @@ -127,48 +105,53 @@ func (s *Server) reserve(ctx context.Context, want int64) (int64, error) { return n, nil } -// release gives back the bytes of reserve after a write to the upload id, -// and counts its files as they are now. -func (s *Server) release(id string, reserved int64) { - _, staged := s.disk.staged(id) - blob, _ := s.disk.size(id) +// release gives back the bytes of reserve after a write. The caller first +// records the files of a write that worked (recordUpload). +func (s *Server) release(reserved int64) { u := &s.uploads u.mu.Lock() defer u.mu.Unlock() u.writing -= reserved - if staged+blob > 0 { - u.onDisk[id] = staged + blob - } } -// countUploads counts the files of the unfinished uploads that an earlier -// process left. -func (s *Server) countUploads(ctx context.Context) { +// recordUpload writes the bytes that the files of the upload id have now in +// its row, with the time of its last use. +func (s *Server) recordUpload(ctx context.Context, id string, used time.Time) error { + _, staged := s.disk.staged(id) + blob, _ := s.disk.size(id) + return s.cfg.Store.SetCacheUpload(ctx, id, staged+blob, used) +} + +// syncUploads records the files of the unfinished uploads that an earlier +// process left, because it can stop between a write and its record. +func (s *Server) syncUploads(ctx context.Context) { list, err := s.cfg.Store.ListUnfinishedCacheEntries(ctx, s.now()) if err != nil { s.cfg.Logger.Error("cache: list unfinished uploads", "err", err) return } for _, e := range list { - s.release(e.ID, 0) + if err := s.recordUpload(ctx, e.ID, e.LastUsedAt); err != nil { + s.cfg.Logger.Warn("cache: record upload", "id", e.ID, "err", err) + } } } // evict removes the least recently used entries until the entries, the // unfinished uploads, and extra more bytes fit in the limit, and the disk -// has MinFreePercent plus evictMargin free. +// has the free part that datadisk.Space.Reclaim asks for. func (s *Server) evict(ctx context.Context, extra int64) error { s.evictMu.Lock() defer s.evictMu.Unlock() - total, err := s.cfg.Store.CacheSize(ctx) - if err != nil { - return err - } - over := int64(0) - if limit := s.Limit(); limit != math.MaxInt64 { - over = total + s.uploads.bytes() + extra - limit + space := s.cfg.Space() + over := space.Reclaim() + if space.Cache > 0 { + size, err := s.cfg.Store.CacheSize(ctx) + if err != nil { + return err + } + over = max(over, size.Entries+size.Uploads+s.uploads.inFlight()+extra-space.Cache) } - over = max(over, s.diskShort(evictMargin)) for over > 0 { oldest, err := s.cfg.Store.LeastRecentCacheEntries(ctx, 64) if err != nil { diff --git a/internal/cache/space_test.go b/internal/cache/space_test.go index e086dcb..6c4916f 100644 --- a/internal/cache/space_test.go +++ b/internal/cache/space_test.go @@ -3,33 +3,13 @@ package cache import ( "bytes" "fmt" - "io/fs" "net/http" "net/url" - "path/filepath" "sync/atomic" "testing" -) -// dirBytes is the size of all files under dir. -func dirBytes(t *testing.T, dir string) int64 { - t.Helper() - var n int64 - err := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - info, err := d.Info() - if err == nil { - n += info.Size() - } - return err - }) - if err != nil { - t.Fatal(err) - } - return n -} + "github.com/runplane/runplane/internal/datadisk" +) // Unfinished uploads count against the limit, so a job cannot fill the disk // with many uploads that it never finalizes. @@ -65,7 +45,8 @@ func TestUnfinishedUploadsShareTheLimit(t *testing.T) { func TestOpenUploadsPerSandbox(t *testing.T) { f := newFixture(t, 0) c := client{t: t, url: f.url} - for i := range maxOpenUploads { + first := c.reserve("layer-0", version) + for i := 1; i < maxOpenUploads; i++ { c.reserve(fmt.Sprintf("layer-%d", i), version) } status, out := c.call("CreateCacheEntry", map[string]any{"key": "one-more", "version": version}) @@ -74,7 +55,12 @@ func TestOpenUploadsPerSandbox(t *testing.T) { } // Another sandbox is not affected, and a finalized upload frees a place. f.via(t, "sb-feature").reserve("feature", version) - c.finalize("layer-0", version, 0) + if status := c.put(first, []byte("layer"), nil); status != http.StatusCreated { + t.Fatalf("Put Blob: %d", status) + } + if out := c.finalize("layer-0", version, len("layer")); out["ok"] != true { + t.Fatalf("finalize: %v", out) + } c.reserve("one-more", version) } @@ -83,10 +69,9 @@ func TestOpenUploadsPerSandbox(t *testing.T) { func TestCacheLeavesFreeDisk(t *testing.T) { f := newFixture(t, 0) var used atomic.Int64 // bytes of other programs on a 10 000-byte disk - f.srv.cfg.DiskSpace = func() (int64, int64, error) { - return 10_000, 10_000 - used.Load() - dirBytes(t, f.dir), nil + f.disk = func() datadisk.Space { + return datadisk.Limits(datadisk.Settings{MinFreePercent: 15}, 10_000, 10_000-used.Load()-dirBytes(t, f.dir)) } - f.srv.cfg.MinFreePercent = func() float64 { return 15 } c := client{t: t, url: f.url} c.save("a", version, bytes.Repeat([]byte("a"), 1000)) c.save("b", version, bytes.Repeat([]byte("b"), 1000)) @@ -110,8 +95,8 @@ func TestCacheLeavesFreeDisk(t *testing.T) { // With a small disk, the limit is a quarter of the disk, not cache_max_gb. func TestLimitIsAQuarterOfTheDiskAtMost(t *testing.T) { - f := newFixture(t, 50<<30) - f.srv.cfg.DiskSpace = func() (int64, int64, error) { return 400, 400, nil } + f := newFixture(t, 0) + f.disk = func() datadisk.Space { return datadisk.Limits(datadisk.Settings{CacheMaxGB: 50}, 400, 400) } c := client{t: t, url: f.url} u := c.reserve("big", version) if status := c.put(u, bytes.Repeat([]byte("x"), 101), nil); status != http.StatusRequestEntityTooLarge { diff --git a/internal/cache/store.go b/internal/cache/stats.go similarity index 52% rename from internal/cache/store.go rename to internal/cache/stats.go index 2817c0e..d9aacca 100644 --- a/internal/cache/store.go +++ b/internal/cache/stats.go @@ -9,14 +9,11 @@ import ( "github.com/runplane/runplane/internal/store" ) -// ErrNotFound means that no cache entry has the ID. -var ErrNotFound = store.ErrNotFound - -// Stats records cache use for each repository in the cache_stats table. The -// server counts on request paths, so Add only adds to counts in memory, and -// Run writes them each minute. -type Stats struct { - st *store.Store +// stats records the cache use of each repository in the cache_stats table. +// The server counts on request paths, so add only adds to counts in memory, +// and run writes them each minute. +type stats struct { + st Store log *slog.Logger now func() time.Time @@ -25,23 +22,23 @@ type Stats struct { } type hourRepo struct { - hour int64 // unix hour + hour time.Time repository string } -func NewStats(st *store.Store, log *slog.Logger) *Stats { - return &Stats{st: st, log: log, now: time.Now, pending: map[hourRepo]store.CacheCounts{}} +func newStats(st Store, log *slog.Logger, now func() time.Time) *stats { + return &stats{st: st, log: log, now: now, pending: map[hourRepo]store.CacheCounts{}} } -func (s *Stats) Add(repository string, c store.CacheCounts) { - k := hourRepo{s.now().Unix() / 3600, repository} +func (s *stats) add(repository string, c store.CacheCounts) { + k := hourRepo{s.now().UTC().Truncate(time.Hour), repository} s.mu.Lock() defer s.mu.Unlock() s.pending[k] = sum(s.pending[k], c) } -// Run writes the counts each minute, and a last time when ctx ends. -func (s *Stats) Run(ctx context.Context) { +// run writes the counts each minute, and a last time when ctx ends. +func (s *stats) run(ctx context.Context) { tick := time.NewTicker(time.Minute) defer tick.Stop() for { @@ -55,15 +52,14 @@ func (s *Stats) Run(ctx context.Context) { } } -// flush writes the pending counts. Counts that fail to write wait for the -// next flush. -func (s *Stats) flush(ctx context.Context) { +// Counts that flush fails to write wait for the next flush. +func (s *stats) flush(ctx context.Context) { s.mu.Lock() batch := s.pending s.pending = map[hourRepo]store.CacheCounts{} s.mu.Unlock() for k, c := range batch { - if err := s.st.AddCacheStats(ctx, time.Unix(k.hour*3600, 0), k.repository, c); err != nil { + if err := s.st.AddCacheStats(ctx, k.hour, k.repository, c); err != nil { s.log.Warn("record cache stats", "repository", k.repository, "err", err) s.mu.Lock() s.pending[k] = sum(s.pending[k], c) diff --git a/internal/cache/twirp.go b/internal/cache/twirp.go index f29e316..01cad69 100644 --- a/internal/cache/twirp.go +++ b/internal/cache/twirp.go @@ -107,9 +107,9 @@ type request struct { base string // scheme and host for signed URLs } -func (s *Server) serveTwirp(w http.ResponseWriter, r *http.Request, trusted bool) { +func (s *Server) serveTwirp(w http.ResponseWriter, r *http.Request, src source) { method := strings.TrimPrefix(r.URL.Path, twirpPrefix) - resp, err := s.twirp(r, method, trusted) + resp, err := s.twirp(r, method, src) if err != nil { var te *twirpError if !errors.As(err, &te) { @@ -122,7 +122,7 @@ func (s *Server) serveTwirp(w http.ResponseWriter, r *http.Request, trusted bool writeJSON(w, http.StatusOK, resp) } -func (s *Server) twirp(r *http.Request, method string, trusted bool) (any, error) { +func (s *Server) twirp(r *http.Request, method string, src source) (any, error) { if r.Method != http.MethodPost { return nil, twirpErr("bad_route", "use POST") } @@ -144,7 +144,7 @@ func (s *Server) twirp(r *http.Request, method string, trusted bool) (any, error return nil, twirpErr("malformed", "read request: %v", err) } - ident, err := s.identify(r, trusted) + ident, err := s.identify(r, src) if errors.Is(err, ErrNoSandbox) { denied := "cache write denied" if method == "GetCacheEntryDownloadURL" { @@ -155,7 +155,7 @@ func (s *Server) twirp(r *http.Request, method string, trusted bool) (any, error if err != nil { return nil, fmt.Errorf("identify sandbox: %w", err) } - req := request{Identity: ident, base: baseURL(r, trusted)} + req := request{Identity: ident, base: baseURL(r, src)} ctx := r.Context() switch body := body.(type) { @@ -183,7 +183,6 @@ func (s *Server) create(ctx context.Context, req request, in *createRequest) (an Ref: req.Ref, Key: in.Key, Version: in.Version, - Path: blobPath(id), CreatedAt: now, LastUsedAt: now, } @@ -198,7 +197,7 @@ func (s *Server) create(ctx context.Context, req request, in *createRequest) (an if err := s.remove(ctx, old); err != nil { return nil, err } - case !errors.Is(err, ErrNotFound): + case !errors.Is(err, store.ErrNotFound): return nil, fmt.Errorf("find entry: %w", err) } if !s.uploads.open(id, req.SandboxID) { @@ -237,7 +236,7 @@ func (s *Server) commit(ctx context.Context, req request, in *finalizeRequest) ( s.mu.Lock() defer s.mu.Unlock() e, err := s.cfg.Store.GetCacheEntryByKey(ctx, req.Repository, req.Ref, in.Key, in.Version) - if errors.Is(err, ErrNotFound) { + if errors.Is(err, store.ErrNotFound) { return finalizeResponse{}, twirpErr("not_found", "no upload for this cache entry") } if err != nil { @@ -256,8 +255,8 @@ func (s *Server) commit(ctx context.Context, req request, in *finalizeRequest) ( return finalizeResponse{}, err case in.SizeBytes > 0 && size != int64(in.SizeBytes): problem = fmt.Sprintf("the upload has %d bytes, not %d", size, in.SizeBytes) - case size > s.Limit(): - problem = fmt.Sprintf("the entry has %d bytes, more than the cache limit of %d bytes", size, s.Limit()) + case size > s.MaxBytes(): + problem = fmt.Sprintf("the entry has %d bytes, more than the cache limit of %d bytes", size, s.MaxBytes()) } if problem != "" { // Drop the upload, so that the job can try again at once. @@ -270,7 +269,7 @@ func (s *Server) commit(ctx context.Context, req request, in *finalizeRequest) ( return finalizeResponse{}, fmt.Errorf("finalize entry: %w", err) } s.uploads.drop(e.ID) - s.cfg.Stats.Add(e.Repository, store.CacheCounts{BytesWritten: size}) + s.stats.add(e.Repository, store.CacheCounts{BytesWritten: size}) s.cfg.Logger.Info("cache: saved", "repository", e.Repository, "ref", e.Ref, "key", e.Key, "bytes", size) return finalizeResponse{OK: true, EntryID: entryID(e)}, nil } @@ -289,8 +288,8 @@ func (s *Server) downloadURL(ctx context.Context, req request, in *downloadReque return nil, twirpErr("permission_denied", "cache read denied: runplane knows no ref of this job") } e, err := s.match(ctx, req, keys, in.Version) - if errors.Is(err, ErrNotFound) { - s.cfg.Stats.Add(req.Repository, store.CacheCounts{Misses: 1}) + if errors.Is(err, store.ErrNotFound) { + s.stats.add(req.Repository, store.CacheCounts{Misses: 1}) return downloadResponse{}, nil } if err != nil { @@ -300,7 +299,7 @@ func (s *Server) downloadURL(ctx context.Context, req request, in *downloadReque if err := s.cfg.Store.TouchCacheEntry(ctx, e.ID, now); err != nil { s.cfg.Logger.Warn("cache: touch entry", "id", e.ID, "err", err) } - s.cfg.Stats.Add(req.Repository, store.CacheCounts{Hits: 1}) + s.stats.add(req.Repository, store.CacheCounts{Hits: 1}) return downloadResponse{ OK: true, SignedDownloadURL: s.sign(req.base, "r", e.ID, now.Add(urlTTL)), @@ -334,14 +333,14 @@ func (s *Server) lookup(ctx context.Context, req request, keys []string, version for _, ref := range req.readRefs() { for _, key := range keys { for _, prefix := range []bool{false, true} { - e, err := s.cfg.Store.MatchCacheEntry(ctx, req.Repository, version, []string{ref}, key, prefix) - if !errors.Is(err, ErrNotFound) { + e, err := s.cfg.Store.MatchCacheEntry(ctx, req.Repository, version, ref, key, prefix) + if !errors.Is(err, store.ErrNotFound) { return e, err } } } } - return store.CacheEntry{}, ErrNotFound + return store.CacheEntry{}, store.ErrNotFound } // dedupe drops repeated strings and keeps the first of each. diff --git a/internal/config/config.go b/internal/config/config.go index 1e88162..884583e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,5 +1,5 @@ -// Package config reads the process configuration from the environment and -// flags, and gives the paths of all runplane files. +// Package config reads the process configuration from the flags, the +// environment, and runplane.env, and gives the paths of all runplane files. package config import ( @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + "net" "os" "path/filepath" "strconv" @@ -15,8 +16,6 @@ import ( "github.com/runplane/runplane/internal/sandbox" ) -const defaultConfigDir = "/etc/runplane" - type Config struct { Listen string // dashboard and API AgentListen string // TLS listener for agents (primary) @@ -24,40 +23,116 @@ type Config struct { DataDir string ConfigDir string Mode sandbox.Mode // empty until the install sets it - Dev bool // relaxed cookies and CORS for the Vite dev server + Dev bool // debug logs, and relaxed cookies and CORS for the Vite dev server DevInsecure bool // lets the devdocker driver operate SSHListen string // SSH gateway for debug shells (primary) } -// Parse reads the RUNPLANE_* environment variables over the defaults, then -// the flags in args. Define the other flags of the subcommand on fs before -// the call. The flag names are the variable names without RUNPLANE_, in -// lowercase, with dashes: -listen, -data-dir. -func Parse(fs *flag.FlagSet, args []string) (Config, error) { - c, err := load() - if err != nil { - return Config{}, err +// Default is the configuration without flags, environment, or +// runplane.env. install.sh writes the same ports and folders to +// runplane.env. +func Default() Config { + return Config{ + Listen: ":8484", + AgentListen: ":8485", + ServicePort: 8486, + DataDir: "/var/lib/runplane", + ConfigDir: "/etc/runplane", + SSHListen: ":2222", + } +} + +// Flag is a flag of the configuration that a command uses. Each command +// also has -config-dir, which selects runplane.env. +type Flag int + +const ( + Listen Flag = iota + AgentListen + ServicePort + DataDir + Mode + Dev + DevInsecure + SSHListen + configDir +) + +// option is one value of Config. Its variable is RUNPLANE_ and the flag +// name in upper case, with _ for -. +type option struct { + name string + usage string + field func(*Config) any // *string, *int, or *bool +} + +var options = [...]option{ + Listen: {"listen", "dashboard and API address", func(c *Config) any { return &c.Listen }}, + AgentListen: {"agent-listen", "TLS address for agents", func(c *Config) any { return &c.AgentListen }}, + ServicePort: {"service-port", "cache and registry mirror port for sandboxes", func(c *Config) any { return &c.ServicePort }}, + DataDir: {"data-dir", "data folder", func(c *Config) any { return &c.DataDir }}, + Mode: {"mode", "sandbox mode: container, vm, or dev", func(c *Config) any { return (*string)(&c.Mode) }}, + Dev: {"dev", "debug logs, and relaxed cookies and CORS for the Vite dev server", func(c *Config) any { return &c.Dev }}, + DevInsecure: {"dev-insecure", "let the devdocker driver operate", func(c *Config) any { return &c.DevInsecure }}, + SSHListen: {"ssh-listen", "SSH gateway address for debug shells", func(c *Config) any { return &c.SSHListen }}, + configDir: {"config-dir", "configuration folder, with runplane.env", func(c *Config) any { return &c.ConfigDir }}, +} + +// Parse reads the configuration of a command. A flag wins over the +// RUNPLANE_* variable of the environment, the environment wins over +// runplane.env in the configuration folder, and runplane.env wins over +// Default. systemd gives the services the variables of runplane.env, but +// sudo starts a command without them, so the file gives an operator command +// the configuration of the service. +// +// fs gets -config-dir and the flags in use. Define the other flags of the +// command on fs before the call. +func Parse(fs *flag.FlagSet, args []string, use ...Flag) (Config, error) { + def := Default() + for _, f := range append(use, configDir) { + o := options[f] + usage := fmt.Sprintf("%s (%s)", o.usage, o.env()) + switch p := o.field(&def).(type) { + case *string: + fs.String(o.name, *p, usage) + case *int: + fs.Int(o.name, *p, usage) + case *bool: + fs.Bool(o.name, *p, usage) + } } - fs.StringVar(&c.Listen, "listen", c.Listen, "dashboard and API address (RUNPLANE_LISTEN)") - fs.StringVar(&c.AgentListen, "agent-listen", c.AgentListen, "TLS address for agents (RUNPLANE_AGENT_LISTEN)") - fs.IntVar(&c.ServicePort, "service-port", c.ServicePort, "cache and registry mirror port for sandboxes (RUNPLANE_SERVICE_PORT)") - fs.StringVar(&c.DataDir, "data-dir", c.DataDir, "data folder (RUNPLANE_DATA_DIR)") - fs.StringVar(&c.ConfigDir, "config-dir", c.ConfigDir, "configuration folder (RUNPLANE_CONFIG_DIR)") - fs.StringVar((*string)(&c.Mode), "mode", string(c.Mode), "sandbox mode: container, vm, or dev (RUNPLANE_MODE)") - fs.BoolVar(&c.Dev, "dev", c.Dev, "relaxed cookies and CORS for the Vite dev server (RUNPLANE_DEV)") - fs.BoolVar(&c.DevInsecure, "dev-insecure", c.DevInsecure, "let the devdocker driver operate (RUNPLANE_DEV_INSECURE)") - fs.StringVar(&c.SSHListen, "ssh-listen", c.SSHListen, "SSH gateway address for debug shells (RUNPLANE_SSH_LISTEN)") if err := fs.Parse(args); err != nil { return Config{}, err } + flags := map[string]string{} + fs.Visit(func(f *flag.Flag) { flags[f.Name] = f.Value.String() }) - switch c.Mode { - case "", sandbox.ModeContainer, sandbox.ModeVM, sandbox.ModeDev: - default: - return Config{}, fmt.Errorf("mode %q: use container, vm, or dev", c.Mode) + dir := cmp.Or(flags["config-dir"], os.Getenv("RUNPLANE_CONFIG_DIR"), def.ConfigDir) + envFile := Config{ConfigDir: dir}.EnvFilePath() + file := readEnvFile(envFile) + c := def + var errs []error + for _, o := range options { + v, set := flags[o.name] + from := "-" + o.name + switch { + case set: + case os.Getenv(o.env()) != "": + v, from = os.Getenv(o.env()), o.env() + case file[o.env()] != "": + v, from = file[o.env()], o.env()+" in "+envFile + default: + continue + } + if err := o.set(&c, v); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", from, err)) + } } - if c.ServicePort < 1 || c.ServicePort > 65535 { - return Config{}, fmt.Errorf("service port %d: use 1 to 65535", c.ServicePort) + if err := errors.Join(errs...); err != nil { + return Config{}, err + } + if err := c.check(); err != nil { + return Config{}, err } // Drivers give the folders to Docker and Firecracker as mount sources, // so they must be absolute. @@ -71,75 +146,63 @@ func Parse(fs *flag.FlagSet, args []string) (Config, error) { return c, nil } -func load() (Config, error) { - c := Config{ - Listen: ":8484", - AgentListen: ":8485", - ServicePort: 8486, - DataDir: "/var/lib/runplane", - ConfigDir: defaultConfigDir, - SSHListen: ":2222", - } - var errs []error - str := func(name string, dst *string) { - if v := os.Getenv(name); v != "" { - *dst = v - } - } - boolean := func(name string, dst *bool) { - v := os.Getenv(name) - if v == "" { - return +func (o option) env() string { + return "RUNPLANE_" + strings.ToUpper(strings.ReplaceAll(o.name, "-", "_")) +} + +func (o option) set(c *Config, v string) error { + switch p := o.field(c).(type) { + case *string: + *p = v + case *int: + n, err := strconv.Atoi(v) + if err != nil { + return fmt.Errorf("%q is not a number", v) } + *p = n + case *bool: b, err := strconv.ParseBool(v) if err != nil { - errs = append(errs, fmt.Errorf("%s: %q is not 0 or 1", name, v)) - return + return fmt.Errorf("%q is not 0 or 1", v) } - *dst = b + *p = b } - str("RUNPLANE_LISTEN", &c.Listen) - str("RUNPLANE_AGENT_LISTEN", &c.AgentListen) - str("RUNPLANE_DATA_DIR", &c.DataDir) - str("RUNPLANE_CONFIG_DIR", &c.ConfigDir) - str("RUNPLANE_MODE", (*string)(&c.Mode)) - str("RUNPLANE_SSH_LISTEN", &c.SSHListen) - boolean("RUNPLANE_DEV", &c.Dev) - boolean("RUNPLANE_DEV_INSECURE", &c.DevInsecure) - if v := os.Getenv("RUNPLANE_SERVICE_PORT"); v != "" { - port, err := strconv.Atoi(v) - if err != nil { - errs = append(errs, fmt.Errorf("RUNPLANE_SERVICE_PORT: %q is not a number", v)) + return nil +} + +func (c Config) check() error { + if c.Mode != "" { + if err := c.Mode.Check(); err != nil { + return err } - c.ServicePort = port } - if err := errors.Join(errs...); err != nil { - return Config{}, fmt.Errorf("read environment: %w", err) + if c.ServicePort < 1 || c.ServicePort > 65535 { + return fmt.Errorf("service port %d: use 1 to 65535", c.ServicePort) } - return c, nil + for _, addr := range []string{c.Listen, c.AgentListen, c.SSHListen} { + if _, _, err := net.SplitHostPort(addr); err != nil { + return fmt.Errorf("listen address %q: use host:port or :port", addr) + } + } + return nil } -// LoadEnvFile sets the RUNPLANE_* variables of runplane.env, the environment -// file of the systemd units, that the environment does not have. sudo starts -// a command without them, so the commands that an operator runs by hand call -// LoadEnvFile before Parse to see the configuration of the service. It reads -// the file in RUNPLANE_CONFIG_DIR, or in /etc/runplane. A file that is -// missing or that the user cannot read changes nothing. -func LoadEnvFile() { - dir := cmp.Or(os.Getenv("RUNPLANE_CONFIG_DIR"), defaultConfigDir) - b, _ := os.ReadFile(filepath.Join(dir, "runplane.env")) +// readEnvFile reads the RUNPLANE_* variables of runplane.env, the +// environment file of the systemd units. A file that is missing or that the +// user cannot read has none. +func readEnvFile(path string) map[string]string { + b, _ := os.ReadFile(path) + vars := map[string]string{} for line := range strings.Lines(string(b)) { key, value, ok := strings.Cut(strings.TrimSpace(line), "=") if !ok || !strings.HasPrefix(key, "RUNPLANE_") { continue } - if _, set := os.LookupEnv(key); set { - continue - } // systemd also takes a value in quotes. if len(value) >= 2 && (value[0] == '"' || value[0] == '\'') && value[len(value)-1] == value[0] { value = value[1 : len(value)-1] } - os.Setenv(key, value) + vars[key] = value } + return vars } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2794f77..d668deb 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,13 +3,33 @@ package config import ( "flag" "io" + "io/fs" "os" "path/filepath" + "strings" "syscall" "testing" ) +var allFlags = []Flag{Listen, AgentListen, ServicePort, DataDir, Mode, Dev, DevInsecure, SSHListen} + +// cleanEnv clears the RUNPLANE_* variables of the shell, and points Parse +// to a configuration folder without runplane.env, so that the environment +// and the runplane.env of this machine have no effect. +func cleanEnv(t *testing.T) string { + t.Helper() + for _, kv := range os.Environ() { + if key, _, _ := strings.Cut(kv, "="); strings.HasPrefix(key, "RUNPLANE_") { + t.Setenv(key, "") + } + } + dir := t.TempDir() + t.Setenv("RUNPLANE_CONFIG_DIR", dir) + return dir +} + func TestParseEnvAndFlags(t *testing.T) { + dir := cleanEnv(t) t.Setenv("RUNPLANE_LISTEN", ":9000") t.Setenv("RUNPLANE_SERVICE_PORT", "9100") t.Setenv("RUNPLANE_DATA_DIR", "rel/data") @@ -17,24 +37,22 @@ func TestParseEnvAndFlags(t *testing.T) { t.Setenv("RUNPLANE_DEV", "1") fs := flag.NewFlagSet("serve", flag.ContinueOnError) - c, err := Parse(fs, []string{"-listen", ":9001", "-mode", "container"}) + c, err := Parse(fs, []string{"-listen", ":9001", "-mode", "container"}, allFlags...) if err != nil { t.Fatal(err) } if c.Listen != ":9001" || c.Mode != "container" { t.Errorf("flags did not override the environment: %+v", c) } - if c.ServicePort != 9100 || !c.Dev || c.AgentListen != ":8485" || c.DevInsecure { + def := Default() + if c.ServicePort != 9100 || !c.Dev || c.AgentListen != def.AgentListen || c.SSHListen != def.SSHListen || c.DevInsecure { t.Errorf("environment or defaults lost: %+v", c) } wd, _ := os.Getwd() // Docker bind-mounts the data folder, so it must be absolute. - if c.DataDir != filepath.Join(wd, "rel/data") || c.ConfigDir != "/etc/runplane" { + if c.DataDir != filepath.Join(wd, "rel/data") || c.ConfigDir != dir { t.Errorf("folders: %q, %q", c.DataDir, c.ConfigDir) } - if c.SSHListen != ":2222" { - t.Errorf("SSH address: %q", c.SSHListen) - } } func TestParseValues(t *testing.T) { @@ -49,8 +67,11 @@ func TestParseValues(t *testing.T) { {"RUNPLANE_DEV", "yes please", false}, {"RUNPLANE_SERVICE_PORT", "http", false}, {"RUNPLANE_SERVICE_PORT", "70000", false}, + {"RUNPLANE_SSH_LISTEN", "127.0.0.1:2200", true}, + {"RUNPLANE_SSH_LISTEN", "2200", false}, } { t.Run(tt.name+"="+tt.value, func(t *testing.T) { + cleanEnv(t) t.Setenv(tt.name, tt.value) fs := flag.NewFlagSet("serve", flag.ContinueOnError) fs.SetOutput(io.Discard) @@ -61,12 +82,30 @@ func TestParseValues(t *testing.T) { } } +// A flag that the command does not use is an error, not a value that has +// no effect. +func TestParseHasOnlyTheFlagsInUse(t *testing.T) { + cleanEnv(t) + fs := flag.NewFlagSet("reset-password", flag.ContinueOnError) + fs.SetOutput(io.Discard) + if _, err := Parse(fs, []string{"-ssh-listen", ":2200"}, DataDir); err == nil { + t.Error("reset-password takes -ssh-listen") + } + fs = flag.NewFlagSet("serve", flag.ContinueOnError) + if c, err := Parse(fs, []string{"-ssh-listen", ":2200"}, DataDir, SSHListen); err != nil || c.SSHListen != ":2200" { + t.Errorf("serve with -ssh-listen: %q, %v", c.SSHListen, err) + } +} + +// The folders hold keys, password hashes, backups, and job logs, so only +// root can open them. The runners folder is the exception: the sandboxes +// read the runner bundles as other users. EnsureDirs also corrects the mode +// of a folder that exists, because MkdirAll keeps it. func TestEnsureDirs(t *testing.T) { old := syscall.Umask(0o077) defer syscall.Umask(old) root := t.TempDir() c := Config{DataDir: filepath.Join(root, "data"), ConfigDir: filepath.Join(root, "etc")} - // A folder that exists with a loose mode gets the right mode. if err := os.MkdirAll(c.CacheDir(), 0o777); err != nil { t.Fatal(err) } @@ -76,54 +115,78 @@ func TestEnsureDirs(t *testing.T) { if err := c.EnsureDirs(); err != nil { t.Fatal(err) } - for path, want := range map[string]os.FileMode{ - c.ConfigDir: 0o700, c.DataDir: 0o700, c.SandboxesDir(): 0o700, c.ImagesDir(): 0o700, - c.RunnersDir(): 0o755, c.CacheDir(): 0o700, c.MirrorDir(): 0o700, c.LogsDir(): 0o700, - c.BackupsDir(): 0o700, c.TLSDir(): 0o700, c.ScratchTemplatesDir(): 0o700, - } { - info, err := os.Stat(path) + modes := map[string]fs.FileMode{} + for _, top := range []string{c.ConfigDir, c.DataDir} { + err := filepath.WalkDir(top, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + info, err := d.Info() + modes[path] = info.Mode().Perm() + return err + }) if err != nil { t.Fatal(err) } - if info.Mode().Perm() != want { - t.Errorf("%s: mode %v, want %v", path, info.Mode().Perm(), want) + } + for path, mode := range modes { + want := fs.FileMode(0o700) + if path == c.RunnersDir() { + want = 0o755 + } + if mode != want { + t.Errorf("%s: mode %v, want %v", path, mode, want) + } + } + for _, path := range []string{c.SandboxesDir(), c.ImagesDir(), c.RunnersDir(), c.CacheDir(), c.MirrorDir(), + c.LogsDir(), c.BackupsDir(), c.TLSDir(), c.ScratchTemplatesDir()} { + if _, ok := modes[path]; !ok { + t.Errorf("EnsureDirs made no folder %s", path) } } } // sudo runplane doctor gets no RUNPLANE_* variables, so it must read the ones -// of the service from runplane.env. The real environment still wins. -func TestLoadEnvFile(t *testing.T) { +// of the service from runplane.env in the configuration folder of +// -config-dir or RUNPLANE_CONFIG_DIR. The real environment and the flags +// still win. +func TestParseReadsTheEnvFile(t *testing.T) { dir := t.TempDir() - t.Setenv("RUNPLANE_CONFIG_DIR", dir) - t.Setenv("RUNPLANE_LISTEN", ":7000") - // LoadEnvFile sets these; t.Setenv restores them after the test. - for _, key := range []string{"RUNPLANE_MODE", "RUNPLANE_DATA_DIR", "RUNPLANE_SSH_LISTEN", "NOT_RUNPLANE"} { - t.Setenv(key, "") - os.Unsetenv(key) - } env := "# Environment of runplane-agent.service\n" + "RUNPLANE_MODE=vm\n" + "RUNPLANE_LISTEN=:9000\n" + "RUNPLANE_DATA_DIR=\"/srv/runplane\"\n" + + "RUNPLANE_AGENT_LISTEN=:9100\n" + "NOT_RUNPLANE=1\n" + "RUNPLANE_SSH_LISTEN=:2200" // no newline at the end if err := os.WriteFile(filepath.Join(dir, "runplane.env"), []byte(env), 0o600); err != nil { t.Fatal(err) } - - LoadEnvFile() - c, err := Parse(flag.NewFlagSet("doctor", flag.ContinueOnError), nil) - if err != nil { - t.Fatal(err) - } - if c.Mode != "vm" || c.DataDir != "/srv/runplane" || c.SSHListen != ":2200" { - t.Errorf("values of the file lost: mode %q, data %q, ssh %q", c.Mode, c.DataDir, c.SSHListen) - } - if c.Listen != ":7000" { - t.Errorf("listen %q, want :7000 from the environment", c.Listen) - } - if _, set := os.LookupEnv("NOT_RUNPLANE"); set { - t.Error("the file set a variable that is not RUNPLANE_*") + for _, tt := range []struct { + name string + env string // RUNPLANE_CONFIG_DIR + args []string + }{ + {"-config-dir", t.TempDir(), []string{"-config-dir", dir, "-agent-listen", ":9200"}}, + {"RUNPLANE_CONFIG_DIR", dir, []string{"-agent-listen", ":9200"}}, + } { + t.Run(tt.name, func(t *testing.T) { + cleanEnv(t) + t.Setenv("RUNPLANE_CONFIG_DIR", tt.env) + t.Setenv("RUNPLANE_LISTEN", ":7000") + c, err := Parse(flag.NewFlagSet("doctor", flag.ContinueOnError), tt.args, allFlags...) + if err != nil { + t.Fatal(err) + } + if c.Mode != "vm" || c.DataDir != "/srv/runplane" || c.SSHListen != ":2200" || c.ConfigDir != dir { + t.Errorf("values of the file lost: mode %q, data %q, ssh %q, config %q", c.Mode, c.DataDir, c.SSHListen, c.ConfigDir) + } + if c.Listen != ":7000" { + t.Errorf("listen %q, want :7000 from the environment", c.Listen) + } + if c.AgentListen != ":9200" { + t.Errorf("agent listen %q, want :9200 from the flag", c.AgentListen) + } + }) } } diff --git a/internal/config/paths.go b/internal/config/paths.go index 98b8d0d..c4c892c 100644 --- a/internal/config/paths.go +++ b/internal/config/paths.go @@ -10,33 +10,48 @@ import ( func (c Config) SecretKeyPath() string { return filepath.Join(c.ConfigDir, "secret.key") } func (c Config) AgentConfigPath() string { return filepath.Join(c.ConfigDir, "agent.json") } +// EnvFilePath is the environment file of the systemd units. install.sh +// writes it. +func (c Config) EnvFilePath() string { return filepath.Join(c.ConfigDir, "runplane.env") } + // AdminKeysPath is the authorized_keys file of the admins: these keys open // the debug shell of every sandbox through the SSH gateway. func (c Config) AdminKeysPath() string { return filepath.Join(c.ConfigDir, "authorized_keys") } -func (c Config) DBPath() string { return filepath.Join(c.DataDir, "runplane.db") } -func (c Config) SandboxesDir() string { return filepath.Join(c.DataDir, "sandboxes") } -func (c Config) SandboxDir(id string) string { return filepath.Join(c.SandboxesDir(), id) } -func (c Config) ImagesDir() string { return filepath.Join(c.DataDir, "images") } -func (c Config) RunnersDir() string { return filepath.Join(c.DataDir, "runners") } -func (c Config) CacheDir() string { return filepath.Join(c.DataDir, "cache") } -func (c Config) MirrorDir() string { return filepath.Join(c.DataDir, "mirror") } -func (c Config) LogsDir() string { return filepath.Join(c.DataDir, "logs") } -func (c Config) BackupsDir() string { return filepath.Join(c.DataDir, "backups") } -func (c Config) TLSDir() string { return filepath.Join(c.DataDir, "tls") } +// SSHKeyPath is the key with which the primary installs agents over SSH. +func (c Config) SSHKeyPath() string { return filepath.Join(c.ConfigDir, "ssh_key") } -// JobLogPath is the stored GitHub log of a completed job. -func (c Config) JobLogPath(jobID string) string { return filepath.Join(c.LogsDir(), jobID+".log") } +func (c Config) DBPath() string { return filepath.Join(c.DataDir, "runplane.db") } +func (c Config) SandboxesDir() string { return filepath.Join(c.DataDir, "sandboxes") } +func (c Config) ImagesDir() string { return filepath.Join(c.DataDir, "images") } +func (c Config) RunnersDir() string { return filepath.Join(c.DataDir, "runners") } +func (c Config) CacheDir() string { return filepath.Join(c.DataDir, "cache") } +func (c Config) MirrorDir() string { return filepath.Join(c.DataDir, "mirror") } +func (c Config) LogsDir() string { return filepath.Join(c.DataDir, "logs") } +func (c Config) BackupsDir() string { return filepath.Join(c.DataDir, "backups") } +func (c Config) TLSDir() string { return filepath.Join(c.DataDir, "tls") } // ScratchTemplatesDir holds the empty ext4 templates of the scratch disks. func (c Config) ScratchTemplatesDir() string { return filepath.Join(c.DataDir, "scratch-templates") } +// JailDir holds the chroots that the Firecracker jailer makes for the VMs. +func (c Config) JailDir() string { return filepath.Join(c.DataDir, "jail") } + // KernelPath is the guest kernel of VM mode. install.sh puts it there. func (c Config) KernelPath() string { return filepath.Join(c.DataDir, "kernel", "vmlinux") } // SSHHostKeyPath is the host key of the SSH gateway. func (c Config) SSHHostKeyPath() string { return filepath.Join(c.DataDir, "ssh_host_ed25519_key") } +// KnownHostsPath holds the host keys of the servers that the primary +// installed over SSH. +func (c Config) KnownHostsPath() string { return filepath.Join(c.DataDir, "known_hosts") } + +// AgentSettingsPath holds the last settings that an agent got from the +// primary. At its start, the agent gives them to its sandbox driver, which +// has sandboxes from before the restart. +func (c Config) AgentSettingsPath() string { return filepath.Join(c.DataDir, "agent-settings.json") } + // EnsureDirs makes the configuration and data folders and sets their modes. // It also corrects the mode of folders that exist. func (c Config) EnsureDirs() error { diff --git a/internal/datadisk/datadisk.go b/internal/datadisk/datadisk.go new file mode 100644 index 0000000..7228e30 --- /dev/null +++ b/internal/datadisk/datadisk.go @@ -0,0 +1,67 @@ +// Package datadisk shares the data disk between the Actions cache and the +// registry mirror. The images, the runner bundles, and the sandboxes use the +// same disk, so the two caches also leave min_free_disk_percent free. +package datadisk + +import "github.com/runplane/runplane/internal/host" + +// evictMargin is how far above min_free_disk_percent a cache frees the disk, +// so that the removal does not run again at each write. +const evictMargin = 5 + +// Settings are the dashboard settings that limit the caches. +type Settings struct { + CacheMaxGB int // cache_max_gb; 0 for no limit of its own + MinFreePercent float64 // min_free_disk_percent +} + +// Space is the room of the two caches on the data disk at one time. +type Space struct { + Cache int64 // the byte limit of the Actions cache; 0 for no limit + Mirror int64 // the byte limit of the registry mirror; 0 for no limit + + total, free int64 + minFree float64 +} + +// Of returns the room of the caches on the disk of dir now. A disk whose +// size runplane cannot read gives the caches their room for an unknown disk. +func Of(dir string, s Settings) Space { + total, free, err := host.DiskSpace(dir) + if err != nil { + return Limits(s, 0, 0) + } + return Limits(s, total, free) +} + +// Limits returns the room of the caches on a data disk of total bytes with +// free bytes free. The cache gets cache_max_gb, at most a quarter of the +// disk. The mirror gets a tenth of the disk, from 2 to 50 GiB. A total of 0 +// means that the disk size is not known. +func Limits(s Settings, total, free int64) Space { + sp := Space{Cache: int64(s.CacheMaxGB) << 30, Mirror: min(max(total/10, 2<<30), 50<<30)} + if total <= 0 { + return sp + } + if sp.Cache <= 0 || sp.Cache > total/4 { + sp.Cache = total / 4 + } + sp.total, sp.free, sp.minFree = total, free, s.MinFreePercent + return sp +} + +// Low reports if the disk has less than min_free_disk_percent free. The +// caches then take no new data. +func (s Space) Low() bool { return s.short(0) > 0 } + +// Reclaim returns the bytes that the caches remove, least recently used +// first: what the disk lacks to have 5 points more than +// min_free_disk_percent free. +func (s Space) Reclaim() int64 { return s.short(evictMargin) } + +func (s Space) short(margin float64) int64 { + if s.total <= 0 { + return 0 + } + return max(0, int64(float64(s.total)*(s.minFree+margin)/100)-s.free) +} diff --git a/internal/datadisk/datadisk_test.go b/internal/datadisk/datadisk_test.go new file mode 100644 index 0000000..1adba33 --- /dev/null +++ b/internal/datadisk/datadisk_test.go @@ -0,0 +1,32 @@ +package datadisk + +import "testing" + +func TestLimits(t *testing.T) { + const gib = 1 << 30 + tests := []struct { + name string + s Settings + total, free int64 + cache, mirror int64 + low bool + reclaim int64 + }{ + {"large disk", Settings{CacheMaxGB: 50, MinFreePercent: 15}, 1000 * gib, 900 * gib, 50 * gib, 50 * gib, false, 0}, + {"a quarter of a small disk", Settings{CacheMaxGB: 50, MinFreePercent: 15}, 40 * gib, 30 * gib, 10 * gib, 4 * gib, false, 0}, + {"the mirror gets 2 GiB at least", Settings{CacheMaxGB: 1}, 10 * gib, 10 * gib, 1 * gib, 2 * gib, false, 0}, + {"no cache_max_gb", Settings{}, 100 * gib, 100 * gib, 25 * gib, 10 * gib, false, 0}, + {"disk size not known", Settings{CacheMaxGB: 50, MinFreePercent: 15}, 0, 0, 50 * gib, 2 * gib, false, 0}, + {"free disk near the minimum", Settings{CacheMaxGB: 50, MinFreePercent: 15}, 100 * gib, 17 * gib, 25 * gib, 10 * gib, false, 3 * gib}, + {"free disk below the minimum", Settings{CacheMaxGB: 50, MinFreePercent: 15}, 100 * gib, 14 * gib, 25 * gib, 10 * gib, true, 6 * gib}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sp := Limits(tt.s, tt.total, tt.free) + if sp.Cache != tt.cache || sp.Mirror != tt.mirror || sp.Low() != tt.low || sp.Reclaim() != tt.reclaim { + t.Errorf("cache %d, mirror %d, low %v, reclaim %d; want %d, %d, %v, %d", + sp.Cache, sp.Mirror, sp.Low(), sp.Reclaim(), tt.cache, tt.mirror, tt.low, tt.reclaim) + } + }) + } +} diff --git a/internal/doctor/clock.go b/internal/doctor/clock.go new file mode 100644 index 0000000..1b8b6bc --- /dev/null +++ b/internal/doctor/clock.go @@ -0,0 +1,53 @@ +package doctor + +import ( + "context" + "encoding/binary" + "fmt" + "net" + "time" +) + +// ntpOffset asks an NTP server for the time, as an SNTP client (RFC 4330). +// A positive offset means that this clock is ahead. +func ntpOffset(ctx context.Context, server string) (time.Duration, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + var d net.Dialer + conn, err := d.DialContext(ctx, "udp", server) + if err != nil { + return 0, fmt.Errorf("ask %s for the time: %w", server, err) + } + defer conn.Close() + deadline, _ := ctx.Deadline() + conn.SetDeadline(deadline) + req := make([]byte, 48) + req[0] = 4<<3 | 3 // version 4, client mode + sent := time.Now() + if _, err := conn.Write(req); err != nil { + return 0, fmt.Errorf("ask %s for the time: %w", server, err) + } + resp := make([]byte, 48) + n, err := conn.Read(resp) + received := time.Now() + if err != nil { + return 0, fmt.Errorf("read the time of %s: %w", server, err) + } + if n < 48 || resp[0]&7 != 4 || resp[1] == 0 { // server mode; stratum 0 is a refusal + return 0, fmt.Errorf("read the time of %s: bad answer", server) + } + serverReceived, serverSent := ntpTime(resp[32:40]), ntpTime(resp[40:48]) + return -(serverReceived.Sub(sent) + serverSent.Sub(received)) / 2, nil +} + +// ntpTime reads a 64-bit NTP timestamp. Seconds without the top bit belong +// to the era that starts in 2036. +func ntpTime(b []byte) time.Time { + sec := int64(binary.BigEndian.Uint32(b[:4])) + if sec&0x80000000 == 0 { + sec += 1 << 32 + } + frac := int64(binary.BigEndian.Uint32(b[4:])) + const unixToNTP = 2208988800 + return time.Unix(sec-unixToNTP, frac*1e9>>32) +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go new file mode 100644 index 0000000..dc02d24 --- /dev/null +++ b/internal/doctor/doctor.go @@ -0,0 +1,347 @@ +// Package doctor checks if a server can run runplane. It runs the general +// checks and the checks of the sandbox driver of the mode, which the driver +// package gives as Requirements. runplane doctor, the setup page, and the +// server page show the result. +package doctor + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "slices" + "strings" + "time" + + "github.com/runplane/runplane/internal/host" + "github.com/runplane/runplane/internal/sandbox" +) + +type Status string + +const ( + StatusOK Status = "ok" + StatusWarn Status = "warn" + StatusFail Status = "fail" +) + +type Check struct { + ID string + Name string + Status Status + Detail string + Fix string // one sentence that says how to fix a check that is not ok + Command string // one shell command that does the fix, if one can +} + +func (c Check) OK(detail string) Check { + c.Status, c.Detail = StatusOK, detail + return c +} + +func (c Check) Warn(detail, fix string) Check { + c.Status, c.Detail, c.Fix = StatusWarn, detail, fix + return c +} + +func (c Check) Fail(detail, fix string) Check { + c.Status, c.Detail, c.Fix = StatusFail, detail, fix + return c +} + +func (c Check) WithCommand(command string) Check { + c.Command = command + return c +} + +// AsRoot is the warning of a check that needs root to read the server. +func (c Check) AsRoot(detail string) Check { + return c.Warn(detail, "Run the doctor as root.").WithCommand("sudo runplane doctor") +} + +// Report is the result of Run: the facts that the doctor read, and the +// checks in the order that the dashboard and the command show them. +type Report struct { + Info host.Info + Checks []Check +} + +func (r Report) Failed() bool { + return slices.ContainsFunc(r.Checks, func(c Check) bool { return c.Status == StatusFail }) +} + +type Options struct { + DataDir string + EnvFile string // the environment file, which sets the listen addresses + Listen []string // TCP addresses of runplane, for example ":8484" + // Serving tells that this process listens on Listen, as the doctor of a + // running server does. Then an address in use needs no owner lookup, + // which works only on Linux. + Serving bool + MinFreeDiskPercent float64 // the min_free_disk_percent setting + Unit string // systemd unit of runplane, for example "runplane"; empty skips its state + Requirements Requirements + // Docker and BaseImage are required. Docker reads the Docker Engine of + // the server. + Docker func(ctx context.Context) (Docker, error) + // BaseImage tries to pull the default builtin image, and returns its + // registry ref and the error of the pull. + BaseImage func(ctx context.Context) (ref string, err error) + // Internet starts a sandbox that connects to the internet. Without it, + // only the running service can do the internet check, because the + // check needs the firewall rules that the service owns. + Internet func(ctx context.Context) error +} + +type Docker struct { + Version string + RootDir string // the data folder of Docker + Firewall string // the firewall backend, for example "iptables" + Runtimes map[string]string // the path of each OCI runtime +} + +// Host is what the doctor read of the server. Requirements.Software adds the +// versions of the sandbox software to Info, and Requirements.Checks read the +// rest. +type Host struct { + host.Info + host.Details + Docker Docker + DockerErr error // why Docker does not answer + Root bool + Systemd bool // the host runs systemd + Unit string // Options.Unit + UnitState string // systemctl is-active of Unit + + dockerSnap bool // Docker comes from the snap package +} + +// Facts reads the facts of this server that the primary keeps. It returns +// an error only when it cannot read the memory or the data disk, and then +// still sets the other facts. +func Facts(ctx context.Context, o Options) (host.Info, error) { + h := o.read(ctx) + return h.Info, h.ResourcesErr +} + +// Run reads this server and checks it. Some checks need root; without root +// they warn. When the mode check fails, Run returns only that check. +func Run(ctx context.Context, o Options) Report { + req := o.Requirements + if req.Mode.Status == StatusFail { + return Report{Checks: []Check{req.Mode}} + } + s := o.scan(ctx) + checks := s.checks(ctx, o) + ref, pullErr := o.BaseImage(ctx) + checks = append(checks, baseImageCheck(s.Host, req.Docker, ref, pullErr), internetCheck(ctx, o.Internet)) + return Report{Info: s.Info, Checks: checks} +} + +type scan struct { + *Host + clock clock + ports []portScan +} + +func (o Options) scan(ctx context.Context) scan { + h := o.read(ctx) + h.Root = os.Geteuid() == 0 + h.Systemd = exists("/run/systemd/system") + if h.Unit = o.Unit; h.Unit != "" { + h.UnitState = UnitState(ctx, h.Unit) + } + return scan{Host: h, clock: readClock(ctx), ports: scanPorts(slices.Concat(o.Listen, o.Requirements.Listen), o.Serving)} +} + +// checks returns the checks before the base image check. +func (s scan) checks(ctx context.Context, o Options) []Check { + req := o.Requirements + cs := []Check{req.Mode, dockerCheck(s.Host, req.Docker)} + if req.Checks != nil { + cs = append(cs, req.Checks(ctx, s.Host)...) + } + cs = append(cs, diskCheck(s.Host, o.DataDir, o.MinFreeDiskPercent), s.clock.check()) + return append(cs, portChecks(s.ports, s.Root, o.EnvFile)...) +} + +func (o Options) read(ctx context.Context) *Host { + info, details := host.Read(ctx, o.DataDir) + h := &Host{Info: info, Details: details} + h.Docker, h.DockerErr = o.Docker(ctx) + h.DockerVersion = h.Docker.Version + // A snap install conflicts with Docker Engine even when it does not answer. + h.dockerSnap = strings.HasPrefix(h.Docker.RootDir, "/var/snap/") || exists("/snap/docker") + if o.Requirements.Software != nil { + o.Requirements.Software(ctx, h) + } + return h +} + +// Format writes the checks for a terminal: one line for each check with a +// status mark, the fix and its command below a check that is not ok, and a +// summary. +func Format(w io.Writer, checks []Check) error { + marks := map[Status]string{StatusOK: "✓", StatusWarn: "!", StatusFail: "✗"} + width := 0 + for _, c := range checks { + width = max(width, len(c.Name)) + } + var b strings.Builder + count := map[Status]int{} + for _, c := range checks { + count[c.Status]++ + fmt.Fprintf(&b, "%s %-*s %s\n", marks[c.Status], width, c.Name, c.Detail) + if c.Status != StatusOK && c.Fix != "" { + fmt.Fprintf(&b, " %-*s Fix: %s\n", width, "", c.Fix) + } + if c.Status != StatusOK && c.Command != "" { + fmt.Fprintf(&b, " %-*s Run: %s\n", width, "", c.Command) + } + } + if count[StatusFail]+count[StatusWarn] == 0 { + b.WriteString("\nAll checks passed.\n") + } else { + fmt.Fprintf(&b, "\n%s, %s.\n", plural(count[StatusFail], "failure"), plural(count[StatusWarn], "warning")) + } + _, err := io.WriteString(w, b.String()) + return err +} + +func plural(n int, word string) string { + if n == 1 { + return "1 " + word + } + return fmt.Sprintf("%d %ss", n, word) +} + +// minFreeDiskBytes is the free space below which the disk check warns. +const minFreeDiskBytes = 20e9 + +// dockerCheck fails only when the sandboxes need Docker. Else Docker only +// builds the images that runplane cannot pull. +func dockerCheck(h *Host, need bool) Check { + c := Check{ID: "docker", Name: "Docker"} + if !need { + if h.Docker.Version != "" { + return c.OK("Docker Engine " + h.Docker.Version) + } + return c.OK("Not running; the sandboxes of this mode do not need Docker") + } + switch { + case h.dockerSnap: + return c.Fail("Docker comes from the snap package, which runplane does not support", "Remove the Docker snap, then install Docker Engine."). + WithCommand("sudo snap remove docker && curl -fsSL https://get.docker.com | sh") + case h.Docker.Version == "" && !h.Root: + return c.AsRoot(fmt.Sprintf("Docker does not answer: %v", h.DockerErr)) + case h.Docker.Version == "": + return c.Fail(fmt.Sprintf("Docker does not answer: %v", h.DockerErr), "Install Docker Engine.").WithCommand("curl -fsSL https://get.docker.com | sh") + } + return c.OK("Docker Engine " + h.Docker.Version) +} + +// diskCheck fails where the scheduler starts no sandbox: at or below +// minPercent free. +func diskCheck(h *Host, dataDir string, minPercent float64) Check { + c := Check{ID: "disk", Name: "Free disk space"} + if h.ResourcesErr != nil || h.DiskBytes <= 0 { + return c.Fail(fmt.Sprintf("Cannot read the data disk: %v", h.ResourcesErr), "Make sure that "+dataDir+" is on a mounted disk.") + } + gigabytes := func(b int64) string { return fmt.Sprintf("%.1f GB", float64(b)/1e9) } + freePercent := float64(h.FreeDiskBytes) * 100 / float64(h.DiskBytes) + detail := fmt.Sprintf("%s free of %s (%.0f%%)", gigabytes(h.FreeDiskBytes), gigabytes(h.DiskBytes), freePercent) + switch { + case freePercent <= minPercent: + return c.Fail(fmt.Sprintf("%s; runplane starts no sandbox at %g%% free or less", detail, minPercent), "Free disk space, or lower min_free_disk_percent.") + case h.FreeDiskBytes < minFreeDiskBytes: + return c.Warn(detail+"; images and jobs need 20 GB or more", "Free disk space, or use a larger disk.") + } + return c.OK(detail) +} + +// baseImageCheck tells where the default image comes from: a pull of ref, +// or a build of its Dockerfile with Docker when the pull fails (pullErr). +// needDocker tells that the docker check above fails without Docker. +func baseImageCheck(h *Host, needDocker bool, ref string, pullErr error) Check { + c := Check{ID: "base_image", Name: "Base image source"} + if pullErr == nil { + return c.OK("Pulls " + ref) + } + why := pullErr.Error() + if len(why) > 200 { + why = why[:200] + "..." + } + switch { + case h.Docker.Version != "": + return c.OK(fmt.Sprintf("Builds the image from its Dockerfile with Docker, because %s cannot be pulled: %s", ref, why)) + case needDocker: + return c.Fail(fmt.Sprintf("Cannot pull %s (%s), and Docker does not answer to build it", ref, why), "Fix the Docker check above.") + case !h.Root: + return c.AsRoot(fmt.Sprintf("Cannot pull %s (%s), and Docker does not answer to this user", ref, why)) + } + return c.Fail(fmt.Sprintf("Cannot pull %s (%s), and Docker is not installed to build it", ref, why), + "Run the install script again: it installs Docker Engine for local image builds. Or make the image public on GHCR.") +} + +func internetCheck(ctx context.Context, probe func(context.Context) error) Check { + c := Check{ID: "internet", Name: "Sandbox internet access"} + if probe == nil { + return c.Warn("Not checked here: only the running runplane service can do this check, because it owns the firewall rules of the sandboxes.", + "Open this server in the dashboard and select Run doctor.") + } + switch err := probe(ctx); { + case errors.Is(err, sandbox.ErrNoProbe): + return c.Warn(fmt.Sprintf("Not checked: %v.", err), "Run runplane doctor again when the default image is ready.") + case err != nil: + return c.Fail(fmt.Sprintf("A sandbox cannot reach the internet: %v", err), "Fix the IP forwarding and firewall checks above.") + } + return c.OK("A sandbox reached the internet") +} + +// The doctor measures the clock against an NTP server, because GitHub +// rejects app tokens from a server with a wrong clock. The Date header of +// GitHub can come from a cache, so it is seconds old. +const ntpServer = "time.cloudflare.com:123" + +type clock struct { + synced bool + syncedErr error // nil when synced is known + skew time.Duration + skewErr error +} + +func readClock(ctx context.Context) clock { + var c clock + c.synced, c.syncedErr = clockSynced() + c.skew, c.skewErr = ntpOffset(ctx, ntpServer) + return c +} + +func (k clock) check() Check { + c := Check{ID: "clock", Name: "Time synchronization"} + const fix, command = "Turn on time sync.", "sudo timedatectl set-ntp true" + off := k.skew.Abs().Round(10 * time.Millisecond) + side := "ahead" + if k.skew < 0 { + side = "behind" + } + switch { + case k.skewErr == nil && off >= time.Minute: + return c.Fail(fmt.Sprintf("The clock is %s %s, so GitHub rejects the app tokens", off, side), fix).WithCommand(command) + case k.syncedErr == nil && !k.synced: + return c.Warn("The kernel clock is not synchronized", fix).WithCommand(command) + case k.skewErr == nil && off >= 5*time.Second: + return c.Warn(fmt.Sprintf("The clock is %s %s", off, side), fix).WithCommand(command) + case k.skewErr == nil: + return c.OK(fmt.Sprintf("Synchronized (%s %s)", off, side)) + case k.syncedErr == nil: + return c.OK("Synchronized") + } + return c.Warn(fmt.Sprintf("Cannot check: %v", k.skewErr), "Allow NTP (UDP port 123) to the internet.") +} + +func exists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go new file mode 100644 index 0000000..dbadaeb --- /dev/null +++ b/internal/doctor/doctor_test.go @@ -0,0 +1,408 @@ +package doctor + +import ( + "context" + "encoding/binary" + "errors" + "net" + "os" + "runtime" + "slices" + "strings" + "syscall" + "testing" + "time" + + "github.com/runplane/runplane/internal/host" + "github.com/runplane/runplane/internal/sandbox" +) + +func open(t *testing.T, path string) *os.File { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { f.Close() }) + return f +} + +// healthy returns the scan of a good host, and the options of a mode whose +// driver adds one check. +func healthy() (scan, Options) { + s := scan{ + Host: &Host{ + Info: host.Info{ + OS: "Ubuntu 24.04.3 LTS", Kernel: "6.8.0-48-generic", Arch: "x64", + Resources: host.Resources{VCPU: 8, MemoryMB: 16000, DiskBytes: 200e9, FreeDiskBytes: 150e9}, + }, + Details: host.Details{OSID: "ubuntu", OSVersion: "24.04", CgroupV2: true, IPForward: true}, + Docker: Docker{Version: "29.5.1", Firewall: "iptables"}, + Root: true, Systemd: true, Unit: "runplane", UnitState: "active", + }, + clock: clock{synced: true, skew: 300 * time.Millisecond}, + ports: []portScan{{addr: ":8484"}}, + } + o := Options{ + MinFreeDiskPercent: 15, + Requirements: Requirements{ + Mode: ModeCheck.OK("Test mode"), + Docker: true, + Checks: func(context.Context, *Host) []Check { return []Check{{ID: "driver", Name: "Driver", Status: StatusOK}} }, + }, + } + return s, o +} + +// A healthy host passes all checks. The dashboard and the command show the +// checks in this order. +func TestHealthyHost(t *testing.T) { + s, o := healthy() + var ids []string + for _, c := range s.checks(t.Context(), o) { + ids = append(ids, c.ID) + if c.Status != StatusOK { + t.Errorf("%s is %s: %s", c.ID, c.Status, c.Detail) + } + } + if want := []string{"mode", "docker", "driver", "disk", "clock", "port_8484"}; !slices.Equal(ids, want) { + t.Errorf("checks %v, want %v", ids, want) + } +} + +// wantFix fails when a check that is not ok has no fix, or a fix with a +// command: the UI shows the fix as text and the command with a copy button. +func wantFix(t *testing.T, c Check) { + t.Helper() + if c.Status != StatusOK && c.Fix == "" { + t.Errorf("%s is %s without a fix", c.ID, c.Status) + } + if strings.Contains(c.Fix, "sudo ") { + t.Errorf("%s fix %q holds a command; put it in Command", c.ID, c.Fix) + } +} + +func TestDoctorStatus(t *testing.T) { + inUse := &net.OpError{Op: "listen", Net: "tcp", Err: os.NewSyscallError("bind", syscall.EADDRINUSE)} + tests := []struct { + name string + change func(*scan, *Options) + id string + want Status + }{ + {"disk below 15%", func(s *scan, _ *Options) { s.FreeDiskBytes = 28e9 }, "disk", StatusFail}, + {"disk at the setting", func(s *scan, _ *Options) { s.FreeDiskBytes = 30e9 }, "disk", StatusFail}, + {"disk below the setting", func(s *scan, o *Options) { s.FreeDiskBytes, o.MinFreeDiskPercent = 50e9, 30 }, "disk", StatusFail}, + {"disk above the setting but small", func(s *scan, _ *Options) { s.DiskBytes, s.FreeDiskBytes = 80e9, 18e9 }, "disk", StatusWarn}, + {"disk unreadable", func(s *scan, _ *Options) { s.DiskBytes, s.ResourcesErr = 0, errors.New("statfs: no such file") }, "disk", StatusFail}, + + {"docker snap", func(s *scan, _ *Options) { s.dockerSnap = true }, "docker", StatusFail}, + {"docker down", func(s *scan, _ *Options) { s.Docker.Version, s.DockerErr = "", errors.New("connection refused") }, "docker", StatusFail}, + {"docker socket without root", func(s *scan, _ *Options) { s.Docker.Version, s.Root = "", false }, "docker", StatusWarn}, + {"no docker in a mode without it", func(s *scan, o *Options) { s.Docker.Version, o.Requirements.Docker = "", false }, "docker", StatusOK}, + + {"clock 2 minutes ahead", func(s *scan, _ *Options) { s.clock.skew = 2 * time.Minute }, "clock", StatusFail}, + {"clock 90 s behind", func(s *scan, _ *Options) { s.clock.skew = -90 * time.Second }, "clock", StatusFail}, + {"clock 10 s ahead", func(s *scan, _ *Options) { s.clock.skew = 10 * time.Second }, "clock", StatusWarn}, + {"clock not synchronized", func(s *scan, _ *Options) { s.clock.synced = false }, "clock", StatusWarn}, + {"clock state unknown, skew small", func(s *scan, _ *Options) { s.clock.syncedErr = errors.New("no adjtimex") }, "clock", StatusOK}, + {"clock unknown", func(s *scan, _ *Options) { + s.clock.syncedErr, s.clock.skewErr = errors.New("no adjtimex"), errors.New("offline") + }, "clock", StatusWarn}, + + {"port used by nginx", func(s *scan, _ *Options) { s.ports[0] = portScan{addr: ":8484", err: inUse, pid: 812, name: "nginx"} }, "port_8484", StatusFail}, + {"port used by runplane", func(s *scan, _ *Options) { + s.ports[0] = portScan{addr: ":8484", err: inUse, pid: 900, name: "runplane"} + }, "port_8484", StatusOK}, + {"port used by this process", func(s *scan, _ *Options) { + s.ports[0] = portScan{addr: ":8484", err: inUse, pid: 901, name: "main", self: true} + }, "port_8484", StatusOK}, + {"port used, owner hidden", func(s *scan, _ *Options) { s.ports[0], s.Root = portScan{addr: ":8484", err: inUse}, false }, "port_8484", StatusWarn}, + {"port used, owner unknown as root", func(s *scan, _ *Options) { s.ports[0] = portScan{addr: ":8484", err: inUse} }, "port_8484", StatusFail}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, o := healthy() + tt.change(&s, &o) + checks := s.checks(t.Context(), o) + i := slices.IndexFunc(checks, func(c Check) bool { return c.ID == tt.id }) + if i < 0 { + t.Fatalf("no %s check", tt.id) + } + if c := checks[i]; c.Status != tt.want { + t.Errorf("%s is %s (%s), want %s", tt.id, c.Status, c.Detail, tt.want) + } + wantFix(t, checks[i]) + }) + } +} + +// The checks that more than one driver adds. +func TestHostChecks(t *testing.T) { + tests := []struct { + name string + check func(*Host) Check + want Status + }{ + {"cgroup v1", func(h *Host) Check { h.CgroupV2 = false; return h.CgroupCheck() }, StatusFail}, + {"no nftables as root", func(h *Host) Check { return h.NFTablesCheck(false) }, StatusFail}, + {"no nftables without root", func(h *Host) Check { h.Root = false; return h.NFTablesCheck(false) }, StatusWarn}, + {"no ip forwarding", func(h *Host) Check { h.IPForward = false; return h.ForwardCheck() }, StatusFail}, + {"no systemd for a driver that needs it", func(h *Host) Check { h.Systemd = false; return h.SystemdCheck("which keeps VMs in scopes") }, StatusFail}, + {"no systemd", func(h *Host) Check { h.Systemd = false; return h.SystemdCheck("") }, StatusWarn}, + {"unit stopped", func(h *Host) Check { h.UnitState = "inactive"; return h.SystemdCheck("") }, StatusWarn}, + {"kernel version unreadable", func(h *Host) Check { + h.Kernel = "custom" + return h.KernelCheck(func(c Check, _ Version) Check { return c.OK("") }) + }, StatusWarn}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, _ := healthy() + c := tt.check(s.Host) + if c.Status != tt.want { + t.Errorf("%s is %s (%s), want %s", c.ID, c.Status, c.Detail, tt.want) + } + wantFix(t, c) + }) + } +} + +// A probe that cannot run here warns; only a probe that ran and did not +// reach the internet fails, because a failed check fails the install. +func TestInternetCheck(t *testing.T) { + probe := func(err error) func(context.Context) error { + return func(context.Context) error { return err } + } + for _, tt := range []struct { + name string + probe func(context.Context) error + want Status + }{ + {"reached", probe(nil), StatusOK}, + {"no image yet", probe(sandbox.ErrNoProbe), StatusWarn}, + {"only the service can probe", nil, StatusWarn}, + {"no internet", probe(errors.New("Get \"https://api.github.com\": dial tcp 140.82.112.6:443: i/o timeout")), StatusFail}, + } { + if c := internetCheck(t.Context(), tt.probe); c.Status != tt.want || c.Fix == "" && tt.want != StatusOK { + t.Errorf("%s: %s %q (fix %q), want %s with a fix", tt.name, c.Status, c.Detail, c.Fix, tt.want) + } + } +} + +func TestFailedModeStopsTheDoctor(t *testing.T) { + ran := false + r := Run(t.Context(), Options{ + Requirements: Requirements{Mode: ModeCheck.Fail(`Unknown mode "kvm"`, "Set RUNPLANE_MODE.")}, + Internet: func(context.Context) error { ran = true; return nil }, + }) + if len(r.Checks) != 1 || r.Checks[0].ID != "mode" || !r.Failed() { + t.Errorf("got %+v, want one failed mode check", r.Checks) + } + if ran { + t.Error("the internet probe ran for an unknown mode") + } +} + +func TestScanPortFindsThisProcess(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("needs Linux, where the port owners come from /proc") + } + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + p := scanPort(l.Addr().String(), false) + if !errors.Is(p.err, syscall.EADDRINUSE) || !p.self || p.pid != os.Getpid() { + t.Errorf("got %+v, want this process as the owner", p) + } +} + +// The doctor of a running server checks its own ports and the ports of its +// driver, also where it cannot find the owner of a port, such as on macOS. +func TestDoctorOfARunningServerKnowsItsPorts(t *testing.T) { + var addrs []string + for range 2 { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + addrs = append(addrs, l.Addr().String()) + } + r := Run(t.Context(), Options{ + DataDir: t.TempDir(), + Listen: addrs[:1], + Serving: true, + Requirements: Requirements{Mode: ModeCheck.OK("Test mode"), Listen: addrs[1:]}, + Docker: func(context.Context) (Docker, error) { return Docker{}, errors.New("no docker") }, + BaseImage: func(context.Context) (string, error) { return "runner:1", nil }, + }) + for _, addr := range addrs { + _, port, _ := net.SplitHostPort(addr) + i := slices.IndexFunc(r.Checks, func(c Check) bool { return c.ID == "port_"+port }) + if i < 0 || r.Checks[i].Status != StatusOK { + t.Errorf("no OK check of the port %s in %+v", port, r.Checks) + } + } +} + +func TestNTPOffset(t *testing.T) { + for _, serverAhead := range []time.Duration{30 * time.Second, -90 * time.Second, 0} { + skew, err := ntpOffset(t.Context(), fakeNTP(t, serverAhead)) + if err != nil { + t.Fatal(err) + } + // This clock is ahead when the server is behind. + if d := skew + serverAhead; d.Abs() > 50*time.Millisecond { + t.Errorf("server ahead %s: got skew %s", serverAhead, skew) + } + } +} + +// fakeNTP answers each request with a clock that is offset from this one. +func fakeNTP(t *testing.T, offset time.Duration) string { + conn, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + go func() { + buf := make([]byte, 48) + for { + _, addr, err := conn.ReadFrom(buf) + if err != nil { + return + } + now := time.Now().Add(offset) + sec := uint64(now.Unix() + 2208988800) + frac := uint64(now.Nanosecond()) << 32 / 1e9 + resp := make([]byte, 48) + resp[0], resp[1] = 4<<3|4, 2 // version 4, server mode, stratum 2 + for _, at := range []int{32, 40} { + binary.BigEndian.PutUint32(resp[at:], uint32(sec)) + binary.BigEndian.PutUint32(resp[at+4:], uint32(frac)) + } + conn.WriteTo(resp, addr) + } + }() + return conn.LocalAddr().String() +} + +func TestBaseImageSource(t *testing.T) { + const ref = "ghcr.io/acme/runplane/runner-ubuntu-24.04:latest" + denied := errors.New("GET https://ghcr.io/v2/acme/runplane/runner-ubuntu-24.04/manifests/latest: DENIED") + tests := []struct { + name string + needDocker bool + change func(*Host) + pullErr error + want Status + detail string + }{ + {"published image", false, func(h *Host) { h.Docker.Version = "" }, nil, StatusOK, "Pulls " + ref}, + {"builds with docker", false, func(*Host) {}, denied, StatusOK, "Builds the image"}, + {"no docker", false, func(h *Host) { h.Docker.Version = "" }, denied, StatusFail, "Docker is not installed"}, + {"docker unknown without root", false, func(h *Host) { h.Docker.Version, h.Root = "", false }, denied, StatusWarn, "Docker does not answer"}, + {"mode with docker builds", true, func(*Host) {}, denied, StatusOK, "Builds the image"}, + {"mode with docker, docker down", true, func(h *Host) { h.Docker.Version = "" }, denied, StatusFail, "Docker does not answer"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, _ := healthy() + tt.change(s.Host) + c := baseImageCheck(s.Host, tt.needDocker, ref, tt.pullErr) + if c.Status != tt.want || !strings.Contains(c.Detail, tt.detail) { + t.Errorf("%s: %q, want %s with %q", c.Status, c.Detail, tt.want, tt.detail) + } + if c.Status != StatusOK && c.Fix == "" { + t.Errorf("%s without a fix", c.Status) + } + }) + } +} + +func TestFormat(t *testing.T) { + tests := []struct { + checks []Check + want string + }{ + {[]Check{ + {ID: "kvm", Name: "KVM", Status: StatusOK, Detail: "/dev/kvm opens read-write", Fix: "not shown", Command: "not shown"}, + {ID: "clock", Name: "Time synchronization", Status: StatusWarn, Detail: "The kernel clock is not synchronized", + Fix: "Turn on time sync.", Command: "sudo timedatectl set-ntp true"}, + {ID: "sysbox", Name: "Sysbox", Status: StatusFail, Detail: "Sysbox is not installed", Fix: "Install Sysbox"}, + }, `✓ KVM /dev/kvm opens read-write +! Time synchronization The kernel clock is not synchronized + Fix: Turn on time sync. + Run: sudo timedatectl set-ntp true +✗ Sysbox Sysbox is not installed + Fix: Install Sysbox + +1 failure, 1 warning. +`}, + {[]Check{{Name: "KVM", Status: StatusOK, Detail: "ok"}}, "✓ KVM ok\n\nAll checks passed.\n"}, + } + for _, tt := range tests { + var b strings.Builder + if err := Format(&b, tt.checks); err != nil { + t.Fatal(err) + } + if b.String() != tt.want { + t.Errorf("got\n%s\nwant\n%s", b.String(), tt.want) + } + } +} + +func TestListenInodes(t *testing.T) { + // Port 8484 (0x2124) has a listening socket and an established one. + got := listenInodes(open(t, "testdata/proc/net-tcp"), 8484) + if !slices.Equal(got, []string{"44817"}) { + t.Errorf("got %v, want [44817]", got) + } +} + +func TestAtLeast(t *testing.T) { + tests := []struct { + have, want string + ok bool + }{ + {"29.5.0", "29.5", true}, + {"29.10.1", "29.5", true}, // numbers, not text + {"28.5.2", "29.5", false}, + {"30.0.0", "29.5", true}, + {"0.7.0", "0.7.1", false}, + {"0.7.1", "0.7.1", true}, + {"0.7.1-0.linux", "0.7.1", true}, + {"0.10.0", "0.7.1", true}, + {"0.6.7+dev", "0.7.1", false}, + {"v1.13.1", "1.13", true}, + {"6.8.0-48-generic", "5.12", true}, + {"5.4.0-200-generic", "5.12", false}, + {"", "0.7.1", false}, + {"unknown", "0.7.1", false}, + {"29", "29.5", false}, // a major number alone is not a version + } + for _, tt := range tests { + if got := AtLeast(tt.have, tt.want); got != tt.ok { + t.Errorf("AtLeast(%q, %q) = %v, want %v", tt.have, tt.want, got, tt.ok) + } + } +} + +func TestFirstVersion(t *testing.T) { + for file, want := range map[string]string{ + "firecracker": "1.13.1", + "jailer": "1.13.1", + "sysbox-runc": "0.7.1", + } { + b, err := os.ReadFile("testdata/tools/" + file) + if err != nil { + t.Fatal(err) + } + if got := firstVersion(string(b)); got != want { + t.Errorf("%s: got %q, want %q", file, got, want) + } + } +} diff --git a/internal/doctor/ports.go b/internal/doctor/ports.go new file mode 100644 index 0000000..fe6d5e1 --- /dev/null +++ b/internal/doctor/ports.go @@ -0,0 +1,101 @@ +package doctor + +import ( + "bufio" + "cmp" + "errors" + "fmt" + "io" + "net" + "os" + "strconv" + "strings" + "syscall" +) + +type portScan struct { + addr string + err error // from listening on addr; nil when the port is free + pid int // the process that listens, when known + name string + self bool // this process listens +} + +func portChecks(ports []portScan, root bool, envFile string) []Check { + var cs []Check + for _, p := range ports { + _, port, _ := net.SplitHostPort(p.addr) + port = cmp.Or(port, p.addr) + c := Check{ID: "port_" + port, Name: "Port " + port} + inUse := errors.Is(p.err, syscall.EADDRINUSE) + switch { + case p.err == nil: + c = c.OK(p.addr + " is free") + case inUse && (p.self || p.name == "runplane"): + c = c.OK("runplane listens on " + p.addr) + case inUse && p.name != "": + c = c.Fail(fmt.Sprintf("%s (pid %d) listens on %s", p.name, p.pid, p.addr), "Stop "+p.name+", or give runplane a different port.") + case inUse && !root: + c = c.AsRoot("Another process listens on " + p.addr) + case inUse: + c = c.Fail("Another process listens on "+p.addr, "Stop that process, or give runplane a different port.") + case errors.Is(p.err, syscall.EACCES): + c = c.Warn("Ports below 1024 need root", "Run runplane as root.") + default: + c = c.Warn(fmt.Sprintf("Cannot listen on %s: %v", p.addr, p.err), "Check the listen address in "+envFile+".") + } + cs = append(cs, c) + } + return cs +} + +func scanPorts(addrs []string, own bool) []portScan { + var ports []portScan + for _, addr := range addrs { + ports = append(ports, scanPort(addr, own)) + } + return ports +} + +// scanPort tries to listen on addr. own tells that this process listens on +// it already. +func scanPort(addr string, own bool) portScan { + p := portScan{addr: addr} + l, err := net.Listen("tcp", addr) + if err == nil { + l.Close() + return p + } + p.err = err + if !errors.Is(err, syscall.EADDRINUSE) { + return p + } + if own { + p.self = true + return p + } + _, port, _ := net.SplitHostPort(addr) + n, _ := strconv.Atoi(port) + p.pid, p.name = portOwner(n) + p.self = p.pid == os.Getpid() + return p +} + +// listenInodes returns the inodes of the sockets that listen on port in a +// /proc/net/tcp or /proc/net/tcp6 table. +func listenInodes(r io.Reader, port int) []string { + const tcpListen = "0A" + var inodes []string + s := bufio.NewScanner(r) + for s.Scan() { + f := strings.Fields(s.Text()) + if len(f) < 10 || f[3] != tcpListen { + continue + } + _, hexPort, _ := strings.Cut(f[1], ":") + if p, err := strconv.ParseUint(hexPort, 16, 16); err == nil && int(p) == port { + inodes = append(inodes, f[9]) + } + } + return inodes +} diff --git a/internal/doctor/requirements.go b/internal/doctor/requirements.go new file mode 100644 index 0000000..f651a9a --- /dev/null +++ b/internal/doctor/requirements.go @@ -0,0 +1,125 @@ +package doctor + +import ( + "cmp" + "context" + "fmt" + "os/exec" + "strings" +) + +// Requirements are what the sandbox driver of a mode needs on the server. +// Each driver package gives its Requirements, with the paths, the users, +// and the versions that the driver uses. The methods of Host below are the +// checks that more than one driver needs. +type Requirements struct { + // Mode is the first check, which names the mode. When it fails, the + // doctor runs no other check. + Mode Check + // Docker tells that the sandboxes need Docker. + Docker bool + // Listen are the TCP addresses of the driver, for the port checks. + Listen []string + // Software reads the versions of the sandbox software into h.Info. + Software func(ctx context.Context, h *Host) + Checks func(ctx context.Context, h *Host) []Check +} + +// ModeCheck is the empty check of Requirements.Mode. +var ModeCheck = Check{ID: "mode", Name: "Mode"} + +func (h *Host) CgroupCheck() Check { + c := Check{ID: "cgroup", Name: "cgroup v2"} + if h.CgroupV2 { + return c.OK("Unified hierarchy at /sys/fs/cgroup") + } + return c.Fail("The host uses cgroup v1", "Add systemd.unified_cgroup_hierarchy=1 to the kernel command line and reboot.") +} + +// NFTablesCheck checks the kernel support of the firewall rules of the +// sandboxes: available is netrules.Available. +func (h *Host) NFTablesCheck(available bool) Check { + c := Check{ID: "nftables", Name: "nftables"} + switch { + case available: + return c.OK("Available") + case !h.Root: + return c.AsRoot("Cannot check without root") + } + return c.Fail("The kernel has no nf_tables support", "Load the nf_tables module.").WithCommand("sudo modprobe nf_tables") +} + +// runplane turns IP forwarding on when it starts, after Docker, and not with +// a sysctl file at boot: Docker sets the FORWARD policy to DROP only when it +// turns on forwarding itself. +func (h *Host) ForwardCheck() Check { + c := Check{ID: "ip_forward", Name: "IP forwarding"} + if h.IPForward { + return c.OK("net.ipv4.ip_forward = 1") + } + c = c.Fail("net.ipv4.ip_forward = 0, so sandboxes have no internet access", "runplane turns on IP forwarding when it starts. Restart runplane.") + if h.Unit != "" { + c = c.WithCommand("sudo systemctl restart " + h.Unit) + } + return c +} + +// SystemdCheck looks at systemd and the unit of runplane. need tells why the +// driver needs systemd; without a need, a host without systemd only warns. +func (h *Host) SystemdCheck(need string) Check { + c := Check{ID: "systemd", Name: "systemd"} + switch { + case !h.Systemd && need != "": + return c.Fail("The host does not run systemd, "+need, "Use a Linux distribution with systemd.") + case !h.Systemd: + return c.Warn("The host does not run systemd", "Use a Linux distribution with systemd.") + case h.Unit == "": + return c.OK("Running") + case h.UnitState == "active": + return c.OK(h.Unit + ".service is active") + } + return c.Warn(fmt.Sprintf("%s.service is %s", h.Unit, cmp.Or(h.UnitState, "unknown")), "Start the "+h.Unit+" service."). + WithCommand("sudo systemctl enable --now " + h.Unit) +} + +func (h *Host) KernelCheck(rules func(c Check, v Version) Check) Check { + c := Check{ID: "kernel", Name: "Kernel"} + v, ok := ParseVersion(h.Kernel) + if !ok { + return c.Warn(fmt.Sprintf("Cannot read the kernel version %q", h.Kernel), "Check that uname -r prints a version such as 6.8.0.").WithCommand("uname -r") + } + return rules(c, v) +} + +// Blocked fails c when one of drops, the nftables chains that drop by +// default, blocks the sandboxes. ifaces names the sandbox interfaces of the +// driver, which the chains must accept. +func Blocked(c Check, drops []string, ifaces string) (Check, bool) { + if len(drops) == 0 { + return c, false + } + return c.Fail(fmt.Sprintf("The nftables chain %s drops by default, so sandboxes cannot reach the internet or the cache", strings.Join(drops, ", ")), + "Accept the traffic of the sandbox interfaces in that chain: "+ifaces+".").WithCommand("sudo nft list ruleset"), true +} + +// OpenForward warns with c when this server forwards the packets between +// all its networks: forwardPolicy is the policy of the iptables FORWARD +// chain. Docker sets the FORWARD policy to DROP only when it turns on IP +// forwarding itself, so an ACCEPT here means that forwarding was on before +// Docker started. Without Docker, the policy is the choice of the host +// firewall. +func (h *Host) OpenForward(c Check, forwardPolicy string) (Check, bool) { + if forwardPolicy == "DROP" || !h.IPForward || h.Docker.Version == "" { + return c, false + } + return c.Warn("The FORWARD policy is ACCEPT, so this server forwards packets between all its networks, not only for the sandboxes", + "Set the policy to DROP. Docker and runplane accept the sandbox traffic with their own rules. "+ + "So that Docker sets DROP at each boot, remove net.ipv4.ip_forward from /etc/sysctl.conf and /etc/sysctl.d: runplane turns on forwarding after Docker starts."). + WithCommand("sudo iptables -P FORWARD DROP"), true +} + +// UnitState is the output of "systemctl is-active unit". +func UnitState(ctx context.Context, unit string) string { + out, _ := exec.CommandContext(ctx, "systemctl", "is-active", unit).Output() + return strings.TrimSpace(string(out)) +} diff --git a/internal/doctor/sys_linux.go b/internal/doctor/sys_linux.go new file mode 100644 index 0000000..9cf2f97 --- /dev/null +++ b/internal/doctor/sys_linux.go @@ -0,0 +1,55 @@ +//go:build linux + +package doctor + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "golang.org/x/sys/unix" +) + +func clockSynced() (bool, error) { + var tx unix.Timex + if _, err := unix.Adjtimex(&tx); err != nil { + return false, fmt.Errorf("read kernel clock state: %w", err) + } + // systemd uses this rule: synchronized when the maximum error is below 16 s. + return tx.Maxerror < 16_000_000, nil +} + +// portOwner finds the process that listens on a TCP port. It needs root to +// see the sockets of other users. +func portOwner(port int) (pid int, name string) { + inodes := map[string]bool{} + for _, path := range []string{"/proc/net/tcp", "/proc/net/tcp6"} { + f, err := os.Open(path) + if err != nil { + continue + } + for _, inode := range listenInodes(f, port) { + inodes["socket:["+inode+"]"] = true + } + f.Close() + } + if len(inodes) == 0 { + return 0, "" + } + procs, _ := filepath.Glob("/proc/[0-9]*") + for _, proc := range procs { + fds, _ := os.ReadDir(filepath.Join(proc, "fd")) + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(proc, "fd", fd.Name())) + if err != nil || !inodes[target] { + continue + } + pid, _ := strconv.Atoi(filepath.Base(proc)) + comm, _ := os.ReadFile(filepath.Join(proc, "comm")) + return pid, strings.TrimSpace(string(comm)) + } + } + return 0, "" +} diff --git a/internal/doctor/sys_other.go b/internal/doctor/sys_other.go new file mode 100644 index 0000000..aaf900c --- /dev/null +++ b/internal/doctor/sys_other.go @@ -0,0 +1,14 @@ +//go:build !linux + +package doctor + +import ( + "errors" + "runtime" +) + +func clockSynced() (bool, error) { + return false, errors.New("no kernel clock state on " + runtime.GOOS) +} + +func portOwner(int) (pid int, name string) { return 0, "" } diff --git a/internal/host/testdata/proc/net-tcp b/internal/doctor/testdata/proc/net-tcp similarity index 100% rename from internal/host/testdata/proc/net-tcp rename to internal/doctor/testdata/proc/net-tcp diff --git a/internal/host/testdata/tools/firecracker b/internal/doctor/testdata/tools/firecracker similarity index 100% rename from internal/host/testdata/tools/firecracker rename to internal/doctor/testdata/tools/firecracker diff --git a/internal/host/testdata/tools/jailer b/internal/doctor/testdata/tools/jailer similarity index 100% rename from internal/host/testdata/tools/jailer rename to internal/doctor/testdata/tools/jailer diff --git a/internal/host/testdata/tools/sysbox-runc b/internal/doctor/testdata/tools/sysbox-runc similarity index 100% rename from internal/host/testdata/tools/sysbox-runc rename to internal/doctor/testdata/tools/sysbox-runc diff --git a/internal/doctor/version.go b/internal/doctor/version.go new file mode 100644 index 0000000..fa3ff9d --- /dev/null +++ b/internal/doctor/version.go @@ -0,0 +1,67 @@ +package doctor + +import ( + "context" + "os/exec" + "slices" + "strconv" + "strings" + "time" +) + +// Version is a major.minor.patch number. Missing parts are 0. +type Version [3]int + +// ParseVersion reads the leading numbers of "v1.13.1", "29.5.0-rc.1", or +// "6.8.0-48-generic". It needs at least a major and a minor number. +func ParseVersion(s string) (Version, bool) { + s = strings.TrimPrefix(s, "v") + if i := strings.IndexFunc(s, func(r rune) bool { return r != '.' && (r < '0' || r > '9') }); i >= 0 { + s = s[:i] + } + parts := strings.Split(strings.TrimSuffix(s, "."), ".") + if len(parts) < 2 || len(parts) > 3 { + return Version{}, false + } + var v Version + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil { + return Version{}, false + } + v[i] = n + } + return v, true +} + +func (v Version) Less(w Version) bool { return slices.Compare(v[:], w[:]) < 0 } + +// AtLeast reports if have is a version and not older than want. +func AtLeast(have, want string) bool { + h, ok := ParseVersion(have) + w, _ := ParseVersion(want) + return ok && !h.Less(w) +} + +// ToolVersion runs "path --version" and returns the first version number in +// the output, or "" if the tool is missing. +func ToolVersion(ctx context.Context, path string) string { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, path, "--version").Output() + if err != nil { + return "" + } + return firstVersion(string(out)) +} + +// firstVersion returns the first version number in the output of a +// "--version" flag, without a leading "v". +func firstVersion(out string) string { + for _, word := range strings.Fields(out) { + if _, ok := ParseVersion(word); ok { + return strings.TrimPrefix(word, "v") + } + } + return "" +} diff --git a/internal/events/events.go b/internal/events/events.go index f7f40ab..509557d 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -1,78 +1,196 @@ -// Package events is the in-process publish and subscribe bus. The API sends -// the events to the browser over SSE. +// Package events is the in-process bus. Each event type of this package is a +// topic and carries its own data, so a publisher cannot send the data of +// another topic. +// +// The bus has one delivery rule for each kind of subscriber. Subscribe never +// loses an event, for the packages that must see each event: notify must see +// each failed job, and github each change of the settings and the pools. +// Watch ends when its client reads too slowly. The SSE stream of the API uses +// Watch, and the browser then connects again and reads the state from the API. package events -import "sync" - -// Event types. All except SettingsChanged are SSE event names. -const ( - SandboxState = "sandbox.state" - JobState = "job.state" - PoolStats = "pool.stats" - ServerState = "server.state" - HostStats = "host.stats" - RunnerVersion = "runner.version" - ImageState = "image.state" - // GitHubState says that the GitHub status changed: the app, the - // installations, or the scale set listeners. It has no data. - GitHubState = "github.state" - BackupState = "backup.state" - - // SettingsChanged is internal: packages that cache settings read them again. - SettingsChanged = "settings.changed" +import ( + "context" + "sync" + + "github.com/runplane/runplane/internal/store" +) + +// Event is an event of the bus. Only the types of this package are events. +type Event interface{ event() } + +// Each of these events has a row after its change. +type ( + SandboxState store.Sandbox + JobState store.Job + ServerState store.Server + RunnerVersion store.RunnerBundle + BackupState store.Backup ) -type Event struct { - Type string - Data any +// ServerRemoved has the ID of a removed server. The browser gets it as it is. +type ServerRemoved struct { + ID string `json:"id"` } -// As returns the data of an event as a T, also when the data is a *T. -func As[T any](data any) (T, bool) { - switch x := data.(type) { - case T: - return x, true - case *T: - if x != nil { - return *x, true - } - } - var zero T - return zero, false +// HostStats is a new host sample of a server. The browser gets it as it is. +type HostStats store.ServerStat + +// ImageState is an image row after a change. +type ImageState store.Image + +// ImageRemoved has the ID of a removed image. The browser gets it as it is. +type ImageRemoved struct { + ID string `json:"id"` +} + +// ImageBuildState is the build of an image on a server after a change. +type ImageBuildState store.ImageBuild + +// PoolStats counts the runners and the waiting jobs of a pool. The browser +// gets it as it is. +type PoolStats struct { + PoolID string `json:"pool_id"` + store.PoolCounts } -// Bus delivers each event to all subscribers. The zero value is ready to use. +// PoolsChanged says that the pools changed, or what they use: a pool, the +// default image, an image, or a server that a pool names. The browser gets +// it as it is. +type PoolsChanged struct{} + +// GitHubState says that the GitHub status changed: the app, the +// installations, or the scale set listeners. +type GitHubState struct{} + +// SettingsChanged has the settings before and after a change. The browser +// gets the API object of the new settings. +type SettingsChanged struct{ Old, New store.Settings } + +func (SandboxState) event() {} +func (JobState) event() {} +func (ServerState) event() {} +func (ServerRemoved) event() {} +func (HostStats) event() {} +func (RunnerVersion) event() {} +func (BackupState) event() {} +func (ImageState) event() {} +func (ImageRemoved) event() {} +func (ImageBuildState) event() {} +func (PoolStats) event() {} +func (PoolsChanged) event() {} +func (GitHubState) event() {} +func (SettingsChanged) event() {} + +// Bus gives each event to its subscribers. The zero value is ready to use. type Bus struct { - mu sync.RWMutex - subs map[chan Event]struct{} + mu sync.RWMutex + sinks map[*sink]struct{} } -// Publish never blocks. A subscriber with a full buffer loses the event. +// sink takes the events of one subscriber. Its put never waits. +type sink struct{ put func(Event) } + +// Publish gives e to each subscriber. It never waits for a subscriber. func (b *Bus) Publish(e Event) { b.mu.RLock() defer b.mu.RUnlock() - for ch := range b.subs { + for s := range b.sinks { + s.put(e) + } +} + +// Subscribe returns each event of type E until ctx ends, in the order of +// Publish. It never loses an event: when the subscriber is slow, its events +// wait in a queue without a size limit. The channel does not close. +func Subscribe[E Event](ctx context.Context, b *Bus) <-chan E { + q := &queue[E]{ch: make(chan E, queueRoom), done: ctx.Done()} + b.add(ctx, func(e Event) { + if x, ok := e.(E); ok { + q.push(x) + } + }) + return q.ch +} + +// Watch returns all events until ctx ends, for a client that can read the +// state again. When more than buffer events wait for the client, the client +// loses the events and the channel closes, so that the client knows. +func (b *Bus) Watch(ctx context.Context, buffer int) <-chan Event { + ch := make(chan Event, buffer) + var mu sync.Mutex + closed := false + b.add(ctx, func(e Event) { + mu.Lock() + defer mu.Unlock() + if closed { + return + } select { case ch <- e: default: + closed = true + close(ch) } - } + }) + return ch } -// Subscribe returns a channel with room for buffer events. Cancel stops the -// delivery and closes the channel; it is safe to call more than one time. -func (b *Bus) Subscribe(buffer int) (<-chan Event, func()) { - ch := make(chan Event, buffer) +func (b *Bus) add(ctx context.Context, put func(Event)) { + s := &sink{put} b.mu.Lock() - if b.subs == nil { - b.subs = make(map[chan Event]struct{}) + if b.sinks == nil { + b.sinks = map[*sink]struct{}{} } - b.subs[ch] = struct{}{} + b.sinks[s] = struct{}{} b.mu.Unlock() - return ch, sync.OnceFunc(func() { + context.AfterFunc(ctx, func() { b.mu.Lock() - delete(b.subs, ch) + delete(b.sinks, s) b.mu.Unlock() - close(ch) }) } + +// queueRoom is the room of a Subscribe channel. More events wait in the queue. +const queueRoom = 64 + +type queue[E any] struct { + ch chan E + done <-chan struct{} + + mu sync.Mutex + wait []E // the events after a full ch, in order +} + +func (q *queue[E]) push(x E) { + q.mu.Lock() + defer q.mu.Unlock() + if len(q.wait) == 0 { + select { + case q.ch <- x: + return + default: + go q.drain() + } + } + q.wait = append(q.wait, x) +} + +// drain moves the waiting events to ch. One drain runs while wait has +// events: push starts it for the first one, and it stops when wait is empty. +func (q *queue[E]) drain() { + q.mu.Lock() + for len(q.wait) > 0 { + x := q.wait[0] + q.mu.Unlock() + select { + case q.ch <- x: + case <-q.done: + return + } + q.mu.Lock() + q.wait = q.wait[1:] + } + q.wait = nil + q.mu.Unlock() +} diff --git a/internal/events/events_test.go b/internal/events/events_test.go index a480a69..c03a847 100644 --- a/internal/events/events_test.go +++ b/internal/events/events_test.go @@ -1,22 +1,27 @@ package events import ( + "context" "sync" "testing" "time" + + "github.com/runplane/runplane/internal/store" ) func TestSlowSubscriberDoesNotBlock(t *testing.T) { var b Bus - slow, cancelSlow := b.Subscribe(1) - defer cancelSlow() - fast, cancelFast := b.Subscribe(10) - defer cancelFast() + ctx := t.Context() + n := 3 * queueRoom + stats := Subscribe[PoolStats](ctx, &b) + slow := b.Watch(ctx, 1) + fast := b.Watch(ctx, 2*n) done := make(chan struct{}) go func() { - for i := range 5 { - b.Publish(Event{Type: JobState, Data: i}) + for i := range n { + b.Publish(PoolStats{PoolCounts: store.PoolCounts{Idle: i}}) + b.Publish(GitHubState{}) } close(done) }() @@ -26,29 +31,87 @@ func TestSlowSubscriberDoesNotBlock(t *testing.T) { t.Fatal("Publish blocked on a full subscriber") } - if e := <-slow; e.Data != 0 { - t.Errorf("slow got %v, want the first event", e.Data) + for i := range n { + select { + case p := <-stats: + if p.Idle != i { + t.Fatalf("pool.stats event %d has idle %d, want the events in order", i, p.Idle) + } + case <-time.After(5 * time.Second): + t.Fatalf("got %d of %d pool.stats events", i, n) + } } select { - case e := <-slow: - t.Errorf("slow got %v, want the other events dropped", e.Data) + case p := <-stats: + t.Errorf("one more pool.stats event %+v, want none", p) default: } - for i := range 5 { - if e := <-fast; e.Data != i || e.Type != JobState { - t.Fatalf("fast event %d = %+v", i, e) + + if e := <-slow; e != (PoolStats{PoolCounts: store.PoolCounts{Idle: 0}}) { + t.Errorf("slow watcher got %+v, want the first event", e) + } + select { + case e, ok := <-slow: + if ok { + t.Errorf("slow watcher got %+v, want its channel closed after it lost events", e) + } + case <-time.After(5 * time.Second): + t.Error("the channel of the slow watcher is open after it lost events") + } + for i := range 2 * n { + if _, ok := <-fast; !ok { + t.Fatalf("fast watcher closed after %d events", i) } } } -func TestCancel(t *testing.T) { - var b Bus - ch, cancel := b.Subscribe(4) - cancel() - cancel() // a second call is safe - b.Publish(Event{Type: SandboxState}) - if _, ok := <-ch; ok { - t.Fatal("the channel is open after cancel") +// Publish puts an event in the channel of each subscriber before it +// returns, so an event that does not come shows that the subscription ended. +func TestSubscriptionEndsWithContext(t *testing.T) { + for _, tt := range []struct { + name string + // subscribe returns got, which reports if an event waits for the subscriber. + subscribe func(t *testing.T, ctx context.Context, b *Bus) (got func() bool) + }{ + {"Subscribe", func(t *testing.T, ctx context.Context, b *Bus) func() bool { + ch := Subscribe[JobState](ctx, b) + return func() bool { + select { + case <-ch: + return true + default: + return false + } + } + }}, + {"Watch", func(t *testing.T, ctx context.Context, b *Bus) func() bool { + ch := b.Watch(ctx, 1) + return func() bool { + select { + case _, ok := <-ch: + if !ok { + t.Fatal("the channel closed, so the test cannot see more events") + } + return true + default: + return false + } + } + }}, + } { + t.Run(tt.name, func(t *testing.T) { + var b Bus + ctx, cancel := context.WithCancel(t.Context()) + got := tt.subscribe(t, ctx, &b) + cancel() + deadline := time.Now().Add(5 * time.Second) + for b.Publish(JobState{}); got(); b.Publish(JobState{}) { + if time.Now().After(deadline) { + t.Fatal("the subscriber gets events after the end of its context") + } + time.Sleep(time.Millisecond) + } + }) } } @@ -62,12 +125,14 @@ func TestPublishWhileCancel(t *testing.T) { case <-stop: return default: - b.Publish(Event{Type: HostStats}) + b.Publish(HostStats{}) } } }) for range 200 { - _, cancel := b.Subscribe(0) + ctx, cancel := context.WithCancel(t.Context()) + b.Watch(ctx, 0) + Subscribe[HostStats](ctx, &b) cancel() } close(stop) diff --git a/internal/image/ext4.go b/internal/ext4/ext4.go similarity index 74% rename from internal/image/ext4.go rename to internal/ext4/ext4.go index 9c7505c..d853fac 100644 --- a/internal/image/ext4.go +++ b/internal/ext4/ext4.go @@ -1,4 +1,6 @@ -package image +// Package ext4 writes the read-only ext4 drives of VM mode: the root disks +// of the images and the runner bundles. +package ext4 import ( "archive/tar" @@ -9,20 +11,21 @@ import ( "path/filepath" "github.com/Microsoft/hcsshim/ext4/tar2ext4" + + "github.com/runplane/runplane/internal/atomicfile" ) // maxDiskSize is the largest ext4 file that runplane makes. The writer // reserves 256 bytes of metadata for each GiB. const maxDiskSize = 256 << 30 -// writeExt4 writes the tar stream from fill as an ext4 file at dst, through -// a temp file and a rename. It returns the file size. -func writeExt4(dst string, fill func(w io.Writer) error) (int64, error) { - f, err := os.CreateTemp(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp-") +// Write writes the tar stream from fill as an ext4 file at dst. It returns +// the file size. +func Write(dst string, fill func(w io.Writer) error) (int64, error) { + f, err := atomicfile.Create(dst, 0o644) if err != nil { return 0, err } - defer os.Remove(f.Name()) defer f.Close() pr, pw := io.Pipe() @@ -42,9 +45,6 @@ func writeExt4(dst string, fill func(w io.Writer) error) (int64, error) { if fillErr := <-filled; err == nil { err = fillErr } - if err == nil { - err = f.Sync() - } if err != nil { return 0, fmt.Errorf("write %s: %w", dst, err) } @@ -52,21 +52,15 @@ func writeExt4(dst string, fill func(w io.Writer) error) (int64, error) { if err != nil { return 0, err } - if err := f.Chmod(0o644); err != nil { - return 0, err - } - if err := f.Close(); err != nil { - return 0, err - } - if err := os.Rename(f.Name(), dst); err != nil { + if err := f.Commit(); err != nil { return 0, err } return st.Size(), nil } -// tarDir writes the files in dir as a tar stream. All files belong to root, +// TarDir writes the files in dir as a tar stream. All files belong to root, // as in the read-only bind mount of container mode. -func tarDir(w io.Writer, dir string) error { +func TarDir(w io.Writer, dir string) error { tw := tar.NewWriter(w) err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { if err != nil { diff --git a/internal/ext4/ext4_test.go b/internal/ext4/ext4_test.go new file mode 100644 index 0000000..797faf9 --- /dev/null +++ b/internal/ext4/ext4_test.go @@ -0,0 +1,28 @@ +package ext4 + +import ( + "archive/tar" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestWriteLeavesNothingOnError(t *testing.T) { + dir := t.TempDir() + dst := filepath.Join(dir, "disk.ext4") + _, err := Write(dst, func(w io.Writer) error { + tw := tar.NewWriter(w) + tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: "a", Size: 1, Mode: 0o644}) + tw.Write([]byte("a")) + return errors.New("network down") + }) + if err == nil || !strings.Contains(err.Error(), "network down") { + t.Fatalf("Write error = %v, want the fill error", err) + } + if left, _ := os.ReadDir(dir); len(left) != 0 { + t.Errorf("folder has %v after a failure, want nothing", left) + } +} diff --git a/internal/ext4/ext4test/ext4test.go b/internal/ext4/ext4test/ext4test.go new file mode 100644 index 0000000..7dd70bc --- /dev/null +++ b/internal/ext4/ext4test/ext4test.go @@ -0,0 +1,65 @@ +// Package ext4test examines the ext4 files of the tests. Its checks need +// e2fsprogs; without it they skip the test, so callers run them in a +// subtest of their own. +package ext4test + +import ( + "encoding/binary" + "os" + "os/exec" + "strings" + "testing" +) + +// Check examines the ext4 superblock magic, and the file system with +// e2fsck. The hcsshim writer does not set the padding bits of the inode +// bitmap. The kernel does not read them on a read-only mount, so the check +// permits that one problem. +func Check(t *testing.T, file string) { + t.Helper() + f, err := os.Open(file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + magic := make([]byte, 2) + if _, err := f.ReadAt(magic, 1080); err != nil { + t.Fatal(err) + } + if got := binary.LittleEndian.Uint16(magic); got != 0xef53 { + t.Fatalf("magic at 1080 = %#x, want 0xef53", got) + } + out, err := exec.Command(tool(t, "e2fsck"), "-fn", file).CombinedOutput() + if exit, ok := err.(*exec.ExitError); err != nil && (!ok || exit.ExitCode() != 4) { + t.Fatalf("e2fsck -fn %s: %v\n%s", file, err, out) + } + for line := range strings.Lines(string(out)) { + if strings.HasSuffix(strings.TrimSpace(line), "? no") && !strings.Contains(line, "Padding at end of inode bitmap is not set") { + t.Errorf("e2fsck -fn %s:\n%s", file, out) + return + } + } +} + +// Debugfs runs a debugfs request on an ext4 file and examines that the +// output has want. +func Debugfs(t *testing.T, file, request, want string) { + t.Helper() + out, err := exec.Command(tool(t, "debugfs"), "-R", request, file).CombinedOutput() + if err != nil { + t.Fatalf("debugfs -R %q: %v\n%s", request, err, out) + } + if !strings.Contains(string(out), want) { + t.Errorf("debugfs -R %q:\n%s\nwant %q", request, out, want) + } +} + +// tool returns the path of a program of e2fsprogs, or skips the test. +func tool(t *testing.T, name string) string { + t.Helper() + bin, err := exec.LookPath(name) + if err != nil { + t.Skipf("needs %s of e2fsprogs to read the ext4 file", name) + } + return bin +} diff --git a/internal/field/field.go b/internal/field/field.go new file mode 100644 index 0000000..e6979e3 --- /dev/null +++ b/internal/field/field.go @@ -0,0 +1,52 @@ +// Package field holds the values of a change that runplane refuses, each at +// the JSON key of its field. The services return them, and the API shows +// each one at its field of the form. +package field + +import ( + "strings" + "unicode" +) + +// Problem is a value that a change cannot have. +type Problem struct { + Key string // for example "name", "dns_servers[1]", or "backup_s3.bucket" + Message string // a sentence for the user + Value any +} + +// Problems is the error of a change with one refused value or more. +type Problems []Problem + +func (p *Problems) Add(key, msg string, value any) { + *p = append(*p, Problem{Key: key, Message: msg, Value: value}) +} + +// Err returns p, or nil without problems. +func (p Problems) Err() error { + if len(p) == 0 { + return nil + } + return p +} + +func (p Problems) Error() string { + msgs := make([]string, len(p)) + for i, x := range p { + msgs[i] = x.Key + ": " + x.Message + } + return strings.Join(msgs, " ") +} + +// Sentence makes the message of an error into a sentence for the user. +func Sentence(err error) string { + msg := []rune(err.Error()) + if len(msg) == 0 { + return "" + } + msg[0] = unicode.ToUpper(msg[0]) + if msg[len(msg)-1] != '.' { + msg = append(msg, '.') + } + return string(msg) +} diff --git a/internal/fleet/auth.go b/internal/fleet/auth.go index c9b5314..1212d1f 100644 --- a/internal/fleet/auth.go +++ b/internal/fleet/auth.go @@ -8,10 +8,11 @@ import ( "crypto/subtle" "encoding/hex" "errors" + "strings" "time" "github.com/runplane/runplane/internal/agent" - "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/ids" "github.com/runplane/runplane/internal/store" ) @@ -68,7 +69,7 @@ func (f *Fleet) register(ctx context.Context, h agent.Hello) (string, string, er // A token made without a name names the server after its host. if v.Name == "" { v.Name = cmp.Or(h.Host.Hostname, "agent-"+v.ID[len(v.ID)-6:]) - if err := f.st.SetServerName(ctx, v.ID, v.Name); err != nil { + if err := f.st.SetServerName(ctx, v.ID, v.Name, now); err != nil { f.log.Warn("name the new server", "server_id", v.ID, "err", err) } } @@ -96,7 +97,7 @@ func (f *Fleet) registerAgain(ctx context.Context, hash, addr string) (string, s return "", "", errBadToken } secret := rand.Text() - if err := f.st.SetServerSecretHash(ctx, u.serverID, hashSecret(secret)); err != nil { + if err := f.st.SetServerSecretHash(ctx, u.serverID, hashSecret(secret), f.now()); err != nil { if errors.Is(err, store.ErrNotFound) { return "", "", errBadToken } @@ -128,14 +129,25 @@ func (f *Fleet) checkSecret(ctx context.Context, h agent.Hello) (string, string, return v.ID, "", nil } -// NewToken makes a one-time install token and the command that installs an -// agent with it. -func (f *Fleet) NewToken(ctx context.Context, base, name, createdBy string) (api.ServerToken, error) { - token, t, err := f.newToken(ctx, name, createdBy, "") +// Token is a one-time install token and the command that installs an agent +// with it. +type Token struct { + ID string + Token string + Command string + ExpiresAt time.Time +} + +// NewToken makes a one-time install token for a new agent. base is the URL +// of the dashboard, and an empty name means the host name of the agent. +func (f *Fleet) NewToken(ctx context.Context, by audit.Actor, base, name string) (Token, error) { + name = strings.TrimSpace(name) + token, t, err := f.newToken(ctx, name, by.UserID, "") if err != nil { - return api.ServerToken{}, err + return Token{}, err } - return api.ServerToken{ID: t.ID, Token: token, Command: f.installCommand(base, token), ExpiresAt: t.ExpiresAt}, nil + f.audit.Record(ctx, by, audit.ServerToken, t.ID, map[string]any{"name": name, "expires_at": t.ExpiresAt}) + return Token{ID: t.ID, Token: token, Command: f.installCommand(base, token), ExpiresAt: t.ExpiresAt}, nil } // newToken records a new install token. serverID is the server row that the diff --git a/internal/fleet/auth_test.go b/internal/fleet/auth_test.go index f686895..ba3fd72 100644 --- a/internal/fleet/auth_test.go +++ b/internal/fleet/auth_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/runplane/runplane/internal/agent" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" "github.com/runplane/runplane/internal/version" @@ -14,13 +15,18 @@ import ( func TestInstallToken(t *testing.T) { ctx := t.Context() h := newHarness(t, newFakeNode("arm64")) - tok, err := h.f.NewToken(ctx, "https://ci.example.com", "build-1", "") + tok, err := h.f.NewToken(ctx, audit.Actor{}, "https://ci.example.com", "build-1") if err != nil { t.Fatal(err) } if !strings.HasPrefix(tok.Token, "rp_srv_") || !tok.ExpiresAt.Equal(h.clock.now().Add(time.Hour)) { t.Errorf("token %q expires at %v, want rp_srv_... for one hour", tok.Token, tok.ExpiresAt) } + entries, _, err := h.st.ListAudit(ctx, "", 10) + if err != nil || len(entries) != 1 || entries[0].Action != "server.token" || entries[0].Target != tok.ID || + string(entries[0].Detail) != `{"expires_at":"2026-11-04T13:00:00Z","name":"build-1"}` { + t.Errorf("audit entries %+v, %v; want the server.token of build-1", entries, err) + } hello := agent.Hello{ Token: tok.Token, Version: version.Version, Mode: sandbox.ModeContainer, Host: newFakeNode("x64").info, Addr: "198.51.100.7", } @@ -67,7 +73,7 @@ func TestInstallToken(t *testing.T) { } // A token without a name names the server after its host. - unnamed, err := h.f.NewToken(ctx, "https://ci.example.com", "", "") + unnamed, err := h.f.NewToken(ctx, audit.Actor{}, "https://ci.example.com", "") if err != nil { t.Fatal(err) } @@ -80,7 +86,7 @@ func TestInstallToken(t *testing.T) { t.Errorf("server name %q of a token without a name, want the host name node-x64", v.Name) } - late, err := h.f.NewToken(ctx, "https://ci.example.com", "build-2", "") + late, err := h.f.NewToken(ctx, audit.Actor{}, "https://ci.example.com", "build-2") if err != nil { t.Fatal(err) } diff --git a/internal/fleet/fleet.go b/internal/fleet/fleet.go index 0aa0331..efeb95a 100644 --- a/internal/fleet/fleet.go +++ b/internal/fleet/fleet.go @@ -1,12 +1,13 @@ // Package fleet keeps the servers of the primary: its own server and the // agents. It records their state and host stats, makes images and runner -// bundles ready on them, installs agents, and tells the scheduler which -// servers can take sandboxes. +// bundles ready on them, installs agents, and tells which servers can take +// sandboxes. package fleet import ( "cmp" "context" + "errors" "log/slog" "maps" "slices" @@ -15,11 +16,12 @@ import ( "time" "github.com/runplane/runplane/internal/agent" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/config" "github.com/runplane/runplane/internal/events" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/notify" "github.com/runplane/runplane/internal/sandbox" + "github.com/runplane/runplane/internal/sshinstall" "github.com/runplane/runplane/internal/store" ) @@ -43,9 +45,23 @@ const ( updateTimeout = 10 * time.Minute ) -// Notifier sends an event to the notification channels that want it. -type Notifier interface { - Notify(ctx context.Context, e notify.Event) error +// ErrNotConnected means that the server has no connection to the primary. +var ErrNotConnected = errors.New("the server is not connected") + +// Runners gives the runner versions for the servers, and records the +// bundle builds of the servers: the runner versions service. +type Runners interface { + // Versions returns the runner versions to try on a server of arch, + // best first, or nil while runplane knows no version. + Versions(ctx context.Context, arch string) ([]string, error) + // Built records the build of a bundle on one server: a ready row with + // its SHA-256, or a failed row with the error. + Built(ctx context.Context, b store.RunnerBundle) +} + +// Pools makes pools: the pools service. +type Pools interface { + Create(ctx context.Context, by audit.Actor, p store.Pool) (store.Pool, error) } type Config struct { @@ -58,29 +74,46 @@ type Config struct { // AgentFingerprint is the fingerprint of the agent certificate. The // install commands and install.sh carry it. AgentFingerprint string - // OnEvent gets each driver event of each server: the HandleEvent method - // of the scheduler. + // Version is the version of runplane on the primary. The agents run + // the same version. + Version string + // Repo is the GitHub repository of the runplane releases. + Repo string + // PublicKey is the release signing key of this binary, or "". The + // install script of the primary carries it. + PublicKey string + // OnEvent gets each driver event of each server. OnEvent func(serverID string, ev sandbox.Event) // CapacityChanged runs when a server comes, goes, changes its status, // or gets its first image or runner: the CapacityChanged method of the // scheduler. CapacityChanged func() - Notifier Notifier - Now func() time.Time // nil: time.Now + Runners Runners + // Pools adds the default label at the first start. + Pools Pools + Now func() time.Time // nil: time.Now } -// Fleet is the set of servers of the primary. type Fleet struct { st *store.Store bus *events.Bus log *slog.Logger + audit audit.Log cfg config.Config local agent.Node fp string + version string + repo string + publicKey string onEvent func(string, sandbox.Event) capacityChanged func() - notifier Notifier + runners Runners + pools Pools now func() time.Time + knownHosts *sshinstall.KnownHosts + + // settings gets each settings change. Run reads it. + settings <-chan events.SettingsChanged // bg is the context of the work that outlives a call. Run cancels it. bg context.Context @@ -89,34 +122,41 @@ type Fleet struct { wg sync.WaitGroup wake chan struct{} // asks Run to provision now - keyMu sync.Mutex // the SSH key and known_hosts - mu sync.Mutex - localID string localAddr string // IPv4 of the primary, for install commands nodes map[string]*node // connected servers by ID downSince map[string]time.Time // agents without a connection - removed map[string]bool // servers that Remove disconnected + removed map[string]bool // servers that RemoveServer disconnected installs map[string]context.CancelFunc tokens map[string]usedToken // recent token uses by hash images map[string]store.Image defaultID string - dropped map[string]bool // images that Images.Remove dropped + dropped map[string]bool // images that RemoveImage dropped updates map[string]agentUpdate // the last update of each agent, by server ID - latest string // newest runner version - minFree float64 // min_free_disk_percent - diskLow map[string]bool - samples []store.ServerStat // host stats for the next flush + samples []store.ServerStat // host stats for the next flush + seen map[string]time.Time // last sample of each agent, for the next flush + + // settingsChanges counts the settings changes that settingsChanged + // sent to the connected servers. + settingsChanges int } // node is a connected server. type node struct { agent.Node id string - sess *agent.Session // nil for the server of the primary arch string addr string // agents: the IP address that the primary sees version string // of runplane on the server + limits sandbox.Limits + + // gcMu prevents an image GC on the server during its pulls. A pull + // holds a read lock until the build row of the server records its + // image, and gcImages holds the write lock. Thus a GC keeps each image + // that a pull made. + gcMu sync.RWMutex + // settingsMu makes the settings sends to the server one at a time. + settingsMu sync.Mutex // Guarded by Fleet.mu. name string @@ -135,19 +175,28 @@ type readyImage struct { at time.Time // when the pull ended } +// New makes the fleet. It takes the settings changes from the bus at once, +// also the changes before Run starts. func New(c Config) *Fleet { bg, stop := context.WithCancel(context.Background()) f := &Fleet{ st: c.Store, bus: c.Bus, log: c.Log, + audit: audit.Log{Store: c.Store, Logger: c.Log}, cfg: c.Config, local: c.Local, fp: c.AgentFingerprint, + version: c.Version, + repo: c.Repo, + publicKey: c.PublicKey, onEvent: c.OnEvent, capacityChanged: c.CapacityChanged, - notifier: c.Notifier, + runners: c.Runners, + pools: c.Pools, now: c.Now, + knownHosts: &sshinstall.KnownHosts{Path: c.Config.KnownHostsPath()}, + settings: events.Subscribe[events.SettingsChanged](bg, c.Bus), bg: bg, stopBg: stop, wake: make(chan struct{}, 1), @@ -159,7 +208,7 @@ func New(c Config) *Fleet { images: map[string]store.Image{}, dropped: map[string]bool{}, updates: map[string]agentUpdate{}, - diskLow: map[string]bool{}, + seen: map[string]time.Time{}, } if f.now == nil { f.now = time.Now @@ -169,33 +218,20 @@ func New(c Config) *Fleet { // Run records the server of the primary, then keeps the state of all // servers until ctx ends: their status, facts, host stats, images, and -// runner bundles. Start it before the agent listener. +// runner bundles. When it ends, it writes the host samples that wait for +// the next flush. func (f *Fleet) Run(ctx context.Context) error { - defer f.stop() - st, err := f.st.GetSettings(ctx) - if err != nil { - return err - } - f.mu.Lock() - f.minFree = st.MinFreeDiskPercent - f.mu.Unlock() - firstStart, err := f.seedImages(ctx) - if err != nil { - return err - } - if err := f.startLocal(ctx); err != nil { - return err - } - if firstStart { - if err := f.seedPool(ctx, st); err != nil { - return err + defer func() { + // The background work adds samples until stop returns. + f.stop() + f.flushStats(context.WithoutCancel(ctx)) + }() + if err := f.start(ctx); err != nil { + if ctx.Err() != nil { + return nil } - } - if err := f.startAgents(ctx); err != nil { return err } - bus, cancel := f.bus.Subscribe(64) - defer cancel() f.provision(ctx) check := time.NewTicker(syncEvery) @@ -209,10 +245,9 @@ func (f *Fleet) Run(ctx context.Context) error { for { select { case <-ctx.Done(): - f.flushStats(context.WithoutCancel(ctx)) return nil - case e := <-bus: - f.handleBus(e) + case c := <-f.settings: + f.settingsChanged(c) case <-check.C: f.syncServers(ctx) case <-flush.C: @@ -227,6 +262,26 @@ func (f *Fleet) Run(ctx context.Context) error { } } +func (f *Fleet) start(ctx context.Context) error { + st, err := f.st.GetSettings(ctx) + if err != nil { + return err + } + firstStart, err := f.seedImages(ctx) + if err != nil { + return err + } + if err := f.startLocal(ctx); err != nil { + return err + } + if firstStart { + if err := f.seedPool(ctx, st); err != nil { + return err + } + } + return f.startAgents(ctx) +} + // stop ends the background work and waits for it. func (f *Fleet) stop() { f.bgMu.Lock() @@ -253,17 +308,25 @@ func (f *Fleet) kick() { } } -// Online returns the connected servers with the status online. -func (f *Fleet) Online() []sandbox.Host { +// Host is a connected server. +type Host struct { + ServerID string + Name string + Arch string + Driver agent.Driver +} + +// Online returns the connected servers with the status online, by name. +func (f *Fleet) Online() []Host { f.mu.Lock() defer f.mu.Unlock() - var hosts []sandbox.Host + var hosts []Host for _, n := range f.nodes { if n.status == store.ServerOnline { - hosts = append(hosts, sandbox.Host{ServerID: n.id, Name: n.name, Arch: n.arch, Driver: n.Driver()}) + hosts = append(hosts, Host{ServerID: n.id, Name: n.name, Arch: n.arch, Driver: n.Driver()}) } } - slices.SortFunc(hosts, func(a, b sandbox.Host) int { + slices.SortFunc(hosts, func(a, b Host) int { return cmp.Or(strings.Compare(a.Name, b.Name), strings.Compare(a.ServerID, b.ServerID)) }) return hosts @@ -271,7 +334,7 @@ func (f *Fleet) Online() []sandbox.Host { // Driver returns the driver of a connected server, also a draining or // disabled one, or nil. -func (f *Fleet) Driver(serverID string) sandbox.Driver { +func (f *Fleet) Driver(serverID string) agent.Driver { if n := f.node(serverID); n != nil { return n.Driver() } @@ -307,6 +370,31 @@ func (f *Fleet) RunnerVersion(serverID string) (string, bool) { return "", false } +// RunnerVersions returns the runner versions that the online servers give +// new sandboxes. +func (f *Fleet) RunnerVersions() []string { + f.mu.Lock() + defer f.mu.Unlock() + var list []string + for _, n := range f.nodes { + if n.status == store.ServerOnline && n.runner != "" && !slices.Contains(list, n.runner) { + list = append(list, n.runner) + } + } + return list +} + +// Limits returns the limits of the driver of a connected server. The fleet +// reads them when the server connects. +func (f *Fleet) Limits(serverID string) (sandbox.Limits, bool) { + f.mu.Lock() + defer f.mu.Unlock() + if n := f.nodes[serverID]; n != nil { + return n.limits, true + } + return sandbox.Limits{}, false +} + func (f *Fleet) node(id string) *node { f.mu.Lock() defer f.mu.Unlock() @@ -319,15 +407,27 @@ func (f *Fleet) connected() []*node { return slices.Collect(maps.Values(f.nodes)) } +// errRemoved refuses the connection of a server that RemoveServer removed. +var errRemoved = errors.New("the server was removed") + // add makes n the connected node of its server and passes its driver // events and host stats on. The channels of an agent close when its -// session ends. -func (f *Fleet) add(n *node) { +// session ends. A server that RemoveServer removed gets no node. If the +// settings changed after settingsChanges, the server gets them again. +func (f *Fleet) add(n *node, settingsChanges int) error { n.images, n.pulls, n.failed = map[string]readyImage{}, map[string]string{}, map[string]time.Time{} f.mu.Lock() + if f.removed[n.id] { + f.mu.Unlock() + return errRemoved + } f.nodes[n.id] = n delete(f.downSince, n.id) + missed := f.settingsChanges != settingsChanges f.mu.Unlock() + if missed { + f.goBg(func(ctx context.Context) { f.sendSettings(ctx, n) }) + } f.goBg(func(ctx context.Context) { onEvent := func(ev sandbox.Event) { f.onEvent(n.id, ev) } onStats := func(s host.Stats) { f.addStats(n, s) } @@ -335,6 +435,7 @@ func (f *Fleet) add(n *node) { f.down(ctx, n) } }) + return nil } // down drops the session of an agent that ended, unless a newer session @@ -363,22 +464,7 @@ func (f *Fleet) publishServer(ctx context.Context, id string) { f.log.Debug("read server for its event", "server_id", id, "err", err) return } - f.bus.Publish(events.Event{Type: events.ServerState, Data: v}) -} - -// handleBus follows the changes that the API makes. -func (f *Fleet) handleBus(e events.Event) { - switch e.Type { - case events.ServerState: - if v, ok := events.As[store.Server](e.Data); ok { - f.serverChanged(v) - } - case events.ImageState: - // The API sends the image row after a new image or a new default. - if _, ok := events.As[store.Image](e.Data); ok { - f.kick() - } - } + f.bus.Publish(events.ServerState(v)) } // serverChanged takes the name and the status of a server row for its @@ -400,17 +486,3 @@ func (f *Fleet) serverChanged(v store.Server) { f.kick() } } - -// notify sends an event to the notification channels in the background. -func (f *Fleet) notify(build func(dashboard string) notify.Event) { - f.goBg(func(ctx context.Context) { - st, err := f.st.GetSettings(ctx) - if err != nil { - f.log.Warn("read settings for a notification", "err", err) - } - e := build(st.DashboardURL()) - if err := f.notifier.Notify(ctx, e); err != nil { - f.log.Warn("send notification", "type", e.Type, "err", err) - } - }) -} diff --git a/internal/fleet/helpers_test.go b/internal/fleet/helpers_test.go index 5178d81..6d5563f 100644 --- a/internal/fleet/helpers_test.go +++ b/internal/fleet/helpers_test.go @@ -17,15 +17,17 @@ import ( "time" "github.com/runplane/runplane/internal/agent" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/config" + "github.com/runplane/runplane/internal/doctor" "github.com/runplane/runplane/internal/events" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/image" - "github.com/runplane/runplane/internal/mirror" - "github.com/runplane/runplane/internal/notify" + "github.com/runplane/runplane/internal/pools" + "github.com/runplane/runplane/internal/runner/versions" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" "github.com/runplane/runplane/internal/tlsutil" + "github.com/runplane/runplane/internal/version" ) // harness is a primary with a fleet, a real agent listener, a fake local @@ -36,14 +38,18 @@ type harness struct { bus *events.Bus f *Fleet clock *clock - notes notes capacity atomic.Int32 // CapacityChanged calls fp string cfg config.Config + version string // of runplane on the primary and on the agents that start log *slog.Logger + // stop ends Fleet.Run and waits for it. + stop func() } -func newHarness(t *testing.T, local *fakeNode) *harness { +// newHarness makes the harness. The primary runs the version "dev", and +// opts can change the fleet config. +func newHarness(t *testing.T, local *fakeNode, opts ...func(*Config)) *harness { t.Helper() dir := t.TempDir() st, err := store.Open(filepath.Join(dir, "runplane.db")) @@ -60,14 +66,22 @@ func newHarness(t *testing.T, local *fakeNode) *harness { t.Fatal(err) } h := &harness{ - t: t, st: st, bus: &events.Bus{}, notes: make(notes, 16), fp: fp, log: testLogger(t), + t: t, st: st, bus: &events.Bus{}, fp: fp, log: testLogger(t), cfg: config.Config{AgentListen: ln.Addr().String(), DataDir: dir, ConfigDir: dir}, clock: &clock{}, } h.clock.ns.Store(time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC).UnixNano()) - h.f = New(Config{ + c := Config{ Store: st, Bus: h.bus, Log: h.log.With("side", "fleet"), Config: h.cfg, Local: local, AgentFingerprint: fp, - OnEvent: func(string, sandbox.Event) {}, CapacityChanged: func() { h.capacity.Add(1) }, Notifier: h.notes, Now: h.clock.now, - }) + Version: "dev", Repo: version.Repo, + OnEvent: func(string, sandbox.Event) {}, CapacityChanged: func() { h.capacity.Add(1) }, Now: h.clock.now, + Runners: versions.New(versions.Config{Store: st, Bus: h.bus}), + Pools: pools.New(pools.Config{Store: st, Bus: h.bus}), + } + for _, o := range opts { + o(&c) + } + h.version = c.Version + h.f = New(c) ctx, cancel := context.WithCancel(context.Background()) l := &agent.Listener{Cert: cert, Authenticate: h.f.Authenticate, OnSession: h.f.AddSession, Logger: h.log.With("side", "listener")} served := make(chan struct{}) @@ -88,19 +102,29 @@ func (h *harness) run() { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- h.f.Run(ctx) }() - h.t.Cleanup(func() { - cancel() - if err := <-done; err != nil { - h.t.Errorf("Run: %v", err) - } + var once sync.Once + h.stop = func() { + once.Do(func() { + cancel() + if err := <-done; err != nil { + h.t.Errorf("Run: %v", err) + } + }) + } + h.t.Cleanup(h.stop) + waitFor(h.t, "the local server", func() bool { + v, err := h.st.GetLocalServer(context.Background()) + return err == nil && h.f.Driver(v.ID) != nil }) - waitFor(h.t, "the local server", func() bool { return h.f.Driver(h.localID()) != nil }) } func (h *harness) localID() string { - h.f.mu.Lock() - defer h.f.mu.Unlock() - return h.f.localID + h.t.Helper() + v, err := h.st.GetLocalServer(context.Background()) + if err != nil { + h.t.Fatal(err) + } + return v.ID } func (h *harness) server(id string) store.Server { @@ -116,7 +140,7 @@ func (h *harness) server(id string) store.Server { // its session. func (h *harness) connectAgent(name string, node *fakeNode) (*testAgent, string) { h.t.Helper() - tok, err := h.f.NewToken(context.Background(), "https://ci.example.com", name, "") + tok, err := h.f.NewToken(context.Background(), audit.Actor{}, "https://ci.example.com", name) if err != nil { h.t.Fatal(err) } @@ -130,11 +154,28 @@ func (h *harness) connectAgent(name string, node *fakeNode) (*testAgent, string) return a, id } +// startGatedAgent starts a new agent whose Configure waits until the test +// closes the returned gate, so that the agent stays in its connect. It +// returns when the agent got the settings of its connect. +func (h *harness) startGatedAgent(name string, node *fakeNode) (*testAgent, chan struct{}) { + h.t.Helper() + gate := make(chan struct{}) + node.configureGate = gate + tok, err := h.f.NewToken(h.t.Context(), audit.Actor{}, "https://ci.example.com", name) + if err != nil { + h.t.Fatal(err) + } + a := h.startAgent(map[string]string{"primary": h.cfg.AgentListen, "fingerprint": h.fp, "token": tok.Token}, node) + waitFor(h.t, "the settings at connect", func() bool { return len(node.args("Configure")) == 1 }) + return a, gate +} + type testAgent struct { - h *harness - path string - node *fakeNode - stop func() + h *harness + path string + node *fakeNode + version string // that the agent sends in its hello + stop func() } func (h *harness) startAgent(file map[string]string, node *fakeNode) *testAgent { @@ -143,7 +184,7 @@ func (h *harness) startAgent(file map[string]string, node *fakeNode) *testAgent if err := os.WriteFile(path, b, 0o600); err != nil { h.t.Fatal(err) } - a := &testAgent{h: h, path: path, node: node} + a := &testAgent{h: h, path: path, node: node, version: h.version} a.start() return a } @@ -153,7 +194,7 @@ func (a *testAgent) start() { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { - agent.Run(ctx, agent.Config{Path: a.path, Node: a.node, Logger: a.h.log.With("side", "agent"), Mirror: func() mirror.Stats { return a.node.mirror }}) + agent.Run(ctx, agent.Config{Path: a.path, Node: a.node, Version: a.version, Logger: a.h.log.With("side", "agent")}) close(done) }() var once sync.Once @@ -166,6 +207,16 @@ func (a *testAgent) start() { a.h.t.Cleanup(a.stop) } +// restart stops the agent, waits until the fleet drops its session, and +// starts the agent with the version. +func (a *testAgent) restart(version string) { + a.stop() + id := a.file()["server_id"] + waitFor(a.h.t, "the end of the session", func() bool { return a.h.f.Driver(id) == nil }) + a.version = version + a.start() +} + func (a *testAgent) file() map[string]string { var file map[string]string b, err := os.ReadFile(a.path) @@ -201,7 +252,7 @@ func recv[T any](t *testing.T, ch <-chan T) T { } // waitEvent reads ch until an event matches. -func waitEvent(t *testing.T, ch <-chan events.Event, match func(events.Event) bool) { +func waitEvent[E any](t *testing.T, ch <-chan E, match func(E) bool) { t.Helper() for { if match(recv(t, ch)) { @@ -216,28 +267,14 @@ type clock struct{ ns atomic.Int64 } func (c *clock) now() time.Time { return time.Unix(0, c.ns.Load()).UTC() } func (c *clock) add(d time.Duration) { c.ns.Add(int64(d)) } -// notes gets the notifications. -type notes chan notify.Event - -func (n notes) Notify(_ context.Context, e notify.Event) error { - select { - case n <- e: - default: - } - return nil -} - -// noDriver leaves out the driver methods that the fleet does not call. -type noDriver = sandbox.Driver - -// fakeNode is a server and its driver. The test chooses the images and -// runner versions that work, and the node records its calls. +// fakeNode is a server, and fakeDriver its driver. The test chooses the +// images and runner versions that work, and the node records its calls. type fakeNode struct { - noDriver info host.Info events chan sandbox.Event stats chan host.Stats - mirror mirror.Stats // what an agent with this node reports + mirror agent.MirrorStats + limits sandbox.Limits mu sync.Mutex sandboxes map[string]sandbox.Sandbox @@ -246,6 +283,9 @@ type fakeNode struct { gates map[string]chan struct{} // pulls of these refs wait until the channel closes updateErr error calls []call + // configureGate and updateGate, if not nil, make Configure and Update + // wait until they close. + configureGate, updateGate chan struct{} } type call struct { @@ -286,7 +326,6 @@ func (n *fakeNode) record(method string, arg any) { n.calls = append(n.calls, call{method, arg}) } -// args returns the argument of each call of method. func (n *fakeNode) args(method string) []any { n.mu.Lock() defer n.mu.Unlock() @@ -299,17 +338,32 @@ func (n *fakeNode) args(method string) []any { return list } -func (n *fakeNode) Driver() sandbox.Driver { return n } -func (n *fakeNode) Facts(context.Context) (host.Info, error) { return n.info, nil } -func (n *fakeNode) Stats() <-chan host.Stats { return n.stats } +func (n *fakeNode) Local() bool { return true } +func (n *fakeNode) Driver() agent.Driver { return fakeDriver{n: n} } +func (n *fakeNode) Stats() <-chan host.Stats { return n.stats } +func (n *fakeNode) Close() error { return nil } + +// Facts records its calls: an agent reads the facts before each connection. +func (n *fakeNode) Facts(context.Context) (host.Info, error) { + n.record("Facts", nil) + return n.info, nil +} + +func (n *fakeNode) MirrorStats(context.Context) (agent.MirrorStats, error) { return n.mirror, nil } -func (n *fakeNode) Doctor(context.Context) ([]host.Check, error) { - return []host.Check{{ID: "kernel", Name: "Kernel", Status: host.StatusOK, Detail: n.info.Kernel}}, nil +func (n *fakeNode) Doctor(context.Context) (doctor.Report, error) { + return doctor.Report{Info: n.info, Checks: []doctor.Check{ + doctor.ModeCheck.OK("Test mode"), + {ID: "kernel", Name: "Kernel", Status: doctor.StatusOK, Detail: n.info.Kernel}, + }}, nil } -func (n *fakeNode) EnsureImage(ctx context.Context, ref string, progress func(image.Progress)) (sandbox.ImageRef, error) { +// imageSize is the size of each image on a fake server. +const imageSize = 734003200 + +func (n *fakeNode) EnsureImage(ctx context.Context, ref string, progress func(agent.Progress)) (agent.Image, error) { n.record("EnsureImage", ref) - progress(image.Progress{Status: image.StatusPulling, Total: 4096}) + progress(agent.Progress{Status: agent.ImagePulling, Total: 4096}) n.mu.Lock() gate := n.gates[ref] n.mu.Unlock() @@ -317,26 +371,26 @@ func (n *fakeNode) EnsureImage(ctx context.Context, ref string, progress func(im select { case <-gate: case <-ctx.Done(): - return sandbox.ImageRef{}, ctx.Err() + return agent.Image{}, ctx.Err() } } n.mu.Lock() defer n.mu.Unlock() digest, ok := n.digests[ref] if !ok { - return sandbox.ImageRef{}, fmt.Errorf("pull %s: not found", ref) + return agent.Image{}, fmt.Errorf("pull %s: not found", ref) } - return sandbox.ImageRef{Ref: ref, Digest: digest}, nil + return agent.Image{ImageRef: sandbox.ImageRef{Ref: ref, Digest: digest}, SizeBytes: imageSize}, nil } -func (n *fakeNode) EnsureRunner(_ context.Context, version, arch string) (image.Bundle, error) { +func (n *fakeNode) EnsureRunner(_ context.Context, version, arch string) (agent.Bundle, error) { n.record("EnsureRunner", version) n.mu.Lock() defer n.mu.Unlock() if !n.runners[version] { - return image.Bundle{}, fmt.Errorf("download runner %s: 404 Not Found", version) + return agent.Bundle{}, fmt.Errorf("download runner %s: 404 Not Found", version) } - return image.Bundle{Version: version, Arch: arch, SHA256: "9e2f" + version, Patched: true, Dir: "/var/lib/runplane/runners/" + version + "-" + arch}, nil + return agent.Bundle{Version: version, Arch: arch, SHA256: "9e2f" + version, Patched: true}, nil } func (n *fakeNode) GCImages(_ context.Context, keep []string) error { @@ -349,32 +403,64 @@ func (n *fakeNode) PruneRunners(_ context.Context, keep []string) error { return nil } -func (n *fakeNode) Configure(_ context.Context, s sandbox.Settings) error { +func (n *fakeNode) Configure(ctx context.Context, s agent.Settings) error { n.record("Configure", s) - return nil + n.mu.Lock() + gate := n.configureGate + n.mu.Unlock() + return wait(ctx, gate) } -func (n *fakeNode) Update(_ context.Context, version string) error { +func (n *fakeNode) Update(ctx context.Context, version string) error { n.record("Update", version) n.mu.Lock() + gate := n.updateGate + n.mu.Unlock() + if err := wait(ctx, gate); err != nil { + return err + } + n.mu.Lock() defer n.mu.Unlock() return n.updateErr } -func (n *fakeNode) Mode() sandbox.Mode { return sandbox.ModeContainer } -func (n *fakeNode) Events() <-chan sandbox.Event { return n.events } +// wait waits until gate closes, if it is not nil. +func wait(ctx context.Context, gate chan struct{}) error { + if gate == nil { + return nil + } + select { + case <-gate: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} -func (n *fakeNode) List(context.Context) ([]sandbox.Sandbox, error) { - n.mu.Lock() - defer n.mu.Unlock() - return slices.SortedFunc(maps.Values(n.sandboxes), func(a, b sandbox.Sandbox) int { return strings.Compare(a.ID, b.ID) }), nil +// fakeDriver leaves out the driver methods that the fleet does not call. +type fakeDriver struct { + agent.Driver + n *fakeNode } -func (n *fakeNode) Stop(_ context.Context, id string) error { - n.record("Stop", id) - n.mu.Lock() - defer n.mu.Unlock() - delete(n.sandboxes, id) +func (d fakeDriver) Mode() sandbox.Mode { return sandbox.ModeContainer } +func (d fakeDriver) Events() <-chan sandbox.Event { return d.n.events } + +func (d fakeDriver) Capacity(context.Context) (sandbox.Capacity, error) { + return sandbox.Capacity{Mode: sandbox.ModeContainer, Arch: d.n.info.Arch, Limits: d.n.limits}, nil +} + +func (d fakeDriver) List(context.Context) ([]sandbox.Sandbox, error) { + d.n.mu.Lock() + defer d.n.mu.Unlock() + return slices.SortedFunc(maps.Values(d.n.sandboxes), func(a, b sandbox.Sandbox) int { return strings.Compare(a.ID, b.ID) }), nil +} + +func (d fakeDriver) Stop(_ context.Context, id string) error { + d.n.record("Stop", id) + d.n.mu.Lock() + defer d.n.mu.Unlock() + delete(d.n.sandboxes, id) return nil } diff --git a/internal/fleet/images.go b/internal/fleet/images.go new file mode 100644 index 0000000..ad77b3c --- /dev/null +++ b/internal/fleet/images.go @@ -0,0 +1,175 @@ +package fleet + +import ( + "context" + "errors" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + + "github.com/runplane/runplane/images" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/field" + "github.com/runplane/runplane/internal/ids" + "github.com/runplane/runplane/internal/store" +) + +// ErrDefaultImage refuses the removal of the default image. +var ErrDefaultImage = errors.New("the default image cannot be removed; make another image the default first") + +// CreateImage adds an image from a registry, and pulls it on each connected +// server. A refused value gives field.Problems, and a ref in use +// store.ErrConflict. +func (f *Fleet) CreateImage(ctx context.Context, by audit.Actor, imgName, ref string) (store.Image, error) { + imgName, ref, err := checkImage(imgName, ref) + if err != nil { + return store.Image{}, err + } + now := f.now() + img := store.Image{ID: ids.New(), Name: imgName, Ref: ref, Source: "registry", CreatedAt: now, UpdatedAt: now} + if err := f.st.CreateImage(ctx, img); err != nil { + return store.Image{}, err + } + f.audit.Record(ctx, by, audit.ImageCreate, img.ID, map[string]any{"name": img.Name, "ref": img.Ref}) + f.bus.Publish(events.ImageState(img)) + f.pullEverywhere(img) + return img, nil +} + +// ImageChange is a change of an image by an admin. A nil field keeps its +// value. +type ImageChange struct { + Name *string + Ref *string // a new ref pulls the image again + // Default true makes the image the default. To change the default, an + // admin makes another image the default. + Default *bool +} + +func (f *Fleet) UpdateImage(ctx context.Context, by audit.Actor, imageID string, c ImageChange) (store.Image, error) { + img, err := f.st.GetImage(ctx, imageID) + if err != nil { + return store.Image{}, err + } + var p field.Problems + switch { + case c.Default != nil && !*c.Default && img.IsDefault: + p.Add("is_default", "Make another image the default instead.", false) + case c.Ref != nil && *c.Ref != img.Ref && img.Source == images.Source: + p.Add("ref", "The ref of a built-in image cannot change. Add a new image instead.", *c.Ref) + } + if err := p.Err(); err != nil { + return store.Image{}, err + } + imgName, ref := img.Name, img.Ref + if c.Name != nil { + imgName = *c.Name + } + if c.Ref != nil { + ref = *c.Ref + } + if imgName, ref, err = checkImage(imgName, ref); err != nil { + return store.Image{}, err + } + refChanged := ref != img.Ref + if imgName != img.Name || refChanged { + img.Name, img.Ref, img.UpdatedAt = imgName, ref, f.now() + if err := f.st.UpdateImage(ctx, img); err != nil { + return store.Image{}, err + } + } + if c.Default != nil && *c.Default && !img.IsDefault { + if err := f.st.SetDefaultImage(ctx, img.ID, f.now()); err != nil { + return store.Image{}, err + } + img.IsDefault = true + // The pools without an image use the default image. + f.bus.Publish(events.PoolsChanged{}) + } + f.audit.Record(ctx, by, audit.ImageUpdate, img.ID, map[string]any{"name": img.Name, "ref": img.Ref, "is_default": c.Default}) + f.bus.Publish(events.ImageState(img)) + f.kick() + if refChanged { + f.pullEverywhere(img) + } + return img, nil +} + +// RemoveImage stops the use of an image and deletes its row. The pools with +// the image change to the default image. The connected servers remove the +// files of the image in the background, because a server removes images +// only after its pulls end. +func (f *Fleet) RemoveImage(ctx context.Context, by audit.Actor, imageID string) error { + img, err := f.st.GetImage(ctx, imageID) + if err != nil { + return err + } + if img.IsDefault { + return ErrDefaultImage + } + f.mu.Lock() + f.dropped[img.ID] = true + delete(f.images, img.ID) + var nodes []*node + for _, n := range f.nodes { + delete(n.images, img.ID) + delete(n.failed, img.ID) + nodes = append(nodes, n) + } + f.mu.Unlock() + if err := f.st.DeleteImage(ctx, img.ID); err != nil { + return err + } + for _, n := range nodes { + f.goBg(func(ctx context.Context) { f.gcImages(ctx, n) }) + } + f.bus.Publish(events.PoolsChanged{}) + f.bus.Publish(events.ImageRemoved{ID: img.ID}) + f.audit.Record(ctx, by, audit.ImageDelete, img.ID, map[string]any{"name": img.Name, "ref": img.Ref}) + return nil +} + +// RefreshImage pulls the image again on each connected server. +func (f *Fleet) RefreshImage(ctx context.Context, by audit.Actor, imageID string) error { + img, err := f.st.GetImage(ctx, imageID) + if err != nil { + return err + } + f.pullEverywhere(img) + f.audit.Record(ctx, by, audit.ImageRefresh, img.ID, map[string]any{"ref": img.Ref}) + return nil +} + +// After pullEverywhere, the servers keep the image that they have until the +// new pull is ready, unless its ref changed. The builds show a failure. +func (f *Fleet) pullEverywhere(img store.Image) { + var list []*node + f.mu.Lock() + f.images[img.ID] = img + if img.IsDefault { + f.defaultID = img.ID + } + for _, n := range f.nodes { + if n.pulls[img.ID] != img.Ref { + n.pulls[img.ID] = img.Ref + list = append(list, n) + } + } + f.mu.Unlock() + for _, n := range list { + f.goBg(func(ctx context.Context) { f.pull(ctx, n, img) }) + } +} + +func checkImage(imgName, ref string) (string, string, error) { + imgName, ref = strings.TrimSpace(imgName), strings.TrimSpace(ref) + var p field.Problems + if imgName == "" { + p.Add("name", "Enter a name.", imgName) + } + if _, err := name.ParseReference(ref); err != nil { + p.Add("ref", "The ref is not a valid image reference, for example ghcr.io/owner/image:tag.", ref) + } + return imgName, ref, p.Err() +} diff --git a/internal/fleet/install.go b/internal/fleet/install.go index 3ca7f1e..083e7d1 100644 --- a/internal/fleet/install.go +++ b/internal/fleet/install.go @@ -4,30 +4,22 @@ import ( "bytes" "cmp" "context" - "errors" "fmt" - "io" - "io/fs" "net" - "net/http" "net/url" - "os" - "path/filepath" - "regexp" - "slices" "strconv" "strings" "time" "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/knownhosts" "github.com/runplane/runplane" - "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/field" "github.com/runplane/runplane/internal/ids" "github.com/runplane/runplane/internal/sshinstall" + "github.com/runplane/runplane/internal/sshkey" "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/update" "github.com/runplane/runplane/internal/version" ) @@ -35,32 +27,33 @@ import ( // connect after the install script ends. const connectWait = time.Minute -// releaseVersion matches the versions that have a release to install. -var releaseVersion = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+$`) +// StepPrefix starts the line of each step in the output of install.sh. +const StepPrefix = "==> " // Script returns install.sh with the values of this primary: the release -// repository, the agent address for the host of the request, the -// certificate fingerprint, the version of this binary if it is a release, -// and the release signing key if the binary has one. -func (f *Fleet) Script(r *http.Request) []byte { - return f.script(f.agentAddr(hostname(r.Host))) +// repository, the agent address for the agents that reach the primary at +// host (host[:port] of the request), the certificate fingerprint, the +// version of this binary if it is a release, and the release signing key if +// the binary has one. +func (f *Fleet) Script(host string) []byte { + return f.script(f.agentAddr(hostname(host))) } func (f *Fleet) script(primary string) []byte { vars := [][2]string{ - {"RUNPLANE_REPO", version.Repo}, + {"RUNPLANE_REPO", f.repo}, {"RUNPLANE_PRIMARY", primary}, {"RUNPLANE_FINGERPRINT", f.fp}, } // An agent gets the release of its primary, so that the two speak the // same protocol. - if releaseVersion.MatchString(version.Version) { - vars = append(vars, [2]string{"RUNPLANE_VERSION", version.Version}) + if version.IsRelease(f.version) { + vars = append(vars, [2]string{"RUNPLANE_VERSION", f.version}) } // The key comes from this binary, not from the release: someone who can // change the release files cannot change it. - if update.PublicKey != "" { - vars = append(vars, [2]string{"RUNPLANE_PUBLIC_KEY", update.PublicKey}) + if f.publicKey != "" { + vars = append(vars, [2]string{"RUNPLANE_PUBLIC_KEY", f.publicKey}) } first, rest, _ := bytes.Cut(runplane.InstallScript, []byte("\n")) var b bytes.Buffer @@ -88,11 +81,11 @@ func (f *Fleet) installCommand(base, token string) string { script := strings.TrimSuffix(u.String(), "/") + "/install.sh" return fmt.Sprintf("curl -fsSL %s | sudo sh -s -- %s", shellQuote(script), flags) } - script := "https://github.com/" + version.Repo + "/releases/latest/download/install.sh" - release := "--repo " + shellQuote(version.Repo) - if releaseVersion.MatchString(version.Version) { - script = "https://github.com/" + version.Repo + "/releases/download/" + version.Version + "/install.sh" - release += " --version " + shellQuote(version.Version) + script := "https://github.com/" + f.repo + "/releases/latest/download/install.sh" + release := "--repo " + shellQuote(f.repo) + if version.IsRelease(f.version) { + script = "https://github.com/" + f.repo + "/releases/download/" + f.version + "/install.sh" + release += " --version " + shellQuote(f.version) } return fmt.Sprintf("curl -fsSL %s | sudo sh -s -- %s %s", shellQuote(script), release, flags) } @@ -104,10 +97,7 @@ func agentFlags(token, primary, fingerprint string) string { // agentAddr is the address of the agent listener for agents that reach the // primary at host. func (f *Fleet) agentAddr(host string) string { - _, port, err := net.SplitHostPort(f.cfg.AgentListen) - if err != nil || port == "" { - port = "8485" - } + _, port, _ := net.SplitHostPort(f.cfg.AgentListen) return net.JoinHostPort(f.publicAddr(host), port) } @@ -143,15 +133,36 @@ func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } +// SSHInstall is a new server that InstallSSH installs over SSH. +type SSHInstall struct { + Name string + Host string // host name or IP address + Port int // 0 means 22 + User string // "" means root; another user needs sudo without a password +} + // InstallSSH makes a server row in the status installing and installs the -// agent over SSH in the background. The status detail shows the output of -// the install script. -func (f *Fleet) InstallSSH(ctx context.Context, base string, req api.SSHInstall) (string, error) { +// agent over SSH in the background. It returns the ID of the new server. +// The status detail shows the output of the install script. A name or a +// host that is not valid gives field.Problems. +func (f *Fleet) InstallSSH(ctx context.Context, by audit.Actor, base string, req SSHInstall) (string, error) { + name, host := strings.TrimSpace(req.Name), strings.TrimSpace(req.Host) + var p field.Problems + if name == "" { + p.Add("name", "Enter a name.", req.Name) + } + if host == "" || strings.ContainsAny(host, " /@") { + p.Add("host", "Enter a host name or an IP address.", req.Host) + } + if err := p.Err(); err != nil { + return "", err + } + req.Name, req.Host, req.Port, req.User = name, host, cmp.Or(req.Port, 22), cmp.Or(req.User, "root") signer, _, err := f.sshKey() if err != nil { return "", err } - t := sshinstall.Target{Addr: net.JoinHostPort(req.Host, strconv.Itoa(cmp.Or(req.Port, 22))), User: cmp.Or(req.User, "root")} + t := sshinstall.Target{Addr: net.JoinHostPort(req.Host, strconv.Itoa(req.Port)), User: req.User} id := ids.New() // Register the install before its row exists: Remove cancels it, and // the start of Run does not mark it as stopped. @@ -178,10 +189,13 @@ func (f *Fleet) InstallSSH(ctx context.Context, base string, req api.SSHInstall) token, _, err := f.newToken(ctx, req.Name, "", id) if err != nil { done() - f.st.DeleteServer(ctx, id) + if derr := f.st.DeleteServer(ctx, id, f.now()); derr != nil { + f.log.Warn("delete the server of a failed install", "server_id", id, "err", derr) + } return "", err } f.publishServer(ctx, id) + f.audit.Record(ctx, by, audit.ServerInstall, id, map[string]any{"name": req.Name, "host": req.Host, "port": req.Port, "user": req.User}) job := installJob{id: id, target: t, signer: signer, primary: f.agentAddr(hostOf(base))} job.command = "sh -s -- " + agentFlags(token, job.primary, f.fp) @@ -218,16 +232,10 @@ func hostOf(base string) string { // agent to connect. The status detail shows the latest line of the script, // at most once a second. func (f *Fleet) install(ctx context.Context, job installJob) { - addr := job.target.Addr - known, err := f.knownHostKey(addr) - if err != nil { - f.installFailed(ctx, job.id, err, "") - return - } var last string var shown time.Time - hostKey, err := sshinstall.Install(ctx, job.target, job.signer, known, job.command, bytes.NewReader(job.script), func(line string) { - line = strings.TrimSpace(strings.TrimPrefix(line, "==> ")) + err := sshinstall.Install(ctx, job.target, job.signer, f.knownHosts, job.command, bytes.NewReader(job.script), func(line string) { + line = strings.TrimSpace(strings.TrimPrefix(line, StepPrefix)) if line == "" { return } @@ -237,19 +245,11 @@ func (f *Fleet) install(ctx context.Context, job installJob) { f.setStatusIf(ctx, job.id, store.ServerInstalling, store.ServerInstalling, truncate(line)) } }) - if known == nil && hostKey != nil { - if err := f.trustHostKey(addr, hostKey); err != nil { - f.log.Warn("save SSH host key", "target", addr, "err", err) - } - } if err != nil { - if strings.Contains(err.Error(), "does not match the known key") { - err = fmt.Errorf("%w. If the server was reinstalled, remove its line from %s", err, f.knownHostsPath()) - } f.installFailed(ctx, job.id, err, last) return } - f.log.Info("agent installed over SSH", "server_id", job.id, "target", addr) + f.log.Info("agent installed over SSH", "server_id", job.id, "target", job.target.Addr) deadline := time.After(connectWait) tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() @@ -278,7 +278,8 @@ func (f *Fleet) installFailed(ctx context.Context, id string, err error, last st f.setStatusIf(ctx, id, store.ServerInstalling, store.ServerUnreachable, truncate(detail)) } -// truncate cuts s to the 500 characters that a status detail keeps. +// truncate cuts the text of a status detail or an update error, which can +// hold the output of a server, to 500 characters. func truncate(s string) string { if r := []rune(s); len(r) > 500 { return string(r[:499]) + "…" @@ -293,49 +294,5 @@ func (f *Fleet) SSHPublicKey(context.Context) (string, error) { } func (f *Fleet) sshKey() (ssh.Signer, string, error) { - f.keyMu.Lock() - defer f.keyMu.Unlock() - return sshinstall.LoadOrCreateKey(filepath.Join(f.cfg.ConfigDir, "ssh_key")) -} - -func (f *Fleet) knownHostsPath() string { return filepath.Join(f.cfg.DataDir, "known_hosts") } - -// knownHostKey returns the host key of addr, or nil if runplane has not -// connected to addr before. -func (f *Fleet) knownHostKey(addr string) (ssh.PublicKey, error) { - f.keyMu.Lock() - defer f.keyMu.Unlock() - data, err := os.ReadFile(f.knownHostsPath()) - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("read known hosts: %w", err) - } - want := knownhosts.Normalize(addr) - for { - _, hosts, key, _, rest, err := ssh.ParseKnownHosts(data) - if errors.Is(err, io.EOF) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("read known hosts %s: %w", f.knownHostsPath(), err) - } - if slices.Contains(hosts, want) { - return key, nil - } - data = rest - } -} - -// trustHostKey adds the host key of addr to known_hosts. -func (f *Fleet) trustHostKey(addr string, key ssh.PublicKey) error { - f.keyMu.Lock() - defer f.keyMu.Unlock() - file, err := os.OpenFile(f.knownHostsPath(), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o600) - if err != nil { - return err - } - _, err = file.WriteString(knownhosts.Line([]string{addr}, key) + "\n") - return errors.Join(err, file.Close()) + return sshkey.LoadOrCreate(f.cfg.SSHKeyPath()) } diff --git a/internal/fleet/install_test.go b/internal/fleet/install_test.go index 6314e05..a79aa0e 100644 --- a/internal/fleet/install_test.go +++ b/internal/fleet/install_test.go @@ -28,10 +28,11 @@ import ( "golang.org/x/crypto/ssh/knownhosts" "github.com/runplane/runplane" - "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/field" + "github.com/runplane/runplane/internal/installtest" "github.com/runplane/runplane/internal/sshinstall/sshtest" "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/update" ) // printValues parses the flags of install.sh and prints the values that an @@ -40,24 +41,6 @@ const printValues = `parse_flags "$@" printf '%s\n' "REPO=$REPO" "VERSION=$VERSION" "PRIMARY=$PRIMARY" "FINGERPRINT=$FINGERPRINT" "TOKEN=$TOKEN" "ROLE=$ROLE" ` -// replaceMain returns install.sh with body in the place of the call of main. -func replaceMain(script []byte, body string) ([]byte, error) { - src, ok := strings.CutSuffix(strings.TrimRight(string(script), "\n"), `main "$@"`) - if !ok { - return nil, errors.New(`install.sh does not end with main "$@"`) - } - return []byte(src + body), nil -} - -// sh runs a shell with stdin and the environment, and returns its output. -func sh(stdin []byte, env []string, args ...string) (string, error) { - cmd := exec.Command("sh", args...) - cmd.Stdin = bytes.NewReader(stdin) - cmd.Env = append(os.Environ(), env...) - out, err := cmd.CombinedOutput() - return string(out), err -} - func values(out string) (map[string]string, error) { m := map[string]string{} for line := range strings.Lines(out) { @@ -75,35 +58,34 @@ func values(out string) (map[string]string, error) { // NewToken makes, and the command that runs the install.sh of the primary // with only a token. func TestInstallValues(t *testing.T) { - h := newHarness(t, newFakeNode("arm64")) - h.run() - _, port, _ := net.SplitHostPort(h.cfg.AgentListen) tests := []struct { name, version string base string // of NewToken; "": install.sh of the primary with a token url string // that curl gets wantVersion string - wantPrimary string + wantHost string // of the agent address of the primary }{ - {"script of a release", "v1.4.2", "", "http://ci.example.com:8484/install.sh", "v1.4.2", "ci.example.com:" + port}, - {"script of a development build", "dev", "", "http://ci.example.com/install.sh", "latest", "ci.example.com:" + port}, - {"script over IPv6", "v1.4.2", "", "http://[2001:db8::5]:8484/install.sh", "v1.4.2", "[2001:db8::5]:" + port}, - {"script over an SSH tunnel", "v1.4.2", "", "http://localhost:9000/install.sh", "v1.4.2", "203.0.113.7:" + port}, - {"command with a public URL", "v1.4.2", "https://ci.example.com", "https://ci.example.com/install.sh", "v1.4.2", "ci.example.com:" + port}, + {"script of a release", "v1.4.2", "", "http://ci.example.com:8484/install.sh", "v1.4.2", "ci.example.com"}, + {"script of a development build", "dev", "", "http://ci.example.com/install.sh", "latest", "ci.example.com"}, + {"script over IPv6", "v1.4.2", "", "http://[2001:db8::5]:8484/install.sh", "v1.4.2", "2001:db8::5"}, + {"script over an SSH tunnel", "v1.4.2", "", "http://localhost:9000/install.sh", "v1.4.2", "203.0.113.7"}, + {"command with a public URL", "v1.4.2", "https://ci.example.com", "https://ci.example.com/install.sh", "v1.4.2", "ci.example.com"}, // Plain HTTP carries no proof of the primary, so the script comes from GitHub. {"command over an SSH tunnel", "v1.4.2", "http://localhost:9000", - "https://github.com/acme/runplane-fork/releases/download/v1.4.2/install.sh", "v1.4.2", "203.0.113.7:" + port}, + "https://github.com/acme/runplane-fork/releases/download/v1.4.2/install.sh", "v1.4.2", "203.0.113.7"}, {"command with no domain", "v1.4.2", "http://203.0.113.7:8484", - "https://github.com/acme/runplane-fork/releases/download/v1.4.2/install.sh", "v1.4.2", "203.0.113.7:" + port}, + "https://github.com/acme/runplane-fork/releases/download/v1.4.2/install.sh", "v1.4.2", "203.0.113.7"}, {"command of a development build", "dev", "http://203.0.113.7:8484", - "https://github.com/acme/runplane-fork/releases/latest/download/install.sh", "latest", "203.0.113.7:" + port}, + "https://github.com/acme/runplane-fork/releases/latest/download/install.sh", "latest", "203.0.113.7"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - setRelease(t, "acme/runplane-fork", tt.version) + h := newHarness(t, newFakeNode("arm64"), func(c *Config) { c.Repo, c.Version = "acme/runplane-fork", tt.version }) + h.run() + _, port, _ := net.SplitHostPort(h.cfg.AgentListen) command, token := "curl -fsSL '"+tt.url+"' | sudo sh -s -- --token rp_srv_TEST", "rp_srv_TEST" if tt.base != "" { - tok, err := h.f.NewToken(t.Context(), tt.base, "build-1", "") + tok, err := h.f.NewToken(t.Context(), audit.Actor{}, tt.base, "build-1") if err != nil { t.Fatal(err) } @@ -112,9 +94,9 @@ func TestInstallValues(t *testing.T) { // The release script on GitHub has no values of this primary. served := runplane.InstallScript if !strings.HasPrefix(tt.url, "https://github.com/") { - served = h.f.Script(httptest.NewRequest("GET", tt.url, nil)) + served = h.f.Script(httptest.NewRequest("GET", tt.url, nil).Host) } - script, err := replaceMain(served, printValues) + script, err := installtest.ReplaceMain(served, printValues) if err != nil { t.Fatal(err) } @@ -122,7 +104,7 @@ func TestInstallValues(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "install.sh"), script, 0o600); err != nil { t.Fatal(err) } - out, err := sh(nil, []string{"DIR=" + dir}, "-c", `curl() { printf '%s\n' "$*" >"$DIR/curl-args"; cat "$DIR/install.sh"; } + out, err := installtest.Sh(nil, []string{"DIR=" + dir}, "-c", `curl() { printf '%s\n' "$*" >"$DIR/curl-args"; cat "$DIR/install.sh"; } sudo() { "$@"; } `+command) if err != nil { @@ -133,7 +115,7 @@ sudo() { "$@"; } } got, err := values(out) want := map[string]string{ - "REPO": "acme/runplane-fork", "VERSION": tt.wantVersion, "PRIMARY": tt.wantPrimary, + "REPO": "acme/runplane-fork", "VERSION": tt.wantVersion, "PRIMARY": net.JoinHostPort(tt.wantHost, port), "FINGERPRINT": h.fp, "TOKEN": token, "ROLE": "agent", } if err != nil || !maps.Equal(got, want) { @@ -143,44 +125,14 @@ sudo() { "$@"; } } } -// TestInstallKeepsRegisteredAgent runs write_config of install.sh again with -// the token that the agent used. The agent.json of the registered agent must -// stay, also when the primary has an IPv6 address. -func TestInstallKeepsRegisteredAgent(t *testing.T) { - script, err := replaceMain(runplane.InstallScript, `CONFIG_DIR=$1 DATA_DIR=$1/data ENV_FILE=$1/runplane.env AGENT_FILE=$1/agent.json -DRY_RUN=0 ROLE=agent MODE=container TOKEN=rp_srv_TEST PRIMARY=$2 FINGERPRINT=sha256:aa -write_config -`) - if err != nil { - t.Fatal(err) - } - for _, primary := range []string{"203.0.113.7:8485", "[2001:db8::5]:8485"} { - dir := t.TempDir() - path := filepath.Join(dir, "agent.json") - registered := "{\n \"primary\": \"" + primary + "\",\n \"fingerprint\": \"sha256:aa\",\n" + - " \"server_id\": \"01jbd5x7k2m4n6p8q9r0s1t2v3\",\n \"secret\": \"s3cret\"\n}\n" - if err := os.WriteFile(path, []byte(registered), 0o600); err != nil { - t.Fatal(err) - } - if out, err := sh(script, nil, "-s", "--", dir, primary); err != nil { - t.Fatalf("install.sh: %v\n%s", err, out) - } - if raw, _ := os.ReadFile(path); string(raw) != registered { - t.Errorf("agent.json of %s after a second install with the same token:\n%s\nwant it unchanged", primary, raw) - } - } -} - // TestInstallScriptSignature runs the release download of the served // install.sh against release files on disk. The script has the release key // of the primary binary, so it installs only a release whose SHA256SUMS has // a signature of that key. func TestInstallScriptSignature(t *testing.T) { - if out, err := exec.Command("openssl", "version").Output(); err != nil || !bytes.HasPrefix(out, []byte("OpenSSL ")) { - t.Skip("install.sh checks signatures with OpenSSL 3 or newer") + if !openSSL3() { + t.Skip("needs OpenSSL 3 or newer: install.sh checks the release signature with openssl pkeyutl -rawin") } - h := newHarness(t, newFakeNode("arm64")) - h.run() pub, key, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) @@ -189,13 +141,13 @@ func TestInstallScriptSignature(t *testing.T) { if err != nil { t.Fatal(err) } - publicKey := update.PublicKey - t.Cleanup(func() { update.PublicKey = publicKey }) - update.PublicKey = base64.StdEncoding.EncodeToString(pub) - setRelease(t, "acme/runplane", "v1.4.2") + h := newHarness(t, newFakeNode("arm64"), func(c *Config) { + c.Repo, c.Version, c.PublicKey = "acme/runplane", "v1.4.2", base64.StdEncoding.EncodeToString(pub) + }) + h.run() // curl takes the files from $RELEASES: install.sh calls it with // "-o " at the end. - script, err := replaceMain(h.f.Script(httptest.NewRequest("GET", "http://ci.example.com/install.sh", nil)), ` + script, err := installtest.ReplaceMain(h.f.Script("ci.example.com"), ` curl() { while [ $# -gt 2 ]; do shift; done cp "$RELEASES/${2#https://github.com/}" "$1" 2>/dev/null || return 22 @@ -243,7 +195,7 @@ cat "$TMP/runplane" t.Fatal(err) } } - out, err := sh(script, []string{"RELEASES=" + releases, "WORK=" + t.TempDir()}, "-s") + out, err := installtest.Sh(script, []string{"RELEASES=" + releases, "WORK=" + t.TempDir()}, "-s") if tt.wantErr == "" { if err != nil || !strings.HasSuffix(out, "\nrelease binary") { t.Errorf("install.sh: %v; want the release binary:\n%s", err, out) @@ -257,6 +209,16 @@ cat "$TMP/runplane" } } +// openSSL3 reports if the openssl command is OpenSSL 3 or newer. LibreSSL +// and OpenSSL 1 cannot check an ed25519 signature of a file. +func openSSL3() bool { + out, err := exec.Command("openssl", "version").Output() + version, ok := bytes.CutPrefix(out, []byte("OpenSSL ")) + major, _, _ := bytes.Cut(version, []byte(".")) + n, aerr := strconv.Atoi(string(major)) + return err == nil && ok && aerr == nil && n >= 3 +} + // releaseArchive returns a release archive whose runplane file holds text. func releaseArchive(t *testing.T, text string) []byte { var b bytes.Buffer @@ -305,12 +267,12 @@ func TestInstallSSH(t *testing.T) { time.Sleep(10 * time.Millisecond) } args, ok := strings.CutPrefix(command, "sudo -n sh -s -- ") - script, err := replaceMain(stdin, printValues) + script, err := installtest.ReplaceMain(stdin, printValues) if !ok || err != nil { fmt.Fprintf(out, "Error: the command %q does not run install.sh with sudo: %v\n", command, err) return 1 } - res, err := sh(script, nil, append([]string{"-s", "--"}, strings.Fields(args)...)...) + res, err := installtest.Sh(script, nil, append([]string{"-s", "--"}, strings.Fields(args)...)...) v, verr := values(res) if err != nil || verr != nil { fmt.Fprintf(out, "Error: %v %v\n%s", err, verr, res) @@ -321,7 +283,7 @@ func TestInstallSSH(t *testing.T) { return 0 }) - id, err := h.f.InstallSSH(ctx, "http://localhost:8484", api.SSHInstall{Name: "build-2", Host: "127.0.0.1", Port: port(srv.Addr), User: "deploy"}) + id, err := h.f.InstallSSH(ctx, audit.Actor{}, "http://localhost:8484", SSHInstall{Name: "build-2", Host: "127.0.0.1", Port: port(srv.Addr), User: "deploy"}) if err != nil { t.Fatal(err) } @@ -329,10 +291,16 @@ func TestInstallSSH(t *testing.T) { if v := h.server(id); v.Name != "build-2" || v.Kind != store.ServerAgent || v.Arch != "x64" { t.Errorf("installed server %+v, want build-2, an x64 agent", v) } + entries, _, err := h.st.ListAudit(ctx, "", 10) + if err != nil || !slices.ContainsFunc(entries, func(e store.AuditEntry) bool { + return e.Action == "server.install" && e.Target == id && string(e.Detail) == fmt.Sprintf(`{"host":"127.0.0.1","name":"build-2","port":%d,"user":"deploy"}`, port(srv.Addr)) + }) { + t.Errorf("audit entries %+v, %v; want the server.install of build-2", entries, err) + } if !sawOutput.Load() { t.Error("the status detail did not show the install output during the install") } - trusted, err := knownhosts.New(filepath.Join(h.cfg.DataDir, "known_hosts")) + trusted, err := knownhosts.New(h.cfg.KnownHostsPath()) if err != nil { t.Fatal(err) } @@ -344,13 +312,13 @@ func TestInstallSSH(t *testing.T) { // A server at a known address with another host key: runplane does not // send the script. srv2 := sshtest.NewServer(t, clientKey, func(string, []byte, io.Writer, io.Writer) uint32 { return 0 }) - f, err := os.OpenFile(filepath.Join(h.cfg.DataDir, "known_hosts"), os.O_APPEND|os.O_WRONLY, 0o600) + f, err := os.OpenFile(h.cfg.KnownHostsPath(), os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { t.Fatal(err) } fmt.Fprintln(f, knownhosts.Line([]string{srv2.Addr}, clientKey)) f.Close() - id2, err := h.f.InstallSSH(ctx, "http://localhost:8484", api.SSHInstall{Name: "build-3", Host: "127.0.0.1", Port: port(srv2.Addr), User: "deploy"}) + id2, err := h.f.InstallSSH(ctx, audit.Actor{}, "http://localhost:8484", SSHInstall{Name: "build-3", Host: "127.0.0.1", Port: port(srv2.Addr), User: "deploy"}) if err != nil { t.Fatal(err) } @@ -363,6 +331,20 @@ func TestInstallSSH(t *testing.T) { } } +// The dashboard form sends the name and the host as the user typed them. +func TestInstallSSHRefusesABadNameOrHost(t *testing.T) { + ctx := t.Context() + h := newHarness(t, newFakeNode("arm64")) + _, err := h.f.InstallSSH(ctx, audit.Actor{}, "http://localhost:8484", SSHInstall{Name: " ", Host: "root@203.0.113.9"}) + var p field.Problems + if !errors.As(err, &p) || len(p) != 2 || p[0].Key != "name" || p[1].Key != "host" { + t.Fatalf("InstallSSH: %v, want problems at name and host", err) + } + if list, err := h.st.ListServers(ctx); err != nil || len(list) != 0 { + t.Errorf("servers after the refused install: %+v, %v; want none", list, err) + } +} + func port(addr string) int { _, p, _ := net.SplitHostPort(addr) n, _ := strconv.Atoi(p) diff --git a/internal/fleet/provision.go b/internal/fleet/provision.go index f264774..089cf9a 100644 --- a/internal/fleet/provision.go +++ b/internal/fleet/provision.go @@ -1,29 +1,24 @@ package fleet import ( + "cmp" "context" "errors" "fmt" - "io/fs" "slices" "strings" - "sync" "time" "github.com/runplane/runplane/images" "github.com/runplane/runplane/internal/agent" - "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/events" "github.com/runplane/runplane/internal/ids" - "github.com/runplane/runplane/internal/image" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/update" + "github.com/runplane/runplane/internal/version" ) -// liveStates are the states of a sandbox that still operates. -var liveStates = []sandbox.State{sandbox.StateStarting, sandbox.StateIdle, sandbox.StateBusy, sandbox.StateStopping} - // seedImages adds the builtin images at the first start, and reports if // this is the first start. func (f *Fleet) seedImages(ctx context.Context) (bool, error) { @@ -31,18 +26,11 @@ func (f *Fleet) seedImages(ctx context.Context) (bool, error) { if err != nil || len(list) > 0 { return false, err } - dirs, err := fs.ReadDir(images.FS, ".") - if err != nil { - return false, fmt.Errorf("list builtin images: %w", err) - } now := f.now() - for _, d := range dirs { - if !d.IsDir() { - continue - } + for _, name := range images.Names { img := store.Image{ - ID: ids.New(), Name: d.Name(), Ref: "builtin:" + d.Name(), IsDefault: d.Name() == images.Default, - Source: "builtin", CreatedAt: now, UpdatedAt: now, + ID: ids.New(), Name: name, Ref: images.Prefix + name, IsDefault: name == images.Default, + Source: images.Source, CreatedAt: now, UpdatedAt: now, } if err := f.st.CreateImage(ctx, img); err != nil { return false, fmt.Errorf("add builtin image: %w", err) @@ -59,10 +47,8 @@ func (f *Fleet) seedPool(ctx context.Context, st store.Settings) error { if err != nil { return err } - p := defaultPool(local.Arch, local.VCPU-st.ReserveVCPU, local.MemoryMB-st.ReserveMemoryMB) - now := f.now() - p.ID, p.CreatedAt, p.UpdatedAt = ids.New(), now, now - if err := f.st.CreatePool(ctx, p); err != nil { + p, err := f.pools.Create(ctx, audit.Actor{}, defaultPool(local.Arch, local.VCPU-st.ReserveVCPU, local.MemoryMB-st.ReserveMemoryMB)) + if err != nil { return fmt.Errorf("add the default label: %w", err) } f.log.Info("added the default label", "label", p.Label, "vcpu", p.VCPU, "memory_mb", p.MemoryMB, "max_runners", p.MaxRunners) @@ -96,9 +82,9 @@ func (f *Fleet) provision(ctx context.Context) { f.log.Warn("list pools", "err", err) return } - bundles, err := f.st.ListRunnerBundles(ctx) + runners, err := f.runnerVersions(ctx) if err != nil { - f.log.Warn("list runner bundles", "err", err) + f.log.Warn("read the runner versions", "err", err) return } type pull struct { @@ -121,9 +107,6 @@ func (f *Fleet) provision(ctx context.Context) { f.defaultID = img.ID } } - if len(bundles) > 0 && update.Compare(bundles[0].Version, f.latest) > 0 { - f.latest = bundles[0].Version - } now := f.now() for _, n := range f.nodes { if n.status != store.ServerOnline { @@ -139,9 +122,10 @@ func (f *Fleet) provision(ctx context.Context) { n.pulls[id] = img.Ref pulls = append(pulls, pull{n, img}) } - if f.latest != "" && n.runner != f.latest && !n.runnerBusy && now.Sub(n.runnerFailed) >= retryAfter { + versions := runners[n.arch] + if len(versions) > 0 && n.runner != versions[0] && !n.runnerBusy && now.Sub(n.runnerFailed) >= retryAfter { n.runnerBusy = true - downloads = append(downloads, download{n, runnerVersions(f.latest, n.arch, bundles)}) + downloads = append(downloads, download{n, versions}) } } f.mu.Unlock() @@ -173,7 +157,7 @@ func (f *Fleet) wanted(n *node, pools []store.Pool) []string { list = append(list, f.defaultID) } for _, p := range pools { - if !p.Enabled || p.Arch != n.arch || len(p.ServerIDs) > 0 && !slices.Contains(p.ServerIDs, n.id) { + if !p.Enabled || !p.CanUse(n.id, n.arch) { continue } if id := p.ImageID; id != "" && !slices.Contains(list, id) { @@ -183,25 +167,30 @@ func (f *Fleet) wanted(n *node, pools []store.Pool) []string { return list } -// runnerVersions returns the versions to try for a server: the newest, then -// up to two older versions that are ready on other servers of the arch. -func runnerVersions(latest, arch string, bundles []store.RunnerBundle) []string { - list := []string{latest} - for _, b := range bundles { // newest first - if len(list) == 3 { - break +// runnerVersions returns the runner versions to try for each arch of the +// online servers. +func (f *Fleet) runnerVersions(ctx context.Context) (map[string][]string, error) { + f.mu.Lock() + var arches []string + for _, n := range f.nodes { + if n.status == store.ServerOnline && !slices.Contains(arches, n.arch) { + arches = append(arches, n.arch) } - if b.Arch == arch && b.Status == store.BundleReady && update.Compare(b.Version, latest) < 0 { - list = append(list, b.Version) + } + f.mu.Unlock() + versions := map[string][]string{} + for _, arch := range arches { + list, err := f.runners.Versions(ctx, arch) + if err != nil { + return nil, err } + versions[arch] = list } - return list + return versions, nil } -// pull makes an image ready on a server and records each step in the image -// build row of the server. A pull again of a ready image, to get a new -// build of its tag, keeps the row ready while it runs and when it fails: -// the server goes on with the image it has. +// pull makes an image ready on a server. When the server gets a new digest +// of the image, pull removes the images that the server does not use. func (f *Fleet) pull(ctx context.Context, n *node, img store.Image) { defer func() { f.mu.Lock() @@ -210,6 +199,19 @@ func (f *Fleet) pull(ctx context.Context, n *node, img store.Image) { } f.mu.Unlock() }() + if f.ensureImage(ctx, n, img) { + f.gcImages(ctx, n) + } +} + +// ensureImage makes an image ready on a server and records each step in the +// image build row of the server. A pull again of a ready image, to get a +// new build of its tag, keeps the row ready while it runs and when it +// fails: the server goes on with the image it has. ensureImage reports if +// the server has a new digest of the image. +func (f *Fleet) ensureImage(ctx context.Context, n *node, img store.Image) bool { + n.gcMu.RLock() + defer n.gcMu.RUnlock() f.mu.Lock() r, had := n.images[img.ID] refresh := had && r.ref == img.Ref @@ -224,13 +226,10 @@ func (f *Fleet) pull(ctx context.Context, n *node, img store.Image) { f.saveBuild(ctx, &b, store.BuildPending, "") } var total int64 - ref, err := n.EnsureImage(ctx, img.Ref, func(p image.Progress) { + ready, err := n.EnsureImage(ctx, img.Ref, func(p agent.Progress) { total = max(total, p.Total) - switch s := string(p.Status); s { - case store.BuildPulling, store.BuildBuilding, store.BuildConverting: - if s != b.Status && !refresh { - f.saveBuild(ctx, &b, s, "") - } + if s, ok := buildStatus(p.Status); ok && s != b.Status && !refresh { + f.saveBuild(ctx, &b, s, "") } }) if err != nil { @@ -241,16 +240,16 @@ func (f *Fleet) pull(ctx context.Context, n *node, img store.Image) { if !refresh { f.saveBuild(ctx, &b, store.BuildFailed, err.Error()) } - return - } - if ref.Digest != before { - b.Digest, b.SizeBytes, b.Path = ref.Digest, total, "" + return false } + // An agent of an earlier release sends no size. Then the largest total + // of the steps takes its place, or the size of the last pull. + b.Digest, b.SizeBytes = ready.Digest, cmp.Or(ready.SizeBytes, total, b.SizeBytes) f.mu.Lock() dropped := f.dropped[img.ID] f.mu.Unlock() if dropped { - return + return false } // The row says ready before the scheduler can use the image. f.saveBuild(ctx, &b, store.BuildReady, "") @@ -258,26 +257,37 @@ func (f *Fleet) pull(ctx context.Context, n *node, img store.Image) { r, had = n.images[img.ID] wasReady := had && r.ref == img.Ref if !f.dropped[img.ID] { - n.images[img.ID] = readyImage{ref: img.Ref, image: ref, at: f.now()} + n.images[img.ID] = readyImage{ref: img.Ref, image: ready.ImageRef, at: f.now()} delete(n.failed, img.ID) } f.mu.Unlock() if !wasReady { f.capacityChanged() } - if before != "" && before != ref.Digest { - f.gcImages(ctx, n, "") - } + return before != "" && before != ready.Digest +} + +// buildStatus gives the status of the image build row for a step of a pull +// that is in progress. +func buildStatus(s agent.ImageStatus) (string, bool) { + switch s { + case agent.ImagePulling: + return store.BuildPulling, true + case agent.ImageBuilding: + return store.BuildBuilding, true + case agent.ImageConverting: + return store.BuildConverting, true + } + return "", false } -// saveBuild records the status of an image build and publishes it. func (f *Fleet) saveBuild(ctx context.Context, b *store.ImageBuild, status, msg string) { b.Status, b.Error, b.UpdatedAt = status, msg, f.now() if err := f.st.UpsertImageBuild(ctx, *b); err != nil { f.log.Warn("record image build", "image", b.ImageID, "server_id", b.ServerID, "err", err) return } - f.bus.Publish(events.Event{Type: events.ImageState, Data: *b}) + f.bus.Publish(events.ImageBuildState(*b)) } // ensureRunner makes the first version in versions that works ready on the @@ -290,16 +300,14 @@ func (f *Fleet) ensureRunner(ctx context.Context, n *node, versions []string) er errs = append(errs, fmt.Errorf("server %s: %w", n.id, err)) // A lost connection says nothing about the bundle. if ctx.Err() == nil && !errors.Is(err, agent.ErrClosed) { - f.saveBundle(ctx, store.RunnerBundle{Version: v, Arch: n.arch, Status: store.BundleFailed, Error: err.Error()}) + f.runners.Built(ctx, store.RunnerBundle{Version: v, Arch: n.arch, Status: store.BundleFailed, Error: err.Error()}) } continue } - f.saveBundle(ctx, store.RunnerBundle{ - Version: v, Arch: b.Arch, SHA256: b.SHA256, Patched: b.Patched, Status: store.BundleReady, Path: b.Dir, - }) + f.runners.Built(ctx, store.RunnerBundle{Version: v, Arch: n.arch, SHA256: b.SHA256, Patched: b.Patched, Status: store.BundleReady}) f.mu.Lock() first := n.runner == "" - if first || update.Compare(v, n.runner) > 0 { + if first || version.Compare(v, n.runner) > 0 { n.runner = v } if i == 0 { @@ -322,113 +330,25 @@ func (f *Fleet) ensureRunner(ctx context.Context, n *node, versions []string) er return errors.Join(errs...) } -// saveBundle records a bundle build and publishes the row when its status -// or error changed. -func (f *Fleet) saveBundle(ctx context.Context, b store.RunnerBundle) { - before, _ := f.st.GetRunnerBundle(ctx, b.Version, b.Arch) - if err := f.st.SetRunnerBundleBuild(ctx, b); err != nil { - f.log.Warn("record runner bundle", "version", b.Version, "arch", b.Arch, "err", err) - return - } - after, err := f.st.GetRunnerBundle(ctx, b.Version, b.Arch) - if err != nil || after.Status == before.Status && after.Error == before.Error { - return - } - f.bus.Publish(events.Event{Type: events.RunnerVersion, Data: after}) -} - -// EnsureLatest makes the runner bundle of version ready on each online -// server. The github package calls it for a new runner release. Servers use -// the version for new sandboxes as soon as it is ready there. -func (f *Fleet) EnsureLatest(ctx context.Context, version string) error { - f.mu.Lock() - if update.Compare(version, f.latest) > 0 { - f.latest = version - } - var list []*node - for _, n := range f.nodes { - if n.status == store.ServerOnline { - list = append(list, n) - } - } - f.mu.Unlock() - errs := make([]error, len(list)) - var wg sync.WaitGroup - for i, n := range list { - wg.Go(func() { errs[i] = f.ensureRunner(ctx, n, []string{version}) }) - } - wg.Wait() - return errors.Join(errs...) -} - -func (f *Fleet) Images() api.Images { return imageService{f} } - -type imageService struct{ f *Fleet } - -// Refresh pulls the image again on each connected server. The servers keep -// the image that they have until the new pull is ready, unless its ref -// changed. -func (s imageService) Refresh(ctx context.Context, imageID string) error { - f := s.f - img, err := f.st.GetImage(ctx, imageID) - if err != nil { - return err - } - var list []*node - f.mu.Lock() - f.images[img.ID] = img - if img.IsDefault { - f.defaultID = img.ID - } - for _, n := range f.nodes { - if n.pulls[img.ID] != img.Ref { - n.pulls[img.ID] = img.Ref - list = append(list, n) - } - } - f.mu.Unlock() - for _, n := range list { - f.goBg(func(ctx context.Context) { f.pull(ctx, n, img) }) - } - return nil -} - -// Remove stops the use of an image and removes its files from the -// connected servers. The API then deletes the image row. -func (s imageService) Remove(ctx context.Context, imageID string) error { - f := s.f - f.mu.Lock() - f.dropped[imageID] = true - delete(f.images, imageID) - var nodes []*node - for _, n := range f.nodes { - delete(n.images, imageID) - delete(n.failed, imageID) - nodes = append(nodes, n) - } - f.mu.Unlock() - for _, n := range nodes { - f.gcImages(ctx, n, imageID) - } - return nil -} - // gcImages removes the images on the server of n that no image row and no -// live sandbox uses. skip is an image whose build does not count. -func (f *Fleet) gcImages(ctx context.Context, n *node, skip string) { +// live sandbox uses. It waits until the pulls on the server record their +// images. +func (f *Fleet) gcImages(ctx context.Context, n *node) { + n.gcMu.Lock() + defer n.gcMu.Unlock() builds, err := f.st.ListImageBuilds(ctx) if err != nil { f.log.Warn("list image builds", "err", err) return } - live, err := f.st.ListSandboxes(ctx, store.SandboxFilter{ServerID: n.id, States: liveStates}) + live, err := f.st.ListSandboxes(ctx, store.SandboxFilter{ServerID: n.id, States: sandbox.LiveStates()}) if err != nil { f.log.Warn("list sandboxes", "err", err) return } var keep []string for _, b := range builds { - if b.ServerID == n.id && b.ImageID != skip && b.Digest != "" { + if b.ServerID == n.id && b.Digest != "" { keep = append(keep, b.Digest) } } @@ -446,7 +366,7 @@ func (f *Fleet) gcImages(ctx context.Context, n *node, skip string) { // versions of its live sandboxes. An agent of an older release has no prune // route; it keeps its bundles. func (f *Fleet) pruneRunners(ctx context.Context, n *node) { - live, err := f.st.ListSandboxes(ctx, store.SandboxFilter{ServerID: n.id, States: liveStates}) + live, err := f.st.ListSandboxes(ctx, store.SandboxFilter{ServerID: n.id, States: sandbox.LiveStates()}) if err != nil { f.log.Warn("list sandboxes", "err", err) return diff --git a/internal/fleet/provision_test.go b/internal/fleet/provision_test.go index d76cf42..4a17c68 100644 --- a/internal/fleet/provision_test.go +++ b/internal/fleet/provision_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/events" "github.com/runplane/runplane/internal/ids" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" @@ -44,6 +46,29 @@ func TestFirstStartHasDefaultImage(t *testing.T) { }) } +// The default label of the first start is a pool change as any other: it +// has an audit entry, and the scheduler and GitHub get events.PoolsChanged. +func TestFirstStartAddsTheDefaultLabel(t *testing.T) { + ctx := t.Context() + h := newHarness(t, newFakeNode("arm64")) + changed := events.Subscribe[events.PoolsChanged](ctx, h.bus) + h.run() + recv(t, changed) + pools, err := h.st.ListPools(ctx) + if err != nil || len(pools) != 1 || pools[0].Label != "runplane" || pools[0].Arch != "arm64" { + t.Fatalf("pools %+v, %v; want the label runplane for arm64", pools, err) + } + // The pools service publishes the event before it records the entry. + var entries []store.AuditEntry + waitFor(t, "the audit entry", func() bool { + entries, _, err = h.st.ListAudit(ctx, "", 10) + return err == nil && len(entries) > 0 + }) + if len(entries) != 1 || entries[0].Action != "pool.create" || entries[0].Target != pools[0].ID || entries[0].UserID != "" { + t.Errorf("audit entries %+v; want one pool.create by runplane", entries) + } +} + func TestProvisionImagesAndRunner(t *testing.T) { ctx := t.Context() local := newFakeNode("arm64") @@ -69,10 +94,10 @@ func TestProvisionImagesAndRunner(t *testing.T) { // 2.330.0 is the newest release, but it does not download on this // server. 2.329.0 is ready on other servers. now := time.Now() - if err := h.st.UpsertRunnerBundle(ctx, store.RunnerBundle{Version: "2.329.0", Arch: "arm64", Status: store.BundleReady, CreatedAt: now}); err != nil { + if err := h.st.SetRunnerBundleBuild(ctx, store.RunnerBundle{Version: "2.329.0", Arch: "arm64", Status: store.BundleReady}, now); err != nil { t.Fatal(err) } - if err := h.st.SetRunnerBundleDates(ctx, "2.330.0", "arm64", &now, nil); err != nil { + if err := h.st.SetRunnerBundleDates(ctx, "2.330.0", "arm64", &now, nil, now); err != nil { t.Fatal(err) } h.run() @@ -110,8 +135,10 @@ func TestProvisionImagesAndRunner(t *testing.T) { if b, _ := h.st.GetImageBuild(ctx, broken, id); !strings.Contains(b.Error, "not found") { t.Errorf("failed build error %q, want the pull error", b.Error) } - if b, _ := h.st.GetImageBuild(ctx, app, id); b.Status != store.BuildReady || b.Digest != "sha256:a001" { - t.Errorf("pool image build %+v, want ready with its digest", b) + // The size is the space of the image on the server, not the bytes of + // the download. + if b, _ := h.st.GetImageBuild(ctx, app, id); b.Status != store.BuildReady || b.Digest != "sha256:a001" || b.SizeBytes != imageSize { + t.Errorf("pool image build %+v, want ready with its digest and size", b) } if v, _ := h.f.RunnerVersion(id); v != "2.329.0" { @@ -121,16 +148,14 @@ func TestProvisionImagesAndRunner(t *testing.T) { t.Errorf("bundle 2.330.0: status %q, error %q; want failed with the download error", b.Status, b.Error) } + // The server tries the newest version again after a wait. local.set(func(n *fakeNode) { n.runners["2.330.0"] = true }) - if err := h.f.EnsureLatest(ctx, "2.330.0"); err != nil { - t.Fatal(err) - } - if v, _ := h.f.RunnerVersion(id); v != "2.330.0" { - t.Errorf("runner version %q after EnsureLatest, want 2.330.0", v) - } + h.clock.add(retryAfter) + h.f.provision(ctx) + waitFor(t, "runner version 2.330.0", func() bool { v, _ := h.f.RunnerVersion(id); return v == "2.330.0" }) b, _ := h.st.GetRunnerBundle(ctx, "2.330.0", "arm64") - if b.Status != store.BundleReady || b.SHA256 != "9e2f2.330.0" || b.Path != "/var/lib/runplane/runners/2.330.0-arm64" || b.ReleasedAt == nil { - t.Errorf("bundle 2.330.0 after EnsureLatest: %+v; want ready with its build and its release date", b) + if b.Status != store.BundleReady || b.SHA256 != "9e2f2.330.0" || b.ReleasedAt == nil { + t.Errorf("bundle 2.330.0 after the second try: %+v; want ready with its build and its release date", b) } // An agent where no runner downloads gets no runner version, and its @@ -167,7 +192,6 @@ func TestRefreshAndRemoveImage(t *testing.T) { return ref.Digest } waitFor(t, "the pool image", func() bool { return digest() == "sha256:a001" }) - images := h.f.Images() // A new pull of the same ref keeps the image in use until it is ready. gate := make(chan struct{}) @@ -175,7 +199,7 @@ func TestRefreshAndRemoveImage(t *testing.T) { n.gates["registry.example.com/app:1"] = gate n.digests["registry.example.com/app:1"] = "sha256:a002" }) - if err := images.Refresh(ctx, app); err != nil { + if err := h.f.RefreshImage(ctx, audit.Actor{}, app); err != nil { t.Fatal(err) } waitFor(t, "the second pull", func() bool { return count(local.args("EnsureImage"), "registry.example.com/app:1") == 2 }) @@ -184,9 +208,10 @@ func TestRefreshAndRemoveImage(t *testing.T) { } close(gate) waitFor(t, "the new digest", func() bool { return digest() == "sha256:a002" }) - if got := local.args("GCImages"); !slices.ContainsFunc(got, func(keep any) bool { return !slices.Contains(keep.([]string), "sha256:a001") }) { - t.Errorf("image GC calls %v after the new digest, want one that drops sha256:a001", got) - } + // The pull calls the GC after it makes the new digest ready. + waitFor(t, "an image GC that drops sha256:a001", func() bool { + return slices.ContainsFunc(local.args("GCImages"), func(keep any) bool { return !slices.Contains(keep.([]string), "sha256:a001") }) + }) // After a new ref, the image of the old ref is not used. gate = make(chan struct{}) @@ -197,7 +222,7 @@ func TestRefreshAndRemoveImage(t *testing.T) { if err := h.st.UpdateImage(ctx, store.Image{ID: app, Name: "app", Ref: "registry.example.com/app:2"}); err != nil { t.Fatal(err) } - if err := images.Refresh(ctx, app); err != nil { + if err := h.f.RefreshImage(ctx, audit.Actor{}, app); err != nil { t.Fatal(err) } if got := digest(); got != "" { @@ -224,12 +249,15 @@ func TestRefreshAndRemoveImage(t *testing.T) { t.Fatal(err) } before, agentBefore := len(local.args("GCImages")), len(agentNode.args("GCImages")) - if err := images.Remove(ctx, app); err != nil { + if err := h.f.RemoveImage(ctx, audit.Actor{}, app); err != nil { t.Fatal(err) } if got := digest(); got != "" { t.Errorf("digest %q after Remove, want none", got) } + waitFor(t, "the image GC on the two servers", func() bool { + return len(local.args("GCImages")) > before && len(agentNode.args("GCImages")) > agentBefore + }) if kept := local.args("GCImages")[before:]; !reflect.DeepEqual(kept, []any{[]string{"sha256:b0b0", "sha256:in-use"}}) { t.Errorf("image GC on the primary kept %v, want one GC that keeps sha256:b0b0 and sha256:in-use", kept) } @@ -238,6 +266,42 @@ func TestRefreshAndRemoveImage(t *testing.T) { } } +// A GC on a server must wait until the pulls there record their images. +// Else the GC does not keep the image of a pull that just ended, and the +// fleet records a ready image that the server removed. +func TestImageGCWaitsForPulls(t *testing.T) { + ctx := t.Context() + local := newFakeNode("arm64") + gate := make(chan struct{}) + local.set(func(n *fakeNode) { + n.digests["registry.example.com/app:1"] = "sha256:a001" + n.digests["registry.example.com/new:1"] = "sha256:e001" + n.gates["registry.example.com/new:1"] = gate + }) + h := newHarness(t, local) + addImage(t, h.st, "ubuntu-24.04", "builtin:ubuntu-24.04", true) + app := addImage(t, h.st, "app", "registry.example.com/app:1", false) + addPool(t, h.st, "app", "arm64", app, true, nil) + h.run() + id := h.localID() + waitFor(t, "the pool image", func() bool { _, ok := h.f.Image(id, app); return ok }) + + if _, err := h.f.CreateImage(ctx, audit.Actor{}, "new", "registry.example.com/new:1"); err != nil { + t.Fatal(err) + } + waitFor(t, "the pull of the new image", func() bool { return count(local.args("EnsureImage"), "registry.example.com/new:1") == 1 }) + if err := h.f.RemoveImage(ctx, audit.Actor{}, app); err != nil { + t.Fatal(err) + } + // Time for a GC that does not wait. + time.Sleep(50 * time.Millisecond) + close(gate) + waitFor(t, "the image GC", func() bool { return len(local.args("GCImages")) > 0 }) + if kept := local.args("GCImages"); !reflect.DeepEqual(kept, []any{[]string{"sha256:b0b0", "sha256:e001"}}) { + t.Errorf("image GC kept %v, want one GC that keeps sha256:b0b0 and the new sha256:e001", kept) + } +} + func count(list []any, v any) int { n := 0 for _, x := range list { @@ -266,15 +330,15 @@ func TestNewRunnerPrunesOldBundlesOnAgents(t *testing.T) { t.Fatal(err) } - if err := h.f.EnsureLatest(ctx, "2.331.0"); err != nil { + now := time.Now() + if err := h.st.SetRunnerBundleDates(ctx, "2.331.0", "arm64", &now, nil, now); err != nil { t.Fatal(err) } + h.f.provision(ctx) waitFor(t, "a prune on the agent that keeps the live version", func() bool { return slices.ContainsFunc(agentNode.args("PruneRunners"), func(keep any) bool { return slices.Equal(keep.([]string), []string{"2.328.0"}) }) }) - if len(local.args("PruneRunners")) == 0 { - t.Error("no prune on the primary") - } + waitFor(t, "a prune on the primary", func() bool { return len(local.args("PruneRunners")) > 0 }) } // The builtin images get a new build each week under the same tag, so a @@ -299,7 +363,12 @@ func TestTaggedImagesArePulledAgainEachDay(t *testing.T) { h.clock.add(23 * time.Hour) h.f.provision(ctx) - time.Sleep(50 * time.Millisecond) + // provision marks each pull that it starts until the pull ends. + waitFor(t, "the end of the pulls", func() bool { + h.f.mu.Lock() + defer h.f.mu.Unlock() + return len(h.f.nodes[id].pulls) == 0 + }) if n := count(local.args("EnsureImage"), "builtin:ubuntu-24.04"); n != 1 { t.Errorf("%d pulls of the default image after 23 hours, want 1", n) } diff --git a/internal/fleet/prune.go b/internal/fleet/prune.go new file mode 100644 index 0000000..77cf049 --- /dev/null +++ b/internal/fleet/prune.go @@ -0,0 +1,18 @@ +package fleet + +import ( + "context" + "errors" + "time" + + "github.com/runplane/runplane/internal/store" +) + +// Prune deletes the install tokens that expired at now and the host samples +// older than store.SampleRetention. +func (f *Fleet) Prune(ctx context.Context, now time.Time) error { + return errors.Join( + f.st.DeleteExpiredServerTokens(ctx, now), + f.st.PruneServerStats(ctx, now.Add(-store.SampleRetention)), + ) +} diff --git a/internal/fleet/servers.go b/internal/fleet/servers.go index f0079d3..678074d 100644 --- a/internal/fleet/servers.go +++ b/internal/fleet/servers.go @@ -5,58 +5,56 @@ import ( "context" "errors" "fmt" - "math" + "maps" + "net/netip" "reflect" + "slices" + "strings" "sync" + "time" "github.com/runplane/runplane/internal/agent" - "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" + "github.com/runplane/runplane/internal/doctor" + "github.com/runplane/runplane/internal/events" + "github.com/runplane/runplane/internal/field" "github.com/runplane/runplane/internal/host" "github.com/runplane/runplane/internal/ids" - "github.com/runplane/runplane/internal/notify" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/version" ) -// startLocal records the server of the primary and connects its node. +// ErrLocalServer means that RemoveServer got the server of the primary. +var ErrLocalServer = errors.New("the primary server cannot be removed") + +// HostCheck is the doctor report of the server of the primary, for the setup +// page. Mode takes the place of the mode check of the report. +type HostCheck struct { + doctor.Report + Mode sandbox.Mode +} + func (f *Fleet) startLocal(ctx context.Context) error { info, err := f.local.Facts(ctx) if err != nil { f.log.Warn("read host facts", "err", err) } - facts := serverFacts(info, f.local.Driver().Mode(), version.Version, info.IPv4) v, err := f.st.GetLocalServer(ctx) - switch { - case errors.Is(err, store.ErrNotFound): + if errors.Is(err, store.ErrNotFound) { now := f.now() v = store.Server{ ID: ids.New(), Name: cmp.Or(info.Hostname, "primary"), Kind: store.ServerLocal, Status: store.ServerOnline, - ServerFacts: facts, CreatedAt: now, UpdatedAt: now, - } - if err := f.st.CreateServer(ctx, v); err != nil { - return fmt.Errorf("record the local server: %w", err) - } - case err != nil: - return fmt.Errorf("read the local server: %w", err) - default: - if err := f.st.UpdateServerFacts(ctx, v.ID, facts); err != nil { - return fmt.Errorf("record the local server: %w", err) - } - if !adminStatus(v.Status) && v.Status != store.ServerOnline { - v.Status = store.ServerOnline - if err := f.st.SetServerStatus(ctx, v.ID, v.Status, ""); err != nil { - return fmt.Errorf("record the local server: %w", err) - } + CreatedAt: now, UpdatedAt: now, } + err = f.st.CreateServer(ctx, v) + } + if err != nil { + return fmt.Errorf("record the local server: %w", err) + } + hello := agent.Hello{Version: f.version, Mode: f.local.Driver().Mode(), Host: info, Addr: info.IPv4} + if err := f.connect(ctx, v.ID, f.local, hello); err != nil { + return fmt.Errorf("record the local server: %w", err) } - f.setRegion(ctx, v.ID, info.Region) - f.mu.Lock() - f.localID, f.localAddr = v.ID, info.IPv4 - f.mu.Unlock() - f.add(&node{Node: f.local, id: v.ID, version: version.Version, name: v.Name, arch: facts.Arch, status: v.Status}) - f.publishServer(ctx, v.ID) - f.log.Info("local server ready", "server_id", v.ID, "name", v.Name, "arch", facts.Arch, "mode", facts.Mode) return nil } @@ -86,64 +84,71 @@ func (f *Fleet) startAgents(ctx context.Context) error { // AddSession takes a new agent session: the OnSession hook of agent.Listener. func (f *Fleet) AddSession(s *agent.Session) { - ctx := f.bg - id, h := s.ServerID(), s.Hello() - f.mu.Lock() - removed := f.removed[id] - f.mu.Unlock() - if removed { - s.Close() - return - } - v, err := f.connect(ctx, id, h) - if err != nil { + id := s.ServerID() + if err := f.connect(f.bg, id, s, s.Hello()); err != nil { f.log.Warn("record the agent connection", "server_id", id, "err", err) s.Close() - return } - n := &node{Node: s, id: id, sess: s, addr: h.Addr, version: h.Version, name: v.Name, arch: h.Host.Arch, status: v.Status} - f.add(n) - f.endUpdate(ctx, id, h.Version) - f.publishServer(ctx, id) - f.log.Info("server connected", "server_id", id, "name", v.Name, "addr", h.Addr, "version", h.Version, "arch", n.arch) - f.goBg(func(ctx context.Context) { - st, err := f.st.GetSettings(ctx) - if err != nil { - f.log.Warn("read settings for an agent", "err", err) - } else { - f.configure(ctx, n, DriverSettings(st)) - } - // The scheduler compares the sandboxes of the server with its rows. - f.capacityChanged() - f.kick() - if f.beginUpdate(id, h.Version, false) { - f.runUpdate(ctx, n) - } - }) } -// connect records the facts of a connected agent and sets it online, unless -// an admin drains or disables it. -func (f *Fleet) connect(ctx context.Context, id string, h agent.Hello) (store.Server, error) { +// connect records a server that connects: its facts, and the status online +// unless an admin drains or disables it. Then it gives the server its +// settings and adds its node, and an agent of another version gets the +// release of the primary. +func (f *Fleet) connect(ctx context.Context, id string, n agent.Node, h agent.Hello) error { v, err := f.st.GetServer(ctx, id) if err != nil { - return v, err + return err } - if err := f.st.UpdateServerFacts(ctx, id, serverFacts(h.Host, h.Mode, h.Version, h.Addr)); err != nil { - return v, err + now := f.now() + if err := f.st.UpdateServerFacts(ctx, id, serverFacts(h.Host, h.Mode, h.Version, h.Addr), now); err != nil { + return err } - f.setRegion(ctx, id, h.Host.Region) - if err := f.st.TouchServer(ctx, id, f.now()); err != nil { - return v, err + // last_seen_at is the last contact of an agent. + if !n.Local() { + if err := f.st.TouchServer(ctx, id, now); err != nil { + return err + } } if !adminStatus(v.Status) { v.Status = store.ServerOnline } detail := "" - if h.Version != version.Version { - detail = fmt.Sprintf("This server has runplane %s and the primary has %s.", h.Version, version.Version) + if h.Version != f.version { + detail = fmt.Sprintf("This server has runplane %s and the primary has %s.", h.Version, f.version) + } + if err := f.st.SetServerStatus(ctx, id, v.Status, detail, now); err != nil { + return err + } + + nd := &node{Node: n, id: id, addr: h.Addr, version: h.Version, name: v.Name, arch: h.Host.Arch, status: v.Status} + // The server gets its settings before the scheduler can start a + // sandbox on it. + f.mu.Lock() + settingsChanges := f.settingsChanges + f.mu.Unlock() + f.sendSettings(ctx, nd) + nd.limits = f.readLimits(ctx, nd) + if n.Local() { + f.mu.Lock() + f.localAddr = h.Addr + f.mu.Unlock() + } + if err := f.add(nd, settingsChanges); err != nil { + return err } - return v, f.st.SetServerStatus(ctx, id, v.Status, detail) + f.endUpdate(ctx, id, h.Version) + f.publishServer(ctx, id) + f.log.Info("server connected", "server_id", id, "name", v.Name, "addr", h.Addr, "version", h.Version, "arch", nd.arch, "mode", h.Mode) + f.goBg(func(ctx context.Context) { + // The scheduler compares the sandboxes of the server with its rows. + f.capacityChanged() + f.kick() + if f.beginUpdate(ctx, id, h.Version, false) == nil { + f.runUpdate(ctx, nd) + } + }) + return nil } // adminStatus reports a status that only an admin sets and clears. @@ -151,14 +156,36 @@ func adminStatus(status string) bool { return status == store.ServerDraining || status == store.ServerDisabled } -func (f *Fleet) configure(ctx context.Context, n *node, s sandbox.Settings) { +// sendSettings gives a server the settings of the store. A send waits for +// the send before it to the same server, and then reads the settings, so +// that the last send has the newest settings. +func (f *Fleet) sendSettings(ctx context.Context, n *node) { + n.settingsMu.Lock() + defer n.settingsMu.Unlock() + st, err := f.st.GetSettings(ctx) + if err != nil { + f.log.Warn("read settings for a server", "server_id", n.id, "err", err) + return + } ctx, cancel := context.WithTimeout(ctx, callTimeout) defer cancel() - if err := n.Configure(ctx, s); err != nil { + if err := n.Configure(ctx, NodeSettings(st)); err != nil { f.log.Warn("send settings to a server", "server_id", n.id, "err", err) } } +// readLimits reads the limits of the driver of a server. A server whose +// capacity cannot be read gets no limits. +func (f *Fleet) readLimits(ctx context.Context, n *node) sandbox.Limits { + ctx, cancel := context.WithTimeout(ctx, callTimeout) + defer cancel() + c, err := n.Driver().Capacity(ctx) + if err != nil { + f.log.Warn("read the capacity of a server", "server_id", n.id, "err", err) + } + return c.Limits +} + // syncServers takes the status changes of the rows, and makes each online // agent unreachable after 60 seconds without a connection. func (f *Fleet) syncServers(ctx context.Context) { @@ -196,20 +223,13 @@ func (f *Fleet) syncServers(ctx context.Context) { } f.failUpdate(ctx, v.ID, "The agent did not connect again after the update.") f.log.Warn("server unreachable", "server_id", v.ID, "name", v.Name, "since", since) - lastSeen := since - if v.LastSeenAt != nil { - lastSeen = *v.LastSeenAt - } - f.notify(func(dashboard string) notify.Event { - return notify.ServerUnreachableEvent(dashboard, v.ID, v.Name, lastSeen) - }) } } // setStatusIf changes the status of a server from one value to another and // publishes the row. It reports whether the status changed. func (f *Fleet) setStatusIf(ctx context.Context, id, from, to, detail string) bool { - ok, err := f.st.SetServerStatusIf(ctx, id, from, to, detail) + ok, err := f.st.SetServerStatusIf(ctx, id, from, to, detail, f.now()) if err != nil { f.log.Warn("set server status", "server_id", id, "status", to, "err", err) return false @@ -232,17 +252,16 @@ func (f *Fleet) refreshFacts(ctx context.Context) { return } facts := serverFacts(info, n.Driver().Mode(), n.version, n.addr) - if n.sess == nil { + if n.Local() { f.mu.Lock() f.localAddr = info.IPv4 f.mu.Unlock() } - regionChanged := f.setRegion(ctx, n.id, info.Region) v, err := f.st.GetServer(ctx, n.id) - if err != nil || v.ServerFacts == facts && !regionChanged { + if err != nil || v.ServerFacts == facts { return } - if err := f.st.UpdateServerFacts(ctx, n.id, facts); err != nil { + if err := f.st.UpdateServerFacts(ctx, n.id, facts, f.now()); err != nil { f.log.Warn("record host facts", "server_id", n.id, "err", err) return } @@ -252,24 +271,115 @@ func (f *Fleet) refreshFacts(ctx context.Context) { wg.Wait() } -// setRegion records the cloud region of a server and reports whether it changed. -func (f *Fleet) setRegion(ctx context.Context, id, region string) bool { - changed, err := f.st.SetServerRegion(ctx, id, region) +// ServerChange is a change of a server by an admin. A nil field keeps its +// value. +type ServerChange struct { + Name *string + Status *string // online, draining, or disabled +} + +// UpdateServer renames a server or changes its status, and returns the +// row. A status that the server cannot get now gives ErrInstalling or +// ErrNotPaused. +func (f *Fleet) UpdateServer(ctx context.Context, by audit.Actor, serverID string, c ServerChange) (store.Server, error) { + old, err := f.st.GetServer(ctx, serverID) if err != nil { - f.log.Warn("record server region", "server_id", id, "err", err) + return store.Server{}, err + } + newStatus := c.Status != nil && *c.Status != old.Status + if newStatus { + if err := statusChange(old.Status, *c.Status); err != nil { + return store.Server{}, err + } + } + now := f.now() + if c.Name != nil { + name := strings.TrimSpace(*c.Name) + if name == "" { + var p field.Problems + p.Add("name", "Enter a name.", *c.Name) + return store.Server{}, p + } + if name != old.Name { + if err := f.st.SetServerName(ctx, old.ID, name, now); err != nil { + return store.Server{}, err + } + } + } + if newStatus { + if err := f.st.SetServerStatus(ctx, old.ID, *c.Status, "", now); err != nil { + return store.Server{}, err + } } - return changed + v, err := f.st.GetServer(ctx, old.ID) + if err != nil { + return store.Server{}, err + } + f.serverChanged(v) + f.bus.Publish(events.ServerState(v)) + changes := map[string][2]any{} + if v.Name != old.Name { + changes["name"] = [2]any{old.Name, v.Name} + } + if v.Status != old.Status { + changes["status"] = [2]any{old.Status, v.Status} + } + if len(changes) > 0 { + f.audit.Record(ctx, by, audit.ServerUpdate, v.ID, map[string]any{"name": v.Name, "status": v.Status, "changes": changes}) + } + return v, nil } -// Remove drains an agent, stops its sandboxes, and closes its connection. -// The API then deletes the row. -func (f *Fleet) Remove(ctx context.Context, serverID string) error { +// The errors of a status that a server cannot get now. +var ( + ErrInstalling = errors.New("the server is still installing") + ErrNotPaused = errors.New("only a draining or disabled server can go online") +) + +// statusChange tells why a server cannot change from one status to another +// by an admin. +func statusChange(from, to string) error { + switch { + case from == store.ServerInstalling: + return ErrInstalling + case to == store.ServerOnline && !adminStatus(from): + return fmt.Errorf("%w, and the server is %s", ErrNotPaused, from) + } + return nil +} + +// Actions are the changes that an admin can make to a server now. +type Actions struct { + Drain bool + Resume bool // online again after a drain or disable + Remove bool + Update bool // install the release of the primary on the agent + RunDoctor bool +} + +// Actions tells which changes an admin can make to the server v now. +func (f *Fleet) Actions(v store.Server) Actions { + connected := f.node(v.ID) != nil + update := f.UpdateState(v) + return Actions{ + Drain: v.Status != store.ServerDraining && statusChange(v.Status, store.ServerDraining) == nil, + Resume: v.Status != store.ServerOnline && statusChange(v.Status, store.ServerOnline) == nil, + Remove: v.Kind != store.ServerLocal, + Update: connected && (update == UpdateAvailable || update == UpdateFailed), + RunDoctor: connected, + } +} + +// RemoveServer drains an agent, stops its sandboxes, closes its connection, +// and deletes its row. The delete can change the servers of pools, or +// disable them. +func (f *Fleet) RemoveServer(ctx context.Context, by audit.Actor, serverID string) error { v, err := f.st.GetServer(ctx, serverID) if err != nil { return err } if v.Kind == store.ServerLocal { - return errors.New("the primary server cannot be removed") + return ErrLocalServer } f.mu.Lock() f.removed[v.ID] = true @@ -286,85 +396,112 @@ func (f *Fleet) Remove(ctx context.Context, serverID string) error { if err := f.st.DeleteServerTokens(ctx, v.ID); err != nil { return err } - if err := f.st.SetServerStatus(ctx, v.ID, store.ServerDraining, "Removing the server."); err != nil { + if err := f.st.SetServerStatus(ctx, v.ID, store.ServerDraining, "Removing the server.", f.now()); err != nil { return err } f.publishServer(ctx, v.ID) f.capacityChanged() - if n == nil { - return nil - } - d := n.Driver() - list, err := d.List(ctx) - if err != nil { - f.log.Warn("list sandboxes of a removed server", "server_id", v.ID, "err", err) - } - for _, sb := range list { - if err := d.Stop(ctx, sb.ID); err != nil { - f.log.Warn("stop sandbox of a removed server", "server_id", v.ID, "sandbox", sb.ID, "err", err) + if n != nil { + d := n.Driver() + list, err := d.List(ctx) + if err != nil { + f.log.Warn("list sandboxes of a removed server", "server_id", v.ID, "err", err) } + for _, sb := range list { + if err := d.Stop(ctx, sb.ID); err != nil { + f.log.Warn("stop sandbox of a removed server", "server_id", v.ID, "sandbox", sb.ID, "err", err) + } + } + n.Close() + f.log.Info("server removed", "server_id", v.ID, "name", v.Name, "sandboxes", len(list)) + } + if err := f.st.DeleteServer(ctx, v.ID, f.now()); err != nil { + return err } - n.sess.Close() - f.log.Info("server removed", "server_id", v.ID, "name", v.Name, "sandboxes", len(list)) + f.bus.Publish(events.ServerRemoved{ID: v.ID}) + f.bus.Publish(events.PoolsChanged{}) + f.audit.Record(ctx, by, audit.ServerDelete, v.ID, map[string]any{"name": v.Name}) return nil } -func (f *Fleet) Doctor(ctx context.Context, serverID string) ([]api.Check, error) { +// Doctor runs the doctor checks on a connected server, and keeps them as +// the last doctor result of the server. +func (f *Fleet) Doctor(ctx context.Context, serverID string) (store.DoctorResult, error) { n := f.node(serverID) if n == nil { - return nil, errors.New("the server is not connected, so its checks cannot run") + return store.DoctorResult{}, ErrNotConnected } - checks, err := n.Doctor(ctx) + r, err := n.Doctor(ctx) if err != nil { - return nil, fmt.Errorf("run the doctor: %w", err) + return store.DoctorResult{}, fmt.Errorf("run the doctor: %w", err) } - return apiChecks(checks), nil + return f.saveDoctor(ctx, serverID, r.Checks) } -// Check reads the server of the primary and runs its doctor checks for the -// setup page. -func (f *Fleet) Check(ctx context.Context) (api.HostCheck, error) { - info, err := f.local.Facts(ctx) +// Check runs the doctor on the server of the primary, for the setup page. +// The server page shows the checks as the last doctor result of the +// primary. +func (f *Fleet) Check(ctx context.Context) (HostCheck, error) { + r, err := f.local.Doctor(ctx) if err != nil { - return api.HostCheck{}, fmt.Errorf("read host facts: %w", err) + return HostCheck{}, fmt.Errorf("run the doctor: %w", err) } - checks, err := f.local.Doctor(ctx) - if err != nil { - return api.HostCheck{}, fmt.Errorf("run the doctor: %w", err) + if local, err := f.st.GetLocalServer(ctx); err == nil { + if _, err := f.saveDoctor(ctx, local.ID, r.Checks); err != nil { + f.log.Warn("save the doctor result", "server_id", local.ID, "err", err) + } } - return api.HostCheck{ - Hostname: info.Hostname, Mode: string(f.local.Driver().Mode()), KVM: info.KVM, Arch: info.Arch, VCPU: info.VCPU, - MemoryMB: info.MemoryMB, DiskBytes: info.DiskBytes, FreeDiskBytes: info.FreeDiskBytes, DiskKind: info.DiskKind, - OS: info.OS, Checks: apiChecks(checks), - }, nil + r.Checks = slices.DeleteFunc(r.Checks, func(c doctor.Check) bool { return c.ID == doctor.ModeCheck.ID }) + return HostCheck{Report: r, Mode: f.local.Driver().Mode()}, nil } -// SettingsChanged sends the new driver settings to each connected server. -func (f *Fleet) SettingsChanged(_ context.Context, prev, next store.Settings) { - f.mu.Lock() - f.minFree = next.MinFreeDiskPercent - f.mu.Unlock() - s := DriverSettings(next) - if reflect.DeepEqual(DriverSettings(prev), s) { +// saveDoctor keeps the checks as the last doctor result of a server. The +// time has the precision of the store, so that the result is the same as +// the row that a later read gives. +func (f *Fleet) saveDoctor(ctx context.Context, serverID string, checks []doctor.Check) (store.DoctorResult, error) { + r := store.DoctorResult{ServerID: serverID, CheckedAt: f.now().UTC().Truncate(time.Millisecond), Checks: []store.DoctorCheck{}} + for _, c := range checks { + r.Checks = append(r.Checks, store.DoctorCheck{ + ID: c.ID, Name: c.Name, Status: string(c.Status), Detail: c.Detail, Fix: c.Fix, Command: c.Command, + }) + } + if err := f.st.SaveDoctorResult(ctx, r); err != nil { + return store.DoctorResult{}, err + } + return r, nil +} + +func (f *Fleet) settingsChanged(c events.SettingsChanged) { + if reflect.DeepEqual(NodeSettings(c.Old), NodeSettings(c.New)) { return } - for _, n := range f.connected() { - f.goBg(func(ctx context.Context) { f.configure(ctx, n, s) }) + f.mu.Lock() + f.settingsChanges++ + nodes := slices.Collect(maps.Values(f.nodes)) + f.mu.Unlock() + for _, n := range nodes { + f.goBg(func(ctx context.Context) { f.sendSettings(ctx, n) }) } } -// DriverSettings are the settings that the sandbox drivers obey. -func DriverSettings(st store.Settings) sandbox.Settings { - return sandbox.Settings{ - ReserveVCPU: st.ReserveVCPU, - ReserveMemoryMB: st.ReserveMemoryMB, - Overcommit: st.CPUOvercommit, - MinFreeDiskPercent: int(math.Ceil(st.MinFreeDiskPercent)), - DNS: st.DNSServers, - EgressAllowlist: st.EgressAllowlist, - BandwidthMbps: st.BandwidthMbps, - RegistryMirror: st.RegistryMirror, - ActionsCache: st.ActionsCache, +// NodeSettings are the settings that each server gets from the primary. +func NodeSettings(st store.Settings) agent.Settings { + var dns []netip.Addr + for _, v := range st.DNSServers { + // The settings service accepts only IPv4 addresses. + if a, err := netip.ParseAddr(v); err == nil { + dns = append(dns, a) + } + } + return agent.Settings{ + Settings: sandbox.Settings{ + DNS: dns, + EgressAllowlist: st.EgressAllowlist, + BandwidthMbps: st.BandwidthMbps, + RegistryMirror: st.RegistryMirror, + ActionsCache: st.ActionsCache, + }, + MinFreeDiskPercent: st.MinFreeDiskPercent, } } @@ -372,6 +509,7 @@ func serverFacts(info host.Info, mode sandbox.Mode, ver, addr string) store.Serv return store.ServerFacts{ Address: cmp.Or(addr, info.IPv4), Provider: info.Provider, + Region: info.Region, OS: info.OS, Kernel: info.Kernel, Arch: info.Arch, @@ -387,11 +525,3 @@ func serverFacts(info host.Info, mode sandbox.Mode, ver, addr string) store.Serv Version: ver, } } - -func apiChecks(checks []host.Check) []api.Check { - out := make([]api.Check, len(checks)) - for i, c := range checks { - out[i] = api.Check{ID: c.ID, Name: c.Name, Status: string(c.Status), Detail: c.Detail, Fix: c.Fix, Command: c.Command} - } - return out -} diff --git a/internal/fleet/servers_test.go b/internal/fleet/servers_test.go index 33cca95..050bf3c 100644 --- a/internal/fleet/servers_test.go +++ b/internal/fleet/servers_test.go @@ -1,8 +1,10 @@ package fleet import ( + "context" "errors" "maps" + "net/netip" "reflect" "slices" "strings" @@ -10,14 +12,11 @@ import ( "time" "github.com/runplane/runplane/internal/agent" - "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/events" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/mirror" - "github.com/runplane/runplane/internal/notify" "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" - "github.com/runplane/runplane/internal/version" ) func TestAgentUnreachableAfterOneMinute(t *testing.T) { @@ -25,8 +24,7 @@ func TestAgentUnreachableAfterOneMinute(t *testing.T) { h := newHarness(t, newFakeNode("arm64")) h.run() a, id := h.connectAgent("build-1", newFakeNode("x64")) - bus, cancel := h.bus.Subscribe(4096) - defer cancel() + servers := events.Subscribe[events.ServerState](ctx, h.bus) a.stop() waitFor(t, "the end of the session", func() bool { return h.f.Driver(id) == nil }) @@ -41,13 +39,9 @@ func TestAgentUnreachableAfterOneMinute(t *testing.T) { if got := h.server(id).Status; got != store.ServerUnreachable { t.Fatalf("status %q 61 seconds after the disconnect, want unreachable", got) } - waitEvent(t, bus, func(e events.Event) bool { - v, ok := e.Data.(store.Server) - return e.Type == events.ServerState && ok && v.ID == id && v.Status == store.ServerUnreachable + waitEvent(t, servers, func(v events.ServerState) bool { + return v.ID == id && v.Status == store.ServerUnreachable }) - if e := recv(t, h.notes); e.Type != notify.ServerUnreachable || e.Data["server_id"] != id || len(h.notes) != 0 { - t.Errorf("notification %+v and %d more, want one server.unreachable for %s", e, len(h.notes), id) - } a.start() waitFor(t, "the agent to connect again", func() bool { return h.f.Driver(id) != nil }) @@ -79,12 +73,11 @@ func TestOnlineServers(t *testing.T) { t.Fatalf("online servers and arches %v, want %v", arches, want) } - // An admin drains the agent, as PATCH /api/servers/{id} does. + // An admin drains the agent. setStatus := func(status string) { - if err := h.st.SetServerStatus(ctx, agentID, status, ""); err != nil { + if _, err := h.f.UpdateServer(ctx, audit.Actor{}, agentID, ServerChange{Status: &status}); err != nil { t.Fatal(err) } - h.bus.Publish(events.Event{Type: events.ServerState, Data: h.server(agentID)}) } before := h.capacity.Load() setStatus(store.ServerDraining) @@ -110,16 +103,16 @@ func TestConnectedAgentStaysOnline(t *testing.T) { h.run() _, id := h.connectAgent("build-x64", newFakeNode("x64")) - if err := h.st.SetServerStatus(ctx, id, store.ServerUnreachable, "No connection to the agent."); err != nil { + if err := h.st.SetServerStatus(ctx, id, store.ServerUnreachable, "No connection to the agent.", h.clock.now()); err != nil { t.Fatal(err) } - h.bus.Publish(events.Event{Type: events.ServerState, Data: h.server(id)}) + h.bus.Publish(events.ServerState(h.server(id))) h.f.syncServers(ctx) if got := h.server(id).Status; got != store.ServerOnline { t.Errorf("status %q of a connected agent, want online", got) } waitFor(t, "the agent in Online", func() bool { - return slices.ContainsFunc(h.f.Online(), func(s sandbox.Host) bool { return s.ServerID == id }) + return slices.ContainsFunc(h.f.Online(), func(s Host) bool { return s.ServerID == id }) }) } @@ -128,8 +121,8 @@ func TestOldSessionEndKeepsNewSession(t *testing.T) { t.Cleanup(h.f.stop) old := &node{Node: newFakeNode("x64"), id: "01srv", status: store.ServerOnline} cur := &node{Node: newFakeNode("x64"), id: "01srv", status: store.ServerOnline} - h.f.add(old) - h.f.add(cur) + h.f.add(old, 0) + h.f.add(cur, 0) // The agent connected again before the end of its old session arrived. h.f.down(t.Context(), old) @@ -143,39 +136,63 @@ func TestOldSessionEndKeepsNewSession(t *testing.T) { } // A connected agent records its region apart from its provider, runs the -// doctor, and adds its registry mirror to the stats of GET /api/caches. +// doctor and keeps the result, adds its registry mirror to the one of the +// primary in the stats of GET /api/caches, and gives the limits of its +// driver for the placement of the pools. func TestConnectedAgents(t *testing.T) { ctx := t.Context() local := newFakeNode("arm64") local.info.Region = "fsn1" + local.mirror = agent.MirrorStats{Hits: 2, Misses: 1, HitBytes: 20, MissBytes: 10, SizeBytes: 30} h := newHarness(t, local) h.run() a, b := newFakeNode("arm64"), newFakeNode("x64") a.info.Region, a.info.Kernel = "nbg1", "6.1.0-agent" - a.mirror = mirror.Stats{Hits: 5, Misses: 2, HitBytes: 500, MissBytes: 200, SizeBytes: 900} - b.mirror = mirror.Stats{Hits: 1, Misses: 3, HitBytes: 100, MissBytes: 300, SizeBytes: 400} + a.mirror = agent.MirrorStats{Hits: 5, Misses: 2, HitBytes: 500, MissBytes: 200, SizeBytes: 900} + a.limits = sandbox.Limits{OverheadMB: 64, MaxVCPU: 32, MaxSandboxes: 100} + b.mirror = agent.MirrorStats{Hits: 1, Misses: 3, HitBytes: 100, MissBytes: 300, SizeBytes: 400} agentA, idA := h.connectAgent("arm-1", a) h.connectAgent("amd-1", b) - regions, err := h.st.ServerRegions(ctx) - if err != nil || regions[h.localID()] != "fsn1" || regions[idA] != "nbg1" { - t.Errorf("regions %v, %v; want fsn1 for the primary and nbg1 for the agent", regions, err) + if local, v := h.server(h.localID()), h.server(idA); local.Region != "fsn1" || v.Region != "nbg1" { + t.Errorf("regions %q and %q, want fsn1 for the primary and nbg1 for the agent", local.Region, v.Region) } if v := h.server(idA); v.Provider != "Hetzner" { t.Errorf("provider %q, want Hetzner without the region", v.Provider) } - want := mirror.Stats{Hits: 6, Misses: 5, HitBytes: 600, MissBytes: 500, SizeBytes: 1300} - if got := h.f.AgentMirrorStats(ctx); got != want { - t.Errorf("agent mirror stats %+v, want %+v", got, want) + want := agent.MirrorStats{Hits: 8, Misses: 6, HitBytes: 620, MissBytes: 510, SizeBytes: 1330} + if got := h.f.MirrorStats(ctx); got != want { + t.Errorf("mirror stats %+v, want %+v", got, want) + } + if got, ok := h.f.Limits(idA); !ok || got != a.limits { + t.Errorf("limits of the agent %+v, %v; want %+v", got, ok, a.limits) + } + ran, err := h.f.Doctor(ctx, idA) + if err != nil || len(ran.Checks) != 2 || ran.Checks[1].Detail != "6.1.0-agent" || ran.Checks[1].Status != "ok" { + t.Errorf("doctor of the agent: %+v, %v", ran, err) + } + // The server page shows the last doctor result of each server. + if kept, err := h.st.GetDoctorResult(ctx, idA); err != nil || !kept.CheckedAt.Equal(ran.CheckedAt) || !reflect.DeepEqual(kept.Checks, ran.Checks) { + t.Errorf("kept doctor result of the agent: %+v, %v; want %+v", kept, err, ran) + } + setup, err := h.f.Check(ctx) + if err != nil { + t.Fatal(err) } - checks, err := h.f.Doctor(ctx, idA) - if err != nil || len(checks) != 1 || checks[0].Detail != "6.1.0-agent" || checks[0].Status != "ok" { - t.Errorf("doctor of the agent: %+v, %v", checks, err) + // The setup page names the mode in its summary, not in the list. + if len(setup.Checks) != 1 || setup.Checks[0].ID != "kernel" { + t.Errorf("checks of the setup page %+v, want only the kernel check", setup.Checks) + } + if kept, err := h.st.GetDoctorResult(ctx, h.localID()); err != nil || len(kept.Checks) != 2 || kept.Checks[1].Detail != local.info.Kernel { + t.Errorf("kept doctor result of the primary after the setup check: %+v, %v", kept, err) } agentA.stop() waitFor(t, "the end of the session", func() bool { return h.f.Driver(idA) == nil }) - if _, err := h.f.Doctor(ctx, idA); err == nil { - t.Error("doctor of a disconnected server: no error") + if _, err := h.f.Doctor(ctx, idA); !errors.Is(err, ErrNotConnected) { + t.Errorf("doctor of a disconnected server: %v, want ErrNotConnected", err) + } + if _, ok := h.f.Limits(idA); ok { + t.Error("a disconnected server has limits") } } @@ -191,18 +208,29 @@ func TestRemoveServer(t *testing.T) { t.Fatal(err) } - if err := h.f.Remove(ctx, h.localID()); err == nil { - t.Error("Remove of the local server: no error") + removed := events.Subscribe[events.ServerRemoved](ctx, h.bus) + + if err := h.f.RemoveServer(ctx, audit.Actor{}, h.localID()); !errors.Is(err, ErrLocalServer) { + t.Errorf("RemoveServer of the local server: %v, want ErrLocalServer", err) } - if err := h.f.Remove(ctx, id); err != nil { + if err := h.f.RemoveServer(ctx, audit.Actor{}, id); err != nil { t.Fatal(err) } + // The browsers drop the server from their lists. + select { + case e := <-removed: + if e.ID != id { + t.Errorf("server.removed of %s, want %s", e.ID, id) + } + default: + t.Error("no server.removed event") + } if got := node.args("Stop"); !reflect.DeepEqual(got, []any{"sb1", "sb2"}) { t.Errorf("stopped sandboxes %v, want sb1 and sb2", got) } waitFor(t, "the end of the session", func() bool { return h.f.Driver(id) == nil }) - if err := h.st.DeleteServer(ctx, id); err != nil { // as the API does - t.Fatal(err) + if _, err := h.st.GetServer(ctx, id); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the row of the removed server: %v, want ErrNotFound", err) } file := a.file() @@ -214,33 +242,141 @@ func TestRemoveServer(t *testing.T) { } } +// The Activity sentences need the old and new name and status of a server. +// An installing server keeps its status, and only a drained or disabled +// server goes online. +func TestUpdateServer(t *testing.T) { + ctx := t.Context() + h := newHarness(t, newFakeNode("arm64")) + h.run() + _, id := h.connectAgent("fsn1-a", newFakeNode("x64")) + now := h.clock.now() + if err := h.st.CreateServer(ctx, store.Server{ + ID: "new", Name: "new", Kind: store.ServerAgent, Status: store.ServerInstalling, CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + status := func(s string) ServerChange { return ServerChange{Status: &s} } + by := audit.Actor{UserID: "u1", IP: "192.0.2.7"} + + // A refused change writes nothing: no event and no audit entry would + // tell about it. + refused := status(store.ServerDraining) + newName := "fsn1-b" + refused.Name = &newName + if _, err := h.f.UpdateServer(ctx, by, "new", refused); !errors.Is(err, ErrInstalling) { + t.Errorf("drain of an installing server: %v, want ErrInstalling", err) + } + if v := h.server("new"); v.Name != "new" { + t.Errorf("name %q after a refused change, want new", v.Name) + } + + change := status(store.ServerDraining) + name := "fsn1-c" + change.Name = &name + v, err := h.f.UpdateServer(ctx, by, id, change) + if err != nil { + t.Fatal(err) + } + if v.Name != "fsn1-c" || v.Status != store.ServerDraining { + t.Errorf("server after the change: %s %s, want fsn1-c draining", v.Name, v.Status) + } + entries, _, err := h.st.ListAudit(ctx, "", 10) + if err != nil { + t.Fatal(err) + } + // The first start and the agent install also record entries. + entries = slices.DeleteFunc(entries, func(e store.AuditEntry) bool { return e.Action != "server.update" }) + want := `{"changes":{"name":["fsn1-a","fsn1-c"],"status":["online","draining"]},"name":"fsn1-c","status":"draining"}` + if len(entries) != 1 || entries[0].Action != "server.update" || string(entries[0].Detail) != want || + entries[0].UserID != "u1" || entries[0].IP != "192.0.2.7" { + t.Errorf("audit entries %+v, want one server.update by u1 with detail %s", entries, want) + } + + if _, err := h.f.UpdateServer(ctx, by, "new", status(store.ServerOnline)); !errors.Is(err, ErrInstalling) { + t.Errorf("online for an installing server: %v, want ErrInstalling", err) + } + if _, err := h.f.UpdateServer(ctx, by, h.localID(), status(store.ServerOnline)); err != nil { + t.Errorf("online to online: %v, want no change", err) + } + if _, err := h.f.UpdateServer(ctx, by, id, status(store.ServerOnline)); err != nil { + t.Errorf("resume of a draining server: %v", err) + } + gone := "gone" + if err := h.st.CreateServer(ctx, store.Server{ + ID: gone, Name: gone, Kind: store.ServerAgent, Status: store.ServerUnreachable, CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if _, err := h.f.UpdateServer(ctx, by, gone, status(store.ServerOnline)); !errors.Is(err, ErrNotPaused) { + t.Errorf("online for an unreachable server: %v, want ErrNotPaused", err) + } +} + +func TestServerActions(t *testing.T) { + h := newHarness(t, newFakeNode("arm64")) + h.run() + _, id := h.connectAgent("fsn1-a", newFakeNode("x64")) + for _, tc := range []struct { + v store.Server + want Actions + }{ + {h.server(h.localID()), Actions{Drain: true, RunDoctor: true}}, + {h.server(id), Actions{Drain: true, Remove: true, RunDoctor: true}}, + {store.Server{ID: "gone", Kind: store.ServerAgent, Status: store.ServerUnreachable}, Actions{Drain: true, Remove: true}}, + {store.Server{ID: "new", Kind: store.ServerAgent, Status: store.ServerInstalling}, Actions{Remove: true}}, + {store.Server{ID: "rest", Kind: store.ServerAgent, Status: store.ServerDraining}, Actions{Resume: true, Remove: true}}, + {store.Server{ID: "off", Kind: store.ServerAgent, Status: store.ServerDisabled}, Actions{Drain: true, Resume: true, Remove: true}}, + } { + if got := h.f.Actions(tc.v); got != tc.want { + t.Errorf("actions of %s %s: %+v, want %+v", tc.v.ID, tc.v.Status, got, tc.want) + } + } +} + func TestSettingsReachServers(t *testing.T) { ctx := t.Context() local := newFakeNode("arm64") h := newHarness(t, local) h.run() + prev, err := h.st.UpdateSettings(ctx, func(s *store.Settings) error { + s.DNSServers, s.EgressAllowlist, s.MinFreeDiskPercent = []string{"1.1.1.1"}, []string{"github.com"}, 10 + s.BandwidthMbps, s.RegistryMirror, s.ActionsCache = 50, true, false + return nil + }) + if err != nil { + t.Fatal(err) + } agentNode := newFakeNode("x64") h.connectAgent("build-x64", agentNode) waitFor(t, "the settings of the new agent", func() bool { return len(agentNode.args("Configure")) == 1 }) - defaults := sandbox.Settings{ - ReserveVCPU: 1, ReserveMemoryMB: 2048, Overcommit: 1, MinFreeDiskPercent: 15, DNS: []string{"1.1.1.1", "8.8.8.8"}, - EgressAllowlist: []string{}, RegistryMirror: true, ActionsCache: true, + stored := agent.Settings{ + Settings: sandbox.Settings{ + DNS: []netip.Addr{netip.MustParseAddr("1.1.1.1")}, EgressAllowlist: []string{"github.com"}, BandwidthMbps: 50, RegistryMirror: true, + }, + MinFreeDiskPercent: 10, } - if got := agentNode.args("Configure")[0]; !reflect.DeepEqual(got, defaults) { - t.Errorf("settings at connect:\n%+v\nwant\n%+v", got, defaults) + if got := agentNode.args("Configure")[0]; !reflect.DeepEqual(got, stored) { + t.Errorf("settings at connect:\n%+v\nwant\n%+v", got, stored) } - prev := store.DefaultSettings() next := prev next.ReserveVCPU, next.ReserveMemoryMB, next.CPUOvercommit = 2, 4096, 1.5 next.MinFreeDiskPercent = 12.5 next.DNSServers, next.EgressAllowlist = []string{"9.9.9.9"}, []string{"10.0.0.0/8"} next.BandwidthMbps, next.RegistryMirror, next.ActionsCache = 100, false, false - h.f.SettingsChanged(ctx, prev, next) - want := sandbox.Settings{ - ReserveVCPU: 2, ReserveMemoryMB: 4096, Overcommit: 1.5, MinFreeDiskPercent: 13, DNS: []string{"9.9.9.9"}, - EgressAllowlist: []string{"10.0.0.0/8"}, BandwidthMbps: 100, + // As the settings service does: it saves the settings, then publishes the change. + if _, err := h.st.UpdateSettings(ctx, func(s *store.Settings) error { + *s = next + return nil + }); err != nil { + t.Fatal(err) + } + h.bus.Publish(events.SettingsChanged{Old: prev, New: next}) + want := agent.Settings{ + Settings: sandbox.Settings{DNS: []netip.Addr{netip.MustParseAddr("9.9.9.9")}, EgressAllowlist: []string{"10.0.0.0/8"}, BandwidthMbps: 100}, + MinFreeDiskPercent: 12.5, } for name, n := range map[string]*fakeNode{"local": local, "agent": agentNode} { waitFor(t, "the new settings on the "+name+" server", func() bool { @@ -250,75 +386,161 @@ func TestSettingsReachServers(t *testing.T) { } } -func TestHostStats(t *testing.T) { +// The fleet sends a server its settings before it adds the server. A +// settings change in that time must also reach the server, else the server +// keeps the old egress allowlist until the next change. +func TestSettingsChangeWhileAnAgentConnects(t *testing.T) { ctx := t.Context() local := newFakeNode("arm64") h := newHarness(t, local) h.run() - bus, cancel := h.bus.Subscribe(4096) - defer cancel() - id := h.localID() - at := h.clock.now().Add(-time.Second) - // With a domain and no public URL, the links go to the domain. + agentNode := newFakeNode("x64") + _, gate := h.startGatedAgent("build-x64", agentNode) + + prev := store.DefaultSettings() + next := prev + next.EgressAllowlist = []string{"10.0.0.0/8"} if _, err := h.st.UpdateSettings(ctx, func(s *store.Settings) error { - s.Domain = "ci.example.com" + *s = next return nil }); err != nil { t.Fatal(err) } - sample := func(diskPercent float64) { - local.stats <- host.Stats{At: at, CPUPercent: 52, MemoryPercent: 38, DiskPercent: diskPercent, Resources: host.Resources{DiskBytes: 500e9}} - at = at.Add(10 * time.Second) - waitEvent(t, bus, func(e events.Event) bool { - st, ok := e.Data.(store.ServerStat) - return e.Type == events.HostStats && ok && st.ServerID == id && st.DiskPercent == diskPercent - }) + h.bus.Publish(events.SettingsChanged{Old: prev, New: next}) + allowlist := func(n *fakeNode) []string { + list := n.args("Configure") + return list[len(list)-1].(agent.Settings).EgressAllowlist } + waitFor(t, "the new settings on the local server", func() bool { return slices.Equal(allowlist(local), []string{"10.0.0.0/8"}) }) + close(gate) + waitFor(t, "the new settings on the agent", func() bool { return slices.Equal(allowlist(agentNode), []string{"10.0.0.0/8"}) }) +} - sample(90) - h.f.flushStats(ctx) - latest, err := h.st.LatestServerStats(ctx) - if err != nil { +// The dashboard saves each switch at once, so two changes can come in one +// second. A send to a server must wait for the send before it: else the +// server can apply the older value last and keep it until the next change. +func TestQuickSettingsChanges(t *testing.T) { + ctx := t.Context() + local := newFakeNode("arm64") + h := newHarness(t, local) + h.run() + gate := make(chan struct{}) + local.set(func(n *fakeNode) { n.configureGate = gate }) + sends := len(local.args("Configure")) + prev := store.DefaultSettings() + change := func(allowlist ...string) { + next := prev + next.EgressAllowlist = allowlist + if _, err := h.st.UpdateSettings(ctx, func(s *store.Settings) error { + *s = next + return nil + }); err != nil { + t.Fatal(err) + } + h.bus.Publish(events.SettingsChanged{Old: prev, New: next}) + prev = next + } + + change("10.0.0.0/8") + waitFor(t, "the first send", func() bool { return len(local.args("Configure")) == sends+1 }) + change("192.168.0.0/16") + // Time for a second send that does not wait. + time.Sleep(50 * time.Millisecond) + if n := len(local.args("Configure")) - sends; n != 1 { + t.Errorf("%d sends while the first send waits, want 1", n) + } + close(gate) + waitFor(t, "the newest settings", func() bool { + list := local.args("Configure") + return slices.Equal(list[len(list)-1].(agent.Settings).EgressAllowlist, []string{"192.168.0.0/16"}) + }) +} + +// An admin can remove a server while it connects. The fleet must not add +// the server then: no row and no sync finds the node again, and the +// scheduler would start sandboxes on it. +func TestRemoveServerWhileItConnects(t *testing.T) { + ctx := t.Context() + h := newHarness(t, newFakeNode("arm64")) + h.run() + agentNode := newFakeNode("x64") + a, gate := h.startGatedAgent("old", agentNode) + id := a.file()["server_id"] + + if err := h.f.RemoveServer(ctx, audit.Actor{}, id); err != nil { t.Fatal(err) } - if st := latest[id]; st.CPUPercent != 52 || st.MemoryPercent != 38 || st.DiskPercent != 90 { - t.Errorf("stored sample %+v, want the one sample", st) + close(gate) + // The agent reads its facts again before it connects again. + waitFor(t, "the end of the session", func() bool { return len(agentNode.args("Facts")) == 2 }) + if h.f.Driver(id) != nil { + t.Error("the removed server is connected") } +} - // 10% free is below the 15% minimum. A second low sample sends nothing; - // after a recovery, the next drop sends again. - sample(91) - sample(50) - sample(95) - var free []float64 - for range 2 { - e := recv(t, h.notes) - if e.Type != notify.DiskLow || e.Data["server_id"] != id || e.URL != "https://ci.example.com/servers/"+id { - t.Errorf("notification %+v, want disk.low for %s with a link to the domain", e, id) - } - free = append(free, e.Data["free_percent"].(float64)) +// Run writes the samples that wait for the next flush when it ends, also +// when the stop comes while Run starts. +func TestHostStats(t *testing.T) { + tests := []struct { + name string + opts []func(*Config) + // beforeStop, if not nil, waits until Run is at the point of the stop. + beforeStop func(t *testing.T, local *fakeNode) + }{ + // Run pulls the default image after its start. + {"stop after the start", nil, func(t *testing.T, local *fakeNode) { + waitFor(t, "the first pull", func() bool { return len(local.args("EnsureImage")) > 0 }) + }}, + // The local server sends samples from its connect, and Run makes the + // default label after it. + {"stop in the start", []func(*Config){holdStart}, nil}, } - slices.Sort(free) - if !slices.Equal(free, []float64{5, 10}) || len(h.notes) != 0 { - t.Errorf("disk.low notifications for free space %v and %d more, want 10%% and 5%%", free, len(h.notes)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + local := newFakeNode("arm64") + h := newHarness(t, local, tt.opts...) + h.run() + stats := events.Subscribe[events.HostStats](ctx, h.bus) + id := h.localID() + + local.stats <- host.Stats{At: h.clock.now(), CPUPercent: 52, MemoryPercent: 38, DiskPercent: 90, Resources: host.Resources{DiskBytes: 500e9}} + waitEvent(t, stats, func(st events.HostStats) bool { return st.ServerID == id }) + if tt.beforeStop != nil { + tt.beforeStop(t, local) + } + h.stop() + latest, err := h.st.LatestServerStats(ctx) + if err != nil { + t.Fatal(err) + } + if st := latest[id]; st.CPUPercent != 52 || st.MemoryPercent != 38 || st.DiskPercent != 90 { + t.Errorf("stored sample %+v, want the one sample", st) + } + }) } } -// setRelease sets the release repository and the version of runplane until -// the test ends. -func setRelease(t *testing.T, repo, ver string) { - oldRepo, oldVer := version.Repo, version.Version - t.Cleanup(func() { version.Repo, version.Version = oldRepo, oldVer }) - version.Repo, version.Version = repo, ver +// holdStart holds Run in its first start until the stop: the default label +// waits for the end of ctx. +func holdStart(c *Config) { c.Pools = heldPools{} } + +type heldPools struct{} + +func (heldPools) Create(ctx context.Context, _ audit.Actor, _ store.Pool) (store.Pool, error) { + <-ctx.Done() + return store.Pool{}, ctx.Err() } +// release makes the primary and its agents run the release v1.4.2. +func release(c *Config) { c.Version = "v1.4.2" } + // An agent must speak the protocol of the primary, so the fleet updates an // agent with another version when it connects. It tries each version one // time for each agent and shows a failure on the server. func TestUpdateAgentOnConnect(t *testing.T) { - h := newHarness(t, newFakeNode("arm64")) + h := newHarness(t, newFakeNode("arm64"), release) h.run() - setRelease(t, version.Repo, "v1.4.2") agentNode := newFakeNode("arm64") agentNode.updateErr = errors.New("download failed") _, id := h.connectAgent("arm-2", agentNode) @@ -336,15 +558,15 @@ func TestUpdateAgentOnConnect(t *testing.T) { for _, tt := range tests { before := len(agentNode.args("Update")) n := &node{Node: agentNode, id: tt.id, version: tt.agent} - if h.f.beginUpdate(n.id, n.version, false) { + if h.f.beginUpdate(t.Context(), n.id, n.version, false) == nil { h.f.runUpdate(t.Context(), n) } if got := agentNode.args("Update")[before:]; !slices.Equal(got, tt.want) { t.Errorf("%s: updates %v, want %v", tt.name, got, tt.want) } } - if v := h.server(id); !strings.Contains(v.StatusDetail, "The update to runplane v1.4.2 failed: download failed.") { - t.Errorf("status detail %q, want the failed update", v.StatusDetail) + if v := h.server(id); !strings.Contains(v.UpdateError, "The update to runplane v1.4.2 failed: download failed.") { + t.Errorf("update error %q, want the failed update", v.UpdateError) } } @@ -352,76 +574,70 @@ func TestUpdateAgentOnConnect(t *testing.T) { // runs until the agent connects again with the new version. func TestUpdateAgentByHand(t *testing.T) { ctx := t.Context() - h := newHarness(t, newFakeNode("arm64")) + h := newHarness(t, newFakeNode("arm64"), release) h.run() - setRelease(t, version.Repo, "v1.4.2") agentNode := newFakeNode("arm64") a, id := h.connectAgent("arm-2", agentNode) - state := func() string { return h.f.UpdateState(h.server(id)) } - // olderAgent makes the connected agent look like an agent of v1.4.1. - olderAgent := func() { - h.f.mu.Lock() - h.f.nodes[id].version = "v1.4.1" - h.f.mu.Unlock() - v := h.server(id) - v.ServerFacts.Version = "v1.4.1" - if err := h.st.UpdateServerFacts(ctx, id, v.ServerFacts); err != nil { - t.Fatal(err) - } - } + state := func() UpdateState { return h.f.UpdateState(h.server(id)) } - if got := state(); got != api.UpdateCurrent { + if got := state(); got != UpdateCurrent { t.Fatalf("update state %q of an agent of the same version, want current", got) } - if err := h.f.UpdateAgent(ctx, id); err == nil { - t.Fatal("UpdateAgent started an update of an agent of the same version") + if err := h.f.UpdateAgent(ctx, audit.Actor{}, id); !errors.Is(err, ErrUpToDate) { + t.Fatalf("UpdateAgent of an agent of the same version: %v, want ErrUpToDate", err) } - if got := h.f.UpdateState(h.server(h.localID())); got != api.UpdateCurrent { + if got := h.f.UpdateState(h.server(h.localID())); got != UpdateCurrent { t.Errorf("update state %q of the primary, want current", got) } - olderAgent() - if got := state(); got != api.UpdateAvailable { - t.Fatalf("update state %q of an older agent, want available", got) - } + // An agent of an older release gets the release of the primary when it + // connects. agentNode.set(func(n *fakeNode) { n.updateErr = errors.New("download failed") }) - if err := h.f.UpdateAgent(ctx, id); err != nil { + a.restart("v1.4.1") + waitFor(t, "the failed update", func() bool { return state() == UpdateFailed }) + if got := agentNode.args("Update"); !slices.Equal(got, []any{"v1.4.2"}) { + t.Errorf("updates %v, want one to v1.4.2", got) + } + // An admin can try a failed version again. The agent holds the update + // until the gate closes, so the connection stays. + gate := make(chan struct{}) + agentNode.set(func(n *fakeNode) { n.updateErr, n.updateGate = nil, gate }) + if err := h.f.UpdateAgent(ctx, audit.Actor{}, id); err != nil { t.Fatal(err) } - waitFor(t, "the failed update", func() bool { return state() == api.UpdateFailed }) - // An admin can try a failed version again. - agentNode.set(func(n *fakeNode) { n.updateErr = nil }) - if err := h.f.UpdateAgent(ctx, id); err != nil { - t.Fatal(err) - } - waitFor(t, "the second update call", func() bool { return len(agentNode.args("Update")) == 2 }) - if got := state(); got != api.UpdateUpdating { + if got := state(); got != UpdateUpdating { t.Fatalf("update state %q before the agent restarts, want updating", got) } - if err := h.f.UpdateAgent(ctx, id); err == nil { - t.Fatal("UpdateAgent started a second update while one runs") + entries, _, err := h.st.ListAudit(ctx, "", 10) + if err != nil || len(entries) == 0 || entries[0].Action != "server.update_agent" || entries[0].Target != id || + string(entries[0].Detail) != `{"from":"v1.4.1","name":"arm-2","to":"v1.4.2"}` { + t.Errorf("audit entries %+v, %v; want the server.update_agent of arm-2 from v1.4.1 to v1.4.2 first", entries, err) + } + if err := h.f.UpdateAgent(ctx, audit.Actor{}, id); !errors.Is(err, ErrUpdateInProgress) { + t.Fatalf("UpdateAgent while an update runs: %v, want ErrUpdateInProgress", err) } - // The agent restarts with the release of the primary. - a.stop() + // The agent replaces its binary and restarts with the release of the + // primary. + close(gate) + // The primary closes the connection when it has the answer. waitFor(t, "the end of the session", func() bool { return h.f.Driver(id) == nil }) - a.start() - waitFor(t, "the end of the update", func() bool { return h.f.Driver(id) != nil && state() == api.UpdateCurrent }) + a.restart("v1.4.2") + waitFor(t, "the end of the update", func() bool { return h.f.Driver(id) != nil && state() == UpdateCurrent }) // An agent that does not come back fails the update. - olderAgent() - if err := h.f.UpdateAgent(ctx, id); err != nil { + a.restart("v1.4.1") + waitFor(t, "the older agent", func() bool { return h.f.Driver(id) != nil && state() == UpdateAvailable }) + if err := h.f.UpdateAgent(ctx, audit.Actor{}, id); err != nil { t.Fatal(err) } - waitFor(t, "the third update call", func() bool { return len(agentNode.args("Update")) == 3 }) - a.stop() waitFor(t, "the end of the session", func() bool { return h.f.Driver(id) == nil }) h.clock.add(61 * time.Second) h.f.syncServers(ctx) - if got := state(); got != api.UpdateFailed { + if got := state(); got != UpdateFailed { t.Fatalf("update state %q of an agent that did not come back, want failed", got) } - if v := h.server(id); v.Status != store.ServerUnreachable || !strings.Contains(v.StatusDetail, "did not connect again") { - t.Errorf("status %q, detail %q; want unreachable after the update", v.Status, v.StatusDetail) + if v := h.server(id); v.Status != store.ServerUnreachable || !strings.Contains(v.UpdateError, "did not connect again") { + t.Errorf("status %q, update error %q; want unreachable after the update", v.Status, v.UpdateError) } } diff --git a/internal/fleet/stats.go b/internal/fleet/stats.go index a54715d..0b4af3f 100644 --- a/internal/fleet/stats.go +++ b/internal/fleet/stats.go @@ -7,8 +7,6 @@ import ( "github.com/runplane/runplane/internal/agent" "github.com/runplane/runplane/internal/events" "github.com/runplane/runplane/internal/host" - "github.com/runplane/runplane/internal/mirror" - "github.com/runplane/runplane/internal/notify" "github.com/runplane/runplane/internal/store" ) @@ -17,8 +15,7 @@ import ( const maxSamples = 10_000 // addStats publishes a host sample of a server and keeps it for the next -// flush. It also sends a notification when the free disk space falls below -// min_free_disk_percent. +// flush. func (f *Fleet) addStats(n *node, s host.Stats) { if s.At.IsZero() { s.At = f.now() @@ -26,50 +23,41 @@ func (f *Fleet) addStats(n *node, s host.Stats) { st := store.ServerStat{ ServerID: n.id, At: s.At.UTC(), CPUPercent: s.CPUPercent, MemoryPercent: s.MemoryPercent, DiskPercent: s.DiskPercent, } - f.bus.Publish(events.Event{Type: events.HostStats, Data: st}) - - free := 100 - s.DiskPercent f.mu.Lock() if len(f.samples) >= maxSamples { f.samples = f.samples[1:] } f.samples = append(f.samples, st) - minFree, name := f.minFree, n.name - low := s.DiskBytes > 0 && free < minFree - alert := low && !f.diskLow[n.id] - f.diskLow[n.id] = low - f.mu.Unlock() - if alert { - f.log.Warn("disk almost full", "server_id", n.id, "free_percent", free) - f.notify(func(dashboard string) notify.Event { - return notify.DiskLowEvent(dashboard, n.id, name, free, int(minFree)) - }) + // The sample of an agent is its last contact. + if !n.Local() && st.At.After(f.seen[n.id]) { + f.seen[n.id] = st.At } + f.mu.Unlock() + f.bus.Publish(events.HostStats(st)) } -// flushStats writes the kept samples, and the time of the last sample of -// each agent as its last contact. +// flushStats writes the kept samples, and the last contact of each agent. func (f *Fleet) flushStats(ctx context.Context) { f.mu.Lock() batch := make([]store.ServerStat, 0, len(f.samples)) - seen := map[string]time.Time{} for _, st := range f.samples { // The row of a removed server is gone, and its samples would fail the batch. - if f.removed[st.ServerID] { - continue + if !f.removed[st.ServerID] { + batch = append(batch, st) } - batch = append(batch, st) - if st.ServerID != f.localID && st.At.After(seen[st.ServerID]) { - seen[st.ServerID] = st.At + } + seen := f.seen + for id := range seen { + if f.removed[id] { + delete(seen, id) } } - f.samples = nil + f.samples, f.seen = nil, map[string]time.Time{} f.mu.Unlock() - if len(batch) == 0 { - return - } - if err := f.st.AppendServerStats(ctx, batch); err != nil { - f.log.Warn("record host stats", "samples", len(batch), "err", err) + if len(batch) > 0 { + if err := f.st.AppendServerStats(ctx, batch); err != nil { + f.log.Warn("record host stats", "samples", len(batch), "err", err) + } } for id, at := range seen { if err := f.st.TouchServer(ctx, id, at); err != nil { @@ -78,24 +66,16 @@ func (f *Fleet) flushStats(ctx context.Context) { } } -// AgentMirrorStats sums the registry mirror stats of the connected agents. -// An agent that does not answer adds nothing. -func (f *Fleet) AgentMirrorStats(ctx context.Context) mirror.Stats { - f.mu.Lock() - var sessions []*agent.Session - for _, n := range f.nodes { - if n.sess != nil { - sessions = append(sessions, n.sess) - } - } - f.mu.Unlock() +// MirrorStats sums the registry mirror stats of the connected servers: +// the api Mirrors service. A server that does not answer adds nothing. +func (f *Fleet) MirrorStats(ctx context.Context) agent.MirrorStats { ctx, cancel := context.WithTimeout(ctx, callTimeout) defer cancel() - var sum mirror.Stats - for _, s := range sessions { - m, err := s.MirrorStats(ctx) + var sum agent.MirrorStats + for _, n := range f.connected() { + m, err := n.MirrorStats(ctx) if err != nil { - f.log.Warn("read mirror stats", "server_id", s.ServerID(), "err", err) + f.log.Warn("read mirror stats", "server_id", n.id, "err", err) continue } sum.Add(m) diff --git a/internal/fleet/update.go b/internal/fleet/update.go index 2103e8c..5c5cd38 100644 --- a/internal/fleet/update.go +++ b/internal/fleet/update.go @@ -2,15 +2,42 @@ package fleet import ( "context" + "errors" "fmt" - "github.com/danielgtaylor/huma/v2" - - "github.com/runplane/runplane/internal/api" + "github.com/runplane/runplane/internal/agent" + "github.com/runplane/runplane/internal/audit" "github.com/runplane/runplane/internal/store" "github.com/runplane/runplane/internal/version" ) +// UpdateState tells whether a server runs the release of the primary. +type UpdateState string + +const ( + // UpdateCurrent: the server runs the version of the primary. + UpdateCurrent UpdateState = "current" + // UpdateAvailable: UpdateAgent installs the release of the primary. + UpdateAvailable UpdateState = "available" + UpdateUpdating UpdateState = "updating" + // UpdateFailed: the last update failed. The update_error of the server + // tells why, and UpdateAgent tries again. + UpdateFailed UpdateState = "failed" + // UpdateNone: runplane cannot update the server, because the primary + // is not a release or the server has not reported its version. + UpdateNone UpdateState = "none" +) + +// The errors of UpdateAgent. +var ( + ErrUpToDate = errors.New("the server runs the version of the primary") + ErrUpdateInProgress = errors.New("an update of the server runs") + ErrCannotUpdate = errors.New("the primary is not a release, or the server has not reported its version") + // errTried means that the update to this version failed before, and + // only an admin tries it again. + errTried = errors.New("the update to this version failed before") +) + // agentUpdate is the last update of an agent to the release of the primary. // An agent must run the release of the primary, so that the two speak the // same protocol. @@ -19,74 +46,84 @@ type agentUpdate struct { // running is true from the start until the agent connects again or // the update fails. running bool - err string // why the update failed } -// UpdateState tells whether a server runs the release of the primary: the -// api Servers.UpdateState. -func (f *Fleet) UpdateState(v store.Server) string { - to := version.Version +func (f *Fleet) UpdateState(v store.Server) UpdateState { f.mu.Lock() - u := f.updates[v.ID] + running := f.updates[v.ID].running f.mu.Unlock() switch { - case u.running: - return api.UpdateUpdating - case v.Kind == store.ServerLocal || v.Version == to: - return api.UpdateCurrent - case !releaseVersion.MatchString(to) || v.Version == "": - return api.UpdateNone - case u.to == to && u.err != "": - return api.UpdateFailed + case running: + return UpdateUpdating + case v.Kind == store.ServerLocal || v.Version == f.version: + return UpdateCurrent + case !version.IsRelease(f.version) || v.Version == "": + return UpdateNone + case v.UpdateError != "": + return UpdateFailed } - return api.UpdateAvailable + return UpdateAvailable } -// UpdateAgent makes a connected agent install the release of the primary: -// the api Servers.UpdateAgent. The update runs in the background. -func (f *Fleet) UpdateAgent(_ context.Context, serverID string) error { +// UpdateAgent makes a connected agent install the release of the primary. +// The update runs in the background. +func (f *Fleet) UpdateAgent(ctx context.Context, by audit.Actor, serverID string) error { n := f.node(serverID) - if n == nil || n.sess == nil { - return huma.Error409Conflict("The server is not connected, so it cannot update.") + if n == nil { + return ErrNotConnected } - if !f.beginUpdate(n.id, n.version, true) { - return huma.Error409Conflict("The server runs the release of the primary, or an update runs.") + if err := f.beginUpdate(ctx, n.id, n.version, true); err != nil { + return err } + f.mu.Lock() + name := n.name + f.mu.Unlock() + f.audit.Record(ctx, by, audit.ServerUpdateAgent, n.id, map[string]any{"name": name, "from": n.version, "to": f.version}) f.goBg(func(ctx context.Context) { f.runUpdate(ctx, n) }) return nil } // beginUpdate marks an update of the agent id, which runs version from, as -// running. It reports false if the primary is not a release, the agent runs -// its version, or an update runs. Without again, it also reports false for a -// version that the fleet tried before on this agent, so that each connection -// does not try again. -func (f *Fleet) beginUpdate(id, from string, again bool) bool { - to := version.Version - if !releaseVersion.MatchString(to) || from == to { - return false +// running, and clears the error of its last update. Without again, it +// refuses a version that the fleet tried before on this agent, so that each +// connection does not try again. +func (f *Fleet) beginUpdate(ctx context.Context, id, from string, again bool) error { + switch { + case from == f.version: + return ErrUpToDate + case !version.IsRelease(f.version) || from == "": + return ErrCannotUpdate } f.mu.Lock() - defer f.mu.Unlock() - if u := f.updates[id]; u.running || u.to == to && !again { - return false + u := f.updates[id] + switch { + case u.running: + f.mu.Unlock() + return ErrUpdateInProgress + case u.to == f.version && !again: + f.mu.Unlock() + return errTried } - f.updates[id] = agentUpdate{to: to, running: true} - return true + f.updates[id] = agentUpdate{to: f.version, running: true} + f.mu.Unlock() + f.setUpdateError(ctx, id, "") + return nil } // runUpdate sends the update to the agent. The agent replaces its binary, -// restarts, and connects again. Its sandboxes keep running. +// restarts, and connects again. Its sandboxes keep running. The agent +// closes the connection for the restart, maybe before its answer arrives, +// so a closed connection does not end the update: the next connection of +// the agent or the unreachable check does. func (f *Fleet) runUpdate(ctx context.Context, n *node) { - to := version.Version f.publishServer(ctx, n.id) - f.log.Info("update agent", "server_id", n.id, "from", n.version, "to", to) + f.log.Info("update agent", "server_id", n.id, "from", n.version, "to", f.version) uctx, cancel := context.WithTimeout(ctx, updateTimeout) - err := n.Update(uctx, to) + err := n.Update(uctx, f.version) cancel() - if err != nil { - f.log.Warn("update agent", "server_id", n.id, "to", to, "err", err) - f.failUpdate(ctx, n.id, fmt.Sprintf("The update to runplane %s failed: %v.", to, err)) + if err != nil && !errors.Is(err, agent.ErrClosed) { + f.log.Warn("update agent", "server_id", n.id, "to", f.version, "err", err) + f.failUpdate(ctx, n.id, fmt.Sprintf("The update to runplane %s failed: %v.", f.version, err)) } } @@ -98,30 +135,30 @@ func (f *Fleet) endUpdate(ctx context.Context, serverID, version string) { f.updates[serverID] = agentUpdate{to: u.to} } f.mu.Unlock() - if u.running && version != u.to { + switch { + case u.running && version != u.to: f.failUpdate(ctx, serverID, fmt.Sprintf("The agent connected again with runplane %s, not %s.", version, u.to)) + case version == f.version: + f.setUpdateError(ctx, serverID, "") } } -// failUpdate ends the running update of a server with an error, which -// also goes to the status detail of the server. func (f *Fleet) failUpdate(ctx context.Context, serverID, why string) { f.mu.Lock() u := f.updates[serverID] if u.running { - f.updates[serverID] = agentUpdate{to: u.to, err: why} + f.updates[serverID] = agentUpdate{to: u.to} } f.mu.Unlock() if !u.running { return } - v, err := f.st.GetServer(ctx, serverID) - if err != nil { - return - } - detail := truncate(why + " Run sudo runplane update on this server.") - if err := f.st.SetServerStatus(ctx, serverID, v.Status, detail); err != nil { - f.log.Warn("set server status", "server_id", serverID, "err", err) - } + f.setUpdateError(ctx, serverID, truncate(why+" Select Update agent to try again, or run curl -fsSL /install.sh | sudo sh on this server.")) f.publishServer(ctx, serverID) } + +func (f *Fleet) setUpdateError(ctx context.Context, serverID, msg string) { + if err := f.st.SetServerUpdateError(ctx, serverID, msg, f.now()); err != nil { + f.log.Warn("record the update error", "server_id", serverID, "err", err) + } +} diff --git a/internal/ghapp/ghapp.go b/internal/ghapp/ghapp.go new file mode 100644 index 0000000..81e082f --- /dev/null +++ b/internal/ghapp/ghapp.go @@ -0,0 +1,119 @@ +// Package ghapp holds the facts of the GitHub App of runplane that need no +// call to GitHub: the account types, the permissions of the app, the labels +// that its pull request moves, and the web pages of an account. +package ghapp + +import ( + "errors" + "net/url" + "slices" + "strconv" + "strings" + + "github.com/runplane/runplane/images" +) + +// The account types of GitHub that runplane runs jobs for. An organization +// has runners for all of its repositories. A personal account has runners +// only for each repository. +const ( + Organization = "Organization" + User = "User" +) + +// DefaultRunnerGroup is the ID of the "Default" runner group. A free +// organization can only use this group, and a repository has only this one. +const DefaultRunnerGroup = 1 + +type Account struct { + Type string `json:"type"` // Organization, User, or another GitHub account type + Login string `json:"login"` +} + +// Supported tells if runplane can run jobs for the account type. +func (a Account) Supported() bool { return a.Type == Organization || a.Type == User } + +// Permission is a permission of a GitHub App, with its access: read or write. +type Permission struct{ Name, Access string } + +// AppPermissions are the permissions of the GitHub App of runplane on the +// account. An organization uses organization runners. A personal account +// can only use repository runners, which need administration. +func (a Account) AppPermissions() []Permission { + runners := Permission{"administration", "write"} + if a.Type == Organization { + runners = Permission{"organization_self_hosted_runners", "write"} + } + return slices.Concat([]Permission{runners, {"actions", "write"}}, PullPermissions, []Permission{{"metadata", "read"}}) +} + +// PullPermissions let runplane open the pull request that moves the jobs of +// a repository to runplane. A change in .github/workflows needs workflows +// next to contents. +var PullPermissions = []Permission{{"contents", "write"}, {"pull_requests", "write"}, {"workflows", "write"}} + +// ErrNoPullPermissions tells that the account of an installation has not +// accepted PullPermissions. An app from before runplane asked for them gets +// them in its settings on GitHub. +var ErrNoPullPermissions = errors.New("the GitHub App needs read and write access to Contents, Pull requests, and Workflows") + +// HostedLabels are the runs-on labels of GitHub-hosted runners that the +// pull request of runplane moves to a pool: ubuntu-latest, and the label of +// the Ubuntu release of the default image. A builtin image has the name of +// the GitHub label of its release. +var HostedLabels = []string{"ubuntu-latest", images.Default} + +// HostedLabelsText gives HostedLabels for a sentence, as "a or b", with +// quote on the two sides of each label. +func HostedLabelsText(quote string) string { + quoted := make([]string, len(HostedLabels)) + for i, l := range HostedLabels { + quoted[i] = quote + l + quote + } + return strings.Join(quoted, " or ") +} + +// NewAppURL is the page on the web host web where the account makes a new +// app from a manifest. state comes back with the manifest code. +func (a Account) NewAppURL(web, state string) string { + return a.settingsURL(web) + "/apps/new?state=" + url.QueryEscape(state) +} + +// AppURL is the settings page of the app slug of the account: its name, +// permissions, and keys. +func (a Account) AppURL(web, slug string) string { + return a.settingsURL(web) + "/apps/" + url.PathEscape(slug) +} + +// AppPermissionsURL is the page where the account changes the permissions of +// the app slug. +func (a Account) AppPermissionsURL(web, slug string) string { + return a.AppURL(web, slug) + "/permissions" +} + +// AppDeleteURL is the page with the "Delete GitHub App" button of the app +// slug. +func (a Account) AppDeleteURL(web, slug string) string { + return a.AppURL(web, slug) + "/advanced" +} + +// InstallationURL is the settings page of the installation id on the +// account, where the account owner chooses the repositories. +func (a Account) InstallationURL(web string, id int64) string { + return a.settingsURL(web) + "/installations/" + strconv.FormatInt(id, 10) +} + +// AppInstallURL is the page that installs the app slug on more accounts or +// repositories. +func AppInstallURL(web, slug string) string { + return web + "/apps/" + url.PathEscape(slug) + "/installations/new" +} + +// settingsURL is the settings page of the account. The pages of its apps +// and of its installations are below it. +func (a Account) settingsURL(web string) string { + if a.Type == Organization { + return web + "/organizations/" + url.PathEscape(a.Login) + "/settings" + } + return web + "/settings" +} diff --git a/internal/ghrest/ghrest.go b/internal/ghrest/ghrest.go new file mode 100644 index 0000000..ad1596d --- /dev/null +++ b/internal/ghrest/ghrest.go @@ -0,0 +1,373 @@ +// Package ghrest calls the GitHub REST API and downloads files from GitHub. +// It sends the headers that GitHub asks for, tries again after network +// errors and rate limits, and reads all pages of a list. +package ghrest + +import ( + "bytes" + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/runplane/runplane/internal/version" +) + +const ( + API = "https://api.github.com" + Web = "https://github.com" +) + +const ( + callTimeout = 30 * time.Second // for one attempt, the body read included + maxRateWait = time.Minute // longest rate limit wait in one call +) + +// Client calls GitHub. The zero value calls api.github.com. +type Client struct { + URL string // the API URL without a trailing slash; API if empty + // RetryWait is the first wait after a network error or a 5xx answer; + // one second if zero. + RetryWait time.Duration +} + +// Call is one REST request. Path is relative to the API URL, or absolute. +type Call struct { + Method string + Path string + Body any // sent as JSON if not nil + Auth Auth // nil for an anonymous call + Timeout time.Duration // of one attempt with its body; 0 for 30 seconds + + download bool +} + +// Auth gives the Authorization header of a call. An empty header makes the +// call anonymous. +type Auth interface { + Header(ctx context.Context) (string, error) +} + +// Renewable is an Auth with a cached credential. After GitHub refuses the +// credential, Do calls Invalidate and tries once more. +type Renewable interface { + Auth + Invalidate() +} + +// APIError is a GitHub answer with an error status. A call that the +// anonymous rate limit stops gets a copy of the answer that set the limit. +type APIError struct { + Method, Path string + Status int + Message string + // RetryAt is set when GitHub rate-limits the call and the wait is + // longer than runplane accepts in one call. + RetryAt time.Time +} + +func (e *APIError) Error() string { + msg := fmt.Sprintf("GitHub %s %s: %d %s", e.Method, e.Path, e.Status, http.StatusText(e.Status)) + if e.Message != "" { + msg += ": " + e.Message + } + if !e.RetryAt.IsZero() { + msg += fmt.Sprintf(" (rate limit, try again after %s)", e.RetryAt.UTC().Format(time.RFC3339)) + } + return msg +} + +// Do sends the call and returns a response with a status below 400. The +// caller closes the body; that also ends the timeout of the attempt. Only a +// GET or a HEAD is sent again after a network error or a 5xx answer. +func (c Client) Do(ctx context.Context, cl Call) (*http.Response, error) { + safe := cl.Method == http.MethodGet || cl.Method == http.MethodHead + renew, _ := cl.Auth.(Renewable) + failures, limits := 0, 0 + for { + var authorization string + if cl.Auth != nil { + var err error + if authorization, err = cl.Auth.Header(ctx); err != nil { + return nil, err + } + } + var shared *budget + if authorization == "" && !cl.download { + shared = anonymous(c.url()) + if err := shared.check(cl); err != nil { + return nil, err + } + } + resp, err := c.send(ctx, cl, authorization) + if err == nil { + if resp.StatusCode < 400 { + return resp, nil + } + apiErr := readAPIError(cl, resp) + switch wait, limited := rateLimitWait(resp, apiErr.Message); { + case limited && wait <= maxRateWait && limits < 3: + limits++ + if err := sleep(ctx, wait); err != nil { + return nil, err + } + continue + case limited: + apiErr.RetryAt = time.Now().Add(wait) + if shared != nil { + shared.hold(*apiErr) + } + return nil, apiErr + case resp.StatusCode == http.StatusUnauthorized && renew != nil: + renew.Invalidate() + renew = nil + continue + case resp.StatusCode < 500: + return nil, apiErr + } + err = apiErr + } + if ctx.Err() != nil || !safe || failures >= 2 { + return nil, err + } + failures++ + if sleep(ctx, cmp.Or(c.RetryWait, time.Second)<<(failures-1)) != nil { + return nil, err + } + } +} + +// JSON sends the call and decodes a JSON answer into out (if not nil). +func (c Client) JSON(ctx context.Context, cl Call, out any) error { + resp, err := c.Do(ctx, cl) + if err != nil { + return err + } + defer resp.Body.Close() + if out == nil || resp.StatusCode == http.StatusNoContent { + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("decode GitHub %s %s: %w", cl.Method, callPath(cl.Path), err) + } + return nil +} + +// List reads all pages of a GET list. key is the field of the page object +// that holds the items, or "" when each page is a JSON array. path must +// have per_page in its query. +func List[T any](ctx context.Context, c Client, a Auth, path, key string) ([]T, error) { + var all []T + for next := path; next != ""; { + resp, err := c.Do(ctx, Call{Method: http.MethodGet, Path: next, Auth: a}) + if err != nil { + return nil, err + } + var page []T + err = decodePage(resp.Body, key, &page) + resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("decode GitHub GET %s: %w", callPath(next), err) + } + all = append(all, page...) + next = nextLink(resp.Header.Get("Link")) + } + return all, nil +} + +// Download sends a GET for a file outside the API, such as a release asset, +// without the API headers and a token. It follows redirects and tries again +// as Do does. Only ctx limits the time. The caller closes the body. +func (c Client) Download(ctx context.Context, url string) (*http.Response, error) { + return c.Do(ctx, Call{Method: http.MethodGet, Path: url, download: true}) +} + +func (c Client) url() string { return cmp.Or(c.URL, API) } + +func (c Client) send(ctx context.Context, cl Call, authorization string) (*http.Response, error) { + u := cl.Path + if strings.HasPrefix(u, "/") { + u = c.url() + u + } + var body io.Reader + if cl.Body != nil { + b, err := json.Marshal(cl.Body) + if err != nil { + return nil, fmt.Errorf("encode request %s %s: %w", cl.Method, callPath(cl.Path), err) + } + body = bytes.NewReader(b) + } + var actx context.Context + var cancel context.CancelFunc + if cl.download { + actx, cancel = context.WithCancel(ctx) + } else { + actx, cancel = context.WithTimeout(ctx, cmp.Or(cl.Timeout, callTimeout)) + } + req, err := http.NewRequestWithContext(actx, cl.Method, u, body) + if err != nil { + cancel() + return nil, fmt.Errorf("make request %s %s: %w", cl.Method, callPath(cl.Path), err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("User-Agent", "runplane/"+version.Version) + if !cl.download { + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + } + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + cancel() + // The url.Error has the full URL, and a signed URL must not go + // to the logs. + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + return nil, fmt.Errorf("GitHub %s %s: %w", cl.Method, callPath(cl.Path), err) + } + resp.Body = cancelOnClose{resp.Body, cancel} + return resp, nil +} + +// callPath is the path of a call without the host and the query. +func callPath(p string) string { + if u, err := url.Parse(p); err == nil { + return u.Path + } + return p +} + +func readAPIError(cl Call, resp *http.Response) *APIError { + e := &APIError{Method: cl.Method, Path: callPath(cl.Path), Status: resp.StatusCode} + defer resp.Body.Close() + if cl.download { + return e + } + b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + var body struct { + Message string `json:"message"` + } + if json.Unmarshal(b, &body) == nil { + e.Message = body.Message + } else { + e.Message = strings.TrimSpace(string(b)) + } + return e +} + +// rateLimitWait tells if a response is a rate limit and how long to wait. +// GitHub sends Retry-After for secondary limits and X-RateLimit-Reset for +// the primary limit. Without either, GitHub asks for at least one minute. +func rateLimitWait(resp *http.Response, message string) (time.Duration, bool) { + if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusTooManyRequests { + return 0, false + } + if s, err := strconv.Atoi(resp.Header.Get("Retry-After")); err == nil && s >= 0 { + return time.Duration(s) * time.Second, true + } + if resp.Header.Get("X-RateLimit-Remaining") == "0" { + if reset, err := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 64); err == nil { + return max(time.Until(time.Unix(reset, 0))+time.Second, 0), true + } + return time.Minute, true + } + if resp.StatusCode == http.StatusTooManyRequests || strings.Contains(strings.ToLower(message), "secondary rate limit") { + return time.Minute, true + } + return 0, false +} + +// budget is the rate limit of the anonymous calls to one API URL. GitHub +// counts these calls by IP address, so all clients of the process share it: +// after GitHub limits one call for longer than a call waits, the other +// calls fail at once until the limit ends. +type budget struct { + mu sync.Mutex + held APIError // the answer that limits the calls until its RetryAt +} + +var budgets sync.Map // of *budget by API URL + +func anonymous(api string) *budget { + b, _ := budgets.LoadOrStore(api, &budget{}) + return b.(*budget) +} + +func (b *budget) check(cl Call) error { + b.mu.Lock() + defer b.mu.Unlock() + if !time.Now().Before(b.held.RetryAt) { + return nil + } + e := b.held + e.Method, e.Path = cl.Method, callPath(cl.Path) + return &e +} + +func (b *budget) hold(e APIError) { + b.mu.Lock() + defer b.mu.Unlock() + b.held = e +} + +func decodePage(r io.Reader, key string, out any) error { + if key == "" { + return json.NewDecoder(r).Decode(out) + } + var obj map[string]json.RawMessage + if err := json.NewDecoder(r).Decode(&obj); err != nil { + return err + } + raw, ok := obj[key] + if !ok || string(raw) == "null" { + return nil + } + return json.Unmarshal(raw, out) +} + +// nextLink returns the rel="next" URL of a Link header. +func nextLink(header string) string { + for part := range strings.SplitSeq(header, ",") { + target, params, ok := strings.Cut(part, ";") + if ok && strings.Contains(params, `rel="next"`) { + return strings.Trim(strings.TrimSpace(target), "<>") + } + } + return "" +} + +type cancelOnClose struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (b cancelOnClose) Close() error { + err := b.ReadCloser.Close() + b.cancel() + return err +} + +func sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/internal/ghrest/ghrest_test.go b/internal/ghrest/ghrest_test.go new file mode 100644 index 0000000..9bd6990 --- /dev/null +++ b/internal/ghrest/ghrest_test.go @@ -0,0 +1,185 @@ +package ghrest + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "testing" + "time" +) + +func TestRetries(t *testing.T) { + reset := time.Now().Add(time.Hour).Truncate(time.Second) + tests := []struct { + name string + method string + first func(w http.ResponseWriter) + wantCalls int + wantWait time.Duration + wantRetry time.Time // RetryAt of the error, zero for no error + }{ + { + name: "secondary limit with Retry-After waits and tries again", method: http.MethodGet, + first: func(w http.ResponseWriter) { + w.Header().Set("Retry-After", "1") + writeJSON(w, http.StatusForbidden, map[string]any{"message": "You have exceeded a secondary rate limit."}) + }, + wantCalls: 2, wantWait: time.Second, + }, + { + name: "429 with Retry-After waits and tries again", method: http.MethodPost, + first: func(w http.ResponseWriter) { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + }, + wantCalls: 2, wantWait: time.Second, + }, + { + name: "primary limit with a far reset fails at once", method: http.MethodGet, + first: func(w http.ResponseWriter) { + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) + writeJSON(w, http.StatusForbidden, map[string]any{"message": "API rate limit exceeded"}) + }, + wantCalls: 1, wantRetry: reset, + }, + { + name: "403 without rate limit signs is no rate limit", method: http.MethodGet, + first: func(w http.ResponseWriter) { + writeJSON(w, http.StatusForbidden, map[string]any{"message": "Resource not accessible by integration"}) + }, + wantCalls: 1, + }, + { + name: "5xx of a read tries again", method: http.MethodGet, + first: func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadGateway) }, + wantCalls: 2, + }, + { + // A POST such as a re-run must not happen twice. + name: "5xx of a write fails at once", method: http.MethodPost, + first: func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadGateway) }, + wantCalls: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls == 1 { + tt.first(w) + return + } + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) + })) + defer srv.Close() + c := Client{URL: srv.URL, RetryWait: time.Millisecond} + start := time.Now() + var out struct{ OK bool } + err := c.JSON(context.Background(), Call{Method: tt.method, Path: "/x"}, &out) + if calls != tt.wantCalls { + t.Errorf("calls = %d, want %d", calls, tt.wantCalls) + } + // Only the second try gets an answer. + if (err == nil) != (tt.wantCalls == 2) || err == nil && !out.OK { + t.Errorf("err = %v, answer = %v", err, out.OK) + } + if waited := time.Since(start); waited < tt.wantWait { + t.Errorf("waited %s, want at least %s", waited, tt.wantWait) + } + var apiErr *APIError + var retryAt time.Time + if errors.As(err, &apiErr) { + retryAt = apiErr.RetryAt + } + if d := retryAt.Sub(tt.wantRetry); d < 0 || d > 5*time.Second { + t.Errorf("RetryAt = %s, want about %s", retryAt, tt.wantRetry) + } + }) + } +} + +// GitHub limits the anonymous calls of an IP address, so after one +// anonymous call gets a long limit, the others wait for its end too. +func TestAnonymousCallsShareTheRateLimit(t *testing.T) { + reset := time.Now().Add(time.Hour).Truncate(time.Second) + calls := map[string]int{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls[r.URL.Path]++ + if r.Header.Get("Authorization") == "" { + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) + writeJSON(w, http.StatusForbidden, map[string]any{"message": "API rate limit exceeded"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{}) + })) + defer srv.Close() + ctx := context.Background() + if err := (Client{URL: srv.URL}).JSON(ctx, Call{Method: http.MethodGet, Path: "/first"}, nil); err == nil { + t.Fatal("the first anonymous call has no error") + } + + err := (Client{URL: srv.URL}).JSON(ctx, Call{Method: http.MethodGet, Path: "/second"}, nil) + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.Path != "/second" { + t.Fatalf("second anonymous call: %v, want a rate limit error", err) + } + if d := apiErr.RetryAt.Sub(reset); d < 0 || d > 5*time.Second { + t.Errorf("RetryAt = %s, want about %s", apiErr.RetryAt, reset) + } + if calls["/second"] != 0 { + t.Errorf("the second anonymous call went to GitHub") + } + if err := (Client{URL: srv.URL}).JSON(ctx, Call{Method: http.MethodGet, Path: "/token", Auth: token("ghs_x")}, nil); err != nil { + t.Errorf("call with a token: %v", err) + } +} + +type token string + +func (t token) Header(context.Context) (string, error) { return "Bearer " + string(t), nil } + +// List follows the rel="next" link of each page, and the Link header of +// GitHub also has the prev, first, and last pages. +func TestList(t *testing.T) { + var srv *httptest.Server + calls := 0 + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + page := func(n int) string { return "<" + srv.URL + "/repos?per_page=2&page=" + strconv.Itoa(n) + ">" } + if calls > 3 { // a List that does not stop + writeJSON(w, http.StatusNotFound, map[string]any{"message": "Not Found"}) + return + } + switch r.URL.Query().Get("page") { + case "", "1": + w.Header().Set("Link", page(2)+`; rel="next", `+page(3)+`; rel="last"`) + writeJSON(w, http.StatusOK, map[string]any{"total_count": 5, "items": []string{"a", "b"}}) + case "2": + w.Header().Set("Link", page(1)+`; rel="prev", `+page(3)+`; rel="next", `+page(3)+`; rel="last", `+page(1)+`; rel="first"`) + writeJSON(w, http.StatusOK, map[string]any{"total_count": 5, "items": []string{"c", "d"}}) + case "3": + w.Header().Set("Link", page(2)+`; rel="prev", `+page(1)+`; rel="first"`) + writeJSON(w, http.StatusOK, map[string]any{"total_count": 5, "items": []string{"e"}}) + default: + writeJSON(w, http.StatusNotFound, map[string]any{"message": "Not Found"}) + } + })) + defer srv.Close() + got, err := List[string](context.Background(), Client{URL: srv.URL}, token("ghs_x"), "/repos?per_page=2", "items") + if err != nil || !slices.Equal(got, []string{"a", "b", "c", "d", "e"}) || calls != 3 { + t.Errorf("List = %q, %v after %d calls; want a to e from 3 pages", got, err, calls) + } +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) +} diff --git a/internal/github/app.go b/internal/github/app.go index 94ffd3e..3626a7e 100644 --- a/internal/github/app.go +++ b/internal/github/app.go @@ -14,30 +14,37 @@ import ( "strings" "time" + "github.com/runplane/runplane/internal/ghapp" + "github.com/runplane/runplane/internal/ghrest" "github.com/runplane/runplane/internal/ids" "github.com/runplane/runplane/internal/store" ) // Manifest returns the GitHub URL for the manifest form and the manifest -// JSON. accountType is "organization" or "user". -func (s *Service) Manifest(ctx context.Context, base, accountType, organization string) (string, string, error) { - if accountType != "organization" && accountType != "user" { - return "", "", fmt.Errorf("unknown account type %q", accountType) - } +// JSON of a new app of owner: an organization with its login, or the +// personal account of the user, with no login. +func (s *Service) Manifest(ctx context.Context, base string, owner ghapp.Account) (string, string, error) { settings, err := s.st.GetSettings(ctx) if err != nil { return "", "", err } state := rand.Text() now := time.Now() + if err := s.st.DeleteExpiredManifestStates(ctx, now); err != nil { + return "", "", err + } err = s.st.CreateManifestState(ctx, store.ManifestState{ - StateHash: hashState(state), AccountType: accountType, Organization: organization, + StateHash: hashState(state), AccountType: owner.Type, Organization: owner.Login, CreatedAt: now, ExpiresAt: now.Add(time.Hour), }) if err != nil { return "", "", err } + permissions := map[string]string{} + for _, p := range owner.AppPermissions() { + permissions[p.Name] = p.Access + } manifest, err := json.Marshal(map[string]any{ "name": appName(settings.InstanceName), "url": base, @@ -46,33 +53,12 @@ func (s *Service) Manifest(ctx context.Context, base, accountType, organization "setup_url": base + "/api/github/setup", "setup_on_update": true, "public": false, - "default_permissions": permissions(accountType), + "default_permissions": permissions, }) if err != nil { return "", "", err } - action := s.web + "/settings/apps/new" - if accountType == "organization" { - action = s.web + "/organizations/" + url.PathEscape(organization) + "/settings/apps/new" - } - return action + "?state=" + url.QueryEscape(state), string(manifest), nil -} - -// permissions are the GitHub App permissions of an account type. An -// organization uses organization runners; a personal account can only -// use repository runners, which need administration. The pullPermissions -// let runplane open the pull request that moves jobs to its labels. -func permissions(accountType string) map[string]string { - p := map[string]string{"actions": "write", "metadata": "read"} - for _, name := range pullPermissions { - p[name] = "write" - } - if accountType == "organization" { - p["organization_self_hosted_runners"] = "write" - } else { - p["administration"] = "write" - } - return p + return owner.NewAppURL(s.web, state), string(manifest), nil } // appName makes a name from the instance name. GitHub App names are unique @@ -87,7 +73,7 @@ func appName(instance string) string { // Callback converts the manifest code into the app and stores it. It // returns the installation page of the new app. -func (s *Service) Callback(ctx context.Context, _, code, state string) (string, error) { +func (s *Service) Callback(ctx context.Context, code, state string) (string, error) { if code == "" || state == "" { return "", errors.New("GitHub did not send the code and the state. Start again") } @@ -99,26 +85,24 @@ func (s *Service) Callback(ctx context.Context, _, code, state string) (string, return "", err } var conv struct { - ID int64 `json:"id"` - Slug string `json:"slug"` - Name string `json:"name"` - ClientID string `json:"client_id"` - ClientSecret string `json:"client_secret"` - WebhookSecret string `json:"webhook_secret"` - PEM string `json:"pem"` - HTMLURL string `json:"html_url"` - Owner account `json:"owner"` - } - err = s.api.json(ctx, call{method: http.MethodPost, path: "/app-manifests/" + url.PathEscape(code) + "/conversions"}, &conv) + ID int64 `json:"id"` + Slug string `json:"slug"` + Name string `json:"name"` + ClientID string `json:"client_id"` + PEM string `json:"pem"` + HTMLURL string `json:"html_url"` + Owner account `json:"owner"` + } + err = s.api.JSON(ctx, ghrest.Call{Method: http.MethodPost, Path: "/app-manifests/" + url.PathEscape(code) + "/conversions"}, &conv) if err != nil { return "", fmt.Errorf("make the GitHub App: %w", err) } // For an organization that does not exist, or where the user cannot make // apps, GitHub makes the app on the personal account. - if requested.AccountType == "organization" && !strings.EqualFold(conv.Owner.Login, requested.Organization) { - made := store.GitHubApp{Slug: conv.Slug, OwnerLogin: conv.Owner.Login, OwnerType: conv.Owner.Type, HTMLURL: conv.HTMLURL} + if requested.AccountType == ghapp.Organization && !strings.EqualFold(conv.Owner.Login, requested.Organization) { + made := store.GitHubApp{Slug: conv.Slug, OwnerLogin: conv.Owner.Login, OwnerType: conv.Owner.Type} return "", fmt.Errorf("GitHub made the app on %s, not on the organization %s: the organization does not exist, "+ - "or you cannot make apps in it. Delete the app at %s, then start again", conv.Owner.Login, requested.Organization, appDeleteURL(made)) + "or you cannot make apps in it. Delete the app at %s, then start again", conv.Owner.Login, requested.Organization, s.appDeleteURL(made)) } if _, err := parseKey([]byte(conv.PEM)); err != nil { return "", err @@ -132,12 +116,7 @@ func (s *Service) Callback(ctx context.Context, _, code, state string) (string, app := store.GitHubApp{ ID: ids.New(), AppID: conv.ID, Slug: conv.Slug, Name: conv.Name, ClientID: conv.ClientID, OwnerLogin: conv.Owner.Login, OwnerType: conv.Owner.Type, HTMLURL: conv.HTMLURL, - PrivateKeyEnc: s.box.Seal([]byte(conv.PEM)), - ClientSecretEnc: s.box.Seal([]byte(conv.ClientSecret)), - CreatedAt: time.Now(), - } - if conv.WebhookSecret != "" { - app.WebhookSecretEnc = s.box.Seal([]byte(conv.WebhookSecret)) + PrivateKeyEnc: s.box.Seal([]byte(conv.PEM)), CreatedAt: time.Now(), } if err := s.st.SaveGitHubApp(ctx, app); err != nil { return "", err @@ -145,7 +124,7 @@ func (s *Service) Callback(ctx context.Context, _, code, state string) (string, s.dropCreds() s.changed() s.log.Info("made GitHub App", "slug", app.Slug, "owner", app.OwnerLogin) - return s.web + "/apps/" + url.PathEscape(app.Slug) + "/installations/new", nil + return ghapp.AppInstallURL(s.web, app.Slug), nil } // Disconnect stops the listeners, deletes the scale sets and their runners @@ -164,22 +143,13 @@ func (s *Service) Disconnect(ctx context.Context) error { } err = s.removeApp(ctx) s.changed() - s.log.Info("disconnected GitHub; delete the app on GitHub", "url", appDeleteURL(c.app)) + s.log.Info("disconnected GitHub; delete the app on GitHub", "url", s.appDeleteURL(c.app)) return err } // appDeleteURL is the GitHub page with the "Delete GitHub App" button. -// It is on the host of the app page, so GitHub Enterprise Server works. -func appDeleteURL(app store.GitHubApp) string { - web := "https://github.com" - if u, err := url.Parse(app.HTMLURL); err == nil && u.Host != "" { - web = u.Scheme + "://" + u.Host - } - slug := url.PathEscape(app.Slug) - if app.OwnerType == "Organization" { - return web + "/organizations/" + url.PathEscape(app.OwnerLogin) + "/settings/apps/" + slug + "/advanced" - } - return web + "/settings/apps/" + slug + "/advanced" +func (s *Service) appDeleteURL(app store.GitHubApp) string { + return ghapp.Account{Type: app.OwnerType, Login: app.OwnerLogin}.AppDeleteURL(s.web, app.Slug) } // removeApp deletes all scale sets on GitHub and all app rows. A scale set @@ -211,9 +181,6 @@ func (s *Service) removeApp(ctx context.Context) error { } } s.dropCreds() - s.checkMu.Lock() - clear(s.checks) - s.checkMu.Unlock() if len(failed) > 0 { return fmt.Errorf("runplane forgot the GitHub App, but GitHub did not delete the scale sets %s. "+ "Remove them in the Actions runner settings on GitHub", strings.Join(failed, ", ")) diff --git a/internal/github/app_test.go b/internal/github/app_test.go index a9ebe5f..cf5b817 100644 --- a/internal/github/app_test.go +++ b/internal/github/app_test.go @@ -2,8 +2,6 @@ package github import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "net/http" @@ -12,28 +10,28 @@ import ( "regexp" "strings" "testing" - "time" + "github.com/runplane/runplane/internal/ghapp" "github.com/runplane/runplane/internal/store" ) func TestManifest(t *testing.T) { tests := []struct { - accountType, org string - wantAction string - wantPermissions map[string]any + owner ghapp.Account + wantAction string + wantPermissions map[string]any }{ - {"organization", "acme", "/organizations/acme/settings/apps/new", + {ghapp.Account{Type: ghapp.Organization, Login: "acme"}, "/organizations/acme/settings/apps/new", map[string]any{"organization_self_hosted_runners": "write", "actions": "write", "metadata": "read", "contents": "write", "pull_requests": "write", "workflows": "write"}}, - {"user", "", "/settings/apps/new", + {ghapp.Account{Type: ghapp.User}, "/settings/apps/new", map[string]any{"administration": "write", "actions": "write", "metadata": "read", "contents": "write", "pull_requests": "write", "workflows": "write"}}, } for _, tt := range tests { - t.Run(tt.accountType, func(t *testing.T) { + t.Run(tt.owner.Type, func(t *testing.T) { e := newEnv(t) - action, manifest, err := e.svc.Manifest(context.Background(), "https://ci.example.com", tt.accountType, tt.org) + action, manifest, err := e.svc.Manifest(context.Background(), "https://ci.example.com", tt.owner) if err != nil { t.Fatal(err) } @@ -52,34 +50,33 @@ func TestManifest(t *testing.T) { if !reflect.DeepEqual(m, want) { t.Errorf("manifest = %v, want %v", m, want) } - - // The store keeps only the hash of the state, for one hour. - var stored string - var expires int64 - err = e.st.DB().QueryRow("SELECT state_hash, expires_at FROM manifest_states").Scan(&stored, &expires) - if err != nil { - t.Fatal(err) - } - if sum := sha256.Sum256([]byte(u.Query().Get("state"))); stored != hex.EncodeToString(sum[:]) { - t.Errorf("stored state %q, want the SHA-256 of the state", stored) - } - if d := time.Until(time.UnixMilli(expires)); d < 59*time.Minute || d > time.Hour { - t.Errorf("state expires in %s, want 1 hour", d) - } }) } } -func TestAppName(t *testing.T) { +// GitHub App names are unique on GitHub, and GitHub refuses names longer +// than 34 characters. +func TestManifestNamesTheApp(t *testing.T) { tests := []struct{ instance, want string }{ - {"", `^runplane-[a-z2-7]{6}$`}, + {"", `^runplane-[a-z2-7]{6}$`}, // the default instance name {" Acme CI ", `^Acme CI-[a-z2-7]{6}$`}, - // GitHub refuses app names longer than 34 characters. {"The continuous integration of Acme Corporation", `^The continuous integration-[a-z2-7]{6}$`}, } for _, tt := range tests { - if got := appName(tt.instance); !regexp.MustCompile(tt.want).MatchString(got) { - t.Errorf("appName(%q) = %q, want %s", tt.instance, got, tt.want) + e := newEnv(t) + ctx := context.Background() + if tt.instance != "" { + if _, err := e.st.UpdateSettings(ctx, func(s *store.Settings) error { s.InstanceName = tt.instance; return nil }); err != nil { + t.Fatal(err) + } + } + _, manifest, err := e.svc.Manifest(ctx, "https://ci.example.com", ghapp.Account{Type: ghapp.User}) + if err != nil { + t.Fatal(err) + } + var m struct{ Name string } + if err := json.Unmarshal([]byte(manifest), &m); err != nil || !regexp.MustCompile(tt.want).MatchString(m.Name) { + t.Errorf("instance %q: app name %q, %v; want %s", tt.instance, m.Name, err, tt.want) } } } @@ -97,14 +94,14 @@ func TestCallbackStoresSealedApp(t *testing.T) { "pem": string(testPEM()), "created_at": "2026-11-04T10:00:00Z", }) }) - action, _, err := e.svc.Manifest(ctx, "https://ci.example.com", "organization", "acme") + action, _, err := e.svc.Manifest(ctx, "https://ci.example.com", ghapp.Account{Type: ghapp.Organization, Login: "acme"}) if err != nil { t.Fatal(err) } u, _ := url.Parse(action) state := u.Query().Get("state") - to, err := e.svc.Callback(ctx, "https://ci.example.com", "code123", state) + to, err := e.svc.Callback(ctx, "code123", state) if err != nil { t.Fatal(err) } @@ -118,19 +115,16 @@ func TestCallbackStoresSealedApp(t *testing.T) { if app.AppID != 4242 || app.ClientID != "Iv23.abc" || app.OwnerLogin != "acme" || app.OwnerType != "Organization" { t.Errorf("app = %+v", app) } - if strings.Contains(string(app.PrivateKeyEnc), "PRIVATE KEY") || strings.Contains(string(app.ClientSecretEnc), "secret-1") { - t.Error("the store has the key or the client secret in clear text") + if strings.Contains(string(app.PrivateKeyEnc), "PRIVATE KEY") { + t.Error("the store has the key in clear text") } if key, err := e.box.Open(app.PrivateKeyEnc); err != nil || string(key) != string(testPEM()) { t.Errorf("sealed key does not open to the PEM: %v", err) } - if secret, err := e.box.Open(app.ClientSecretEnc); err != nil || string(secret) != "secret-1" { - t.Errorf("sealed client secret = %q, %v", secret, err) - } // A state works one time, and a bad state does not reach GitHub. for _, s := range []string{state, "bad-state"} { - if _, err := e.svc.Callback(ctx, "https://ci.example.com", "code123", s); err == nil { + if _, err := e.svc.Callback(ctx, "code123", s); err == nil { t.Errorf("callback with the state %q succeeded", s) } } @@ -153,15 +147,15 @@ func TestCallbackRejectsAppOfAnotherAccount(t *testing.T) { "pem": string(testPEM()), "created_at": "2026-11-04T10:00:00Z", }) }) - action, _, err := e.svc.Manifest(ctx, "https://ci.example.com", "organization", "Telemetry") + action, _, err := e.svc.Manifest(ctx, "https://ci.example.com", ghapp.Account{Type: ghapp.Organization, Login: "Telemetry"}) if err != nil { t.Fatal(err) } u, _ := url.Parse(action) - _, err = e.svc.Callback(ctx, "https://ci.example.com", "code123", u.Query().Get("state")) + _, err = e.svc.Callback(ctx, "code123", u.Query().Get("state")) if err == nil || !strings.Contains(err.Error(), "ada") || !strings.Contains(err.Error(), "Telemetry") || - !strings.Contains(err.Error(), "https://github.com/settings/apps/runplane-ab12cd/advanced") { + !strings.Contains(err.Error(), e.srv.URL+"/settings/apps/runplane-ab12cd/advanced") { t.Fatalf("err = %v, want the two accounts and the delete page of the app", err) } app, err := e.st.GetGitHubApp(ctx) @@ -205,20 +199,3 @@ func TestDisconnectNamesScaleSetsLeftOnGitHub(t *testing.T) { t.Errorf("the app is still there after a GitHub failure: %v", err) } } - -func TestAppDeleteURL(t *testing.T) { - tests := []struct { - ownerType, htmlURL, want string - }{ - {"Organization", "", "https://github.com/organizations/acme/settings/apps/runplane-ab12cd/advanced"}, - {"User", "", "https://github.com/settings/apps/runplane-ab12cd/advanced"}, - {"Organization", "https://git.example.com/apps/runplane-ab12cd", - "https://git.example.com/organizations/acme/settings/apps/runplane-ab12cd/advanced"}, - } - for _, tt := range tests { - got := appDeleteURL(store.GitHubApp{Slug: "runplane-ab12cd", OwnerLogin: "acme", OwnerType: tt.ownerType, HTMLURL: tt.htmlURL}) - if got != tt.want { - t.Errorf("appDeleteURL(%s, %q) = %q, want %q", tt.ownerType, tt.htmlURL, got, tt.want) - } - } -} diff --git a/internal/github/auth.go b/internal/github/auth.go index 2911ce8..01a6931 100644 --- a/internal/github/auth.go +++ b/internal/github/auth.go @@ -12,6 +12,7 @@ import ( "github.com/golang-jwt/jwt/v5" + "github.com/runplane/runplane/internal/ghrest" "github.com/runplane/runplane/internal/store" ) @@ -56,7 +57,6 @@ func (s *Service) creds(ctx context.Context) (*creds, error) { return s.cred, nil } -// dropCreds drops the cached app and its installation tokens. func (s *Service) dropCreds() { s.credMu.Lock() s.cred = nil @@ -91,7 +91,7 @@ func signJWT(key *rsa.PrivateKey, issuer string, now time.Time) (string, error) type appAuth struct{ s *Service } -func (a appAuth) header(ctx context.Context) (string, error) { +func (a appAuth) Header(ctx context.Context) (string, error) { c, err := a.s.creds(ctx) if err != nil { return "", err @@ -103,14 +103,12 @@ func (a appAuth) header(ctx context.Context) (string, error) { return "Bearer " + signed, nil } -func (appAuth) invalidate() {} - type installAuth struct { s *Service id int64 } -func (a installAuth) header(ctx context.Context) (string, error) { +func (a installAuth) Header(ctx context.Context) (string, error) { t, err := a.s.installationToken(ctx, a.id) if err != nil { return "", err @@ -118,7 +116,7 @@ func (a installAuth) header(ctx context.Context) (string, error) { return "Bearer " + t, nil } -func (a installAuth) invalidate() { +func (a installAuth) Invalidate() { a.s.tokMu.Lock() delete(a.s.tokens, a.id) a.s.tokMu.Unlock() @@ -142,10 +140,10 @@ func (s *Service) installationToken(ctx context.Context, installationID int64) ( Token string `json:"token"` ExpiresAt time.Time `json:"expires_at"` } - err := s.api.json(ctx, call{ - method: http.MethodPost, - path: fmt.Sprintf("/app/installations/%d/access_tokens", installationID), - auth: appAuth{s}, + err := s.api.JSON(ctx, ghrest.Call{ + Method: http.MethodPost, + Path: fmt.Sprintf("/app/installations/%d/access_tokens", installationID), + Auth: appAuth{s}, }, &out) if err != nil { return "", fmt.Errorf("make installation token: %w", err) diff --git a/internal/github/auth_test.go b/internal/github/auth_test.go index 4459bab..1027823 100644 --- a/internal/github/auth_test.go +++ b/internal/github/auth_test.go @@ -5,29 +5,11 @@ import ( "net/http" "testing" "time" - - "github.com/golang-jwt/jwt/v5" ) -func TestSignJWT(t *testing.T) { - now := time.Unix(1_800_000_000, 0) - raw, err := signJWT(testKey(), testClientID, now) - if err != nil { - t.Fatal(err) - } - var claims jwt.RegisteredClaims - _, err = jwt.ParseWithClaims(raw, &claims, func(*jwt.Token) (any, error) { return &testKey().PublicKey, nil }, - jwt.WithValidMethods([]string{"RS256"}), jwt.WithTimeFunc(func() time.Time { return now })) - if err != nil { - t.Fatalf("parse JWT: %v", err) - } - // GitHub accepts at most 10 minutes from iat to exp. - if claims.Issuer != testClientID || claims.IssuedAt.Unix() != 1_800_000_000-60 || claims.ExpiresAt.Unix() != 1_800_000_000+540 { - t.Errorf("claims = iss %q, iat %d, exp %d; want 60 s before and 9 min after now", claims.Issuer, - claims.IssuedAt.Unix(), claims.ExpiresAt.Unix()) - } -} - +// GitHub tokens live one hour. runplane uses a token again until 5 minutes +// before its end, and asks for a new one after GitHub refuses it. The fake +// GitHub checks the app JWT of each new token (refer to checkJWT). func TestInstallationToken(t *testing.T) { tests := []struct { name string @@ -38,24 +20,23 @@ func TestInstallationToken(t *testing.T) { }{ {"reused while it lives longer than 5 minutes", time.Hour, "", 3, 1}, {"renewed in its last 5 minutes", 4 * time.Minute, "", 3, 3}, - {"renewed once after GitHub refused it", time.Hour, "Bearer ghs_7_1", 4, 2}, + {"renewed once after GitHub refused it", time.Hour, "Bearer ghs_10_1", 4, 2}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := newEnv(t) - e.connect() + e := acmeEnv(t) e.tokenTTL = tt.ttl calls := 0 - e.mux.HandleFunc("GET /api/v3/repos/acme/web", func(w http.ResponseWriter, r *http.Request) { + e.mux.HandleFunc("GET /api/v3/repos/acme/web/actions/runs/555", func(w http.ResponseWriter, r *http.Request) { calls++ if r.Header.Get("Authorization") == tt.refuse { writeJSON(w, http.StatusUnauthorized, map[string]any{"message": "Bad credentials"}) return } - writeJSON(w, http.StatusOK, map[string]any{}) + writeJSON(w, http.StatusOK, runJSON("push", true, 100)) }) for range 3 { - if err := e.svc.api.json(context.Background(), call{method: http.MethodGet, path: "/repos/acme/web", auth: installAuth{e.svc, 7}}, nil); err != nil { + if _, err := e.svc.CheckJob(context.Background(), "acme/web", 555); err != nil { t.Fatal(err) } } diff --git a/internal/github/github.go b/internal/github/github.go index c2f98bf..5194b66 100644 --- a/internal/github/github.go +++ b/internal/github/github.go @@ -1,6 +1,6 @@ // Package github connects runplane to GitHub: the GitHub App manifest flow, // app and installation tokens, REST calls, runner scale sets with their -// listeners, job data for the scheduler, and runner releases. +// listeners, and job data for the scheduler. package github import ( @@ -13,10 +13,8 @@ import ( "sync/atomic" "time" - "github.com/actions/scaleset/listener" - "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/notify" + "github.com/runplane/runplane/internal/ghrest" "github.com/runplane/runplane/internal/scheduler" "github.com/runplane/runplane/internal/secret" "github.com/runplane/runplane/internal/store" @@ -26,49 +24,40 @@ import ( type Scheduler interface { // Attach gives the scheduler the client of a scale set whose listener // starts. The listener sends each message to the returned Scaler. - Attach(set store.ScaleSet, c scheduler.ScaleSetClient) listener.Scaler + Attach(set store.ScaleSet, c scheduler.ScaleSetClient) scheduler.Scaler // Detach tells the scheduler that the scale set has no listener now. // The scheduler must not use its client after this call. Detach(scaleSetID string) } -// Bundles makes the runner bundles of a version on each server. -type Bundles interface { - EnsureLatest(ctx context.Context, version string) error -} - -type Notifier interface { - Notify(ctx context.Context, e notify.Event) error -} - type Config struct { Store *store.Store Box *secret.Box Bus *events.Bus Log *slog.Logger Scheduler Scheduler - Bundles Bundles - Notifier Notifier } // Service implements api.GitHub and scheduler.GitHub. type Service struct { - st *store.Store - box *secret.Box - bus *events.Bus - log *slog.Logger - sched Scheduler - bundles Bundles - notify Notifier - api *rest - web string - owner string // names the listener sessions on GitHub + st *store.Store + box *secret.Box + bus *events.Bus + log *slog.Logger + sched Scheduler + api ghrest.Client + web string + owner string // names the listener sessions on GitHub // ops serializes the changes of installations and scale sets. - ops sync.Mutex - poke chan struct{} - - // base is the parent context of the listeners. Run cancels it. + ops sync.Mutex + // The changes of the settings and the pools, from New on, so that Run + // misses no change that comes before it starts. + settings <-chan events.SettingsChanged + pools <-chan events.PoolsChanged + + // base is the parent context of the listeners and of the subscription. + // Run cancels it. base context.Context stop context.CancelFunc @@ -81,13 +70,6 @@ type Service struct { lmu sync.Mutex listeners map[string]*setListener - checkMu sync.Mutex - checks map[int64]runCheck // CheckJob results by run ID - - // warned has the runner versions with a sent end date warning. Only - // releaseLoop uses it. - warned map[string]bool - pollEvent atomic.Int64 // unix ms of the last github.state event for a poll } @@ -100,32 +82,23 @@ func New(cfg Config) *Service { bus: cfg.Bus, log: cfg.Log, sched: cfg.Scheduler, - bundles: cfg.Bundles, - notify: cfg.Notifier, - api: newREST("https://api.github.com"), - web: "https://github.com", + web: ghrest.Web, owner: cmp.Or(owner, "runplane"), - poke: make(chan struct{}, 1), + settings: events.Subscribe[events.SettingsChanged](base, cfg.Bus), + pools: events.Subscribe[events.PoolsChanged](base, cfg.Bus), base: base, stop: stop, tokens: map[int64]token{}, listeners: map[string]*setListener{}, - checks: map[int64]runCheck{}, - warned: map[string]bool{}, } } // Run syncs the installations and starts the scale set listeners, then // keeps them in step with GitHub, the pools, and the settings. It syncs each -// 5 minutes and checks the runner releases each hour. It returns after ctx -// ends and all listeners have stopped. +// 5 minutes, and does one reconcile for the changes of the pools and the +// settings that wait. It returns after ctx ends and all listeners have +// stopped. func (s *Service) Run(ctx context.Context) { - settings, unsubscribe := s.bus.Subscribe(64) - defer unsubscribe() - - var wg sync.WaitGroup - wg.Go(func() { s.releaseLoop(ctx) }) - s.syncAll(ctx) tick := time.NewTicker(5 * time.Minute) defer tick.Stop() @@ -134,26 +107,30 @@ func (s *Service) Run(ctx context.Context) { case <-ctx.Done(): s.stop() s.stopListeners() - wg.Wait() return case <-tick.C: s.syncAll(ctx) - case <-s.poke: + case <-s.pools: + s.dropChanges() + s.reconcileNow(ctx) + case <-s.settings: + s.dropChanges() s.reconcileNow(ctx) - case e := <-settings: - if e.Type == events.SettingsChanged { - s.reconcileNow(ctx) - } } } } -// PoolsChanged makes the scale sets agree with the pools soon, without -// blocking. -func (s *Service) PoolsChanged(context.Context) { - select { - case s.poke <- struct{}{}: - default: +// dropChanges takes the changes of the settings and the pools that wait, +// because the next reconcile reads the pools and the settings after all of +// them. +func (s *Service) dropChanges() { + for { + select { + case <-s.pools: + case <-s.settings: + default: + return + } } } @@ -181,7 +158,6 @@ func (s *Service) reconcileNow(ctx context.Context) { s.reconcile(ctx) } -// anyInstallation returns an installation that is not suspended. func (s *Service) anyInstallation(ctx context.Context) (store.Installation, bool) { list, err := s.st.ListInstallations(ctx) if err != nil { @@ -197,7 +173,7 @@ func (s *Service) anyInstallation(ctx context.Context) (store.Installation, bool // changed tells the browsers that the GitHub status changed. func (s *Service) changed() { - s.bus.Publish(events.Event{Type: events.GitHubState}) + s.bus.Publish(events.GitHubState{}) } // polledChanged tells the browsers about a new poll time, at most once in @@ -220,3 +196,14 @@ func (q quiet) Enabled(ctx context.Context, l slog.Level) bool { func (q quiet) WithAttrs(a []slog.Attr) slog.Handler { return quiet{q.Handler.WithAttrs(a)} } func (q quiet) WithGroup(n string) slog.Handler { return quiet{q.Handler.WithGroup(n)} } + +func sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/internal/github/github_test.go b/internal/github/github_test.go index 224fdb1..4d567c7 100644 --- a/internal/github/github_test.go +++ b/internal/github/github_test.go @@ -6,23 +6,26 @@ import ( "time" "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/sandbox" "github.com/runplane/runplane/internal/store" ) +// A personal account has a scale set for each pool and repository whose +// jobs runplane takes: not for a disabled pool, an archived repository, or +// a public repository while allow_public_repos is off. func TestRunFollowsSettingsAndPools(t *testing.T) { e := newEnv(t) e.connect() f := e.actions() - e.installations().set([]map[string]any{{ + e.inst.set([]map[string]any{{ "id": 20, "app_id": testAppID, "suspended_at": nil, "account": map[string]any{"login": "octo", "id": 2, "type": "User"}, }}, map[string][]map[string]any{"ghs_20_": { {"id": 200, "full_name": "octo/app", "private": true}, {"id": 201, "full_name": "octo/site", "private": false}, + {"id": 202, "full_name": "octo/old", "private": true, "archived": true}, }}) e.pool("p1", "runplane", true) + e.pool("p0", "runplane-off", false) ctx := context.Background() - e.st.UpdateSettings(ctx, func(s *store.Settings) error { s.Mode = sandbox.ModeVM; return nil }) runCtx, cancel := context.WithCancel(ctx) done := make(chan struct{}) @@ -43,18 +46,18 @@ func TestRunFollowsSettingsAndPools(t *testing.T) { eventually(t, "one scale set for the private repository", listening(1)) e.st.UpdateSettings(ctx, func(s *store.Settings) error { s.AllowPublicRepos = true; return nil }) - e.bus.Publish(events.Event{Type: events.SettingsChanged}) + e.bus.Publish(events.SettingsChanged{}) eventually(t, "a scale set for the public repository too", listening(2)) e.pool("p2", "runplane-big", true) - e.svc.PoolsChanged(ctx) + e.bus.Publish(events.PoolsChanged{}) eventually(t, "scale sets for the new pool", listening(4)) scopes := map[string]int{} for _, sc := range f.state().scopes { scopes[sc]++ } - if scopes["octo--app"] != 2 || scopes["octo--site"] != 2 { - t.Errorf("scale sets by repository = %v, want two in each", scopes) + if len(scopes) != 2 || scopes["octo--app"] != 2 || scopes["octo--site"] != 2 { + t.Errorf("scale sets by repository = %v, want two in octo/app and in octo/site", scopes) } cancel() @@ -72,32 +75,20 @@ func TestRunFollowsSettingsAndPools(t *testing.T) { } } +// A listener polls GitHub all the time. The browsers learn the time of the +// last poll from a github.state event, but at most once in 30 seconds. func TestGitHubStateEvents(t *testing.T) { - e := newEnv(t) - e.connect() - published, unsubscribe := e.bus.Subscribe(16) - defer unsubscribe() - // Polls publish at most once in 30 seconds. - t0 := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) - for _, step := range []struct { - at time.Duration - want int - }{ - {0, 1}, - {10 * time.Second, 0}, - {29 * time.Second, 0}, - {31 * time.Second, 1}, - {40 * time.Second, 0}, - } { - e.svc.polledChanged(t0.Add(step.at)) - if got := countEvents(published, events.GitHubState); got != step.want { - t.Errorf("poll at +%s: %d github.state events, want %d", step.at, got, step.want) - } + e, f, _ := listeningEnv(t) + published := events.Subscribe[events.GitHubState](t.Context(), e.bus) + polls := len(f.state().polls) + eventually(t, "10 more polls", func() bool { return len(f.state().polls) >= polls+10 }) + if n := countEvents(published); n > 1 { + t.Errorf("%d github.state events for 10 polls, want at most 1", n) } if err := e.svc.Disconnect(context.Background()); err != nil { t.Fatal(err) } - if countEvents(published, events.GitHubState) == 0 { + if countEvents(published) == 0 { t.Error("no github.state event after the app was removed") } } diff --git a/internal/github/helpers_test.go b/internal/github/helpers_test.go index 6c816d0..0b9ec4f 100644 --- a/internal/github/helpers_test.go +++ b/internal/github/helpers_test.go @@ -24,11 +24,10 @@ import ( "time" "github.com/actions/scaleset" - "github.com/actions/scaleset/listener" "github.com/golang-jwt/jwt/v5" "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/notify" + "github.com/runplane/runplane/internal/ghrest" "github.com/runplane/runplane/internal/scheduler" "github.com/runplane/runplane/internal/secret" "github.com/runplane/runplane/internal/store" @@ -63,9 +62,8 @@ type env struct { srv *httptest.Server svc *Service sched *fakeScheduler - bundles *fakeBundles - notifier *fakeNotifier - tokens atomic.Int32 // installation tokens that the fake made + inst *fakeInstallations // the installations that Sync reads + tokens atomic.Int32 // installation tokens that the fake made tokenTTL time.Duration } @@ -81,8 +79,7 @@ func newEnv(t *testing.T) *env { t.Fatal(err) } e := &env{t: t, st: st, box: box, bus: &events.Bus{}, mux: http.NewServeMux(), tokenTTL: time.Hour, - sched: &fakeScheduler{got: make(chan *scaleset.RunnerScaleSetMessage, 16)}, - bundles: &fakeBundles{}, notifier: &fakeNotifier{}} + sched: &fakeScheduler{got: make(chan scheduler.Message, 16)}} e.srv = httptest.NewServer(e.mux) t.Cleanup(e.srv.Close) e.mux.HandleFunc("POST /api/v3/app/installations/{id}/access_tokens", func(w http.ResponseWriter, r *http.Request) { @@ -92,9 +89,11 @@ func newEnv(t *testing.T) *env { "expires_at": time.Now().Add(e.tokenTTL), }) }) - e.svc = New(Config{Store: st, Box: box, Bus: e.bus, Log: slog.New(slog.DiscardHandler), Scheduler: e.sched, - Bundles: e.bundles, Notifier: e.notifier}) - e.svc.web, e.svc.api.base = e.srv.URL, e.srv.URL+"/api/v3" + e.inst = e.installations() + e.svc = New(Config{Store: st, Box: box, Bus: e.bus, Log: slog.New(slog.DiscardHandler), Scheduler: e.sched}) + // Config has no option for the GitHub hosts, so the test sets them. + e.svc.web = e.srv.URL + e.svc.api = ghrest.Client{URL: e.srv.URL + "/api/v3", RetryWait: time.Millisecond} t.Cleanup(func() { e.svc.stop() e.svc.stopListeners() @@ -102,6 +101,14 @@ func newEnv(t *testing.T) *env { return e } +// sync reads the installations of the fake and makes the scale sets agree. +func (e *env) sync() { + e.t.Helper() + if err := e.svc.Sync(context.Background()); err != nil { + e.t.Fatal(err) + } +} + // connect stores the GitHub App, as the manifest flow does. func (e *env) connect() { e.t.Helper() @@ -122,6 +129,8 @@ func acmeEnv(t *testing.T) *env { return e } +// install records the installation in the store, and in the fake, so that +// a Sync keeps it. func (e *env) install(id int64, login, accountType string) { e.t.Helper() err := e.st.UpsertInstallation(context.Background(), store.Installation{ @@ -130,6 +139,10 @@ func (e *env) install(id int64, login, accountType string) { if err != nil { e.t.Fatal(err) } + e.inst.mu.Lock() + e.inst.list = append(e.inst.list, map[string]any{"id": id, "app_id": testAppID, "suspended_at": nil, + "account": map[string]any{"login": login, "id": id, "type": accountType}}) + e.inst.mu.Unlock() } func (e *env) pool(id, label string, enabled bool) { @@ -142,11 +155,21 @@ func (e *env) pool(id, label string, enabled bool) { } } -// checkJWT verifies an app JWT with the public key of the test app. +// checkJWT accepts only an app JWT that GitHub accepts: RS256 with the key +// of the app and the client ID as the issuer, an issue time that is not in +// the future, and at most 10 minutes from the issue time to the expiry. func (e *env) checkJWT(r *http.Request) { raw := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") - _, err := jwt.Parse(raw, func(*jwt.Token) (any, error) { return &testKey().PublicKey, nil }, - jwt.WithValidMethods([]string{"RS256"}), jwt.WithIssuer(testClientID)) + var claims jwt.RegisteredClaims + _, err := jwt.ParseWithClaims(raw, &claims, func(*jwt.Token) (any, error) { return &testKey().PublicKey, nil }, + jwt.WithValidMethods([]string{"RS256"}), jwt.WithIssuer(testClientID), jwt.WithIssuedAt(), jwt.WithExpirationRequired()) + switch { + case err != nil: + case claims.IssuedAt == nil: + err = fmt.Errorf("no iat") + case claims.ExpiresAt.Sub(claims.IssuedAt.Time) > 10*time.Minute: + err = fmt.Errorf("iat %s and exp %s are more than 10 minutes apart", claims.IssuedAt, claims.ExpiresAt) + } if err != nil { e.t.Errorf("%s %s: bad app JWT: %v", r.Method, r.URL.Path, err) } @@ -158,15 +181,13 @@ func writeJSON(w http.ResponseWriter, status int, v any) { json.NewEncoder(w).Encode(v) } -// countEvents counts the events of type typ on ch so far. -func countEvents(ch <-chan events.Event, typ string) int { +// countEvents counts the events on ch so far. +func countEvents[E any](ch <-chan E) int { n := 0 for { select { - case e := <-ch: - if e.Type == typ { - n++ - } + case <-ch: + n++ default: return n } @@ -185,53 +206,46 @@ func eventually(t *testing.T, what string, cond func() bool) { } } -type fakeBundles struct{ versions []string } - -func (f *fakeBundles) EnsureLatest(_ context.Context, version string) error { - f.versions = append(f.versions, version) - return nil -} - -type fakeNotifier struct{ sent []notify.Event } - -func (f *fakeNotifier) Notify(_ context.Context, e notify.Event) error { - f.sent = append(f.sent, e) - return nil -} - // fakeScheduler acquires every available job, like the real scheduler does -// for jobs that it accepts. It sends the messages from GitHub to got. +// for jobs that it accepts. It sends the messages with jobs to got. type fakeScheduler struct { last scheduler.ScaleSetClient // of the last attached scale set detached []string max int - got chan *scaleset.RunnerScaleSetMessage + got chan scheduler.Message + + mu sync.Mutex + acquired []int64 // the keys that AcquireJobs gave back } -func (f *fakeScheduler) Attach(_ store.ScaleSet, c scheduler.ScaleSetClient) listener.Scaler { +func (f *fakeScheduler) Attach(_ store.ScaleSet, c scheduler.ScaleSetClient) scheduler.Scaler { f.last = c c.SetMaxRunners(f.max) - return scalerFunc(func(ctx context.Context, msg *scaleset.RunnerScaleSetMessage) error { - if msg == nil || msg.MessageID == listener.InitialMessageID { + return scalerFunc(func(ctx context.Context, m scheduler.Message) error { + if len(m.Available) == 0 { return nil } - var ids []int64 - for _, j := range msg.JobAvailableMessages { - ids = append(ids, j.RunnerRequestID) + var keys []int64 + for _, j := range m.Available { + keys = append(keys, j.RunnerRequestID) } - if _, err := c.AcquireJobs(ctx, ids); err != nil { + acquired, err := c.AcquireJobs(ctx, keys) + if err != nil { return err } - f.got <- msg + f.mu.Lock() + f.acquired = append(f.acquired, acquired...) + f.mu.Unlock() + f.got <- m return nil }) } func (f *fakeScheduler) Detach(id string) { f.detached = append(f.detached, id) } -type scalerFunc func(context.Context, *scaleset.RunnerScaleSetMessage) error +type scalerFunc func(context.Context, scheduler.Message) error -func (f scalerFunc) Scale(ctx context.Context, m *scaleset.RunnerScaleSetMessage) error { +func (f scalerFunc) Scale(ctx context.Context, m scheduler.Message) error { return f(ctx, m) } @@ -242,6 +256,7 @@ type fakeActions struct { actionsState nextID int conflict bool // the session endpoint answers 409 + noUpdate bool // the update of a scale set fails agents map[int64]string // runner ID -> "busy", "gone", or "broken"; others are removable revokeNext bool // the next admin token is revoked at once revoked string @@ -327,8 +342,12 @@ func (e *env) actions() *fakeActions { json.NewDecoder(r.Body).Decode(&s) s.ID, _ = strconv.Atoi(r.PathValue("id")) f.mu.Lock() + defer f.mu.Unlock() + if f.noUpdate { + actionsError(w, http.StatusBadRequest, "InvalidRunnerScaleSetException") + return + } f.sets[s.ID] = s - f.mu.Unlock() writeJSON(w, http.StatusOK, s) }) m.HandleFunc("DELETE "+sets+"/{id}", func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/github/installations.go b/internal/github/installations.go index ced69e2..d89b8a2 100644 --- a/internal/github/installations.go +++ b/internal/github/installations.go @@ -7,13 +7,15 @@ import ( "net/http" "time" + "github.com/runplane/runplane/internal/ghapp" + "github.com/runplane/runplane/internal/ghrest" + "github.com/runplane/runplane/internal/pages" "github.com/runplane/runplane/internal/store" ) type account struct { - Login string `json:"login"` + ghapp.Account ID int64 `json:"id"` - Type string `json:"type"` // Organization | User AvatarURL string `json:"avatar_url"` } @@ -30,34 +32,32 @@ type installation struct { // Setup records the installation that GitHub sends the browser back with, // reads its repositories, and makes its scale sets. It returns the first job // page the first time GitHub connects, and the GitHub settings page after. -func (s *Service) Setup(ctx context.Context, _ string, installationID int64, setupAction string) (string, error) { +func (s *Service) Setup(ctx context.Context, installationID int64, setupAction string) (string, error) { s.ops.Lock() defer s.ops.Unlock() c, err := s.creds(ctx) if err != nil { return "", err } - known, err := s.st.ListInstallations(ctx) + connected, err := s.st.GitHubConnected(ctx) if err != nil { return "", err } - connected := len(known) > 0 if setupAction == "request" || installationID == 0 { - // An organization owner must approve the request on GitHub first. - if connected { - return "/settings/github?requested=1", nil - } - return "/setup/github?requested=1", nil + return pages.GitHubRequested(connected), nil } - var inst installation - err = s.api.json(ctx, call{method: http.MethodGet, path: fmt.Sprintf("/app/installations/%d", installationID), auth: appAuth{s}}, &inst) + inst, err := s.readInstallation(ctx, installationID) if err != nil { - return "", fmt.Errorf("read the installation: %w", err) + return "", err } if inst.AppID != c.app.AppID { return "", fmt.Errorf("installation %d belongs to a different GitHub App", installationID) } + if !inst.Account.Supported() { + return "", fmt.Errorf("installation %d is on the %s account %s: runplane runs jobs only for organizations "+ + "and personal accounts", installationID, inst.Account.Type, inst.Account.Login) + } if err := s.saveInstallation(ctx, inst); err != nil { return "", err } @@ -66,9 +66,9 @@ func (s *Service) Setup(ctx context.Context, _ string, installationID int64, set } s.reconcile(ctx) if connected { - return "/settings/github", nil + return pages.GitHubSettings, nil } - return "/setup/first-job", nil + return pages.FirstJob, nil } // Sync reads the installations, their repositories, and suspensions now, @@ -88,7 +88,7 @@ func (s *Service) syncInstallations(ctx context.Context) error { if _, err := s.creds(ctx); err != nil { return err } - remote, err := list[installation](ctx, s.api, appAuth{s}, "/app/installations?per_page=100", "") + remote, err := ghrest.List[installation](ctx, s.api, appAuth{s}, "/app/installations?per_page=100", "") if err != nil { return fmt.Errorf("list installations: %w", err) } @@ -99,7 +99,7 @@ func (s *Service) syncInstallations(ctx context.Context) error { seen := map[int64]bool{} var errs []error for _, inst := range remote { - if inst.Account.Type != "Organization" && inst.Account.Type != "User" { + if !inst.Account.Supported() { continue } seen[inst.ID] = true @@ -127,6 +127,15 @@ func (s *Service) syncInstallations(ctx context.Context) error { return errors.Join(errs...) } +func (s *Service) readInstallation(ctx context.Context, id int64) (installation, error) { + var inst installation + err := s.api.JSON(ctx, ghrest.Call{Method: http.MethodGet, Path: fmt.Sprintf("/app/installations/%d", id), Auth: appAuth{s}}, &inst) + if err != nil { + return installation{}, fmt.Errorf("read the installation: %w", err) + } + return inst, nil +} + func (s *Service) saveInstallation(ctx context.Context, inst installation) error { return s.st.UpsertInstallation(ctx, store.Installation{ ID: inst.ID, AppID: inst.AppID, AccountLogin: inst.Account.Login, AccountID: inst.Account.ID, @@ -139,7 +148,7 @@ func (s *Service) saveInstallation(ctx context.Context, inst installation) error // syncRepositories stores the repositories that the installation can use. // The GitHub repository object has the JSON names of store.Repository. func (s *Service) syncRepositories(ctx context.Context, installationID int64) error { - repos, err := list[store.Repository](ctx, s.api, installAuth{s, installationID}, "/installation/repositories?per_page=100", "repositories") + repos, err := ghrest.List[store.Repository](ctx, s.api, installAuth{s, installationID}, "/installation/repositories?per_page=100", "repositories") if err != nil { return fmt.Errorf("list repositories of installation %d: %w", installationID, err) } diff --git a/internal/github/installations_test.go b/internal/github/installations_test.go index 3dc59d2..4001494 100644 --- a/internal/github/installations_test.go +++ b/internal/github/installations_test.go @@ -3,7 +3,9 @@ package github import ( "context" "errors" + "fmt" "net/http" + "strconv" "strings" "sync" "testing" @@ -26,14 +28,12 @@ func (e *env) installations() *fakeInstallations { e.checkJWT(r) f.mu.Lock() defer f.mu.Unlock() - page := 0 - if r.URL.Query().Get("page") == "2" { - page = 1 + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + page = max(page, 1) + if page < len(f.list) { + w.Header().Set("Link", fmt.Sprintf(`<%s/api/v3/app/installations?per_page=100&page=%d>; rel="next"`, e.srv.URL, page+1)) } - if page == 0 && len(f.list) > 1 { - w.Header().Set("Link", `<`+e.srv.URL+`/api/v3/app/installations?per_page=100&page=2>; rel="next"`) - } - writeJSON(w, http.StatusOK, f.list[page:min(page+1, len(f.list))]) + writeJSON(w, http.StatusOK, f.list[min(page-1, len(f.list)):min(page, len(f.list))]) }) e.mux.HandleFunc("GET /api/v3/installation/repositories", func(w http.ResponseWriter, r *http.Request) { f.mu.Lock() @@ -59,7 +59,7 @@ func TestSyncInstallations(t *testing.T) { e := newEnv(t) e.connect() ctx := context.Background() - f := e.installations() + f := e.inst org := map[string]any{"id": 10, "app_id": testAppID, "suspended_at": nil, "account": map[string]any{"login": "acme", "id": 1, "type": "Organization"}} user := map[string]any{"id": 20, "app_id": testAppID, "suspended_at": nil, @@ -114,7 +114,6 @@ func TestSyncInstallations(t *testing.T) { t.Errorf("the scale set of the removed installation is still there: %v", err) } - // The suspension ends. org["suspended_at"] = nil if err := e.svc.Sync(ctx); err != nil { t.Fatal(err) @@ -130,7 +129,7 @@ func TestSetupRecordsInstallation(t *testing.T) { e := newEnv(t) e.connect() ctx := context.Background() - f := e.installations() + f := e.inst f.set(nil, map[string][]map[string]any{"ghs_77_": {{"id": 300, "full_name": "octo/app", "private": true}}}) appID := testAppID e.mux.HandleFunc("GET /api/v3/app/installations/77", func(w http.ResponseWriter, r *http.Request) { @@ -147,7 +146,7 @@ func TestSetupRecordsInstallation(t *testing.T) { {"request", "/settings/github?requested=1"}, } for _, tt := range tests { - if to, err := e.svc.Setup(ctx, "https://ci.example.com", 77, tt.action); err != nil || to != tt.want { + if to, err := e.svc.Setup(ctx, 77, tt.action); err != nil || to != tt.want { t.Errorf("Setup(%s) = %q, %v; want %q", tt.action, to, err, tt.want) } } @@ -161,7 +160,27 @@ func TestSetupRecordsInstallation(t *testing.T) { } appID = 99 - if _, err := e.svc.Setup(ctx, "https://ci.example.com", 77, "install"); err == nil { + if _, err := e.svc.Setup(ctx, 77, "install"); err == nil { t.Error("setup accepted an installation of a different app") } } + +// runplane runs jobs only for organizations and personal accounts, so +// Setup refuses an installation that Sync would delete. +func TestSetupRefusesOtherAccountTypes(t *testing.T) { + e := newEnv(t) + e.connect() + ctx := context.Background() + e.mux.HandleFunc("GET /api/v3/app/installations/78", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "id": 78, "app_id": testAppID, "repository_selection": "all", "suspended_at": nil, + "account": map[string]any{"login": "acme-corp", "id": 6, "type": "Enterprise"}, + }) + }) + if _, err := e.svc.Setup(ctx, 78, "install"); err == nil || !strings.Contains(err.Error(), "Enterprise") { + t.Errorf("Setup of an enterprise installation: err = %v, want a refusal", err) + } + if _, err := e.st.GetInstallation(ctx, 78); !errors.Is(err, store.ErrNotFound) { + t.Errorf("installation row: err = %v, want none", err) + } +} diff --git a/internal/github/jobs.go b/internal/github/jobs.go index f9b2110..fb20142 100644 --- a/internal/github/jobs.go +++ b/internal/github/jobs.go @@ -5,13 +5,13 @@ import ( "context" "errors" "fmt" - "maps" + "io" "net/http" "slices" "strings" "time" - "github.com/runplane/runplane/internal/atomicfile" + "github.com/runplane/runplane/internal/ghrest" "github.com/runplane/runplane/internal/scheduler" "github.com/runplane/runplane/internal/store" ) @@ -55,15 +55,8 @@ type workflowJob struct { // CheckJob tells if the repository is private and if the run comes from a // fork, and gives the run data for the cache refs. It reads the run, // because the scale set message has no head repository; the run also has -// the current private flag. The result is cached by run ID for one hour, so -// the jobs of one run cost one request. +// the current private flag. func (s *Service) CheckJob(ctx context.Context, repository string, runID int64) (scheduler.JobCheck, error) { - s.checkMu.Lock() - cached, ok := s.checks[runID] - s.checkMu.Unlock() - if ok && time.Since(cached.at) <= time.Hour { - return cached.JobCheck, nil - } run, err := s.run(ctx, repository, runID) if err != nil { if unreadable(err) { @@ -71,27 +64,16 @@ func (s *Service) CheckJob(ctx context.Context, repository string, runID int64) } return scheduler.JobCheck{}, err } - c := scheduler.JobCheck{Private: run.Repository.Private, Fork: run.fork(), Ref: run.ref()} - now := time.Now() - s.checkMu.Lock() - maps.DeleteFunc(s.checks, func(_ int64, old runCheck) bool { return now.Sub(old.at) > time.Hour }) - s.checks[runID] = runCheck{c, now} - s.checkMu.Unlock() - return c, nil -} - -type runCheck struct { - scheduler.JobCheck - at time.Time + return scheduler.JobCheck{Private: run.Repository.Private, Fork: run.fork(), Ref: run.ref()}, nil } // unreadable tells if an error stays when runplane reads the run again: the // app has no installation for the repository, or GitHub hides the run (404) // or refuses it for a reason other than a rate limit (403). func unreadable(err error) bool { - var e *apiError + var e *ghrest.APIError if errors.As(err, &e) { - return e.status == http.StatusNotFound || e.status == http.StatusForbidden && e.retryAt.IsZero() + return e.Status == http.StatusNotFound || e.Status == http.StatusForbidden && e.RetryAt.IsZero() } return errors.Is(err, errNoInstallation) } @@ -117,31 +99,47 @@ func (run workflowRun) fork() bool { return run.HeadRepository.ID != run.Repository.ID } -// Enrich reads the run and its jobs and writes the enrichment fields. For a -// cancelled job, it also writes another result that GitHub has: the -// scheduler cancels a job whose sandbox it lost. -func (s *Service) Enrich(ctx context.Context, job store.Job) error { - _, err := s.enrich(ctx, job) - return err -} - -// enrich is Enrich that returns the GitHub job ID. -func (s *Service) enrich(ctx context.Context, job store.Job) (int64, error) { - if job.RunID == 0 { - return 0, fmt.Errorf("enrich job %s: the job has no run ID", job.ID) +// Enrich reads the run and its jobs, and gives the run data of the job and +// its result if GitHub completed it. +func (s *Service) Enrich(ctx context.Context, job store.Job) (scheduler.Enrichment, error) { + gj, err := s.githubJob(ctx, job) + if err != nil { + return scheduler.Enrichment{}, err } run, err := s.run(ctx, job.Repository, job.RunID) if err != nil { - return 0, err + return scheduler.Enrichment{}, err + } + ref := run.ref() + e := scheduler.Enrichment{JobEnrichment: store.JobEnrichment{ + GitHubJobID: gj.ID, WorkflowName: run.Name, RunNumber: run.RunNumber, + RunAttempt: cmp.Or(gj.RunAttempt, run.RunAttempt), HeadBranch: run.HeadBranch, HeadSHA: run.HeadSHA, + CommitMessage: run.HeadCommit.Message, Event: run.Event, + // The triggering actor started this attempt; the actor started the first. + Actor: cmp.Or(run.TriggeringActor.Login, run.Actor.Login), + PRNumber: ref.PRNumber, BaseRef: ref.BaseRef, HTMLURL: gj.HTMLURL, Steps: gj.Steps, QueuedAt: gj.CreatedAt, + }} + if gj.Status == "completed" { + e.Result = store.ResultOf(gj.Conclusion) + if gj.CompletedAt != nil { + e.FinishedAt = *gj.CompletedAt + } + } + return e, nil +} + +func (s *Service) githubJob(ctx context.Context, job store.Job) (workflowJob, error) { + if job.RunID == 0 { + return workflowJob{}, fmt.Errorf("find the GitHub job of %s: the job has no run ID", job.ID) } a, err := s.repoAuth(ctx, job.Repository) if err != nil { - return 0, err + return workflowJob{}, err } - jobs, err := list[workflowJob](ctx, s.api, a, + jobs, err := ghrest.List[workflowJob](ctx, s.api, a, fmt.Sprintf("/repos/%s/actions/runs/%d/jobs?filter=all&per_page=100", job.Repository, job.RunID), "jobs") if err != nil { - return 0, fmt.Errorf("list jobs of run %d: %w", job.RunID, err) + return workflowJob{}, fmt.Errorf("list jobs of run %d: %w", job.RunID, err) } var runnerName string if job.SandboxID != "" { @@ -151,31 +149,9 @@ func (s *Service) enrich(ctx context.Context, job store.Job) (int64, error) { } gj, ok := matchJob(jobs, job.JobName, runnerName, job.RunAttempt) if !ok { - return 0, fmt.Errorf("enrich job %s: run %d has no job %q", job.ID, job.RunID, job.JobName) + return workflowJob{}, fmt.Errorf("find the GitHub job of %s: run %d has no job %q", job.ID, job.RunID, job.JobName) } - - ref := run.ref() - err = s.st.SetJobEnrichment(ctx, job.ID, store.JobEnrichment{ - GitHubJobID: gj.ID, WorkflowName: run.Name, RunNumber: run.RunNumber, - RunAttempt: cmp.Or(gj.RunAttempt, run.RunAttempt), HeadBranch: run.HeadBranch, HeadSHA: run.HeadSHA, - CommitMessage: run.HeadCommit.Message, Event: run.Event, - // The triggering actor started this attempt; the actor started the first. - Actor: cmp.Or(run.TriggeringActor.Login, run.Actor.Login), - PRNumber: ref.PRNumber, BaseRef: ref.BaseRef, HTMLURL: gj.HTMLURL, Steps: gj.Steps, QueuedAt: gj.CreatedAt, - }) - if err != nil { - return 0, err - } - if r := conclusionResult(gj.Conclusion); job.Result == store.ResultCancelled && gj.Status == "completed" && r != "" && r != store.ResultCancelled { - m := store.JobMessage{RunnerRequestID: job.RunnerRequestID, State: store.JobCompleted, Result: r} - if gj.CompletedAt != nil { - m.FinishedAt = *gj.CompletedAt - } - if _, err := s.st.UpsertJob(ctx, m); err != nil { - return 0, err - } - } - return gj.ID, nil + return gj, nil } // Steps reads the GitHub job: one request. @@ -188,7 +164,7 @@ func (s *Service) Steps(ctx context.Context, job store.Job) ([]store.JobStep, er return nil, err } var gj workflowJob - err = s.api.json(ctx, call{method: http.MethodGet, path: fmt.Sprintf("/repos/%s/actions/jobs/%d", job.Repository, job.GitHubJobID), auth: a}, &gj) + err = s.api.JSON(ctx, ghrest.Call{Method: http.MethodGet, Path: fmt.Sprintf("/repos/%s/actions/jobs/%d", job.Repository, job.GitHubJobID), Auth: a}, &gj) if err != nil { return nil, fmt.Errorf("read job %d: %w", job.GitHubJobID, err) } @@ -199,22 +175,6 @@ func (s *Service) Steps(ctx context.Context, job store.Job) ([]store.JobStep, er return gj.Steps, nil } -// conclusionResult maps the conclusion of a GitHub job to a job result, or -// returns "" if the job has none yet. -func conclusionResult(c string) string { - switch c { - case "": - return "" - case "success", "neutral": - return store.ResultSucceeded - case "cancelled": - return store.ResultCancelled - case "skipped": - return store.ResultSkipped - } - return store.ResultFailed -} - // matchJob finds the GitHub job of a runplane job. The runner name is // unique, so it wins. Without it, the job name decides: in the given run // attempt, or in the newest attempt when the attempt is 0 (unknown). @@ -240,36 +200,27 @@ func matchJob(jobs []workflowJob, name, runnerName string, attempt int) (workflo return best, found } -// DownloadLog writes the complete log of the job from GitHub to path. It -// writes a temporary file next to path and renames it, so path is never a -// partial log. -func (s *Service) DownloadLog(ctx context.Context, job store.Job, path string) error { +// DownloadLog returns the complete log of the job from GitHub. The caller +// must close it. +func (s *Service) DownloadLog(ctx context.Context, job store.Job) (io.ReadCloser, error) { id, err := s.githubJobID(ctx, job) if err != nil { - return err + return nil, err } a, err := s.repoAuth(ctx, job.Repository) if err != nil { - return err + return nil, err } - // GitHub answers with a redirect to a short-lived blob URL. The blob - // store must not get the token, so runplane follows the redirect itself. - resp, err := s.api.do(ctx, call{ - method: http.MethodGet, path: fmt.Sprintf("/repos/%s/actions/jobs/%d/logs", job.Repository, id), - auth: a, timeout: logTimeout, noRedirect: true, + // GitHub answers with a redirect to a short-lived blob URL on a + // different host. net/http does not send the token to that host. + resp, err := s.api.Do(ctx, ghrest.Call{ + Method: http.MethodGet, Path: fmt.Sprintf("/repos/%s/actions/jobs/%d/logs", job.Repository, id), + Auth: a, Timeout: logTimeout, }) if err != nil { - return fmt.Errorf("download log of job %d: %w", id, err) - } - if loc := resp.Header.Get("Location"); resp.StatusCode >= 300 && loc != "" { - resp.Body.Close() - resp, err = s.api.do(ctx, call{method: http.MethodGet, path: loc, timeout: logTimeout}) - if err != nil { - return fmt.Errorf("download log of job %d: %w", id, err) - } + return nil, fmt.Errorf("download log of job %d: %w", id, err) } - defer resp.Body.Close() - return atomicfile.Write(path, resp.Body, 0o600) + return resp.Body, nil } const logTimeout = 10 * time.Minute @@ -283,20 +234,19 @@ func (s *Service) Rerun(ctx context.Context, job store.Job) error { if err != nil { return err } - err = s.api.json(ctx, call{method: http.MethodPost, path: fmt.Sprintf("/repos/%s/actions/jobs/%d/rerun", job.Repository, id), auth: a}, nil) + err = s.api.JSON(ctx, ghrest.Call{Method: http.MethodPost, Path: fmt.Sprintf("/repos/%s/actions/jobs/%d/rerun", job.Repository, id), Auth: a}, nil) if err != nil { return fmt.Errorf("re-run job %d: %w", id, err) } return nil } -// githubJobID returns the GitHub job ID, and enriches the job first if the -// ID is not known yet. func (s *Service) githubJobID(ctx context.Context, job store.Job) (int64, error) { if job.GitHubJobID != 0 { return job.GitHubJobID, nil } - return s.enrich(ctx, job) + gj, err := s.githubJob(ctx, job) + return gj.ID, err } func (s *Service) run(ctx context.Context, repository string, runID int64) (workflowRun, error) { @@ -305,7 +255,7 @@ func (s *Service) run(ctx context.Context, repository string, runID int64) (work return workflowRun{}, err } var run workflowRun - err = s.api.json(ctx, call{method: http.MethodGet, path: fmt.Sprintf("/repos/%s/actions/runs/%d", repository, runID), auth: a}, &run) + err = s.api.JSON(ctx, ghrest.Call{Method: http.MethodGet, Path: fmt.Sprintf("/repos/%s/actions/runs/%d", repository, runID), Auth: a}, &run) if err != nil { return workflowRun{}, fmt.Errorf("read run %d of %s: %w", runID, repository, err) } @@ -316,7 +266,7 @@ var errNoInstallation = errors.New("no installation of the GitHub App can read") // repoAuth returns the installation that can read the repository: the one // that lists it, or else the one of the repository owner. -func (s *Service) repoAuth(ctx context.Context, repository string) (auth, error) { +func (s *Service) repoAuth(ctx context.Context, repository string) (ghrest.Auth, error) { owner, name, ok := strings.Cut(repository, "/") if !ok || owner == "" || name == "" || strings.Contains(name, "/") { return nil, fmt.Errorf("bad repository name %q", repository) diff --git a/internal/github/jobs_test.go b/internal/github/jobs_test.go index e5e939e..071a642 100644 --- a/internal/github/jobs_test.go +++ b/internal/github/jobs_test.go @@ -5,8 +5,6 @@ import ( "errors" "io" "net/http" - "os" - "path/filepath" "strconv" "strings" "testing" @@ -45,7 +43,6 @@ var testJobs = []map[string]any{ {"id": 903, "run_attempt": 2, "name": "lint", "runner_name": "runplane-01def", "html_url": "https://x/903"}, } -// serveRun serves run 555 of acme/web with the jobs. func (e *env) serveRun(jobs []map[string]any) { e.mux.HandleFunc("GET /api/v3/repos/acme/web/actions/runs/555", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, runJSON("push", true, 100)) @@ -78,19 +75,12 @@ func TestCheckJob(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { e := acmeEnv(t) - calls := 0 e.mux.HandleFunc("GET /api/v3/repos/acme/web/actions/runs/555", func(w http.ResponseWriter, r *http.Request) { - calls++ writeJSON(w, http.StatusOK, runJSON(tt.event, tt.private, tt.headRepoID)) }) - for range 2 { - got, err := e.svc.CheckJob(context.Background(), "acme/web", 555) - if err != nil || got != tt.want { - t.Errorf("CheckJob = %+v, %v; want %+v", got, err, tt.want) - } - } - if calls != 1 { - t.Errorf("run read %d times, want 1 (cached by run ID)", calls) + got, err := e.svc.CheckJob(context.Background(), "acme/web", 555) + if err != nil || got != tt.want { + t.Errorf("CheckJob = %+v, %v; want %+v", got, err, tt.want) } }) } @@ -113,7 +103,6 @@ func TestCheckJobErrors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { e := acmeEnv(t) - e.svc.api.retryWait = time.Millisecond e.mux.HandleFunc("GET /api/v3/repos/acme/web/actions/runs/555", func(w http.ResponseWriter, r *http.Request) { if tt.limited { w.Header().Set("X-RateLimit-Remaining", "0") @@ -138,84 +127,100 @@ func TestEnrich(t *testing.T) { t.Fatal(err) } job, err := e.st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: 1, State: store.JobRunning, Repository: "acme/web", - JobName: "build", RunID: 555, QueuedAt: time.Now()}) + JobName: "build", RunID: 555, QueuedAt: time.Now()}, time.Now()) if err != nil { t.Fatal(err) } job.SandboxID = "sb1" - if err := e.svc.Enrich(ctx, job); err != nil { + got, err := e.svc.Enrich(ctx, job) + if err != nil { t.Fatal(err) } - got, _ := e.st.GetJob(ctx, job.ID) if got.GitHubJobID != 902 || got.RunAttempt != 2 || got.HTMLURL != "https://x/902" { t.Errorf("matched job %d attempt %d %s, want 902 by runner name", got.GitHubJobID, got.RunAttempt, got.HTMLURL) } if got.WorkflowName != "CI" || got.RunNumber != 81 || got.HeadBranch != "fix-cache" || got.HeadSHA != "3f2a9c1e" || - got.CommitMessage != "Fix the cache key" || got.Actor != "bob" || got.PRNumber != 17 || got.EnrichedAt == nil { + got.CommitMessage != "Fix the cache key" || got.Actor != "bob" || got.PRNumber != 17 || got.Result != "" { t.Errorf("enrichment = %+v", got) } if len(got.Steps) != 2 || got.Steps[1].Name != "Run tests" || got.Steps[1].Conclusion != "failure" || got.Steps[1].StartedAt == nil { t.Errorf("steps = %+v", got.Steps) } + // The scheduler records the data: Enrich does not change the job. + if stored, _ := e.st.GetJob(ctx, job.ID); stored.GitHubJobID != 0 || stored.EnrichedAt != nil { + t.Errorf("stored job = GitHub job %d, enriched at %v; want no change", stored.GitHubJobID, stored.EnrichedAt) + } } -// The scheduler guesses "cancelled" for a job whose sandbox it lost; the -// finish step then takes the result from GitHub. -func TestEnrichReplacesTheResultOfALostJob(t *testing.T) { +// Enrich gives the result of a job that GitHub completed, so that the +// scheduler can replace a result that it guessed. +func TestEnrichGivesTheGitHubResult(t *testing.T) { e := acmeEnv(t) ctx := context.Background() e.serveRun([]map[string]any{{"id": 905, "run_attempt": 2, "name": "build", "status": "completed", "conclusion": "failure", "completed_at": "2026-11-04T10:07:00Z"}}) - job, err := e.st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: 1, State: store.JobCompleted, Result: store.ResultCancelled, - Repository: "acme/web", JobName: "build", RunID: 555, QueuedAt: time.Now(), FinishedAt: time.Now()}) + job := store.Job{ID: "j1", RunnerRequestID: 1, State: store.JobCompleted, Result: store.ResultCancelled, ResultGuessed: true, + Repository: "acme/web", JobName: "build", RunID: 555} + got, err := e.svc.Enrich(ctx, job) if err != nil { t.Fatal(err) } - if err := e.svc.Enrich(ctx, job); err != nil { - t.Fatal(err) - } - got, _ := e.st.GetJob(ctx, job.ID) finished := time.Date(2026, 11, 4, 10, 7, 0, 0, time.UTC) - if got.Result != store.ResultFailed || got.FinishedAt == nil || !got.FinishedAt.Equal(finished) { - t.Errorf("job = result %q, finished %v; want failed from GitHub at %v", got.Result, got.FinishedAt, finished) + if got.Result != store.ResultFailed || !got.FinishedAt.Equal(finished) { + t.Errorf("result %q, finished %v; want failed from GitHub at %v", got.Result, got.FinishedAt, finished) } } -func TestMatchJob(t *testing.T) { - jobs := []workflowJob{ - {ID: 901, RunAttempt: 1, Name: "build", RunnerName: "runplane-old"}, - {ID: 902, RunAttempt: 2, Name: "build", RunnerName: "runplane-01abc"}, - {ID: 903, RunAttempt: 2, Name: "lint", RunnerName: "runplane-01def"}, - {ID: 904, RunAttempt: 2, Name: "deploy"}, +// Enrich finds the GitHub job of a runplane job in the jobs of all run +// attempts. The runner name of the sandbox is unique, so it wins. Without +// it, the job name decides: in the run attempt of the job, or in the newest +// attempt when the attempt is 0 (unknown). +func TestEnrichFindsTheGitHubJob(t *testing.T) { + e := acmeEnv(t) + ctx := context.Background() + e.serveRun([]map[string]any{ + {"id": 901, "run_attempt": 1, "name": "build", "runner_name": "runplane-old"}, + {"id": 902, "run_attempt": 2, "name": "build", "runner_name": "runplane-01abc"}, + {"id": 903, "run_attempt": 2, "name": "lint", "runner_name": "runplane-01def"}, + {"id": 904, "run_attempt": 2, "name": "deploy"}, + }) + for id, runner := range map[string]string{"old": "runplane-old", "upper": "RUNPLANE-OLD", "zzz": "runplane-zzz"} { + if err := e.st.InsertSandbox(ctx, store.Sandbox{ID: id, PoolID: "p1", ServerID: "s1", Mode: sandbox.ModeContainer, + State: sandbox.StateBusy, RunnerName: runner, CreatedAt: time.Now()}); err != nil { + t.Fatal(err) + } } tests := []struct { - name, job, runner string - attempt int - want int64 + name, job, sandbox string + attempt int + want int64 // 0: no GitHub job }{ - {"runner name wins over the newest attempt", "build", "runplane-old", 0, 901}, - {"runner name without regard to case", "build", "RUNPLANE-01ABC", 0, 902}, + {"runner name wins over the newest attempt", "build", "old", 0, 901}, + {"runner name without regard to case", "build", "upper", 0, 901}, {"name in the newest attempt without a runner", "build", "", 0, 902}, {"name in the known attempt", "build", "", 1, 901}, - {"unknown runner falls back to the name", "lint", "runplane-zzz", 0, 903}, + {"unknown runner falls back to the name", "lint", "zzz", 0, 903}, {"no job of the attempt", "deploy", "", 1, 0}, {"no job of the name", "test", "", 0, 0}, } for _, tt := range tests { - got, ok := matchJob(jobs, tt.job, tt.runner, tt.attempt) - if ok != (tt.want != 0) || got.ID != tt.want { - t.Errorf("%s: matchJob = %d, %v; want %d", tt.name, got.ID, ok, tt.want) + got, err := e.svc.Enrich(ctx, store.Job{ID: "j1", Repository: "acme/web", RunID: 555, JobName: tt.job, + RunAttempt: tt.attempt, SandboxID: tt.sandbox}) + if (err == nil) != (tt.want != 0) || got.GitHubJobID != tt.want { + t.Errorf("%s: GitHub job %d, %v; want %d", tt.name, got.GitHubJobID, err, tt.want) } } } func TestDownloadLogFollowsRedirectWithoutToken(t *testing.T) { e := acmeEnv(t) + // GitHub sends the log from a blob store on a different host. + blob := strings.Replace(e.srv.URL, "127.0.0.1", "localhost", 1) e.mux.HandleFunc("GET /api/v3/repos/acme/web/actions/jobs/902/logs", func(w http.ResponseWriter, r *http.Request) { if !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ghs_10_") { t.Errorf("log request without the installation token") } - http.Redirect(w, r, e.srv.URL+"/blob/job-902.txt?sig=abc", http.StatusFound) + http.Redirect(w, r, blob+"/blob/job-902.txt?sig=abc", http.StatusFound) }) e.mux.HandleFunc("GET /blob/job-902.txt", func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "" || r.URL.Query().Get("sig") != "abc" { @@ -223,32 +228,25 @@ func TestDownloadLogFollowsRedirectWithoutToken(t *testing.T) { } io.WriteString(w, "2026-11-04T10:00:00Z Run tests\nok\n") }) - dir := t.TempDir() - path := filepath.Join(dir, "job1.log") - if err := e.svc.DownloadLog(context.Background(), store.Job{ID: "job1", Repository: "acme/web", GitHubJobID: 902}, path); err != nil { + log, err := e.svc.DownloadLog(context.Background(), store.Job{ID: "job1", Repository: "acme/web", GitHubJobID: 902}) + if err != nil { t.Fatal(err) } - if b, err := os.ReadFile(path); err != nil || string(b) != "2026-11-04T10:00:00Z Run tests\nok\n" { + defer log.Close() + if b, err := io.ReadAll(log); err != nil || string(b) != "2026-11-04T10:00:00Z Run tests\nok\n" { t.Errorf("log = %q, %v", b, err) } - if entries, _ := os.ReadDir(dir); len(entries) != 1 { - t.Errorf("folder has %d files, want only the log", len(entries)) - } } -func TestDownloadLogKeepsOldFileOnFailure(t *testing.T) { +func TestDownloadLogOfAnExpiredLog(t *testing.T) { e := acmeEnv(t) e.mux.HandleFunc("GET /api/v3/repos/acme/web/actions/jobs/902/logs", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusGone, map[string]any{"message": "Log has expired"}) }) - path := filepath.Join(t.TempDir(), "job1.log") - os.WriteFile(path, []byte("live log"), 0o600) - if err := e.svc.DownloadLog(context.Background(), store.Job{ID: "job1", Repository: "acme/web", GitHubJobID: 902}, path); err == nil { + if log, err := e.svc.DownloadLog(context.Background(), store.Job{ID: "job1", Repository: "acme/web", GitHubJobID: 902}); err == nil { + log.Close() t.Fatal("DownloadLog succeeded for an expired log") } - if b, _ := os.ReadFile(path); string(b) != "live log" { - t.Errorf("log = %q, want the earlier file", b) - } } func TestRerunFindsJobIDFirst(t *testing.T) { @@ -261,7 +259,7 @@ func TestRerunFindsJobIDFirst(t *testing.T) { writeJSON(w, http.StatusCreated, map[string]any{}) }) job, err := e.st.UpsertJob(ctx, store.JobMessage{RunnerRequestID: 2, State: store.JobCompleted, Result: store.ResultFailed, - Repository: "acme/web", JobName: "lint", RunID: 555, QueuedAt: time.Now()}) + Repository: "acme/web", JobName: "lint", RunID: 555, QueuedAt: time.Now()}, time.Now()) if err != nil { t.Fatal(err) } @@ -271,6 +269,9 @@ func TestRerunFindsJobIDFirst(t *testing.T) { if len(rerun) != 1 || rerun[0] != "903" { t.Errorf("re-run jobs = %v, want [903]", rerun) } + if got, _ := e.st.GetJob(ctx, job.ID); got.GitHubJobID != 0 || got.Result != store.ResultFailed { + t.Errorf("job after the re-run = GitHub job %d, result %q; want no change", got.GitHubJobID, got.Result) + } } func TestSteps(t *testing.T) { diff --git a/internal/github/listen.go b/internal/github/listen.go index dbb0e66..574d561 100644 --- a/internal/github/listen.go +++ b/internal/github/listen.go @@ -16,11 +16,13 @@ import ( "github.com/actions/scaleset" "github.com/actions/scaleset/listener" + "github.com/runplane/runplane/internal/ghapp" "github.com/runplane/runplane/internal/scheduler" "github.com/runplane/runplane/internal/store" ) -// setListener runs the listener of one scale set. It is also the +// setListener runs the listener of one scale set. It is the listener.Scaler +// of the scale set, which gives each message to the scheduler, and the // scheduler.ScaleSetClient of the scale set. type setListener struct { s *Service @@ -30,16 +32,18 @@ type setListener struct { cancel context.CancelFunc done chan struct{} max atomic.Int64 + scaler scheduler.Scaler - mu sync.Mutex - cl *scaleset.Client - label string - githubID int64 - session *scaleset.MessageSessionClient - running *listener.Listener - status string // the last written status and error, to skip equal writes - errText string - answered bool // GitHub answered a poll since the last restart + mu sync.Mutex + cl *scaleset.Client + label string + githubID int64 + session *scaleset.MessageSessionClient + running *listener.Listener + offered map[int64]int64 // request IDs by job key, of the jobs that the last message offered + answered bool // GitHub answered a poll since the last restart + listenErr string // the last error of the listener, "" while it listens + renameErr string // the error of the last rename, "" if it worked } // startListener attaches the scale set to the scheduler and starts its @@ -54,13 +58,13 @@ func (s *Service) startListener(ctx context.Context, row store.ScaleSet, t targe l := &setListener{ s: s, id: row.ID, target: t, cl: cl, cancel: cancel, done: make(chan struct{}), log: s.log.With("scale_set", t.pool.Label, "owner", t.owner()), - label: t.pool.Label, githubID: row.GitHubID, status: row.Status, errText: row.Error, + label: t.pool.Label, githubID: row.GitHubID, } - scaler := s.sched.Attach(row, l) + l.scaler = s.sched.Attach(row, l) s.lmu.Lock() s.listeners[row.ID] = l s.lmu.Unlock() - go l.run(lctx, scaler) + go l.run(lctx) return l } @@ -94,17 +98,16 @@ func (s *Service) stopListeners() { } } -// run listens until ctx ends. After a failure it waits 1 second, then -// twice as long after each further failure, up to 1 minute. -func (l *setListener) run(ctx context.Context, scaler listener.Scaler) { +// run listens until ctx ends. +func (l *setListener) run(ctx context.Context) { defer close(l.done) wait := time.Second for { - err := l.listen(ctx, scaler) + err := l.listen(ctx) if ctx.Err() != nil { return } - if l.setStatus(ctx, store.ScaleSetError, err.Error()) { + if l.setListenErr(ctx, err.Error()) { l.log.Warn("scale set listener stopped", "err", err) } if errors.Is(err, scaleset.UnauthorizedError) { @@ -143,9 +146,7 @@ func (l *setListener) client() *scaleset.Client { return l.cl } -// listen makes the scale set on GitHub if necessary, opens a session, and -// runs the listener on it. -func (l *setListener) listen(ctx context.Context, scaler listener.Scaler) error { +func (l *setListener) listen(ctx context.Context) error { id, err := l.ensureRemote(ctx) if err != nil { return fmt.Errorf("make the scale set on GitHub: %w", err) @@ -190,11 +191,69 @@ func (l *setListener) listen(ctx context.Context, scaler listener.Scaler) error l.mu.Unlock() }() - l.setStatus(ctx, store.ScaleSetListening, "") + l.setListenErr(ctx, "") if st := session.Session().Statistics; st != nil { l.record(ctx, st, time.Now()) } - return run.Run(ctx, scaler) + return run.Run(ctx, l) +} + +// setListenErr records the error of the listener, "" while it listens, and +// writes the status. It tells if the status changed. +func (l *setListener) setListenErr(ctx context.Context, msg string) bool { + l.mu.Lock() + l.listenErr = msg + l.mu.Unlock() + return l.report(ctx) +} + +// report writes the status of the scale set: an error if the last rename +// or the listener failed, listening while the listener has a session, and +// else nothing. It tells if the status changed. While a listener runs, only +// the listener writes the status of its row. +func (l *setListener) report(ctx context.Context) bool { + l.mu.Lock() + msg, listening := cmp.Or(l.renameErr, l.listenErr), l.session != nil + l.mu.Unlock() + switch { + case msg != "": + return l.s.setStatus(ctx, l.id, store.ScaleSetError, msg) + case listening: + return l.s.setStatus(ctx, l.id, store.ScaleSetListening, "") + } + return false +} + +// rename gives the scale set the label of its pool, on GitHub and in its +// row. A failed rename stays the error of the scale set until a rename +// works. +func (l *setListener) rename(ctx context.Context, row store.ScaleSet, label string) { + renameErr := "" + if row.GitHubID != 0 { + _, err := l.client().UpdateRunnerScaleSet(ctx, int(row.GitHubID), scaleSetSpec(label, int(row.RunnerGroupID))) + if err != nil { + renameErr = "rename the scale set to " + label + " on GitHub: " + err.Error() + } + } + if renameErr == "" { + if err := l.s.st.SetScaleSetRemote(ctx, row.ID, row.GitHubID, row.RunnerGroupID, label, time.Now()); err != nil { + l.log.Warn("rename scale set", "err", err) + return + } + l.setLabel(label) + } + l.mu.Lock() + l.renameErr = renameErr + l.mu.Unlock() + l.report(ctx) +} + +func (l *setListener) Scale(ctx context.Context, msg *scaleset.RunnerScaleSetMessage) error { + m, offered := message(l.log, msg) + l.mu.Lock() + l.offered = offered + l.mu.Unlock() + return l.scaler.Scale(ctx, m) } // ensureRemote returns the GitHub ID of the scale set. It finds the scale @@ -208,22 +267,22 @@ func (l *setListener) ensureRemote(ctx context.Context) (int64, error) { return id, nil } cl := l.client() - set, err := cl.GetRunnerScaleSet(ctx, defaultRunnerGroup, label) + set, err := cl.GetRunnerScaleSet(ctx, ghapp.DefaultRunnerGroup, label) if err != nil { return 0, err } hasLabel := func(x scaleset.Label) bool { return strings.EqualFold(x.Name, label) } switch { case set == nil: - set, err = cl.CreateRunnerScaleSet(ctx, scaleSetSpec(label, defaultRunnerGroup)) + set, err = cl.CreateRunnerScaleSet(ctx, scaleSetSpec(label, ghapp.DefaultRunnerGroup)) case !slices.ContainsFunc(set.Labels, hasLabel) || !set.RunnerSetting.DisableUpdate: - set, err = cl.UpdateRunnerScaleSet(ctx, set.ID, scaleSetSpec(label, cmp.Or(set.RunnerGroupID, defaultRunnerGroup))) + set, err = cl.UpdateRunnerScaleSet(ctx, set.ID, scaleSetSpec(label, cmp.Or(set.RunnerGroupID, ghapp.DefaultRunnerGroup))) } if err != nil { return 0, err } - group := int64(cmp.Or(set.RunnerGroupID, defaultRunnerGroup)) - if err := l.s.st.SetScaleSetRemote(ctx, l.id, int64(set.ID), group, label); err != nil { + group := int64(cmp.Or(set.RunnerGroupID, ghapp.DefaultRunnerGroup)) + if err := l.s.st.SetScaleSetRemote(ctx, l.id, int64(set.ID), group, label, time.Now()); err != nil { return 0, err } l.mu.Lock() @@ -237,31 +296,13 @@ func (l *setListener) clearRemote(ctx context.Context) { l.githubID = 0 label := l.label l.mu.Unlock() - if err := l.s.st.SetScaleSetRemote(ctx, l.id, 0, defaultRunnerGroup, label); err != nil { + if err := l.s.st.SetScaleSetRemote(ctx, l.id, 0, ghapp.DefaultRunnerGroup, label, time.Now()); err != nil { l.log.Warn("clear scale set ID", "err", err) } } -// setStatus writes the status if it changed and tells if it did. -func (l *setListener) setStatus(ctx context.Context, status, msg string) bool { - msg = truncate(msg, 1000) - l.mu.Lock() - same := l.status == status && l.errText == msg - l.status, l.errText = status, msg - l.mu.Unlock() - if !same { - l.s.setStatus(ctx, l.id, status, msg) - } - return !same -} - func (l *setListener) record(ctx context.Context, st *scaleset.RunnerScaleSetStatistic, at time.Time) { - err := l.s.st.SetScaleSetStats(ctx, l.id, store.ScaleSetStats{ - AvailableJobs: st.TotalAvailableJobs, AcquiredJobs: st.TotalAcquiredJobs, AssignedJobs: st.TotalAssignedJobs, - RunningJobs: st.TotalRunningJobs, RegisteredRunners: st.TotalRegisteredRunners, - BusyRunners: st.TotalBusyRunners, IdleRunners: st.TotalIdleRunners, - }, at) - if err != nil && !errors.Is(err, store.ErrNotFound) { + if err := l.s.st.SetScaleSetStats(ctx, l.id, statsOf(*st), at); err != nil && !errors.Is(err, store.ErrNotFound) { l.log.Warn("record scale set statistics", "err", err) } } @@ -286,7 +327,7 @@ func (p pollRecorder) GetMessage(ctx context.Context, lastMessageID, maxCapacity } else if err := l.s.st.TouchScaleSetPoll(ctx, l.id, now); err != nil && !errors.Is(err, store.ErrNotFound) { l.log.Warn("record scale set poll", "err", err) } - if !l.setStatus(ctx, store.ScaleSetListening, "") { + if !l.setListenErr(ctx, "") { l.s.polledChanged(now) } return msg, nil @@ -304,14 +345,40 @@ func (l *setListener) labelNow() string { return l.label } -func (l *setListener) AcquireJobs(ctx context.Context, requestIDs []int64) ([]int64, error) { +// AcquireJobs acquires the jobs of the last message with these keys. GitHub +// acquires a job by its request ID, and it can send the request ID 0 for +// more than one job (see jobKey). The acquired zeros then go to the keys in +// their order. +func (l *setListener) AcquireJobs(ctx context.Context, keys []int64) ([]int64, error) { l.mu.Lock() - session := l.session + session, offered := l.session, l.offered l.mu.Unlock() if session == nil { return nil, fmt.Errorf("acquire jobs: the listener of scale set %s has no session", l.labelNow()) } - return session.AcquireJobs(ctx, requestIDs) + var ids []int64 + byID := map[int64][]int64{} + for _, k := range keys { + if id, ok := offered[k]; ok { + ids = append(ids, id) + byID[id] = append(byID[id], k) + } + } + if len(ids) == 0 { + return nil, nil + } + acquired, err := session.AcquireJobs(ctx, ids) + if err != nil { + return nil, err + } + var out []int64 + for _, id := range acquired { + if k := byID[id]; len(k) > 0 { + out = append(out, k[0]) + byID[id] = k[1:] + } + } + return out, nil } func (l *setListener) GenerateJITConfig(ctx context.Context, runnerName string) (scheduler.JITConfig, error) { @@ -329,7 +396,7 @@ func (l *setListener) GenerateJITConfig(ctx context.Context, runnerName string) if cfg.Runner == nil || cfg.EncodedJITConfig == "" { return scheduler.JITConfig{}, errors.New("make JIT config: GitHub sent no runner") } - return scheduler.JITConfig{Encoded: cfg.EncodedJITConfig, RunnerID: int64(cfg.Runner.ID), RunnerName: cfg.Runner.Name}, nil + return scheduler.JITConfig{Encoded: cfg.EncodedJITConfig, RunnerID: int64(cfg.Runner.ID)}, nil } func (l *setListener) RemoveRunner(ctx context.Context, runnerID int64) error { diff --git a/internal/github/message.go b/internal/github/message.go new file mode 100644 index 0000000..96222c8 --- /dev/null +++ b/internal/github/message.go @@ -0,0 +1,97 @@ +package github + +import ( + "cmp" + "hash/fnv" + "log/slog" + + "github.com/actions/scaleset" + + "github.com/runplane/runplane/internal/scheduler" + "github.com/runplane/runplane/internal/store" +) + +// message converts a listener message to the scheduler message. It also +// returns the request ID of each offered job by the key of its job row, +// because GitHub acquires a job by its request ID. +func message(log *slog.Logger, msg *scaleset.RunnerScaleSetMessage) (scheduler.Message, map[int64]int64) { + var m scheduler.Message + offered := map[int64]int64{} + if msg == nil { + return m, offered + } + if msg.Statistics != nil { + st := statsOf(*msg.Statistics) + m.Stats = &st + } + for _, j := range msg.JobAvailableMessages { + log.Debug("job message", "type", "available", "request", j.RunnerRequestID, "job", j.JobID, "name", j.JobDisplayName) + jm := jobMessage(store.JobQueued, j.JobMessageBase) + offered[jm.RunnerRequestID] = j.RunnerRequestID + m.Available = append(m.Available, jm) + } + for _, j := range msg.JobAssignedMessages { + log.Debug("job message", "type", "assigned", "request", j.RunnerRequestID, "job", j.JobID, "name", j.JobDisplayName) + m.Assigned = append(m.Assigned, jobMessage(store.JobAssigned, j.JobMessageBase)) + } + for _, j := range msg.JobStartedMessages { + log.Debug("job message", "type", "started", "request", j.RunnerRequestID, "job", j.JobID, "name", j.JobDisplayName, "runner", j.RunnerName) + jm := jobMessage(store.JobRunning, j.JobMessageBase) + jm.RunnerName = j.RunnerName + m.Started = append(m.Started, jm) + } + for _, j := range msg.JobCompletedMessages { + log.Debug("job message", "type", "completed", "request", j.RunnerRequestID, "job", j.JobID, "name", j.JobDisplayName, "result", j.Result) + jm := jobMessage(store.JobCompleted, j.JobMessageBase) + jm.RunnerName = j.RunnerName + jm.Result = cmp.Or(store.ResultOf(j.Result), store.ResultFailed) + m.Completed = append(m.Completed, jm) + } + return m, offered +} + +func jobMessage(state string, m scaleset.JobMessageBase) scheduler.JobMessage { + repo := m.RepositoryName + if m.OwnerName != "" { + repo = m.OwnerName + "/" + m.RepositoryName + } + return scheduler.JobMessage{JobMessage: store.JobMessage{ + RunnerRequestID: jobKey(m), + State: state, + Repository: repo, + WorkflowRef: m.JobWorkflowRef, + JobName: m.JobDisplayName, + RunID: m.WorkflowRunID, + Event: m.EventName, + Labels: m.RequestLabels, + QueuedAt: m.QueueTime, + AssignedAt: m.ScaleSetAssignTime, + StartedAt: m.RunnerAssignTime, + FinishedAt: m.FinishTime, + }} +} + +// jobKey gives the key of the job row of a message. GitHub sends +// runnerRequestId 0 in scale set messages and names each job by its jobId, +// a UUID. For such a message the key comes from the jobId: a negative +// number, so it cannot match a real request ID. +func jobKey(m scaleset.JobMessageBase) int64 { + if m.RunnerRequestID != 0 || m.JobID == "" { + return m.RunnerRequestID + } + h := fnv.New64a() + h.Write([]byte(m.JobID)) + return -int64(h.Sum64()>>1) - 1 +} + +func statsOf(x scaleset.RunnerScaleSetStatistic) store.ScaleSetStats { + return store.ScaleSetStats{ + AvailableJobs: x.TotalAvailableJobs, + AcquiredJobs: x.TotalAcquiredJobs, + AssignedJobs: x.TotalAssignedJobs, + RunningJobs: x.TotalRunningJobs, + RegisteredRunners: x.TotalRegisteredRunners, + BusyRunners: x.TotalBusyRunners, + IdleRunners: x.TotalIdleRunners, + } +} diff --git a/internal/github/pulls.go b/internal/github/pulls.go index 78071fc..0150e4b 100644 --- a/internal/github/pulls.go +++ b/internal/github/pulls.go @@ -8,10 +8,14 @@ import ( "net/http" "net/url" "regexp" + "slices" "strings" - "github.com/danielgtaylor/huma/v2" - + "github.com/runplane/runplane/internal/field" + "github.com/runplane/runplane/internal/ghapp" + "github.com/runplane/runplane/internal/ghrest" + "github.com/runplane/runplane/internal/pages" + "github.com/runplane/runplane/internal/policy" "github.com/runplane/runplane/internal/store" ) @@ -19,25 +23,23 @@ import ( // It has no slash, so a branch named runplane cannot block it. const pullBranch = "runplane-runs-on" -// pullPermissions are the installation permissions that OpenPullRequest -// uses. A change in .github/workflows needs workflows next to contents. -var pullPermissions = []string{"contents", "pull_requests", "workflows"} - -// hostedRunsOn matches a runs-on line with one GitHub-hosted label that a -// pool can take: ubuntu-latest, or ubuntu-24.04, the release of the runplane -// base image. Quotes and a comment can follow the label. -var hostedRunsOn = regexp.MustCompile(`(?m)^([ \t]*(?:-[ \t]+)?runs-on:[ \t]*["']?)ubuntu-(?:latest|24\.04)(["']?[ \t]*(?:#[^\r\n]*)?\r?)$`) +// hostedRunsOn matches a runs-on line with one label of ghapp.HostedLabels. +// Quotes and a comment can follow the label. +var hostedRunsOn = regexp.MustCompile(`(?m)^([ \t]*(?:-[ \t]+)?runs-on:[ \t]*["']?)(?:` + anyOf(ghapp.HostedLabels) + + `)(["']?[ \t]*(?:#[^\r\n]*)?\r?)$`) // OpenPullRequest opens a pull request in the repository that sets runs-on -// to label in each job on ubuntu-latest or ubuntu-24.04, and returns its -// number and URL. While the pull request of an earlier call is open, it -// returns that one. base is the browser URL of runplane, for the -// description. +// to label in each job on one of ghapp.HostedLabels, and returns its number +// and URL. While the pull request of an earlier call is open, it returns +// that one. base is the browser URL of runplane, for the description. +// +// The jobs of the pull request must start, so it returns field.Problems at +// "label" when no enabled pool has the label, and at "repository" when the +// app has no access to the repository, runplane takes no jobs of it, or it +// has no job to move. It returns ghapp.ErrNoPullPermissions when the +// account has not accepted ghapp.PullPermissions. func (s *Service) OpenPullRequest(ctx context.Context, base, repository, label string) (int, string, error) { - repo, err := s.st.GetRepositoryByName(ctx, repository) - if errors.Is(err, store.ErrNotFound) { - return 0, "", huma.Error404NotFound("The GitHub App has no access to " + repository + ".") - } + repo, err := s.pullRepository(ctx, repository, label) if err != nil { return 0, "", err } @@ -81,15 +83,15 @@ func (s *Service) OpenPullRequest(ctx context.Context, base, repository, label s return 0, "", err } err = r.json(ctx, http.MethodPost, "/git/commits", map[string]any{ - "message": "Run jobs on runplane\n\nThe jobs on ubuntu-latest and ubuntu-24.04 use the runplane label " + label + ".", + "message": "Run jobs on runplane\n\nThe jobs on " + ghapp.HostedLabelsText("") + " use the runplane label " + label + ".", "tree": made.SHA, "parents": []string{branch.Commit.SHA}, }, &made) if err != nil { return 0, "", err } err = r.json(ctx, http.MethodPost, "/git/refs", map[string]any{"ref": "refs/heads/" + pullBranch, "sha": made.SHA}, nil) - var apiErr *apiError - if errors.As(err, &apiErr) && apiErr.status == http.StatusUnprocessableEntity { + var apiErr *ghrest.APIError + if errors.As(err, &apiErr) && apiErr.Status == http.StatusUnprocessableEntity { // The branch of an earlier pull request that is closed now. err = r.json(ctx, http.MethodPatch, "/git/refs/heads/"+pullBranch, map[string]any{"sha": made.SHA, "force": true}, nil) } @@ -113,7 +115,6 @@ type pullRequest struct { HTMLURL string `json:"html_url"` } -// treeFile is a file of a new Git tree, with its full content. type treeFile struct { Path string `json:"path"` Mode string `json:"mode"` @@ -121,34 +122,62 @@ type treeFile struct { Content string `json:"content"` } -// checkPullPermissions returns a 403 problem if the account of the -// installation did not accept pullPermissions. An app from before runplane -// asked for them gets them in its settings on GitHub. +// pullRepository returns the repository of a pull request that moves jobs +// to the pool label, or field.Problems when the jobs would not start. +func (s *Service) pullRepository(ctx context.Context, repository, label string) (store.Repository, error) { + pools, err := s.st.ListPools(ctx) + if err != nil { + return store.Repository{}, err + } + if !slices.ContainsFunc(pools, func(p store.Pool) bool { return p.Enabled && p.Label == label }) { + return store.Repository{}, field.Problems{{Key: "label", Message: "No enabled pool has this label.", Value: label}} + } + repo, err := s.st.GetRepositoryByName(ctx, repository) + if errors.Is(err, store.ErrNotFound) { + return store.Repository{}, repositoryProblem(repository, "The GitHub App has no access to "+repository+".") + } + if err != nil { + return store.Repository{}, err + } + inst, err := s.st.GetInstallation(ctx, repo.InstallationID) + if err != nil { + return store.Repository{}, err + } + if inst.SuspendedAt != nil { + return store.Repository{}, repositoryProblem(repository, "GitHub suspended the app installation on "+inst.AccountLogin+".") + } + settings, err := s.st.GetSettings(ctx) + if err != nil { + return store.Repository{}, err + } + if !policy.TakesJobs(settings, repo) { + return store.Repository{}, repositoryProblem(repository, "runplane takes no jobs of "+repo.FullName+", so the moved jobs would not start.") + } + return repo, nil +} + func (s *Service) checkPullPermissions(ctx context.Context, installationID int64) error { - var inst installation - err := s.api.json(ctx, call{method: http.MethodGet, path: fmt.Sprintf("/app/installations/%d", installationID), auth: appAuth{s}}, &inst) + inst, err := s.readInstallation(ctx, installationID) if err != nil { - return fmt.Errorf("read the installation: %w", err) + return err } - for _, name := range pullPermissions { - if inst.Permissions[name] != "write" { - return huma.Error403Forbidden("The GitHub App needs read and write access to Contents, Pull requests, and Workflows. " + - "Add these permissions in the app settings on GitHub, then accept them for " + inst.Account.Login + ".") + for _, p := range ghapp.PullPermissions { + if inst.Permissions[p.Name] != p.Access { + return fmt.Errorf("%w. Add these permissions in the app settings on GitHub, then accept them for %s", + ghapp.ErrNoPullPermissions, inst.Account.Login) } } return nil } -// repoREST sends REST calls about one repository as its installation. type repoREST struct { - api *rest - auth auth + api ghrest.Client + auth ghrest.Auth name string // owner/name } -// json sends a call to path below /repos/owner/name. func (r repoREST) json(ctx context.Context, method, path string, body, out any) error { - return r.api.json(ctx, call{method: method, path: "/repos/" + r.name + path, body: body, auth: r.auth}, out) + return r.api.JSON(ctx, ghrest.Call{Method: method, Path: "/repos/" + r.name + path, Body: body, Auth: r.auth}, out) } // moveWorkflows reads the workflows at the commit ref and returns the ones @@ -160,9 +189,9 @@ func (r repoREST) moveWorkflows(ctx context.Context, ref, label string) ([]treeF SHA string `json:"sha"` } err := r.json(ctx, http.MethodGet, "/contents/.github/workflows?ref="+ref, nil, &entries) - var apiErr *apiError - if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound { - return nil, 0, huma.Error422UnprocessableEntity(r.name + " has no workflows in .github/workflows.") + var apiErr *ghrest.APIError + if errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound { + return nil, 0, repositoryProblem(r.name, r.name+" has no workflows in .github/workflows.") } if err != nil { return nil, 0, err @@ -191,31 +220,44 @@ func (r repoREST) moveWorkflows(ctx context.Context, ref, label string) ([]treeF jobs += n } if jobs == 0 { - return nil, 0, huma.Error422UnprocessableEntity("No job in the workflows of " + r.name + " runs on ubuntu-latest or ubuntu-24.04.") + return nil, 0, repositoryProblem(r.name, "No job in the workflows of "+r.name+" runs on "+ghapp.HostedLabelsText("")+".") } return files, jobs, nil } +func repositoryProblem(repository, msg string) error { + return field.Problems{{Key: "repository", Message: msg, Value: repository}} +} + // moveJobs sets runs-on to label in each job of the workflow that runs on -// ubuntu-latest or ubuntu-24.04, and counts these jobs. The other lines -// stay as they are. +// one of ghapp.HostedLabels, and counts these jobs. The other lines stay as +// they are. func moveJobs(workflow, label string) (string, int) { n := len(hostedRunsOn.FindAllStringIndex(workflow, -1)) return hostedRunsOn.ReplaceAllString(workflow, "${1}"+label+"${2}"), n } +// anyOf is a regexp term that matches each of the texts. +func anyOf(texts []string) string { + quoted := make([]string, len(texts)) + for i, t := range texts { + quoted[i] = regexp.QuoteMeta(t) + } + return strings.Join(quoted, "|") +} + func pullDescription(base, label string, jobs int, files []treeFile) string { var b strings.Builder noun := "jobs" if jobs == 1 { noun = "job" } - fmt.Fprintf(&b, "This pull request sets `runs-on` to `%s` for %d %s on `ubuntu-latest` or `ubuntu-24.04`. "+ + fmt.Fprintf(&b, "This pull request sets `runs-on` to `%s` for %d %s on %s. "+ "These jobs then run on your runplane servers. Workflows that run on pull requests use the change for this pull request too, "+ - "so you can see the jobs in runplane before you merge.\n\nChanged workflows:\n\n", label, jobs, noun) + "so you can see the jobs in runplane before you merge.\n\nChanged workflows:\n\n", label, jobs, noun, ghapp.HostedLabelsText("`")) for _, f := range files { fmt.Fprintf(&b, "- `%s`\n", f.Path) } - fmt.Fprintf(&b, "\nJobs in runplane: %s/jobs\n", base) + fmt.Fprintf(&b, "\nJobs in runplane: %s%s\n", base, pages.Jobs) return b.String() } diff --git a/internal/github/pulls_test.go b/internal/github/pulls_test.go index 40a0a6c..5392d99 100644 --- a/internal/github/pulls_test.go +++ b/internal/github/pulls_test.go @@ -12,32 +12,52 @@ import ( "testing" "time" - "github.com/danielgtaylor/huma/v2" - + "github.com/runplane/runplane/internal/field" + "github.com/runplane/runplane/internal/ghapp" "github.com/runplane/runplane/internal/store" ) -func TestMoveJobs(t *testing.T) { +// The pull request sets runs-on to the label of the pool on each line with +// one GitHub-hosted label of the default image. The other lines stay as +// they are, and a workflow with no such line is no pull request. +func TestPullRequestMovesTheJobs(t *testing.T) { tests := []struct { - name, in, want string - jobs int + name, in, want string // want == in: no job moves }{ - {"ubuntu-latest", " runs-on: ubuntu-latest\n", " runs-on: big\n", 1}, - {"quoted ubuntu-24.04 with a comment", " runs-on: 'ubuntu-24.04' # pinned\n", " runs-on: 'big' # pinned\n", 1}, - {"CRLF line ends", "jobs:\r\n a:\r\n runs-on: ubuntu-latest\r\n", "jobs:\r\n a:\r\n runs-on: big\r\n", 1}, - {"matrix entry", " - runs-on: ubuntu-latest\n", " - runs-on: big\n", 1}, - {"two jobs", " a:\n runs-on: ubuntu-latest\n b:\n runs-on: ubuntu-24.04", " a:\n runs-on: big\n b:\n runs-on: big", 2}, - {"other release", " runs-on: ubuntu-22.04\n", " runs-on: ubuntu-22.04\n", 0}, - {"larger runner", " runs-on: ubuntu-latest-8-cores\n", " runs-on: ubuntu-latest-8-cores\n", 0}, - {"expression", " runs-on: ${{ matrix.os }}\n", " runs-on: ${{ matrix.os }}\n", 0}, - {"label list", " runs-on: [ubuntu-latest]\n", " runs-on: [ubuntu-latest]\n", 0}, - {"other key", " os: ubuntu-latest\n", " os: ubuntu-latest\n", 0}, + {"ubuntu-latest", " runs-on: ubuntu-latest\n", " runs-on: big\n"}, + {"quoted ubuntu-24.04 with a comment", " runs-on: 'ubuntu-24.04' # pinned\n", " runs-on: 'big' # pinned\n"}, + {"CRLF line ends", "jobs:\r\n a:\r\n runs-on: ubuntu-latest\r\n", "jobs:\r\n a:\r\n runs-on: big\r\n"}, + {"matrix entry", " - runs-on: ubuntu-latest\n", " - runs-on: big\n"}, + {"two jobs", " a:\n runs-on: ubuntu-latest\n b:\n runs-on: ubuntu-24.04", " a:\n runs-on: big\n b:\n runs-on: big"}, + {"other release", " runs-on: ubuntu-22.04\n", " runs-on: ubuntu-22.04\n"}, + {"larger runner", " runs-on: ubuntu-latest-8-cores\n", " runs-on: ubuntu-latest-8-cores\n"}, + {"expression", " runs-on: ${{ matrix.os }}\n", " runs-on: ${{ matrix.os }}\n"}, + {"label list", " runs-on: [ubuntu-latest]\n", " runs-on: [ubuntu-latest]\n"}, + {"other key", " os: ubuntu-latest\n", " os: ubuntu-latest\n"}, } for _, tt := range tests { - got, jobs := moveJobs(tt.in, "big") - if got != tt.want || jobs != tt.jobs { - t.Errorf("%s: moveJobs = %q, %d; want %q, %d", tt.name, got, jobs, tt.want, tt.jobs) - } + t.Run(tt.name, func(t *testing.T) { + e := acmeEnv(t) + f := e.repo() + f.workflows = map[string]string{"ci.yml": tt.in} + _, _, err := e.svc.OpenPullRequest(context.Background(), "https://ci.example.com", "acme/web", "big") + w := f.written() + if tt.want == tt.in { + if errorKind(err) != "repository problem" || len(w) != 0 { + t.Errorf("err = %v, writes = %v; want the repository problem and no writes", err, w) + } + return + } + if err != nil { + t.Fatal(err) + } + want := map[string]any{"base_tree": "t1", "tree": []any{map[string]any{ + "path": ".github/workflows/ci.yml", "mode": "100644", "type": "blob", "content": tt.want, + }}} + if got := w["POST /git/trees"]; !reflect.DeepEqual(got, want) { + t.Errorf("tree = %v, want %v", got, want) + } + }) } } @@ -84,6 +104,8 @@ func (e *env) repo() *fakeRepo { }); err != nil { e.t.Fatal(err) } + e.pool("p1", "big", true) + e.pool("p2", "off", false) f := &fakeRepo{ permissions: map[string]string{"actions": "write", "contents": "write", "metadata": "read", "pull_requests": "write", "workflows": "write"}, workflows: map[string]string{"ci.yml": ciWorkflow, "release.yaml": "jobs:\n build:\n runs-on: macos-latest\n", "README.md": "runs-on: ubuntu-latest\n"}, @@ -224,27 +246,45 @@ func TestOpenPullRequest(t *testing.T) { } }) + // The jobs of the pull request must start. + setRepo := func(e *env, r store.Repository) { + r.ID, r.FullName, r.DefaultBranch, r.UpdatedAt = 100, "acme/web", "main", time.Now() + if err := e.st.ReplaceRepositories(ctx, 10, []store.Repository{r}); err != nil { + t.Fatal(err) + } + } + suspend := func(e *env) { + now := time.Now() + if err := e.st.UpsertInstallation(ctx, store.Installation{ID: 10, AppID: testAppID, AccountLogin: "acme", + AccountType: "Organization", SuspendedAt: &now, CreatedAt: now, UpdatedAt: now}); err != nil { + t.Fatal(err) + } + } errorTests := []struct { - name string - repository string - change func(f *fakeRepo) - wantStatus int + name string + repository, label string + change func(e *env, f *fakeRepo) + want string // errorKind }{ - {"unknown repository", "acme/other", func(*fakeRepo) {}, http.StatusNotFound}, - {"no workflows permission", "acme/web", func(f *fakeRepo) { delete(f.permissions, "workflows") }, http.StatusForbidden}, - {"read-only contents", "acme/web", func(f *fakeRepo) { f.permissions["contents"] = "read" }, http.StatusForbidden}, - {"no workflows", "acme/web", func(f *fakeRepo) { f.workflows = nil }, http.StatusUnprocessableEntity}, - {"no job on ubuntu-latest", "acme/web", func(f *fakeRepo) { delete(f.workflows, "ci.yml") }, http.StatusUnprocessableEntity}, + {"unknown repository", "acme/other", "big", func(*env, *fakeRepo) {}, "repository problem"}, + {"label of a disabled pool", "acme/web", "off", func(*env, *fakeRepo) {}, "label problem"}, + {"label of no pool", "acme/web", "gone", func(*env, *fakeRepo) {}, "label problem"}, + {"suspended installation", "acme/web", "big", func(e *env, _ *fakeRepo) { suspend(e) }, "repository problem"}, + {"public repository", "acme/web", "big", func(e *env, _ *fakeRepo) { setRepo(e, store.Repository{}) }, "repository problem"}, + {"archived repository", "acme/web", "big", func(e *env, _ *fakeRepo) { setRepo(e, store.Repository{Private: true, Archived: true}) }, "repository problem"}, + {"no workflows permission", "acme/web", "big", func(_ *env, f *fakeRepo) { delete(f.permissions, "workflows") }, "no permissions"}, + {"read-only contents", "acme/web", "big", func(_ *env, f *fakeRepo) { f.permissions["contents"] = "read" }, "no permissions"}, + {"no workflows", "acme/web", "big", func(_ *env, f *fakeRepo) { f.workflows = nil }, "repository problem"}, + {"no job on ubuntu-latest", "acme/web", "big", func(_ *env, f *fakeRepo) { delete(f.workflows, "ci.yml") }, "repository problem"}, } for _, tt := range errorTests { t.Run(tt.name, func(t *testing.T) { e := acmeEnv(t) f := e.repo() - tt.change(f) - _, _, err := e.svc.OpenPullRequest(ctx, "https://ci.example.com", tt.repository, "big") - var se huma.StatusError - if !errors.As(err, &se) || se.GetStatus() != tt.wantStatus { - t.Errorf("err = %v, want status %d", err, tt.wantStatus) + tt.change(e, f) + _, _, err := e.svc.OpenPullRequest(ctx, "https://ci.example.com", tt.repository, tt.label) + if got := errorKind(err); got != tt.want { + t.Errorf("err = %v (%s), want %s", err, got, tt.want) } if w := f.written(); len(w) != 0 { t.Errorf("writes = %v, want none", w) @@ -252,3 +292,16 @@ func TestOpenPullRequest(t *testing.T) { }) } } + +// errorKind names the errors of OpenPullRequest that the API shows to the +// user. +func errorKind(err error) string { + var p field.Problems + switch { + case errors.Is(err, ghapp.ErrNoPullPermissions): + return "no permissions" + case errors.As(err, &p) && len(p) == 1: + return p[0].Key + " problem" + } + return "another error" +} diff --git a/internal/github/releases.go b/internal/github/releases.go deleted file mode 100644 index 5bb53e6..0000000 --- a/internal/github/releases.go +++ /dev/null @@ -1,206 +0,0 @@ -package github - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/url" - "slices" - "strings" - "time" - - "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/notify" -) - -func (s *Service) releaseLoop(ctx context.Context) { - for { - if err := s.checkReleases(ctx); err != nil && ctx.Err() == nil { - s.log.Warn("check runner releases", "err", err) - } - if sleep(ctx, time.Hour) != nil { - return - } - } -} - -// checkReleases records the latest actions/runner release for the server -// arches, asks for its bundles, reads the end dates of all known versions, -// and warns about a version in use that ends soon. -func (s *Service) checkReleases(ctx context.Context) error { - var a auth // a token has a higher rate limit than anonymous reads - if inst, ok := s.anyInstallation(ctx); ok { - a = installAuth{s, inst.ID} - } - var rel struct { - TagName string `json:"tag_name"` - PublishedAt *time.Time `json:"published_at"` - } - if err := s.api.json(ctx, call{method: http.MethodGet, path: "/repos/actions/runner/releases/latest", auth: a}, &rel); err != nil { - return fmt.Errorf("read the latest runner release: %w", err) - } - latest := strings.TrimPrefix(rel.TagName, "v") - if latest == "" { - return errors.New("read the latest runner release: no tag name") - } - arches, err := s.serverArches(ctx) - if err != nil { - return err - } - // The row comes first, so that the bundle builder updates it. - for _, arch := range arches { - s.setBundleDates(ctx, latest, arch, rel.PublishedAt, nil) - } - if err := s.bundles.EnsureLatest(ctx, latest); err != nil { - s.log.Warn("make runner bundles", "version", latest, "err", err) - } - if err := s.readEndDates(ctx); err != nil { - s.log.Warn("read runner end dates", "err", err) - } - return s.warnEnds(ctx, arches) -} - -// readEndDates reads the end date of each known runner version with the -// runner deprecations API (GitHub changelog, 3 September 2026). The API -// needs an organization or a repository, so it uses an installation. -func (s *Service) readEndDates(ctx context.Context) error { - scope, a, ok := s.deprecationScope(ctx) - if !ok { - return nil - } - bundles, err := s.st.ListRunnerBundles(ctx) - if err != nil { - return err - } - var errs []error - done := map[string]bool{} - for _, b := range bundles { - if done[b.Version] { - continue - } - done[b.Version] = true - var d struct { - RegistrationDeprecatesAt *time.Time `json:"registration_deprecates_at"` - RuntimeDeprecatesAt *time.Time `json:"runtime_deprecates_at"` - } - err := s.api.json(ctx, call{ - method: http.MethodGet, path: scope + "/actions/runners/deprecations/" + url.PathEscape(b.Version), auth: a, - }, &d) - var apiErr *apiError - if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound { - continue - } - if err != nil { - errs = append(errs, err) - continue - } - end := earliest(d.RuntimeDeprecatesAt, d.RegistrationDeprecatesAt) - if end == nil { - continue - } - for _, x := range bundles { - if x.Version == b.Version { - s.setBundleDates(ctx, x.Version, x.Arch, nil, end) - } - } - } - return errors.Join(errs...) -} - -// earliest returns the earlier time. Each new sandbox registers a JIT -// runner and then runs a job, so the version ends at the first of the -// registration and runtime end dates. -func earliest(a, b *time.Time) *time.Time { - switch { - case a == nil: - return b - case b == nil || a.Before(*b): - return a - } - return b -} - -// deprecationScope returns the REST path of an organization or repository -// that the app can read, with its installation. An organization comes -// first, because its answer does not depend on one repository. -func (s *Service) deprecationScope(ctx context.Context) (string, auth, bool) { - insts, err := s.st.ListInstallations(ctx) - if err != nil { - return "", nil, false - } - var repoScope string - var repoAuth auth - for _, inst := range insts { - switch { - case inst.SuspendedAt != nil: - case inst.AccountType == "Organization": - return "/orgs/" + url.PathEscape(inst.AccountLogin), installAuth{s, inst.ID}, true - case repoScope == "": - if repos, err := s.st.ListRepositories(ctx, inst.ID); err == nil && len(repos) > 0 { - repoScope, repoAuth = "/repos/"+repos[0].FullName, installAuth{s, inst.ID} - } - } - } - return repoScope, repoAuth, repoScope != "" -} - -// setBundleDates writes the dates of one bundle row and publishes the row -// when it changed. -func (s *Service) setBundleDates(ctx context.Context, version, arch string, releasedAt, endDate *time.Time) { - before, berr := s.st.GetRunnerBundle(ctx, version, arch) - if err := s.st.SetRunnerBundleDates(ctx, version, arch, releasedAt, endDate); err != nil { - s.log.Warn("record runner version", "version", version, "arch", arch, "err", err) - return - } - after, err := s.st.GetRunnerBundle(ctx, version, arch) - if err != nil { - return - } - if berr == nil && sameTime(before.ReleasedAt, after.ReleasedAt) && sameTime(before.EndDate, after.EndDate) { - return - } - s.bus.Publish(events.Event{Type: events.RunnerVersion, Data: after}) -} - -// warnEnds sends one notification for each runner version that new -// sandboxes use and that GitHub stops in less than 14 days. -func (s *Service) warnEnds(ctx context.Context, arches []string) error { - settings, err := s.st.GetSettings(ctx) - if err != nil { - return err - } - for _, arch := range arches { - b, err := s.st.LatestReadyBundle(ctx, arch) - if err != nil || b.EndDate == nil || time.Until(*b.EndDate) > 14*24*time.Hour || s.warned[b.Version] { - continue - } - s.log.Warn("runner version ends soon", "version", b.Version, "end_date", b.EndDate) - if err := s.notify.Notify(ctx, notify.RunnerDeprecationEvent(settings.DashboardURL(), b.Version, *b.EndDate)); err != nil { - return fmt.Errorf("send runner end date warning: %w", err) - } - s.warned[b.Version] = true - } - return nil -} - -func (s *Service) serverArches(ctx context.Context) ([]string, error) { - servers, err := s.st.ListServers(ctx) - if err != nil { - return nil, err - } - var arches []string - for _, srv := range servers { - if srv.Arch != "" && !slices.Contains(arches, srv.Arch) { - arches = append(arches, srv.Arch) - } - } - return arches, nil -} - -func sameTime(a, b *time.Time) bool { - if a == nil || b == nil { - return a == b - } - return a.Equal(*b) -} diff --git a/internal/github/releases_test.go b/internal/github/releases_test.go deleted file mode 100644 index fe6f9a2..0000000 --- a/internal/github/releases_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package github - -import ( - "context" - "net/http" - "slices" - "testing" - "time" - - "github.com/runplane/runplane/internal/events" - "github.com/runplane/runplane/internal/notify" - "github.com/runplane/runplane/internal/store" -) - -func TestCheckReleases(t *testing.T) { - tests := []struct { - name string - runtime time.Duration // runtime end of the version in use, from now - wantWarns int - }{ - {"end in 10 days warns once", 10 * 24 * time.Hour, 1}, - {"end in 20 days does not warn", 20 * 24 * time.Hour, 0}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - e := acmeEnv(t) - ctx := context.Background() - for i, arch := range []string{"x64", "arm64", "x64"} { - id := string(rune('a' + i)) - err := e.st.CreateServer(ctx, store.Server{ID: id, Name: id, Kind: store.ServerAgent, Status: store.ServerOnline, - ServerFacts: store.ServerFacts{Arch: arch}, CreatedAt: time.Now(), UpdatedAt: time.Now()}) - if err != nil { - t.Fatal(err) - } - } - for _, arch := range []string{"x64", "arm64"} { - err := e.st.UpsertRunnerBundle(ctx, store.RunnerBundle{Version: "2.330.0", Arch: arch, SHA256: "sha-" + arch, - Patched: true, Status: store.BundleReady, Path: "/runners/2.330.0-" + arch, CreatedAt: time.Now()}) - if err != nil { - t.Fatal(err) - } - } - runtimeEnd := time.Now().Add(tt.runtime).UTC().Truncate(time.Second) - e.mux.HandleFunc("GET /api/v3/repos/actions/runner/releases/latest", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]any{"tag_name": "v2.331.0", "published_at": "2026-11-01T12:00:00Z"}) - }) - e.mux.HandleFunc("GET /api/v3/orgs/acme/actions/runners/deprecations/{version}", func(w http.ResponseWriter, r *http.Request) { - switch r.PathValue("version") { - case "2.330.0": - writeJSON(w, http.StatusOK, map[string]any{"runner_version": "2.330.0", - "registration_deprecates_at": runtimeEnd.Add(48 * time.Hour), "runtime_deprecates_at": runtimeEnd}) - case "2.331.0": - writeJSON(w, http.StatusOK, map[string]any{"runner_version": "2.331.0", - "registration_deprecates_at": nil, "runtime_deprecates_at": nil}) - default: - writeJSON(w, http.StatusNotFound, map[string]any{"message": "Not Found"}) - } - }) - published, unsubscribe := e.bus.Subscribe(16) - defer unsubscribe() - - if err := e.svc.checkReleases(ctx); err != nil { - t.Fatal(err) - } - if !slices.Equal(e.bundles.versions, []string{"2.331.0"}) { - t.Errorf("EnsureLatest got %v", e.bundles.versions) - } - released := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC) - for _, arch := range []string{"x64", "arm64"} { - latest, err := e.st.GetRunnerBundle(ctx, "2.331.0", arch) - if err != nil || latest.Status != store.BundlePending || latest.ReleasedAt == nil || !latest.ReleasedAt.Equal(released) || latest.EndDate != nil { - t.Errorf("2.331.0 %s = %+v, %v", arch, latest, err) - } - old, _ := e.st.GetRunnerBundle(ctx, "2.330.0", arch) - if old.Status != store.BundleReady || old.SHA256 != "sha-"+arch || old.Path == "" || !old.Patched { - t.Errorf("the end date write changed the build columns: %+v", old) - } - if old.EndDate == nil || !old.EndDate.Equal(runtimeEnd) { - t.Errorf("2.330.0 %s end date = %v, want the runtime end %v", arch, old.EndDate, runtimeEnd) - } - } - if n := countEvents(published, events.RunnerVersion); n != 4 { - t.Errorf("published %d runner.version events, want 4", n) - } - sent := e.notifier.sent - if len(sent) != tt.wantWarns || len(sent) > 0 && (sent[0].Type != notify.RunnerDeprecation || sent[0].Data["version"] != "2.330.0") { - t.Errorf("warnings = %+v, want %d for 2.330.0", sent, tt.wantWarns) - } - - // The next hour changes nothing. - if err := e.svc.checkReleases(ctx); err != nil { - t.Fatal(err) - } - if len(e.notifier.sent) != tt.wantWarns || countEvents(published, events.RunnerVersion) != 0 { - t.Errorf("the second check sent %d warnings and published events, want %d and none", len(e.notifier.sent), tt.wantWarns) - } - }) - } -} diff --git a/internal/github/rest.go b/internal/github/rest.go deleted file mode 100644 index c35774f..0000000 --- a/internal/github/rest.go +++ /dev/null @@ -1,297 +0,0 @@ -package github - -import ( - "bytes" - "cmp" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/runplane/runplane/internal/version" -) - -// apiError is a GitHub REST response with an error status. -type apiError struct { - method, path string - status int - message string - // retryAt is set when GitHub rate-limits the request and the wait is - // longer than runplane accepts in one call. - retryAt time.Time -} - -func (e *apiError) Error() string { - msg := fmt.Sprintf("GitHub API %s %s: %d %s", e.method, e.path, e.status, http.StatusText(e.status)) - if e.message != "" { - msg += ": " + e.message - } - if !e.retryAt.IsZero() { - msg += fmt.Sprintf(" (rate limit, try again after %s)", e.retryAt.UTC().Format(time.RFC3339)) - } - return msg -} - -// auth gives the Authorization header of a request. -type auth interface { - header(ctx context.Context) (string, error) - // invalidate drops a cached credential after GitHub refused it. - invalidate() -} - -// rest is a small client for the GitHub REST API. -type rest struct { - http *http.Client - noRedirect *http.Client - base string // API URL without a trailing slash - retryWait time.Duration // first wait after a network error or a 5xx -} - -const ( - callTimeout = 30 * time.Second // for one attempt, the body read included - maxRateWait = time.Minute // longest rate limit wait in one call -) - -func newREST(base string) *rest { - return &rest{ - http: &http.Client{}, - noRedirect: &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, - base: base, - retryWait: time.Second, - } -} - -// call is one REST request. path is relative to the API URL or absolute -// (pagination links). -type call struct { - method string - path string - body any // sent as JSON when not nil - auth auth // nil = anonymous - timeout time.Duration // 0 = callTimeout - noRedirect bool -} - -// do sends the call and returns a response with a status below 400. The -// caller closes the body; closing it also ends the attempt's timeout. do -// tries again after up to three short rate limit waits, after network errors -// and 5xx answers of safe methods, and once after a refused cached token. -func (c *rest) do(ctx context.Context, cl call) (*http.Response, error) { - safe := cl.method == http.MethodGet || cl.method == http.MethodHead - refreshed := false - failures, limits := 0, 0 - for { - var authorization string - if cl.auth != nil { - var err error - if authorization, err = cl.auth.header(ctx); err != nil { - return nil, err - } - } - resp, err := c.send(ctx, cl, authorization) - if err == nil { - if resp.StatusCode < 400 { - return resp, nil - } - apiErr := readAPIError(cl.method, cl.path, resp) - switch wait, limited := rateLimitWait(resp, apiErr.message); { - case limited && wait <= maxRateWait && limits < 3: - limits++ - if err := sleep(ctx, wait); err != nil { - return nil, err - } - continue - case limited: - apiErr.retryAt = time.Now().Add(wait) - return nil, apiErr - case resp.StatusCode == http.StatusUnauthorized && cl.auth != nil && !refreshed: - cl.auth.invalidate() - refreshed = true - continue - case resp.StatusCode < 500: - return nil, apiErr - } - err = apiErr - } - if ctx.Err() != nil || !safe || failures >= 2 { - return nil, err - } - failures++ - if sleep(ctx, c.retryWait<<(failures-1)) != nil { - return nil, err - } - } -} - -// send sends one attempt. Closing the response body ends its timeout. -func (c *rest) send(ctx context.Context, cl call, authorization string) (*http.Response, error) { - u := cl.path - if strings.HasPrefix(u, "/") { - u = c.base + u - } - var body io.Reader - if cl.body != nil { - b, err := json.Marshal(cl.body) - if err != nil { - return nil, fmt.Errorf("encode request %s %s: %w", cl.method, cl.path, err) - } - body = bytes.NewReader(b) - } - actx, cancel := context.WithTimeout(ctx, cmp.Or(cl.timeout, callTimeout)) - req, err := http.NewRequestWithContext(actx, cl.method, u, body) - if err != nil { - cancel() - return nil, fmt.Errorf("make request %s %s: %w", cl.method, cl.path, err) - } - if body != nil { - req.Header.Set("Content-Type", "application/json") - } - req.Header.Set("Accept", "application/vnd.github+json") - req.Header.Set("X-GitHub-Api-Version", "2022-11-28") - req.Header.Set("User-Agent", "runplane/"+version.Version) - if authorization != "" { - req.Header.Set("Authorization", authorization) - } - hc := c.http - if cl.noRedirect { - hc = c.noRedirect - } - resp, err := hc.Do(req) - if err != nil { - cancel() - return nil, fmt.Errorf("GitHub API %s %s: %w", cl.method, cl.path, err) - } - resp.Body = cancelOnClose{resp.Body, cancel} - return resp, nil -} - -// json sends the call and decodes a JSON answer into out (if not nil). -func (c *rest) json(ctx context.Context, cl call, out any) error { - resp, err := c.do(ctx, cl) - if err != nil { - return err - } - defer resp.Body.Close() - if out == nil || resp.StatusCode == http.StatusNoContent { - return nil - } - if err := json.NewDecoder(resp.Body).Decode(out); err != nil { - return fmt.Errorf("decode GitHub API %s %s: %w", cl.method, cl.path, err) - } - return nil -} - -// list reads all pages of a GET list. key is the field of the page object -// that holds the items, or "" when each page is a JSON array. path must -// have per_page in its query. -func list[T any](ctx context.Context, c *rest, a auth, path, key string) ([]T, error) { - var all []T - for next := path; next != ""; { - resp, err := c.do(ctx, call{method: http.MethodGet, path: next, auth: a}) - if err != nil { - return nil, err - } - var page []T - err = decodePage(resp.Body, key, &page) - resp.Body.Close() - if err != nil { - return nil, fmt.Errorf("decode GitHub API GET %s: %w", next, err) - } - all = append(all, page...) - next = nextLink(resp.Header.Get("Link")) - } - return all, nil -} - -func decodePage(r io.Reader, key string, out any) error { - if key == "" { - return json.NewDecoder(r).Decode(out) - } - var obj map[string]json.RawMessage - if err := json.NewDecoder(r).Decode(&obj); err != nil { - return err - } - raw, ok := obj[key] - if !ok || string(raw) == "null" { - return nil - } - return json.Unmarshal(raw, out) -} - -// nextLink returns the rel="next" URL of a Link header. -func nextLink(header string) string { - for part := range strings.SplitSeq(header, ",") { - target, params, ok := strings.Cut(part, ";") - if ok && strings.Contains(params, `rel="next"`) { - return strings.Trim(strings.TrimSpace(target), "<>") - } - } - return "" -} - -func readAPIError(method, path string, resp *http.Response) *apiError { - e := &apiError{method: method, path: path, status: resp.StatusCode} - if u, err := url.Parse(path); err == nil && u.IsAbs() { - e.path = u.Path - } - b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) - resp.Body.Close() - var body struct { - Message string `json:"message"` - } - if json.Unmarshal(b, &body) == nil { - e.message = body.Message - } else { - e.message = strings.TrimSpace(string(b)) - } - return e -} - -// rateLimitWait tells if a response is a rate limit and how long to wait. -// GitHub sends Retry-After for secondary limits and X-RateLimit-Reset for -// the primary limit. Without either, GitHub asks for at least one minute. -func rateLimitWait(resp *http.Response, message string) (time.Duration, bool) { - if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusTooManyRequests { - return 0, false - } - if s, err := strconv.Atoi(resp.Header.Get("Retry-After")); err == nil && s >= 0 { - return time.Duration(s) * time.Second, true - } - if resp.Header.Get("X-RateLimit-Remaining") == "0" { - if reset, err := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 64); err == nil { - return max(time.Until(time.Unix(reset, 0))+time.Second, 0), true - } - return time.Minute, true - } - if resp.StatusCode == http.StatusTooManyRequests || strings.Contains(strings.ToLower(message), "secondary rate limit") { - return time.Minute, true - } - return 0, false -} - -type cancelOnClose struct { - io.ReadCloser - cancel context.CancelFunc -} - -func (b cancelOnClose) Close() error { - err := b.ReadCloser.Close() - b.cancel() - return err -} - -func sleep(ctx context.Context, d time.Duration) error { - t := time.NewTimer(d) - defer t.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-t.C: - return nil - } -} diff --git a/internal/github/rest_test.go b/internal/github/rest_test.go deleted file mode 100644 index 9d51212..0000000 --- a/internal/github/rest_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package github - -import ( - "context" - "errors" - "net/http" - "net/http/httptest" - "strconv" - "testing" - "time" -) - -func TestRESTRetries(t *testing.T) { - reset := time.Now().Add(time.Hour).Truncate(time.Second) - tests := []struct { - name string - method string - first func(w http.ResponseWriter) - wantCalls int - wantWait time.Duration - wantRetry time.Time // RetryAt of the error, zero for no error - }{ - { - name: "secondary limit with Retry-After waits and tries again", method: http.MethodGet, - first: func(w http.ResponseWriter) { - w.Header().Set("Retry-After", "1") - writeJSON(w, http.StatusForbidden, map[string]any{"message": "You have exceeded a secondary rate limit."}) - }, - wantCalls: 2, wantWait: time.Second, - }, - { - name: "429 with Retry-After waits and tries again", method: http.MethodPost, - first: func(w http.ResponseWriter) { - w.Header().Set("Retry-After", "1") - w.WriteHeader(http.StatusTooManyRequests) - }, - wantCalls: 2, wantWait: time.Second, - }, - { - name: "primary limit with a far reset fails at once", method: http.MethodGet, - first: func(w http.ResponseWriter) { - w.Header().Set("X-RateLimit-Remaining", "0") - w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) - writeJSON(w, http.StatusForbidden, map[string]any{"message": "API rate limit exceeded"}) - }, - wantCalls: 1, wantRetry: reset, - }, - { - name: "403 without rate limit signs is no rate limit", method: http.MethodGet, - first: func(w http.ResponseWriter) { - writeJSON(w, http.StatusForbidden, map[string]any{"message": "Resource not accessible by integration"}) - }, - wantCalls: 1, - }, - { - name: "5xx of a read tries again", method: http.MethodGet, - first: func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadGateway) }, - wantCalls: 2, - }, - { - // A POST such as a re-run must not happen twice. - name: "5xx of a write fails at once", method: http.MethodPost, - first: func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadGateway) }, - wantCalls: 1, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - calls := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls++ - if calls == 1 { - tt.first(w) - return - } - writeJSON(w, http.StatusOK, map[string]any{"ok": true}) - })) - defer srv.Close() - c := newREST(srv.URL) - c.retryWait = time.Millisecond - start := time.Now() - var out struct{ OK bool } - err := c.json(context.Background(), call{method: tt.method, path: "/x"}, &out) - if calls != tt.wantCalls { - t.Errorf("calls = %d, want %d", calls, tt.wantCalls) - } - // Only the second try gets an answer. - if (err == nil) != (tt.wantCalls == 2) || err == nil && !out.OK { - t.Errorf("err = %v, answer = %v", err, out.OK) - } - if waited := time.Since(start); waited < tt.wantWait { - t.Errorf("waited %s, want at least %s", waited, tt.wantWait) - } - var apiErr *apiError - var retryAt time.Time - if errors.As(err, &apiErr) { - retryAt = apiErr.retryAt - } - if d := retryAt.Sub(tt.wantRetry); d < 0 || d > 5*time.Second { - t.Errorf("RetryAt = %s, want about %s", retryAt, tt.wantRetry) - } - }) - } -} - -func TestNextLink(t *testing.T) { - h := `; rel="prev", ; rel="next", ; rel="last"` - if got := nextLink(h); got != "https://api.github.com/app/installations?page=3" { - t.Errorf("nextLink = %q", got) - } - if got := nextLink(`; rel="first"`); got != "" { - t.Errorf("nextLink on the last page = %q, want empty", got) - } -} diff --git a/internal/github/scalesets.go b/internal/github/scalesets.go index cbf4ebe..4c0daaa 100644 --- a/internal/github/scalesets.go +++ b/internal/github/scalesets.go @@ -14,17 +14,18 @@ import ( "github.com/actions/scaleset" + "github.com/runplane/runplane/internal/ghapp" + "github.com/runplane/runplane/internal/ghrest" "github.com/runplane/runplane/internal/ids" + "github.com/runplane/runplane/internal/policy" "github.com/runplane/runplane/internal/scheduler" "github.com/runplane/runplane/internal/store" "github.com/runplane/runplane/internal/version" ) -// defaultRunnerGroup is the ID of the "Default" runner group. A free -// organization can only use this group, and a repository has only this one. -const defaultRunnerGroup = 1 - -// target is one scale set that runplane wants on GitHub. +// target is one scale set that runplane wants on GitHub. An organization +// has one for each pool, and a personal account one for each pool and +// repository. type target struct { pool store.Pool inst store.Installation @@ -59,11 +60,12 @@ func (t target) owner() string { return t.inst.AccountLogin } -func (t target) runnersPath() string { +// path is the REST path of the runner owner. +func (t target) path() string { if t.repo != nil { - return "/repos/" + t.repo.FullName + "/actions/runners" + return "/repos/" + t.repo.FullName } - return "/orgs/" + url.PathEscape(t.inst.AccountLogin) + "/actions/runners" + return "/orgs/" + url.PathEscape(t.inst.AccountLogin) } // targets returns the scale sets that the pools, installations, and @@ -83,7 +85,7 @@ func (s *Service) targets(ctx context.Context) (map[string]target, error) { } byInst := map[int64][]store.Repository{} for _, inst := range insts { - if inst.AccountType != "User" { + if inst.AccountType != ghapp.User { continue } repos, err := s.st.ListRepositories(ctx, inst.ID) @@ -98,17 +100,14 @@ func (s *Service) targets(ctx context.Context) (map[string]target, error) { continue } for _, inst := range insts { - if inst.AccountType == "Organization" { + if inst.AccountType == ghapp.Organization { t := target{pool: pool, inst: inst} out[rowKey(t.row())] = t continue } + // A repository whose jobs runplane refuses gets no scale set. for _, r := range byInst[inst.ID] { - // The scheduler takes a public job only if each runner of its - // pool is in a VM. The setting mode does not tell that: each - // server runs its own mode, and a mode change applies after a - // restart. - if r.Archived || (!r.Private && !settings.AllowPublicRepos) { + if !policy.TakesJobs(settings, r) { continue } t := target{pool: pool, inst: inst, repo: &r} @@ -174,11 +173,8 @@ func (s *Service) reconcileSets(ctx context.Context) error { func (s *Service) keepSet(ctx context.Context, row store.ScaleSet, t target) { if t.inst.SuspendedAt != nil { s.stopListener(row.ID) - msg := "GitHub suspended the app installation on " + t.inst.AccountLogin + - ". runplane listens again when the installation is active." - if row.Status != store.ScaleSetError || row.Error != msg { - s.setStatus(ctx, row.ID, store.ScaleSetError, msg) - } + s.setStatus(ctx, row.ID, store.ScaleSetError, "GitHub suspended the app installation on "+t.inst.AccountLogin+ + ". runplane listens again when the installation is active.") return } s.lmu.Lock() @@ -190,23 +186,8 @@ func (s *Service) keepSet(ctx context.Context, row store.ScaleSet, t target) { } } if row.Name != t.pool.Label { - s.rename(ctx, l, row, t.pool.Label) - } -} - -func (s *Service) rename(ctx context.Context, l *setListener, row store.ScaleSet, label string) { - if row.GitHubID != 0 { - _, err := l.client().UpdateRunnerScaleSet(ctx, int(row.GitHubID), scaleSetSpec(label, int(row.RunnerGroupID))) - if err != nil { - s.setStatus(ctx, row.ID, store.ScaleSetError, "rename the scale set to "+label+" on GitHub: "+err.Error()) - return - } + l.rename(ctx, row, t.pool.Label) } - if err := s.st.SetScaleSetRemote(ctx, row.ID, row.GitHubID, row.RunnerGroupID, label); err != nil { - s.log.Warn("rename scale set", "scale_set", row.ID, "err", err) - return - } - l.setLabel(label) } // removeSet stops the listener and deletes the scale set on GitHub, then @@ -220,9 +201,7 @@ func (s *Service) removeSet(ctx context.Context, row store.ScaleSet) { return } if err == nil && t.inst.SuspendedAt == nil && (t.repo == nil || !t.repo.Archived) { - if row.Status != store.ScaleSetDeleting { - s.setStatus(ctx, row.ID, store.ScaleSetDeleting, "") - } + s.setStatus(ctx, row.ID, store.ScaleSetDeleting, "") if err := s.deleteRemote(ctx, row.ID); err != nil { s.setStatus(ctx, row.ID, store.ScaleSetDeleting, "delete the scale set on GitHub: "+err.Error()) return @@ -340,17 +319,16 @@ type runnerLabel struct { // listRunners returns the runners of one scale set. The REST runner object // has no scale set ID, so a runner belongs to the scale set when it has the -// scale set label and a name that starts with "