diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 194ed9c..ab626a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,17 @@ concurrency: permissions: contents: read +# Docker Hub limits the anonymous pulls of an address, and the runners share +# theirs: the jobs that pull log in with a read-only token of the +# repository's secrets (DOCKERHUB_USERNAME, DOCKERHUB_TOKEN), as the runner +# and as root (install.sh runs as root), when the secrets are there. BuildKit +# also asks Google's mirror of Docker Hub first. +env: + BUILDKITD_MIRROR: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + jobs: checks: name: Lint, typecheck, test, build @@ -41,9 +52,16 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Log in to Docker Hub + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 with: driver-opts: image=moby/buildkit:v0.33.1@sha256:cec9f139f45e93c5c69c60f8b07cfad9f43f4ef6b6a6cd917527fea5ff2e3dea + buildkitd-config-inline: ${{ env.BUILDKITD_MIRROR }} - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . @@ -65,6 +83,12 @@ jobs: node-version: 22 # The script only needs node; no pnpm here to cache for. package-manager-cache: false + - name: Log in to Docker Hub + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Create, call, update and delete examples/node-postgres run: scripts/e2e-engine.sh @@ -80,9 +104,19 @@ jobs: with: node-version: 22 package-manager-cache: false + - name: Log in to Docker Hub + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Log root in to Docker Hub too + if: env.DOCKERHUB_USERNAME != '' + run: sudo install -D -m 600 ~/.docker/config.json /root/.docker/config.json - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 with: driver-opts: image=moby/buildkit:v0.33.1@sha256:cec9f139f45e93c5c69c60f8b07cfad9f43f4ef6b6a6cd917527fea5ff2e3dea + buildkitd-config-inline: ${{ env.BUILDKITD_MIRROR }} - name: Build the image uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d0c4b6..847dd35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ - Compose: an environment has 4 CPUs and 4096 processes to share between its services (`SPAWNER_ENV_CPUS`, `SPAWNER_ENV_PIDS`), as it has its memory, and no service gets raw sockets (`NET_RAW`). A file asking more CPUs or processes than that is refused. - Uploads: links are followed through the links they lead to once extracted. - CLI: `spawner login` opens http and https URLs only; untracked `.env-*`, `.npmrc`, private keys and `*.pem` files stay home like `.env` files; a Dockerfile is read only when it is a regular file of 1 MiB at most; the MCP server fences what comes from the environment and tells the model to read it as data. +- **A harder installation**: + - The installer no longer makes a token of the installation with every scope (`SPAWNER_BOOTSTRAP_TOKEN`) unless given one; installations that have one keep it until you empty its line in `.env`. + - It refuses a version older than the installed one (its database only migrates forward; `--allow-downgrade` after restoring a backup), a data directory other than the installation's, and system directories; it never makes new secrets over an installation that lost them, never upgrades without a backup (it starts Postgres for it), writes `.env` and `dns.env` in one move, keeps a trailing `=` of their values, and no longer shows the DNS credentials typed. + - The data directory is mode 700, rendered compose files and logs are readable by Spawner only, Traefik runs without capabilities but binding ports, Postgres and Traefik have memory limits, Spawner's code belongs to root, and Prisma's telemetry is off. A local install over HTTP warns that it serves every interface; the development stack listens on 127.0.0.1. + - git reaches GitHub, GitLab and Bitbucket with the host keys they publish, even the first time. - **Source repositories are chosen by admins**: the other sources of `spawner.yaml` must come from repositories listed in the project (Projects, Edit, "Source repositories"); `spawner up` refuses the others before sending anything. A branch could otherwise make Spawner clone any repository its deploy keys reach, or an address of the internal network. The upgrade lists for each project the repositories its live environments already use. - **Terminals and log streams close when their access goes**: deactivating a user, changing their role or revoking a token now closes the terminals and the streams of logs they had open, at once, and an expired token's within 30 seconds. - **Uploads stop at their first limit**: an archive is read as a stream that stops at the first file, byte or decompressed byte over the limits, so that a compression bomb costs no more than a legitimate archive; a person has 5 deploys of uploaded code waiting to start at most. diff --git a/Dockerfile b/Dockerfile index f453c48..96853e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,9 @@ FROM node:22-bookworm-slim AS base RUN apt-get update \ && apt-get install -y --no-install-recommends openssl ca-certificates \ && rm -rf /var/lib/apt/lists/* -RUN npm install -g pnpm@8.15.0 FROM base AS build +RUN npm install -g pnpm@8.15.0 ENV TURBO_TELEMETRY_DISABLED=1 # Set by the release workflow from its tag (2.0.0-rc.1): the version the CLI # bundle and the API report. Empty: the version of the package.json files. @@ -38,14 +38,16 @@ RUN apt-get update \ COPY --from=docker:29-cli /usr/local/bin/docker /usr/local/bin/docker COPY --from=docker:29-cli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins WORKDIR /app -COPY --from=build --chown=node:node /out ./ -COPY --from=build --chown=node:node /repo/apps/web/dist ./web -COPY --from=build --chown=node:node /repo/apps/cli/dist/spawner.cjs ./cli/spawner +# Owned by root: Spawner, which runs as node, cannot change its own code. +COPY --from=build /out ./ +COPY --from=build /repo/apps/web/dist ./web +COPY --from=build /repo/apps/cli/dist/spawner.cjs ./cli/spawner # The installer of this version: an update from the dashboard runs it with # --upgrade, from a short-lived container of this image. firewall.nft: the # rules the spawner-firewall container of the stack loads on the host. COPY install.sh firewall.nft LICENSE NOTICE ./ ENV NODE_ENV=production \ + CHECKPOINT_DISABLE=1 \ PORT=3000 \ WEB_DIST_PATH=/app/web \ SPAWNER_CLI_PATH=/app/cli/spawner \ diff --git a/apps/api/docker-entrypoint.sh b/apps/api/docker-entrypoint.sh index 5399d90..b051181 100755 --- a/apps/api/docker-entrypoint.sh +++ b/apps/api/docker-entrypoint.sh @@ -14,11 +14,15 @@ if [ "$(id -u)" = "0" ]; then fi # The data directory and its first level may come from the host or from # Docker, as root, when it creates the mount source of Traefik: Spawner - # writes in all of them. + # writes in all of them. The data directory is Spawner's alone (the + # containers that mount parts of it do not go through it); route files + # stay readable by Traefik, which runs without capabilities. if [ -n "$SPAWNER_DATA_DIR" ]; then mkdir -p "$SPAWNER_DATA_DIR" chown node:node "$SPAWNER_DATA_DIR" 2>/dev/null || true + chmod 700 "$SPAWNER_DATA_DIR" 2>/dev/null || true find "$SPAWNER_DATA_DIR" -mindepth 1 -maxdepth 1 -type d -exec chown node:node {} + 2>/dev/null || true + find "$SPAWNER_DATA_DIR/traefik" -maxdepth 1 -type f -name '*.yaml' -exec chmod 644 {} + 2>/dev/null || true fi exec setpriv --reuid=node --regid=node --init-groups "$0" "$@" fi diff --git a/apps/api/known_hosts b/apps/api/known_hosts new file mode 100644 index 0000000..40609a9 --- /dev/null +++ b/apps/api/known_hosts @@ -0,0 +1,13 @@ +# Host keys of GitHub, GitLab and Bitbucket, as they publish them (api.github.com/meta, +# docs.gitlab.com, bitbucket.org/site/ssh), checked against their published fingerprints. +# git connects to these hosts with these keys only; other hosts are trusted on first use. +# Update it when one of them rotates a key. +github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl +github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= +github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= +bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQeJzhupRu0u0cdegZIa8e86EG2qOCsIsD1Xw0xSeiPDlCr7kq97NLmMbpKTX6Esc30NuoqEEHCuc7yWtwp8dI76EEEB1VqY9QJq6vk+aySyboD5QF61I/1WeTwu+deCbgKMGbUijeXhtfbxSxm6JwGrXrhBdofTsbKRUsrN1WoNgUa8uqN1Vx6WAJw1JHPhglEGGHea6QICwJOAr/6mrui/oB7pkaWKHj3z7d1IC4KWLtY47elvjbaTlkN04Kc/5LFEirorGYVbt15kAUlqGM65pk6ZBxtaO3+30LVlORZkxOh+LKL/BvbZ/iRNhItLqNyieoQj/uh/7Iv4uyH/cV/0b4WDSd3DptigWq84lJubb9t/DnZlrJazxyDCulTmKdOR7vs9gMTo+uoIrPSb8ScTtvw65+odKAlBj59dhnVp9zd7QUojOpXlL62Aw56U4oO+FALuevvMjiWeavKhJqlR7i5n9srYcrNV7ttmDw7kf/97P5zauIhxcjX+xHv4M= +bitbucket.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPIQmuzMBuKdWeF4+a2sjSSpBK0iqitSQ+5BM9KhpexuGt20JpTVM7u5BDZngncgrqDMbWdxMWWOGtZ9UgbqgZE= +bitbucket.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIazEu89wgQZ4bqs3d63QSMzYVa0MuJ2e2gKTKqu+UUO +gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY= +gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf +gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9 diff --git a/apps/api/package.json b/apps/api/package.json index 5abbe62..433ccd3 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -7,7 +7,8 @@ "files": [ "dist", "prisma", - "docker-entrypoint.sh" + "docker-entrypoint.sh", + "known_hosts" ], "scripts": { "build": "nest build", diff --git a/apps/api/src/modules/engine/git-keys.service.ts b/apps/api/src/modules/engine/git-keys.service.ts index ac76ecd..a0048ba 100644 --- a/apps/api/src/modules/engine/git-keys.service.ts +++ b/apps/api/src/modules/engine/git-keys.service.ts @@ -13,6 +13,8 @@ export interface RepoKeyInfo { } const GLOBAL_KEY = "id_spawner"; +/** apps/api/known_hosts, from dist/ as from src/. */ +const PUBLISHED_KNOWN_HOSTS = path.resolve(__dirname, "..", "..", "..", "known_hosts"); /** * SSH deploy keys. Each repository can have its own read-only key @@ -30,6 +32,14 @@ export class GitKeysService { return path.join(this.config.keysDir, "known_hosts"); } + /** + * The host keys of GitHub, GitLab and Bitbucket shipped with Spawner: git + * connects to them with these keys only, even the first time. + */ + get publishedKnownHostsPath(): string { + return PUBLISHED_KNOWN_HOSTS; + } + /** * The private key to use for a repository: its own key if it has one, * else the global key, else none. diff --git a/apps/api/src/modules/engine/git-mirror.service.ts b/apps/api/src/modules/engine/git-mirror.service.ts index af82aed..d1627a7 100644 --- a/apps/api/src/modules/engine/git-mirror.service.ts +++ b/apps/api/src/modules/engine/git-mirror.service.ts @@ -179,8 +179,9 @@ export class GitMirrorService { /** * Environment of git commands: no prompt, no system config, SSH and HTTPS - * transports only, and the repository's deploy key. Hosts are trusted on - * first use and pinned in known_hosts afterwards. + * transports only, and the repository's deploy key. GitHub, GitLab and + * Bitbucket are reached with their published host keys only; other hosts + * are trusted on first use and pinned in known_hosts afterwards. */ private env(repoUrl: string): Record { const ssh = [ @@ -191,6 +192,8 @@ export class GitMirrorService { "StrictHostKeyChecking=accept-new", "-o", `UserKnownHostsFile=${this.keys.knownHostsPath}`, + "-o", + `GlobalKnownHostsFile=${this.keys.publishedKnownHostsPath}`, ]; const key = this.keys.keyPathFor(repoUrl); if (key) { diff --git a/apps/api/src/modules/engine/job-logs.service.ts b/apps/api/src/modules/engine/job-logs.service.ts index 509114c..870c1df 100644 --- a/apps/api/src/modules/engine/job-logs.service.ts +++ b/apps/api/src/modules/engine/job-logs.service.ts @@ -28,7 +28,7 @@ export class JobLogsService { return; } const text = size + line.length > MAX_LOG_BYTES ? "[log truncated at 5 MiB]\n" : `${new Date().toISOString()} ${line}\n`; - fs.appendFileSync(this.storage.jobLogPath(jobId), text); + fs.appendFileSync(this.storage.jobLogPath(jobId), text, { mode: 0o600 }); this.sizes.set(jobId, size + Buffer.byteLength(text)); this.events.emit(jobId, text); } diff --git a/apps/api/src/modules/engine/log-archive.service.ts b/apps/api/src/modules/engine/log-archive.service.ts index fec8a35..0ee1607 100644 --- a/apps/api/src/modules/engine/log-archive.service.ts +++ b/apps/api/src/modules/engine/log-archive.service.ts @@ -40,7 +40,7 @@ export class LogArchiveService { continue; } const lines = await this.docker.logLines(container.Id, { tail: SCAN_LINES }).catch(() => [] as RawLogLine[]); - fs.writeFileSync(path.join(dir, `${service}${SUFFIX}`), gzipSync(lastBytes(lines, ARCHIVE_BYTES)), { mode: 0o640 }); + fs.writeFileSync(path.join(dir, `${service}${SUFFIX}`), gzipSync(lastBytes(lines, ARCHIVE_BYTES)), { mode: 0o600 }); archived.push(service); } return archived; diff --git a/apps/api/src/modules/engine/router.service.ts b/apps/api/src/modules/engine/router.service.ts index b68c9a0..9ececb5 100644 --- a/apps/api/src/modules/engine/router.service.ts +++ b/apps/api/src/modules/engine/router.service.ts @@ -45,6 +45,8 @@ export const WAKE_PATH = "/api/v1/wake"; /** Traefik applies a changed file within a couple of seconds. */ const ROUTE_WAIT_MS = 15_000; +/** Route files hold no secret; Traefik, root without capabilities, reads them as any other user would. */ +const ROUTES_MODE = 0o644; const ROUTE_POLL_MS = 250; /** Header of the answers of Spawner's waiting page. */ export const WAKE_HEADER = "x-spawner-wake"; @@ -114,6 +116,7 @@ export class RouterService implements OnModuleInit, OnApplicationBootstrap { }, }, }), + ROUTES_MODE, ); } @@ -198,7 +201,7 @@ export class RouterService implements OnModuleInit, OnApplicationBootstrap { routers[id] = { ...this.router(exposure.host, id), middlewares: exposure.auth === "none" ? PUBLIC_MIDDLEWARES : PREVIEW_MIDDLEWARES }; services[id] = { loadBalancer: { servers: [{ url: `http://${exposure.service}.${network}:${exposure.port}` }] } }; } - this.storage.writeAtomic(this.storage.traefikFile(environmentId), stringify({ http: { routers, services } })); + this.storage.writeAtomic(this.storage.traefikFile(environmentId), stringify({ http: { routers, services } }), ROUTES_MODE); } /** @@ -217,7 +220,7 @@ export class RouterService implements OnModuleInit, OnApplicationBootstrap { middlewares: exposure.auth === "none" ? [WAKE_MIDDLEWARE] : ["spawner-preview-auth", WAKE_MIDDLEWARE], }; } - this.storage.writeAtomic(this.storage.traefikFile(environmentId), stringify({ http: { routers } })); + this.storage.writeAtomic(this.storage.traefikFile(environmentId), stringify({ http: { routers } }), ROUTES_MODE); try { await this.docker.disconnectNetwork(`${composeProject}_default`, this.config.traefikContainer); } catch (error) { diff --git a/apps/api/src/modules/engine/storage.service.spec.ts b/apps/api/src/modules/engine/storage.service.spec.ts index 821df8a..c819ee4 100644 --- a/apps/api/src/modules/engine/storage.service.spec.ts +++ b/apps/api/src/modules/engine/storage.service.spec.ts @@ -63,3 +63,16 @@ describe("StorageService.removeTree", () => { await expect(new StorageService(config).removeTree(lockedSource())).rejects.toMatchObject({ code: expect.stringMatching(/EACCES|EPERM/) }); }); }); + +describe("StorageService.writeAtomic", () => { + it("writes files Spawner alone reads, unless a mode says otherwise", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "spawner-atomic-")); + const storage = new StorageService(config); + storage.writeAtomic(path.join(root, "compose.rendered.yaml"), "secret: x\n"); + storage.writeAtomic(path.join(root, "routes.yaml"), "http: {}\n", 0o644); + expect(fs.statSync(path.join(root, "compose.rendered.yaml")).mode & 0o777).toBe(0o600); + expect(fs.statSync(path.join(root, "routes.yaml")).mode & 0o777).toBe(0o644); + expect(fs.readdirSync(root).sort()).toEqual(["compose.rendered.yaml", "routes.yaml"]); + fs.rmSync(root, { recursive: true, force: true }); + }); +}); diff --git a/apps/api/src/modules/engine/storage.service.ts b/apps/api/src/modules/engine/storage.service.ts index 98d9ded..89f20d4 100644 --- a/apps/api/src/modules/engine/storage.service.ts +++ b/apps/api/src/modules/engine/storage.service.ts @@ -119,11 +119,12 @@ export class StorageService implements OnModuleInit { /** * Writes a file through a temporary file and a rename, so readers such as - * Traefik never see a half-written file. + * Traefik never see a half-written file; readable by Spawner only unless + * a mode says otherwise. */ - writeAtomic(target: string, content: string): void { + writeAtomic(target: string, content: string, mode = 0o600): void { const temporary = `${target}.${process.pid}.${Date.now()}.tmp`; - fs.writeFileSync(temporary, content, { mode: 0o640 }); + fs.writeFileSync(temporary, content, { mode }); fs.renameSync(temporary, target); } } diff --git a/docker-compose.yml b/docker-compose.yml index d976245..cbf6f70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,10 @@ # SPAWNER_DATA_DIR is mounted at the same path in the container, so the paths # Spawner writes in compose files are the same for the Docker daemon. # +# Ports listen on 127.0.0.1 only, unless SPAWNER_HTTP_BIND says otherwise +# (0.0.0.0 for the end-to-end test, whose environments call the previews +# through the host). +# # The firewall of install.sh's stack (firewall.nft) changes the host's rules # for every container of Docker's bridge networks: here it only starts with # --profile firewall, as the end-to-end test does. @@ -35,13 +39,16 @@ services: image: traefik:v3.5 container_name: spawner-traefik restart: unless-stopped + cap_drop: [ALL] + cap_add: [NET_BIND_SERVICE] + security_opt: [no-new-privileges:true] command: - "--providers.file.directory=/etc/traefik/dynamic" - "--providers.file.watch=true" - "--entrypoints.web.address=:80" - "--log.level=INFO" ports: - - "${SPAWNER_HTTP_PORT:-80}:80" + - "${SPAWNER_HTTP_BIND:-127.0.0.1}:${SPAWNER_HTTP_PORT:-80}:80" volumes: - ${SPAWNER_DATA_DIR:?set SPAWNER_DATA_DIR to an absolute path}/traefik:/etc/traefik/dynamic:ro networks: @@ -56,7 +63,7 @@ services: # A memory exhaustion of Spawner stops at its container, not at the host. mem_limit: ${SPAWNER_MEMORY_LIMIT:-1g} ports: - - "8080:3000" + - "${SPAWNER_HTTP_BIND:-127.0.0.1}:8080:3000" environment: - NODE_ENV=${NODE_ENV:-development} - PORT=3000 diff --git a/docs/configuration.md b/docs/configuration.md index ef70eb2..845d77a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -24,7 +24,7 @@ The installer sets some variables itself, in `/opt/spawner/compose.yaml` and `/o | `SPAWNER_SECRET` | generated into `/secret.key` | Master secret, from which the keys that sign tokens and sessions and encrypt settings derive (installer) | | `SESSION_SECRET` | derived from the master secret | Signs the session cookie | | `SESSION_MAX_AGE` | `86400000` (24 hours) | Dashboard sessions, in milliseconds | -| `SPAWNER_BOOTSTRAP_TOKEN` | none | Bearer token of the installation, with every scope and no user, for scripts and CI (installer) | +| `SPAWNER_BOOTSTRAP_TOKEN` | none | Bearer token of the installation, with every scope and no user, for scripts and CI; the installer sets it only when given one | | `GITHUB_CLIENT_ID`, `GITHUB_CLIENT_SECRET` | none | GitHub login, until it is configured from the settings page | | `GITHUB_ORG`, `GITHUB_TEAM` | none | Restrict the GitHub login to the members of an organization, and of a team | | `GITHUB_CALLBACK_URL` | `/api/v1/auth/github/callback` | The OAuth callback | diff --git a/docs/install.md b/docs/install.md index 4d1f332..3306ae2 100644 --- a/docs/install.md +++ b/docs/install.md @@ -79,6 +79,7 @@ curl -fsSL https://github.com/Flosk6/Spawner/releases/latest/download/install.sh | `--tls off` | Plain HTTP, for a local install (below) [`SPAWNER_TLS`] | | `--image ` | An image of your own instead of `ghcr.io/flosk6/spawner` [`SPAWNER_IMAGE`] | | `--yes` | Ask nothing; fail when an answer is missing | +| `--allow-downgrade` | Install a version older than the installed one, after restoring a backup of its database ([operations](operations.md#upgrading)) | ### DNS providers @@ -93,7 +94,7 @@ curl -fsSL https://github.com/Flosk6/Spawner/releases/latest/download/install.sh | Gandi | `GANDIV5_PERSONAL_ACCESS_TOKEN` | A personal access token allowed to manage the domain's technical configuration | | Others | as listed for the provider on [the lego documentation](https://go-acme.github.io/lego/dns/) | `--dns-provider --dns-env KEY=VALUE ...` | -The credentials go to `/opt/spawner/dns.env`, readable by root only, and only Traefik receives them. +The credentials go to `/opt/spawner/dns.env`, readable by root only, and only Traefik receives them. Typed answers to the questions for credentials are not shown. ## What the installer changes on the server @@ -107,7 +108,7 @@ The credentials go to `/opt/spawner/dns.env`, readable by root only, and only Tr - **zram**, if you accept it: `zram-tools`, half of the memory, zstd. - **nftables** (`nft`), if it is missing, and **`/etc/systemd/system/docker.service.d/spawner-firewall.conf`**: Docker loads the rules of `/opt/spawner/firewall.nft` before it starts any container, so that after a reboot no environment runs a moment without them ([security](security.md#isolation)). The rules sit in a table of their own (`inet spawner`), which Docker, ufw and firewalld leave alone. - **`/opt/spawner`**: `compose.yaml`, `.env` (settings and secrets), `dns.env`, `firewall.nft`, `backups/`; mode 700. -- **`/var/lib/spawner`**: the data directory (repository mirrors, environment sources, routes, job logs, deploy keys). +- **`/var/lib/spawner`**: the data directory (repository mirrors, environment sources, routes, job logs, deploy keys), mode 700. Nothing else: no Node.js on the host, no system upgrade, no cron. Spawner itself, Postgres and Traefik run as containers named `spawner`, `spawner-postgres` and `spawner-traefik`; `spawner-firewall` loads the rules of the environments again every minute. diff --git a/docs/operations.md b/docs/operations.md index 8fb3751..6493222 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -7,7 +7,7 @@ Running a Spawner server day to day: where things are, backups, restores, upgrad | Path | What it holds | |---|---| | `/opt/spawner/compose.yaml` | The stack: `spawner`, `spawner-postgres`, `spawner-traefik`, `spawner-firewall`. Written by the installer; rerun it rather than editing this file | -| `/opt/spawner/.env` | Version, domain, memory of the Spawner container (`SPAWNER_MEMORY_LIMIT`), and the secrets: `SPAWNER_SECRET`, `POSTGRES_PASSWORD`, `SPAWNER_BOOTSTRAP_TOKEN` | +| `/opt/spawner/.env` | Version, domain, memory of the Spawner container (`SPAWNER_MEMORY_LIMIT`), and the secrets: `SPAWNER_SECRET`, `POSTGRES_PASSWORD`, and `SPAWNER_BOOTSTRAP_TOKEN` when the installer was given one | | `/opt/spawner/dns.env` | Credentials of the DNS provider, for Traefik | | `/opt/spawner/spawner.env` | Settings of your own (below); the installer never overwrites it | | `/opt/spawner/firewall.nft` | The rules for the containers of Docker's bridge networks, which Docker loads before it starts ([security](security.md#isolation)); copied from the image by each run of the installer | @@ -84,7 +84,7 @@ The dashboard is away for about a minute, environments keep running, and the pag curl -fsSL https://github.com/Flosk6/Spawner/releases/latest/download/install.sh | sudo bash -s -- --upgrade ``` -`--version 2.1.0` picks a version. If the new version does not start, the installer goes back to the previous one by itself. To go back later, install the previous version with `--upgrade --version ` and restore the backup the upgrade took: migrations only go forward. +`--version 2.1.0` picks a version. If the new version does not start, the installer goes back to the previous one by itself. To go back later, restore the backup the upgrade took (it holds the database of the previous version), then install that version with `--upgrade --version --allow-downgrade`: migrations only go forward, so the installer refuses an older version without that option. ## Disk @@ -116,6 +116,6 @@ When the disk fills up anyway: delete the environments nobody uses any more, sho ## Secrets -- **The bootstrap token** (`SPAWNER_BOOTSTRAP_TOKEN` in `.env`) has every scope, for scripts of the installation: change it in `.env`, then `docker compose --project-directory /opt/spawner --env-file /opt/spawner/.env up -d`. +- **The bootstrap token** (`SPAWNER_BOOTSTRAP_TOKEN` in `.env`) has every scope and no user, for scripts of the installation. The installer makes one only when given (`sudo env SPAWNER_BOOTSTRAP_TOKEN=... bash install.sh`); installations from 2.1 and before have one: empty the line in `.env` to turn it off, or change it, then `docker compose --project-directory /opt/spawner --env-file /opt/spawner/.env up -d`. Personal tokens, with an expiry and a user, are better for scripts. - **The master secret** (`SPAWNER_SECRET`) signs sessions and tokens and encrypts settings and secret variables. Changing it logs everyone out, invalidates every personal token and makes the encrypted settings and secret variables unreadable: do it only if it leaked, then set the GitHub login and the secret variables again. - **The database password** (`POSTGRES_PASSWORD`) never leaves the server; the database is not reachable from outside the stack. diff --git a/docs/security.md b/docs/security.md index d961872..161f3c1 100644 --- a/docs/security.md +++ b/docs/security.md @@ -37,7 +37,7 @@ Service names, network aliases and hostnames are the names Docker's DNS answers - **Every container** gets `no-new-privileges`, no raw sockets (`NET_RAW` dropped: no forged packets on the environment's network), a memory limit (512 MiB per service by default, 2 GiB per environment), a CPU limit (1 per service by default, 4 per environment), a process limit (512 per service by default, 4096 per environment), a restart policy, and compressed logs with a cap. A `tmpfs` counts against the memory of its service, as the kernel charges it there. - **The Spawner container** has a memory limit (1 GiB by default, `install.sh --memory-limit`): if the API runs out of memory, it restarts without taking the server down. - **Uploads** are checked entry by entry before extraction: no absolute paths, no `..`, no links leaving the archive, no devices or hard links, and limits on size and file count. Once extracted, every link is followed through the links it leads to, as the kernel would: a chain of links that each stay inside cannot leave either. The check reads the archive as a stream and stops at the first limit crossed, the decompressed size included, so a compression bomb costs no more than a legitimate archive. A person has 5 deploys of uploaded code waiting to start at most, since each keeps its archives on disk until it runs. -- **Git** runs without a shell, with a minimal environment, no system configuration, `ssh` and `https` only, SSH in batch mode against a known_hosts file, the repository URL after `--`, and validated refs. Deploy keys are read-only and per repository. +- **Git** runs without a shell, with a minimal environment, no system configuration, `ssh` and `https` only, SSH in batch mode, the repository URL after `--`, and validated refs. GitHub, GitLab and Bitbucket are reached with the host keys they publish, shipped with Spawner, even the first time; other hosts are trusted on first use, then pinned. Deploy keys are read-only and per repository. - **Repositories are chosen by admins.** A branch names its other sources in `spawner.yaml`, but Spawner clones, or lists the branches of, only the project's repository and the source repositories an admin listed for it: a branch cannot point Spawner, and the global deploy key, at another repository or at an address of the internal network. - **External programs** (git, docker compose) run with argument arrays, never through a shell. - **Logs** of services are served as `text/plain` with `nosniff`. @@ -76,7 +76,10 @@ Previews still share the preview domain with each other and with the dashboard: ## Secrets -- **The master secret** (`SPAWNER_SECRET`, in `/opt/spawner/.env`, mode 600) derives the keys that sign tokens, encrypt settings and sign sessions. Keep a copy: without it, the encrypted settings and the secret project variables cannot be read back after a restore. +- **The master secret** (`SPAWNER_SECRET`, in `/opt/spawner/.env`, mode 600) derives the keys that sign tokens, encrypt settings and sign sessions. Keep a copy: without it, the encrypted settings and the secret project variables cannot be read back after a restore. The installer never makes a new one over an installation that lost it: it stops and says so. +- **The data directory** (`/var/lib/spawner`, mode 700) belongs to the user Spawner runs as, uid 1000, which is also the first user of most cloud images (`ubuntu`, `debian`): treat that account as an admin of Spawner. Rendered compose files (with the secret variables), job logs, archived logs and terminal recordings are readable by Spawner only; Traefik's route files hold no secret. +- **No token of the installation by default.** `SPAWNER_BOOTSTRAP_TOKEN` (every scope, no user, no expiry) exists only when given to the installer; scripts use personal tokens. +- **The stack**: Traefik runs without capabilities but binding ports, Spawner cannot change its own code (it belongs to root), Postgres and Traefik have memory limits, and Prisma's telemetry is off. - **Project variables** marked secret are encrypted at rest, never shown again, and masked in job logs. They still reach the containers as environment variables: anyone who may run commands in an environment can read them. Give previews test credentials only. - **DNS credentials** (`/opt/spawner/dns.env`, mode 600) go to Traefik alone. - **The Docker socket** is mounted into Spawner: whoever controls Spawner controls the host. That is why the API checks every request, compose files go through the policy, and the server must be dedicated to previews. diff --git a/install.sh b/install.sh index f747343..3f979ef 100755 --- a/install.sh +++ b/install.sh @@ -27,6 +27,8 @@ # (default: 1g) [SPAWNER_MEMORY_LIMIT] # --yes ask nothing, and fail when an answer is missing # --upgrade back the database up, then move to --version +# --allow-downgrade install a version older than the one installed (its +# database only migrates forward: restore its backup) # --uninstall stop Spawner and its environments; the data stays # --purge with --uninstall, delete the environments and the data too # @@ -35,7 +37,9 @@ # nothing else is installed on the host, apart from Docker itself, nftables # (the rules of firewall.nft, loaded before Docker by a drop-in of # docker.service) and zram if asked. spawner.env holds settings of your own, -# which the installer never overwrites. +# which the installer never overwrites. SPAWNER_BOOTSTRAP_TOKEN, when set, +# becomes a token of the installation with every scope, for scripts: none is +# made otherwise. set -euo pipefail @@ -73,6 +77,8 @@ MEMORY_LIMIT="${SPAWNER_MEMORY_LIMIT:-}" ASSUME_YES=false ACTION=install PURGE=false +ALLOW_DOWNGRADE=false +BOOTSTRAP_TOKEN="${SPAWNER_BOOTSTRAP_TOKEN:-}" DNS_VARS=() declare -A DNS_VALUES=() DOCKER_INSTALLED_NOW=false @@ -134,6 +140,18 @@ ask() { printf -v "$variable" '%s' "${answer:-$default}" } +# A secret, such as a token of the DNS provider: not shown while typed. +# ask_secret VARIABLE "Question" +ask_secret() { + local variable=$1 question=$2 answer="" + if [ "$ASSUME_YES" = true ] || [ ! -r /dev/tty ]; then + die "$question: give it with an option (see --help)" + fi + read -rs -p "$question (hidden): " answer /dev/null | grep -q "^$MARKER" || die "$INSTALL_DIR holds an installation this installer did not make (an older Spawner?): back it up and remove it first" - local key value - while IFS='=' read -r key value; do + local line key value previous_data_dir="" + while IFS= read -r line || [ -n "$line" ]; do + [[ "$line" == *=* && "$line" != \#* ]] || continue + key=${line%%=*} + value=${line#*=} case "$key" in - SPAWNER_DATA_DIR) DATA_DIR=${DATA_DIR:-$value} ;; + SPAWNER_DATA_DIR) previous_data_dir=$value ;; SPAWNER_PREVIEW_DOMAIN) DOMAIN=${DOMAIN:-$value} ;; ACME_EMAIL) EMAIL=${EMAIL:-$value} ;; SPAWNER_DNS_PROVIDER) DNS_PROVIDER=${DNS_PROVIDER:-$value} ;; @@ -217,10 +240,31 @@ load_previous() { SPAWNER_MEMORY_LIMIT) MEMORY_LIMIT=${MEMORY_LIMIT:-$value} ;; SPAWNER_SECRET) SECRET=$value ;; POSTGRES_PASSWORD) POSTGRES_PASSWORD=$value ;; - SPAWNER_BOOTSTRAP_TOKEN) BOOTSTRAP_TOKEN=$value ;; + SPAWNER_BOOTSTRAP_TOKEN) BOOTSTRAP_TOKEN=${BOOTSTRAP_TOKEN:-$value} ;; esac done <"$ENV_FILE" + INSTALLED=true + if [ -n "$previous_data_dir" ]; then + [ -z "$DATA_DIR" ] || [ "$DATA_DIR" = "$previous_data_dir" ] || + die "the data of this installation is in $previous_data_dir, not $DATA_DIR: moving it is not supported" + DATA_DIR=$previous_data_dir + fi DATA_DIR=${DATA_DIR:-/var/lib/spawner} + check_directory SPAWNER_DATA_DIR "$DATA_DIR" +} + +# Refuses a directory the installer would write to, or delete with --purge, +# that is not one of its own: relative, with "..", or a directory of the +# system. +check_directory() { + local name=$1 dir=$2 + [[ "$dir" == /* && "$dir" != *"/../"* && "$dir" != *"/.." && "$dir" != *"/./"* ]] || die "$name must be an absolute path without . or .., not \"$dir\"" + dir=${dir%/} + case "${dir:-/}" in + / | /bin | /boot | /dev | /etc | /home | /lib | /lib64 | /media | /mnt | /opt | /proc | /root | /run | /sbin | /srv | /sys | /tmp | /usr | /usr/* | /var | /var/lib | /var/log) + die "$name cannot be $dir, a directory of the system: use a directory of its own, such as /var/lib/spawner" + ;; + esac } # --- Checks ------------------------------------------------------------------- @@ -281,6 +325,7 @@ ask_questions() { DOMAIN=${DOMAIN:-localtest.me} DNS_PROVIDER=none ok "local install over HTTP, on $DOMAIN" + warn "plain HTTP on port 80 of every interface: whoever reaches this machine reaches the dashboard and the previews unencrypted. Keep it to a laptop or a private network." return fi if [ -z "$DOMAIN" ]; then @@ -333,7 +378,11 @@ ask_dns_credentials() { DNS_VALUES[${entry%%=*}]=${entry#*=} done if [ -f "$DNS_ENV_FILE" ]; then - while IFS='=' read -r name value; do + local line + while IFS= read -r line || [ -n "$line" ]; do + [[ "$line" == *=* ]] || continue + name=${line%%=*} + value=${line#*=} [ -n "$name" ] && [ -z "${DNS_VALUES[$name]:-}" ] && DNS_VALUES[$name]=$value done <"$DNS_ENV_FILE" fi @@ -342,17 +391,20 @@ ask_dns_credentials() { value=${!name:-} if [ -z "$value" ] && [ "$name" = OVH_ENDPOINT ]; then ask value "$name (ovh-eu, ovh-ca, ovh-us)" ovh-eu - elif [ -z "$value" ]; then + elif [ -z "$value" ] && [ "$name" = AWS_REGION ]; then ask value "$name" + elif [ -z "$value" ]; then + ask_secret value "$name" fi DNS_VALUES[$name]=$value done if [ ${#names[@]} -eq 0 ] && [ ${#DNS_VALUES[@]} -eq 0 ]; then [ "$ASSUME_YES" != true ] && [ -r /dev/tty ] || die "--dns-provider $DNS_PROVIDER needs its credentials: --dns-env KEY=VALUE" say "Give the variables of $DNS_PROVIDER listed on https://go-acme.github.io/lego/dns/$DNS_PROVIDER/, one KEY=VALUE per line, then an empty line:" - while read -r entry "$ENV_FILE" <"$ENV_FILE.new" <"$DNS_ENV_FILE" + mv "$ENV_FILE.new" "$ENV_FILE" + : >"$DNS_ENV_FILE.new" if [ "$wildcard" = true ]; then local name for name in "${!DNS_VALUES[@]}"; do - printf '%s=%s\n' "$name" "${DNS_VALUES[$name]}" >>"$DNS_ENV_FILE" + printf '%s=%s\n' "$name" "${DNS_VALUES[$name]}" >>"$DNS_ENV_FILE.new" done fi + mv "$DNS_ENV_FILE.new" "$DNS_ENV_FILE" if [ ! -f "$SETTINGS_FILE" ]; then cat >"$SETTINGS_FILE" <<'SETTINGS' # Settings of your own for Spawner, one VARIABLE=value per line: the installer @@ -573,6 +644,7 @@ services: image: $POSTGRES_IMAGE container_name: spawner-postgres restart: unless-stopped + mem_limit: 1g environment: POSTGRES_DB: spawner POSTGRES_USER: spawner @@ -590,6 +662,12 @@ services: image: $TRAEFIK_IMAGE container_name: spawner-traefik restart: unless-stopped + mem_limit: 512m + # Root in its container, for the certificates it already stored, but + # without the capabilities of root: it binds ports, nothing else. + cap_drop: [ALL] + cap_add: [NET_BIND_SERVICE] + security_opt: [no-new-privileges:true] command: - --providers.file.directory=/etc/traefik/dynamic - --providers.file.watch=true @@ -786,8 +864,20 @@ summary() { # --- Upgrade and removal ------------------------------------------------------ +# Saves the database before an upgrade. Postgres is started for it when it +# was stopped: an upgrade never goes on without a backup. backup_database() { - docker ps --format '{{.Names}}' | grep -qx spawner-postgres || return 0 + if ! docker ps --format '{{.Names}}' | grep -qx spawner-postgres; then + [ -f "$COMPOSE_FILE" ] || die "Postgres is not running and $COMPOSE_FILE is missing: the database cannot be backed up before the upgrade" + say "Starting Postgres to back the database up" + compose up -d postgres >/dev/null 2>&1 || die "Postgres does not start: the database cannot be backed up before the upgrade (docker logs spawner-postgres)" + local ready=false + for _ in $(seq 1 30); do + docker exec spawner-postgres pg_isready -U spawner -d spawner >/dev/null 2>&1 && { ready=true; break; } + sleep 2 + done + [ "$ready" = true ] || die "Postgres does not answer: the database cannot be backed up before the upgrade (docker logs spawner-postgres)" + fi local file file="$INSTALL_DIR/backups/spawner-$(date -u +%Y%m%d-%H%M%S)-${PREVIOUS_VERSION:-unknown}.sql.gz" docker exec spawner-postgres pg_dump -U spawner -d spawner | gzip >"$file" @@ -849,7 +939,7 @@ roll_back() { uninstall() { [ "$(id -u)" -eq 0 ] || die "run it as root" if [ "$PURGE" = true ]; then - say "This deletes Spawner, every environment it runs with their data, its database and $DATA_DIR." + say "This deletes Spawner, every environment it runs with their data, its database, $DATA_DIR, and $INSTALL_DIR with the backups of the database." if [ "$ASSUME_YES" != true ]; then local answer="" [ -r /dev/tty ] || die "confirm with --yes" @@ -891,6 +981,7 @@ uninstall() { main() { parse_args "$@" + check_directory SPAWNER_INSTALL_DIR "$INSTALL_DIR" case "$ACTION" in uninstall) load_previous diff --git a/scripts/e2e-engine.sh b/scripts/e2e-engine.sh index ebb3c5f..405ba3e 100755 --- a/scripts/e2e-engine.sh +++ b/scripts/e2e-engine.sh @@ -60,6 +60,8 @@ ROOT="$PWD" export SPAWNER_DATA_DIR="${SPAWNER_DATA_DIR:-$PWD/local-data/e2e}" export SPAWNER_HTTP_PORT="${SPAWNER_HTTP_PORT:-80}" +# Environments call the previews through the host, as on a server. +export SPAWNER_HTTP_BIND=0.0.0.0 export SPAWNER_BOOTSTRAP_TOKEN="${SPAWNER_BOOTSTRAP_TOKEN:-$(node -e "console.log(require('crypto').randomBytes(24).toString('hex'))")}" # The dashboard must be under the preview domain to hand out preview cookies, # whatever a local .env says. @@ -123,14 +125,20 @@ export SPAWNER_CONFIG_DIR="$WORK/cli-config" CLI_SERVER="http://spawner.localtest.me:${SPAWNER_HTTP_PORT}" spawner() { node "$WORK/spawner" "$@"; } +# in_data PATH: whether PATH exists in the data directory, which is +# Spawner's alone (0700): read through a container, as root. +in_data() { + docker run --rm -v "$SPAWNER_DATA_DIR:/data:ro" alpine:3.20 test -e "/data/$1" +} + # leftovers ENV_ID: what a deleted environment left behind, if anything. leftovers() { local found="" [ -z "$(docker ps -aq --filter "label=dev.spawner.env=$1")" ] || found+=" containers" [ -z "$(docker volume ls -q --filter "label=dev.spawner.env=$1")" ] || found+=" volumes" [ -z "$(docker network ls -q --filter "label=dev.spawner.env=$1")" ] || found+=" networks" - [ ! -e "$SPAWNER_DATA_DIR/traefik/$1.yaml" ] || found+=" routing" - [ ! -e "$SPAWNER_DATA_DIR/envs/$1" ] || found+=" sources" + ! in_data "traefik/$1.yaml" || found+=" routing" + ! in_data "envs/$1" || found+=" sources" echo "$found" } @@ -175,7 +183,7 @@ cleanup() { docker run --rm --network host --cap-add NET_ADMIN --entrypoint nft spawner-e2e-firewall delete table inet spawner >/dev/null 2>&1 || true # Each run builds Spawner's image again: removing it keeps old builds from piling up (the build cache stays). "${COMPOSE[@]}" down -v --remove-orphans --rmi local >/dev/null 2>&1 || true - docker run --rm -v "$SPAWNER_DATA_DIR:/data" alpine:3.20 sh -c 'rm -rf /data/*' >/dev/null 2>&1 || true + docker run --rm -v "$SPAWNER_DATA_DIR:/data" alpine:3.20 sh -c 'rm -rf /data/* && chmod 777 /data' >/dev/null 2>&1 || true rm -rf "$SPAWNER_DATA_DIR" rm -rf "$WORK" else @@ -438,7 +446,7 @@ done [ "$(api GET "/jobs/$foreign_job" | json 'v.status + " " + v.errorCode')" = "failed invalid" ] || fail "a source from an unlisted repository should be refused: $(api GET "/jobs/$foreign_job")" api GET "/jobs/$foreign_job/logs" | grep -q 'sources.other.repo: source "other" comes from https://github.com/acme/other.git, which is not among the source repositories' \ || fail "the job log should name the refused repository: $(api GET "/jobs/$foreign_job/logs")" -[ ! -e "$SPAWNER_DATA_DIR/envs/$foreign_id/src/other" ] || fail "nothing should be cloned for a refused source" +! in_data "envs/$foreign_id/src/other" || fail "nothing should be cloned for a refused source" wait_job "$(api DELETE "/envs/$foreign_id" | json 'v.job.id')" set +e foreign=$(cd "$FOREIGN" && spawner up --json 2>/dev/null) @@ -701,7 +709,7 @@ wait_status "$bind_id" deleted 150 || fail "an expired environment should be del [[ "$(api GET "/envs/$bind_id/events" | json 'v.events.map((e) => e.message).join("|")')" == *"Deletion started by Spawner (expired)"* ]] || fail "the timeline should say it expired" left=$(leftovers "$bind_id") [ -z "$left" ] || fail "the expired environment left behind:$left" -[ ! -e "$SPAWNER_DATA_DIR/envs/$bind_id" ] || fail "the files written by the container are left behind" +! in_data "envs/$bind_id" || fail "the files written by the container are left behind" pass "once expired, the environment disappeared with everything it held, files written as root included" foreign="e2e-foreign-$$" diff --git a/scripts/e2e-installer.sh b/scripts/e2e-installer.sh index aee061e..8d1c9fb 100755 --- a/scripts/e2e-installer.sh +++ b/scripts/e2e-installer.sh @@ -17,9 +17,10 @@ # 4. updates from the dashboard (the API of its Update button) to a newer # version, then to a version that does not start: Spawner goes back to # the version before by itself, with its database -# 5. upgrades with the installer: the database is backed up, the -# environment keeps running -# 6. runs it again without options: the secrets stay +# 5. upgrades with the installer: a version older than the one installed +# is refused; the database is backed up, the environment keeps running +# 6. runs it again without options: the secrets stay, and another data +# directory is refused # 7. removes everything with --uninstall --purge: nothing may be left, the # rules and Docker's drop-in included # @@ -80,7 +81,8 @@ INSTALLED="spawner-e2e:$VERSION" sudo docker tag "$IMAGE" "$INSTALLED" sudo mkdir -p /opt/spawner echo "SPAWNER_RELEASES_URL=file:///var/lib/spawner/e2e-releases.json" | sudo tee /opt/spawner/spawner.env >/dev/null -sudo bash install.sh --tls off --domain localtest.me --image "$INSTALLED" --yes --min-disk 5 | tee "$WORK/install.log" +# A token of the installation for the API calls of this test: the installer makes none by itself. +sudo env SPAWNER_BOOTSTRAP_TOKEN="$(openssl rand -hex 24)" bash install.sh --tls off --domain localtest.me --image "$INSTALLED" --yes --min-disk 5 2>&1 | tee "$WORK/install.log" grep -q "Spawner $VERSION is running" "$WORK/install.log" || fail "the installer should end with its summary, for Spawner $VERSION" [ "$(secret SPAWNER_VERSION)" = "$VERSION" ] || fail "the installation should be of Spawner $VERSION, not $(secret SPAWNER_VERSION)" grep -q "First admin http://spawner.localtest.me/invite/" "$WORK/install.log" || fail "the installer should print the first admin link" @@ -89,6 +91,11 @@ for file in .env dns.env spawner.env; do [ "$(sudo stat -c %a "/opt/spawner/$file")" = "600" ] || fail "$file should be readable by root only, not $(sudo stat -c %a "/opt/spawner/$file")" done sudo grep -q '"log-driver": "local"' /etc/docker/daemon.json || fail "Docker should keep compressed, capped logs" +grep -q "plain HTTP on port 80 of every interface" "$WORK/install.log" || fail "a local install should warn that it serves plain HTTP on every interface" +[ "$(sudo stat -c %a /var/lib/spawner)" = "700" ] || fail "the data directory should be Spawner's alone, not $(sudo stat -c %a /var/lib/spawner)" +[ "$(sudo docker inspect -f '{{json .HostConfig.CapDrop}} {{json .HostConfig.CapAdd}}' spawner-traefik)" = '["ALL"] ["CAP_NET_BIND_SERVICE"]' ] \ + || fail "Traefik should run without capabilities but binding ports: $(sudo docker inspect -f '{{json .HostConfig.CapDrop}} {{json .HostConfig.CapAdd}}' spawner-traefik)" +[ "$(sudo docker exec spawner stat -c %U /app/dist/main.js)" = "root" ] || fail "Spawner's code should belong to root, not to the user it runs as" SPAWNER_TOKEN=$(secret SPAWNER_BOOTSTRAP_TOKEN) export SPAWNER_URL="$DASHBOARD" SPAWNER_TOKEN SPAWNER_CONFIG_DIR="$WORK/cli-config" pass "install.sh set Spawner up: the dashboard answers on $DASHBOARD, the first admin link is printed" @@ -104,6 +111,8 @@ cd "$WORK/app" up=$(spawner up e2e-demo --wait --json) || fail "spawner up failed: $up" env_id=$(echo "$up" | json 'v.environment.id') [ "$(echo "$up" | json 'v.environment.status')" = "ready" ] || fail "the environment should be ready: $up" +[ "$(sudo stat -c %a "/var/lib/spawner/envs/$env_id/compose.rendered.yaml")" = "600" ] || fail "the rendered compose file, with the secrets, should be Spawner's alone" +[ "$(sudo stat -c %a "/var/lib/spawner/traefik/$env_id.yaml")" = "644" ] || fail "the routes should stay readable by Traefik" url=$(spawner url e2e-demo --with-token --json) page=$(curl -fsS -H "Host: $(echo "$url" | json 'new URL(v.url).host')" -H "X-Spawner-Preview: $(echo "$url" | json 'v.header.value')" http://127.0.0.1/) [[ "$page" == *"Hello from Spawner (e2e-demo)"* ]] || fail "the environment should answer through Traefik: $page" @@ -239,19 +248,28 @@ pass "an update to a version that does not start went back to $NEXT by itself, d step "Upgrading" secret_before=$(secret SPAWNER_SECRET) echo "SPAWNER_BUILD_CONCURRENCY=3" | sudo tee -a /opt/spawner/spawner.env >/dev/null -sudo bash install.sh --upgrade --image "$IMAGE" --yes | tee "$WORK/upgrade.log" +if sudo bash install.sh --upgrade --image "$IMAGE" --version "$VERSION" --yes >"$WORK/downgrade.log" 2>&1; then + fail "an upgrade to $VERSION, older than the installed $NEXT, should be refused" +fi +grep -q "$VERSION is older than the installed $NEXT" "$WORK/downgrade.log" || fail "the refusal should say why: $(tail -3 "$WORK/downgrade.log")" +[ "$(api GET /info | json 'v.version')" = "$NEXT" ] || fail "a refused downgrade should leave Spawner as it was" +sudo bash install.sh --upgrade --image "spawner-e2e:$NEXT" --version "$NEXT" --yes | tee "$WORK/upgrade.log" [ -n "$(sudo find /opt/spawner/backups -name 'spawner-*.sql.gz' -size +1k)" ] || fail "the upgrade should back the database up" [ "$(secret SPAWNER_SECRET)" = "$secret_before" ] || fail "the upgrade should keep the secrets" [ "$(api GET "/envs/$env_id" | json 'v.status')" = "ready" ] || fail "the environment should survive the upgrade" [ "$(sudo docker exec spawner printenv SPAWNER_BUILD_CONCURRENCY)" = "3" ] || fail "the settings of spawner.env should survive the upgrade and apply" -pass "the upgrade backed the database up and kept the secrets, spawner.env and the environment" +pass "a downgrade is refused; the upgrade backed the database up and kept the secrets, spawner.env and the environment" step "Running it again" +if sudo env SPAWNER_DATA_DIR=/srv/elsewhere bash install.sh --yes >"$WORK/elsewhere.log" 2>&1; then + fail "another data directory than the installation's should be refused" +fi +grep -q "the data of this installation is in /var/lib/spawner" "$WORK/elsewhere.log" || fail "the refusal should name the data directory: $(tail -3 "$WORK/elsewhere.log")" sudo bash install.sh --yes | tee "$WORK/again.log" [ "$(secret SPAWNER_SECRET)" = "$secret_before" ] || fail "running it again should keep the secrets" [ "$(secret SPAWNER_PREVIEW_DOMAIN)" = "localtest.me" ] || fail "running it again should keep the domain" [ "$(curl -fsS "$DASHBOARD/api/v1/healthz" | json 'v.status')" = "ok" ] || fail "the dashboard should answer after a second run" -pass "a second run keeps the domain, the secrets and the environment" +pass "a second run keeps the domain, the secrets and the environment, and refuses another data directory" step "Removing everything" sudo bash install.sh --uninstall --purge --yes