diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 668af4c53a..17d57d2210 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -233,6 +233,21 @@ Daemon control endpoints use the `/_sandbox/*` namespace, with `/health` at the root. AgentSandbox forwards those routes separately from the public preview contract. +For v8 (Blocks) sites the daemon also serves the Deco content protocol over the +working tree, so the site editor edits a sandbox the way it edits a local +`deco serve`: `POST /_sandbox/rpc` (JSON-RPC: `describe`, `schema.get`, +`blocks.list`, `blocks.apply`) and `PUT /_sandbox/assets/` (uploads into +`public/assets`). Both need the daemon token. Content lives in +`/.deco/blocks/*.json` and is committed and pushed like code; there is +no CDN draft in a sandbox. Commits take the worktree lock, so they serialize +with fs writes, publish, discard and autosave. The `.deco/.blocks.lock` and +`.deco/.tx-*/` files they use are listed in `.git/info/exclude`. The +implementation is a Go port of `@decocms/blocks/protocol`, and +`daemon-e2e/daemon.content-protocol.e2e.test.ts` runs the published conformance +suite and a byte-for-byte parity check against it. Bump the pinned +`@decocms/blocks` there when the protocol changes. `/_sandbox/decofile` (v7) is +unchanged and reflects protocol writes. + ## Export surface | Import | Purpose | diff --git a/packages/sandbox/daemon-go/README.md b/packages/sandbox/daemon-go/README.md index 1a65b13c5f..04194cab59 100644 --- a/packages/sandbox/daemon-go/README.md +++ b/packages/sandbox/daemon-go/README.md @@ -57,6 +57,7 @@ Two properties the consumer depends on, both asserted in `daemon-e2e/`: | `internal/auth/` | Bearer-token authentication | | `internal/telemetry/` | OTLP metrics export | | `internal/worktree/` | Worktree lock | +| `internal/content/` | Content protocol (`/_sandbox/rpc`, `/_sandbox/assets/*`): Go port of `@decocms/blocks/protocol`, checked by the conformance and parity e2e | ## Startup contract diff --git a/packages/sandbox/daemon-go/internal/routes/content.go b/packages/sandbox/daemon-go/internal/routes/content.go new file mode 100644 index 0000000000..a7a3034c78 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/routes/content.go @@ -0,0 +1,125 @@ +package routes + +import ( + "net/http" + "path/filepath" + "sync" + "time" + + "github.com/decocms/studio/sandbox-daemon/internal/config" + "github.com/decocms/studio/sandbox-daemon/internal/content" + "github.com/decocms/studio/sandbox-daemon/internal/gitx" + "github.com/decocms/studio/sandbox-daemon/internal/paths" + "github.com/decocms/studio/sandbox-daemon/internal/worktree" +) + +// ContentDeps wires the content protocol (`POST /_sandbox/rpc`, `PUT +// /_sandbox/assets/`) to the working tree. +type ContentDeps struct { + RepoDir string + Store *config.Store + // TreeLock serializes commits and uploads with every other tree mutation + // (fs writes, publish, discard, rebase, autosave). Reads never take it. + TreeLock *worktree.Lock + // OnWrite gets the repo-relative path of every file a commit or upload + // touched: the same hook the fs routes call, so `file-changed`, the + // `decofile` version and the branch status follow protocol writes too. + OnWrite func(relPath string) + // ServerVersion is reported by describe. OPEN: no build stamps one yet. + ServerVersion string +} + +// Content serves the content protocol for the app root the workload config +// points at (the package path, like /_sandbox/decofile). One handler per app +// root: it holds that root's hash and body caches. +type Content struct { + deps ContentDeps + + mu sync.Mutex + root string + handler *content.Handler +} + +// treeLockWait bounds a protocol write's wait for the working-tree lock. +var treeLockWait = 10 * time.Second + +func NewContent(deps ContentDeps) *Content { + return &Content{deps: deps} +} + +func (c *Content) appRoot() string { + pmPath := "" + if cfg := c.deps.Store.Read(); cfg != nil { + pmPath = cfg.PmPath() + } + return paths.ResolvePmRoot(c.deps.RepoDir, pmPath) +} + +// repoRel is the slash-separated path of target inside the repo. +func (c *Content) repoRel(target string) string { + rel, err := filepath.Rel(c.deps.RepoDir, target) + if err != nil { + return target + } + return filepath.ToSlash(rel) +} + +func (c *Content) current() *content.Handler { + root := c.appRoot() + c.mu.Lock() + defer c.mu.Unlock() + if c.handler != nil && c.root == root { + return c.handler + } + // A commit's lock file and transaction folders must never reach a user + // branch through autosave or the shutdown `git add -A`. + decoRel := c.repoRel(filepath.Join(root, ".deco")) + gitx.EnsureExclude(c.deps.RepoDir, "/"+decoRel+"/.blocks.lock") + gitx.EnsureExclude(c.deps.RepoDir, "/"+decoRel+"/.tx-*/") + + store := content.NewFSStore(content.FSOptions{ + Root: root, + RepoRoot: c.deps.RepoDir, + ContainWithin: c.deps.RepoDir, + // Bounded well under Studio's 30 s proxy timeout: a write waiting on + // a long publish/rebase/autosave is refused (Unavailable, retry) + // instead of landing after the editor already reported it failed. + Exclusive: func() (func(), bool) { + if c.deps.TreeLock == nil { + return func() {}, true + } + return c.deps.TreeLock.AcquireWithin(treeLockWait) + }, + }) + blocksDir := filepath.Join(root, ".deco", "blocks") + assetsDir := filepath.Join(root, "public", "assets") + notify := func(path string) { + if c.deps.OnWrite != nil { + c.deps.OnWrite(c.repoRel(path)) + } + } + c.root = root + c.handler = content.NewHandler(content.Options{ + Store: store, + ServerName: "studio-sandbox-daemon", + ServerVersion: c.deps.ServerVersion, + OnCommit: func(files []string) { + for _, f := range files { + notify(filepath.Join(blocksDir, f)) + } + }, + OnAsset: func(name string) { notify(filepath.Join(assetsDir, name)) }, + }) + return c.handler +} + +// RPC serves `/_sandbox/rpc` (any method: non-POST answers 405, as the +// protocol requires). +func (c *Content) RPC(w http.ResponseWriter, r *http.Request) { + c.current().ServeRPC(w, r) +} + +// Assets serves `/_sandbox/assets/`. +func (c *Content) Assets(w http.ResponseWriter, r *http.Request) { + c.current().ServeAssets(w, r) +} diff --git a/packages/sandbox/daemon-go/internal/routes/content_test.go b/packages/sandbox/daemon-go/internal/routes/content_test.go new file mode 100644 index 0000000000..b7716fc88b --- /dev/null +++ b/packages/sandbox/daemon-go/internal/routes/content_test.go @@ -0,0 +1,179 @@ +package routes + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/decocms/studio/sandbox-daemon/internal/config" + "github.com/decocms/studio/sandbox-daemon/internal/worktree" +) + +func rpcPost(t *testing.T, c *Content, body string) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(http.MethodPost, "/_sandbox/rpc", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + c.RPC(w, r) + return w +} + +// A protocol write under the package path reports repo-relative paths through +// the fs routes' hook (file-changed, the decofile version, branch status), and +// keeps its lock and transaction files out of git. +func TestContentFollowsThePackagePathAndReportsWrites(t *testing.T) { + repo := t.TempDir() + if out, err := exec.Command("git", "init", "-q", repo).CombinedOutput(); err != nil { + t.Skipf("git init: %v %s", err, out) + } + app := filepath.Join(repo, "apps", "web") + os.MkdirAll(filepath.Join(app, ".deco", "blocks"), 0o755) + store := config.NewStore() + store.Hydrate(&config.TenantConfig{ + Application: &config.Application{ + PackageManager: &config.PackageManagerConfig{Path: config.Str("apps/web")}, + }, + }) + var written []string + c := NewContent(ContentDeps{ + RepoDir: repo, + Store: store, + TreeLock: &worktree.Lock{}, + OnWrite: func(p string) { written = append(written, p) }, + }) + + w := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"describe"}`) + if !strings.Contains(w.Body.String(), `"root":"apps/web"`) || !strings.Contains(w.Body.String(), `"dir":"apps/web/public/assets"`) { + t.Fatalf("describe: %s", w.Body) + } + w = rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"pages-Home Page":{"path":"/"},"Header":{}}}}`) + if strings.Contains(w.Body.String(), `"error"`) { + t.Fatalf("apply: %s", w.Body) + } + sort.Strings(written) + want := []string{"apps/web/.deco/blocks/Header.json", "apps/web/.deco/blocks/pages-Home%20Page.json"} + if !reflect.DeepEqual(written, want) { + t.Errorf("OnWrite: %v, want %v", written, want) + } + + r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1, 2})) + r.Header.Set("Content-Type", "image/png") + aw := httptest.NewRecorder() + c.Assets(aw, r) + if aw.Code != 201 || written[len(written)-1] != "apps/web/public/assets/logo.png" { + t.Errorf("upload: %d %s %v", aw.Code, aw.Body, written) + } + + exclude, _ := os.ReadFile(filepath.Join(repo, ".git", "info", "exclude")) + for _, line := range []string{"/apps/web/.deco/.blocks.lock", "/apps/web/.deco/.tx-*/"} { + if !strings.Contains(string(exclude), line+"\n") { + t.Errorf("exclude lacks %s:\n%s", line, exclude) + } + } +} + +// Commits wait for the working-tree lock, so a publish or an fs write never +// interleaves with one. +func TestContentCommitsTakeTheTreeLock(t *testing.T) { + repo := t.TempDir() + os.MkdirAll(filepath.Join(repo, ".deco", "blocks"), 0o755) + lock := &worktree.Lock{} + c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: lock}) + + release := lock.Acquire() + done := make(chan string, 1) + go func() { + done <- rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`).Body.String() + }() + select { + case body := <-done: + t.Fatalf("committed while the tree was locked: %s", body) + case <-time.After(200 * time.Millisecond): + } + // Reads never wait. + if w := rpcPost(t, c, `{"jsonrpc":"2.0","id":2,"method":"blocks.list"}`); w.Code != 200 { + t.Errorf("list under the lock: %d", w.Code) + } + release() + if body := <-done; strings.Contains(body, `"error"`) { + t.Errorf("apply: %s", body) + } +} + +// A write that can't get the tree lock in time is refused (Unavailable, retry +// later) and never lands afterwards — the editor has already given up on it. +func TestContentWritesGiveUpOnABusyTree(t *testing.T) { + prev := treeLockWait + treeLockWait = 100 * time.Millisecond + defer func() { treeLockWait = prev }() + repo := t.TempDir() + os.MkdirAll(filepath.Join(repo, ".deco", "blocks"), 0o755) + lock := &worktree.Lock{} + c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: lock}) + + release := lock.Acquire() + body := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`).Body.String() + if !strings.Contains(body, `"message":"the working tree is busy"`) || !strings.Contains(body, `"retryAfterMs":500`) { + t.Errorf("busy tree: %s", body) + } + r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1})) + r.Header.Set("Content-Type", "image/png") + aw := httptest.NewRecorder() + c.Assets(aw, r) + if aw.Code < 500 { + t.Errorf("upload on a busy tree: %d %s", aw.Code, aw.Body) + } + release() + for _, p := range []string{".deco/blocks/x.json", "public/assets/logo.png"} { + if _, err := os.Stat(filepath.Join(repo, p)); err == nil { + t.Errorf("%s landed after the request was refused", p) + } + } +} + +// A storage folder symlinked outside the working tree is never read or +// written through (stricter than the TS storage; see FSOptions.ContainWithin). +func TestContentStaysInsideTheWorkingTree(t *testing.T) { + repo, outside := t.TempDir(), t.TempDir() + os.MkdirAll(filepath.Join(outside, "blocks"), 0o755) + os.WriteFile(filepath.Join(outside, "blocks", "secret.json"), []byte(`{}`), 0o644) + if err := os.Symlink(outside, filepath.Join(repo, ".deco")); err != nil { + t.Skipf("symlink: %v", err) + } + os.MkdirAll(filepath.Join(outside, "pub"), 0o755) + os.Symlink(filepath.Join(outside, "pub"), filepath.Join(repo, "public")) + c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: &worktree.Lock{}}) + + if w := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"describe"}`); strings.Contains(w.Body.String(), `"error"`) { + t.Errorf("describe: %s", w.Body) + } + for _, body := range []string{ + `{"jsonrpc":"2.0","id":1,"method":"blocks.list"}`, + `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`, + } { + if w := rpcPost(t, c, body); !strings.Contains(w.Body.String(), "resolves outside") { + t.Errorf("%s: %s", body, w.Body) + } + } + r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1})) + r.Header.Set("Content-Type", "image/png") + aw := httptest.NewRecorder() + c.Assets(aw, r) + if aw.Code < 400 { + t.Errorf("upload: %d %s", aw.Code, aw.Body) + } + if entries, _ := os.ReadDir(filepath.Join(outside, "pub")); len(entries) != 0 { + t.Errorf("wrote outside the tree: %v", entries) + } + if _, err := os.Stat(filepath.Join(outside, "blocks", "x.json")); err == nil { + t.Error("committed outside the tree") + } +} diff --git a/packages/sandbox/daemon-go/internal/worktree/lock.go b/packages/sandbox/daemon-go/internal/worktree/lock.go index 986eb980a4..2d912c57e1 100644 --- a/packages/sandbox/daemon-go/internal/worktree/lock.go +++ b/packages/sandbox/daemon-go/internal/worktree/lock.go @@ -7,7 +7,10 @@ // publish or discard that runs mid-write commits or destroys a half-written file. package worktree -import "sync" +import ( + "sync" + "time" +) // Lock guards the working tree. Held by the mutating fs routes and by every git // operation that reads or rewrites the whole checkout. @@ -27,3 +30,19 @@ func (l *Lock) Acquire() func() { l.mu.Lock() return l.mu.Unlock } + +// AcquireWithin is Acquire bounded by d: ok is false (and nothing is held) +// when the tree stayed busy that long. For callers whose client gives up +// after a while, so a write never lands after its request was abandoned. +func (l *Lock) AcquireWithin(d time.Duration) (release func(), ok bool) { + deadline := time.Now().Add(d) + for { + if l.mu.TryLock() { + return l.mu.Unlock, true + } + if !time.Now().Before(deadline) { + return nil, false + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/packages/sandbox/daemon-go/main.go b/packages/sandbox/daemon-go/main.go index d9393859d2..f4e50815fe 100644 --- a/packages/sandbox/daemon-go/main.go +++ b/packages/sandbox/daemon-go/main.go @@ -126,6 +126,7 @@ type sandboxHandlers struct { tasksList, tasksGet, tasksDelete http.HandlerFunc tasksKill, tasksKillAll, tasksStream http.HandlerFunc toolsSync, exec http.HandlerFunc + contentRPC, contentAssets http.HandlerFunc fs, git, setup map[string]http.HandlerFunc } @@ -624,6 +625,15 @@ func (d *daemon) registerSandboxRoutes(mux *http.ServeMux, pre string, h sandbox mux.HandleFunc("POST "+pre+"/tasks/{id}/kill", d.authed(h.tasksKill)) mux.HandleFunc("POST "+pre+"/tools/sync", d.authed(h.toolsSync)) + + // The content protocol over the working tree (v8 sites). Every method but + // OPTIONS (the CORS preflight) reaches the handler, which answers anything + // but POST (PUT for uploads) with the protocol's own 405. Commits and + // uploads take the tree lock themselves; reads never do. + for _, method := range []string{"GET", "POST", "PUT", "PATCH", "DELETE"} { + mux.HandleFunc(method+" "+pre+"/rpc", d.authed(h.contentRPC)) + mux.HandleFunc(method+" "+pre+"/assets/{name...}", d.authed(h.contentAssets)) + } for _, step := range []string{"clone", "install", "start"} { mux.HandleFunc("POST "+pre+"/setup/"+step, d.authed(h.setup[step])) } @@ -1225,6 +1235,13 @@ func main() { GetConfigured: func() bool { return d.store.Read() != nil }, }) + contentRoutes := routes.NewContent(routes.ContentDeps{ + RepoDir: repoDir, + Store: d.store, + TreeLock: &d.treeLock, + OnWrite: fsDeps.OnWorkingTreeWrite, + }) + h := sandboxHandlers{ scripts: routes.Scripts(func() []string { if cached, ok := d.orchestrator.DiscoveredScripts(); ok { @@ -1252,7 +1269,9 @@ func main() { GetDecofileVersion: d.getDecofileVersion, OnDecofileVersionUnknown: func() { go d.announceDecofileVersion() }, }), - decofile: routes.Decofile(d.decofileDeps), + decofile: routes.Decofile(d.decofileDeps), + contentRPC: contentRoutes.RPC, + contentAssets: contentRoutes.Assets, configRead: routes.ConfigRead(routes.ConfigDeps{ DaemonBootId: bootId, Store: d.store,