feat(daemon): mount /_sandbox/rpc and /assets; bounded tree-lock wait

packages/sandbox/README.md

@@ -233,6 +233,21 @@ Daemon control endpoints use the `/_sandbox/*` namespace, with `/health` at the
 root. AgentSandbox forwards those routes separately from the public preview
 contract.
 
+For v8 (Blocks) sites the daemon also serves the Deco content protocol over the
+working tree, so the site editor edits a sandbox the way it edits a local
+`deco serve`: `POST /_sandbox/rpc` (JSON-RPC: `describe`, `schema.get`,
+`blocks.list`, `blocks.apply`) and `PUT /_sandbox/assets/<name>` (uploads into
+`public/assets`). Both need the daemon token. Content lives in
+`<app root>/.deco/blocks/*.json` and is committed and pushed like code; there is
+no CDN draft in a sandbox. Commits take the worktree lock, so they serialize
+with fs writes, publish, discard and autosave. The `.deco/.blocks.lock` and
+`.deco/.tx-*/` files they use are listed in `.git/info/exclude`. The
+implementation is a Go port of `@decocms/blocks/protocol`, and
+`daemon-e2e/daemon.content-protocol.e2e.test.ts` runs the published conformance
+suite and a byte-for-byte parity check against it. Bump the pinned
+`@decocms/blocks` there when the protocol changes. `/_sandbox/decofile` (v7) is
+unchanged and reflects protocol writes.
+
 ## Export surface
 
 | Import | Purpose |

packages/sandbox/daemon-go/README.md

@@ -57,6 +57,7 @@ Two properties the consumer depends on, both asserted in `daemon-e2e/`:
 | `internal/auth/` | Bearer-token authentication |
 | `internal/telemetry/` | OTLP metrics export |
 | `internal/worktree/` | Worktree lock |
+| `internal/content/` | Content protocol (`/_sandbox/rpc`, `/_sandbox/assets/*`): Go port of `@decocms/blocks/protocol`, checked by the conformance and parity e2e |
 
 ## Startup contract
 

packages/sandbox/daemon-go/internal/routes/content.go

@@ -0,0 +1,125 @@
+package routes
+
+import (
+	"net/http"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/decocms/studio/sandbox-daemon/internal/config"
+	"github.com/decocms/studio/sandbox-daemon/internal/content"
+	"github.com/decocms/studio/sandbox-daemon/internal/gitx"
+	"github.com/decocms/studio/sandbox-daemon/internal/paths"
+	"github.com/decocms/studio/sandbox-daemon/internal/worktree"
+)
+
+// ContentDeps wires the content protocol (`POST /_sandbox/rpc`, `PUT
+// /_sandbox/assets/<name>`) to the working tree.
+type ContentDeps struct {
+	RepoDir string
+	Store   *config.Store
+	// TreeLock serializes commits and uploads with every other tree mutation
+	// (fs writes, publish, discard, rebase, autosave). Reads never take it.
+	TreeLock *worktree.Lock
+	// OnWrite gets the repo-relative path of every file a commit or upload
+	// touched: the same hook the fs routes call, so `file-changed`, the
+	// `decofile` version and the branch status follow protocol writes too.
+	OnWrite func(relPath string)
+	// ServerVersion is reported by describe. OPEN: no build stamps one yet.
+	ServerVersion string
+}
+
+// Content serves the content protocol for the app root the workload config
+// points at (the package path, like /_sandbox/decofile). One handler per app
+// root: it holds that root's hash and body caches.
+type Content struct {
+	deps ContentDeps
+
+	mu      sync.Mutex
+	root    string
+	handler *content.Handler
+}
+
+// treeLockWait bounds a protocol write's wait for the working-tree lock.
+var treeLockWait = 10 * time.Second
+
+func NewContent(deps ContentDeps) *Content {
+	return &Content{deps: deps}
+}
+
+func (c *Content) appRoot() string {
+	pmPath := ""
+	if cfg := c.deps.Store.Read(); cfg != nil {
+		pmPath = cfg.PmPath()
+	}
+	return paths.ResolvePmRoot(c.deps.RepoDir, pmPath)
+}
+
+// repoRel is the slash-separated path of target inside the repo.
+func (c *Content) repoRel(target string) string {
+	rel, err := filepath.Rel(c.deps.RepoDir, target)
+	if err != nil {
+		return target
+	}
+	return filepath.ToSlash(rel)
+}
+
+func (c *Content) current() *content.Handler {
+	root := c.appRoot()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.handler != nil && c.root == root {
+		return c.handler
+	}
+	// A commit's lock file and transaction folders must never reach a user
+	// branch through autosave or the shutdown `git add -A`.
+	decoRel := c.repoRel(filepath.Join(root, ".deco"))
+	gitx.EnsureExclude(c.deps.RepoDir, "/"+decoRel+"/.blocks.lock")
+	gitx.EnsureExclude(c.deps.RepoDir, "/"+decoRel+"/.tx-*/")
+
+	store := content.NewFSStore(content.FSOptions{
+		Root:          root,
+		RepoRoot:      c.deps.RepoDir,
+		ContainWithin: c.deps.RepoDir,
+		// Bounded well under Studio's 30 s proxy timeout: a write waiting on
+		// a long publish/rebase/autosave is refused (Unavailable, retry)
+		// instead of landing after the editor already reported it failed.
+		Exclusive: func() (func(), bool) {
+			if c.deps.TreeLock == nil {
+				return func() {}, true
+			}
+			return c.deps.TreeLock.AcquireWithin(treeLockWait)
+		},
+	})
+	blocksDir := filepath.Join(root, ".deco", "blocks")
+	assetsDir := filepath.Join(root, "public", "assets")
+	notify := func(path string) {
+		if c.deps.OnWrite != nil {
+			c.deps.OnWrite(c.repoRel(path))
+		}
+	}
+	c.root = root
+	c.handler = content.NewHandler(content.Options{
+		Store:         store,
+		ServerName:    "studio-sandbox-daemon",
+		ServerVersion: c.deps.ServerVersion,
+		OnCommit: func(files []string) {
+			for _, f := range files {
+				notify(filepath.Join(blocksDir, f))
+			}
+		},
+		OnAsset: func(name string) { notify(filepath.Join(assetsDir, name)) },
+	})
+	return c.handler
+}
+
+// RPC serves `/_sandbox/rpc` (any method: non-POST answers 405, as the
+// protocol requires).
+func (c *Content) RPC(w http.ResponseWriter, r *http.Request) {
+	c.current().ServeRPC(w, r)
+}
+
+// Assets serves `/_sandbox/assets/<name>`.
+func (c *Content) Assets(w http.ResponseWriter, r *http.Request) {
+	c.current().ServeAssets(w, r)
+}

packages/sandbox/daemon-go/internal/routes/content_test.go

@@ -0,0 +1,179 @@
+package routes
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/decocms/studio/sandbox-daemon/internal/config"
+	"github.com/decocms/studio/sandbox-daemon/internal/worktree"
+)
+
+func rpcPost(t *testing.T, c *Content, body string) *httptest.ResponseRecorder {
+	t.Helper()
+	r := httptest.NewRequest(http.MethodPost, "/_sandbox/rpc", strings.NewReader(body))
+	r.Header.Set("Content-Type", "application/json")
+	w := httptest.NewRecorder()
+	c.RPC(w, r)
+	return w
+}
+
+// A protocol write under the package path reports repo-relative paths through
+// the fs routes' hook (file-changed, the decofile version, branch status), and
+// keeps its lock and transaction files out of git.
+func TestContentFollowsThePackagePathAndReportsWrites(t *testing.T) {
+	repo := t.TempDir()
+	if out, err := exec.Command("git", "init", "-q", repo).CombinedOutput(); err != nil {
+		t.Skipf("git init: %v %s", err, out)
+	}
+	app := filepath.Join(repo, "apps", "web")
+	os.MkdirAll(filepath.Join(app, ".deco", "blocks"), 0o755)
+	store := config.NewStore()
+	store.Hydrate(&config.TenantConfig{
+		Application: &config.Application{
+			PackageManager: &config.PackageManagerConfig{Path: config.Str("apps/web")},
+		},
+	})
+	var written []string
+	c := NewContent(ContentDeps{
+		RepoDir:  repo,
+		Store:    store,
+		TreeLock: &worktree.Lock{},
+		OnWrite:  func(p string) { written = append(written, p) },
+	})
+
+	w := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"describe"}`)
+	if !strings.Contains(w.Body.String(), `"root":"apps/web"`) || !strings.Contains(w.Body.String(), `"dir":"apps/web/public/assets"`) {
+		t.Fatalf("describe: %s", w.Body)
+	}
+	w = rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"pages-Home Page":{"path":"/"},"Header":{}}}}`)
+	if strings.Contains(w.Body.String(), `"error"`) {
+		t.Fatalf("apply: %s", w.Body)
+	}
+	sort.Strings(written)
+	want := []string{"apps/web/.deco/blocks/Header.json", "apps/web/.deco/blocks/pages-Home%20Page.json"}
+	if !reflect.DeepEqual(written, want) {
+		t.Errorf("OnWrite: %v, want %v", written, want)
+	}
+
+	r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1, 2}))
+	r.Header.Set("Content-Type", "image/png")
+	aw := httptest.NewRecorder()
+	c.Assets(aw, r)
+	if aw.Code != 201 || written[len(written)-1] != "apps/web/public/assets/logo.png" {
+		t.Errorf("upload: %d %s %v", aw.Code, aw.Body, written)
+	}
+
+	exclude, _ := os.ReadFile(filepath.Join(repo, ".git", "info", "exclude"))
+	for _, line := range []string{"/apps/web/.deco/.blocks.lock", "/apps/web/.deco/.tx-*/"} {
+		if !strings.Contains(string(exclude), line+"\n") {
+			t.Errorf("exclude lacks %s:\n%s", line, exclude)
+		}
+	}
+}
+
+// Commits wait for the working-tree lock, so a publish or an fs write never
+// interleaves with one.
+func TestContentCommitsTakeTheTreeLock(t *testing.T) {
+	repo := t.TempDir()
+	os.MkdirAll(filepath.Join(repo, ".deco", "blocks"), 0o755)
+	lock := &worktree.Lock{}
+	c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: lock})
+
+	release := lock.Acquire()
+	done := make(chan string, 1)
+	go func() {
+		done <- rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`).Body.String()
+	}()
+	select {
+	case body := <-done:
+		t.Fatalf("committed while the tree was locked: %s", body)
+	case <-time.After(200 * time.Millisecond):
+	}
+	// Reads never wait.
+	if w := rpcPost(t, c, `{"jsonrpc":"2.0","id":2,"method":"blocks.list"}`); w.Code != 200 {
+		t.Errorf("list under the lock: %d", w.Code)
+	}
+	release()
+	if body := <-done; strings.Contains(body, `"error"`) {
+		t.Errorf("apply: %s", body)
+	}
+}
+
+// A write that can't get the tree lock in time is refused (Unavailable, retry
+// later) and never lands afterwards — the editor has already given up on it.
+func TestContentWritesGiveUpOnABusyTree(t *testing.T) {
+	prev := treeLockWait
+	treeLockWait = 100 * time.Millisecond
+	defer func() { treeLockWait = prev }()
+	repo := t.TempDir()
+	os.MkdirAll(filepath.Join(repo, ".deco", "blocks"), 0o755)
+	lock := &worktree.Lock{}
+	c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: lock})
+
+	release := lock.Acquire()
+	body := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`).Body.String()
+	if !strings.Contains(body, `"message":"the working tree is busy"`) || !strings.Contains(body, `"retryAfterMs":500`) {
+		t.Errorf("busy tree: %s", body)
+	}
+	r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1}))
+	r.Header.Set("Content-Type", "image/png")
+	aw := httptest.NewRecorder()
+	c.Assets(aw, r)
+	if aw.Code < 500 {
+		t.Errorf("upload on a busy tree: %d %s", aw.Code, aw.Body)
+	}
+	release()
+	for _, p := range []string{".deco/blocks/x.json", "public/assets/logo.png"} {
+		if _, err := os.Stat(filepath.Join(repo, p)); err == nil {
+			t.Errorf("%s landed after the request was refused", p)
+		}
+	}
+}
+
+// A storage folder symlinked outside the working tree is never read or
+// written through (stricter than the TS storage; see FSOptions.ContainWithin).
+func TestContentStaysInsideTheWorkingTree(t *testing.T) {
+	repo, outside := t.TempDir(), t.TempDir()
+	os.MkdirAll(filepath.Join(outside, "blocks"), 0o755)
+	os.WriteFile(filepath.Join(outside, "blocks", "secret.json"), []byte(`{}`), 0o644)
+	if err := os.Symlink(outside, filepath.Join(repo, ".deco")); err != nil {
+		t.Skipf("symlink: %v", err)
+	}
+	os.MkdirAll(filepath.Join(outside, "pub"), 0o755)
+	os.Symlink(filepath.Join(outside, "pub"), filepath.Join(repo, "public"))
+	c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: &worktree.Lock{}})
+
+	if w := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"describe"}`); strings.Contains(w.Body.String(), `"error"`) {
+		t.Errorf("describe: %s", w.Body)
+	}
+	for _, body := range []string{
+		`{"jsonrpc":"2.0","id":1,"method":"blocks.list"}`,
+		`{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`,
+	} {
+		if w := rpcPost(t, c, body); !strings.Contains(w.Body.String(), "resolves outside") {
+			t.Errorf("%s: %s", body, w.Body)
+		}
+	}
+	r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1}))
+	r.Header.Set("Content-Type", "image/png")
+	aw := httptest.NewRecorder()
+	c.Assets(aw, r)
+	if aw.Code < 400 {
+		t.Errorf("upload: %d %s", aw.Code, aw.Body)
+	}
+	if entries, _ := os.ReadDir(filepath.Join(outside, "pub")); len(entries) != 0 {
+		t.Errorf("wrote outside the tree: %v", entries)
+	}
+	if _, err := os.Stat(filepath.Join(outside, "blocks", "x.json")); err == nil {
+		t.Error("committed outside the tree")
+	}
+}

packages/sandbox/daemon-go/internal/worktree/lock.go

@@ -7,7 +7,10 @@
 // publish or discard that runs mid-write commits or destroys a half-written file.
 package worktree
 
-import "sync"
+import (
+	"sync"
+	"time"
+)
 
 // Lock guards the working tree. Held by the mutating fs routes and by every git
 // operation that reads or rewrites the whole checkout.
@@ -27,3 +30,19 @@ func (l *Lock) Acquire() func() {
 	l.mu.Lock()
 	return l.mu.Unlock
 }
+
+// AcquireWithin is Acquire bounded by d: ok is false (and nothing is held)
+// when the tree stayed busy that long. For callers whose client gives up
+// after a while, so a write never lands after its request was abandoned.
+func (l *Lock) AcquireWithin(d time.Duration) (release func(), ok bool) {
+	deadline := time.Now().Add(d)
+	for {
+		if l.mu.TryLock() {
+			return l.mu.Unlock, true
+		}
+		if !time.Now().Before(deadline) {
+			return nil, false
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+}