From e2df3f576b5250e60eda39a80ee23330a46a1afd Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:53:52 -0300 Subject: [PATCH 01/20] feat(daemon): JS-exact JSON for the content protocol Adds a JSON encoder/decoder whose number formatting and string escaping match JSON.stringify exactly, so content the daemon writes is byte-identical to what deco serve writes. The tests are the spec. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- packages/sandbox/daemon-go/go.mod | 2 +- .../daemon-go/internal/content/jsjson.go | 752 ++++++++++++++++++ .../daemon-go/internal/content/jsjson_test.go | 90 +++ 3 files changed, 843 insertions(+), 1 deletion(-) create mode 100644 packages/sandbox/daemon-go/internal/content/jsjson.go create mode 100644 packages/sandbox/daemon-go/internal/content/jsjson_test.go diff --git a/packages/sandbox/daemon-go/go.mod b/packages/sandbox/daemon-go/go.mod index 5fa67abd2e..10b5610b6f 100644 --- a/packages/sandbox/daemon-go/go.mod +++ b/packages/sandbox/daemon-go/go.mod @@ -10,6 +10,7 @@ require ( go.opentelemetry.io/otel/metric v1.44.0 go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 + golang.org/x/text v0.37.0 ) require ( @@ -24,7 +25,6 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/grpc v1.81.1 // indirect diff --git a/packages/sandbox/daemon-go/internal/content/jsjson.go b/packages/sandbox/daemon-go/internal/content/jsjson.go new file mode 100644 index 0000000000..d2a912170a --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/jsjson.go @@ -0,0 +1,752 @@ +package content + +// JavaScript's JSON, byte for byte. +// +// The reference server (`@decocms/blocks/protocol`, TypeScript) stores a block +// as `JSON.stringify(entry, null, 2) + "\n"` and answers with JSON.stringify'd +// bodies. `deco serve` on a laptop writes the same bytes, so a sandbox edit and +// a local edit of one entry must not differ by a single byte, or every save +// churns the git diff. encoding/json can't do that: it reorders nothing but +// loses JS property order (integer-like keys first), formats numbers its own +// way, escapes <, > and &, and has no room for the lone UTF-16 surrogates a +// JS string may hold. +// +// Values are: nil (null), bool, float64, string, []any and *Object. Strings are +// WTF-8: valid UTF-8, except that a lone surrogate from a `\udXXX` escape is +// kept as its 3-byte generalized encoding, so it round-trips the way it does +// in JS. + +import ( + "strconv" + "strings" + "unicode/utf8" +) + +// Object is a JS object: own string keys in property order. +type Object struct { + keys []string + vals map[string]any +} + +// NewObject builds an object from alternating keys and values. +func NewObject(kv ...any) *Object { + o := &Object{vals: map[string]any{}} + for i := 0; i+1 < len(kv); i += 2 { + o.Set(kv[i].(string), kv[i+1]) + } + return o +} + +// arrayIndex reports whether key is a canonical array index (0 to 2^32-2): +// such keys come first, in ascending order, in every JS property listing. +func arrayIndex(key string) (uint64, bool) { + if key == "" || len(key) > 10 || (len(key) > 1 && key[0] == '0') { + return 0, false + } + var n uint64 + for i := 0; i < len(key); i++ { + c := key[i] + if c < '0' || c > '9' { + return 0, false + } + n = n*10 + uint64(c-'0') + } + return n, n <= 4294967294 +} + +// Set defines key: an existing key keeps its position, like a JS assignment. +func (o *Object) Set(key string, value any) { + if o.vals == nil { + o.vals = map[string]any{} + } + if _, ok := o.vals[key]; ok { + o.vals[key] = value + return + } + o.vals[key] = value + if n, ok := arrayIndex(key); ok { + i := 0 + for ; i < len(o.keys); i++ { + m, isIndex := arrayIndex(o.keys[i]) + if !isIndex || m > n { + break + } + } + o.keys = append(o.keys, "") + copy(o.keys[i+1:], o.keys[i:]) + o.keys[i] = key + return + } + o.keys = append(o.keys, key) +} + +// Get returns an own property. +func (o *Object) Get(key string) (any, bool) { + if o == nil { + return nil, false + } + v, ok := o.vals[key] + return v, ok +} + +// Has reports an own property. +func (o *Object) Has(key string) bool { + _, ok := o.Get(key) + return ok +} + +// Keys returns the own keys in JS property order. +func (o *Object) Keys() []string { + if o == nil { + return nil + } + return o.keys +} + +// Len is the number of own keys. +func (o *Object) Len() int { + if o == nil { + return 0 + } + return len(o.keys) +} + +// ---------------------------------------------------------------- decoding + +// decodeUTF8 decodes bytes like the WHATWG TextDecoder: every maximal +// invalid subpart becomes one U+FFFD (fatal: an error instead), and a leading +// BOM is dropped when stripBOM is set (TextDecoder's default). +func decodeUTF8(b []byte, fatal, stripBOM bool) (string, bool) { + if stripBOM && len(b) >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF { + b = b[3:] + } + if utf8.Valid(b) { + return string(b), true + } + if fatal { + return "", false + } + var sb strings.Builder + sb.Grow(len(b)) + i := 0 + for i < len(b) { + c := b[i] + if c < 0x80 { + sb.WriteByte(c) + i++ + continue + } + need, lower, upper := 0, byte(0x80), byte(0xBF) + switch { + case c >= 0xC2 && c <= 0xDF: + need = 1 + case c >= 0xE0 && c <= 0xEF: + need = 2 + if c == 0xE0 { + lower = 0xA0 + } else if c == 0xED { + upper = 0x9F + } + case c >= 0xF0 && c <= 0xF4: + need = 3 + if c == 0xF0 { + lower = 0x90 + } else if c == 0xF4 { + upper = 0x8F + } + default: + sb.WriteRune(utf8.RuneError) + i++ + continue + } + j := i + 1 + ok := true + for k := 0; k < need; k++ { + if j >= len(b) || b[j] < lower || b[j] > upper { + ok = false + break + } + lower, upper = 0x80, 0xBF + j++ + } + if !ok { + sb.WriteRune(utf8.RuneError) + i = j + continue + } + sb.Write(b[i:j]) + i = j + } + return sb.String(), true +} + +// maxParseDepth bounds nesting so a hostile body can't exhaust the stack. +// OPEN: JS parses any depth and only fails later (stringify's call stack); a +// body nested deeper than this is a parse error here. +const maxParseDepth = 10000 + +type parser struct { + s string + i int + depth int +} + +type parseError struct{ msg string } + +func (e *parseError) Error() string { return e.msg } + +// ParseJSON is JSON.parse over text. +func ParseJSON(text string) (any, error) { + p := &parser{s: text} + p.ws() + v, err := p.value() + if err != nil { + return nil, err + } + p.ws() + if p.i != len(p.s) { + return nil, p.fail("Unexpected non-whitespace character after JSON") + } + return v, nil +} + +func (p *parser) fail(msg string) error { + return &parseError{msg: msg + " at position " + strconv.Itoa(p.i)} +} + +func (p *parser) ws() { + for p.i < len(p.s) { + switch p.s[p.i] { + case ' ', '\t', '\n', '\r': + p.i++ + default: + return + } + } +} + +func (p *parser) value() (any, error) { + if p.i >= len(p.s) { + return nil, p.fail("Unexpected end of JSON input") + } + switch c := p.s[p.i]; { + case c == '{': + return p.object() + case c == '[': + return p.array() + case c == '"': + return p.str() + case c == '-' || (c >= '0' && c <= '9'): + return p.number() + case strings.HasPrefix(p.s[p.i:], "true"): + p.i += 4 + return true, nil + case strings.HasPrefix(p.s[p.i:], "false"): + p.i += 5 + return false, nil + case strings.HasPrefix(p.s[p.i:], "null"): + p.i += 4 + return nil, nil + } + return nil, p.fail("Unexpected token") +} + +func (p *parser) enter() error { + p.depth++ + if p.depth > maxParseDepth { + return p.fail("JSON nested too deeply") + } + return nil +} + +func (p *parser) object() (any, error) { + if err := p.enter(); err != nil { + return nil, err + } + defer func() { p.depth-- }() + p.i++ // { + o := &Object{vals: map[string]any{}} + p.ws() + if p.i < len(p.s) && p.s[p.i] == '}' { + p.i++ + return o, nil + } + for { + p.ws() + if p.i >= len(p.s) || p.s[p.i] != '"' { + return nil, p.fail("Expected property name") + } + k, err := p.str() + if err != nil { + return nil, err + } + p.ws() + if p.i >= len(p.s) || p.s[p.i] != ':' { + return nil, p.fail("Expected ':' after property name") + } + p.i++ + p.ws() + v, err := p.value() + if err != nil { + return nil, err + } + o.Set(k.(string), v) + p.ws() + if p.i >= len(p.s) { + return nil, p.fail("Unexpected end of JSON input") + } + if p.s[p.i] == ',' { + p.i++ + continue + } + if p.s[p.i] == '}' { + p.i++ + return o, nil + } + return nil, p.fail("Expected ',' or '}' after property value") + } +} + +func (p *parser) array() (any, error) { + if err := p.enter(); err != nil { + return nil, err + } + defer func() { p.depth-- }() + p.i++ // [ + out := []any{} + p.ws() + if p.i < len(p.s) && p.s[p.i] == ']' { + p.i++ + return out, nil + } + for { + p.ws() + v, err := p.value() + if err != nil { + return nil, err + } + out = append(out, v) + p.ws() + if p.i >= len(p.s) { + return nil, p.fail("Unexpected end of JSON input") + } + if p.s[p.i] == ',' { + p.i++ + continue + } + if p.s[p.i] == ']' { + p.i++ + return out, nil + } + return nil, p.fail("Expected ',' or ']' after array element") + } +} + +func hexVal(c byte) int { + switch { + case c >= '0' && c <= '9': + return int(c - '0') + case c >= 'a' && c <= 'f': + return int(c-'a') + 10 + case c >= 'A' && c <= 'F': + return int(c-'A') + 10 + } + return -1 +} + +func (p *parser) hex4() (rune, bool) { + if p.i+4 > len(p.s) { + return 0, false + } + var r rune + for k := 0; k < 4; k++ { + h := hexVal(p.s[p.i+k]) + if h < 0 { + return 0, false + } + r = r<<4 | rune(h) + } + p.i += 4 + return r, true +} + +// appendWTF8 appends a code point, surrogates included (generalized UTF-8). +func appendWTF8(sb *strings.Builder, r rune) { + if r >= 0xD800 && r <= 0xDFFF { + sb.WriteByte(byte(0xE0 | (r >> 12))) + sb.WriteByte(byte(0x80 | ((r >> 6) & 0x3F))) + sb.WriteByte(byte(0x80 | (r & 0x3F))) + return + } + sb.WriteRune(r) +} + +func (p *parser) str() (any, error) { + p.i++ // " + var sb strings.Builder + start := p.i + for { + if p.i >= len(p.s) { + return nil, p.fail("Unterminated string in JSON") + } + c := p.s[p.i] + switch { + case c == '"': + sb.WriteString(p.s[start:p.i]) + p.i++ + return sb.String(), nil + case c < 0x20: + return nil, p.fail("Bad control character in string literal in JSON") + case c == '\\': + sb.WriteString(p.s[start:p.i]) + p.i++ + if p.i >= len(p.s) { + return nil, p.fail("Unterminated string in JSON") + } + e := p.s[p.i] + p.i++ + switch e { + case '"', '\\', '/': + sb.WriteByte(e) + case 'b': + sb.WriteByte('\b') + case 'f': + sb.WriteByte('\f') + case 'n': + sb.WriteByte('\n') + case 'r': + sb.WriteByte('\r') + case 't': + sb.WriteByte('\t') + case 'u': + r, ok := p.hex4() + if !ok { + return nil, p.fail("Bad Unicode escape in JSON") + } + // A high surrogate followed by an escaped low one is one code point. + if r >= 0xD800 && r <= 0xDBFF && strings.HasPrefix(p.s[p.i:], `\u`) { + save := p.i + p.i += 2 + if lo, ok := p.hex4(); ok && lo >= 0xDC00 && lo <= 0xDFFF { + r = 0x10000 + (r-0xD800)<<10 + (lo - 0xDC00) + } else { + p.i = save + } + } + appendWTF8(&sb, r) + default: + p.i-- + return nil, p.fail("Bad escaped character in JSON") + } + start = p.i + default: + p.i++ + } + } +} + +func isDigit(c byte) bool { return c >= '0' && c <= '9' } + +func (p *parser) number() (any, error) { + start := p.i + if p.s[p.i] == '-' { + p.i++ + } + if p.i >= len(p.s) { + return nil, p.fail("No number after minus sign in JSON") + } + if p.s[p.i] == '0' { + p.i++ + } else if isDigit(p.s[p.i]) { + for p.i < len(p.s) && isDigit(p.s[p.i]) { + p.i++ + } + } else { + return nil, p.fail("No number after minus sign in JSON") + } + if p.i < len(p.s) && p.s[p.i] == '.' { + p.i++ + if p.i >= len(p.s) || !isDigit(p.s[p.i]) { + return nil, p.fail("Unterminated fractional number in JSON") + } + for p.i < len(p.s) && isDigit(p.s[p.i]) { + p.i++ + } + } + if p.i < len(p.s) && (p.s[p.i] == 'e' || p.s[p.i] == 'E') { + p.i++ + if p.i < len(p.s) && (p.s[p.i] == '+' || p.s[p.i] == '-') { + p.i++ + } + if p.i >= len(p.s) || !isDigit(p.s[p.i]) { + return nil, p.fail("Exponent part is missing a number in JSON") + } + for p.i < len(p.s) && isDigit(p.s[p.i]) { + p.i++ + } + } + // Overflow is ±Inf, as in JS (stringified as null). + f, _ := strconv.ParseFloat(p.s[start:p.i], 64) + return f, nil +} + +// ---------------------------------------------------------------- encoding + +// FormatNumber is ECMAScript Number::toString(10). +func FormatNumber(f float64) string { + if f != f { + return "NaN" + } + if f == 0 { + return "0" + } + if f < 0 { + return "-" + FormatNumber(-f) + } + if f > 1.7976931348623157e308 { + return "Infinity" + } + e := strconv.FormatFloat(f, 'e', -1, 64) // d.ddde±XX + mant, expPart, _ := strings.Cut(e, "e") + digits := strings.Replace(mant, ".", "", 1) + exp, _ := strconv.Atoi(expPart) + k := len(digits) + n := exp + 1 + switch { + case k <= n && n <= 21: + return digits + strings.Repeat("0", n-k) + case 0 < n && n <= 21: + return digits[:n] + "." + digits[n:] + case -6 < n && n <= 0: + return "0." + strings.Repeat("0", -n) + digits + } + sign := "+" + if n-1 < 0 { + sign = "-" + } + abs := n - 1 + if abs < 0 { + abs = -abs + } + if k == 1 { + return digits + "e" + sign + strconv.Itoa(abs) + } + return digits[:1] + "." + digits[1:] + "e" + sign + strconv.Itoa(abs) +} + +// nextCodePoint decodes one WTF-8 code point: a lone surrogate's 3-byte +// encoding comes back as the surrogate itself. +func nextCodePoint(s string, i int) (rune, int) { + if i+2 < len(s) && s[i] == 0xED && s[i+1] >= 0xA0 && s[i+1] <= 0xBF { + return rune(0xD000) | rune(s[i+1]&0x3F)<<6 | rune(s[i+2]&0x3F), 3 + } + r, size := utf8.DecodeRuneInString(s[i:]) + return r, size +} + +const lowerHex = "0123456789abcdef" + +// QuoteJSON is JSON.stringify(string). +func QuoteJSON(sb *strings.Builder, s string) { + sb.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + c := s[i] + if c >= 0x20 && c != '"' && c != '\\' && c < 0x80 { + i++ + continue + } + if c < 0x80 { + sb.WriteString(s[start:i]) + switch c { + case '"': + sb.WriteString(`\"`) + case '\\': + sb.WriteString(`\\`) + case '\b': + sb.WriteString(`\b`) + case '\f': + sb.WriteString(`\f`) + case '\n': + sb.WriteString(`\n`) + case '\r': + sb.WriteString(`\r`) + case '\t': + sb.WriteString(`\t`) + default: + sb.WriteString(`\u00`) + sb.WriteByte(lowerHex[c>>4]) + sb.WriteByte(lowerHex[c&0xF]) + } + i++ + start = i + continue + } + r, size := nextCodePoint(s, i) + if r >= 0xD800 && r <= 0xDFFF { + sb.WriteString(s[start:i]) + sb.WriteString(`\u`) + sb.WriteByte(lowerHex[(r>>12)&0xF]) + sb.WriteByte(lowerHex[(r>>8)&0xF]) + sb.WriteByte(lowerHex[(r>>4)&0xF]) + sb.WriteByte(lowerHex[r&0xF]) + i += size + start = i + continue + } + i += size + } + sb.WriteString(s[start:]) + sb.WriteByte('"') +} + +// Stringify is JSON.stringify(v) (indent 0) or JSON.stringify(v, null, indent). +func Stringify(v any, indent int) string { + var sb strings.Builder + writeValue(&sb, v, indent, "") + return sb.String() +} + +func writeValue(sb *strings.Builder, v any, indent int, current string) { + switch x := v.(type) { + case nil: + sb.WriteString("null") + case bool: + if x { + sb.WriteString("true") + } else { + sb.WriteString("false") + } + case float64: + if x != x || x > 1.7976931348623157e308 || x < -1.7976931348623157e308 { + sb.WriteString("null") + } else { + sb.WriteString(FormatNumber(x)) + } + case int: + sb.WriteString(FormatNumber(float64(x))) + case string: + QuoteJSON(sb, x) + case []any: + if len(x) == 0 { + sb.WriteString("[]") + return + } + inner := current + strings.Repeat(" ", indent) + sb.WriteByte('[') + for i, item := range x { + if i > 0 { + sb.WriteByte(',') + } + if indent > 0 { + sb.WriteByte('\n') + sb.WriteString(inner) + } + writeValue(sb, item, indent, inner) + } + if indent > 0 { + sb.WriteByte('\n') + sb.WriteString(current) + } + sb.WriteByte(']') + case *Object: + if x.Len() == 0 { + sb.WriteString("{}") + return + } + inner := current + strings.Repeat(" ", indent) + sb.WriteByte('{') + for i, k := range x.keys { + if i > 0 { + sb.WriteByte(',') + } + if indent > 0 { + sb.WriteByte('\n') + sb.WriteString(inner) + } + QuoteJSON(sb, k) + sb.WriteByte(':') + if indent > 0 { + sb.WriteByte(' ') + } + writeValue(sb, x.vals[k], indent, inner) + } + if indent > 0 { + sb.WriteByte('\n') + sb.WriteString(current) + } + sb.WriteByte('}') + case rawJSON: + sb.WriteString(string(x)) + default: + panic("content: unsupported JSON value") + } +} + +// rawJSON is an already-serialized value, spliced in verbatim. +type rawJSON string + +// ---------------------------------------------------------------- strings + +// utf16Units returns s as UTF-16 code units (JS string semantics). +func utf16Units(s string) []uint16 { + out := make([]uint16, 0, len(s)) + for i := 0; i < len(s); { + r, size := nextCodePoint(s, i) + i += size + if r >= 0x10000 { + r -= 0x10000 + out = append(out, uint16(0xD800+(r>>10)), uint16(0xDC00+(r&0x3FF))) + } else { + out = append(out, uint16(r)) + } + } + return out +} + +// jsLength is a JS string's .length. +func jsLength(s string) int { + n := 0 + for i := 0; i < len(s); { + r, size := nextCodePoint(s, i) + i += size + if r >= 0x10000 { + n += 2 + } else { + n++ + } + } + return n +} + +// compareJS orders two strings like JS's `<`: by UTF-16 code units. +func compareJS(a, b string) int { + // Byte order equals code-unit order unless a supplementary code point + // meets U+E000..U+FFFF; compare in UTF-16 only when it might matter. + ascii := true + for i := 0; i < len(a) && ascii; i++ { + ascii = a[i] < 0x80 + } + for i := 0; i < len(b) && ascii; i++ { + ascii = b[i] < 0x80 + } + if ascii { + return strings.Compare(a, b) + } + x, y := utf16Units(a), utf16Units(b) + for i := 0; i < len(x) && i < len(y); i++ { + if x[i] != y[i] { + if x[i] < y[i] { + return -1 + } + return 1 + } + } + switch { + case len(x) < len(y): + return -1 + case len(x) > len(y): + return 1 + } + return 0 +} diff --git a/packages/sandbox/daemon-go/internal/content/jsjson_test.go b/packages/sandbox/daemon-go/internal/content/jsjson_test.go new file mode 100644 index 0000000000..42070e3635 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/jsjson_test.go @@ -0,0 +1,90 @@ +package content + +import ( + "math" + "testing" +) + +// Expected strings are what V8's JSON.stringify / Number#toString print. +func TestFormatNumberMatchesJS(t *testing.T) { + cases := map[float64]string{ + 0: "0", math.Copysign(0, -1): "0", 42: "42", -7: "-7", 0.1: "0.1", + 1e21: "1e+21", 1e20: "100000000000000000000", 1e-7: "1e-7", 1e-6: "0.000001", + 123456789012345680000: "123456789012345680000", 9007199254740991: "9007199254740991", + 5e-324: "5e-324", 1.7976931348623157e308: "1.7976931348623157e+308", + 1.5e-7: "1.5e-7", 0.000123: "0.000123", 1.25e21: "1.25e+21", 100: "100", 0.5: "0.5", + } + for f, want := range cases { + if got := FormatNumber(f); got != want { + t.Errorf("FormatNumber(%v) = %q, want %q", f, got, want) + } + } +} + +func TestStringifyMatchesJSONStringify(t *testing.T) { + cases := []struct{ in, compact, pretty string }{ + {`{"b":1,"a":[true,null],"10":"x","2":{}}`, `{"2":{},"10":"x","b":1,"a":[true,null]}`, + "{\n \"2\": {},\n \"10\": \"x\",\n \"b\": 1,\n \"a\": [\n true,\n null\n ]\n}"}, + {`[]`, `[]`, `[]`}, + {`{"s":"\u0000\u001f\n\t\"\\\/<>&\u2028\u00e9"}`, "{\"s\":\"\\u0000\\u001f\\n\\t\\\"\\\\/<>&\u2028é\"}", ""}, + {`{"lone":"\ud800x\udfff","pair":"\ud83d\ude00"}`, "{\"lone\":\"\\ud800x\\udfff\",\"pair\":\"😀\"}", ""}, + {`{"n":[1E2,-0,1e400,0.1e-6]}`, `{"n":[100,0,null,1e-7]}`, ""}, + {`{"a":1,"a":2,"b":3}`, `{"a":2,"b":3}`, ""}, + {`{"4294967294":1,"4294967295":2,"01":3,"0":4}`, `{"0":4,"4294967294":1,"4294967295":2,"01":3}`, ""}, + } + for _, c := range cases { + v, err := ParseJSON(c.in) + if err != nil { + t.Fatalf("ParseJSON(%s): %v", c.in, err) + } + if got := Stringify(v, 0); got != c.compact { + t.Errorf("Stringify(%s) = %s, want %s", c.in, got, c.compact) + } + if c.pretty != "" { + if got := Stringify(v, 2); got != c.pretty { + t.Errorf("Stringify(%s, 2) = %q, want %q", c.in, got, c.pretty) + } + } + } +} + +func TestParseJSONRejectsWhatJSONParseRejects(t *testing.T) { + for _, in := range []string{ + "", "{", "{not json", "[1,]", `{"a":1,}`, "01", "1.", ".5", "+1", "NaN", "'x'", + "\"\t\"", `"\x"`, `"\u12"`, "[1] 2", "tru", "\ufeff{}", "{\"a\" 1}", + } { + if _, err := ParseJSON(in); err == nil { + t.Errorf("ParseJSON(%q) succeeded", in) + } + } + for _, in := range []string{" {} ", "\t\n\r[ ]", "-0", "1e5", "\"\\u00e9\""} { + if _, err := ParseJSON(in); err != nil { + t.Errorf("ParseJSON(%q): %v", in, err) + } + } +} + +func TestDecodeUTF8LikeTextDecoder(t *testing.T) { + // One U+FFFD per maximal invalid subpart, a BOM stripped. + got, _ := decodeUTF8([]byte{0xEF, 0xBB, 0xBF, 'a', 0xE2, 0x82, 'b', 0xED, 0xA0, 0x80, 0xFF}, false, true) + if want := "a\uFFFDb\uFFFD\uFFFD\uFFFD\uFFFD"; got != want { + t.Errorf("decodeUTF8 = %q, want %q", got, want) + } + if _, ok := decodeUTF8([]byte{0xC3}, true, true); ok { + t.Error("fatal decode accepted invalid UTF-8") + } + kept, _ := decodeUTF8([]byte{0xEF, 0xBB, 0xBF, 'x'}, false, false) + if kept != "\uFEFFx" { + t.Errorf("BOM not kept: %q", kept) + } +} + +func TestCompareJSUsesUTF16Order(t *testing.T) { + // In UTF-8 byte order U+1F600 sorts after U+FF5E; in UTF-16 (JS) it sorts before. + if compareJS("\U0001F600", "\uFF5E") >= 0 { + t.Error("a surrogate pair must sort before U+FF5E, as in JS") + } + if compareJS("a", "b") >= 0 || compareJS("b", "a") <= 0 || compareJS("a", "a") != 0 || compareJS("a", "ab") >= 0 { + t.Error("ASCII order") + } +} From 248278346dfd2a5fada9c71ea690c303278138fe Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:54:38 -0300 Subject: [PATCH 02/20] feat(daemon): block keys/file names and protocol errors Maps block keys to file names with encodeURIComponent parity and resolves the spellings a key may have on disk; adds the protocol's error codes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../daemon-go/internal/content/errors.go | 120 ++++++ .../daemon-go/internal/content/keys.go | 365 ++++++++++++++++++ .../daemon-go/internal/content/keys_test.go | 186 +++++++++ 3 files changed, 671 insertions(+) create mode 100644 packages/sandbox/daemon-go/internal/content/errors.go create mode 100644 packages/sandbox/daemon-go/internal/content/keys.go create mode 100644 packages/sandbox/daemon-go/internal/content/keys_test.go diff --git a/packages/sandbox/daemon-go/internal/content/errors.go b/packages/sandbox/daemon-go/internal/content/errors.go new file mode 100644 index 0000000000..36c1777da8 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/errors.go @@ -0,0 +1,120 @@ +package content + +import "strconv" + +// Error codes: JSON-RPC 2.0's, then the content protocol's. Ported from +// `@decocms/blocks/protocol` errors.ts. +const ( + CodeParseError = -32700 + CodeInvalidRequest = -32600 + CodeMethodNotFound = -32601 + CodeInvalidParams = -32602 + CodeInternalError = -32603 + CodeNotFound = -32001 + CodeConflict = -32002 + CodeInvalidBlock = -32003 + CodeReadOnly = -32005 + CodeUnsupported = -32006 + CodeLimitExceeded = -32007 + CodeUnavailable = -32008 +) + +// ProtocolError is a method failure: the JSON-RPC `error` object. +type ProtocolError struct { + Code int + Message string + Data any // nil: no `data` member +} + +func (e *ProtocolError) Error() string { return e.Message } + +// JSON is the error object, `{code, message, data?}`. +func (e *ProtocolError) JSON() *Object { + o := NewObject("code", float64(e.Code), "message", e.Message) + if e.Data != nil { + o.Set("data", e.Data) + } + return o +} + +// Violation is one rule a blocks.apply breaks. +type Violation struct { + Name string + Pointer *string // set for a violation about one value + Rule string + Message string +} + +func (v Violation) json() *Object { + o := NewObject("name", v.Name) + if v.Pointer != nil { + o.Set("pointer", *v.Pointer) + } + o.Set("rule", v.Rule) + o.Set("message", v.Message) + return o +} + +func errNotFound(message string) *ProtocolError { + return &ProtocolError{Code: CodeNotFound, Message: message} +} + +func errConflict(entries *Object) *ProtocolError { + return &ProtocolError{Code: CodeConflict, Message: "a precondition failed", Data: NewObject("entries", entries)} +} + +func errInvalidBlock(violations []Violation) *ProtocolError { + message := strconv.Itoa(len(violations)) + " invalid blocks" + if len(violations) == 1 { + message = `invalid block "` + violations[0].Name + `": ` + violations[0].Message + } + list := make([]any, len(violations)) + for i, v := range violations { + list[i] = v.json() + } + return &ProtocolError{Code: CodeInvalidBlock, Message: message, Data: NewObject("violations", list)} +} + +func errReadOnly() *ProtocolError { + return &ProtocolError{Code: CodeReadOnly, Message: "this endpoint is read-only"} +} + +func errUnsupported(message string) *ProtocolError { + return &ProtocolError{Code: CodeUnsupported, Message: message} +} + +func errLimitExceeded(message string, data *Object) *ProtocolError { + e := &ProtocolError{Code: CodeLimitExceeded, Message: message} + if data != nil { + e.Data = data + } + return e +} + +func errUnavailable(message string, retryAfterMs int) *ProtocolError { + e := &ProtocolError{Code: CodeUnavailable, Message: message} + if retryAfterMs >= 0 { + e.Data = NewObject("retryAfterMs", float64(retryAfterMs)) + } + return e +} + +func errInvalidParams(message string) *ProtocolError { + return &ProtocolError{Code: CodeInvalidParams, Message: message} +} + +func errInvalidRequest(message string) *ProtocolError { + return &ProtocolError{Code: CodeInvalidRequest, Message: message} +} + +func errMethodNotFound(method string) *ProtocolError { + return &ProtocolError{Code: CodeMethodNotFound, Message: `unknown method "` + method + `"`} +} + +func errParse() *ProtocolError { + return &ProtocolError{Code: CodeParseError, Message: "invalid JSON"} +} + +func errInternal() *ProtocolError { + return &ProtocolError{Code: CodeInternalError, Message: "internal error"} +} diff --git a/packages/sandbox/daemon-go/internal/content/keys.go b/packages/sandbox/daemon-go/internal/content/keys.go new file mode 100644 index 0000000000..5a5cdcd260 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/keys.go @@ -0,0 +1,365 @@ +package content + +// The one file-name rule for saved blocks, ported from +// `@decocms/blocks/protocol/keys` (keys.ts): +// +// - name to file: encodeURIComponent(name) + ".json", directly in .deco/blocks; +// - file to name: the stem decoded exactly once (the raw stem when that fails); +// - spellings of one name (files that decode to the same name after repeated +// decoding): the file whose entry has a `path` wins, then the one that took +// more decoding, then the lowest file name; +// - file content: JSON.stringify(entry, null, 2) + "\n". + +import ( + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +const ( + blockFileExtension = ".json" + // MaxEncodedNameBytes keeps `.json` under a 255-byte file name. + MaxEncodedNameBytes = 250 +) + +var windowsDeviceNames = func() map[string]bool { + m := map[string]bool{"CON": true, "PRN": true, "AUX": true, "NUL": true} + for i := 1; i <= 9; i++ { + m["COM"+string(rune('0'+i))] = true + m["LPT"+string(rune('0'+i))] = true + } + return m +}() + +var sourceExtensions = []string{".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"} + +// uriError is JS's URIError: thrown (panicked) by encodeURIComponent on a lone +// surrogate. Like any unexpected throw in the TS server, it surfaces as an +// Internal error (-32603) for the call. +type uriError struct{} + +func isURIUnreserved(c byte) bool { + return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || + strings.IndexByte("-_.!~*'()", c) >= 0 +} + +const upperHex = "0123456789ABCDEF" + +// encodeURIComponent is JS's encodeURIComponent; it panics with uriError on a +// lone surrogate, as JS throws. +func encodeURIComponent(s string) string { + var sb strings.Builder + for i := 0; i < len(s); { + c := s[i] + if c < 0x80 { + if isURIUnreserved(c) { + sb.WriteByte(c) + } else { + sb.WriteByte('%') + sb.WriteByte(upperHex[c>>4]) + sb.WriteByte(upperHex[c&0xF]) + } + i++ + continue + } + r, size := nextCodePoint(s, i) + if (r >= 0xD800 && r <= 0xDFFF) || (r == utf8.RuneError && size == 1) { + panic(uriError{}) + } + for k := 0; k < size; k++ { + b := s[i+k] + sb.WriteByte('%') + sb.WriteByte(upperHex[b>>4]) + sb.WriteByte(upperHex[b&0xF]) + } + i += size + } + return sb.String() +} + +// decodeURIComponent is JS's decodeURIComponent; ok is false where JS throws. +func decodeURIComponent(s string) (string, bool) { + if strings.IndexByte(s, '%') < 0 { + return s, true + } + var sb strings.Builder + octet := func(i int) (byte, bool) { + if i+2 >= len(s) || s[i] != '%' { + return 0, false + } + h, l := hexVal(s[i+1]), hexVal(s[i+2]) + if h < 0 || l < 0 { + return 0, false + } + return byte(h<<4 | l), true + } + for i := 0; i < len(s); { + if s[i] != '%' { + sb.WriteByte(s[i]) + i++ + continue + } + b, ok := octet(i) + if !ok { + return "", false + } + i += 3 + if b < 0x80 { + sb.WriteByte(b) + continue + } + n := 0 + for mask := byte(0x80); b&mask != 0 && n < 8; mask >>= 1 { + n++ + } + if n == 1 || n > 4 { + return "", false + } + octets := []byte{b} + for k := 1; k < n; k++ { + c, ok := octet(i) + if !ok || c&0xC0 != 0x80 { + return "", false + } + octets = append(octets, c) + i += 3 + } + r, size := utf8.DecodeRune(octets) + if size != n || (r == utf8.RuneError && size == 1) { + return "", false + } + sb.Write(octets) + } + return sb.String(), true +} + +// BlockFileName is the file a saved block named name is stored in. +func BlockFileName(name string) string { + return encodeURIComponent(name) + blockFileExtension +} + +// IsBlockFileName reports a saved-block file name: `.json`, a stem, no folder, +// not a dotfile. +func IsBlockFileName(file string) bool { + return strings.HasSuffix(file, blockFileExtension) && + !strings.HasPrefix(file, ".") && + len(file) > len(blockFileExtension) && + !strings.Contains(file, "/") && + !strings.Contains(file, `\`) +} + +func stem(file string) string { + return strings.TrimSuffix(file, blockFileExtension) +} + +// BlockNameFromFile is the entry name stored in file: the stem decoded once. +func BlockNameFromFile(file string) string { + raw := stem(file) + if name, ok := decodeURIComponent(raw); ok { + return name + } + return raw +} + +// FullyDecodeFileName decodes file's stem until it stops changing: the +// spelling-group key, and how many decodes it took. +func FullyDecodeFileName(file string) (string, int) { + name := stem(file) + passes := 0 + for strings.Contains(name, "%") { + next, ok := decodeURIComponent(name) + if !ok || next == name { + break + } + name = next + passes++ + } + return name, passes +} + +// SpellingKey is the spelling-group key of a saved-block name. +func SpellingKey(name string) string { + key, _ := FullyDecodeFileName(BlockFileName(name)) + return key +} + +// SerializeBlock is the bytes a saved block is stored as. +func SerializeBlock(entry any) string { + return Stringify(entry, 2) + "\n" +} + +// entryHasPath reports a page-like entry: a non-empty string `path`. +func entryHasPath(entry any) bool { + o, ok := entry.(*Object) + if !ok { + return false + } + p, ok := o.Get("path") + s, isString := p.(string) + return ok && isString && len(s) > 0 +} + +// spellingCandidate is one file holding (a spelling of) a saved block. +type spellingCandidate struct { + File string + Version string + HasPath bool + Value *Object // nil when the body wasn't read +} + +func compareSpellings(a, b *spellingCandidate) int { + if a.HasPath != b.HasPath { + if a.HasPath { + return -1 + } + return 1 + } + _, passesA := FullyDecodeFileName(a.File) + _, passesB := FullyDecodeFileName(b.File) + if passesA != passesB { + return passesB - passesA + } + return compareJS(a.File, b.File) +} + +type resolvedSpelling struct { + Name string + Winner *spellingCandidate + Shadowed []*spellingCandidate +} + +// resolveSpellings groups candidates by spelling key and picks one winner per +// group, returned in file-name order of the winners. +func resolveSpellings(candidates []*spellingCandidate) []resolvedSpelling { + var order []string + groups := map[string][]*spellingCandidate{} + for _, c := range candidates { + key, _ := FullyDecodeFileName(c.File) + if _, ok := groups[key]; !ok { + order = append(order, key) + } + groups[key] = append(groups[key], c) + } + winners := make([]resolvedSpelling, 0, len(order)) + for _, key := range order { + group := append([]*spellingCandidate(nil), groups[key]...) + sort.SliceStable(group, func(i, j int) bool { return compareSpellings(group[i], group[j]) < 0 }) + winners = append(winners, resolvedSpelling{ + Name: BlockNameFromFile(group[0].File), + Winner: group[0], + Shadowed: group[1:], + }) + } + sort.SliceStable(winners, func(i, j int) bool { + return compareJS(winners[i].Winner.File, winners[j].Winner.File) < 0 + }) + // A Map keyed by name: a later winner with the same name replaces the + // earlier one in place (as `new Map(entries)` does). + out := make([]resolvedSpelling, 0, len(winners)) + index := map[string]int{} + for _, w := range winners { + if i, ok := index[w.Name]; ok { + out[i] = w + continue + } + index[w.Name] = len(out) + out = append(out, w) + } + return out +} + +// NameViolation is a rule a name breaks. +type NameViolation struct { + Reason string + Message string +} + +// jsLower is String.prototype.toLowerCase, close enough: per-code-point +// lowercase that leaves lone surrogates alone, with U+0130's special mapping. +// OPEN: the context-sensitive final-sigma rule isn't ported. +func jsLower(s string) string { + ascii := true + for i := 0; i < len(s) && ascii; i++ { + ascii = s[i] < 0x80 + } + if ascii { + return strings.ToLower(s) + } + var sb strings.Builder + for i := 0; i < len(s); { + r, size := nextCodePoint(s, i) + switch { + case r >= 0xD800 && r <= 0xDFFF, r == utf8.RuneError && size == 1: + sb.WriteString(s[i : i+size]) + case r == 0x130: + sb.WriteString("i̇") + default: + sb.WriteRune(unicode.ToLower(r)) + } + i += size + } + return sb.String() +} + +// checkBlockName reports every rule a name to save breaks. existing, when +// non-nil, refuses a new name differing from an existing one only in case. +func checkBlockName(name string, existing []string) []NameViolation { + if name == "" { + return []NameViolation{{"empty", "the name is empty"}} + } + var out []NameViolation + if strings.Contains(name, `\`) || strings.Contains(name, "\x00") { + out = append(out, NameViolation{"invalid-character", "the name contains a backslash or a NUL character"}) + } + if strings.Contains(name, "..") { + out = append(out, NameViolation{"dot-dot", `the name contains ".."`}) + } + if strings.HasPrefix(name, ".") { + out = append(out, NameViolation{"leading-dot", `the name starts with ".", which would make its file hidden`}) + } + if name == "__proto__" { + out = append(out, NameViolation{"reserved", `the name "__proto__" is reserved`}) + } + encoded := encodeURIComponent(name) + if len(encoded) > MaxEncodedNameBytes { + out = append(out, NameViolation{"too-long", "the encoded name is over 250 bytes"}) + } + deviceStem := strings.ToUpper(strings.SplitN(encoded, ".", 2)[0]) + if windowsDeviceNames[deviceStem] { + out = append(out, NameViolation{"device-name", `"` + deviceStem + `" is a Windows device name`}) + } + lower := jsLower(name) + for _, ext := range sourceExtensions { + if strings.HasSuffix(lower, ext) { + out = append(out, NameViolation{"source-extension", `the name ends in "` + ext + `", which would shadow a source module`}) + break + } + } + if existing != nil { + isNew := true + collidesWith := "" + found := false + for _, e := range existing { + if e == name { + isNew = false + break + } + if !found && jsLower(e) == lower { + collidesWith, found = e, true + } + } + if isNew && found { + out = append(out, NameViolation{"case-collision", `the name differs from the existing entry "` + collidesWith + `" only in letter case`}) + } + } + return out +} + +// checkDeletedName: anything non-empty can be deleted. +func checkDeletedName(name string) []NameViolation { + if name == "" { + return []NameViolation{{"empty", "the name is empty"}} + } + return nil +} diff --git a/packages/sandbox/daemon-go/internal/content/keys_test.go b/packages/sandbox/daemon-go/internal/content/keys_test.go new file mode 100644 index 0000000000..2f677626d1 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/keys_test.go @@ -0,0 +1,186 @@ +package content + +import ( + "reflect" + "strings" + "testing" +) + +// Ported from keys.test.ts. + +func TestBlockFileName(t *testing.T) { + cases := map[string]string{ + "pages-Home%20Page-6f1e": "pages-Home%2520Page-6f1e.json", + "collections/blog/posts/abc": "collections%2Fblog%2Fposts%2Fabc.json", + "Header": "Header.json", + "pages-Home Page": "pages-Home%20Page.json", + "Cores dos preços": "Cores%20dos%20pre%C3%A7os.json", + "50% off": "50%25%20off.json", + "a!~*'()b": "a!~*'()b.json", + } + for name, file := range cases { + if got := BlockFileName(name); got != file { + t.Errorf("BlockFileName(%q) = %q, want %q", name, got, file) + } + } +} + +func TestEncodeURIComponentPanicsOnALoneSurrogate(t *testing.T) { + defer func() { + if _, ok := recover().(uriError); !ok { + t.Error("expected a uriError panic") + } + }() + encodeURIComponent("x\xed\xa0\x80") +} + +func TestBlockNameFromFileDecodesOnce(t *testing.T) { + cases := map[string]string{ + "pages-Home%2520Page-6f1e.json": "pages-Home%20Page-6f1e", + "collections%2Fblog%2Fposts%2Fabc.json": "collections/blog/posts/abc", + "pages-Home%20Page.json": "pages-Home Page", + "Header.json": "Header", + "50% off.json": "50% off", + "bad%E0%A4%A.json": "bad%E0%A4%A", + "bad%C0%80.json": "bad%C0%80", // overlong: URIError in JS + "sur%ED%A0%80.json": "sur%ED%A0%80", + } + for file, name := range cases { + if got := BlockNameFromFile(file); got != name { + t.Errorf("BlockNameFromFile(%q) = %q, want %q", file, got, name) + } + } + for _, name := range []string{"a", "pages-Home%20Page-6f1e", "x/y", "50% off", "é ü 世界", "%", "%%25"} { + if got := BlockNameFromFile(BlockFileName(name)); got != name { + t.Errorf("round trip of %q gave %q", name, got) + } + } +} + +func TestSpellings(t *testing.T) { + check := func(file, name string, passes int) { + t.Helper() + n, p := FullyDecodeFileName(file) + if n != name || p != passes { + t.Errorf("FullyDecodeFileName(%q) = %q, %d; want %q, %d", file, n, p, name, passes) + } + } + check("pages-Home%2520Page.json", "pages-Home Page", 2) + check("pages-Home%20Page.json", "pages-Home Page", 1) + check("Header.json", "Header", 0) + check("50% off.json", "50% off", 0) + if SpellingKey("pages-Home%20Page") != SpellingKey("pages-Home Page") || SpellingKey("A%2520B") != "A B" { + t.Error("spelling keys") + } + + names := func(rs []resolvedSpelling) []string { + var out []string + for _, r := range rs { + out = append(out, r.Name) + } + return out + } + // A path wins, then more decoding, then the lowest file name. + a := &spellingCandidate{File: "pages-Home%2520Page.json"} + b := &spellingCandidate{File: "pages-Home%20Page.json", HasPath: true} + if r := resolveSpellings([]*spellingCandidate{a, b}); r[0].Winner != b { + t.Error("the file with a path must win") + } + bot := &spellingCandidate{File: "pages-Home%2520Page.json", HasPath: true} + legacy := &spellingCandidate{File: "pages-Home%20Page.json", HasPath: true} + r := resolveSpellings([]*spellingCandidate{legacy, bot}) + if !reflect.DeepEqual(names(r), []string{"pages-Home%20Page"}) || r[0].Winner != bot || r[0].Shadowed[0] != legacy { + t.Errorf("more decoding must win: %+v", r) + } + upper := &spellingCandidate{File: "A%2FB.json"} + lower := &spellingCandidate{File: "A%2fB.json"} + if r := resolveSpellings([]*spellingCandidate{lower, upper}); r[0].Winner != upper || r[0].Name != "A/B" { + t.Error("the lowest file name must win") + } + three := resolveSpellings([]*spellingCandidate{{File: "x%2520y.json"}, {File: "x%20y.json"}, {File: "x y.json"}}) + if three[0].Name != "x%20y" || three[0].Shadowed[0].File != "x%20y.json" || three[0].Shadowed[1].File != "x y.json" { + t.Errorf("three spellings: %+v", three) + } + if got := names(resolveSpellings([]*spellingCandidate{{File: "b.json"}, {File: "a.json"}})); !reflect.DeepEqual(got, []string{"a", "b"}) { + t.Errorf("distinct names: %v", got) + } +} + +func TestSerializeBlockAndFileNames(t *testing.T) { + v, _ := ParseJSON(`{"a":1,"b":[true]}`) + if got := SerializeBlock(v); got != "{\n \"a\": 1,\n \"b\": [\n true\n ]\n}\n" { + t.Errorf("SerializeBlock = %q", got) + } + for file, want := range map[string]bool{ + "a.json": true, ".json": false, "a.ts": false, "a/b.json": false, + ".env.json": false, "..json": false, ".DS_Store.json": false, `a\b.json`: false, + } { + if IsBlockFileName(file) != want { + t.Errorf("IsBlockFileName(%q) != %v", file, want) + } + } + for entry, want := range map[string]bool{`{"path":"/"}`: true, `{"path":""}`: false, `{"path":1}`: false, `null`: false, `[]`: false} { + v, _ := ParseJSON(entry) + if entryHasPath(v) != want { + t.Errorf("entryHasPath(%s) != %v", entry, want) + } + } +} + +func reasons(name string, existing []string) []string { + var out []string + for _, v := range checkBlockName(name, existing) { + out = append(out, v.Reason) + } + return out +} + +func TestCheckBlockName(t *testing.T) { + cases := [][2]string{ + {"", "empty"}, {`a\b`, "invalid-character"}, {"a\x00b", "invalid-character"}, + {"a..b", "dot-dot"}, {"..", "dot-dot"}, {".env", "leading-dot"}, {".", "leading-dot"}, + {".hidden page", "leading-dot"}, {"__proto__", "reserved"}, {"CON", "device-name"}, + {"con", "device-name"}, {"nul.backup", "device-name"}, {"COM1", "device-name"}, + {"lpt9", "device-name"}, {"widget.ts", "source-extension"}, {"Widget.TSX", "source-extension"}, + {"helper.js", "source-extension"}, + } + for _, c := range cases { + got := reasons(c[0], nil) + found := false + for _, r := range got { + found = found || r == c[1] + } + if !found { + t.Errorf("checkBlockName(%q) = %v, want %s", c[0], got, c[1]) + } + } + if got := reasons(strings.Repeat("a", 250), nil); got != nil { + t.Errorf("250 bytes: %v", got) + } + if got := reasons(strings.Repeat("a", 251), nil); !reflect.DeepEqual(got, []string{"too-long"}) { + t.Errorf("251 bytes: %v", got) + } + if got := reasons(strings.Repeat("é", 41), nil); got != nil { + t.Errorf("41 é: %v", got) + } + if got := reasons(strings.Repeat("é", 42), nil); !reflect.DeepEqual(got, []string{"too-long"}) { + t.Errorf("42 é: %v", got) + } + if got := reasons("header", []string{"Header"}); !reflect.DeepEqual(got, []string{"case-collision"}) { + t.Errorf("case collision: %v", got) + } + if reasons("Header", []string{"Header"}) != nil || reasons("Footer", []string{"Header"}) != nil { + t.Error("an update or another name isn't a collision") + } + for _, name := range []string{"Header", "pages-Home Page-6f1e", "collections/blog/posts/abc", "COM", "CONsole", "a.json", "a.b", "end."} { + if got := reasons(name, nil); got != nil { + t.Errorf("%q: %v", name, got) + } + } + if got := reasons(`x..y\z.ts`, nil); !reflect.DeepEqual(got, []string{"invalid-character", "dot-dot", "source-extension"}) { + t.Errorf("every rule: %v", got) + } + if checkDeletedName("widget.ts") != nil || checkDeletedName("CON") != nil || checkDeletedName("")[0].Reason != "empty" { + t.Error("deleted names") + } +} From 4fd78e40a0e0b77b4bb944406a463f1e592e728d Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:54:43 -0300 Subject: [PATCH 03/20] feat(daemon): filesystem content store over the working tree A content store over the sandbox working tree: contained to the tree, writes under a commit lock, durable writes and rollback on failure. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../daemon-go/internal/content/fsstore.go | 874 ++++++++++++++++++ .../daemon-go/internal/content/stat_darwin.go | 18 + .../daemon-go/internal/content/stat_linux.go | 18 + 3 files changed, 910 insertions(+) create mode 100644 packages/sandbox/daemon-go/internal/content/fsstore.go create mode 100644 packages/sandbox/daemon-go/internal/content/stat_darwin.go create mode 100644 packages/sandbox/daemon-go/internal/content/stat_linux.go diff --git a/packages/sandbox/daemon-go/internal/content/fsstore.go b/packages/sandbox/daemon-go/internal/content/fsstore.go new file mode 100644 index 0000000000..1bce4432ba --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/fsstore.go @@ -0,0 +1,874 @@ +package content + +// The filesystem storage, ported from `@decocms/blocks/protocol/storage/fs` +// (index.ts, hash.ts, lock.ts, transaction.ts). Inside the app root: +// +// - .deco/blocks/*.json — the saved blocks; the only thing a commit writes; +// - .deco/schema.gen.json, else .deco/meta.gen.json — read only; +// - .deco/secrets.pub — read only; +// - public/assets — uploads, never overwriting a file. +// +// A file's version is its git blob hash. Commits are serialized in process and +// across processes (.deco/.blocks.lock), applied with staged files and atomic +// renames, rolled back on failure, and the folder is fsynced before a commit +// returns. A crashed commit can leave .deco/.blocks.lock (taken over after +// 30 s) and .deco/.tx-* folders (swept by the next commit) behind. + +import ( + "crypto/rand" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "errors" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + defaultAssetsDir = "public/assets" + DefaultAssetsMaxBytes = 25 * 1024 * 1024 + lockFileName = ".blocks.lock" + transactionPrefix = ".tx-" +) + +// storageNotFound: there's no .deco folder at all. +type storageNotFound struct{ msg string } + +func (e *storageNotFound) Error() string { return e.msg } + +// storageInvalidFile: a name that can't be a saved-block file. +type storageInvalidFile struct{ file string } + +func (e *storageInvalidFile) Error() string { return "not a saved-block file name: " + e.file } + +// storageUnavailable: the storage failed or is busy. +type storageUnavailable struct { + msg string + retryAfterMs int // -1: none +} + +func (e *storageUnavailable) Error() string { return e.msg } + +// storageFile is one saved-block file in a snapshot. +type storageFile struct { + File string + Version string + Size int64 +} + +type storageSnapshot struct { + Revision string + Files []storageFile +} + +type storedBody struct { + Text string + Version string +} + +type storedSchema struct { + Version string + Text string +} + +type filePut struct { + File string + Content string +} + +type commitAttempt struct { + Put []filePut + Delete []string + Expected map[string]*string // nil value: must not exist + // ExpectedSchemaVersion is checked when CheckSchema is set (nil: no schema). + CheckSchema bool + ExpectedSchemaVersion *string +} + +type commitResult struct { + Stale bool + Revision string + Versions map[string]string +} + +// FSOptions configure a filesystem storage. +type FSOptions struct { + // Root is the app root: the folder that contains .deco/. + Root string + // RepoRoot is what describe reports paths relative to. + RepoRoot string + // AssetsMaxBytes is the largest upload (default 25 MiB). + AssetsMaxBytes int64 + // LockTimeout bounds the wait for another process's lock (default 10 s). + LockTimeout time.Duration + // ContainWithin, when set, refuses (as NotFound) to read or write when + // .deco, .deco/blocks or the assets folder resolve, through symlinks, + // outside it. + // OPEN: stricter than the TS FS storage, which follows symlinks. + ContainWithin string + // Exclusive, when set, is held around every commit and upload (the daemon's + // working-tree lock), before the in-process and file locks. ok false means + // it couldn't be had in time: the write is refused as Unavailable (retry), + // never left to land after the client gave up. + Exclusive func() (release func(), ok bool) +} + +type fingerprinted struct { + fingerprint string + version string +} + +// FSStore is the filesystem storage of one app root. +type FSStore struct { + root, repoRoot, decoDir, blocksDir, assetsDir string + assetsMaxBytes int64 + lockTimeout, lockStale time.Duration + exclusive func() (func(), bool) + containWithin string + + hashMu sync.Mutex + hashCache map[string]fingerprinted + + commitMu sync.Mutex // the in-process commit queue + swept bool +} + +// NewFSStore opens the storage of an app root. +func NewFSStore(o FSOptions) *FSStore { + root, _ := filepath.Abs(o.Root) + repoRoot := o.RepoRoot + if repoRoot == "" { + repoRoot = root + } + repoRoot, _ = filepath.Abs(repoRoot) + s := &FSStore{ + root: root, + repoRoot: repoRoot, + decoDir: filepath.Join(root, ".deco"), + blocksDir: filepath.Join(root, ".deco", "blocks"), + assetsDir: filepath.Join(root, filepath.FromSlash(defaultAssetsDir)), + assetsMaxBytes: o.AssetsMaxBytes, + lockTimeout: o.LockTimeout, + lockStale: 30 * time.Second, + exclusive: o.Exclusive, + containWithin: o.ContainWithin, + hashCache: map[string]fingerprinted{}, + } + if s.assetsMaxBytes <= 0 { + s.assetsMaxBytes = DefaultAssetsMaxBytes + } + if s.lockTimeout <= 0 { + s.lockTimeout = 10 * time.Second + } + return s +} + +// checkContained answers storageNotFound when a storage folder resolves +// outside ContainWithin (see FSOptions). +func (s *FSStore) checkContained() error { + if s.containWithin == "" { + return nil + } + within, err := resolveExisting(s.containWithin) + if err != nil { + return wrapIOError(err) + } + for _, dir := range []string{s.decoDir, s.blocksDir, s.assetsDir} { + real, err := resolveExisting(dir) + if err != nil { + return wrapIOError(err) + } + rel, err := filepath.Rel(within, real) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return &storageNotFound{msg: dir + " resolves outside " + s.containWithin} + } + } + return nil +} + +// resolveExisting resolves the symlinks of path's longest existing prefix +// and appends the rest (what a later MkdirAll would create). +func resolveExisting(path string) (string, error) { + path = filepath.Clean(path) + rest := "" + for { + real, err := filepath.EvalSymlinks(path) + if err == nil { + return filepath.Join(real, rest), nil + } + if !isMissing(err) { + return "", err + } + parent := filepath.Dir(path) + if parent == path { + return filepath.Join(path, rest), nil + } + rest = filepath.Join(filepath.Base(path), rest) + path = parent + } +} + +// Root is the absolute app root. +func (s *FSStore) Root() string { return s.root } + +func isMissing(err error) bool { + return errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) +} + +func exists(path string) (bool, error) { + if _, err := os.Lstat(path); err != nil { + if isMissing(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func readBytesOrNil(path string) ([]byte, error) { + b, err := os.ReadFile(path) + if err != nil { + if isMissing(err) { + return nil, nil + } + return nil, err + } + return b, nil +} + +// gitBlobHash is `git hash-object`: sha1("blob \0" + bytes). +func gitBlobHash(b []byte) string { + h := sha1.New() + h.Write([]byte("blob " + strconv.Itoa(len(b)) + "\x00")) + h.Write(b) + return hex.EncodeToString(h.Sum(nil)) +} + +// revisionOf is SHA-256 over the sorted `file\0version` listing. +func revisionOf(files []storageFile) string { + listing := make([]string, len(files)) + for i, f := range files { + listing[i] = f.File + "\x00" + f.Version + } + sort.SliceStable(listing, func(i, j int) bool { return compareJS(listing[i], listing[j]) < 0 }) + sum := sha256.Sum256([]byte(strings.Join(listing, "\n"))) + return hex.EncodeToString(sum[:]) +} + +func toPosixRel(base, target string) string { + rel, err := filepath.Rel(base, target) + if err != nil || rel == "." { + return "." + } + return filepath.ToSlash(rel) +} + +type storageDescription struct { + Root string + AssetsDir string + AssetsMaxBytes int64 +} + +func (s *FSStore) describe() storageDescription { + return storageDescription{ + Root: toPosixRel(s.repoRoot, s.root), + AssetsDir: toPosixRel(s.repoRoot, s.assetsDir), + AssetsMaxBytes: s.assetsMaxBytes, + } +} + +func assertBlockFile(file string) error { + if !IsBlockFileName(file) { + return &storageInvalidFile{file: file} + } + return nil +} + +func wrapIOError(err error) error { + var nf *storageNotFound + var un *storageUnavailable + var inv *storageInvalidFile + if errors.As(err, &nf) || errors.As(err, &un) || errors.As(err, &inv) { + return err + } + // OPEN: the message text is Go's, not Node's (`filesystem error: ENOENT: …`). + return &storageUnavailable{msg: "filesystem error: " + err.Error(), retryAfterMs: -1} +} + +func (s *FSStore) hashFile(file string) (*storageFile, error) { + path := filepath.Join(s.blocksDir, file) + info, err := os.Stat(path) + if err != nil { + if isMissing(err) { + return nil, nil + } + return nil, err + } + if !info.Mode().IsRegular() { + return nil, nil + } + fingerprint := statFingerprint(info) + s.hashMu.Lock() + cached, ok := s.hashCache[file] + s.hashMu.Unlock() + if ok && cached.fingerprint == fingerprint { + return &storageFile{File: file, Version: cached.version, Size: info.Size()}, nil + } + b, err := os.ReadFile(path) + if err != nil { + if isMissing(err) { + return nil, nil + } + return nil, err + } + version := gitBlobHash(b) + s.hashMu.Lock() + s.hashCache[file] = fingerprinted{fingerprint: fingerprint, version: version} + s.hashMu.Unlock() + return &storageFile{File: file, Version: version, Size: int64(len(b))}, nil +} + +func (s *FSStore) freshVersion(file string) (*string, error) { + b, err := os.ReadFile(filepath.Join(s.blocksDir, file)) + if err != nil { + if isMissing(err) { + return nil, nil + } + return nil, err + } + v := gitBlobHash(b) + return &v, nil +} + +func (s *FSStore) readSchema() (*storedSchema, error) { + for _, name := range []string{"schema.gen.json", "meta.gen.json"} { + b, err := readBytesOrNil(filepath.Join(s.decoDir, name)) + if err != nil { + return nil, err + } + if b != nil { + text, _ := decodeUTF8(b, false, true) + return &storedSchema{Version: gitBlobHash(b), Text: text}, nil + } + } + return nil, nil +} + +func (s *FSStore) readSecretsPublicKey() (*string, error) { + b, err := readBytesOrNil(filepath.Join(s.decoDir, "secrets.pub")) + if err != nil || b == nil { + return nil, err + } + // readFile(path, "utf8") keeps a BOM. + text, _ := decodeUTF8(b, false, false) + return &text, nil +} + +func (s *FSStore) snapshotRaw() (storageSnapshot, error) { + ok, err := exists(s.decoDir) + if err != nil { + return storageSnapshot{}, err + } + if !ok { + return storageSnapshot{}, &storageNotFound{msg: "no .deco folder in " + s.root} + } + var names []string + entries, err := os.ReadDir(s.blocksDir) + if err != nil && !isMissing(err) { + return storageSnapshot{}, err + } + for _, e := range entries { + // Node would see a name that isn't UTF-8 with U+FFFD in it, and then + // fail to stat it: skipping it is the same outcome. + if e.Type().IsRegular() && IsBlockFileName(e.Name()) && strings.ToValidUTF8(e.Name(), "�") == e.Name() { + names = append(names, e.Name()) + } + } + files := make([]storageFile, 0, len(names)) + for _, name := range names { + f, err := s.hashFile(name) + if err != nil { + return storageSnapshot{}, err + } + if f != nil { + files = append(files, *f) + } + } + sort.SliceStable(files, func(i, j int) bool { return compareJS(files[i].File, files[j].File) < 0 }) + present := map[string]bool{} + for _, n := range names { + present[n] = true + } + s.hashMu.Lock() + for k := range s.hashCache { + if !present[k] { + delete(s.hashCache, k) + } + } + s.hashMu.Unlock() + return storageSnapshot{Revision: revisionOf(files), Files: files}, nil +} + +func (s *FSStore) snapshot() (storageSnapshot, error) { + snap, err := s.snapshotRaw() + if err != nil { + return snap, wrapIOError(err) + } + return snap, nil +} + +// readFiles reads current bodies with the version of the bytes read; a file +// that vanished is left out. +func (s *FSStore) readFiles(files []string) (map[string]storedBody, error) { + out := map[string]storedBody{} + for _, file := range files { + if err := assertBlockFile(file); err != nil { + return nil, err + } + b, err := readBytesOrNil(filepath.Join(s.blocksDir, file)) + if err != nil { + return nil, err + } + if b != nil { + text, _ := decodeUTF8(b, false, true) + out[file] = storedBody{Text: text, Version: gitBlobHash(b)} + } + } + return out, nil +} + +func (s *FSStore) commit(attempt commitAttempt) (commitResult, error) { + for _, p := range attempt.Put { + if err := assertBlockFile(p.File); err != nil { + return commitResult{}, err + } + } + for _, f := range attempt.Delete { + if err := assertBlockFile(f); err != nil { + return commitResult{}, err + } + } + for f := range attempt.Expected { + if err := assertBlockFile(f); err != nil { + return commitResult{}, err + } + } + ok, err := exists(s.decoDir) + if err != nil { + return commitResult{}, wrapIOError(err) + } + if !ok { + return commitResult{}, &storageNotFound{msg: "no .deco folder in " + s.root} + } + result, err := s.withCommitLock(func() (commitResult, error) { + if !s.swept { + if err := sweepStaleTransactions(s.decoDir); err != nil { + return commitResult{}, err + } + s.swept = true + } + if attempt.CheckSchema { + schema, err := s.readSchema() + if err != nil { + return commitResult{}, err + } + var current *string + if schema != nil { + current = &schema.Version + } + if !sameVersion(current, attempt.ExpectedSchemaVersion) { + return commitResult{Stale: true}, nil + } + } + for file, expected := range attempt.Expected { + fresh, err := s.freshVersion(file) + if err != nil { + return commitResult{}, err + } + if !sameVersion(fresh, expected) { + return commitResult{Stale: true}, nil + } + } + if err := applyFileChange(s.blocksDir, s.decoDir, attempt.Put, attempt.Delete); err != nil { + return commitResult{}, err + } + versions := map[string]string{} + for _, p := range attempt.Put { + versions[p.File] = gitBlobHash([]byte(p.Content)) + } + after, err := s.snapshotRaw() + if err != nil { + return commitResult{}, err + } + return commitResult{Revision: after.Revision, Versions: versions}, nil + }) + if err != nil { + return commitResult{}, wrapIOError(err) + } + return result, nil +} + +func sameVersion(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +// errTreeBusy: the working tree stayed locked (a publish, rebase or +// autosave) for longer than the daemon waits. +var errTreeBusy = &storageUnavailable{msg: "the working tree is busy", retryAfterMs: 500} + +// holdExclusive takes the Exclusive lock, if any. +func (s *FSStore) holdExclusive() (func(), error) { + if s.exclusive == nil { + return func() {}, nil + } + release, ok := s.exclusive() + if !ok { + return nil, errTreeBusy + } + return release, nil +} + +func (s *FSStore) withCommitLock(fn func() (commitResult, error)) (commitResult, error) { + releaseTree, err := s.holdExclusive() + if err != nil { + return commitResult{}, err + } + defer releaseTree() + s.commitMu.Lock() + defer s.commitMu.Unlock() + release, err := acquireFileLock(s.decoDir, s.lockTimeout, s.lockStale) + if err != nil { + return commitResult{}, err + } + defer release() + return fn() +} + +func acquireFileLock(decoDir string, timeout, stale time.Duration) (func(), error) { + path := filepath.Join(decoDir, lockFileName) + deadline := time.Now().Add(timeout) + for { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err == nil { + _, werr := f.WriteString(strconv.Itoa(os.Getpid()) + "\n") + cerr := f.Close() + if werr != nil || cerr != nil { + os.Remove(path) + return nil, errors.Join(werr, cerr) + } + return func() { os.Remove(path) }, nil + } + if !errors.Is(err, fs.ErrExist) { + return nil, err + } + if info, statErr := os.Stat(path); statErr == nil && time.Since(info.ModTime()) > stale { + // A crashed writer left its lock behind. + os.Remove(path) + continue + } + if time.Now().After(deadline) { + return nil, &storageUnavailable{msg: "another process is writing .deco/blocks", retryAfterMs: 500} + } + time.Sleep(20 * time.Millisecond) + } +} + +// ---------------------------------------------------------------- transaction + +func writeDurably(path, content string) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := f.WriteString(content); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + return f.Close() +} + +// syncDir flushes a folder's entries; a no-op where that's unsupported. +func syncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + if ignorableSyncError(err) { + return nil + } + return err + } + defer f.Close() + if err := f.Sync(); err != nil && !ignorableSyncError(err) { + return err + } + return nil +} + +func ignorableSyncError(err error) bool { + for _, e := range []error{syscall.EISDIR, syscall.EPERM, syscall.EACCES, syscall.EINVAL, syscall.EBADF, syscall.ENOTSUP} { + if errors.Is(err, e) { + return true + } + } + return false +} + +func copyFile(src, dst string) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + info, err := os.Stat(src) + if err != nil { + return err + } + return os.WriteFile(dst, b, info.Mode().Perm()) +} + +// preserve hard-links source to backup (a copy where links aren't supported); +// false when there's nothing to preserve. +func preserve(source, backup string) (bool, error) { + err := os.Link(source, backup) + if err == nil { + return true, nil + } + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if errors.Is(err, syscall.EPERM) || errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.EXDEV) || errors.Is(err, syscall.EOPNOTSUPP) { + if cerr := copyFile(source, backup); cerr != nil { + if errors.Is(cerr, fs.ErrNotExist) { + return false, nil + } + return false, cerr + } + return true, nil + } + return false, err +} + +func sameFile(a, b string) (bool, error) { + x, err := os.Stat(a) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + y, err := os.Stat(b) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + return os.SameFile(x, y), nil +} + +// effectiveDeletes drops a delete that, on a case-insensitive filesystem, is +// the very file a put renames over. +func effectiveDeletes(dir string, put []filePut, deletes []string) ([]string, error) { + var out []string + seen := map[string]bool{} + putSet := map[string]bool{} + for _, p := range put { + putSet[p.File] = true + } + for _, file := range deletes { + if seen[file] { + continue + } + seen[file] = true + if putSet[file] { + continue + } + twin := "" + for _, p := range put { + if jsLower(p.File) == jsLower(file) { + twin = p.File + break + } + } + if twin != "" { + same, err := sameFile(filepath.Join(dir, file), filepath.Join(dir, twin)) + if err != nil { + return nil, err + } + if same { + continue + } + } + out = append(out, file) + } + return out, nil +} + +// applyFileChange applies puts and deletes to dir atomically: every file +// lands, or dir is left as it was. scratch holds the transaction folder and +// must be on dir's filesystem. +func applyFileChange(dir, scratch string, put []filePut, deletes []string) (err error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + deletes, err = effectiveDeletes(dir, put, deletes) + if err != nil { + return err + } + tx, err := os.MkdirTemp(scratch, transactionPrefix) + if err != nil { + return err + } + defer os.RemoveAll(tx) + staged := filepath.Join(tx, "new") + backups := filepath.Join(tx, "old") + if err := os.Mkdir(staged, 0o755); err != nil { + return err + } + if err := os.Mkdir(backups, 0o755); err != nil { + return err + } + + type touch struct { + file string + backup string // "" when there was nothing to preserve + } + var touched []touch + isTouched := map[string]bool{} + defer func() { + if err == nil { + return + } + for _, t := range touched { + target := filepath.Join(dir, t.file) + if t.backup != "" { + os.Rename(t.backup, target) + } else { + os.Remove(target) + } + } + }() + + for i, p := range put { + if err = writeDurably(filepath.Join(staged, strconv.Itoa(i)), p.Content); err != nil { + return err + } + } + targets := make([]string, 0, len(put)+len(deletes)) + for _, p := range put { + targets = append(targets, p.File) + } + targets = append(targets, deletes...) + for _, file := range targets { + if isTouched[file] { + continue + } + backup := filepath.Join(backups, strconv.Itoa(len(touched))) + var kept bool + if kept, err = preserve(filepath.Join(dir, file), backup); err != nil { + return err + } + if !kept { + backup = "" + } + isTouched[file] = true + touched = append(touched, touch{file: file, backup: backup}) + } + for i, p := range put { + if err = os.Rename(filepath.Join(staged, strconv.Itoa(i)), filepath.Join(dir, p.File)); err != nil { + return err + } + } + for _, file := range deletes { + if rmErr := os.Remove(filepath.Join(dir, file)); rmErr != nil && !errors.Is(rmErr, fs.ErrNotExist) { + err = rmErr + return err + } + } + err = syncDir(dir) + return err +} + +// sweepStaleTransactions removes the transaction folders a crashed commit +// left; call it only under the commit lock. +func sweepStaleTransactions(scratch string) error { + entries, err := os.ReadDir(scratch) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), transactionPrefix) { + os.RemoveAll(filepath.Join(scratch, e.Name())) + } + } + return nil +} + +// ---------------------------------------------------------------- assets + +// suffixedAssetName is the name to try when name is taken: a short random +// suffix before the extension. +func suffixedAssetName(name string) string { + var b [3]byte + rand.Read(b[:]) + suffix := hex.EncodeToString(b[:]) + if dot := strings.LastIndex(name, "."); dot > 0 { + return name[:dot] + "-" + suffix + name[dot:] + } + return name + "-" + suffix +} + +// putAsset stores an upload, never overwriting a file; it returns the stored name. +func (s *FSStore) putAsset(name string, body []byte) (string, error) { + if name == "" || strings.ContainsAny(name, `/\`) || strings.HasPrefix(name, ".") { + return "", errors.New("not an asset file name: " + name) + } + if err := s.checkContained(); err != nil { + return "", err + } + releaseTree, err := s.holdExclusive() + if err != nil { + return "", err + } + defer releaseTree() + if err := os.MkdirAll(s.assetsDir, 0o755); err != nil { + return "", err + } + candidate := name + for attempt := 0; ; attempt++ { + path := filepath.Join(s.assetsDir, candidate) + if filepath.Dir(path) != s.assetsDir { + return "", errors.New("invalid asset path") + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + if !errors.Is(err, fs.ErrExist) || attempt > 20 { + return "", err + } + candidate = suffixedAssetName(name) + continue + } + _, werr := f.Write(body) + serr := f.Sync() + cerr := f.Close() + if err := errors.Join(werr, serr, cerr); err != nil { + return "", err + } + return candidate, nil + } +} diff --git a/packages/sandbox/daemon-go/internal/content/stat_darwin.go b/packages/sandbox/daemon-go/internal/content/stat_darwin.go new file mode 100644 index 0000000000..b6f0240b7f --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/stat_darwin.go @@ -0,0 +1,18 @@ +package content + +import ( + "os" + "strconv" + "syscall" +) + +// statFingerprint is `ino:size:mtimeNs:ctimeNs`: an unchanged fingerprint +// means an unchanged file, so a poll only rehashes what changed. +func statFingerprint(info os.FileInfo) string { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return strconv.FormatInt(info.Size(), 10) + ":" + strconv.FormatInt(info.ModTime().UnixNano(), 10) + } + return strconv.FormatUint(uint64(st.Ino), 10) + ":" + strconv.FormatInt(st.Size, 10) + ":" + + strconv.FormatInt(st.Mtimespec.Nano(), 10) + ":" + strconv.FormatInt(st.Ctimespec.Nano(), 10) +} diff --git a/packages/sandbox/daemon-go/internal/content/stat_linux.go b/packages/sandbox/daemon-go/internal/content/stat_linux.go new file mode 100644 index 0000000000..b6290d5e02 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/stat_linux.go @@ -0,0 +1,18 @@ +package content + +import ( + "os" + "strconv" + "syscall" +) + +// statFingerprint is `ino:size:mtimeNs:ctimeNs`: an unchanged fingerprint +// means an unchanged file, so a poll only rehashes what changed. +func statFingerprint(info os.FileInfo) string { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return strconv.FormatInt(info.Size(), 10) + ":" + strconv.FormatInt(info.ModTime().UnixNano(), 10) + } + return strconv.FormatUint(uint64(st.Ino), 10) + ":" + strconv.FormatInt(st.Size, 10) + ":" + + strconv.FormatInt(st.Mtim.Nano(), 10) + ":" + strconv.FormatInt(st.Ctim.Nano(), 10) +} From f0ba46de89b9edc5398770c4f82af24d778650f0 Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:55:19 -0300 Subject: [PATCH 04/20] refactor(api): extract gen-artifact + commit message from commit-coalescer Moves regenerateGenArtifact and decofileCommitMessage out of commit-coalescer.ts into decofile/gen-artifact.ts so the hosted publish path (later in this stack) can reuse them. No behavior change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- apps/api/src/decofile/commit-coalescer.ts | 60 ++++++------------- apps/api/src/decofile/gen-artifact.ts | 71 +++++++++++++++++++++++ 2 files changed, 88 insertions(+), 43 deletions(-) create mode 100644 apps/api/src/decofile/gen-artifact.ts diff --git a/apps/api/src/decofile/commit-coalescer.ts b/apps/api/src/decofile/commit-coalescer.ts index e5b3df6a33..f0ba25411a 100644 --- a/apps/api/src/decofile/commit-coalescer.ts +++ b/apps/api/src/decofile/commit-coalescer.ts @@ -1,9 +1,5 @@ -import { - appendCoAuthorTrailer, - type CoAuthorIdentity, -} from "@decocms/sandbox/shared"; -import { blockKeyToFileStem, mergeBlocks } from "@decocms/shared/decofile"; -import { repoIdentityKey } from "@decocms/shared/git-providers"; +import type { CoAuthorIdentity } from "@decocms/sandbox/shared"; +import { blockKeyToFileStem } from "@decocms/shared/decofile"; import { exponentialBackoffWithJitter, sleep } from "@decocms/shared/std"; import { type FileChange, @@ -15,9 +11,9 @@ import { blockEntriesInTree, blocksDirPath, primeBlobCache, - resolveBlockContents, resolveOrCreateHead, } from "./read-decofile"; +import { decofileCommitMessage, regenerateGenArtifact } from "./gen-artifact"; /** * Per-(virtualMcpId, branch) commit coalescer. Autosaves arrive every ~700ms @@ -174,33 +170,24 @@ async function commitBatch(batch: Batch): Promise { if (writes.length === 0) return headSha; - // Repos that track the merged artifact get it regenerated in-commit; - // gitignored repos (the common case) never have the tree entry. - const genPath = packagePath - ? `${packagePath}/.deco/blocks.gen.json` - : ".deco/blocks.gen.json"; - if (tree.some((e) => e.type === "blob" && e.path === genPath)) { - const files = await resolveBlockContents( - client, - nextBlocks.values(), - blobMemo, - ); - const { decofile: genContent, skipped } = mergeBlocks(files); - if (skipped.length > 0) { - console.warn("decofile gen: dropped blocks that were not valid JSON", { - repo: repoIdentityKey(client.repo), - branch, - packagePath, - blocks: skipped.map((s) => s.key), - }); - } - writes.push({ path: genPath, content: genContent }); - } + const gen = await regenerateGenArtifact({ + client, + tree, + packagePath, + branch, + nextBlocks: nextBlocks.values(), + memo: blobMemo, + }); + if (gen) writes.push(gen); try { const { sha } = await client.commitFiles({ branch, - message: commitMessage(batch), + message: decofileCommitMessage( + [...batch.set.keys()], + [...batch.del], + batch.deps.coAuthor, + ), expectedHead: headSha, changes: writes, }); @@ -217,16 +204,3 @@ async function commitBatch(batch: Batch): Promise { } } } - -function commitMessage(batch: Batch): string { - const summarize = (keys: string[]): string => { - const shown = keys.slice(0, 3).join(", "); - return keys.length > 3 ? `${shown} (+${keys.length - 3} more)` : shown; - }; - const parts: string[] = []; - if (batch.set.size > 0) - parts.push(`update ${summarize([...batch.set.keys()])}`); - if (batch.del.size > 0) parts.push(`delete ${summarize([...batch.del])}`); - const subject = `chore(decofile): ${parts.join("; ")}`; - return appendCoAuthorTrailer(subject, batch.deps.coAuthor); -} diff --git a/apps/api/src/decofile/gen-artifact.ts b/apps/api/src/decofile/gen-artifact.ts new file mode 100644 index 0000000000..ed945e9043 --- /dev/null +++ b/apps/api/src/decofile/gen-artifact.ts @@ -0,0 +1,71 @@ +import { + appendCoAuthorTrailer, + type CoAuthorIdentity, +} from "@decocms/sandbox/shared"; +import { mergeBlocks } from "@decocms/shared/decofile"; +import { repoIdentityKey } from "@decocms/shared/git-providers"; +import type { FileChange, RepoContentClient, TreeEntry } from "@/git-providers"; +import { type BlockSource, resolveBlockContents } from "./read-decofile"; + +/** Repo-relative path of the merged `blocks.gen.json` artifact. */ +function genArtifactPath(packagePath: string | null): string { + return packagePath + ? `${packagePath}/.deco/blocks.gen.json` + : ".deco/blocks.gen.json"; +} + +/** + * The regenerated `blocks.gen.json`, for a commit that changes `.deco/blocks`. + * + * Repos that track the merged artifact get it rewritten in the same commit, so + * it never disagrees with the block files; gitignored repos (the common case) + * have no tree entry and get `null`. `nextBlocks` is the post-commit view of + * the blocks dir; `memo` is the blob memo a compare-and-swap retry loop + * threads through, so a retry never re-reads a blob it already resolved. + */ +export async function regenerateGenArtifact(params: { + client: RepoContentClient; + tree: TreeEntry[]; + packagePath: string | null; + branch: string; + nextBlocks: Iterable; + memo: Map; +}): Promise { + const { client, tree, packagePath, branch } = params; + const genPath = genArtifactPath(packagePath); + if (!tree.some((e) => e.type === "blob" && e.path === genPath)) return null; + const files = await resolveBlockContents( + client, + params.nextBlocks, + params.memo, + ); + const { decofile, skipped } = mergeBlocks(files); + if (skipped.length > 0) { + console.warn("decofile gen: dropped blocks that were not valid JSON", { + repo: repoIdentityKey(client.repo), + branch, + packagePath, + blocks: skipped.map((s) => s.key), + }); + } + return { path: genPath, content: decofile }; +} + +/** The commit message of a decofile write: what changed, plus the co-author. */ +export function decofileCommitMessage( + setKeys: string[], + deleteKeys: string[], + coAuthor: CoAuthorIdentity | null | undefined, +): string { + const summarize = (keys: string[]): string => { + const shown = keys.slice(0, 3).join(", "); + return keys.length > 3 ? `${shown} (+${keys.length - 3} more)` : shown; + }; + const parts: string[] = []; + if (setKeys.length > 0) parts.push(`update ${summarize(setKeys)}`); + if (deleteKeys.length > 0) parts.push(`delete ${summarize(deleteKeys)}`); + return appendCoAuthorTrailer( + `chore(decofile): ${parts.join("; ")}`, + coAuthor, + ); +} From 77d7a4e3222d27be7b142159b37dac4afe8ff6f1 Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:56:12 -0300 Subject: [PATCH 05/20] chore(api): depend on @decocms/blocks; run Studio CLI by path Adds `@decocms/blocks@8.1.0-next.7` to apps/api (the hosted path uses its decofile/release helpers). The Dockerfile and the tarball smoke test now start Studio by the path of its CLI, and a test pins the v7 secret-loader exemption. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- apps/api/Dockerfile | 6 +- apps/api/package.json | 1 + apps/api/scripts/smoke-tarball.ts | 20 ++- apps/api/src/cli/deco-bin.test.ts | 77 +++++++++++ .../src/decofile/legacy-secret-guard.test.ts | 128 ++++++++++++++++++ bun.lock | 5 + 6 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/cli/deco-bin.test.ts create mode 100644 apps/api/src/decofile/legacy-secret-guard.test.ts diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index e0054d2ca1..eeaa920ba2 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -58,4 +58,8 @@ ENV DATABASE_URL=file:/app/data/mesh.db # The CLI handles migrations automatically on startup # Use --skip-migrations if you want to manage migrations separately -CMD ["bun", "run", "deco", "--no-tui", "--no-local-mode"] +# Runs Studio's CLI by path, not through the `deco` bin name: another installed +# package may declare a `deco` bin too (@decocms/blocks does), and whichever +# wins `node_modules/.bin/deco` would start instead of Studio +# (src/cli/deco-bin.test.ts). +CMD ["bun", "run", "node_modules/decocms/dist/server/cli.js", "--no-tui", "--no-local-mode"] diff --git a/apps/api/package.json b/apps/api/package.json index 83f4c6f4e1..304f61eb85 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -67,6 +67,7 @@ "@better-auth/sso": "1.4.1", "@decocms/better-auth": "1.5.17", "@decocms/bindings": "workspace:*", + "@decocms/blocks": "8.1.0-next.7", "@decocms/mcp-utils": "workspace:*", "@decocms/runtime": "workspace:*", "@decocms/sandbox": "workspace:*", diff --git a/apps/api/scripts/smoke-tarball.ts b/apps/api/scripts/smoke-tarball.ts index 61123b9302..89edc321fa 100644 --- a/apps/api/scripts/smoke-tarball.ts +++ b/apps/api/scripts/smoke-tarball.ts @@ -25,7 +25,7 @@ */ import { $ } from "bun"; -import { mkdtemp, writeFile } from "fs/promises"; +import { mkdtemp, realpath, writeFile } from "fs/promises"; import { join } from "path"; import { tmpdir } from "os"; @@ -71,7 +71,25 @@ if (!(await Bun.file(clientIndex).exists())) { // eagerly during load, so a missing external crashes here before // --version prints — same symptom a real consumer would hit. const cliBin = join(scratch, "node_modules", ".bin", "deco"); +const studioCli = join( + scratch, + "node_modules", + "decocms", + "dist", + "server", + "cli.js", +); +// Another installed package with a `deco` bin (@decocms/blocks has one) can +// win the link and start instead of Studio: the bin must be Studio's CLI. +if ((await realpath(cliBin)) !== (await realpath(studioCli))) { + console.error( + `node_modules/.bin/deco resolves to ${await realpath(cliBin)}, not Studio's ${studioCli}`, + ); + process.exit(1); +} await $`${cliBin} --version`.cwd(scratch); +// What the Docker image runs (apps/api/Dockerfile CMD): the CLI by path. +await $`bun run node_modules/decocms/dist/server/cli.js --version`.cwd(scratch); console.log( "✅ Smoke test passed — browser assets are present and every external resolves at startup.", diff --git a/apps/api/src/cli/deco-bin.test.ts b/apps/api/src/cli/deco-bin.test.ts new file mode 100644 index 0000000000..bd5fd33836 --- /dev/null +++ b/apps/api/src/cli/deco-bin.test.ts @@ -0,0 +1,77 @@ +/** + * The published `decocms` package and its Docker image must start Studio's + * own CLI. `@decocms/blocks` also declares a `deco` bin: as a runtime + * dependency, `bun add decocms` linked whichever `deco` won into + * `node_modules/.bin`, and the image started the Blocks CLI + * (`unknown command "--no-tui"`). The server bundle inlines what it needs, so + * no runtime dependency may declare a `deco` bin, and the image runs the CLI + * by path. `scripts/smoke-tarball.ts` checks the same on the packed tarball. + */ + +import { describe, expect, test } from "bun:test"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +const API_ROOT = join(import.meta.dir, "..", ".."); + +interface PackageJson { + name: string; + bin?: string | Record; + dependencies?: Record; + optionalDependencies?: Record; +} + +function readPackage(dir: string): PackageJson { + return JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); +} + +function binNames(pkg: PackageJson): string[] { + if (!pkg.bin) return []; + if (typeof pkg.bin === "string") return [pkg.name.split("/").pop()!]; + return Object.keys(pkg.bin); +} + +const api = readPackage(API_ROOT); +const runtimeDeps = { + ...api.dependencies, + ...api.optionalDependencies, +}; + +describe("the deco bin", () => { + test("is Studio's CLI", () => { + expect(api.name).toBe("decocms"); + expect(api.bin).toEqual({ deco: "./dist/server/cli.js" }); + }); + + test("@decocms/blocks is bundled, never a runtime dependency", () => { + expect(Object.keys(runtimeDeps)).not.toContain("@decocms/blocks"); + }); + + test("no installed runtime dependency declares another deco bin", () => { + const clashes = Object.keys(runtimeDeps).filter((name) => { + const dir = join(API_ROOT, "node_modules", name); + // Optional platform packages may be missing on this machine. + if (!existsSync(join(dir, "package.json"))) return false; + return binNames(readPackage(dir)).includes("deco"); + }); + expect(clashes).toEqual([]); + }); + + test("the Docker image runs the CLI by path", () => { + const dockerfile = readFileSync(join(API_ROOT, "Dockerfile"), "utf8"); + const cmd = dockerfile + .split("\n") + .filter((line) => line.startsWith("CMD ")) + .pop(); + expect(cmd).toBeDefined(); + const argv = JSON.parse(cmd!.slice("CMD ".length)) as string[]; + // `bun add` of the tarball installs it at node_modules/decocms. + const cliPath = join( + "node_modules", + api.name, + (api.bin as Record).deco!, + ); + expect(argv.slice(0, 3)).toEqual(["bun", "run", cliPath]); + expect(argv).toContain("--no-tui"); + }); +}); diff --git a/apps/api/src/decofile/legacy-secret-guard.test.ts b/apps/api/src/decofile/legacy-secret-guard.test.ts new file mode 100644 index 0000000000..397416a532 --- /dev/null +++ b/apps/api/src/decofile/legacy-secret-guard.test.ts @@ -0,0 +1,128 @@ +/** + * The content protocol's secret guard on a legacy (v7) site. + * + * A v7 `website/loaders/secret.ts` block marks its `encrypted` string + * `"format": "secret"`, but that string is the site's own hex ciphertext, not + * a v8 `Secret` field. The guard must let it through unchanged, and stay strict + * for v8 `Secret` fields and `secret` blocks. The published + * `@decocms/blocks@8.1.0-next.3` lacks the exemption, so Studio carries it as + * `patches/@decocms%2Fblocks@8.1.0-next.3.patch` until a release includes it. + */ + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createContentHandler } from "@decocms/blocks/protocol/server"; +import { createFsStorage } from "@decocms/blocks/protocol/storage/fs"; + +const LOADER = "website/loaders/secret.ts"; + +/** A v7 `meta.gen.json` with the secret loader, an app that uses it, and a v8 section. */ +const meta = { + manifest: { + blocks: { + loaders: { [LOADER]: { $ref: "#/definitions/c2VjcmV0" } }, + apps: { "site/apps/site.ts": { $ref: "#/definitions/c2l0ZQ==" } }, + sections: { newsletter: { $ref: "#/definitions/bmV3c2xldHRlcg==" } }, + }, + }, + schema: { + definitions: { + c2VjcmV0: { + type: "object", + properties: { + name: { type: "string" }, + encrypted: { type: "string", format: "secret" }, + }, + }, + "c2l0ZQ==": { + type: "object", + properties: { apiKey: { $ref: "#/definitions/c2VjcmV0" } }, + }, + "bmV3c2xldHRlcg==": { + type: "object", + properties: { apiKey: { type: "string", format: "secret" } }, + }, + }, + }, +}; + +let root: string; +let handler: (request: Request) => Promise; + +beforeAll(async () => { + root = await mkdtemp(join(tmpdir(), "legacy-secret-guard-")); + await mkdir(join(root, ".deco", "blocks"), { recursive: true }); + await writeFile(join(root, ".deco", "meta.gen.json"), JSON.stringify(meta)); + handler = createContentHandler(createFsStorage({ root }), { + server: { name: "test", version: "0" }, + }); +}); + +afterAll(async () => { + await rm(root, { recursive: true, force: true }); +}); + +async function apply(set: Record) { + const res = await handler( + new Request("http://localhost/rpc", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "blocks.apply", + params: { set }, + }), + }), + ); + return (await res.json()) as { + result?: { revision: string }; + error?: { code: number; message: string }; + }; +} + +describe("secret guard on a legacy site", () => { + test("saves a v7 secret loader block's hex ciphertext unchanged", async () => { + const body = await apply({ + site: { + __resolveType: "site/apps/site.ts", + apiKey: { + __resolveType: LOADER, + name: "API_KEY", + encrypted: "0a1b2c3d", + }, + }, + }); + expect(body.error).toBeUndefined(); + expect(body.result?.revision).toEqual(expect.any(String)); + }); + + test("still refuses plain text in a v8 Secret field", async () => { + const body = await apply({ + news: { __resolveType: "newsletter", apiKey: "plain-text" }, + }); + expect(body.error).toBeDefined(); + }); + + test("still refuses a malformed v8 secret block", async () => { + const body = await apply({ + news: { + __resolveType: "newsletter", + apiKey: { __resolveType: "secret", ciphertext: "0a1b2c3d" }, + }, + }); + expect(body.error).toBeDefined(); + }); + + test("still refuses a v7 loader block in a v8 Secret field", async () => { + const body = await apply({ + news: { + __resolveType: "newsletter", + apiKey: { __resolveType: LOADER, encrypted: "0a1b2c3d" }, + }, + }); + expect(body.error).toBeDefined(); + }); +}); diff --git a/bun.lock b/bun.lock index 4211163fea..f0a78f0795 100644 --- a/bun.lock +++ b/bun.lock @@ -54,6 +54,7 @@ "@better-auth/sso": "1.4.1", "@decocms/better-auth": "1.5.17", "@decocms/bindings": "workspace:*", + "@decocms/blocks": "8.1.0-next.7", "@decocms/mcp-utils": "workspace:*", "@decocms/runtime": "workspace:*", "@decocms/sandbox": "workspace:*", @@ -704,6 +705,8 @@ "@decocms/bindings": ["@decocms/bindings@workspace:packages/bindings"], + "@decocms/blocks": ["@decocms/blocks@8.1.0-next.7", "", { "dependencies": { "ajv": "^8.20.0", "zod": "^4.4.3" }, "peerDependencies": { "react": "^19.0.0", "typescript": "^5.0.0" }, "bin": { "deco": "bin/deco.js" } }, "sha512-ULmBDjpBCjM/w0HikMtgZXHYA46WANE8wvqSbcvfaFGhV8CrpCAtAbSEB8ya/IfOYGd4sVhAFlz+CNarzTBZIA=="], + "@decocms/e2e": ["@decocms/e2e@workspace:packages/e2e"], "@decocms/harness-runner": ["@decocms/harness-runner@workspace:packages/harness-runner"], @@ -3660,6 +3663,8 @@ "@decocms/better-auth/better-call": ["better-call@1.1.5", "", { "dependencies": { "@better-auth/utils": "^0.3.0", "@better-fetch/fetch": "^1.1.4", "rou3": "^0.7.10", "set-cookie-parser": "^2.7.1" }, "peerDependencies": { "zod": "^4.0.0" }, "optionalPeers": ["zod"] }, "sha512-nQJ3S87v6wApbDwbZ++FrQiSiVxWvZdjaO+2v6lZJAG2WWggkB2CziUDjPciz3eAt9TqfRursIQMZIcpkBnvlw=="], + "@decocms/blocks/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "@decocms/native/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "@decocms/sandbox-controller/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], From 98ee981dfee25b40e466508cc051d35d1333413d Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:56:40 -0300 Subject: [PATCH 06/20] feat(daemon): secret guard and ciphertext checks Refuses plaintext in v8 Secret fields and validates ciphertext shape; the v7 secret loader stays exempt. Its unit tests (secrets_test.go) land in the next PR, because they also cover servablePublicKey from methods.go. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../daemon-go/internal/content/ciphertext.go | 149 +++++++++ .../daemon-go/internal/content/secrets.go | 307 ++++++++++++++++++ 2 files changed, 456 insertions(+) create mode 100644 packages/sandbox/daemon-go/internal/content/ciphertext.go create mode 100644 packages/sandbox/daemon-go/internal/content/secrets.go diff --git a/packages/sandbox/daemon-go/internal/content/ciphertext.go b/packages/sandbox/daemon-go/internal/content/ciphertext.go new file mode 100644 index 0000000000..2ef07bd238 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/ciphertext.go @@ -0,0 +1,149 @@ +package content + +// The `secret` block's ciphertext format and the secrets public key, ported +// from `@decocms/blocks` v8/ciphertext.ts: +// +// v1... +// +// each segment canonical unpadded base64url; wrappedKey 256/384/512 bytes, iv +// 12 bytes, ciphertext at least 16 (the GCM tag). + +import ( + "encoding/base64" + "regexp" + "strings" +) + +var ciphertextPattern = regexp.MustCompile(`^v1\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$`) + +// decodeBase64URL decodes canonical unpadded base64url; ok is false otherwise. +func decodeBase64URL(text string) ([]byte, bool) { + if len(text)%4 == 1 { + return nil, false + } + b, err := base64.RawURLEncoding.Strict().DecodeString(text) + if err != nil { + return nil, false + } + return b, base64.RawURLEncoding.EncodeToString(b) == text +} + +// isWellFormedCiphertext reports a well-formed secret ciphertext. +func isWellFormedCiphertext(v any) bool { + text, ok := v.(string) + if !ok || !ciphertextPattern.MatchString(text) { + return false + } + parts := strings.Split(text, ".") + key, ok1 := decodeBase64URL(parts[1]) + iv, ok2 := decodeBase64URL(parts[2]) + ct, ok3 := decodeBase64URL(parts[3]) + if !ok1 || !ok2 || !ok3 { + return false + } + switch len(key) { + case 256, 384, 512: + default: + return false + } + return len(iv) == 12 && len(ct) >= 16 +} + +// isJSWhitespace is JS's \s (WhiteSpace and LineTerminator). +func isJSWhitespace(r rune) bool { + switch r { + case '\t', '\n', '\v', '\f', '\r', ' ', 0xA0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, 0xFEFF: + return true + } + return r >= 0x2000 && r <= 0x200A +} + +func jsTrim(s string) string { + return strings.TrimFunc(s, isJSWhitespace) +} + +var pemBodyPattern = regexp.MustCompile(`^[A-Za-z0-9+/]+={0,2}$`) + +// pemBlock is one `-----BEGIN X-----…-----END X-----` match. +type pemBlock struct { + start, end int + label string + body string +} + +// pemBlocks finds every match of /-----BEGIN ([A-Z ]+)-----([\s\S]*?)-----END \1-----/g. +func pemBlocks(pem string) []pemBlock { + var out []pemBlock + const begin = "-----BEGIN " + from := 0 + for from < len(pem) { + at := strings.Index(pem[from:], begin) + if at < 0 { + break + } + start := from + at + i := start + len(begin) + j := i + for j < len(pem) && (pem[j] == ' ' || pem[j] >= 'A' && pem[j] <= 'Z') { + j++ + } + if j == i || !strings.HasPrefix(pem[j:], "-----") { + from = start + 1 + continue + } + label := pem[i:j] + bodyStart := j + 5 + end := "-----END " + label + "-----" + k := strings.Index(pem[bodyStart:], end) + if k < 0 { + from = start + 1 + continue + } + out = append(out, pemBlock{start: start, end: bodyStart + k + len(end), label: label, body: pem[bodyStart : bodyStart+k]}) + from = bodyStart + k + len(end) + } + return out +} + +// atob is the WHATWG forgiving-base64 decode (whitespace already removed). +func atob(s string) ([]byte, bool) { + if len(s)%4 == 0 { + s = strings.TrimSuffix(s, "=") + s = strings.TrimSuffix(s, "=") + } + if len(s)%4 == 1 || strings.ContainsAny(s, "=") { + return nil, false + } + b, err := base64.RawStdEncoding.DecodeString(s) + return b, err == nil +} + +// publicKeyDerFromPem returns the DER bytes of a single `PUBLIC KEY` PEM +// block, nil for anything else. +func publicKeyDerFromPem(pem string) []byte { + blocks := pemBlocks(pem) + if len(blocks) != 1 || blocks[0].label != "PUBLIC KEY" { + return nil + } + block := blocks[0] + whole := pem[block.start:block.end] + // String.prototype.replace with a string pattern: the first occurrence. + rest := strings.Replace(pem, whole, "", 1) + if jsTrim(rest) != "" { + return nil + } + body := strings.Map(func(r rune) rune { + if isJSWhitespace(r) { + return -1 + } + return r + }, block.body) + if !pemBodyPattern.MatchString(body) { + return nil + } + der, ok := atob(body) + if !ok { + return nil + } + return der +} diff --git a/packages/sandbox/daemon-go/internal/content/secrets.go b/packages/sandbox/daemon-go/internal/content/secrets.go new file mode 100644 index 0000000000..5c005dfc4d --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/secrets.go @@ -0,0 +1,307 @@ +package content + +// The secret guard, ported from `@decocms/blocks/protocol` secrets.ts: a +// `Secret` field (schema `format: "secret"`) must hold a `secret` block with a +// well-formed ciphertext, or a variant block whose every value does; every +// `secret` block anywhere must carry a well-formed ciphertext. A legacy (v7) +// secret loader block (`…/loaders/secret.ts`) is walked without its +// definition: its `encrypted` string already holds the site's own ciphertext. + +import ( + "regexp" + "strconv" + "strings" +) + +const ( + secretBlockType = "secret" + secretFormat = "secret" + lazyType = "lazy" + maxSecretDepth = 512 +) + +var multivariateTypes = map[string]bool{"multivariate": true, "website/flags/multivariate.ts": true} + +var legacySecretLoader = regexp.MustCompile(`(^|/)loaders/secret\.ts$`) + +var definitionRef = regexp.MustCompile("^#/definitions/([^\\n\\r\u2028\u2029]+)$") + +// objectPrototypeKeys are the names `key in object` finds on +// Object.prototype: the secret guard's property lookup sees them on every +// schema object, so they must shadow `additionalProperties` the same way. +var objectPrototypeKeys = map[string]bool{ + "constructor": true, "__defineGetter__": true, "__defineSetter__": true, + "hasOwnProperty": true, "__lookupGetter__": true, "__lookupSetter__": true, + "isPrototypeOf": true, "propertyIsEnumerable": true, "toString": true, + "valueOf": true, "__proto__": true, "toLocaleString": true, +} + +func asObject(v any) (*Object, bool) { + o, ok := v.(*Object) + return o, ok && o != nil +} + +func prop(v any, key string) any { + if o, ok := asObject(v); ok { + x, _ := o.Get(key) + return x + } + return nil +} + +func resolveTypeOf(v any) (string, bool) { + s, ok := prop(v, "__resolveType").(string) + return s, ok +} + +func escapePointer(key string) string { + return strings.ReplaceAll(strings.ReplaceAll(key, "~", "~0"), "/", "~1") +} + +func isSecretFieldSchema(schema any) bool { + f, ok := prop(schema, "format").(string) + return ok && f == secretFormat +} + +// blockIndex maps every manifest block key to its schema, first group wins. +func blockIndex(meta *Object) map[string]any { + index := map[string]any{} + var groups []any + switch b := prop(prop(meta, "manifest"), "blocks").(type) { + case *Object: + for _, k := range b.Keys() { + v, _ := b.Get(k) + groups = append(groups, v) + } + case []any: + groups = b + } + for _, g := range groups { + group, ok := asObject(g) + if !ok { + continue + } + for _, k := range group.Keys() { + if _, seen := index[k]; !seen { + v, _ := group.Get(k) + index[k] = v + } + } + } + return index +} + +type secretWalker struct { + name string + meta *Object + index map[string]any + violations []Violation +} + +func (w *secretWalker) report(pointer, rule, message string) { + p := pointer + w.violations = append(w.violations, Violation{Name: w.name, Pointer: &p, Rule: rule, Message: message}) +} + +// definition looks key up in schema.definitions (non-objects read as nil: +// for the guard, a missing definition and a non-object one are the same). +func (w *secretWalker) definition(key string) any { + switch defs := prop(prop(w.meta, "schema"), "definitions").(type) { + case *Object: + v, _ := defs.Get(key) + return v + case []any: + if n, ok := arrayIndex(key); ok && n < uint64(len(defs)) { + return defs[n] + } + } + return nil +} + +func (w *secretWalker) deref(schema any) any { + current := schema + for hops := 0; hops < 32; hops++ { + ref, ok := prop(current, "$ref").(string) + if _, isObj := asObject(current); !isObj || !ok { + break + } + m := definitionRef.FindStringSubmatch(ref) + if m == nil { + return current + } + key, ok := decodeURIComponent(m[1]) + if !ok { + // decodeURIComponent threw: an unexpected throw, as in JS. + panic(uriError{}) + } + key = strings.ReplaceAll(strings.ReplaceAll(key, "~1", "/"), "~0", "~") + current = w.definition(key) + } + return current +} + +func (w *secretWalker) branches(schema any, depth int) []*Object { + node, ok := asObject(w.deref(schema)) + if !ok || depth > 16 { + return nil + } + out := []*Object{node} + for _, combinator := range []string{"allOf", "anyOf", "oneOf"} { + if list, ok := prop(node, combinator).([]any); ok { + for _, item := range list { + out = append(out, w.branches(item, depth+1)...) + } + } + } + return out +} + +func (w *secretWalker) isSecretField(schema any) bool { + for _, b := range w.branches(schema, 0) { + if isSecretFieldSchema(b) { + return true + } + } + return false +} + +// propertySchema returns the schema of key; found is false where JS gives undefined. +func (w *secretWalker) propertySchema(schema any, key string) (any, bool) { + branches := w.branches(schema, 0) + for _, b := range branches { + if props, ok := asObject(prop(b, "properties")); ok { + if v, own := props.Get(key); own { + return v, true + } + if objectPrototypeKeys[key] { + // An inherited member: never a schema object. + return nil, true + } + } + } + for _, b := range branches { + if ap, ok := asObject(prop(b, "additionalProperties")); ok { + return ap, true + } + } + return nil, false +} + +func (w *secretWalker) itemSchema(schema any) (any, bool) { + for _, b := range w.branches(schema, 0) { + if items, ok := asObject(prop(b, "items")); ok { + return items, true + } + } + return nil, false +} + +func (w *secretWalker) checkSecretValue(value any, pointer string, depth int) { + if depth > maxSecretDepth { + w.report(pointer, "too-deep", "the value is nested too deeply") + return + } + typ, hasType := resolveTypeOf(value) + if hasType && typ == secretBlockType { + w.walkBlock(value.(*Object), pointer, depth) + return + } + if hasType && multivariateTypes[typ] { + variants, ok := prop(value, "variants").([]any) + if !ok { + w.report(pointer, "secret-field", "a Secret field holds a variant block without variants") + return + } + for i, variant := range variants { + at := pointer + "/variants/" + strconv.Itoa(i) + v, ok := asObject(variant) + if !ok { + w.report(at, "secret-field", "a variant must be an object") + continue + } + if rule, has := v.Get("rule"); has { + w.walk(rule, nil, false, at+"/rule", depth+1) + } + inner, _ := v.Get("value") + innerAt := at + "/value" + if t, ok := resolveTypeOf(inner); ok && t == lazyType { + inner = prop(inner, "value") + innerAt += "/value" + } + w.checkSecretValue(inner, innerAt, depth+1) + } + return + } + w.report(pointer, "secret-field", `a Secret field must hold a "secret" block with a well-formed ciphertext, never plain text`) +} + +func (w *secretWalker) walkBlock(block *Object, pointer string, depth int) { + typ, _ := resolveTypeOf(block) + if typ == secretBlockType && !isWellFormedCiphertext(prop(block, "ciphertext")) { + w.report(pointer, "secret-ciphertext", `a "secret" block must carry a well-formed "ciphertext" (v1...)`) + } + var definition any + hasDefinition := false + if w.meta != nil && !legacySecretLoader.MatchString(typ) { + if w.index == nil { + w.index = blockIndex(w.meta) + } + definition, hasDefinition = w.index[typ] + } + w.walkObject(block, definition, hasDefinition, pointer, depth) +} + +func (w *secretWalker) walkObject(object *Object, schema any, hasSchema bool, pointer string, depth int) { + for _, key := range object.Keys() { + if key == "__resolveType" { + continue + } + child, _ := object.Get(key) + var childSchema any + hasChild := false + if hasSchema { + childSchema, hasChild = w.propertySchema(schema, key) + } + w.walk(child, childSchema, hasChild, pointer+"/"+escapePointer(key), depth+1) + } +} + +// walk checks value; hasSchema is false where JS's schema is undefined. +func (w *secretWalker) walk(value any, schema any, hasSchema bool, pointer string, depth int) { + if depth > maxSecretDepth { + w.report(pointer, "too-deep", "the value is nested too deeply") + return + } + if hasSchema && w.isSecretField(schema) { + w.checkSecretValue(value, pointer, depth) + return + } + if list, ok := value.([]any); ok { + var items any + hasItems := false + if hasSchema { + items, hasItems = w.itemSchema(schema) + } + for i, item := range list { + w.walk(item, items, hasItems, pointer+"/"+strconv.Itoa(i), depth+1) + } + return + } + obj, ok := asObject(value) + if !ok { + return + } + if _, typed := resolveTypeOf(obj); typed { + w.walkBlock(obj, pointer, depth) + return + } + w.walkObject(obj, schema, hasSchema, pointer, depth) +} + +// checkSecrets checks one entry against the secret guard. meta is the parsed +// schema; without one only the secret blocks' ciphertexts are checked. +func checkSecrets(name string, entry any, meta *Object) []Violation { + w := &secretWalker{name: name, meta: meta} + w.walk(entry, nil, false, "", 0) + return w.violations +} From 3911ab53b4a4a1db1928058427cdfd4dbf56b15f Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:56:46 -0300 Subject: [PATCH 07/20] feat(daemon): content-protocol RPC (describe, schema.get, blocks.list, blocks.apply) The content-protocol JSON-RPC handler: envelope and batch handling, the four methods, blocks.apply ifMatch and retry, and request body limits. Also adds secrets_test.go (the secret guard from the previous PR plus describe's servablePublicKey). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../daemon-go/internal/content/content.go | 250 +++++++++ .../daemon-go/internal/content/handler.go | 278 ++++++++++ .../daemon-go/internal/content/methods.go | 511 ++++++++++++++++++ .../daemon-go/internal/content/params.go | 144 +++++ .../daemon-go/internal/content/params_test.go | 47 ++ .../sandbox/daemon-go/internal/content/rpc.go | 147 +++++ .../internal/content/secrets_test.go | 227 ++++++++ 7 files changed, 1604 insertions(+) create mode 100644 packages/sandbox/daemon-go/internal/content/content.go create mode 100644 packages/sandbox/daemon-go/internal/content/handler.go create mode 100644 packages/sandbox/daemon-go/internal/content/methods.go create mode 100644 packages/sandbox/daemon-go/internal/content/params.go create mode 100644 packages/sandbox/daemon-go/internal/content/params_test.go create mode 100644 packages/sandbox/daemon-go/internal/content/rpc.go create mode 100644 packages/sandbox/daemon-go/internal/content/secrets_test.go diff --git a/packages/sandbox/daemon-go/internal/content/content.go b/packages/sandbox/daemon-go/internal/content/content.go new file mode 100644 index 0000000000..9b14328e46 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/content.go @@ -0,0 +1,250 @@ +package content + +// Turns a storage snapshot into the entry map, applying the file-name rule: +// decode once, resolve spellings, report the rest. Ported from +// server/content.ts and server/bodyCache.ts. + +import ( + "container/list" + "sort" + "strconv" + "sync" +) + +const ( + maxBlockBytes = 1 * 1024 * 1024 + maxSnapshotReads = 3 + defaultCacheBytes = 32 * 1024 * 1024 +) + +type parsedBody struct { + ok bool + value *Object + kind string // invalid-json | not-an-object | too-large + message string + bytes int +} + +// parseBody parses one stored entry, enforcing the per-entry byte limit. +func parseBody(text string) parsedBody { + bytes := len(text) + if bytes > maxBlockBytes { + return parsedBody{kind: "too-large", bytes: bytes, message: "the file is over " + strconv.Itoa(maxBlockBytes) + " bytes"} + } + v, err := ParseJSON(text) + if err != nil { + // OPEN: the message is this parser's, not V8's. + return parsedBody{kind: "invalid-json", bytes: bytes, message: err.Error()} + } + o, ok := asObject(v) + if !ok { + return parsedBody{kind: "not-an-object", bytes: bytes, message: "the file doesn't hold a JSON object"} + } + return parsedBody{ok: true, value: o, bytes: bytes} +} + +// bodyCache is a bounded LRU of parsed bodies keyed by file and version. +type bodyCache struct { + mu sync.Mutex + maxBytes int + total int + order *list.List // front: oldest + entries map[string]*list.Element +} + +type cacheItem struct { + key string + body parsedBody +} + +func newBodyCache(maxBytes int) *bodyCache { + return &bodyCache{maxBytes: maxBytes, order: list.New(), entries: map[string]*list.Element{}} +} + +func (c *bodyCache) get(file, version string) (parsedBody, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[file+"\x00"+version] + if !ok { + return parsedBody{}, false + } + c.order.MoveToBack(e) + return e.Value.(*cacheItem).body, true +} + +func (c *bodyCache) set(file, version string, body parsedBody) { + if body.bytes > c.maxBytes { + return + } + c.mu.Lock() + defer c.mu.Unlock() + key := file + "\x00" + version + if e, ok := c.entries[key]; ok { + c.total -= e.Value.(*cacheItem).body.bytes + c.order.Remove(e) + } + c.entries[key] = c.order.PushBack(&cacheItem{key: key, body: body}) + c.total += body.bytes + for c.total > c.maxBytes { + oldest := c.order.Front() + item := oldest.Value.(*cacheItem) + c.order.Remove(oldest) + delete(c.entries, item.key) + c.total -= item.body.bytes + } +} + +type loadedEntry struct { + File string + Version string + Value *Object // nil when the body wasn't needed +} + +type namedEntry struct { + Name string + loadedEntry +} + +type diagnostic struct { + File string + Kind string + Message string + Name string // shadowed only + Winner string // shadowed only +} + +func (d diagnostic) json() *Object { + o := NewObject("file", d.File, "kind", d.Kind) + if d.Kind == "shadowed" { + o.Set("name", d.Name) + o.Set("winner", d.Winner) + } + o.Set("message", d.Message) + return o +} + +type loadedContent struct { + snapshot storageSnapshot + // entries in file-name order of the winning files + entries []namedEntry + // every saved-block file by spelling key, keys in first-seen order + groupOrder []string + groups map[string][]storageFile + diagnostics []diagnostic + moved bool +} + +func (h *Handler) loadContent(snap storageSnapshot, readAll bool) (*loadedContent, error) { + var files []storageFile + for _, f := range snap.Files { + if IsBlockFileName(f.File) { + files = append(files, f) + } + } + c := &loadedContent{snapshot: snap, groups: map[string][]storageFile{}} + for _, f := range files { + key, _ := FullyDecodeFileName(f.File) + if _, ok := c.groups[key]; !ok { + c.groupOrder = append(c.groupOrder, key) + } + c.groups[key] = append(c.groups[key], f) + } + + parsed := map[string]parsedBody{} + var toRead []storageFile + for _, key := range c.groupOrder { + group := c.groups[key] + if !readAll && len(group) < 2 { + continue + } + for _, f := range group { + if f.Size > maxBlockBytes { + parsed[f.File] = parsedBody{kind: "too-large", bytes: int(f.Size), message: "the file is over " + strconv.Itoa(maxBlockBytes) + " bytes"} + continue + } + if hit, ok := h.cache.get(f.File, f.Version); ok { + parsed[f.File] = hit + } else { + toRead = append(toRead, f) + } + } + } + if len(toRead) > 0 { + names := make([]string, len(toRead)) + for i, f := range toRead { + names[i] = f.File + } + bodies, err := h.store.readFiles(names) + if err != nil { + return nil, err + } + for _, f := range toRead { + read, ok := bodies[f.File] + if !ok { + c.moved = true // vanished since the snapshot + continue + } + body := parseBody(read.Text) + // Cached under the version of the bytes actually read. + h.cache.set(f.File, read.Version, body) + if read.Version != f.Version { + c.moved = true + continue + } + parsed[f.File] = body + } + } + + var candidates []*spellingCandidate + for _, f := range files { + body, ok := parsed[f.File] + if !ok { + if !readAll { + candidates = append(candidates, &spellingCandidate{File: f.File, Version: f.Version}) + } + continue + } + if !body.ok { + c.diagnostics = append(c.diagnostics, diagnostic{File: f.File, Kind: body.kind, Message: body.message}) + continue + } + candidates = append(candidates, &spellingCandidate{File: f.File, Version: f.Version, HasPath: entryHasPath(body.value), Value: body.value}) + } + + for _, r := range resolveSpellings(candidates) { + c.entries = append(c.entries, namedEntry{Name: r.Name, loadedEntry: loadedEntry{File: r.Winner.File, Version: r.Winner.Version, Value: r.Winner.Value}}) + for _, loser := range r.Shadowed { + c.diagnostics = append(c.diagnostics, diagnostic{ + File: loser.File, Kind: "shadowed", Name: r.Name, Winner: r.Winner.File, + Message: `another spelling of "` + r.Name + `" wins: ` + r.Winner.File, + }) + } + } + sort.SliceStable(c.diagnostics, func(i, j int) bool { + return compareJS(c.diagnostics[i].File, c.diagnostics[j].File) < 0 + }) + return c, nil +} + +// loadCurrentContent takes a snapshot and loads it, taking a new one when a +// file changed while its body was read. shortCircuit ends the read right +// after the snapshot (a conditional read that's "not modified"). +func (h *Handler) loadCurrentContent(readAll bool, shortCircuit func(storageSnapshot) bool) (storageSnapshot, *loadedContent, error) { + for read := 1; read <= maxSnapshotReads; read++ { + snap, err := h.store.snapshot() + if err != nil { + return snap, nil, err + } + if shortCircuit != nil && shortCircuit(snap) { + return snap, nil, nil + } + c, err := h.loadContent(snap, readAll) + if err != nil { + return snap, nil, err + } + if !c.moved { + return snap, c, nil + } + } + return storageSnapshot{}, nil, errUnavailable("saved blocks kept changing while being read; retry shortly", 250) +} diff --git a/packages/sandbox/daemon-go/internal/content/handler.go b/packages/sandbox/daemon-go/internal/content/handler.go new file mode 100644 index 0000000000..079dd57059 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/handler.go @@ -0,0 +1,278 @@ +// Package content serves the Deco content protocol (`deco-content` v1: +// JSON-RPC 2.0 with describe, schema.get, blocks.list and blocks.apply, plus +// `PUT …/assets/` uploads) over the sandbox's working tree. +// +// It is a port of the TypeScript reference in `@decocms/blocks/protocol` +// (server/, storage/fs, keys, secrets), behaviour-identical by design: same +// files and bytes, same error codes and shapes, same secret guard. The +// conformance suite `@decocms/blocks/protocol/conformance` runs against it in +// daemon-e2e, so drift fails CI. Port changes from the TS source; don't invent. +package content + +import ( + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "strings" + "sync" +) + +const maxRequestBytes = 8 * 1024 * 1024 + +// Options configure a Handler. +type Options struct { + Store *FSStore + ServerName string + ServerVersion string + // OnCommit is called after blocks.apply lands, with the block files + // (inside .deco/blocks) it wrote or deleted. + OnCommit func(files []string) + // OnAsset is called after an upload, with the stored file name. + OnAsset func(name string) + // Logf receives errors that become Internal errors. + Logf func(format string, args ...any) +} + +// Handler serves the protocol and uploads for one app root. +type Handler struct { + store *FSStore + serverName string + serverVersion string + pollIntervalMs int + maxCommitAttempts int + retryMinMs, retryMaxMs int + cache *bodyCache + onCommit func([]string) + onAsset func(string) + logFn func(string, ...any) + schemaMu sync.Mutex + schemaVersion string + schemaMeta *Object +} + +// NewHandler builds a handler over a filesystem storage. +func NewHandler(o Options) *Handler { + h := &Handler{ + store: o.Store, + serverName: o.ServerName, + serverVersion: o.ServerVersion, + pollIntervalMs: defaultPollMs, + maxCommitAttempts: defaultCommitTries, + retryMinMs: 50, + retryMaxMs: 200, + cache: newBodyCache(defaultCacheBytes), + onCommit: o.OnCommit, + onAsset: o.OnAsset, + logFn: o.Logf, + } + if h.serverName == "" { + h.serverName = "deco-blocks" + } + if h.serverVersion == "" { + h.serverVersion = "unknown" + } + return h +} + +func (h *Handler) logf(format string, args ...any) { + if h.logFn != nil { + h.logFn(format, args...) + return + } + slog.Warn("content protocol", "msg", fmt.Sprintf(format, args...)) +} + +// ---------------------------------------------------------------- HTTP plumbing + +var errBodyTooLarge = errors.New("body too large") + +type bodyEncodingError struct{ msg string } + +func (e *bodyEncodingError) Error() string { return e.msg } + +func readLimited(r io.Reader, limit int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > limit { + return nil, errBodyTooLarge + } + return b, nil +} + +// readBody reads at most limit bytes, decompressing a gzip body (the limit +// applies after decompression, so gzip can't bypass it). +func readBody(r *http.Request, limit int64) ([]byte, error) { + encoding := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Encoding"))) + if encoding == "" { + encoding = "identity" + } + if encoding == "identity" && r.ContentLength > limit { + return nil, errBodyTooLarge + } + if r.Body == nil || r.Body == http.NoBody { + return []byte{}, nil + } + if encoding == "identity" { + return readLimited(r.Body, limit) + } + if encoding != "gzip" { + return nil, &bodyEncodingError{msg: `unsupported Content-Encoding "` + encoding + `"`} + } + zr, err := gzip.NewReader(r.Body) + if err != nil { + return nil, &bodyEncodingError{msg: "the gzip body is corrupt"} + } + b, err := readLimited(zr, limit) + if err != nil { + if errors.Is(err, errBodyTooLarge) { + return nil, err + } + return nil, &bodyEncodingError{msg: "the gzip body is corrupt"} + } + return b, nil +} + +// jsNumber is JS's Number(string), for the q-values of Accept-Encoding. +func jsNumber(s string) float64 { + s = jsTrim(s) + if s == "" { + return 0 + } + switch s { + case "Infinity", "+Infinity": + return 1 + case "-Infinity": + return -1 + } + if len(s) > 2 && s[0] == '0' && strings.ContainsRune("xXoObB", rune(s[1])) { + base := map[byte]int{'x': 16, 'X': 16, 'o': 8, 'O': 8, 'b': 2, 'B': 2}[s[1]] + n, err := strconv.ParseUint(s[2:], base, 64) + if err != nil { + return -1 + } + return float64(n) + } + for _, c := range s { + if !(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-') { + return -1 // NaN: never > 0 + } + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + var ne *strconv.NumError + if errors.As(err, &ne) && errors.Is(ne.Err, strconv.ErrRange) { + return f + } + return -1 + } + return f +} + +func acceptsGzip(r *http.Request) bool { + header := strings.Join(r.Header.Values("Accept-Encoding"), ", ") + if header == "" { + return false + } + for _, part := range strings.Split(header, ",") { + pieces := strings.Split(strings.ToLower(strings.TrimSpace(part)), ";") + coding := strings.TrimSpace(pieces[0]) + if coding != "gzip" && coding != "*" { + continue + } + q, hasQ := "", false + for _, p := range pieces[1:] { + if p = strings.TrimSpace(p); strings.HasPrefix(p, "q=") { + q, hasQ = p[2:], true + break + } + } + if !hasQ || jsNumber(q) > 0 { + return true + } + } + return false +} + +const gzipThresholdBytes = 1024 + +// writeJSON is the TS jsonResponse: JSON, no-store, gzip when accepted. +func writeJSON(w http.ResponseWriter, r *http.Request, body string, status int, extra map[string]string) { + h := w.Header() + h.Set("Content-Type", "application/json; charset=utf-8") + h.Set("Cache-Control", "no-store") + h.Set("Vary", "Accept-Encoding") + for k, v := range extra { + h.Set(k, v) + } + if len(body) < gzipThresholdBytes || !acceptsGzip(r) { + h.Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(status) + io.WriteString(w, body) + return + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + io.WriteString(zw, body) + zw.Close() + h.Set("Content-Encoding", "gzip") + h.Set("Content-Length", strconv.Itoa(buf.Len())) + w.WriteHeader(status) + w.Write(buf.Bytes()) +} + +func errorBody(e *ProtocolError) string { + return Stringify(NewObject("jsonrpc", "2.0", "id", nil, "error", e.JSON()), 0) +} + +func isJSONContentType(r *http.Request) bool { + values := r.Header.Values("Content-Type") + if len(values) == 0 { + return false + } + t := strings.Join(values, ", ") + return strings.ToLower(strings.TrimSpace(strings.SplitN(t, ";", 2)[0])) == "application/json" +} + +// ServeRPC serves the protocol endpoint. +func (h *Handler) ServeRPC(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, r, errorBody(errInvalidRequest("use POST")), 405, map[string]string{"Allow": "POST"}) + return + } + if !isJSONContentType(r) { + writeJSON(w, r, errorBody(errInvalidRequest("Content-Type must be application/json")), 415, nil) + return + } + raw, err := readBody(r, maxRequestBytes) + if err != nil { + var enc *bodyEncodingError + switch { + case errors.Is(err, errBodyTooLarge): + writeJSON(w, r, errorBody(errLimitExceeded("the request body is over "+strconv.Itoa(maxRequestBytes)+" bytes", NewObject("limit", "maxRequestBytes"))), 413, nil) + case errors.As(err, &enc): + writeJSON(w, r, errorBody(errInvalidRequest(enc.msg)), 415, nil) + default: + // The client went away mid-body. + writeJSON(w, r, errorBody(errInternal()), 500, nil) + } + return + } + text, ok := decodeUTF8(raw, true, true) + if !ok { + writeJSON(w, r, errorBody(errParse()), 200, nil) + return + } + body, err := ParseJSON(text) + if err != nil { + writeJSON(w, r, errorBody(errParse()), 200, nil) + return + } + writeJSON(w, r, h.dispatch(body), 200, nil) +} diff --git a/packages/sandbox/daemon-go/internal/content/methods.go b/packages/sandbox/daemon-go/internal/content/methods.go new file mode 100644 index 0000000000..55d0308960 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/methods.go @@ -0,0 +1,511 @@ +package content + +// The four methods, ported from server/methods/read.ts and apply.ts. + +import ( + "math/rand" + "strconv" + "strings" + "time" +) + +const ( + maxOpsPerApply = 500 + maxPublicKeyBytes = 16 * 1024 + assetsURLPrefix = "/assets/" + defaultPollMs = 2000 + defaultCommitTries = 3 +) + +// servablePublicKey is the key describe may serve: a single PUBLIC KEY PEM +// block of at most 16 KiB; anything else is never broadcast. +func servablePublicKey(text *string) *string { + if text == nil || len(*text) > maxPublicKeyBytes || strings.Contains(*text, "PRIVATE KEY") { + return nil + } + if publicKeyDerFromPem(*text) == nil { + return nil + } + return text +} + +func (h *Handler) describe() (any, error) { + desc := h.store.describe() + stored, err := h.store.readSecretsPublicKey() + if err != nil { + return nil, err + } + publicKey := servablePublicKey(stored) + if stored != nil && publicKey == nil { + h.logf(".deco/secrets.pub isn't a single PUBLIC KEY PEM block; describe reports no key") + } + var secrets any + if publicKey != nil { + secrets = NewObject("publicKey", *publicKey) + } + return NewObject( + "protocol", "deco-content", + "version", NewObject("major", 1.0, "minor", 0.0), + "server", NewObject("name", h.serverName, "version", h.serverVersion), + "kind", "working-tree", + "readOnly", false, + "root", desc.Root, + "schemaFormat", "deco-meta@1", + "pollIntervalMs", float64(h.pollIntervalMs), + "preview", nil, + "assets", NewObject("dir", desc.AssetsDir, "urlPrefix", assetsURLPrefix, "maxBytes", float64(desc.AssetsMaxBytes)), + "secrets", secrets, + ), nil +} + +// parseSchema parses schema text; a file caught mid-write is Unavailable, +// never served torn. +func parseSchema(text string) (*Object, error) { + v, err := ParseJSON(text) + if err != nil { + return nil, errUnavailable("the schema file is being written; retry shortly", 500) + } + o, ok := asObject(v) + if !ok { + return nil, errUnavailable("the schema file doesn't hold a JSON object", 500) + } + return o, nil +} + +func ifNoneMatchOf(params any) (string, bool) { + v, ok := prop(params, "ifNoneMatch").(string) + return v, ok +} + +func (h *Handler) schemaGet(params any) (any, error) { + stored, err := h.store.readSchema() + if err != nil { + return nil, err + } + if stored == nil { + // No schema yet is a state, not an error; the snapshot still refuses a + // site without a .deco folder. + if _, err := h.store.snapshot(); err != nil { + return nil, err + } + return NewObject("notModified", false, "version", nil, "resolvedRef", nil, "schema", nil), nil + } + if inm, ok := ifNoneMatchOf(params); ok && inm == stored.Version { + return NewObject("notModified", true, "version", stored.Version), nil + } + meta, err := h.parsedSchemaFor(stored) + if err != nil { + return nil, err + } + return NewObject("notModified", false, "version", stored.Version, "resolvedRef", nil, "schema", meta), nil +} + +// parsedSchemaFor parses a stored schema, reusing the last parse of the same version. +func (h *Handler) parsedSchemaFor(stored *storedSchema) (*Object, error) { + h.schemaMu.Lock() + if h.schemaVersion == stored.Version && h.schemaMeta != nil { + meta := h.schemaMeta + h.schemaMu.Unlock() + return meta, nil + } + h.schemaMu.Unlock() + meta, err := parseSchema(stored.Text) + if err != nil { + return nil, err + } + h.schemaMu.Lock() + h.schemaVersion, h.schemaMeta = stored.Version, meta + h.schemaMu.Unlock() + return meta, nil +} + +func (h *Handler) blocksList(params any) (any, error) { + inm, hasINM := ifNoneMatchOf(params) + snap, content, err := h.loadCurrentContent(true, func(s storageSnapshot) bool { + return hasINM && inm == s.Revision + }) + if err != nil { + return nil, err + } + if content == nil { + return NewObject("notModified", true, "revision", snap.Revision, "resolvedRef", nil), nil + } + blocks, versions := NewObject(), NewObject() + for _, e := range content.entries { + if e.Value == nil { + continue + } + blocks.Set(e.Name, e.Value) + versions.Set(e.Name, e.Version) + } + diagnostics := make([]any, len(content.diagnostics)) + for i, d := range content.diagnostics { + diagnostics[i] = d.json() + } + return NewObject( + "notModified", false, + "revision", snap.Revision, + "resolvedRef", nil, + "blocks", blocks, + "versions", versions, + "diagnostics", diagnostics, + ), nil +} + +// ---------------------------------------------------------------- blocks.apply + +type setOp struct { + name string + value any +} + +type guardOp struct { + name string + version any // a string, or nil (must not exist); unvalidated under __proto__ +} + +type normalizedApply struct { + set []setOp + delete []string // names to delete that aren't also set + ifMatch []guardOp +} + +func normalize(params any) (*normalizedApply, error) { + a := &normalizedApply{} + setNames := map[string]bool{} + if set, ok := asObject(prop(params, "set")); ok { + for _, k := range set.Keys() { + v, _ := set.Get(k) + a.set = append(a.set, setOp{name: k, value: v}) + setNames[k] = true + } + } + if list, ok := prop(params, "delete").([]any); ok { + seen := map[string]bool{} + for _, item := range list { + name := item.(string) + if seen[name] { + continue + } + seen[name] = true + if !setNames[name] { + a.delete = append(a.delete, name) + } + } + } + ops := len(a.set) + len(a.delete) + if ops > maxOpsPerApply { + return nil, errLimitExceeded(strconv.Itoa(ops)+" names in one blocks.apply; the limit is "+strconv.Itoa(maxOpsPerApply), NewObject("limit", "maxOpsPerApply")) + } + if guards, ok := asObject(prop(params, "ifMatch")); ok { + for _, k := range guards.Keys() { + v, _ := guards.Get(k) + a.ifMatch = append(a.ifMatch, guardOp{name: k, version: v}) + } + } + return a, nil +} + +func nameViolations(name string, vs []NameViolation) []Violation { + out := make([]Violation, len(vs)) + for i, v := range vs { + out[i] = Violation{Name: name, Rule: v.Reason, Message: v.Message} + } + return out +} + +// validateStatic checks everything that doesn't depend on stored content. +func validateStatic(a *normalizedApply, meta *Object) ([]Violation, map[string]string) { + var violations []Violation + bodies := map[string]string{} + bySpelling := map[string]string{} + for _, op := range a.set { + violations = append(violations, nameViolations(op.name, checkBlockName(op.name, nil))...) + key := SpellingKey(op.name) + if twin, ok := bySpelling[key]; ok { + violations = append(violations, Violation{Name: op.name, Rule: "spelling-collision", Message: `"` + op.name + `" and "` + twin + `" are spellings of the same entry`}) + } else { + bySpelling[key] = op.name + } + if _, ok := asObject(op.value); !ok { + violations = append(violations, Violation{Name: op.name, Rule: "not-an-object", Message: "an entry must be a JSON object"}) + continue + } + body := SerializeBlock(op.value) + if len(body) > maxBlockBytes { + violations = append(violations, Violation{Name: op.name, Rule: "too-large", Message: "the entry is over " + strconv.Itoa(maxBlockBytes) + " bytes"}) + continue + } + bodies[op.name] = body + violations = append(violations, checkSecrets(op.name, op.value, meta)...) + } + for _, name := range a.delete { + violations = append(violations, nameViolations(name, checkDeletedName(name))...) + } + return violations, bodies +} + +type spelled struct { + name string + entry loadedEntry +} + +func entriesBySpelling(c *loadedContent) map[string]spelled { + out := map[string]spelled{} + for _, e := range c.entries { + key, _ := FullyDecodeFileName(e.File) + out[key] = spelled{name: e.Name, entry: e.loadedEntry} + } + return out +} + +// validateAgainst checks the rules that depend on the existing entries. +func validateAgainst(c *loadedContent, bySpelling map[string]spelled, a *normalizedApply) []Violation { + var violations []Violation + existing := make([]string, 0, len(c.entries)) + for _, e := range c.entries { + existing = append(existing, e.Name) + } + for _, op := range a.set { + if _, ok := bySpelling[SpellingKey(op.name)]; ok { + continue + } + names := append([]string(nil), existing...) + for _, other := range a.set { + if other.name != op.name { + names = append(names, other.name) + } + } + for _, v := range checkBlockName(op.name, names) { + if v.Reason == "case-collision" { + violations = append(violations, Violation{Name: op.name, Rule: v.Reason, Message: v.Message}) + } + } + } + return violations +} + +// orderedSet keeps insertion order, like a JS Set. +type orderedSet struct { + items []string + has map[string]bool +} + +func (s *orderedSet) add(v string) { + if s.has == nil { + s.has = map[string]bool{} + } + if !s.has[v] { + s.has[v] = true + s.items = append(s.items, v) + } +} + +func (s *orderedSet) remove(v string) { + if !s.has[v] { + return + } + delete(s.has, v) + for i, x := range s.items { + if x == v { + s.items = append(s.items[:i], s.items[i+1:]...) + return + } + } +} + +type applyPlan struct { + put []filePut + delete []string + expected map[string]*string +} + +// plan writes encode(name) and deletes every other spelling; only guarded +// entries (all their spellings) become commit expectations. +func plan(c *loadedContent, bySpelling map[string]spelled, a *normalizedApply, bodies map[string]string) applyPlan { + versionOf := map[string]string{} + for _, f := range c.snapshot.Files { + if IsBlockFileName(f.File) { + versionOf[f.File] = f.Version + } + } + groupFiles := func(name string) []string { + var out []string + for _, f := range c.groups[SpellingKey(name)] { + out = append(out, f.File) + } + return out + } + p := applyPlan{expected: map[string]*string{}} + putIndex := map[string]int{} + var deletes orderedSet + for _, op := range a.set { + file := BlockFileName(op.name) + if i, ok := putIndex[file]; ok { + p.put[i].Content = bodies[op.name] + } else { + putIndex[file] = len(p.put) + p.put = append(p.put, filePut{File: file, Content: bodies[op.name]}) + } + for _, other := range groupFiles(op.name) { + if other != file { + deletes.add(other) + } + } + } + for _, name := range a.delete { + file := BlockFileName(name) + if _, ok := versionOf[file]; ok { + deletes.add(file) + } + if _, ok := bySpelling[SpellingKey(name)]; ok { + for _, other := range groupFiles(name) { + deletes.add(other) + } + } + } + for _, f := range p.put { + deletes.remove(f.File) + } + for _, g := range a.ifMatch { + for _, file := range append([]string{BlockFileName(g.name)}, groupFiles(g.name)...) { + if IsBlockFileName(file) { + if v, ok := versionOf[file]; ok { + v := v + p.expected[file] = &v + } else { + p.expected[file] = nil + } + } + } + } + p.delete = deletes.items + return p +} + +// checkIfMatch compares each guard against the entry under any spelling. +func checkIfMatch(bySpelling map[string]spelled, a *normalizedApply) *Object { + mismatches := NewObject() + failed := false + for _, g := range a.ifMatch { + var actual any + if e, ok := bySpelling[SpellingKey(g.name)]; ok { + actual = e.entry.Version + } + if actual != g.version { + // `mismatches.__proto__ = …` sets the prototype in JS: no own entry. + if g.name != "__proto__" { + mismatches.Set(g.name, NewObject("expected", g.version, "actual", actual)) + } + failed = true + } + } + if failed { + return mismatches + } + return nil +} + +// resultFromVersions reports every name written or deleted, plus null for +// each other spelling the commit deleted. +func resultFromVersions(a *normalizedApply, revision string, fileVersions map[string]string, deleted []string) *Object { + versions := NewObject() + touched := map[string]bool{} + for _, op := range a.set { + if v, ok := fileVersions[BlockFileName(op.name)]; ok { + versions.Set(op.name, v) + } else { + versions.Set(op.name, nil) + } + touched[SpellingKey(op.name)] = true + } + for _, name := range a.delete { + versions.Set(name, nil) + touched[SpellingKey(name)] = true + } + for _, file := range deleted { + name := BlockNameFromFile(file) + key, _ := FullyDecodeFileName(file) + if !versions.Has(name) && touched[key] { + versions.Set(name, nil) + } + } + return NewObject("revision", revision, "versions", versions) +} + +func (h *Handler) loadSchemaForApply() (*string, *Object, error) { + stored, err := h.store.readSchema() + if err != nil { + return nil, nil, err + } + if stored == nil { + return nil, nil, nil + } + meta, err := h.parsedSchemaFor(stored) + if err != nil { + return nil, nil, err + } + v := stored.Version + return &v, meta, nil +} + +func (h *Handler) backoff(attempt int) { + ms := (float64(h.retryMinMs) + rand.Float64()*float64(max(0, h.retryMaxMs-h.retryMinMs))) * float64(attempt) + if ms > 0 { + time.Sleep(time.Duration(ms * float64(time.Millisecond))) + } +} + +func (h *Handler) blocksApply(params any) (any, error) { + a, err := normalize(params) + if err != nil { + return nil, err + } + for attempt := 1; attempt <= h.maxCommitAttempts; attempt++ { + schemaVersion, meta, err := h.loadSchemaForApply() + if err != nil { + return nil, err + } + violations, bodies := validateStatic(a, meta) + snap, content, err := h.loadCurrentContent(false, nil) + if err != nil { + return nil, err + } + bySpelling := entriesBySpelling(content) + violations = append(violations, validateAgainst(content, bySpelling, a)...) + if len(violations) > 0 { + return nil, errInvalidBlock(violations) + } + if mismatches := checkIfMatch(bySpelling, a); mismatches != nil { + return nil, errConflict(mismatches) + } + p := plan(content, bySpelling, a, bodies) + if len(p.put) == 0 && len(p.delete) == 0 { + return resultFromVersions(a, snap.Revision, nil, nil), nil + } + result, err := h.store.commit(commitAttempt{ + Put: p.put, + Delete: p.delete, + Expected: p.expected, + CheckSchema: len(a.set) > 0, + ExpectedSchemaVersion: schemaVersion, + }) + if err != nil { + return nil, err + } + if !result.Stale { + if h.onCommit != nil { + files := make([]string, 0, len(p.put)+len(p.delete)) + for _, f := range p.put { + files = append(files, f.File) + } + h.onCommit(append(files, p.delete...)) + } + return resultFromVersions(a, result.Revision, result.Versions, p.delete), nil + } + if attempt < h.maxCommitAttempts { + h.backoff(attempt) + } + } + return nil, errUnavailable("storage kept changing; gave up after "+strconv.Itoa(h.maxCommitAttempts)+" commit attempts", 250) +} diff --git a/packages/sandbox/daemon-go/internal/content/params.go b/packages/sandbox/daemon-go/internal/content/params.go new file mode 100644 index 0000000000..b7739d5792 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/params.go @@ -0,0 +1,144 @@ +package content + +// Parameter validation for the four methods, ported from params.ts (a zod 4 +// strictObject per method). Unknown parameters are refused, so a guard the +// server doesn't understand never turns into an unguarded write. Messages +// follow zod 4's wording; like zod, a `__proto__` key is never validated. + +import ( + "strconv" + "strings" +) + +const maxOpaqueLength = 1024 + +func jsTypeName(v any) string { + switch v.(type) { + case nil: + return "null" + case bool: + return "boolean" + case float64: + return "number" + case string: + return "string" + case []any: + return "array" + } + return "object" +} + +type issues []string + +func (is *issues) add(path, message string) { + *is = append(*is, path+": "+message) +} + +// opaque is z.string().min(1).max(1024) (nullable: also null). +func (is *issues) opaque(path string, v any, nullable bool) { + if v == nil && nullable { + return + } + if s, ok := v.(string); ok { + n := jsLength(s) + if n < 1 { + is.add(path, "Too small: expected string to have >=1 characters") + } else if n > maxOpaqueLength { + is.add(path, "Too big: expected string to have <=1024 characters") + } + return + } + is.add(path, "Invalid input: expected string, received "+jsTypeName(v)) + // zod 4 still runs the length checks on anything with a length. + if list, ok := v.([]any); ok { + if len(list) < 1 { + is.add(path, "Too small: expected array to have >=1 items") + } else if len(list) > maxOpaqueLength { + is.add(path, "Too big: expected array to have <=1024 items") + } + } +} + +var paramShapes = map[string][]string{ + "describe": {}, + "schema.get": {"ifNoneMatch"}, + "blocks.list": {"ifNoneMatch"}, + "blocks.apply": {"set", "delete", "ifMatch"}, +} + +// validateParams checks params for method (present is false when the request +// had no `params`, which reads as {}). +func validateParams(method string, params any, present bool) *ProtocolError { + if !present { + return nil + } + obj, ok := asObject(params) + if !ok { + return errInvalidParams("params must be an object") + } + shape := paramShapes[method] + var is issues + for _, key := range shape { + v, has := obj.Get(key) + if !has { + continue + } + switch key { + case "ifNoneMatch": + is.opaque(key, v, false) + case "set": + if _, ok := asObject(v); !ok { + is.add(key, "Invalid input: expected record, received "+jsTypeName(v)) + } + case "delete": + list, ok := v.([]any) + if !ok { + is.add(key, "Invalid input: expected array, received "+jsTypeName(v)) + break + } + for i, item := range list { + if _, ok := item.(string); !ok { + is.add(key+"."+strconv.Itoa(i), "Invalid input: expected string, received "+jsTypeName(item)) + } + } + case "ifMatch": + record, ok := asObject(v) + if !ok { + is.add(key, "Invalid input: expected record, received "+jsTypeName(v)) + break + } + for _, k := range record.Keys() { + if k == "__proto__" { + continue + } + item, _ := record.Get(k) + is.opaque(key+"."+k, item, true) + } + } + } + var unknown []string + for _, k := range obj.Keys() { + if k == "__proto__" { + continue + } + known := false + for _, s := range shape { + if s == k { + known = true + break + } + } + if !known { + unknown = append(unknown, `"`+k+`"`) + } + } + if len(unknown) == 1 { + is = append(is, "unknown parameter "+unknown[0]) + } else if len(unknown) > 1 { + is = append(is, "unknown parameters "+strings.Join(unknown, ", ")) + } + if len(is) > 0 { + return errInvalidParams(strings.Join(is, "; ")) + } + return nil +} diff --git a/packages/sandbox/daemon-go/internal/content/params_test.go b/packages/sandbox/daemon-go/internal/content/params_test.go new file mode 100644 index 0000000000..570802c4d6 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/params_test.go @@ -0,0 +1,47 @@ +package content + +import "testing" + +// Ported from params.test.ts; the messages are zod 4's, captured from the +// published package. +func TestValidateParams(t *testing.T) { + ok := func(method, params string) { + t.Helper() + if err := validateParams(method, mustParse(t, params), true); err != nil { + t.Errorf("%s %s: %v", method, params, err.Message) + } + } + fail := func(method, params, message string) { + t.Helper() + err := validateParams(method, mustParse(t, params), true) + if err == nil || err.Code != CodeInvalidParams || err.Message != message { + t.Errorf("%s %s:\n got %+v\nwant %q", method, params, err, message) + } + } + if validateParams("describe", nil, false) != nil { + t.Error("absent params read as {}") + } + ok("describe", `{}`) + fail("describe", `null`, "params must be an object") + fail("describe", `[]`, "params must be an object") + fail("describe", `"x"`, "params must be an object") + fail("describe", `{"a":1}`, `unknown parameter "a"`) + fail("describe", `{"a":1,"b":2}`, `unknown parameters "a", "b"`) + ok("describe", `{"__proto__":5}`) + fail("blocks.list", `{"ifNoneMatch":5}`, "ifNoneMatch: Invalid input: expected string, received number") + fail("blocks.list", `{"ifNoneMatch":""}`, "ifNoneMatch: Too small: expected string to have >=1 characters") + fail("blocks.list", `{"ifNoneMatch":[]}`, "ifNoneMatch: Invalid input: expected string, received array; ifNoneMatch: Too small: expected array to have >=1 items") + fail("blocks.list", `{"zz":1,"ifNoneMatch":5}`, `ifNoneMatch: Invalid input: expected string, received number; unknown parameter "zz"`) + fail("schema.get", `{"ifNoneMatch":null}`, "ifNoneMatch: Invalid input: expected string, received null") + ok("schema.get", `{"ifNoneMatch":"v"}`) + fail("blocks.apply", `{"set":[]}`, "set: Invalid input: expected record, received array") + fail("blocks.apply", `{"delete":[1,"a",null]}`, "delete.0: Invalid input: expected string, received number; delete.2: Invalid input: expected string, received null") + fail("blocks.apply", `{"ifMatch":{"a.b":5,"10":true,"2":null,"c":""}}`, + "ifMatch.10: Invalid input: expected string, received boolean; ifMatch.a.b: Invalid input: expected string, received number; ifMatch.c: Too small: expected string to have >=1 characters") + fail("blocks.apply", `{"set":1,"delete":2,"ifMatch":3,"q":1}`, + `set: Invalid input: expected record, received number; delete: Invalid input: expected array, received number; ifMatch: Invalid input: expected record, received number; unknown parameter "q"`) + for _, removed := range []string{"ref", "refs", "requestKey", "ifSchemaMatch", "ifUnmodifiedSince"} { + fail("blocks.apply", `{"set":{},"`+removed+`":"x"}`, `unknown parameter "`+removed+`"`) + } + ok("blocks.apply", `{"set":{"a":[]},"delete":["a"],"ifMatch":{"a":null,"__proto__":5}}`) +} diff --git a/packages/sandbox/daemon-go/internal/content/rpc.go b/packages/sandbox/daemon-go/internal/content/rpc.go new file mode 100644 index 0000000000..cc0009dd0f --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/rpc.go @@ -0,0 +1,147 @@ +package content + +// The JSON-RPC 2.0 layer, ported from server/rpc.ts: every request needs an +// id; a batch runs in order, holds at most 10 calls and isn't atomic. + +import ( + "errors" + "strings" +) + +const maxBatchCalls = 10 + +var methodNames = map[string]bool{"describe": true, "schema.get": true, "blocks.list": true, "blocks.apply": true} + +var envelopeMembers = map[string]bool{"jsonrpc": true, "id": true, "method": true, "params": true} + +type envelope struct { + id any // string or float64 + method string + params any + hasParams bool +} + +func parseEnvelope(value any) (*envelope, *ProtocolError, any) { + raw, isObject := asObject(value) + var id any + if raw != nil { + switch v := prop(raw, "id").(type) { + case string, float64: + id = v + } + } + if !isObject { + return nil, errInvalidRequest("a request must be an object"), nil + } + if jsonrpc, ok := prop(raw, "jsonrpc").(string); !ok || jsonrpc != "2.0" { + return nil, errInvalidRequest(`jsonrpc must be "2.0"`), id + } + if id == nil { + return nil, errInvalidRequest("every request needs a string or number id"), nil + } + method, ok := prop(raw, "method").(string) + if !ok { + return nil, errInvalidRequest("method must be a string"), id + } + for _, key := range raw.Keys() { + if !envelopeMembers[key] { + return nil, errInvalidRequest(`unknown request member "` + key + `"`), id + } + } + if !methodNames[method] { + return nil, errMethodNotFound(method), id + } + params, has := raw.Get("params") + return &envelope{id: id, method: method, params: params, hasParams: has}, nil, id +} + +func errorResponse(id any, e *ProtocolError) string { + return Stringify(NewObject("jsonrpc", "2.0", "id", id, "error", e.JSON()), 0) +} + +// toProtocolError maps storage errors to protocol errors; nil for unknown ones. +func toProtocolError(err error) *ProtocolError { + var pe *ProtocolError + var nf *storageNotFound + var inv *storageInvalidFile + var un *storageUnavailable + switch { + case errors.As(err, &pe): + return pe + case errors.As(err, &nf): + return errNotFound(nf.msg) + case errors.As(err, &inv): + name := BlockNameFromFile(inv.file) + return errInvalidBlock([]Violation{{Name: name, Rule: "unsupported-name", Message: `this storage can't hold the entry "` + name + `"`}}) + case errors.As(err, &un): + return errUnavailable(un.msg, un.retryAfterMs) + } + return nil +} + +func (h *Handler) run(e *envelope) (any, error) { + if perr := validateParams(e.method, e.params, e.hasParams); perr != nil { + return nil, perr + } + if e.method != "describe" { + if err := h.store.checkContained(); err != nil { + return nil, err + } + } + switch e.method { + case "describe": + return h.describe() + case "schema.get": + return h.schemaGet(e.params) + case "blocks.list": + return h.blocksList(e.params) + default: + return h.blocksApply(e.params) + } +} + +func (h *Handler) call(value any) (out string) { + e, perr, id := parseEnvelope(value) + if perr != nil { + return errorResponse(id, perr) + } + defer func() { + // An unexpected throw (a URIError from a lone surrogate, a bug) is an + // Internal error for this call only, as in the TS server. + if r := recover(); r != nil { + if _, isURI := r.(uriError); !isURI { + h.logf("content protocol: %s panicked: %v", e.method, r) + } + out = errorResponse(e.id, errInternal()) + } + }() + result, err := h.run(e) + if err != nil { + if known := toProtocolError(err); known != nil { + return errorResponse(e.id, known) + } + h.logf("content protocol: %s failed: %v", e.method, err) + return errorResponse(e.id, errInternal()) + } + return Stringify(NewObject("jsonrpc", "2.0", "id", e.id, "result", result), 0) +} + +// dispatch runs a parsed body (one request or a batch) and returns the +// serialized response body. +func (h *Handler) dispatch(body any) string { + list, isBatch := body.([]any) + if !isBatch { + return h.call(body) + } + if len(list) == 0 { + return errorResponse(nil, errInvalidRequest("an empty batch")) + } + if len(list) > maxBatchCalls { + return errorResponse(nil, errLimitExceeded("a batch holds at most 10 calls", NewObject("limit", "maxBatchCalls"))) + } + parts := make([]string, len(list)) + for i, item := range list { + parts[i] = h.call(item) + } + return "[" + strings.Join(parts, ",") + "]" +} diff --git a/packages/sandbox/daemon-go/internal/content/secrets_test.go b/packages/sandbox/daemon-go/internal/content/secrets_test.go new file mode 100644 index 0000000000..9dd20f8197 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/secrets_test.go @@ -0,0 +1,227 @@ +package content + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "reflect" + "strings" + "testing" +) + +// Ported from secrets.test.ts, ciphertext.test.ts and __tests__/fixtures.ts. + +const schemaFixtureJSON = `{ + "manifest": {"blocks": { + "sections": {"hero": {"$ref": "#/definitions/aGVybw=="}, "newsletter": {"$ref": "#/definitions/bmV3c2xldHRlcg=="}}, + "loaders": {"multivariate": {"$ref": "#/definitions/bXY="}, "lazy": {"$ref": "#/definitions/bGF6eQ=="}, + "website/loaders/secret.ts": {"$ref": "#/definitions/djc="}}, + "content": {"settings": {"$ref": "#/definitions/c2V0dGluZ3M="}} + }}, + "schema": {"definitions": { + "aGVybw==": {"type": "object", "properties": {"title": {"type": "string"}, "padding": {"type": "string"}}}, + "bmV3c2xldHRlcg==": {"type": "object", "properties": {"listId": {"type": "string"}, "apiKey": {"type": "string", "format": "secret"}}}, + "c2V0dGluZ3M=": {"type": "object", "properties": { + "integrations": {"type": "array", "items": {"type": "object", "properties": {"token": {"$ref": "#/definitions/U2VjcmV0"}, "label": {"type": "string"}}}}, + "nested": {"anyOf": [{"type": "object", "properties": {"key": {"$ref": "#/definitions/U2VjcmV0"}}}]}, + "extra": {"type": "object", "additionalProperties": {"$ref": "#/definitions/U2VjcmV0"}} + }}, + "U2VjcmV0": {"type": "string", "format": "secret"}, + "bXY=": {"type": "object", "properties": {"variants": {"type": "array"}}}, + "bGF6eQ==": {"type": "object", "properties": {"value": {}}}, + "djc=": {"type": "object", "properties": {"name": {"type": "string"}, "encrypted": {"type": "string", "format": "secret"}}} + }} +}` + +func mustParse(t *testing.T, text string) any { + t.Helper() + v, err := ParseJSON(text) + if err != nil { + t.Fatalf("ParseJSON: %v", err) + } + return v +} + +func b64(n int, b byte) string { + return base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{b}, n)) +} + +func ciphertextWithLengths(key, iv, ct int) string { + return "v1." + b64(key, 3) + "." + b64(iv, 4) + "." + b64(ct, 5) +} + +var validCiphertext = ciphertextWithLengths(384, 12, 32) + +func secretJSON(c string) string { return `{"__resolveType":"secret","ciphertext":"` + c + `"}` } + +func ruleList(t *testing.T, entry string) []string { + t.Helper() + meta, _ := asObject(mustParse(t, schemaFixtureJSON)) + var out []string + for _, v := range checkSecrets("entry", mustParse(t, entry), meta) { + out = append(out, v.Rule+"@"+*v.Pointer) + } + return out +} + +func TestCiphertextFormat(t *testing.T) { + for _, c := range []string{ciphertextWithLengths(256, 12, 16), ciphertextWithLengths(384, 12, 32), ciphertextWithLengths(512, 12, 1024)} { + if !isWellFormedCiphertext(c) { + t.Errorf("refused %s…", c[:20]) + } + } + refused := []any{ + "", "v1.", "v1.hunter2", "v1.my-api-key_123", "v1.QUJD.ZGVm", + "v2" + validCiphertext[2:], ciphertextWithLengths(255, 12, 16), ciphertextWithLengths(384, 16, 32), + ciphertextWithLengths(384, 12, 15), validCiphertext + ".QUJD", validCiphertext + "==", + validCiphertext[:len(validCiphertext)-1] + "+", strings.Replace(validCiphertext, ".", ". ", 1), + "hunter2", 42.0, nil, + } + for _, c := range refused { + if isWellFormedCiphertext(c) { + t.Errorf("accepted %v", c) + } + } + // Non-canonical base64url (stray low bits) is refused: one value, one spelling. + if _, ok := decodeBase64URL("QUJE"); !ok { + t.Error("canonical") + } + if _, ok := decodeBase64URL("QUI"); !ok { + t.Error("unpadded") + } + if _, ok := decodeBase64URL("QUJ"); ok { + t.Error("stray low bits") + } +} + +func publicKeyPEM(t *testing.T) string { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + der, _ := x509.MarshalPKIXPublicKey(&key.PublicKey) + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})) +} + +func TestPublicKeyDerFromPem(t *testing.T) { + pub := publicKeyPEM(t) + if publicKeyDerFromPem(pub) == nil || publicKeyDerFromPem("\n "+pub+"\n\n") == nil { + t.Error("a single PUBLIC KEY block is accepted, surrounding whitespace too") + } + for _, bad := range []string{ + "", "hello", pub + pub, pub + "trailing text", + strings.ReplaceAll(pub, "PUBLIC KEY", "RSA PRIVATE KEY"), + strings.Replace(pub, "-----END PUBLIC KEY-----", "-----END PRIVATE KEY-----", 1), + strings.Replace(pub, "M", "*", 1), + } { + if publicKeyDerFromPem(bad) != nil { + t.Errorf("accepted %q", bad[:min(len(bad), 40)]) + } + } + if s := servablePublicKey(&pub); s == nil { + t.Error("servable") + } + withPrivate := pub + "\n# PRIVATE KEY" + if servablePublicKey(&withPrivate) != nil { + t.Error("a PRIVATE KEY mention is never served") + } + huge := pub + strings.Repeat(" ", 16*1024) + if servablePublicKey(&huge) != nil { + t.Error("over 16 KiB") + } +} + +func TestSecretGuard(t *testing.T) { + newsletter := func(apiKey string) string { + return `{"__resolveType":"newsletter","listId":"l1","apiKey":` + apiKey + `}` + } + expect := func(entry string, want ...string) { + t.Helper() + got := ruleList(t, entry) + if len(want) == 0 { + want = nil + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s:\n got %v\nwant %v", entry, got, want) + } + } + expect(newsletter(`"hunter2"`), "secret-field@/apiKey") + for _, v := range []string{`42`, `null`, `{"value":"hunter2"}`, `{"__resolveType":"MyKey"}`} { + expect(newsletter(v), "secret-field@/apiKey") + } + expect(newsletter(secretJSON(validCiphertext))) + expect(newsletter(secretJSON("hunter2")), "secret-ciphertext@/apiKey") + expect(newsletter(`{"__resolveType":"secret"}`), "secret-ciphertext@/apiKey") + expect(`{"__resolveType":"newsletter","listId":"l1"}`) + expect(`{"__resolveType":"settings", + "integrations":[{"token":`+secretJSON(validCiphertext)+`,"label":"ok"},{"token":"plain","label":"leak"}], + "nested":{"key":"plain"},"extra":{"a":`+secretJSON(validCiphertext)+`,"b":"plain"}}`, + "secret-field@/integrations/1/token", "secret-field@/nested/key", "secret-field@/extra/b") + expect(`{"__resolveType":"page","sections":[{"__resolveType":"newsletter","apiKey":"plain"},{"__resolveType":"hero","title":"x"}]}`, + "secret-field@/sections/0/apiKey") + variants := func(values ...string) string { + var parts []string + for _, v := range values { + parts = append(parts, `{"rule":{"__resolveType":"always"},"value":{"__resolveType":"lazy","value":`+v+`}}`) + } + return newsletter(`{"__resolveType":"multivariate","variants":[` + strings.Join(parts, ",") + `]}`) + } + expect(variants(secretJSON(validCiphertext), secretJSON(validCiphertext))) + expect(variants(secretJSON(validCiphertext), `"plain"`), "secret-field@/apiKey/variants/1/value/value") + expect(newsletter(`{"__resolveType":"multivariate"}`), "secret-field@/apiKey") + expect(newsletter(`{"__resolveType":"website/flags/multivariate.ts","variants":[{"rule":{"__resolveType":"always"},"value":"plain"}]}`), + "secret-field@/apiKey/variants/0/value") + expect(`{"__resolveType":"hero","title":`+secretJSON("bad")+`,"list":[`+secretJSON(validCiphertext)+`]}`, "secret-ciphertext@/title") + expect(`{"__resolveType":"hero","title":"plain text is fine here"}`) + expect(`{"__resolveType":"unknown-type","apiKey":"not a known Secret field"}`) + expect(`{"__resolveType":"settings","extra":{"constructor":"plain","x/~y":"plain"}}`, + "secret-field@/extra/constructor", "secret-field@/extra/x~1~0y") + + // Where a schema has `properties`, an Object.prototype name is found there + // (`key in properties` in JS) and never falls back to additionalProperties. + box, _ := asObject(mustParse(t, `{"manifest":{"blocks":{"s":{"box":{"type":"object","properties":{"title":{"type":"string"}},"additionalProperties":{"type":"string","format":"secret"}}}}}}`)) + inherited := checkSecrets("e", mustParse(t, `{"__resolveType":"box","constructor":"plain","toString":"plain","other":"plain","title":"t"}`), box) + if len(inherited) != 1 || *inherited[0].Pointer != "/other" { + t.Errorf("inherited names: %+v", inherited) + } + + // Without a schema only the ciphertexts are checked. + noSchema := checkSecrets("e", mustParse(t, `{"__resolveType":"hero","title":`+secretJSON("bad")+`}`), nil) + if len(noSchema) != 1 || noSchema[0].Rule != "secret-ciphertext" { + t.Errorf("no schema: %+v", noSchema) + } + top := checkSecrets("e", mustParse(t, secretJSON("bad")), nil) + if len(top) != 1 || *top[0].Pointer != "" { + t.Errorf("top-level secret block: %+v", top) + } +} + +func TestLegacySecretLoaderIsExempt(t *testing.T) { + // The v7 loader's `encrypted` is marked format: secret but holds the + // site's own ciphertext: it is walked without its definition. + const loader = `"website/loaders/secret.ts"` + got := ruleList(t, `{"__resolveType":"settings","apps":{"__resolveType":`+loader+`,"name":"API_KEY","encrypted":"0a1b2c"}}`) + if got != nil { + t.Errorf("legacy loader: %v", got) + } + got = ruleList(t, `{"__resolveType":`+loader+`,"encrypted":`+secretJSON("bad")+`}`) + if !reflect.DeepEqual(got, []string{"secret-ciphertext@/encrypted"}) { + t.Errorf("a secret block inside the loader is still checked: %v", got) + } + got = ruleList(t, `{"__resolveType":"site/loaders/secret.ts","encrypted":"0a1b2c"}`) + if got != nil { + t.Errorf("any app's loaders/secret.ts: %v", got) + } +} + +func TestSecretGuardNamesTheEntry(t *testing.T) { + meta, _ := asObject(mustParse(t, schemaFixtureJSON)) + v := checkSecrets("Newsletter", mustParse(t, `{"__resolveType":"newsletter","apiKey":"x"}`), meta) + if len(v) != 1 || v[0].Name != "Newsletter" || v[0].Rule != "secret-field" || *v[0].Pointer != "/apiKey" { + t.Errorf("%+v", v) + } +} From 6f8c7ec3efc968795d0d4a95b8dad0d8795bbec0 Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:56:50 -0300 Subject: [PATCH 08/20] feat(daemon): asset uploads + handler tests Asset uploads with name sanitizing, plus HTTP-level tests for the store, secrets, RPC and uploads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../daemon-go/internal/content/assets.go | 181 +++++ .../daemon-go/internal/content/server_test.go | 694 ++++++++++++++++++ 2 files changed, 875 insertions(+) create mode 100644 packages/sandbox/daemon-go/internal/content/assets.go create mode 100644 packages/sandbox/daemon-go/internal/content/server_test.go diff --git a/packages/sandbox/daemon-go/internal/content/assets.go b/packages/sandbox/daemon-go/internal/content/assets.go new file mode 100644 index 0000000000..55c4a7ab4b --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/assets.go @@ -0,0 +1,181 @@ +package content + +// Uploads beside the protocol, ported from server/assets.ts and assets.ts: +// `PUT …/assets/` with the file's own image, video, font or PDF type; +// the name's extension must match it, SVG is refused, a taken name gets a +// short suffix, and the answer is the path the field stores: /assets/. + +import ( + "errors" + "net/http" + "regexp" + "strconv" + "strings" + + "golang.org/x/text/unicode/norm" +) + +var assetTypes = map[string][]string{ + "image/png": {".png"}, + "image/jpeg": {".jpg", ".jpeg"}, + "image/webp": {".webp"}, + "image/avif": {".avif"}, + "image/gif": {".gif"}, + "image/x-icon": {".ico"}, + "image/vnd.microsoft.icon": {".ico"}, + "video/mp4": {".mp4"}, + "video/webm": {".webm"}, + "font/woff2": {".woff2"}, + "font/woff": {".woff"}, + "font/ttf": {".ttf"}, + "font/otf": {".otf"}, + "application/font-woff": {".woff"}, + "application/x-font-ttf": {".ttf"}, + "application/vnd.ms-fontobject": {".eot"}, + "application/pdf": {".pdf"}, +} + +func mediaType(contentType string) string { + return strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) +} + +func assetExtensions(contentType string, present bool) []string { + if !present { + return nil + } + return assetTypes[mediaType(contentType)] +} + +// assetNameForType fits a sanitized name to its type: a name without an +// extension gets the type's; "" when the extension doesn't match. +func assetNameForType(name string, extensions []string) string { + if extensions == nil { + return "" + } + dot := strings.LastIndex(name, ".") + if dot <= 0 { + return name + extensions[0] + } + ext := strings.ToLower(name[dot:]) + for _, e := range extensions { + if e == ext { + return name[:dot] + ext + } + } + return "" +} + +var ( + combiningMarks = regexp.MustCompile("[̀-ͯ]") + notNameChars = regexp.MustCompile(`[^A-Za-z0-9._-]+`) + repeatedDashes = regexp.MustCompile(`-{2,}`) + leadingDotsDash = regexp.MustCompile(`^[.-]+`) + dashBeforeDot = regexp.MustCompile(`-+(\.|$)`) +) + +// sanitizeAssetName normalizes an upload's name: its last path segment, with +// anything but letters, digits, ".", "_" and "-" replaced; "" when nothing +// usable is left. +func sanitizeAssetName(raw string) string { + name, ok := decodeURIComponent(raw) + if !ok { + name = raw + } + if i := strings.LastIndexAny(name, `\/`); i >= 0 { + name = name[i+1:] + } + name = norm.NFKD.String(name) + name = combiningMarks.ReplaceAllString(name, "") + name = notNameChars.ReplaceAllString(name, "-") + name = repeatedDashes.ReplaceAllString(name, "-") + name = leadingDotsDash.ReplaceAllString(name, "") + name = dashBeforeDot.ReplaceAllString(name, "$1") + if name == "" || name == "." || name == ".." { + return "" + } + if len(name) > 200 { + ext := "" + if dot := strings.LastIndex(name, "."); dot > 0 { + ext = name[dot:] + if len(ext) > 16 { + ext = ext[:16] + } + } + name = name[:200-len(ext)] + ext + } + return name +} + +var assetErrorStatus = map[int]int{ + CodeReadOnly: 403, + CodeUnsupported: 404, + CodeLimitExceeded: 413, + CodeInvalidRequest: 400, +} + +func (h *Handler) assetFail(w http.ResponseWriter, r *http.Request, e *ProtocolError, status int) { + if status == 0 { + status = assetErrorStatus[e.Code] + if status == 0 { + status = 400 + } + } + writeJSON(w, r, Stringify(NewObject("error", e.JSON()), 0), status, nil) +} + +// ServeAssets serves `PUT …/assets/`. +func (h *Handler) ServeAssets(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + writeJSON(w, r, Stringify(NewObject("error", errInvalidRequest("use PUT").JSON()), 0), 405, map[string]string{"Allow": "PUT"}) + return + } + desc := h.store.describe() + contentType := strings.Join(r.Header.Values("Content-Type"), ", ") + extensions := assetExtensions(contentType, len(r.Header.Values("Content-Type")) > 0) + if extensions == nil { + h.assetFail(w, r, errInvalidRequest("uploads must be an image, video, font or PDF"), 415) + return + } + path := r.URL.EscapedPath() + sanitized := "" + if marker := strings.LastIndex(path, assetsURLPrefix); marker >= 0 { + sanitized = sanitizeAssetName(path[marker+len(assetsURLPrefix):]) + } + if sanitized == "" { + h.assetFail(w, r, errInvalidRequest("PUT /assets/ needs a file name"), 0) + return + } + name := assetNameForType(sanitized, extensions) + if name == "" { + h.assetFail(w, r, errInvalidRequest("the file name's extension doesn't match its content type ("+contentType+")"), 415) + return + } + body, err := readBody(r, desc.AssetsMaxBytes) + if err != nil { + var enc *bodyEncodingError + switch { + case errors.Is(err, errBodyTooLarge): + h.assetFail(w, r, errLimitExceeded("uploads are limited to "+strconv.FormatInt(desc.AssetsMaxBytes, 10)+" bytes", nil), 0) + case errors.As(err, &enc): + h.assetFail(w, r, errInvalidRequest(enc.msg), 415) + default: + h.assetFail(w, r, errInternal(), 500) + } + return + } + if len(body) == 0 { + h.assetFail(w, r, errInvalidRequest("the upload is empty"), 0) + return + } + stored, err := h.store.putAsset(name, body) + if err != nil { + // OPEN: the TS handler lets this throw (its server answers 500). + h.logf("content protocol: upload failed: %v", err) + h.assetFail(w, r, errInternal(), 500) + return + } + if h.onAsset != nil { + h.onAsset(stored) + } + writeJSON(w, r, Stringify(NewObject("path", assetsURLPrefix+stored), 0), 201, nil) +} diff --git a/packages/sandbox/daemon-go/internal/content/server_test.go b/packages/sandbox/daemon-go/internal/content/server_test.go new file mode 100644 index 0000000000..e603ed564c --- /dev/null +++ b/packages/sandbox/daemon-go/internal/content/server_test.go @@ -0,0 +1,694 @@ +package content + +import ( + "bytes" + "compress/gzip" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "io" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// Ported from server/handler.test.ts, server/methods.test.ts, +// server/apply.test.ts, server/assets.test.ts and storage/fs/*.test.ts. + +type site struct { + t *testing.T + root string + h *Handler + mu sync.Mutex + commits [][]string +} + +func newSite(t *testing.T, withSchema bool) *site { + t.Helper() + root := t.TempDir() + os.MkdirAll(filepath.Join(root, ".deco", "blocks"), 0o755) + os.WriteFile(filepath.Join(root, ".deco", "index.ts"), []byte("export default {};\n"), 0o644) + if withSchema { + os.WriteFile(filepath.Join(root, ".deco", "schema.gen.json"), []byte(schemaFixtureJSON), 0o644) + } + s := &site{t: t, root: root} + s.h = NewHandler(Options{ + Store: NewFSStore(FSOptions{Root: root, RepoRoot: root}), + OnCommit: func(files []string) { + s.mu.Lock() + s.commits = append(s.commits, files) + s.mu.Unlock() + }, + Logf: func(string, ...any) {}, + }) + return s +} + +func (s *site) blockPath(file string) string { return filepath.Join(s.root, ".deco", "blocks", file) } + +func (s *site) writeFile(file, content string) { + s.t.Helper() + if err := os.WriteFile(s.blockPath(file), []byte(content), 0o644); err != nil { + s.t.Fatal(err) + } +} + +type rpcResponse struct { + ID any `json:"id"` + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + Data json.RawMessage `json:"data"` + } `json:"error"` +} + +func (s *site) post(body string, headers map[string]string) *httptest.ResponseRecorder { + r := httptest.NewRequest("POST", "/rpc", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + for k, v := range headers { + r.Header.Set(k, v) + } + w := httptest.NewRecorder() + s.h.ServeRPC(w, r) + return w +} + +func (s *site) call(method string, params any) rpcResponse { + s.t.Helper() + envelope := map[string]any{"jsonrpc": "2.0", "id": 1, "method": method} + if params != nil { + envelope["params"] = params + } + body, _ := json.Marshal(envelope) + w := s.post(string(body), nil) + var res rpcResponse + if err := json.Unmarshal(w.Body.Bytes(), &res); err != nil { + s.t.Fatalf("%s: %v (%s)", method, err, w.Body.String()) + } + return res +} + +func (s *site) result(method string, params any, out any) { + s.t.Helper() + res := s.call(method, params) + if res.Error != nil { + s.t.Fatalf("%s failed: %d %s %s", method, res.Error.Code, res.Error.Message, res.Error.Data) + } + if err := json.Unmarshal(res.Result, out); err != nil { + s.t.Fatal(err) + } +} + +func (s *site) errorCode(method string, params any) int { + s.t.Helper() + res := s.call(method, params) + if res.Error == nil { + s.t.Fatalf("%s succeeded: %s", method, res.Result) + } + return res.Error.Code +} + +type listResult struct { + NotModified bool `json:"notModified"` + Revision string `json:"revision"` + Blocks map[string]json.RawMessage `json:"blocks"` + Versions map[string]string `json:"versions"` + Diagnostics []map[string]string `json:"diagnostics"` +} + +type applyResult struct { + Revision string `json:"revision"` + Versions map[string]*string `json:"versions"` +} + +func (s *site) list() listResult { + var r listResult + s.result("blocks.list", nil, &r) + return r +} + +func (s *site) apply(params any) applyResult { + var r applyResult + s.result("blocks.apply", params, &r) + return r +} + +func gitHash(b []byte) string { + h := sha1.New() + h.Write([]byte("blob " + strconv.Itoa(len(b)) + "\x00")) + h.Write(b) + return hex.EncodeToString(h.Sum(nil)) +} + +// ---------------------------------------------------------------- HTTP + +func TestHandlerHTTP(t *testing.T) { + s := newSite(t, true) + get := httptest.NewRecorder() + s.h.ServeRPC(get, httptest.NewRequest("GET", "/rpc", nil)) + if get.Code != 405 || get.Header().Get("Allow") != "POST" || !strings.Contains(get.Body.String(), `"code":-32600`) { + t.Errorf("GET: %d %s", get.Code, get.Body) + } + form := httptest.NewRecorder() + r := httptest.NewRequest("POST", "/rpc", strings.NewReader("a=1")) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + s.h.ServeRPC(form, r) + if form.Code != 415 { + t.Errorf("form post: %d", form.Code) + } + if w := s.post(`{"jsonrpc":"2.0","id":1,"method":"describe"}`, map[string]string{"Content-Type": "Application/JSON; charset=utf-8"}); w.Code != 200 { + t.Errorf("JSON with parameters: %d", w.Code) + } + for _, body := range []string{"{not json", "", "\xff"} { + w := s.post(body, nil) + if w.Code != 200 || !strings.Contains(w.Body.String(), `"code":-32700`) || !strings.Contains(w.Body.String(), `"id":null`) { + t.Errorf("%q: %d %s", body, w.Code, w.Body) + } + } + w := s.post(`{"jsonrpc":"2.0","id":1,"method":"describe"}`, nil) + if w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Type") != "application/json; charset=utf-8" || w.Header().Get("Vary") != "Accept-Encoding" { + t.Errorf("headers: %v", w.Header()) + } + + // The request limit applies whatever Content-Length says, and after gzip. + big := `{"jsonrpc":"2.0","id":1,"method":"describe","params":{"x":"` + strings.Repeat("x", maxRequestBytes) + `"}}` + if w := s.post(big, nil); w.Code != 413 || !strings.Contains(w.Body.String(), `"limit":"maxRequestBytes"`) { + t.Errorf("oversized: %d %.200s", w.Code, w.Body) + } + var zipped bytes.Buffer + zw := gzip.NewWriter(&zipped) + zw.Write([]byte(big)) + zw.Close() + r = httptest.NewRequest("POST", "/rpc", &zipped) + r.Header.Set("Content-Type", "application/json") + r.Header.Set("Content-Encoding", "gzip") + w = httptest.NewRecorder() + s.h.ServeRPC(w, r) + if w.Code != 413 { + t.Errorf("oversized gzip: %d", w.Code) + } + zipped.Reset() + zw = gzip.NewWriter(&zipped) + zw.Write([]byte(`{"jsonrpc":"2.0","id":7,"method":"describe"}`)) + zw.Close() + r = httptest.NewRequest("POST", "/rpc", &zipped) + r.Header.Set("Content-Type", "application/json") + r.Header.Set("Content-Encoding", "gzip") + w = httptest.NewRecorder() + s.h.ServeRPC(w, r) + if w.Code != 200 || !strings.Contains(w.Body.String(), `"id":7`) { + t.Errorf("gzip body: %d %s", w.Code, w.Body) + } + for _, enc := range []string{"br", "gzip"} { + r = httptest.NewRequest("POST", "/rpc", strings.NewReader("not gzip")) + r.Header.Set("Content-Type", "application/json") + r.Header.Set("Content-Encoding", enc) + w = httptest.NewRecorder() + s.h.ServeRPC(w, r) + if w.Code != 415 { + t.Errorf("%s: %d", enc, w.Code) + } + } + + // Responses of 1 KiB or more are gzipped when accepted. + s.apply(map[string]any{"set": map[string]any{"big": map[string]any{"text": strings.Repeat("x", 4096)}}}) + for header, want := range map[string]string{"gzip": "gzip", "gzip;q=0": "", "*": "gzip", "br": "", "identity, gzip;q=0.5": "gzip"} { + w := s.post(`{"jsonrpc":"2.0","id":1,"method":"blocks.list"}`, map[string]string{"Accept-Encoding": header}) + if got := w.Header().Get("Content-Encoding"); got != want { + t.Errorf("Accept-Encoding %q: Content-Encoding %q", header, got) + } + if want == "gzip" { + zr, _ := gzip.NewReader(w.Body) + plain, _ := io.ReadAll(zr) + if !strings.Contains(string(plain), `"blocks"`) { + t.Errorf("gzip body: %.100s", plain) + } + } + } +} + +func TestEnvelopesAndBatches(t *testing.T) { + s := newSite(t, true) + cases := map[string]string{ + `{"jsonrpc":"2.0","method":"describe"}`: `{"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"every request needs a string or number id"}}`, + `{"jsonrpc":"1.0","id":"a","method":"describe"}`: `{"jsonrpc":"2.0","id":"a","error":{"code":-32600,"message":"jsonrpc must be \"2.0\""}}`, + `{"jsonrpc":"2.0","id":1,"method":"blocks.rename"}`: `{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"unknown method \"blocks.rename\""}}`, + `{"jsonrpc":"2.0","id":1,"method":"describe","x":1}`: `{"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"unknown request member \"x\""}}`, + `{"jsonrpc":"2.0","id":1,"method":4}`: `{"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"method must be a string"}}`, + `[]`: `{"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"an empty batch"}}`, + `[1]`: `[{"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"a request must be an object"}}]`, + `{"jsonrpc":"2.0","id":1,"method":"describe","params":[]}`: `{"jsonrpc":"2.0","id":1,"error":{"code":-32602,"message":"params must be an object"}}`, + } + for in, want := range cases { + if got := s.post(in, nil).Body.String(); got != want { + t.Errorf("%s:\n got %s\nwant %s", in, got, want) + } + } + calls := make([]string, 11) + for i := range calls { + calls[i] = `{"jsonrpc":"2.0","id":` + strconv.Itoa(i) + `,"method":"describe"}` + } + if got := s.post("["+strings.Join(calls, ",")+"]", nil).Body.String(); !strings.Contains(got, `"limit":"maxBatchCalls"`) { + t.Errorf("11 calls: %s", got) + } + var items []rpcResponse + json.Unmarshal(s.post(`[{"jsonrpc":"2.0","id":1,"method":"describe"},{"jsonrpc":"2.0","id":2,"method":"nope"},{"jsonrpc":"2.0","id":"three","method":"blocks.list"}]`, nil).Body.Bytes(), &items) + if len(items) != 3 || items[0].Error != nil || items[1].Error.Code != CodeMethodNotFound || items[2].Error != nil || items[2].ID != "three" { + t.Errorf("batch: %+v", items) + } + // Not atomic: the first write lands though the second fails. + json.Unmarshal(s.post(`[{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"kept":{}}}},{"jsonrpc":"2.0","id":2,"method":"blocks.apply","params":{"set":{"bad":[]}}}]`, nil).Body.Bytes(), &items) + if items[0].Error != nil || items[1].Error.Code != CodeInvalidBlock { + t.Errorf("batch writes: %+v", items) + } + if _, ok := s.list().Blocks["kept"]; !ok { + t.Error("the first write must land") + } + // A lone surrogate in a name is an Internal error for that call only. + if got := s.post(`{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x\ud800":{}}}}`, nil).Body.String(); got != `{"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"internal error"}}` { + t.Errorf("lone surrogate: %s", got) + } +} + +// ---------------------------------------------------------------- reads + +func TestDescribe(t *testing.T) { + s := newSite(t, false) + pub := publicKeyPEM(t) + os.WriteFile(filepath.Join(s.root, ".deco", "secrets.pub"), []byte(pub), 0o644) + var d map[string]any + s.result("describe", map[string]any{}, &d) + want := map[string]any{ + "protocol": "deco-content", "version": map[string]any{"major": 1.0, "minor": 0.0}, + "server": map[string]any{"name": "deco-blocks", "version": "unknown"}, "kind": "working-tree", + "readOnly": false, "root": ".", "schemaFormat": "deco-meta@1", "pollIntervalMs": 2000.0, "preview": nil, + "assets": map[string]any{"dir": "public/assets", "urlPrefix": "/assets/", "maxBytes": float64(25 * 1024 * 1024)}, + "secrets": map[string]any{"publicKey": pub}, + } + if !reflect.DeepEqual(d, want) { + t.Errorf("describe:\n got %v\nwant %v", d, want) + } + os.WriteFile(filepath.Join(s.root, ".deco", "secrets.pub"), []byte(pub+"-----BEGIN PRIVATE KEY-----\nAA==\n-----END PRIVATE KEY-----\n"), 0o644) + s.result("describe", nil, &d) + if d["secrets"] != nil { + t.Error("a file holding a private key is never served") + } + nested := NewHandler(Options{Store: NewFSStore(FSOptions{Root: filepath.Join(s.root, "apps", "site"), RepoRoot: s.root})}) + if desc := nested.store.describe(); desc.Root != "apps/site" || desc.AssetsDir != "apps/site/public/assets" { + t.Errorf("nested root: %+v", desc) + } +} + +func TestSchemaGet(t *testing.T) { + s := newSite(t, false) + var r map[string]any + s.result("schema.get", nil, &r) + if !reflect.DeepEqual(r, map[string]any{"notModified": false, "version": nil, "resolvedRef": nil, "schema": nil}) { + t.Errorf("no schema: %v", r) + } + os.WriteFile(filepath.Join(s.root, ".deco", "meta.gen.json"), []byte(`{"from":"meta"}`), 0o644) + s.result("schema.get", nil, &r) + if r["version"] != gitHash([]byte(`{"from":"meta"}`)) || !reflect.DeepEqual(r["schema"], map[string]any{"from": "meta"}) { + t.Errorf("meta.gen.json fallback: %v", r) + } + os.WriteFile(filepath.Join(s.root, ".deco", "schema.gen.json"), []byte(`{"from":"schema"}`), 0o644) + s.result("schema.get", nil, &r) + version := r["version"].(string) + if !reflect.DeepEqual(r["schema"], map[string]any{"from": "schema"}) { + t.Errorf("schema.gen.json first: %v", r) + } + r = nil + s.result("schema.get", map[string]any{"ifNoneMatch": version}, &r) + if !reflect.DeepEqual(r, map[string]any{"notModified": true, "version": version}) { + t.Errorf("not modified: %v", r) + } + os.WriteFile(filepath.Join(s.root, ".deco", "schema.gen.json"), []byte(`{"torn`), 0o644) + res := s.call("schema.get", nil) + if res.Error == nil || res.Error.Code != CodeUnavailable || string(res.Error.Data) != `{"retryAfterMs":500}` { + t.Errorf("torn schema: %+v", res.Error) + } + os.RemoveAll(filepath.Join(s.root, ".deco")) + if code := s.errorCode("schema.get", nil); code != CodeNotFound { + t.Errorf("no .deco: %d", code) + } + if code := s.errorCode("blocks.list", nil); code != CodeNotFound { + t.Errorf("no .deco list: %d", code) + } +} + +func TestBlocksList(t *testing.T) { + s := newSite(t, false) + s.writeFile("Header.json", `{"a":1}`) + s.writeFile("pages-Home%20Page.json", `{"path":"/"}`) + s.writeFile("pages-Home%2520Page.json", `{"v":"bot"}`) // shadowed: the other has a path + s.writeFile("broken.json", `{"a":`) + s.writeFile("list.json", `[1]`) + s.writeFile("huge.json", `{"t":"`+strings.Repeat("x", maxBlockBytes)+`"}`) + s.writeFile("constructor.json", `{"proto":true}`) + s.writeFile(".hidden.json", `{}`) + s.writeFile("notes.txt", `{}`) + os.Mkdir(s.blockPath("dir.json"), 0o755) + + l := s.list() + var names []string + for n := range l.Blocks { + names = append(names, n) + } + sort.Strings(names) + if !reflect.DeepEqual(names, []string{"Header", "constructor", "pages-Home Page"}) { + t.Errorf("names: %v", names) + } + if l.Versions["Header"] != gitHash([]byte(`{"a":1}`)) { + t.Error("versions are git blob hashes") + } + kinds := map[string]string{} + for _, d := range l.Diagnostics { + kinds[d["file"]] = d["kind"] + } + want := map[string]string{"broken.json": "invalid-json", "list.json": "not-an-object", "huge.json": "too-large", "pages-Home%2520Page.json": "shadowed"} + if !reflect.DeepEqual(kinds, want) { + t.Errorf("diagnostics: %v", l.Diagnostics) + } + var again map[string]any + s.result("blocks.list", map[string]any{"ifNoneMatch": l.Revision}, &again) + if !reflect.DeepEqual(again, map[string]any{"notModified": true, "revision": l.Revision, "resolvedRef": nil}) { + t.Errorf("not modified: %v", again) + } + // A changed file is served fresh, not from the cache. + s.writeFile("Header.json", `{"a":2}`) + if l2 := s.list(); string(l2.Blocks["Header"]) != `{"a":2}` || l2.Revision == l.Revision { + t.Errorf("changed file: %s", l2.Blocks["Header"]) + } +} + +// ---------------------------------------------------------------- writes + +func TestApplyWritesTheReferenceBytes(t *testing.T) { + s := newSite(t, false) + r := s.apply(map[string]any{"set": map[string]any{"pages-Home Page": map[string]any{"path": "/", "b": []any{true}}}}) + file := "pages-Home%20Page.json" + b, err := os.ReadFile(s.blockPath(file)) + if err != nil { + t.Fatal(err) + } + if string(b) != "{\n \"b\": [\n true\n ],\n \"path\": \"/\"\n}\n" { + t.Errorf("bytes: %q", b) + } + if *r.Versions["pages-Home Page"] != gitHash(b) { + t.Error("version") + } + if l := s.list(); l.Revision != r.Revision { + t.Error("the listed revision is the one apply returned") + } + if !reflect.DeepEqual(s.commits, [][]string{{file}}) { + t.Errorf("OnCommit: %v", s.commits) + } +} + +func TestApplySemantics(t *testing.T) { + s := newSite(t, true) + s.apply(map[string]any{"set": map[string]any{"a": map[string]any{"v": 1}, "b": map[string]any{"v": 2}}}) + r := s.apply(map[string]any{"set": map[string]any{"c": map[string]any{}}, "delete": []any{"a", "missing"}}) + if r.Versions["a"] != nil || r.Versions["missing"] != nil || r.Versions["c"] == nil { + t.Errorf("set + delete: %+v", r.Versions) + } + both := s.apply(map[string]any{"set": map[string]any{"both": map[string]any{}}, "delete": []any{"both"}}) + if both.Versions["both"] == nil { + t.Error("set wins over delete") + } + // Every violation at once, nothing written. + res := s.call("blocks.apply", map[string]any{"set": map[string]any{ + "good": map[string]any{}, "arr": []any{}, "nothing": nil, ".hidden": map[string]any{}, + "n": map[string]any{"__resolveType": "newsletter", "apiKey": "plain"}, + }}) + var data struct{ Violations []map[string]string } + json.Unmarshal(res.Error.Data, &data) + if res.Error.Code != CodeInvalidBlock || len(data.Violations) != 4 || res.Error.Message != "4 invalid blocks" { + t.Errorf("violations: %s %s", res.Error.Message, res.Error.Data) + } + if _, ok := s.list().Blocks["good"]; ok { + t.Error("nothing is written when anything is invalid") + } + if code := s.errorCode("blocks.apply", map[string]any{"set": map[string]any{"big": map[string]any{"t": strings.Repeat("x", maxBlockBytes)}}}); code != CodeInvalidBlock { + t.Errorf("too large: %d", code) + } + names := make([]any, 501) + for i := range names { + names[i] = "n" + strconv.Itoa(i) + } + res = s.call("blocks.apply", map[string]any{"delete": names}) + if res.Error == nil || res.Error.Code != CodeLimitExceeded || string(res.Error.Data) != `{"limit":"maxOpsPerApply"}` { + t.Errorf("501 names: %+v", res.Error) + } + // A case variant of an existing (or another new) name is refused. + if code := s.errorCode("blocks.apply", map[string]any{"set": map[string]any{"B": map[string]any{}}}); code != CodeInvalidBlock { + t.Errorf("case collision: %d", code) + } + if code := s.errorCode("blocks.apply", map[string]any{"set": map[string]any{"New": map[string]any{}, "new": map[string]any{}}}); code != CodeInvalidBlock { + t.Errorf("case collision among new names: %d", code) + } + if r := s.apply(map[string]any{"delete": []any{"widget.ts"}}); r.Versions["widget.ts"] != nil { + t.Error("a source-extension name can be deleted") + } + // An empty apply commits nothing. + before := len(s.commits) + s.apply(map[string]any{}) + if len(s.commits) != before { + t.Error("an empty apply must not commit") + } +} + +func TestApplySpellingsAndGuards(t *testing.T) { + s := newSite(t, false) + s.writeFile("Home%2520Page.json", `{"v":0}`) + created := s.list().Versions["Home%20Page"] + // Create-only under another spelling fails, reporting the existing version. + res := s.call("blocks.apply", map[string]any{"set": map[string]any{"Home Page": map[string]any{}}, "ifMatch": map[string]any{"Home Page": nil}}) + if res.Error == nil || res.Error.Code != CodeConflict || string(res.Error.Data) != `{"entries":{"Home Page":{"expected":null,"actual":"`+created+`"}}}` { + t.Errorf("create-only: %+v", res.Error) + } + if code := s.errorCode("blocks.apply", map[string]any{"set": map[string]any{"Home%20Page": map[string]any{}, "Home Page": map[string]any{}}}); code != CodeInvalidBlock { + t.Errorf("two spellings in one write: %d", code) + } + // A guard on one spelling holds for the other; the other spelling goes. + r := s.apply(map[string]any{"set": map[string]any{"Home Page": map[string]any{"v": 1}}, "ifMatch": map[string]any{"Home Page": created}}) + if r.Versions["Home%20Page"] != nil || r.Versions["Home Page"] == nil { + t.Errorf("versions: %+v", r.Versions) + } + if _, err := os.Stat(s.blockPath("Home%2520Page.json")); !os.IsNotExist(err) { + t.Error("the other spelling must be deleted in the same commit") + } + // Deleting any spelling deletes every spelling. + s.writeFile("Home%2520Page.json", `{"v":0}`) + r = s.apply(map[string]any{"delete": []any{"Home Page"}}) + if len(s.list().Blocks) != 0 || len(r.Versions) != 2 { + t.Errorf("delete every spelling: %+v %v", r.Versions, s.list().Blocks) + } + // Last writer wins without guards. + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + s.apply(map[string]any{"set": map[string]any{"lww": map[string]any{"i": i}, "own" + strconv.Itoa(i): map[string]any{}}}) + }(i) + } + wg.Wait() + if l := s.list(); len(l.Blocks) != 9 { + t.Errorf("concurrent writers: %d entries", len(l.Blocks)) + } +} + +func TestApplyRetriesThenGivesUp(t *testing.T) { + s := newSite(t, false) + s.h.retryMinMs, s.h.retryMaxMs = 0, 0 + s.apply(map[string]any{"set": map[string]any{"g": map[string]any{"v": 1}}}) + version := *s.apply(map[string]any{"set": map[string]any{"g": map[string]any{"v": 2}}}).Versions["g"] + // Change the guarded file behind every commit attempt's back. + n := 0 + s.h.store.exclusive = func() (func(), bool) { + n++ + s.writeFile("g.json", `{"v":"other `+strconv.Itoa(n)+`"}`) + return func() {}, true + } + // The guard is rechecked against a fresh snapshot: a Conflict, not a stale write. + res := s.call("blocks.apply", map[string]any{"set": map[string]any{"g": map[string]any{"v": 3}}, "ifMatch": map[string]any{"g": version}}) + if res.Error == nil || res.Error.Code != CodeConflict { + t.Errorf("guard after a move: %+v", res.Error) + } + // A schema that keeps changing exhausts the attempts. + s.h.store.exclusive = func() (func(), bool) { + n++ + os.WriteFile(filepath.Join(s.root, ".deco", "schema.gen.json"), []byte(`{"n":`+strconv.Itoa(n)+`}`), 0o644) + return func() {}, true + } + res = s.call("blocks.apply", map[string]any{"set": map[string]any{"x": map[string]any{}}}) + if res.Error == nil || res.Error.Code != CodeUnavailable || res.Error.Message != "storage kept changing; gave up after 3 commit attempts" { + t.Errorf("gave up: %+v", res.Error) + } +} + +// ---------------------------------------------------------------- storage + +func TestCommitLockAndTransactions(t *testing.T) { + s := newSite(t, false) + deco := filepath.Join(s.root, ".deco") + // A crashed commit's transaction folder is swept by the next commit. + os.MkdirAll(filepath.Join(deco, ".tx-crashed", "new"), 0o755) + // Another process's live lock is waited for, a stale one taken over. + lock := filepath.Join(deco, lockFileName) + os.WriteFile(lock, []byte("1\n"), 0o644) + s.h.store.lockTimeout = 100 * time.Millisecond + res := s.call("blocks.apply", map[string]any{"set": map[string]any{"x": map[string]any{}}}) + if res.Error == nil || res.Error.Code != CodeUnavailable || string(res.Error.Data) != `{"retryAfterMs":500}` { + t.Errorf("held lock: %+v", res.Error) + } + old := time.Now().Add(-time.Minute) + os.Chtimes(lock, old, old) + s.apply(map[string]any{"set": map[string]any{"x": map[string]any{}}}) + entries, _ := os.ReadDir(deco) + for _, e := range entries { + if e.Name() == lockFileName || strings.HasPrefix(e.Name(), ".tx-") { + t.Errorf("left behind: %s", e.Name()) + } + } +} + +func TestApplyFileChangeRollsBack(t *testing.T) { + dir := t.TempDir() + scratch := t.TempDir() + os.WriteFile(filepath.Join(dir, "a.json"), []byte("old a"), 0o644) + os.WriteFile(filepath.Join(dir, "gone.json"), []byte("old gone"), 0o644) + // The second rename fails: its target is a non-empty folder. + os.MkdirAll(filepath.Join(dir, "b.json", "x"), 0o755) + err := applyFileChange(dir, scratch, []filePut{{"a.json", "new a"}, {"b.json", "new b"}}, []string{"gone.json"}) + if err == nil { + t.Fatal("expected the change to fail") + } + if b, _ := os.ReadFile(filepath.Join(dir, "a.json")); string(b) != "old a" { + t.Errorf("a.json not restored: %q", b) + } + if b, _ := os.ReadFile(filepath.Join(dir, "gone.json")); string(b) != "old gone" { + t.Errorf("gone.json not restored: %q", b) + } + if left, _ := os.ReadDir(scratch); len(left) != 0 { + t.Errorf("transaction folder left: %v", left) + } + if err := applyFileChange(dir, scratch, []filePut{{"a.json", "new a"}}, []string{"gone.json", "never.json"}); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(filepath.Join(dir, "a.json")); string(b) != "new a" { + t.Error("write") + } + if _, err := os.Stat(filepath.Join(dir, "gone.json")); !os.IsNotExist(err) { + t.Error("delete") + } +} + +func TestStorageRefusesNonBlockFiles(t *testing.T) { + s := newSite(t, false) + for _, file := range []string{"../x.json", ".env.json", "a/b.json", "x.ts"} { + _, err := s.h.store.commit(commitAttempt{Put: []filePut{{file, "{}"}}}) + if pe := toProtocolError(err); pe == nil || pe.Code != CodeInvalidBlock { + t.Errorf("%s: %v", file, err) + } + } +} + +// ---------------------------------------------------------------- assets + +func (s *site) upload(name, contentType string, body []byte, method string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, "/assets/"+name, bytes.NewReader(body)) + if contentType != "" { + r.Header.Set("Content-Type", contentType) + } + w := httptest.NewRecorder() + s.h.ServeAssets(w, r) + return w +} + +func TestAssets(t *testing.T) { + s := newSite(t, false) + png := []byte{0x89, 'P', 'N', 'G'} + w := s.upload("banner.png", "image/png", png, "PUT") + if w.Code != 201 || w.Body.String() != `{"path":"/assets/banner.png"}` { + t.Fatalf("upload: %d %s", w.Code, w.Body) + } + if b, _ := os.ReadFile(filepath.Join(s.root, "public", "assets", "banner.png")); !bytes.Equal(b, png) { + t.Error("stored bytes") + } + w = s.upload("banner.png", "image/png", []byte{1}, "PUT") + var out struct{ Path string } + json.Unmarshal(w.Body.Bytes(), &out) + if w.Code != 201 || out.Path == "/assets/banner.png" || !strings.HasPrefix(out.Path, "/assets/banner-") || !strings.HasSuffix(out.Path, ".png") { + t.Errorf("a taken name gets a suffix: %s", out.Path) + } + if b, _ := os.ReadFile(filepath.Join(s.root, "public", "assets", "banner.png")); !bytes.Equal(b, png) { + t.Error("never overwrites") + } + for _, c := range []struct{ name, typ string }{ + {"page.html", "text/html"}, {"evil.html", "image/png"}, {"evil.js", "image/jpeg"}, + {"logo.svg", "image/svg+xml"}, {"data.json", "application/json"}, {"x.png", ""}, + } { + if w := s.upload(c.name, c.typ, png, "PUT"); w.Code != 415 || !strings.Contains(w.Body.String(), `"code":-32600`) { + t.Errorf("%s as %s: %d", c.name, c.typ, w.Code) + } + } + if w := s.upload("Logo", "image/webp", png, "PUT"); w.Body.String() != `{"path":"/assets/Logo.webp"}` { + t.Errorf("extension added: %s", w.Body) + } + if w := s.upload("Photo.JPEG", "image/jpeg", png, "PUT"); w.Body.String() != `{"path":"/assets/Photo.jpeg"}` { + t.Errorf("extension lowercased: %s", w.Body) + } + if w := s.upload("x.png", "image/png", png, "POST"); w.Code != 405 || w.Header().Get("Allow") != "PUT" { + t.Errorf("POST: %d", w.Code) + } + if w := s.upload("empty.png", "image/png", nil, "PUT"); w.Code != 400 { + t.Errorf("empty: %d", w.Code) + } + if w := s.upload("", "image/png", png, "PUT"); w.Code != 400 { + t.Errorf("no name: %d", w.Code) + } + if w := s.upload("..%2F..%2Fescape.png", "image/png", png, "PUT"); w.Body.String() != `{"path":"/assets/escape.png"}` { + t.Errorf("stays inside the asset folder: %s", w.Body) + } + s.h.store.assetsMaxBytes = 3 + if w := s.upload("big.png", "image/png", png, "PUT"); w.Code != 413 || !strings.Contains(w.Body.String(), "uploads are limited to 3 bytes") { + t.Errorf("too big: %d %s", w.Code, w.Body) + } +} + +func TestSanitizeAssetName(t *testing.T) { + cases := map[string]string{ + "Bänner (1).PNG": "Banner-1.PNG", + "a%20b.png": "a-b.png", + "dir/sub\\file.png": "file.png", + "...hidden.png": "hidden.png", + "--x--.png": "x.png", + "x-.png": "x.png", + "ｆｕｌｌｗｉｄｔｈ.png": "fullwidth.png", + "%E0%A4%A.png": "E0-A4-A.png", + "..": "", + "日本語": "", + "a" + strings.Repeat("b", 300) + ".png": "a" + strings.Repeat("b", 195) + ".png", + } + for in, want := range cases { + if got := sanitizeAssetName(in); got != want { + t.Errorf("sanitizeAssetName(%q) = %q, want %q", in, got, want) + } + } + if got := suffixedAssetName("banner.jpg"); len(got) != len("banner-abcdef.jpg") || !strings.HasPrefix(got, "banner-") || !strings.HasSuffix(got, ".jpg") { + t.Errorf("suffix: %s", got) + } + if got := suffixedAssetName("noext"); !strings.HasPrefix(got, "noext-") || len(got) != len("noext-abcdef") { + t.Errorf("suffix without extension: %s", got) + } +} From b1881104e9ccb569b65d61af7d2c49bd6b8e2fd2 Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:56:55 -0300 Subject: [PATCH 09/20] feat(daemon): mount /_sandbox/rpc and /assets; bounded tree-lock wait Mounts the content protocol and asset uploads on the daemon behind its auth, and adds worktree.Lock.AcquireWithin so a write never lands after its client gave up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- packages/sandbox/README.md | 15 ++ packages/sandbox/daemon-go/README.md | 1 + .../daemon-go/internal/routes/content.go | 125 ++++++++++++ .../daemon-go/internal/routes/content_test.go | 179 ++++++++++++++++++ .../daemon-go/internal/worktree/lock.go | 21 +- packages/sandbox/daemon-go/main.go | 21 +- 6 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 packages/sandbox/daemon-go/internal/routes/content.go create mode 100644 packages/sandbox/daemon-go/internal/routes/content_test.go diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 668af4c53a..17d57d2210 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -233,6 +233,21 @@ Daemon control endpoints use the `/_sandbox/*` namespace, with `/health` at the root. AgentSandbox forwards those routes separately from the public preview contract. +For v8 (Blocks) sites the daemon also serves the Deco content protocol over the +working tree, so the site editor edits a sandbox the way it edits a local +`deco serve`: `POST /_sandbox/rpc` (JSON-RPC: `describe`, `schema.get`, +`blocks.list`, `blocks.apply`) and `PUT /_sandbox/assets/` (uploads into +`public/assets`). Both need the daemon token. Content lives in +`/.deco/blocks/*.json` and is committed and pushed like code; there is +no CDN draft in a sandbox. Commits take the worktree lock, so they serialize +with fs writes, publish, discard and autosave. The `.deco/.blocks.lock` and +`.deco/.tx-*/` files they use are listed in `.git/info/exclude`. The +implementation is a Go port of `@decocms/blocks/protocol`, and +`daemon-e2e/daemon.content-protocol.e2e.test.ts` runs the published conformance +suite and a byte-for-byte parity check against it. Bump the pinned +`@decocms/blocks` there when the protocol changes. `/_sandbox/decofile` (v7) is +unchanged and reflects protocol writes. + ## Export surface | Import | Purpose | diff --git a/packages/sandbox/daemon-go/README.md b/packages/sandbox/daemon-go/README.md index 1a65b13c5f..04194cab59 100644 --- a/packages/sandbox/daemon-go/README.md +++ b/packages/sandbox/daemon-go/README.md @@ -57,6 +57,7 @@ Two properties the consumer depends on, both asserted in `daemon-e2e/`: | `internal/auth/` | Bearer-token authentication | | `internal/telemetry/` | OTLP metrics export | | `internal/worktree/` | Worktree lock | +| `internal/content/` | Content protocol (`/_sandbox/rpc`, `/_sandbox/assets/*`): Go port of `@decocms/blocks/protocol`, checked by the conformance and parity e2e | ## Startup contract diff --git a/packages/sandbox/daemon-go/internal/routes/content.go b/packages/sandbox/daemon-go/internal/routes/content.go new file mode 100644 index 0000000000..a7a3034c78 --- /dev/null +++ b/packages/sandbox/daemon-go/internal/routes/content.go @@ -0,0 +1,125 @@ +package routes + +import ( + "net/http" + "path/filepath" + "sync" + "time" + + "github.com/decocms/studio/sandbox-daemon/internal/config" + "github.com/decocms/studio/sandbox-daemon/internal/content" + "github.com/decocms/studio/sandbox-daemon/internal/gitx" + "github.com/decocms/studio/sandbox-daemon/internal/paths" + "github.com/decocms/studio/sandbox-daemon/internal/worktree" +) + +// ContentDeps wires the content protocol (`POST /_sandbox/rpc`, `PUT +// /_sandbox/assets/`) to the working tree. +type ContentDeps struct { + RepoDir string + Store *config.Store + // TreeLock serializes commits and uploads with every other tree mutation + // (fs writes, publish, discard, rebase, autosave). Reads never take it. + TreeLock *worktree.Lock + // OnWrite gets the repo-relative path of every file a commit or upload + // touched: the same hook the fs routes call, so `file-changed`, the + // `decofile` version and the branch status follow protocol writes too. + OnWrite func(relPath string) + // ServerVersion is reported by describe. OPEN: no build stamps one yet. + ServerVersion string +} + +// Content serves the content protocol for the app root the workload config +// points at (the package path, like /_sandbox/decofile). One handler per app +// root: it holds that root's hash and body caches. +type Content struct { + deps ContentDeps + + mu sync.Mutex + root string + handler *content.Handler +} + +// treeLockWait bounds a protocol write's wait for the working-tree lock. +var treeLockWait = 10 * time.Second + +func NewContent(deps ContentDeps) *Content { + return &Content{deps: deps} +} + +func (c *Content) appRoot() string { + pmPath := "" + if cfg := c.deps.Store.Read(); cfg != nil { + pmPath = cfg.PmPath() + } + return paths.ResolvePmRoot(c.deps.RepoDir, pmPath) +} + +// repoRel is the slash-separated path of target inside the repo. +func (c *Content) repoRel(target string) string { + rel, err := filepath.Rel(c.deps.RepoDir, target) + if err != nil { + return target + } + return filepath.ToSlash(rel) +} + +func (c *Content) current() *content.Handler { + root := c.appRoot() + c.mu.Lock() + defer c.mu.Unlock() + if c.handler != nil && c.root == root { + return c.handler + } + // A commit's lock file and transaction folders must never reach a user + // branch through autosave or the shutdown `git add -A`. + decoRel := c.repoRel(filepath.Join(root, ".deco")) + gitx.EnsureExclude(c.deps.RepoDir, "/"+decoRel+"/.blocks.lock") + gitx.EnsureExclude(c.deps.RepoDir, "/"+decoRel+"/.tx-*/") + + store := content.NewFSStore(content.FSOptions{ + Root: root, + RepoRoot: c.deps.RepoDir, + ContainWithin: c.deps.RepoDir, + // Bounded well under Studio's 30 s proxy timeout: a write waiting on + // a long publish/rebase/autosave is refused (Unavailable, retry) + // instead of landing after the editor already reported it failed. + Exclusive: func() (func(), bool) { + if c.deps.TreeLock == nil { + return func() {}, true + } + return c.deps.TreeLock.AcquireWithin(treeLockWait) + }, + }) + blocksDir := filepath.Join(root, ".deco", "blocks") + assetsDir := filepath.Join(root, "public", "assets") + notify := func(path string) { + if c.deps.OnWrite != nil { + c.deps.OnWrite(c.repoRel(path)) + } + } + c.root = root + c.handler = content.NewHandler(content.Options{ + Store: store, + ServerName: "studio-sandbox-daemon", + ServerVersion: c.deps.ServerVersion, + OnCommit: func(files []string) { + for _, f := range files { + notify(filepath.Join(blocksDir, f)) + } + }, + OnAsset: func(name string) { notify(filepath.Join(assetsDir, name)) }, + }) + return c.handler +} + +// RPC serves `/_sandbox/rpc` (any method: non-POST answers 405, as the +// protocol requires). +func (c *Content) RPC(w http.ResponseWriter, r *http.Request) { + c.current().ServeRPC(w, r) +} + +// Assets serves `/_sandbox/assets/`. +func (c *Content) Assets(w http.ResponseWriter, r *http.Request) { + c.current().ServeAssets(w, r) +} diff --git a/packages/sandbox/daemon-go/internal/routes/content_test.go b/packages/sandbox/daemon-go/internal/routes/content_test.go new file mode 100644 index 0000000000..b7716fc88b --- /dev/null +++ b/packages/sandbox/daemon-go/internal/routes/content_test.go @@ -0,0 +1,179 @@ +package routes + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/decocms/studio/sandbox-daemon/internal/config" + "github.com/decocms/studio/sandbox-daemon/internal/worktree" +) + +func rpcPost(t *testing.T, c *Content, body string) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(http.MethodPost, "/_sandbox/rpc", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + c.RPC(w, r) + return w +} + +// A protocol write under the package path reports repo-relative paths through +// the fs routes' hook (file-changed, the decofile version, branch status), and +// keeps its lock and transaction files out of git. +func TestContentFollowsThePackagePathAndReportsWrites(t *testing.T) { + repo := t.TempDir() + if out, err := exec.Command("git", "init", "-q", repo).CombinedOutput(); err != nil { + t.Skipf("git init: %v %s", err, out) + } + app := filepath.Join(repo, "apps", "web") + os.MkdirAll(filepath.Join(app, ".deco", "blocks"), 0o755) + store := config.NewStore() + store.Hydrate(&config.TenantConfig{ + Application: &config.Application{ + PackageManager: &config.PackageManagerConfig{Path: config.Str("apps/web")}, + }, + }) + var written []string + c := NewContent(ContentDeps{ + RepoDir: repo, + Store: store, + TreeLock: &worktree.Lock{}, + OnWrite: func(p string) { written = append(written, p) }, + }) + + w := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"describe"}`) + if !strings.Contains(w.Body.String(), `"root":"apps/web"`) || !strings.Contains(w.Body.String(), `"dir":"apps/web/public/assets"`) { + t.Fatalf("describe: %s", w.Body) + } + w = rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"pages-Home Page":{"path":"/"},"Header":{}}}}`) + if strings.Contains(w.Body.String(), `"error"`) { + t.Fatalf("apply: %s", w.Body) + } + sort.Strings(written) + want := []string{"apps/web/.deco/blocks/Header.json", "apps/web/.deco/blocks/pages-Home%20Page.json"} + if !reflect.DeepEqual(written, want) { + t.Errorf("OnWrite: %v, want %v", written, want) + } + + r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1, 2})) + r.Header.Set("Content-Type", "image/png") + aw := httptest.NewRecorder() + c.Assets(aw, r) + if aw.Code != 201 || written[len(written)-1] != "apps/web/public/assets/logo.png" { + t.Errorf("upload: %d %s %v", aw.Code, aw.Body, written) + } + + exclude, _ := os.ReadFile(filepath.Join(repo, ".git", "info", "exclude")) + for _, line := range []string{"/apps/web/.deco/.blocks.lock", "/apps/web/.deco/.tx-*/"} { + if !strings.Contains(string(exclude), line+"\n") { + t.Errorf("exclude lacks %s:\n%s", line, exclude) + } + } +} + +// Commits wait for the working-tree lock, so a publish or an fs write never +// interleaves with one. +func TestContentCommitsTakeTheTreeLock(t *testing.T) { + repo := t.TempDir() + os.MkdirAll(filepath.Join(repo, ".deco", "blocks"), 0o755) + lock := &worktree.Lock{} + c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: lock}) + + release := lock.Acquire() + done := make(chan string, 1) + go func() { + done <- rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`).Body.String() + }() + select { + case body := <-done: + t.Fatalf("committed while the tree was locked: %s", body) + case <-time.After(200 * time.Millisecond): + } + // Reads never wait. + if w := rpcPost(t, c, `{"jsonrpc":"2.0","id":2,"method":"blocks.list"}`); w.Code != 200 { + t.Errorf("list under the lock: %d", w.Code) + } + release() + if body := <-done; strings.Contains(body, `"error"`) { + t.Errorf("apply: %s", body) + } +} + +// A write that can't get the tree lock in time is refused (Unavailable, retry +// later) and never lands afterwards — the editor has already given up on it. +func TestContentWritesGiveUpOnABusyTree(t *testing.T) { + prev := treeLockWait + treeLockWait = 100 * time.Millisecond + defer func() { treeLockWait = prev }() + repo := t.TempDir() + os.MkdirAll(filepath.Join(repo, ".deco", "blocks"), 0o755) + lock := &worktree.Lock{} + c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: lock}) + + release := lock.Acquire() + body := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`).Body.String() + if !strings.Contains(body, `"message":"the working tree is busy"`) || !strings.Contains(body, `"retryAfterMs":500`) { + t.Errorf("busy tree: %s", body) + } + r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1})) + r.Header.Set("Content-Type", "image/png") + aw := httptest.NewRecorder() + c.Assets(aw, r) + if aw.Code < 500 { + t.Errorf("upload on a busy tree: %d %s", aw.Code, aw.Body) + } + release() + for _, p := range []string{".deco/blocks/x.json", "public/assets/logo.png"} { + if _, err := os.Stat(filepath.Join(repo, p)); err == nil { + t.Errorf("%s landed after the request was refused", p) + } + } +} + +// A storage folder symlinked outside the working tree is never read or +// written through (stricter than the TS storage; see FSOptions.ContainWithin). +func TestContentStaysInsideTheWorkingTree(t *testing.T) { + repo, outside := t.TempDir(), t.TempDir() + os.MkdirAll(filepath.Join(outside, "blocks"), 0o755) + os.WriteFile(filepath.Join(outside, "blocks", "secret.json"), []byte(`{}`), 0o644) + if err := os.Symlink(outside, filepath.Join(repo, ".deco")); err != nil { + t.Skipf("symlink: %v", err) + } + os.MkdirAll(filepath.Join(outside, "pub"), 0o755) + os.Symlink(filepath.Join(outside, "pub"), filepath.Join(repo, "public")) + c := NewContent(ContentDeps{RepoDir: repo, Store: config.NewStore(), TreeLock: &worktree.Lock{}}) + + if w := rpcPost(t, c, `{"jsonrpc":"2.0","id":1,"method":"describe"}`); strings.Contains(w.Body.String(), `"error"`) { + t.Errorf("describe: %s", w.Body) + } + for _, body := range []string{ + `{"jsonrpc":"2.0","id":1,"method":"blocks.list"}`, + `{"jsonrpc":"2.0","id":1,"method":"blocks.apply","params":{"set":{"x":{}}}}`, + } { + if w := rpcPost(t, c, body); !strings.Contains(w.Body.String(), "resolves outside") { + t.Errorf("%s: %s", body, w.Body) + } + } + r := httptest.NewRequest(http.MethodPut, "/_sandbox/assets/logo.png", bytes.NewReader([]byte{1})) + r.Header.Set("Content-Type", "image/png") + aw := httptest.NewRecorder() + c.Assets(aw, r) + if aw.Code < 400 { + t.Errorf("upload: %d %s", aw.Code, aw.Body) + } + if entries, _ := os.ReadDir(filepath.Join(outside, "pub")); len(entries) != 0 { + t.Errorf("wrote outside the tree: %v", entries) + } + if _, err := os.Stat(filepath.Join(outside, "blocks", "x.json")); err == nil { + t.Error("committed outside the tree") + } +} diff --git a/packages/sandbox/daemon-go/internal/worktree/lock.go b/packages/sandbox/daemon-go/internal/worktree/lock.go index 986eb980a4..2d912c57e1 100644 --- a/packages/sandbox/daemon-go/internal/worktree/lock.go +++ b/packages/sandbox/daemon-go/internal/worktree/lock.go @@ -7,7 +7,10 @@ // publish or discard that runs mid-write commits or destroys a half-written file. package worktree -import "sync" +import ( + "sync" + "time" +) // Lock guards the working tree. Held by the mutating fs routes and by every git // operation that reads or rewrites the whole checkout. @@ -27,3 +30,19 @@ func (l *Lock) Acquire() func() { l.mu.Lock() return l.mu.Unlock } + +// AcquireWithin is Acquire bounded by d: ok is false (and nothing is held) +// when the tree stayed busy that long. For callers whose client gives up +// after a while, so a write never lands after its request was abandoned. +func (l *Lock) AcquireWithin(d time.Duration) (release func(), ok bool) { + deadline := time.Now().Add(d) + for { + if l.mu.TryLock() { + return l.mu.Unlock, true + } + if !time.Now().Before(deadline) { + return nil, false + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/packages/sandbox/daemon-go/main.go b/packages/sandbox/daemon-go/main.go index d9393859d2..f4e50815fe 100644 --- a/packages/sandbox/daemon-go/main.go +++ b/packages/sandbox/daemon-go/main.go @@ -126,6 +126,7 @@ type sandboxHandlers struct { tasksList, tasksGet, tasksDelete http.HandlerFunc tasksKill, tasksKillAll, tasksStream http.HandlerFunc toolsSync, exec http.HandlerFunc + contentRPC, contentAssets http.HandlerFunc fs, git, setup map[string]http.HandlerFunc } @@ -624,6 +625,15 @@ func (d *daemon) registerSandboxRoutes(mux *http.ServeMux, pre string, h sandbox mux.HandleFunc("POST "+pre+"/tasks/{id}/kill", d.authed(h.tasksKill)) mux.HandleFunc("POST "+pre+"/tools/sync", d.authed(h.toolsSync)) + + // The content protocol over the working tree (v8 sites). Every method but + // OPTIONS (the CORS preflight) reaches the handler, which answers anything + // but POST (PUT for uploads) with the protocol's own 405. Commits and + // uploads take the tree lock themselves; reads never do. + for _, method := range []string{"GET", "POST", "PUT", "PATCH", "DELETE"} { + mux.HandleFunc(method+" "+pre+"/rpc", d.authed(h.contentRPC)) + mux.HandleFunc(method+" "+pre+"/assets/{name...}", d.authed(h.contentAssets)) + } for _, step := range []string{"clone", "install", "start"} { mux.HandleFunc("POST "+pre+"/setup/"+step, d.authed(h.setup[step])) } @@ -1225,6 +1235,13 @@ func main() { GetConfigured: func() bool { return d.store.Read() != nil }, }) + contentRoutes := routes.NewContent(routes.ContentDeps{ + RepoDir: repoDir, + Store: d.store, + TreeLock: &d.treeLock, + OnWrite: fsDeps.OnWorkingTreeWrite, + }) + h := sandboxHandlers{ scripts: routes.Scripts(func() []string { if cached, ok := d.orchestrator.DiscoveredScripts(); ok { @@ -1252,7 +1269,9 @@ func main() { GetDecofileVersion: d.getDecofileVersion, OnDecofileVersionUnknown: func() { go d.announceDecofileVersion() }, }), - decofile: routes.Decofile(d.decofileDeps), + decofile: routes.Decofile(d.decofileDeps), + contentRPC: contentRoutes.RPC, + contentAssets: contentRoutes.Assets, configRead: routes.ConfigRead(routes.ConfigDeps{ DaemonBootId: bootId, Store: d.store, From f9c02ed23b2e9939160f8e8ed768396b5aefa197 Mon Sep 17 00:00:00 2001 From: gimenes Date: Thu, 8 Oct 2026 14:57:03 -0300 Subject: [PATCH 10/20] feat(hosted): delivery bucket settings, store and CDN purge Adds the hosted delivery bucket settings (R2/S3), `deliveryStore` (read/write release objects with ETag handling) and `deliveryPurge` (CDN purge with one retry), plus shared test helpers. Nothing calls them yet; the routes land in #10/#11. A temporary knip `ignoreIssues` entry for `apps/api/src/hosted/*.ts` (unused exports only) is removed in PR 11 of this track. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- apps/api/src/file-storage/upload-policy.ts | 2 +- apps/api/src/hosted/delivery-purge.test.ts | 169 +++++++++++++++ apps/api/src/hosted/delivery-purge.ts | 98 +++++++++ apps/api/src/hosted/delivery-store.test.ts | 61 ++++++ apps/api/src/hosted/delivery-store.ts | 219 +++++++++++++++++++ apps/api/src/hosted/hosted-test-helpers.ts | 234 +++++++++++++++++++++ apps/api/src/settings/resolve-config.ts | 15 +- apps/api/src/settings/types.ts | 20 ++ knip.jsonc | 4 +- 9 files changed, 819 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/hosted/delivery-purge.test.ts create mode 100644 apps/api/src/hosted/delivery-purge.ts create mode 100644 apps/api/src/hosted/delivery-store.test.ts create mode 100644 apps/api/src/hosted/delivery-store.ts create mode 100644 apps/api/src/hosted/hosted-test-helpers.ts diff --git a/apps/api/src/file-storage/upload-policy.ts b/apps/api/src/file-storage/upload-policy.ts index 14aa4195c4..05678d0664 100644 --- a/apps/api/src/file-storage/upload-policy.ts +++ b/apps/api/src/file-storage/upload-policy.ts @@ -33,7 +33,7 @@ export const MAX_UPLOAD_BYTES = 100 * 1024 * 1024; * browsers do NOT execute scripts when SVG is loaded as a pure image. * - Top-level navigation, ``, `