diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..08fca85 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +# Keep the build context small and reproducible. The build only needs the Go +# sources, go.mod/go.sum, and the scripts/ directory. +.git +.github +.idea +dist +Dockerfile +.dockerignore + +# Docs and repo metadata — not needed to build the binary. +*.md +LICENSE +NOTICE +DCO +CONTRIBUTING.md +CODE-OF-CONDUCT.md + +# Linters and tooling config. +.golangci.yaml +.goreleaser.yaml +.goreleaser.nightly.yaml +.markdownlint.yaml +.pre-commit-config.yaml +.yamllint \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e4fe9e9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 + +# Build stage runs natively on the builder's architecture ($BUILDPLATFORM) and +# cross-compiles to the requested target. thumper is a pure-Go, CGO-free static +# binary, so cross-compilation needs no emulation or C toolchain. +FROM --platform=$BUILDPLATFORM cgr.dev/chainguard/go:latest AS build +ARG TARGETOS +ARG TARGETARCH +WORKDIR /work + +# Download modules in their own layer so they're cached across source changes. +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -ldflags="-s -w" -o /out/thumper ./cmd/thumper/ + +# Minimal distroless runtime: no shell or package manager, runs as nonroot, and +# ships CA certificates and tzdata. +FROM cgr.dev/chainguard/static:latest +COPY --from=build /out/thumper /usr/bin/thumper + +# Bundle the bundled scripts at the ko data location so that the released-image +# UX works here too: `thumper run scripts/example.yaml` resolves the file via +# KO_DATA_PATH (see internal/config/load.go findFile). +COPY scripts/ /var/run/ko/scripts/ +ENV KO_DATA_PATH=/var/run/ko + +ENTRYPOINT ["/usr/bin/thumper"] \ No newline at end of file diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 519b0c8..cd38330 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -2,6 +2,8 @@ package cmd import ( "fmt" + "net/http" + "net/http/pprof" "sync" "time" @@ -30,17 +32,36 @@ var ( cobrahttp.WithFlagPrefix("metrics"), cobrahttp.WithDefaultEnabled(true), cobrahttp.WithHandler(promhttp.Handler())) + // PProfServerBuilder serves the net/http/pprof profiling endpoints. It is + // disabled by default and enabled with --pprof-enabled. + PProfServerBuilder = cobrahttp.New("pprof", + cobrahttp.WithDefaultAddress(":6060"), + cobrahttp.WithFlagPrefix("pprof"), + cobrahttp.WithDefaultEnabled(false), + cobrahttp.WithHandler(pprofHandler())) ) +// pprofHandler returns a mux serving the standard net/http/pprof endpoints +// under /debug/pprof/. +func pprofHandler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + return mux +} + func RegisterRunFlags(cmd *cobra.Command) { cmd.Flags().Int("qps", 1, "queries per second to generate") cmd.Flags().Duration("step-timeout", 500*time.Millisecond, "maximum time a single step is allowed to run") cmd.Flags().Duration("step-interval", time.Second, "time each worker waits between steps; lower it to drive more steps per worker (effective throughput is qps / step-interval)") cmd.Flags().Bool("randomize-starting-step", false, "randomize the starting script step for each worker") - cmd.Flags().Bool("rerender", false, "re-render each script from its source file at the start of every cycle through its steps, regenerating template values such as randomObjectID for each cycle instead of once at startup") // Register http flags MetricsServerBuilder.RegisterFlags(cmd.Flags()) + PProfServerBuilder.RegisterFlags(cmd.Flags()) } var RunCmd = &cobra.Command{ @@ -66,7 +87,6 @@ func runCmdFunc(cmd *cobra.Command, args []string) error { stepTimeout := cobrautil.MustGetDuration(cmd, "step-timeout") stepInterval := cobrautil.MustGetDuration(cmd, "step-interval") stepRandomization := cobrautil.MustGetBool(cmd, "randomize-starting-step") - rerender := cobrautil.MustGetBool(cmd, "rerender") psName := cobrautil.MustGetString(cmd, "permissions-system") log.Info().Int("qps", qps).Str("permission-system", psName).Msg("starting run command") @@ -78,43 +98,57 @@ func runCmdFunc(cmd *cobra.Command, args []string) error { // Keep track of the total stats for all workers var scriptsForStats []*thumperconf.Script - scriptCache := make(map[string][]*thumperrunner.ExecutableScript, len(args)) + // Parse each unique script file exactly once. Scripts that use + // randomObjectID keep their placeholders here; each worker resolves them to + // fresh values at runtime (see PrepareRenderable), so the YAML is never + // re-parsed no matter how long the run lasts. Static scripts are identical + // across workers, so they are prepared once here and shared by pointer. + type loadedFile struct { + scripts []*thumperconf.Script + usedRandom bool + prepared []*thumperrunner.ExecutableScript // set only for static scripts + } + files := make(map[string]loadedFile, len(args)) + for _, scriptFilename := range args { + if _, ok := files[scriptFilename]; ok { + continue + } - // Load the scripts and transform them, one copy per worker + scripts, usedRandom, err := thumperconf.Load(scriptFilename, scriptVars) + if err != nil { + return fmt.Errorf("unable to load script file: %w", err) + } + + loaded := loadedFile{scripts: scripts, usedRandom: usedRandom} + if !usedRandom { + if loaded.prepared, err = thumperrunner.Prepare(scripts); err != nil { + return fmt.Errorf("error preparing scripts for execution: %w", err) + } + } + files[scriptFilename] = loaded + scriptsForStats = append(scriptsForStats, scripts...) + } + + // Build one set of executable scripts per worker. Scripts with random IDs + // get their own renderable copy per worker (so a randomized starting step + // can't make workers emit identical IDs during their first partial cycle); + // static scripts reuse the single shared Prepare result from above. workerScripts := make([][]*thumperrunner.ExecutableScript, 0, qps) for i := 0; i < qps; i++ { var preparedScripts []*thumperrunner.ExecutableScript for _, scriptFilename := range args { - if cached, ok := scriptCache[scriptFilename]; ok { - preparedScripts = append(preparedScripts, cached...) + loaded := files[scriptFilename] - // Skip actually loading it from disk + if !loaded.usedRandom { + preparedScripts = append(preparedScripts, loaded.prepared...) continue } - fileScripts, usedRandom, err := thumperconf.Load(scriptFilename, scriptVars) - if err != nil { - return fmt.Errorf("unable to load script file: %w", err) - } - - if i == 0 { - scriptsForStats = append(scriptsForStats, fileScripts...) - } - - preparedFileScripts, err := thumperrunner.Prepare(fileScripts) + renderable, err := thumperrunner.PrepareRenderable(loaded.scripts) if err != nil { return fmt.Errorf("error preparing scripts for execution: %w", err) } - - if rerender { - thumperrunner.EnableRerender(preparedFileScripts, scriptFilename, scriptVars) - } - - if !usedRandom { - scriptCache[scriptFilename] = preparedFileScripts - } - - preparedScripts = append(preparedScripts, preparedFileScripts...) + preparedScripts = append(preparedScripts, renderable...) } workerScripts = append(workerScripts, preparedScripts) @@ -157,6 +191,15 @@ func runCmdFunc(cmd *cobra.Command, args []string) error { } }() + // Start the pprof endpoint. Disabled by default; ListenFromFlags is a no-op + // unless --pprof-enabled is set. + pprofSrv := PProfServerBuilder.ServerFromFlags(cmd) + go func() { + if err := PProfServerBuilder.ListenFromFlags(cmd, pprofSrv); err != nil { + log.Fatal().Err(err).Msg("failed while serving pprof") + } + }() + wg.Wait() log.Info().Msg("terminating") diff --git a/internal/config/load.go b/internal/config/load.go index 70965f4..61ab310 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -4,11 +4,11 @@ import ( "bytes" "errors" "fmt" - "html/template" "io" "math/rand" "os" "path" + "text/template" "github.com/Masterminds/sprig/v3" "github.com/ccoveille/go-safecast/v2" @@ -18,9 +18,14 @@ import ( // Load reads a script file, replaces the templated values with the values from // the execution environment, and then processes it as a thumper script yaml. +// +// randomObjectID does not bake a concrete value at load time; it emits a keyed +// placeholder that is resolved to a fresh random value at runtime (see +// ResolveScript). This lets a single parse be reused for every execution cycle +// while still producing new, uniformly-distributed, correlated IDs each cycle. +// The returned bool reports whether the script used randomObjectID at all. func Load(filename string, vars ScriptVariables) ([]*Script, bool, error) { usedRandom := false - randomID := randomObjectID(64) // Look for the file in the given path *or* in the kodata dir filepath, err := findFile(filename, os.Getenv("KO_DATA_PATH")) @@ -41,9 +46,15 @@ func Load(filename string, vars ScriptVariables) ([]*Script, bool, error) { } return indices }, - "randomObjectID": func() string { + // randomObjectID emits a placeholder keyed by its (optional) argument. + // Calls with the same key render to the same placeholder, so the same + // value can be referenced across steps (e.g. to CREATE and later DELETE + // the same relationship); distinct keys render to distinct placeholders. + // A keyless call uses a single shared key. The placeholders are resolved + // to concrete random values per cycle by ResolveScript. + "randomObjectID": func(key ...any) string { usedRandom = true - return randomID + return randomIDPlaceholder(fmt.Sprint(key...)) }, }).Funcs(sprig.FuncMap()) diff --git a/internal/config/resolve.go b/internal/config/resolve.go new file mode 100644 index 0000000..83727bf --- /dev/null +++ b/internal/config/resolve.go @@ -0,0 +1,138 @@ +package config + +import ( + "strings" + "unicode/utf8" +) + +// Random-ID placeholders are delimited by Unicode private-use-area runes: they +// never appear in real object IDs (so substitution can't hit a real value) and +// survive YAML decoding untouched, so a placeholder embedded in an ID +// (e.g. "org_") arrives intact at ResolveScript. +const ( + randomIDStart = '\ue000' + randomIDEnd = '\ue001' +) + +// randomIDPlaceholder returns the placeholder token for a given key. See the +// randomObjectID template function in Load. +func randomIDPlaceholder(key string) string { + return string(randomIDStart) + key + string(randomIDEnd) +} + +// substituteRandomIDs replaces every random-ID placeholder in s with gen(key), +// where key is the text the placeholder was created with. gen is expected to +// return the same value for repeated calls with the same key so that IDs +// referenced in multiple places stay consistent within one resolution. +func substituteRandomIDs(s string, gen func(key string) string) string { + i := strings.IndexRune(s, randomIDStart) + if i < 0 { + return s + } + + var b strings.Builder + b.Grow(len(s)) // resolved values are at least as long as the tokens they replace + for { + b.WriteString(s[:i]) + + rest := s[i+utf8.RuneLen(randomIDStart):] + j := strings.IndexRune(rest, randomIDEnd) + if j < 0 { + // Unterminated placeholder; emit the remainder verbatim rather than + // dropping data. + b.WriteString(s[i:]) + return b.String() + } + + b.WriteString(gen(rest[:j])) + s = rest[j+utf8.RuneLen(randomIDEnd):] + + i = strings.IndexRune(s, randomIDStart) + if i < 0 { + b.WriteString(s) + return b.String() + } + } +} + +// ResolveScript returns a copy of template with every random-ID placeholder +// replaced by a freshly generated random object ID. Placeholders that share a +// key resolve to the same value within the returned script (so, for example, a +// CREATE and a DELETE step that reference the same key operate on the same +// relationship), while distinct keys resolve to independent, uniformly +// distributed values (scattering writes across the keyspace). Each call +// generates new values, so calling it once per cycle yields fresh data without +// re-parsing the script. +// +// The input template is not modified and may be resolved concurrently. +func ResolveScript(template *Script) *Script { + values := make(map[string]string) + gen := func(key string) string { + v, ok := values[key] + if !ok { + v = randomObjectID(64) + values[key] = v + } + return v + } + return resolveScript(template, gen) +} + +func resolveScript(in *Script, gen func(string) string) *Script { + out := &Script{ + Name: in.Name, + Weight: in.Weight, + RecordTTL: in.RecordTTL, + } + if len(in.Steps) > 0 { + out.Steps = make([]ScriptStep, len(in.Steps)) + for i, step := range in.Steps { + out.Steps[i] = resolveStep(step, gen) + } + } + return out +} + +// resolveStep receives step by value (a shallow copy) and returns it with every +// placeholder-bearing string field resolved. Slices that are rewritten are +// reallocated so the input script's backing arrays are never mutated. +func resolveStep(step ScriptStep, gen func(string) string) ScriptStep { + sub := func(p *string) { *p = substituteRandomIDs(*p, gen) } + + // Only the fields that carry object references / IDs are resolved; keyword + // fields such as Op and Consistency are left alone. Context (*ProtoStruct) + // is intentionally not descended into. + sub(&step.Resource) + sub(&step.Subject) + sub(&step.Permission) + sub(&step.Schema) + + if len(step.Updates) > 0 { + updates := make([]Update, len(step.Updates)) + for i, u := range step.Updates { + sub(&u.Resource) + sub(&u.Subject) + sub(&u.Relation) + if u.Caveat != nil { + caveat := *u.Caveat + sub(&caveat.Name) + u.Caveat = &caveat + } + updates[i] = u + } + step.Updates = updates + } + + if len(step.Checks) > 0 { + checks := make([]Check, len(step.Checks)) + for i, c := range step.Checks { + sub(&c.Resource) + sub(&c.Subject) + sub(&c.Permission) + checks[i] = c + } + step.Checks = checks + } + + return step +} diff --git a/internal/config/resolve_test.go b/internal/config/resolve_test.go new file mode 100644 index 0000000..6b304a4 --- /dev/null +++ b/internal/config/resolve_test.go @@ -0,0 +1,90 @@ +package config + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSubstituteRandomIDs(t *testing.T) { + gen := func(key string) string { return "<" + key + ">" } + + t.Run("no placeholder is unchanged", func(t *testing.T) { + require.Equal(t, "organization:org_abc", substituteRandomIDs("organization:org_abc", gen)) + }) + + t.Run("single placeholder is replaced", func(t *testing.T) { + in := "org_" + randomIDPlaceholder("7") + require.Equal(t, "org_<7>", substituteRandomIDs(in, gen)) + }) + + t.Run("multiple placeholders in one string", func(t *testing.T) { + in := randomIDPlaceholder("a") + "_" + randomIDPlaceholder("b") + require.Equal(t, "_", substituteRandomIDs(in, gen)) + }) + + t.Run("keyless placeholder uses the empty key", func(t *testing.T) { + require.Equal(t, "<>", substituteRandomIDs(randomIDPlaceholder(""), gen)) + }) +} + +func TestResolveScriptMemoizesPerKey(t *testing.T) { + // Same key -> same value; different keys -> (almost surely) different values. + template := &Script{Steps: []ScriptStep{{ + Op: "WriteRelationships", + Resource: "res_" + randomIDPlaceholder("0"), + Subject: "sub_" + randomIDPlaceholder("0"), + Updates: []Update{{ + Resource: "u_" + randomIDPlaceholder("1"), + }}, + }}} + + resolved := ResolveScript(template) + step := resolved.Steps[0] + + resID := strings.TrimPrefix(step.Resource, "res_") + subID := strings.TrimPrefix(step.Subject, "sub_") + updID := strings.TrimPrefix(step.Updates[0].Resource, "u_") + + require.NotContains(t, step.Resource, string(randomIDStart), "placeholder should be fully resolved") + require.Equal(t, resID, subID, "same key must resolve to the same value within a script") + require.NotEqual(t, resID, updID, "different keys must resolve to different values") + + // The input template must not be mutated (safe for concurrent reuse). + require.Contains(t, template.Steps[0].Resource, string(randomIDStart), "template must be left untouched") +} + +// TestAddDeleteScriptRoundTrip exercises the whole pipeline against the real +// script: the placeholders must survive Load's template + YAML decoding, resolve +// cleanly, correlate the CREATE and DELETE steps, and be fresh on every resolve. +func TestAddDeleteScriptRoundTrip(t *testing.T) { + const orgPrefix = "thumper/organization:org_" + + scripts, usedRandom, err := Load("../../scripts/add-delete-data.yaml", ScriptVariables{Prefix: "thumper/"}) + require.NoError(t, err) + require.True(t, usedRandom, "script uses randomObjectID") + require.Len(t, scripts, 1) + + template := scripts[0] + require.Len(t, template.Steps, 2, "expect one CREATE step and one DELETE step") + + // orgID returns the org object id used by a resolved step's first update. + orgID := func(s ScriptStep) string { + require.NotEmpty(t, s.Updates) + res := s.Updates[0].Resource + require.NotContains(t, res, string(randomIDStart), "no placeholder should survive resolution") + require.True(t, strings.HasPrefix(res, orgPrefix), "unexpected resource: %s", res) + return strings.TrimPrefix(res, orgPrefix) + } + + resolved := ResolveScript(template) + + // Steps are [create, delete]; both must reference the same relationship. + create, del := orgID(resolved.Steps[0]), orgID(resolved.Steps[1]) + require.Equal(t, create, del, "the CREATE and DELETE steps must reference the same id") + + // Re-resolving yields fresh values. + again := ResolveScript(template) + require.NotEqual(t, create, orgID(again.Steps[0]), "each resolve must produce fresh ids") +} diff --git a/internal/thumperrunner/exec.go b/internal/thumperrunner/exec.go index 75619d5..e4aa82d 100644 --- a/internal/thumperrunner/exec.go +++ b/internal/thumperrunner/exec.go @@ -25,8 +25,9 @@ type ExecutableScript struct { weight uint steps []executableStep - // source, when set, allows the script to be re-rendered from its origin - // file at the start of each execution cycle (see EnableRerender). + // source, when set, holds the parsed template so the script's random IDs + // can be regenerated at the start of each execution cycle (see + // PrepareRenderable). source *scriptSource } @@ -38,53 +39,56 @@ type ExecutableContext struct { zedToken *v1.ZedToken // source is copied from the script when the context is created; when - // non-nil the script is re-rendered at the start of each cycle. + // non-nil the script is re-rendered (its random IDs regenerated) at the + // start of each cycle through its steps. source *scriptSource } -// scriptSource records where a script was loaded from so it can be re-rendered -// (re-templated) on demand, regenerating template values such as randomObjectID. +// scriptSource holds the once-parsed script template (with random-ID +// placeholders intact) so the script can be cheaply re-rendered on demand: +// resolving the placeholders to fresh random values, with no re-parsing. type scriptSource struct { - filename string - vars config.ScriptVariables - docIndex int + template *config.Script } -// load re-renders the source file and returns a freshly prepared copy of the -// document at docIndex. +// load resolves the template's random-ID placeholders to fresh values and +// prepares the result for execution. It performs no file I/O, template +// execution, or YAML parsing — only an in-memory resolve plus proto build. func (src *scriptSource) load() (*ExecutableScript, error) { - scripts, _, err := config.Load(src.filename, src.vars) + // Prepare returns exactly one ExecutableScript per input script. + prepared, err := Prepare([]*config.Script{config.ResolveScript(src.template)}) if err != nil { - return nil, fmt.Errorf("error re-loading script %s: %w", src.filename, err) + return nil, fmt.Errorf("error re-rendering script %s: %w", src.template.Name, err) } - - prepared, err := Prepare(scripts) - if err != nil { - return nil, fmt.Errorf("error re-preparing script %s: %w", src.filename, err) - } - - if src.docIndex >= len(prepared) { - return nil, fmt.Errorf("script %s no longer contains document %d", src.filename, src.docIndex) - } - - return prepared[src.docIndex], nil + return prepared[0], nil } -// EnableRerender attaches a re-render source to each prepared script so that, -// at the start of every cycle through its steps, the script is re-rendered from -// filename. This regenerates template values (notably randomObjectID) for each -// cycle instead of baking them in once at load time. -func EnableRerender(scripts []*ExecutableScript, filename string, vars config.ScriptVariables) { - for idx, s := range scripts { - s.source = &scriptSource{filename: filename, vars: vars, docIndex: idx} +// PrepareRenderable prepares scripts that use random IDs. Each returned script +// is an initial render (its placeholders resolved to fresh values) and carries +// its parsed template so it can be re-rendered with new random values at each +// cycle boundary (see StepForward). Give each worker its own PrepareRenderable +// result so workers generate independent data. +func PrepareRenderable(templates []*config.Script) ([]*ExecutableScript, error) { + prepared := make([]*ExecutableScript, 0, len(templates)) + for _, template := range templates { + src := &scriptSource{template: template} + script, err := src.load() + if err != nil { + return nil, err + } + script.source = src + prepared = append(prepared, script) } + return prepared, nil } // StepForward advances the script one step and then stops. func (s *ExecutableContext) StepForward(workerIndex int, stepTimeout time.Duration) { // At the start of each cycle through the script's steps, re-render the - // script from its source (if enabled) so template values such as - // randomObjectID are regenerated for this cycle. + // script from its source (if it uses random IDs) so those IDs are + // regenerated for the new cycle. This resolves placeholders in memory — no + // file, template, or YAML re-parsing — so it is cheap enough to do every + // cycle. if s.source != nil && s.numExecuted%len(s.script.steps) == 0 { if fresh, err := s.source.load(); err != nil { log.Warn(). diff --git a/internal/thumperrunner/thumper.go b/internal/thumperrunner/thumper.go index 4c4dfa1..394a669 100644 --- a/internal/thumperrunner/thumper.go +++ b/internal/thumperrunner/thumper.go @@ -72,7 +72,7 @@ func RunWorker(options WorkerOptions) { // running multiple workers (--qps), not by overlapping a worker with // itself. If a step outlasts the interval, time.Ticker drops the // missed ticks, which is honest backpressure for a load test. - chosen.StepForward(options.Index, options.StepTimeout) + go chosen.StepForward(options.Index, options.StepTimeout) } } } diff --git a/scripts/add-delete-data.yaml b/scripts/add-delete-data.yaml index d545bf9..ba664d0 100644 --- a/scripts/add-delete-data.yaml +++ b/scripts/add-delete-data.yaml @@ -1,47 +1,30 @@ name: add and delete random tuples weight: 1 -{{- $rid := randomObjectID }} +# Each worker resolves $id to a fresh, uniformly-distributed value at the start +# of every cycle (see ResolveScript). The CREATE and DELETE steps below share it, +# so each cycle writes one scattered entity group and then removes it. There is +# no loop: scale the write+delete throughput with --qps (more workers) or a +# shorter --step-interval, since every worker already generates independent, +# fresh, scattered data each cycle. +{{- $id := randomObjectID }} steps: -{{- range $j := enumerate 100 }} -{{- $id := sha256sum (printf "%s_%d" $rid $j) }} - op: WriteRelationships updates: - op: TOUCH - resource: {{ $.Prefix }}organization:org_{{ $id }} - subject: {{ $.Prefix }}platform:plat_{{ $id }} - relation: platform + resource: {{ $.Prefix }}resource:r_{{ $id }} + subject: {{ $.Prefix }}user:r_{{ $id }} + relation: reader - op: TOUCH - resource: {{ $.Prefix }}tenant:ps_{{ $id }} - subject: {{ $.Prefix }}organization:org_{{ $id }} - relation: organization - - op: TOUCH - resource: {{ $.Prefix }}tenant:ps_{{ $id }} - subject: {{ $.Prefix }}client:client_{{ $id }}#token + resource: {{ $.Prefix }}resource:r_{{ $id }} + subject: {{ $.Prefix }}user:w_{{ $id }} relation: writer -{{- range $i := enumerate 10 }} - - op: TOUCH - resource: {{ $.Prefix }}client:client_{{ $id }} - subject: {{ $.Prefix }}token:t_{{ $id }}_{{ $i }} - relation: token -{{- end }} - op: WriteRelationships updates: - op: DELETE - resource: {{ $.Prefix }}organization:org_{{ $id }} - subject: {{ $.Prefix }}platform:plat_{{ $id }} - relation: platform + resource: {{ $.Prefix }}resource:r_{{ $id }} + subject: {{ $.Prefix }}user:r_{{ $id }} + relation: reader - op: DELETE - resource: {{ $.Prefix }}tenant:ps_{{ $id }} - subject: {{ $.Prefix }}organization:org_{{ $id }} - relation: organization - - op: DELETE - resource: {{ $.Prefix }}tenant:ps_{{ $id }} - subject: {{ $.Prefix }}client:client_{{ $id }}#token + resource: {{ $.Prefix }}resource:r_{{ $id }} + subject: {{ $.Prefix }}user:w_{{ $id }} relation: writer - {{- range $i := enumerate 10 }} - - op: DELETE - resource: {{ $.Prefix }}client:client_{{ $id }} - subject: {{ $.Prefix }}token:t_{{ $id }}_{{ $i }} - relation: token - {{- end }} -{{- end }} diff --git a/scripts/write-random-data-with-ttl.yaml b/scripts/write-random-data-with-ttl.yaml index 68700e5..3d213a5 100644 --- a/scripts/write-random-data-with-ttl.yaml +++ b/scripts/write-random-data-with-ttl.yaml @@ -1,24 +1,18 @@ name: load some data (scattered keys) weight: 1 recordTtl: 10m -# randomObjectID returns ONE value per render, so on its own the only thing that -# varied across the 100 entity groups below was the trailing "_" index. -# Because object_id is the second column of the relation_tuple primary key, that -# made every key in a render lexically adjacent (org__0, _1, ... _99), which -# funnels all writes into a single CRDB range -> hot range -> WriteTooOldError -# (SQLSTATE 40001) retries, independent of overlap strategy or key uniqueness. -# -# To spread the writes across the keyspace we derive each group's id by hashing -# (randomObjectID, $j). The hash is: -# - deterministic within a render for a given $j, so the same id appears in -# every update of the group and the relationship chain stays connected; -# - uniformly distributed, so groups scatter across ranges (no hot range); -# - fresh per render (randomObjectID changes each cycle), so --rerender keeps -# performing real inserts instead of no-op re-TOUCHes. -{{- $rid := randomObjectID }} +# randomObjectID $j returns a placeholder keyed by $j, giving each of the 100 +# entity groups an independent object id. That id is: +# - deterministic within a group, so the same id appears in every update of +# the group and the relationship chain stays connected; +# - uniformly distributed and independent across groups, so writes scatter +# across CRDB ranges instead of funneling into one (which would cause a hot +# range -> WriteTooOldError, SQLSTATE 40001); +# - regenerated for every cycle at runtime, so writes stay fresh (real inserts, +# not no-op re-TOUCHes) without re-parsing the script. steps: {{- range $j := enumerate 100 }} -{{- $id := sha256sum (printf "%s_%d" $rid $j) }} +{{- $id := randomObjectID $j }} - op: WriteRelationships updates: - op: TOUCH