From 6b767d983815ebdecc485bf8a8fcbfad422c5829 Mon Sep 17 00:00:00 2001 From: Mukesh Kumar Chaudhary Date: Mon, 16 Nov 2026 11:47:09 +0545 Subject: [PATCH] refactor!: one way to hold, observe and approve writes - Options.Plan becomes Options.HoldWrites. Held writes return *agent.Pending like any write that needs approval, and Result.Planned is removed. - Options.OnCall is removed. OnEvent carries every call, with its input, and taskreport's audit log reads call events. - agent.Apply is removed. Plans are approved through agent.Plans. - codemode.WithApproval(ctx) replaces setting Recorder.Approved, which is now unexported. - Call.Planned is renamed Held, and Recorder.Pending() is renamed Held(). None of these were released. --- CHANGELOG.md | 9 ++- SECURITY.md | 2 +- agent/agent.go | 46 +++++--------- agent/effects_test.go | 42 ++++++------ agent/event.go | 4 +- agent/plan.go | 25 ++------ codemode/codemode.go | 2 +- codemode/record.go | 35 ++++++---- codemode/source_test.go | 6 +- docs/content/approval.md | 11 ++-- docs/content/configuration.md | 17 +++-- docs/content/environments.md | 13 ++-- docs/content/existing-app.md | 95 +++++++++++++++------------- docs/content/scaling-up.md | 11 ++-- docs/content/tenants.md | 6 +- docs/content/using-it.md | 2 +- docs/content/worker.md | 2 +- docs/content/writes.md | 52 +++++++-------- examples/shop/README.md | 2 +- examples/taskreport/server/limits.go | 31 +++++---- examples/taskreport/server/server.go | 21 ++++-- mcpserver/mcpserver.go | 6 +- 22 files changed, 219 insertions(+), 221 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b0d6c5..bd90424 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,7 +137,14 @@ version is 0, a minor bump may break the API. - `mcpsource` maps JSON Schema `integer` to `integer` rather than `number`, records `required` fields, and falls back to a tool's title or name when it has no description. `mcpsource.Source` is now an alias of `codemode.Source`. -- `agent.Apply` runs writes that need approval: applying is the approval. +- One way to hold writes, one hook, one way to approve: + - `Options.Plan` is now `Options.HoldWrites`, and held writes come back as + `*agent.Pending` like any write needing approval. `Result.Planned` is gone. + - `Options.OnCall` is gone; `OnEvent` carries every call, with its input. + - `agent.Apply` is gone; plans are approved through `agent.Plans`. + - `codemode.WithApproval(ctx)` runs writes a person already approved, such as + a button they pressed, instead of setting `Recorder.Approved`. + - `codemode.Call.Planned` is `Held`, and `Recorder.Pending()` is `Held()`. - `mcpserver`'s `run_program` returns an error naming the writes held for approval instead of reporting success. - Docs lead with integrating codemode: a five-step summary on the landing page diff --git a/SECURITY.md b/SECURITY.md index 44d42a4..063f61b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -31,7 +31,7 @@ the host, outside the program's reach. your app holds for the signed-in user. - **Credentials stay in the host.** The bridge carries `(name, input) → output` and nothing else. -- **Rate limits and audit** per user, via `agent.Options.OnCall`. +- **Rate limits and audit** per user, via `agent.Options.OnEvent`. ## Reporting a vulnerability diff --git a/agent/agent.go b/agent/agent.go index de565bc..e79494e 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -28,11 +28,8 @@ type Options struct { CacheCaller string // Includes repair attempts; nil disables streaming. It must not block. OnChunk func(string) - // Plan records writes instead of executing them; review Result.Planned, then pass it to Apply. - // ponytail: a planned write returns {} to the program, so logic that reads a write's output plans wrongly. - Plan bool - // nil disables it. It must not block. - OnCall func(codemode.Call) + // HoldWrites holds every write in a plan, as if each needed approval: a dry run. + HoldWrites bool // Stops repair attempts once spent; zero disables. Only providers that report usage count. MaxTokens int64 // Plans keeps writes held for approval until someone decides; nil returns them in Pending only. @@ -87,9 +84,9 @@ type Result struct { Attempts int Cached bool Timing Timing - Calls []codemode.Call - Planned []codemode.Call - Usage llm.Usage + // Calls is every capability call across all attempts, held ones included. + Calls []codemode.Call + Usage llm.Usage } var ErrTokenBudget = errors.New("token budget exhausted") @@ -211,8 +208,7 @@ func generate(ctx context.Context, mode codemode.CodeMode, model llm.Client, req result.Calls = append(result.Calls, recorder.Calls()...) if err == nil { result.Answer = answer - result.Planned = planned(recorder) - return result, pending(ctx, recorder, result.Planned, request, options) + return result, pending(ctx, recorder, request, options) } if stop := unretryable(recorder, err); stop != nil { return result, stop @@ -279,20 +275,16 @@ func fromCache(ctx context.Context, mode codemode.CodeMode, key program.Key, opt _ = options.Cache.Forget(ctx, entry.ID) return result, false, unretryable(recorder, err) } - result.Planned = planned(recorder) return result, true, nil } func run(ctx context.Context, mode codemode.CodeMode, source string, options Options, timing *Timing) (any, *codemode.Recorder, error) { - recorder := &codemode.Recorder{Plan: options.Plan, OnCall: func(call codemode.Call) { + recorder := &codemode.Recorder{HoldWrites: options.HoldWrites, OnCall: func(call codemode.Call) { kind := EventCallDone - if call.Planned { + if call.Held { kind = EventCallHeld } - options.emit(Event{Kind: kind, Name: call.Name, Duration: call.Duration, Err: call.Error}) - if options.OnCall != nil { - options.OnCall(call) - } + options.emit(Event{Kind: kind, Name: call.Name, Input: call.Input, Duration: call.Duration, Err: call.Error}) }} running := time.Now() answer, err := options.Runner.Run(codemode.WithRecorder(ctx, recorder), mode, source) @@ -303,10 +295,16 @@ func run(ctx context.Context, mode codemode.CodeMode, source string, options Opt return answer, recorder, err } -func pending(ctx context.Context, recorder *codemode.Recorder, held []codemode.Call, request string, options Options) error { - if !recorder.Pending() { +func pending(ctx context.Context, recorder *codemode.Recorder, request string, options Options) error { + if !recorder.Held() { return nil } + var held []codemode.Call + for _, call := range recorder.Calls() { + if call.Held { + held = append(held, call) + } + } if options.Plans == nil { return &Pending{Plan: Plan{RunID: options.runID, Request: request, Calls: held, Status: PlanPending}} } @@ -327,16 +325,6 @@ func unretryable(recorder *codemode.Recorder, err error) error { return nil } -func planned(recorder *codemode.Recorder) []codemode.Call { - var writes []codemode.Call - for _, call := range recorder.Calls() { - if call.Planned { - writes = append(writes, call) - } - } - return writes -} - func complete(ctx context.Context, model llm.Client, system, prompt string, chunk func(string)) (string, error) { if chunk != nil { if streamer, ok := model.(llm.Streamer); ok { diff --git a/agent/effects_test.go b/agent/effects_test.go index 1c6f5ed..3d339fa 100644 --- a/agent/effects_test.go +++ b/agent/effects_test.go @@ -83,31 +83,30 @@ func TestRunStopsOnAPermanentError(t *testing.T) { } } -func TestPlanRecordsWritesAndApplyRunsThem(t *testing.T) { +func TestHoldWritesIsADryRunThatApprovalApplies(t *testing.T) { mode, executed := effects(t) author := &model{responses: []string{ `const { todos } = codemode.execute("todos.list", {}); codemode.execute("todos.complete", {id: todos[0].id}); - return "planned";`, + return "held";`, }} + plans := NewPlans(NewMemoryPlanStore(), PlanOptions{}) - result, err := Run(context.Background(), mode, author, "finish all", Options{Runner: newRunner(t), Plan: true}) - if err != nil { - t.Fatal(err) + _, err := Run(context.Background(), mode, author, "finish all", Options{Runner: newRunner(t), HoldWrites: true, Plans: plans, PlanOwner: "ada"}) + var pending *Pending + if !errors.As(err, &pending) { + t.Fatalf("err = %v, want *Pending", err) } if len(*executed) != 1 || (*executed)[0] != "todo/List" { t.Fatalf("executed = %v, want only the read", *executed) } - if len(result.Planned) != 1 || string(result.Planned[0].Input) != `{"id":1}` { - t.Fatalf("planned = %+v", result.Planned) + if len(pending.Calls) != 1 || string(pending.Calls[0].Input) != `{"id":1}` { + t.Fatalf("held = %+v", pending.Calls) } - calls, err := Apply(context.Background(), mode, result.Planned) - if err != nil { - t.Fatal(err) - } - if len(calls) != 1 || len(*executed) != 2 || (*executed)[1] != "todo/Complete" { - t.Fatalf("calls = %+v, executed = %v", calls, *executed) + plan, err := plans.Approve(context.Background(), mode, "ada", pending.ID) + if err != nil || len(plan.Applied) != 1 || len(*executed) != 2 || (*executed)[1] != "todo/Complete" { + t.Fatalf("plan = %+v, executed = %v, err = %v", plan, *executed, err) } } @@ -131,15 +130,17 @@ func TestRunStopsRepairingWhenTheTokenBudgetIsSpent(t *testing.T) { } } -func TestOnCallSeesEachCall(t *testing.T) { +func TestOnEventSeesEachCallWithItsInput(t *testing.T) { mode, _ := effects(t) author := &model{responses: []string{`codemode.execute("todos.list", {}); return codemode.execute("todos.list", {}).todos.length;`}} var seen []string - _, err := Run(context.Background(), mode, author, "list", Options{Runner: newRunner(t), OnCall: func(call codemode.Call) { - seen = append(seen, call.Name) + _, err := Run(context.Background(), mode, author, "list", Options{Runner: newRunner(t), OnEvent: func(event Event) { + if event.Kind == EventCallDone { + seen = append(seen, event.Name+string(event.Input)) + } }}) - if err != nil || len(seen) != 2 { + if err != nil || len(seen) != 2 || seen[0] != "todos.list{}" { t.Fatalf("seen = %v, err = %v", seen, err) } } @@ -147,16 +148,17 @@ func TestOnCallSeesEachCall(t *testing.T) { func TestRunNeverCachesAProgramThatCanWrite(t *testing.T) { for name, source := range map[string]string{ "wrote": `codemode.execute("todos.complete", {id: 1}); return "done";`, - "planned": `codemode.execute("todos.complete", {id: 1}); return "planned";`, + "held": `codemode.execute("todos.complete", {id: 1}); return "held";`, "skipped": `const { todos } = codemode.execute("todos.list", {}); if (todos.length === 0) { codemode.execute("todos.complete", {id: 1}); } return "ok";`, } { t.Run(name, func(t *testing.T) { mode, _ := effects(t) author := &model{responses: []string{source, source}} - options := Options{Runner: newRunner(t), Cache: program.NewExactCache(), Plan: name == "planned"} + options := Options{Runner: newRunner(t), Cache: program.NewExactCache(), HoldWrites: name == "held"} for run := 1; run <= 2; run++ { result, err := Run(context.Background(), mode, author, "finish todo 1", options) - if err != nil { + var pending *Pending + if err != nil && !errors.As(err, &pending) { t.Fatalf("run %d: %v", run, err) } if result.Cached { diff --git a/agent/event.go b/agent/event.go index 91c0ef0..54caecb 100644 --- a/agent/event.go +++ b/agent/event.go @@ -2,6 +2,7 @@ package agent import ( "context" + "encoding/json" "fmt" "log/slog" "time" @@ -17,8 +18,9 @@ type Event struct { PlanID string // Stage is where a run failed: cache, search, generate, or run. Stage string - // Name is the capability, on call events. + // Name and Input are the capability and what the program sent, on call events. Name string + Input json.RawMessage Attempt int Count int Duration time.Duration diff --git a/agent/plan.go b/agent/plan.go index 3488d2c..02f08db 100644 --- a/agent/plan.go +++ b/agent/plan.go @@ -88,9 +88,9 @@ func (plan Plan) decidedError() error { return fmt.Errorf("plan %s is already %s (%s): %w", plan.ID, plan.Status, detail, ErrPlanDecided) } -// Pending is the error a run returns when a write is held for approval. It is -// the stored plan, so ID and ExpiresAt are what the client needs; without -// Options.Plans, ID is empty and Calls go to Apply. +// Pending is the error a run returns when a write is held. It is the stored +// plan, so ID and ExpiresAt are what the client needs. Without Options.Plans +// the plan is not stored and ID is empty: show Calls, but nothing can apply them. type Pending struct { Plan } @@ -283,8 +283,7 @@ func (plans *Plans) save(ctx context.Context, plan *Plan) error { func (plans *Plans) apply(ctx context.Context, mode codemode.CodeMode, plan Plan) (Plan, error) { plans.emit(Event{Kind: EventPlanApplying, RunID: plan.RunID, PlanID: plan.ID, Count: len(plan.Calls) - len(plan.Applied)}) - recorder := &codemode.Recorder{Approved: true} - approved := codemode.WithRecorder(ctx, recorder) + approved := codemode.WithApproval(ctx) for index := len(plan.Applied); index < len(plan.Calls); index++ { call := plan.Calls[index] started := time.Now() @@ -294,7 +293,7 @@ func (plans *Plans) apply(ctx context.Context, mode codemode.CodeMode, plan Plan result.Error = err.Error() } plan.Applied = append(plan.Applied, result) - plans.emit(Event{Kind: EventCallDone, RunID: plan.RunID, PlanID: plan.ID, Name: call.Name, Duration: result.Duration, Err: result.Error}) + plans.emit(Event{Kind: EventCallDone, RunID: plan.RunID, PlanID: plan.ID, Name: call.Name, Input: call.Input, Duration: result.Duration, Err: result.Error}) if err != nil { failure := fmt.Errorf("plan %s: call %d of %d (%s): %w", plan.ID, index+1, len(plan.Calls), call.Name, err) _ = plan.move(PlanFailed, plans.options.Now(), "", failure.Error()) @@ -333,20 +332,6 @@ func planError(id string, err error) error { return fmt.Errorf("plan %s: %w", id, err) } -// Apply runs calls with approval granted, for a plan kept by the host itself. It -// stops at the first failure; each call still passes validation, the registry -// and the transport. -func Apply(ctx context.Context, mode codemode.CodeMode, calls []codemode.Call) ([]codemode.Call, error) { - recorder := &codemode.Recorder{Approved: true} - ctx = codemode.WithRecorder(ctx, recorder) - for _, call := range calls { - if _, err := mode.Execute(ctx, call.Name, call.Input); err != nil { - return recorder.Calls(), err - } - } - return recorder.Calls(), nil -} - type memoryPlanStore struct { mutex sync.Mutex plans map[string][]byte diff --git a/codemode/codemode.go b/codemode/codemode.go index e57a8d2..52c06cb 100644 --- a/codemode/codemode.go +++ b/codemode/codemode.go @@ -113,7 +113,7 @@ func (runtime *runtime) Execute(ctx context.Context, name string, input json.Raw } call := Call{Name: name, Input: input, ReadOnly: item.ReadOnly, NeedsApproval: item.NeedsApproval} if recorder.hold(item) { - call.Planned = true + call.Held = true call.Output = json.RawMessage("{}") recorder.add(call) return call.Output, nil diff --git a/codemode/record.go b/codemode/record.go index b49bd7a..5351c29 100644 --- a/codemode/record.go +++ b/codemode/record.go @@ -24,8 +24,9 @@ type Call struct { Output json.RawMessage `json:"output,omitempty"` Error string `json:"error,omitempty"` ReadOnly bool `json:"read_only,omitempty"` - Planned bool `json:"planned,omitempty"` - // The capability is held for approval; Planned says whether this call was. + // Held is a write recorded instead of run; it waits in a plan. + Held bool `json:"held,omitempty"` + // The capability needs approval; Held says whether this call was held. NeedsApproval bool `json:"needs_approval,omitempty"` Duration time.Duration `json:"duration"` @@ -36,16 +37,22 @@ func (call Call) Err() error { return call.err } // Safe for concurrent use. type Recorder struct { - // Each planned write returns {} to the program. - Plan bool - // Approved runs writes that need approval; set only when applying an approved plan. - Approved bool + // HoldWrites holds every write, not only those that need approval. A held + // write returns {} to the program. + HoldWrites bool // It must not block. OnCall func(Call) - mutex sync.Mutex - calls []Call - holding bool + approved bool + mutex sync.Mutex + calls []Call + holding bool +} + +// WithApproval runs every write, including those that need approval. Use it +// when a person has already said yes, such as a delete button they pressed. +func WithApproval(ctx context.Context) context.Context { + return WithRecorder(ctx, &Recorder{approved: true}) } // hold reports whether a write is recorded instead of run. Once one write is held @@ -56,14 +63,14 @@ func (recorder *Recorder) hold(item capability.Capability) bool { } recorder.mutex.Lock() defer recorder.mutex.Unlock() - if item.NeedsApproval && !recorder.Approved { + if (item.NeedsApproval && !recorder.approved) || recorder.HoldWrites { recorder.holding = true } - return recorder.Plan || recorder.holding + return recorder.holding } -// Pending reports whether a write is waiting for approval. -func (recorder *Recorder) Pending() bool { +// Held reports whether any write was held instead of run. +func (recorder *Recorder) Held() bool { recorder.mutex.Lock() defer recorder.mutex.Unlock() return recorder.holding @@ -87,7 +94,7 @@ func (recorder *Recorder) Calls() []Call { // A failed write may still have committed. func (recorder *Recorder) Wrote() bool { for _, call := range recorder.Calls() { - if !call.ReadOnly && !call.Planned { + if !call.ReadOnly && !call.Held { return true } } diff --git a/codemode/source_test.go b/codemode/source_test.go index a2d2552..c4e8759 100644 --- a/codemode/source_test.go +++ b/codemode/source_test.go @@ -95,12 +95,12 @@ func TestAWriteThatNeedsApprovalHoldsItAndEveryLaterWrite(t *testing.T) { if strings.Join(*called, ",") != "rename,list" { t.Fatalf("ran %v; want the first rename and the read only", *called) } - if !recorder.Pending() { + if !recorder.Held() { t.Fatal("recorder does not report a pending write") } var held []string for _, call := range recorder.Calls() { - if call.Planned { + if call.Held { held = append(held, call.Name) } } @@ -109,7 +109,7 @@ func TestAWriteThatNeedsApprovalHoldsItAndEveryLaterWrite(t *testing.T) { } *called = nil - approved := WithRecorder(context.Background(), &Recorder{Approved: true}) + approved := WithApproval(context.Background()) if _, err := mode.Execute(approved, "tasks.delete", nil); err != nil || strings.Join(*called, ",") != "delete" { t.Fatalf("approved delete: err = %v, ran %v", err, *called) } diff --git a/docs/content/approval.md b/docs/content/approval.md index 75d8019..7d22978 100644 --- a/docs/content/approval.md +++ b/docs/content/approval.md @@ -117,8 +117,7 @@ A delete from the task board is a button the user pressed, so the server runs it with approval granted: ```go -approved := codemode.WithRecorder(ctx, &codemode.Recorder{Approved: true}) -output, err := mode.Execute(approved, "tasks.delete", input) +output, err := mode.Execute(codemode.WithApproval(ctx), "tasks.delete", input) ``` ## Limits @@ -129,6 +128,8 @@ output, err := mode.Execute(approved, "tasks.delete", input) - A plan is decided by its owner. Approval by another role is not built yet. - MCP clients cannot approve: `run_program` refuses a program that reaches such a write and names what did not run. -- Without `Options.Plans`, `Pending.ID` is empty; keep `Pending.Calls` yourself - and run them with `agent.Apply`. -- `Options.Plan` still holds *every* write for a dry run, approval or not. +- Without `Options.Plans`, nothing is stored and `Pending.ID` is empty: the + calls can be shown but not approved. `agent.NewMemoryPlanStore()` is enough + for a single process. +- `Options.HoldWrites` holds *every* write for one run, approval or not — a dry + run through the same plan. diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 1fbafb6..8b951d1 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -89,22 +89,21 @@ The last check runs one trivial program through the worker inside `New`, so a missing binary, a version mismatch, a bad `docker run` or a missing image fails at startup, not on the first request. -## Observing calls +## Observing runs -There is no tracing dependency. `agent.Options.OnCall` sees every capability -call as it finishes; feed it to whatever the host already runs. It must not block. +There is no tracing dependency. `agent.Options.OnEvent` sees every step of a run, +each capability call included, with the run's ID; feed it to whatever the host +already runs. It must not block. ```go agent.Run(ctx, mode, model, request, agent.Options{ - Runner: runner, - OnCall: func(call codemode.Call) { - slog.Info("capability", "user", user, "name", call.Name, - "ms", call.Duration.Milliseconds(), "error", call.Error) - }, + Runner: runner, + OnEvent: agent.LogEvents(slog.Default().With("user", user)), }) ``` -The same hook is where an audit log or an OpenTelemetry span per call goes. +The same hook is where an audit log or an OpenTelemetry span goes; see +[Observability](/observability). ## Commands diff --git a/docs/content/environments.md b/docs/content/environments.md index 7a22031..cfcc5f7 100644 --- a/docs/content/environments.md +++ b/docs/content/environments.md @@ -14,7 +14,7 @@ change. | Program cache | in memory | in memory per replica, or shared through a store | | Limits | defaults (30 s, 50 calls) | set to your workload | | Who writes prompts | you | your users, so treat every program as hostile | -| Audit | `-log debug` | one line per capability call through `OnCall` | +| Audit | `-log debug` | one line per step through `OnEvent` | ## One `main.go` for both @@ -84,10 +84,7 @@ func main() { Runner: runner, Cache: cache, CacheCaller: tenant + "|" + strings.Join(caller.Roles, ","), - OnCall: func(call codemode.Call) { - slog.Info("capability", "user", user, "roles", caller.Roles, "name", call.Name, - "ms", call.Duration.Milliseconds(), "error", call.Error) - }, + OnEvent: agent.LogEvents(slog.Default().With("user", user, "roles", caller.Roles)), }) if err != nil { http.Error(writer, err.Error(), http.StatusBadGateway) @@ -243,8 +240,8 @@ Around the sandbox, the host still owns: them from the program's input. - **Fairness.** A per-user rate limit and a cap on programs running at once, before `agent.Run`. -- **Audit.** One line per call from `OnCall`: user, role, capability, a hash of - the input. +- **Audit.** One line per call from `OnEvent` (`call.done` and `call.held`): + user, role, capability, a hash of the input. - **Read-only where possible.** A reporting product can drop writes from the registry entirely: `full.Select(func(c capability.Capability) bool { return c.ReadOnly })`. @@ -276,7 +273,7 @@ Then, for the environment: - `runner.Name()` in the first log line says `child process` or `container` (or you decided in-process is right). - `codemode-worker` is the same version as the library in `go.mod`. - Timeout, call budget and heap cap are set, not defaulted. -- `OnCall` writes somewhere you will look. +- `OnEvent` writes somewhere you will look. - The handler deadline is well above `SANDBOX_TIMEOUT`, so a slow program fails as a sandbox timeout, not a dropped connection. Every variable is listed in [Configuration](/configuration); the threat model per diff --git a/docs/content/existing-app.md b/docs/content/existing-app.md index 6fa79c8..0040c72 100644 --- a/docs/content/existing-app.md +++ b/docs/content/existing-app.md @@ -403,6 +403,7 @@ type Assistant struct { model llm.Client runner sandbox.Runner cache program.Cache + plans *agent.Plans } func New(service *crm.Service) (*Assistant, error) { @@ -418,14 +419,16 @@ func New(service *crm.Service) (*Assistant, error) { if err != nil { return nil, err } - return &Assistant{crm: service, full: full, model: model, runner: runner, cache: program.NewExactCache()}, nil + // Use agent/sqlplans on the CRM's database so a confirmation can come after a restart. + plans := agent.NewPlans(agent.NewMemoryPlanStore(), agent.PlanOptions{}) + return &Assistant{crm: service, full: full, model: model, runner: runner, cache: program.NewExactCache(), plans: plans}, nil } func (assistant *Assistant) Close() error { return assistant.runner.Close() } -// Ask answers one question for the signed-in user. With plan set, reads run but -// writes are returned in Result.Planned for the user to confirm. -func (assistant *Assistant) Ask(ctx context.Context, question string, plan bool) (agent.Result, error) { +// Ask answers one question for the signed-in user. Reads run; every write is +// held in a plan the user confirms, and err is then an *agent.Pending. +func (assistant *Assistant) Ask(ctx context.Context, question string) (agent.Result, error) { session, _ := auth.From(ctx) mode, err := assistant.modeFor(session) if err != nil { @@ -436,19 +439,21 @@ func (assistant *Assistant) Ask(ctx context.Context, question string, plan bool) Cache: assistant.cache, // instructions name the user, so a cached program is theirs alone CacheCaller: session.Org + "|" + session.UserID + "|" + strings.Join(slices.Sorted(slices.Values(session.Roles)), ","), - Plan: plan, + HoldWrites: true, + Plans: assistant.plans, + PlanOwner: session.Org + "|" + session.UserID, Instructions: instructions(session), }) } -// Apply runs writes the user confirmed, through the same registry and scope. -func (assistant *Assistant) Apply(ctx context.Context, planned []codemode.Call) ([]codemode.Call, error) { +// Confirm runs a held plan once, through this user's registry and scope as they are now. +func (assistant *Assistant) Confirm(ctx context.Context, planID string) (agent.Plan, error) { session, _ := auth.From(ctx) mode, err := assistant.modeFor(session) if err != nil { - return nil, err + return agent.Plan{}, err } - return agent.Apply(ctx, mode, planned) + return assistant.plans.Approve(ctx, mode, session.Org+"|"+session.UserID, planID) } // modeFor narrows the registry to what this user's roles allow; the roles come @@ -533,66 +538,70 @@ func ask(helper *assistant.Assistant) http.HandlerFunc { http.Error(writer, err.Error(), http.StatusBadRequest) return } - // Plan: reads run now, writes wait for the user to confirm them. - result, err := helper.Ask(request.Context(), body.Question, true) - if err != nil { - http.Error(writer, err.Error(), http.StatusBadGateway) - return + // Reads run now; writes wait in a plan for the user to confirm. + result, err := helper.Ask(request.Context(), body.Question) + var pending *agent.Pending + switch { + case errors.As(err, &pending): + json.NewEncoder(writer).Encode(map[string]any{"program": result.Source, "plan": pending.Plan}) + case err != nil: + http.Error(writer, err.Error(), http.StatusBadGateway) // names the run and the stage + default: + json.NewEncoder(writer).Encode(map[string]any{"answer": agent.Answer(result.Answer), "program": result.Source}) } - json.NewEncoder(writer).Encode(map[string]any{ - "answer": agent.Answer(result.Answer), - "program": result.Source, - "planned": result.Planned, // empty unless the question asked for changes - }) } } func apply(helper *assistant.Assistant) http.HandlerFunc { return func(writer http.ResponseWriter, request *http.Request) { var body struct { - Planned []codemode.Call `json:"planned"` + PlanID string `json:"plan_id"` } - if err := json.NewDecoder(http.MaxBytesReader(writer, request.Body, 1<<16)).Decode(&body); err != nil { + if err := json.NewDecoder(http.MaxBytesReader(writer, request.Body, 1<<12)).Decode(&body); err != nil { http.Error(writer, err.Error(), http.StatusBadRequest) return } - // Each call goes through this user's registry and scope again, so an - // edited plan cannot do more than the user could. - calls, err := helper.Apply(request.Context(), body.Planned) + // Only the ID comes back: the plan's calls stay on the server, so the + // client cannot change what the user confirmed. + plan, err := helper.Confirm(request.Context(), body.PlanID) if err != nil { - http.Error(writer, err.Error(), http.StatusUnprocessableEntity) + http.Error(writer, err.Error(), http.StatusUnprocessableEntity) // names the plan and the call return } - json.NewEncoder(writer).Encode(map[string]any{"applied": calls}) + json.NewEncoder(writer).Encode(map[string]any{"plan": plan}) } } ``` ### Step 6: writes, confirmed by the user -The ask route always runs with `Plan: true`. Reads execute; writes are recorded -and returned instead of run: +The ask route always runs with `HoldWrites: true`. Reads execute; writes are +held in a plan and returned instead of run: ```json { - "answer": "...", - "planned": [ - {"name": "deals.move", "input": {"deal_id": 2, "stage": "won"}}, - {"name": "deals.logActivity", "input": {"deal_id": 2, "kind": "note", "note": "Contract signed."}} - ] + "program": "...", + "plan": { + "id": "3f2a9c1e...", + "status": "pending", + "expires_at": "2026-11-17T12:00:00Z", + "calls": [ + {"name": "deals.move", "input": {"deal_id": 2, "stage": "won"}}, + {"name": "deals.logActivity", "input": {"deal_id": 2, "kind": "note", "note": "Contract signed."}} + ] + } } ``` -Show `planned` as a confirmation ("Move *Parc expansion* to **won**, log a -note?"). When the user confirms, send it to `/api/assistant/apply`. +Show `plan.calls` as a confirmation ("Move *Parc expansion* to **won**, log a +note?"). When the user confirms, send `{"plan_id": ...}` to `/api/assistant/apply`. -- **When `planned` is not empty, show the plan, not the answer.** A planned write - returns `{}` to the program, so the preview answer can say "moved deal - undefined". The plan is the truth; the answer is a guess about it. -- **An edited plan cannot do more than the user can.** `Apply` sends every call - through this user's registry and `execute` again. We replayed ada's plan as a - user from another org: `execute deals.move: not found (permanent)`, and nothing - changed. +- **When there is a plan, show the plan, not the answer.** A held write returns + `{}` to the program, so any preview answer is a guess; the plan is the truth. +- **The client sends only the ID.** The calls stay on the server, a plan applies + once, and only for the user it belongs to: another org's user gets + `no such plan`. `Approve` sends every call through this user's registry and + `execute` again, so a role lost since then is a role lost now. ### Step 7: roll it out @@ -604,7 +613,7 @@ note?"). When the user confirms, send it to `/api/assistant/apply`. | Production, many customer orgs | plus a container worker | see [Running the worker](/worker#step-5-a-container-per-run) | Then add what any feature that acts for users needs: a per-user rate limit and -an audit log through `agent.Options.OnCall` ([Roles and tenants](/tenants#sandbox-fairness-and-audit)), +an audit log through `agent.Options.OnEvent` ([Roles and tenants](/tenants#sandbox-fairness-and-audit)), and start with reads only if writes need more review. ### What it did diff --git a/docs/content/scaling-up.md b/docs/content/scaling-up.md index 216a794..a8a07cb 100644 --- a/docs/content/scaling-up.md +++ b/docs/content/scaling-up.md @@ -252,16 +252,19 @@ what a tenant is: - **Fairness.** Limit each user's request rate and how many programs they run at once, before `agent.Run`. One heavy user should cost themselves latency, not everyone. -- **Audit.** `agent.Options.OnCall` sees every capability call. Write one line +- **Audit.** `agent.Options.OnEvent` sees every capability call. Write one line per call with the user, role, capability and a hash of the input: enough to answer "who called what", without the log becoming a second copy of the data. ```go agent.Run(ctx, mode, model, request, agent.Options{ Runner: runner, - OnCall: func(call codemode.Call) { - sum := sha256.Sum256(call.Input) - audit.Log(identity.User, identity.Role, call.Name, hex.EncodeToString(sum[:]), call.Error) + OnEvent: func(event agent.Event) { + if event.Kind != agent.EventCallDone && event.Kind != agent.EventCallHeld { + return + } + sum := sha256.Sum256(event.Input) + audit.Log(event.RunID, identity.User, identity.Role, event.Name, hex.EncodeToString(sum[:]), event.Err) }, }) ``` diff --git a/docs/content/tenants.md b/docs/content/tenants.md index 5a54309..0a1c2eb 100644 --- a/docs/content/tenants.md +++ b/docs/content/tenants.md @@ -388,7 +388,7 @@ if instructionsMentionUser { | Sandbox | per run | one shared runner; process isolation or a container worker for untrusted tenants ([Running the worker](/worker)) | | Rate limit | per tenant **and** per user | so one user cannot spend a tenant's quota, and one tenant cannot starve others | | Concurrency | per tenant | a cap on programs running at once | -| Audit | per call | `OnCall` with tenant, user, roles, capability, input hash | +| Audit | per call | `OnEvent` with tenant, user, roles, capability, input hash | ## The whole request @@ -428,8 +428,8 @@ func (server *Server) ask(writer http.ResponseWriter, request *http.Request) { Runner: server.runner, // shared; SANDBOX_ISOLATION decides in-process or worker Cache: server.cache, // keyed by the capability set already; CacheCaller adds anything else in the prompt Instructions: "You answer for one user of one organisation. Say so when a request needs data outside it.", - OnCall: func(call codemode.Call) { - server.audit(identity, call) + OnEvent: func(event agent.Event) { + server.audit(identity, event) // call.done and call.held carry the capability and input }, }) if err != nil { diff --git a/docs/content/using-it.md b/docs/content/using-it.md index eda7ef3..c30f829 100644 --- a/docs/content/using-it.md +++ b/docs/content/using-it.md @@ -590,7 +590,7 @@ This is the one checklist; the other pages link here. - **Set the sandbox bounds deliberately** — `Timeout` and `MaxCalls`, or `SANDBOX_TIMEOUT` and `SANDBOX_MAX_CALLS`. The defaults — 50 calls, 30 seconds — are demo numbers. - **Log `Result.Source`.** When an answer is wrong, the program is the evidence. - **Regenerate the registry in CI.** A drifted `registry.json` is a silent failure. -- **Confirm writes** a user did not plainly ask for: `agent.Options{Plan: true}`, then `agent.Apply` — see [Writes and retries](/writes). +- **Confirm writes** a user did not plainly ask for: mark them `NeedsApproval`, or hold every write with `HoldWrites`, and apply through `agent.Plans` — see [Approving writes](/approval). - **Run programs out of process** if they will ever be shaped by untrusted end users: `SANDBOX_ISOLATION=process` through `sandbox.NewFromEnv`, and a container worker for tenants you do not trust. See [Development and production](/environments). - **Blank-import exactly what you run.** An unregistered backend fails at `New`, not at startup — see [what your binary links](/dependencies). diff --git a/docs/content/worker.md b/docs/content/worker.md index 6026c95..decd0be 100644 --- a/docs/content/worker.md +++ b/docs/content/worker.md @@ -277,7 +277,7 @@ The worker reads one `run` message with the source and limits, applies its own limits, then runs the same engine your app would have run in-process. That engine is given a `CodeMode` whose `Execute` writes a `call` line and waits for the `result` line. Your app answers each `call` with its real `CodeMode`, so the -registry, roles, `Recorder` and `OnCall` all work unchanged. A `done` or `error` +registry, roles, `Recorder` and `OnEvent` all work unchanged. A `done` or `error` line ends the run, and your app kills the worker's process group either way. Your app trusts nothing the worker says. It counts calls itself, checks every diff --git a/docs/content/writes.md b/docs/content/writes.md index 4bb84bd..515dac7 100644 --- a/docs/content/writes.md +++ b/docs/content/writes.md @@ -43,37 +43,27 @@ user what changed. ## Approving writes first -To hold only the writes that matter, such as a refund, mark them -`NeedsApproval` and apply the plan by ID: see [Approving writes](/approval). -To dry-run every write, use `Options.Plan`. +Mark the writes that need a person with `NeedsApproval`: the program runs its +reads, holds those writes, and `agent.Run` returns `*agent.Pending`. See +[Approving writes](/approval). -With `Options.Plan`, reads run but writes are recorded instead of executed and -returned in `Result.Planned`. Show them, then apply exactly those: +To hold *every* write for one run — a dry run — set `HoldWrites`. It is the same +flow: the held writes become a plan, and `plans.Approve` applies them. ```go -result, err := agent.Run(ctx, mode, model, request, agent.Options{Runner: runner, Plan: true}) -// ... user approves result.Planned ... -calls, err := agent.Apply(ctx, mode, result.Planned) -``` - -```mermaid -sequenceDiagram - participant App - participant Agent - participant Sandbox - App->>Agent: Run(Plan: true) - Agent->>Sandbox: program - Sandbox-->>Agent: reads executed, writes recorded - Agent-->>App: Result.Planned - App->>App: user approves - App->>Agent: Apply(planned) - Agent-->>App: calls made +_, err := agent.Run(ctx, mode, model, request, agent.Options{ + Runner: runner, HoldWrites: true, Plans: plans, PlanOwner: user.ID, +}) +var pending *agent.Pending +if errors.As(err, &pending) { + // show pending.Calls; on approval: plans.Approve(ctx, mode, user.ID, pending.ID) +} ``` -A planned write returns `{}` to the program, so a program that uses a write's +A held write returns `{}` to the program, so a program that uses a write's output — the ID of a task it just created, say — cannot be planned accurately. -`Apply` passes each call through the registry and your transport again, so it -is no more trusted than a direct call. +Approving passes each call through validation, the registry and your transport +again, so it is no more trusted than a direct call. ## Response size @@ -84,13 +74,15 @@ not what it allocates. For a hard memory limit, set `SANDBOX_ISOLATION=process`: ## Watching a run -`Options.OnCall` receives each call as it completes — the hook for tracing, -metrics, or a live progress view. Code mode imports no tracing SDK; start your -OpenTelemetry span there, or wrap your `ExecuteFunc` and `llm.Client`. +`Options.OnEvent` receives each step of the run as it happens, including every +capability call with its name, input, duration and error — the hook for audit, +tracing, metrics, or a live progress view. See [Observability](/observability). ```go -agent.Options{Runner: runner, OnCall: func(call codemode.Call) { - log.Printf("%s %s err=%q", call.Name, call.Duration, call.Error) +agent.Options{Runner: runner, OnEvent: func(event agent.Event) { + if event.Kind == agent.EventCallDone { + log.Printf("%s %s err=%q", event.Name, event.Duration, event.Err) + } }} ``` diff --git a/examples/shop/README.md b/examples/shop/README.md index 9f09102..20627d8 100644 --- a/examples/shop/README.md +++ b/examples/shop/README.md @@ -5,7 +5,7 @@ capabilities from typed Go functions, an answer that comes back as a Go struct, and a refund that waits for you to approve it. ``` -main.go the shop: structs, two functions, gosource, RunAs, ApplyPlan +main.go the shop: structs, two functions, gosource, RunAs, Plans.Approve main_test.go typed answer, approval, access and validation, with no model ``` diff --git a/examples/taskreport/server/limits.go b/examples/taskreport/server/limits.go index 67496bc..45eff90 100644 --- a/examples/taskreport/server/limits.go +++ b/examples/taskreport/server/limits.go @@ -12,7 +12,7 @@ import ( "golang.org/x/time/rate" - "github.com/mukezhz/codemode/codemode" + "github.com/mukezhz/codemode/agent" "github.com/mukezhz/codemode/examples/taskreport/auth" ) @@ -90,24 +90,23 @@ type auditRecord struct { Role string `json:"role"` Capability string `json:"capability"` InputHash string `json:"input_sha256"` - Planned bool `json:"planned,omitempty"` + Held bool `json:"held,omitempty"` Error string `json:"error,omitempty"` DurationMS int64 `json:"duration_ms"` } -func (auditor *auditor) hook(identity auth.Identity) func(codemode.Call) { - if auditor == nil { - return nil - } - return func(call codemode.Call) { - sum := sha256.Sum256(call.Input) - line, _ := json.Marshal(auditRecord{ - Time: time.Now().UTC(), User: identity.User, Role: string(identity.Role), Capability: call.Name, - InputHash: hex.EncodeToString(sum[:]), Planned: call.Planned, Error: call.Error, - DurationMS: call.Duration.Milliseconds(), - }) - auditor.mutex.Lock() - auditor.writer.Write(append(line, '\n')) - auditor.mutex.Unlock() +// record takes the call events of one caller's runs and approved plans. +func (auditor *auditor) record(identity auth.Identity, event agent.Event) { + if auditor == nil || (event.Kind != agent.EventCallDone && event.Kind != agent.EventCallHeld) { + return } + sum := sha256.Sum256(event.Input) + line, _ := json.Marshal(auditRecord{ + Time: event.Time.UTC(), User: identity.User, Role: string(identity.Role), Capability: event.Name, + InputHash: hex.EncodeToString(sum[:]), Held: event.Kind == agent.EventCallHeld, Error: event.Err, + DurationMS: event.Duration.Milliseconds(), + }) + auditor.mutex.Lock() + auditor.writer.Write(append(line, '\n')) + auditor.mutex.Unlock() } diff --git a/examples/taskreport/server/server.go b/examples/taskreport/server/server.go index 0acc214..683adad 100644 --- a/examples/taskreport/server/server.go +++ b/examples/taskreport/server/server.go @@ -240,7 +240,7 @@ func (server *Server) execute(writer http.ResponseWriter, request *http.Request, return } // A button the user pressed is their approval, so a delete from the board runs at once. - approved := codemode.WithRecorder(request.Context(), &codemode.Recorder{Approved: true}) + approved := codemode.WithApproval(request.Context()) output, err := mode.Execute(approved, name, encoded) if err != nil { fail(writer, http.StatusBadRequest, err) @@ -372,10 +372,9 @@ func (server *Server) answer( // A program targets one role's capabilities, so cache per role. CacheCaller: string(identity.Role), OnChunk: onChunk, - OnCall: server.audit.hook(identity), Plans: server.options.Plans, - OnEvent: agent.LogEvents(log.With("user", identity.User)), PlanOwner: identity.User, + OnEvent: server.events(identity, log), }) var held *agent.Pending if errors.As(err, &held) { @@ -427,6 +426,15 @@ func (server *Server) answer( }, http.StatusOK, nil } +// events logs every step of a run and audits its calls, for one caller. +func (server *Server) events(identity auth.Identity, log *slog.Logger) func(agent.Event) { + logged := agent.LogEvents(log.With("user", identity.User)) + return func(event agent.Event) { + logged(event) + server.audit.record(identity, event) + } +} + func (server *Server) listPlans(writer http.ResponseWriter, request *http.Request) { identity, err := auth.From(request.Context()) if err != nil { @@ -455,10 +463,9 @@ func (server *Server) approvePlan(writer http.ResponseWriter, request *http.Requ return } plan, err := server.options.Plans.Approve(request.Context(), mode, identity.User, request.PathValue("id")) - if hook := server.audit.hook(identity); hook != nil { - for _, call := range plan.Applied { - hook(call) - } + for _, call := range plan.Applied { + server.audit.record(identity, agent.Event{Kind: agent.EventCallDone, Time: time.Now(), Name: call.Name, + Input: call.Input, Duration: call.Duration, Err: call.Error}) } server.decided(writer, plan, err) } diff --git a/mcpserver/mcpserver.go b/mcpserver/mcpserver.go index 62ac945..7ebc031 100644 --- a/mcpserver/mcpserver.go +++ b/mcpserver/mcpserver.go @@ -125,13 +125,13 @@ func New(modeFor ModeFunc, runner sandbox.Runner, options Options) (*mcp.Server, value, err := runner.Run(codemode.WithRecorder(ctx, recorder), mode, input.Source) output := RunOutput{Value: value, Calls: []Call{}} for _, call := range recorder.Calls() { - output.Calls = append(output.Calls, Call{Name: call.Name, Error: call.Error, DurationMS: call.Duration.Milliseconds(), Held: call.Planned}) + output.Calls = append(output.Calls, Call{Name: call.Name, Error: call.Error, DurationMS: call.Duration.Milliseconds(), Held: call.Held}) } // ponytail: no apply_plan tool yet, so a held write is refused rather than approved here. - if err == nil && recorder.Pending() { + if err == nil && recorder.Held() { var held []string for _, call := range recorder.Calls() { - if call.Planned { + if call.Held { held = append(held, call.Name) } }