Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,14 @@ version is 0, a minor bump may break the API.
- `mcpsource` maps JSON Schema `integer` to `integer` rather than `number`,
records `required` fields, and falls back to a tool's title or name when it has
no description. `mcpsource.Source` is now an alias of `codemode.Source`.
- `agent.Apply` runs writes that need approval: applying is the approval.
- One way to hold writes, one hook, one way to approve:
- `Options.Plan` is now `Options.HoldWrites`, and held writes come back as
`*agent.Pending` like any write needing approval. `Result.Planned` is gone.
- `Options.OnCall` is gone; `OnEvent` carries every call, with its input.
- `agent.Apply` is gone; plans are approved through `agent.Plans`.
- `codemode.WithApproval(ctx)` runs writes a person already approved, such as
a button they pressed, instead of setting `Recorder.Approved`.
- `codemode.Call.Planned` is `Held`, and `Recorder.Pending()` is `Held()`.
- `mcpserver`'s `run_program` returns an error naming the writes held for
approval instead of reporting success.
- Docs lead with integrating codemode: a five-step summary on the landing page
Expand Down
2 changes: 1 addition & 1 deletion SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ the host, outside the program's reach.
your app holds for the signed-in user.
- **Credentials stay in the host.** The bridge carries `(name, input) → output`
and nothing else.
- **Rate limits and audit** per user, via `agent.Options.OnCall`.
- **Rate limits and audit** per user, via `agent.Options.OnEvent`.

## Reporting a vulnerability

Expand Down
46 changes: 17 additions & 29 deletions agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,8 @@ type Options struct {
CacheCaller string
// Includes repair attempts; nil disables streaming. It must not block.
OnChunk func(string)
// Plan records writes instead of executing them; review Result.Planned, then pass it to Apply.
// ponytail: a planned write returns {} to the program, so logic that reads a write's output plans wrongly.
Plan bool
// nil disables it. It must not block.
OnCall func(codemode.Call)
// HoldWrites holds every write in a plan, as if each needed approval: a dry run.
HoldWrites bool
// Stops repair attempts once spent; zero disables. Only providers that report usage count.
MaxTokens int64
// Plans keeps writes held for approval until someone decides; nil returns them in Pending only.
Expand Down Expand Up @@ -87,9 +84,9 @@ type Result struct {
Attempts int
Cached bool
Timing Timing
Calls []codemode.Call
Planned []codemode.Call
Usage llm.Usage
// Calls is every capability call across all attempts, held ones included.
Calls []codemode.Call
Usage llm.Usage
}

var ErrTokenBudget = errors.New("token budget exhausted")
Expand Down Expand Up @@ -211,8 +208,7 @@ func generate(ctx context.Context, mode codemode.CodeMode, model llm.Client, req
result.Calls = append(result.Calls, recorder.Calls()...)
if err == nil {
result.Answer = answer
result.Planned = planned(recorder)
return result, pending(ctx, recorder, result.Planned, request, options)
return result, pending(ctx, recorder, request, options)
}
if stop := unretryable(recorder, err); stop != nil {
return result, stop
Expand Down Expand Up @@ -279,20 +275,16 @@ func fromCache(ctx context.Context, mode codemode.CodeMode, key program.Key, opt
_ = options.Cache.Forget(ctx, entry.ID)
return result, false, unretryable(recorder, err)
}
result.Planned = planned(recorder)
return result, true, nil
}

func run(ctx context.Context, mode codemode.CodeMode, source string, options Options, timing *Timing) (any, *codemode.Recorder, error) {
recorder := &codemode.Recorder{Plan: options.Plan, OnCall: func(call codemode.Call) {
recorder := &codemode.Recorder{HoldWrites: options.HoldWrites, OnCall: func(call codemode.Call) {
kind := EventCallDone
if call.Planned {
if call.Held {
kind = EventCallHeld
}
options.emit(Event{Kind: kind, Name: call.Name, Duration: call.Duration, Err: call.Error})
if options.OnCall != nil {
options.OnCall(call)
}
options.emit(Event{Kind: kind, Name: call.Name, Input: call.Input, Duration: call.Duration, Err: call.Error})
}}
running := time.Now()
answer, err := options.Runner.Run(codemode.WithRecorder(ctx, recorder), mode, source)
Expand All @@ -303,10 +295,16 @@ func run(ctx context.Context, mode codemode.CodeMode, source string, options Opt
return answer, recorder, err
}

func pending(ctx context.Context, recorder *codemode.Recorder, held []codemode.Call, request string, options Options) error {
if !recorder.Pending() {
func pending(ctx context.Context, recorder *codemode.Recorder, request string, options Options) error {
if !recorder.Held() {
return nil
}
var held []codemode.Call
for _, call := range recorder.Calls() {
if call.Held {
held = append(held, call)
}
}
if options.Plans == nil {
return &Pending{Plan: Plan{RunID: options.runID, Request: request, Calls: held, Status: PlanPending}}
}
Expand All @@ -327,16 +325,6 @@ func unretryable(recorder *codemode.Recorder, err error) error {
return nil
}

func planned(recorder *codemode.Recorder) []codemode.Call {
var writes []codemode.Call
for _, call := range recorder.Calls() {
if call.Planned {
writes = append(writes, call)
}
}
return writes
}

func complete(ctx context.Context, model llm.Client, system, prompt string, chunk func(string)) (string, error) {
if chunk != nil {
if streamer, ok := model.(llm.Streamer); ok {
Expand Down
42 changes: 22 additions & 20 deletions agent/effects_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,31 +83,30 @@ func TestRunStopsOnAPermanentError(t *testing.T) {
}
}

func TestPlanRecordsWritesAndApplyRunsThem(t *testing.T) {
func TestHoldWritesIsADryRunThatApprovalApplies(t *testing.T) {
mode, executed := effects(t)
author := &model{responses: []string{
`const { todos } = codemode.execute("todos.list", {});
codemode.execute("todos.complete", {id: todos[0].id});
return "planned";`,
return "held";`,
}}
plans := NewPlans(NewMemoryPlanStore(), PlanOptions{})

result, err := Run(context.Background(), mode, author, "finish all", Options{Runner: newRunner(t), Plan: true})
if err != nil {
t.Fatal(err)
_, err := Run(context.Background(), mode, author, "finish all", Options{Runner: newRunner(t), HoldWrites: true, Plans: plans, PlanOwner: "ada"})
var pending *Pending
if !errors.As(err, &pending) {
t.Fatalf("err = %v, want *Pending", err)
}
if len(*executed) != 1 || (*executed)[0] != "todo/List" {
t.Fatalf("executed = %v, want only the read", *executed)
}
if len(result.Planned) != 1 || string(result.Planned[0].Input) != `{"id":1}` {
t.Fatalf("planned = %+v", result.Planned)
if len(pending.Calls) != 1 || string(pending.Calls[0].Input) != `{"id":1}` {
t.Fatalf("held = %+v", pending.Calls)
}

calls, err := Apply(context.Background(), mode, result.Planned)
if err != nil {
t.Fatal(err)
}
if len(calls) != 1 || len(*executed) != 2 || (*executed)[1] != "todo/Complete" {
t.Fatalf("calls = %+v, executed = %v", calls, *executed)
plan, err := plans.Approve(context.Background(), mode, "ada", pending.ID)
if err != nil || len(plan.Applied) != 1 || len(*executed) != 2 || (*executed)[1] != "todo/Complete" {
t.Fatalf("plan = %+v, executed = %v, err = %v", plan, *executed, err)
}
}

Expand All @@ -131,32 +130,35 @@ func TestRunStopsRepairingWhenTheTokenBudgetIsSpent(t *testing.T) {
}
}

func TestOnCallSeesEachCall(t *testing.T) {
func TestOnEventSeesEachCallWithItsInput(t *testing.T) {
mode, _ := effects(t)
author := &model{responses: []string{`codemode.execute("todos.list", {}); return codemode.execute("todos.list", {}).todos.length;`}}
var seen []string

_, err := Run(context.Background(), mode, author, "list", Options{Runner: newRunner(t), OnCall: func(call codemode.Call) {
seen = append(seen, call.Name)
_, err := Run(context.Background(), mode, author, "list", Options{Runner: newRunner(t), OnEvent: func(event Event) {
if event.Kind == EventCallDone {
seen = append(seen, event.Name+string(event.Input))
}
}})
if err != nil || len(seen) != 2 {
if err != nil || len(seen) != 2 || seen[0] != "todos.list{}" {
t.Fatalf("seen = %v, err = %v", seen, err)
}
}

func TestRunNeverCachesAProgramThatCanWrite(t *testing.T) {
for name, source := range map[string]string{
"wrote": `codemode.execute("todos.complete", {id: 1}); return "done";`,
"planned": `codemode.execute("todos.complete", {id: 1}); return "planned";`,
"held": `codemode.execute("todos.complete", {id: 1}); return "held";`,
"skipped": `const { todos } = codemode.execute("todos.list", {}); if (todos.length === 0) { codemode.execute("todos.complete", {id: 1}); } return "ok";`,
} {
t.Run(name, func(t *testing.T) {
mode, _ := effects(t)
author := &model{responses: []string{source, source}}
options := Options{Runner: newRunner(t), Cache: program.NewExactCache(), Plan: name == "planned"}
options := Options{Runner: newRunner(t), Cache: program.NewExactCache(), HoldWrites: name == "held"}
for run := 1; run <= 2; run++ {
result, err := Run(context.Background(), mode, author, "finish todo 1", options)
if err != nil {
var pending *Pending
if err != nil && !errors.As(err, &pending) {
t.Fatalf("run %d: %v", run, err)
}
if result.Cached {
Expand Down
4 changes: 3 additions & 1 deletion agent/event.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package agent

import (
"context"
"encoding/json"
"fmt"
"log/slog"
"time"
Expand All @@ -17,8 +18,9 @@ type Event struct {
PlanID string
// Stage is where a run failed: cache, search, generate, or run.
Stage string
// Name is the capability, on call events.
// Name and Input are the capability and what the program sent, on call events.
Name string
Input json.RawMessage
Attempt int
Count int
Duration time.Duration
Expand Down
25 changes: 5 additions & 20 deletions agent/plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,9 @@ func (plan Plan) decidedError() error {
return fmt.Errorf("plan %s is already %s (%s): %w", plan.ID, plan.Status, detail, ErrPlanDecided)
}

// Pending is the error a run returns when a write is held for approval. It is
// the stored plan, so ID and ExpiresAt are what the client needs; without
// Options.Plans, ID is empty and Calls go to Apply.
// Pending is the error a run returns when a write is held. It is the stored
// plan, so ID and ExpiresAt are what the client needs. Without Options.Plans
// the plan is not stored and ID is empty: show Calls, but nothing can apply them.
type Pending struct {
Plan
}
Expand Down Expand Up @@ -283,8 +283,7 @@ func (plans *Plans) save(ctx context.Context, plan *Plan) error {

func (plans *Plans) apply(ctx context.Context, mode codemode.CodeMode, plan Plan) (Plan, error) {
plans.emit(Event{Kind: EventPlanApplying, RunID: plan.RunID, PlanID: plan.ID, Count: len(plan.Calls) - len(plan.Applied)})
recorder := &codemode.Recorder{Approved: true}
approved := codemode.WithRecorder(ctx, recorder)
approved := codemode.WithApproval(ctx)
for index := len(plan.Applied); index < len(plan.Calls); index++ {
call := plan.Calls[index]
started := time.Now()
Expand All @@ -294,7 +293,7 @@ func (plans *Plans) apply(ctx context.Context, mode codemode.CodeMode, plan Plan
result.Error = err.Error()
}
plan.Applied = append(plan.Applied, result)
plans.emit(Event{Kind: EventCallDone, RunID: plan.RunID, PlanID: plan.ID, Name: call.Name, Duration: result.Duration, Err: result.Error})
plans.emit(Event{Kind: EventCallDone, RunID: plan.RunID, PlanID: plan.ID, Name: call.Name, Input: call.Input, Duration: result.Duration, Err: result.Error})
if err != nil {
failure := fmt.Errorf("plan %s: call %d of %d (%s): %w", plan.ID, index+1, len(plan.Calls), call.Name, err)
_ = plan.move(PlanFailed, plans.options.Now(), "", failure.Error())
Expand Down Expand Up @@ -333,20 +332,6 @@ func planError(id string, err error) error {
return fmt.Errorf("plan %s: %w", id, err)
}

// Apply runs calls with approval granted, for a plan kept by the host itself. It
// stops at the first failure; each call still passes validation, the registry
// and the transport.
func Apply(ctx context.Context, mode codemode.CodeMode, calls []codemode.Call) ([]codemode.Call, error) {
recorder := &codemode.Recorder{Approved: true}
ctx = codemode.WithRecorder(ctx, recorder)
for _, call := range calls {
if _, err := mode.Execute(ctx, call.Name, call.Input); err != nil {
return recorder.Calls(), err
}
}
return recorder.Calls(), nil
}

type memoryPlanStore struct {
mutex sync.Mutex
plans map[string][]byte
Expand Down
2 changes: 1 addition & 1 deletion codemode/codemode.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func (runtime *runtime) Execute(ctx context.Context, name string, input json.Raw
}
call := Call{Name: name, Input: input, ReadOnly: item.ReadOnly, NeedsApproval: item.NeedsApproval}
if recorder.hold(item) {
call.Planned = true
call.Held = true
call.Output = json.RawMessage("{}")
recorder.add(call)
return call.Output, nil
Expand Down
35 changes: 21 additions & 14 deletions codemode/record.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ type Call struct {
Output json.RawMessage `json:"output,omitempty"`
Error string `json:"error,omitempty"`
ReadOnly bool `json:"read_only,omitempty"`
Planned bool `json:"planned,omitempty"`
// The capability is held for approval; Planned says whether this call was.
// Held is a write recorded instead of run; it waits in a plan.
Held bool `json:"held,omitempty"`
// The capability needs approval; Held says whether this call was held.
NeedsApproval bool `json:"needs_approval,omitempty"`
Duration time.Duration `json:"duration"`

Expand All @@ -36,16 +37,22 @@ func (call Call) Err() error { return call.err }

// Safe for concurrent use.
type Recorder struct {
// Each planned write returns {} to the program.
Plan bool
// Approved runs writes that need approval; set only when applying an approved plan.
Approved bool
// HoldWrites holds every write, not only those that need approval. A held
// write returns {} to the program.
HoldWrites bool
// It must not block.
OnCall func(Call)

mutex sync.Mutex
calls []Call
holding bool
approved bool
mutex sync.Mutex
calls []Call
holding bool
}

// WithApproval runs every write, including those that need approval. Use it
// when a person has already said yes, such as a delete button they pressed.
func WithApproval(ctx context.Context) context.Context {
return WithRecorder(ctx, &Recorder{approved: true})
}

// hold reports whether a write is recorded instead of run. Once one write is held
Expand All @@ -56,14 +63,14 @@ func (recorder *Recorder) hold(item capability.Capability) bool {
}
recorder.mutex.Lock()
defer recorder.mutex.Unlock()
if item.NeedsApproval && !recorder.Approved {
if (item.NeedsApproval && !recorder.approved) || recorder.HoldWrites {
recorder.holding = true
}
return recorder.Plan || recorder.holding
return recorder.holding
}

// Pending reports whether a write is waiting for approval.
func (recorder *Recorder) Pending() bool {
// Held reports whether any write was held instead of run.
func (recorder *Recorder) Held() bool {
recorder.mutex.Lock()
defer recorder.mutex.Unlock()
return recorder.holding
Expand All @@ -87,7 +94,7 @@ func (recorder *Recorder) Calls() []Call {
// A failed write may still have committed.
func (recorder *Recorder) Wrote() bool {
for _, call := range recorder.Calls() {
if !call.ReadOnly && !call.Planned {
if !call.ReadOnly && !call.Held {
return true
}
}
Expand Down
6 changes: 3 additions & 3 deletions codemode/source_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@ func TestAWriteThatNeedsApprovalHoldsItAndEveryLaterWrite(t *testing.T) {
if strings.Join(*called, ",") != "rename,list" {
t.Fatalf("ran %v; want the first rename and the read only", *called)
}
if !recorder.Pending() {
if !recorder.Held() {
t.Fatal("recorder does not report a pending write")
}
var held []string
for _, call := range recorder.Calls() {
if call.Planned {
if call.Held {
held = append(held, call.Name)
}
}
Expand All @@ -109,7 +109,7 @@ func TestAWriteThatNeedsApprovalHoldsItAndEveryLaterWrite(t *testing.T) {
}

*called = nil
approved := WithRecorder(context.Background(), &Recorder{Approved: true})
approved := WithApproval(context.Background())
if _, err := mode.Execute(approved, "tasks.delete", nil); err != nil || strings.Join(*called, ",") != "delete" {
t.Fatalf("approved delete: err = %v, ran %v", err, *called)
}
Expand Down
Loading
Loading