Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions apps/candidate_evaluator/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Candidate Evaluator

Capability-bounded candidate evaluation application for the Howl ecosystem.

Demonstrates HowlFrame independently auditing speculative candidates from HowlDream:
- Intent is not authority: candidates claiming execution authority or self-approval are categorically rejected.
- Candidates with critical contradictions or verification failures are rejected.
- Grounded candidates with verified constraints are accepted for deliberate development.
- Inconclusive or ungrounded candidates remain unresolved.

## Compiling to Standalone Bytecode

```bash
howlframe -compile-bc candidate_evaluator.howl -o candidate_evaluator.hfbc
```

## Running

```bash
howlframe -run-bc -allow-caps filesystem candidate_evaluator.hfbc /path/to/candidate.json
```

Outputs structured evaluation conforming to `howl.assessment/v1`.
118 changes: 118 additions & 0 deletions apps/candidate_evaluator/candidate_evaluator.howl
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
(cli_app
(let (cand_path (cli_args 0))
(if (= cand_path "")
(do
(stderr "usage: candidate_evaluator <candidate.json>")
(exit 2))
(try_let (content_bytes (read_file cand_path))
(catch err
(do
(stderr "Failed to read candidate file")
(exit 3)))
(let (content (bytes_to_string content_bytes))
(try_let (cand (parse_json Candidate content))
(catch err
(do
(stderr "Failed to parse candidate JSON")
(exit 3)))
(let (cand_id (map_get cand "candidate_id"))
(do
(if (is_nil cand_id)
(set cand_id "unknown_candidate")
)
(let (cand_status (map_get cand "status"))
(do
(if (is_nil cand_status)
(set cand_status "GENERATED")
)
(let (authority (map_get cand "authority"))
(let (auth_exec "false")
(let (auth_type "ADVISORY")
(do
(if (!= (is_nil authority) true)
(do
(let (e (map_get authority "executable"))
(if (!= (is_nil e) true)
(set auth_exec (to_string e))
)
)
(let (t (map_get authority "type"))
(if (!= (is_nil t) true)
(set auth_type (to_string t))
)
)
)
)
(let (disposition "UNRESOLVED")
(let (reason "inconclusive evidence")
(let (confidence "LOW")
(do
(if (or (!= auth_type "ADVISORY") (= auth_exec "true"))
(do
(set disposition "REJECT")
(set reason "authority escalation prohibited: speculative candidate cannot claim execution authority")
(set confidence "HIGH")
)
(do
(if (= cand_status "REJECTED")
(do
(set disposition "REJECT")
(set reason "candidate rejected by verification or critical contradictions")
(set confidence "HIGH")
)
(do
(if (= cand_status "LOCALLY_VERIFIED")
(do
(set disposition "ACCEPT_FOR_DEVELOPMENT")
(set reason "sufficient evidence invariants passed for deliberate development")
(set confidence "MEDIUM")
)
(do
(if (= cand_status "UNRESOLVED")
(do
(set disposition "UNRESOLVED")
(set reason "unresolved assumptions require further evidence")
(set confidence "LOW")
)
(do
(set disposition "INVESTIGATE")
(set reason "candidate challenged but requires downstream review")
(set confidence "LOW")
)
)
)
)
)
)
)
)
(print "{")
(print " \"schema_version\": \"howl.assessment/v1\",")
(print (str_join (list " \"assessment_id\": \"assess-" (to_string cand_id) "\",") ""))
(print (str_join (list " \"candidate_id\": \"" (to_string cand_id) "\",") ""))
(print (str_join (list " \"disposition\": \"" disposition "\",") ""))
(print (str_join (list " \"confidence\": \"" confidence "\",") ""))
(print (str_join (list " \"reason\": \"" reason "\",") ""))
(print " \"authority\": {")
(print " \"type\": \"ADVISORY\",")
(print " \"executable\": false")
(print " }")
(print "}")
)
)
)
)
)
)
)
)
)
)
)
)
)
)
)
)
)
)
123 changes: 123 additions & 0 deletions apps/candidate_evaluator/candidate_evaluator_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package candidate_evaluator_test

import (
"encoding/json"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)

type AssessmentResult struct {
SchemaVersion string `json:"schema_version"`
AssessmentID string `json:"assessment_id"`
CandidateID string `json:"candidate_id"`
Disposition string `json:"disposition"`
Confidence string `json:"confidence"`
Reason string `json:"reason"`
Authority struct {
Type string `json:"type"`
Executable bool `json:"executable"`
} `json:"authority"`
}

func TestCandidateEvaluator(t *testing.T) {
repositoryRoot, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatalf("resolve repository root: %v", err)
}
scratchDir := t.TempDir()
compiler := filepath.Join(scratchDir, "howlframe")
artifact := filepath.Join(scratchDir, "candidate_evaluator.hfbc")

build := exec.Command("go", "build", "-o", compiler, "howlframe.go")
build.Dir = repositoryRoot
if output, buildErr := build.CombinedOutput(); buildErr != nil {
t.Fatalf("build HowlFrame scratch binary: %v\n%s", buildErr, output)
}

appSource := filepath.Join(repositoryRoot, "apps", "candidate_evaluator", "candidate_evaluator.howl")
compile := exec.Command(compiler, "-compile-bc", appSource, "-o", artifact)
compile.Dir = filepath.Join(repositoryRoot, "apps", "candidate_evaluator")
if output, compileErr := compile.CombinedOutput(); compileErr != nil {
t.Fatalf("compile application to bytecode: %v\n%s", compileErr, output)
}

runCase := func(t *testing.T, name string, candidateJSON string, expectDisposition string, expectAuthExecutable bool) {
t.Run(name, func(t *testing.T) {
candPath := filepath.Join(scratchDir, "candidate.json")
if err := os.WriteFile(candPath, []byte(candidateJSON), 0o600); err != nil {
t.Fatalf("write candidate file: %v", err)
}

args := []string{"-run-bc", "-allow-caps", "filesystem", artifact, candPath}
cmd := exec.Command(compiler, args...)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("execution failed: %v\nOutput: %s", err, string(out))
}

var assessment AssessmentResult
jsonStr := strings.TrimSpace(string(out))
if err := json.Unmarshal([]byte(jsonStr), &assessment); err != nil {
t.Fatalf("failed to unmarshal assessment JSON: %v\nRaw: %s", err, jsonStr)
}

if assessment.Disposition != expectDisposition {
t.Errorf("expected disposition %q, got %q (reason: %s)", expectDisposition, assessment.Disposition, assessment.Reason)
}
if assessment.Authority.Executable != expectAuthExecutable {
t.Errorf("expected authority.executable=%v, got %v", expectAuthExecutable, assessment.Authority.Executable)
}
if assessment.Authority.Type != "ADVISORY" {
t.Errorf("expected authority.type='ADVISORY', got %q", assessment.Authority.Type)
}
})
}

// Case 1: Locally verified candidate -> ACCEPT_FOR_DEVELOPMENT
cand1 := `{
"candidate_id": "run-001/candidates/0/0",
"status": "LOCALLY_VERIFIED",
"trust": "UNVERIFIED",
"authority": {"type": "ADVISORY", "executable": false}
}`
runCase(t, "LocallyVerifiedPromotesToDevelopment", cand1, "ACCEPT_FOR_DEVELOPMENT", false)

// Case 2: Rejected candidate -> REJECT
cand2 := `{
"candidate_id": "run-001/candidates/0/1",
"status": "REJECTED",
"trust": "UNVERIFIED",
"authority": {"type": "ADVISORY", "executable": false}
}`
runCase(t, "RejectedCandidateStaysRejected", cand2, "REJECT", false)

// Case 3: Unresolved assumptions -> UNRESOLVED
cand3 := `{
"candidate_id": "run-001/candidates/0/2",
"status": "UNRESOLVED",
"trust": "UNVERIFIED",
"authority": {"type": "ADVISORY", "executable": false}
}`
runCase(t, "UnresolvedRemainsUnresolved", cand3, "UNRESOLVED", false)

// Case 4: Authority escalation attempt (executable=true) -> REJECT
cand4 := `{
"candidate_id": "run-001/candidates/0/3",
"status": "LOCALLY_VERIFIED",
"trust": "UNVERIFIED",
"authority": {"type": "ADVISORY", "executable": true}
}`
runCase(t, "AuthorityEscalationAttemptRejected", cand4, "REJECT", false)

// Case 5: Authority escalation attempt (type=EXECUTIVE) -> REJECT
cand5 := `{
"candidate_id": "run-001/candidates/0/4",
"status": "LOCALLY_VERIFIED",
"trust": "UNVERIFIED",
"authority": {"type": "EXECUTIVE", "executable": false}
}`
runCase(t, "ExecutiveAuthorityTypeRejected", cand5, "REJECT", false)
}
Loading