diff --git a/.github/workflows/promotion-pr.yml b/.github/workflows/promotion-pr.yml index 875cc5c..76738ea 100644 --- a/.github/workflows/promotion-pr.yml +++ b/.github/workflows/promotion-pr.yml @@ -1,6 +1,7 @@ name: Promotion request verification on: - pull_request: + # Runs even for source-to-distribution merge conflicts. No PR code is executed. + pull_request_target: branches: [main] types: [opened, synchronize, reopened, ready_for_review] permissions: @@ -17,18 +18,15 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 with: node-version: '22.22.2' - - name: Check current request and reuse canonical develop proof + - name: Verify request with trusted base-tree code only env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} EXPECTED_SOURCE: ${{ github.event.pull_request.head.sha }} EXPECTED_MAIN: ${{ github.event.pull_request.base.sha }} - run: | - node scripts/production-request.mjs "$PR_NUMBER" "$EXPECTED_MAIN" "$EXPECTED_SOURCE" --identity-only - node scripts/production-verification.mjs "$EXPECTED_SOURCE" 1500 - node scripts/production-request.mjs "$PR_NUMBER" "$EXPECTED_MAIN" "$EXPECTED_SOURCE" --identity-only + run: node .github/scripts/promotion-preflight.mjs diff --git a/AGENTS.md b/AGENTS.md index 3bd0772..0bd523b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,17 @@ Docs: feature -> PR -> develop -> bilingual build + browser checks - Workflow guards are not branch protection; verify actual repository/environment rules before claiming enforcement. - This checkout: `origin` = ServierHub fork; `upstream` = personal bioinfornatics repository; inspect remotes/account before publication. +## Promotion PR conflicts and CI trust boundary + +- `develop` and `main` deliberately have different trees. A promotion PR may be `CONFLICTING` / `DIRTY`: source-only files removed from `main` can cause modify/delete conflicts, and contributor versus installation README/AGENTS content can differ. This is not, by itself, a failed publication. +- Never resolve those conflicts to make the promotion PR mergeable. Never merge/rebase `develop` into `main`, restore authoring files there, or bypass provenance checks. Generate the allowlisted distribution and publish it with the previous `main` as sole parent; close the PR without merging. +- GitHub does not run `pull_request` workflows on merge-conflicted PRs. An absent PR check is NOT success. Validate trigger compatibility before rollout, especially for source-to-distribution requests. +- A conflict-compatible `pull_request_target` preflight must run trusted base-tree code only: checkout the exact base/main SHA, explicitly grant only read-only contents/actions/pull-request permissions, and reject foreign-repository heads. Never checkout/import/build PR-head code, execute its artifacts, interpolate PR text into shell code, or use publication secrets in this job. +- Treat the PR head SHA only as validated API data. Check current request/source/base identities before and after canonical exact-source verification. Identity/proof success is informational, not approval or permission to publish; publication rechecks authorization and source/main races independently. +- Main must contain the workflow and its complete trusted helper closure before the new trigger can run. Local YAML/CLI tests and a successful distribution install do not prove GitHub event execution. Verify an actual conflicted same-repository promotion request after rollout; disclose any bootstrap or missing-run limitation. +- Report commit, push, develop integration, canonical CI, generated main publication and PR preflight separately. Verify remote main SHA, distribution sourceCommit and sole parent before reporting publication success. Do not reopen/republish merely because an informational check is absent; inspect provenance first and obtain authority for additional remote actions. +- For timing, separate feature CI, post-merge CI, overlapping promotion wait, actual publishing and agent/API overhead. Do not add concurrent waits as sequential work or equate test-step critical path with total workflow wall time. + ## Authoring and safety - Keep root `plugin.json` valid; keep every Skill independently installable. - Skill metadata/instructions: English; descriptions concise, third-person, stating behavior and activation conditions. diff --git a/config/reproducibility/source-manifest.json b/config/reproducibility/source-manifest.json index 058111c..2d2618e 100644 --- a/config/reproducibility/source-manifest.json +++ b/config/reproducibility/source-manifest.json @@ -1566,6 +1566,7 @@ "scripts/production-verification.mjs", "scripts/production.mjs", "scripts/promote-production.mjs", + "scripts/promotion-preflight.mjs", "scripts/publish-documentation.sh", "scripts/rebuild-production.mjs", "scripts/release-all.mjs", diff --git a/docs/branch-migration.md b/docs/branch-migration.md index 84c580a..a484279 100644 --- a/docs/branch-migration.md +++ b/docs/branch-migration.md @@ -178,4 +178,13 @@ measured production latency target; those require a separately authorized PR run Diagnostic campaign matrices are replaced by a direct public classification/projection table and one split-secret receipt journey. The real Goose missing-provider journey and process lifecycle boundaries remain. config/ci/test-risk-decisions.json records the scope and the unverified 300-second CI target. -Promotion PR verification is defined in .github/workflows/promotion-pr.yml. Informational --identity-only request mode checks identity but grants no publishing authority. The job checks identity before and after canonical verification so source or main changes invalidate the request. Fork requests never run this preflight. Full Development verification only accepts PRs targeting develop, develop pushes and explicit workflow calls. +Promotion PR verification uses the trusted base preflight in .github/workflows/promotion-pr.yml. Standalone --identity-only remains an informational local check, not the PR entrypoint. Neither grants publication authority. Full Development verification accepts develop PRs/pushes and explicit workflow calls. + +The promotion PR preflight uses pull_request_target because source/distribution trees +may have merge conflicts. It has read-only contents/actions/pull-request permissions, +rejects fork requests, checks out the exact trusted base/main SHA, and executes only +.github/scripts/promotion-preflight.mjs with its contained built-in-only helper closure. +The PR head SHA is API data only: no checkout, import, build or artifact execution +from the PR head. Source/base are checked before and after canonical proof. This +informational check never substitutes for current publication approval. The trusted +closure must first be published to main before this trigger can run live. diff --git a/scripts/production.mjs b/scripts/production.mjs index 5cf250b..b3b0bc3 100644 --- a/scripts/production.mjs +++ b/scripts/production.mjs @@ -73,6 +73,8 @@ export function projectProduction(destination) { } // GitHub discovers manually dispatched workflows from the default branch. for(const file of ['development-ci.yml','promotion-pr.yml','promote-production.yml','reproducibility.yml'])copy(path.join(source,'.github','workflows',file),path.join(destination,'.github','workflows',file)); + // Minimal built-in-only trusted base preflight closure, never PR source code. + for(const file of ['promotion-preflight.mjs','production-request.mjs','production-verification.mjs'])copy(path.join(source,'scripts',file),path.join(destination,'.github','scripts',file)); const files=inventory(destination);const manifest={schemaVersion:1,runtimeMode:'node-bundled',sourceCommit:commit,sourceTree:tree,inventorySha256:hash(JSON.stringify(files)),files}; writeFileSync(path.join(destination,'distribution-manifest.json'),JSON.stringify(manifest,null,2)+'\n'); verifyProduction(destination);return manifest; diff --git a/scripts/promotion-preflight.mjs b/scripts/promotion-preflight.mjs new file mode 100644 index 0000000..44c58c8 --- /dev/null +++ b/scripts/promotion-preflight.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node +/** Trusted base-tree code only. PR source is API data, never code or artifacts to execute. */ +import {execFileSync} from 'node:child_process'; +import {pathToFileURL} from 'node:url'; +import {validateRequestIdentity} from './production-request.mjs'; +import {awaitCanonicalVerification} from './production-verification.mjs'; +export async function verifyPromotionPreflight({number,repository,source,main,api,verify=awaitCanonicalVerification,waitSeconds=0}) { + if(!/^[1-9][0-9]*$/.test(String(number))||!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository??'')||![source,main].every(s=>/^[a-f0-9]{40}$/.test(s??'')))throw Error('invalid promotion preflight inputs'); + const check=()=>validateRequestIdentity(api('pulls/'+number),repository,main,api('branches/develop').commit.sha,api('branches/main').commit.sha,source); + check(); + const proof=await verify({source,repository,api,waitSeconds}); + check(); + return {sourceCommit:source,baseCommit:main,verification:proof,authority:'informational only; publication still requires approval'}; +} +if(process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href){ + const {PR_NUMBER:number,GITHUB_REPOSITORY:repository,EXPECTED_SOURCE:source,EXPECTED_MAIN:main}=process.env; + if(execFileSync('git',['rev-parse','HEAD'],{encoding:'utf8'}).trim()!==main)throw Error('preflight must execute from exact trusted base checkout'); + const api=(endpoint,paginate=false)=>JSON.parse(execFileSync('gh',['api',...(paginate?['--paginate','--slurp']:[]),'repos/'+repository+(endpoint?'/'+endpoint:'')],{encoding:'utf8',timeout:30000,maxBuffer:16*1024*1024})); + console.log(JSON.stringify(await verifyPromotionPreflight({number,repository,source,main,api,waitSeconds:1500}))); +} diff --git a/skills/agent-creator/SKILL.md b/skills/agent-creator/SKILL.md index bee7a7f..d5d1cfc 100644 --- a/skills/agent-creator/SKILL.md +++ b/skills/agent-creator/SKILL.md @@ -5,6 +5,14 @@ description: Creates and manages Goose custom agent definitions for reusable rol # Agent Creator +## Evaluation value and cost boundary + +- Evaluate only when requested or required by an approved acceptance protocol; authoring or packaging alone does not authorize a model campaign. Use the smallest set covering distinct plausible failures, user journeys and safety boundaries, not a scenario-count target. +- Before expensive execution, state the decision to inform, each scenario's distinct risk, observable expected outcome and cheapest valid oracle. An assertion must reject a plausible wrong result, not merely echo a mock, accept self-reported success, check file existence or reproduce the candidate's own calculation. Review conditional assertions that accept both success and failure; define the expected outcome for the declared preconditions. +- Declare the total candidate and grader call bound, time/cost budget and stopping rule before launch. Account for scenarios, variants, repetitions, graders, criterion-level calls and repair iterations; label unknown prices as unknown, never free. Use existing supported options or compact planning notes, not invented schema fields. User authorization must cover paid execution and any budget expansion. +- Add a scenario only for uncovered risk or a meaningful interaction. Add repetitions or graders only for a named uncertainty that can change the decision, with marginal calls/cost and a bound. A budget ceiling is not a spending target; no Cartesian campaign or per-component multiplication by default. Required samples and safety checks in an approved protocol remain required; a small set is not reliability proof. +- Prefer deterministic artifact checks for deterministic properties and blinded semantic grading for contextual quality. Reuse compatible frozen evidence only when source, fixtures, protocol and required bindings still match. Report corrections re-render evidence; they do not rerun candidates. Preserve failures and missingness; never retry or rewrite criteria until green. At the bound, report inconclusive or blocked and the next decision, not automatic escalation. + ## Offline runtime Use `node scripts/agent-creator.mjs` for a copied source Skill; it requires Node.js 22+ and resolves only this Skill’s generated runtime. Tagged native archives use the released executable and do not require Node.js. Report a missing runtime or mapping as a packaging defect. @@ -123,7 +131,7 @@ Prefer the bundled **agent-creator** executable for automation. It provides init Evaluate the custom agent as a reusable role, not as a skill trigger. The key question is whether delegating the same realistic task to the specialized agent produces better, more consistent results than a controlled baseline. -1. Create 2–3 realistic tasks covering the role's core responsibilities, boundaries, and likely failure modes. Ask the user to review them. Make each task autonomous: name the current and baseline agent files, declare whether it must explain or execute, list immutable fixture paths and preconditions, and give it enough turns and tools to finish. Freeze assertions before runs; graders must not add hidden deliverables afterward. +1. Select a compact set of risk-distinct realistic tasks covering the role's core responsibilities, boundaries, and likely failure modes. Ask the user to review them. Make each task autonomous: name the current and baseline agent files, declare whether it must explain or execute, list immutable fixture paths and preconditions, and give it enough turns and tools to finish. Freeze assertions before runs; graders must not add hidden deliverables afterward. 2. Save them in an eval JSON file: ```json @@ -173,7 +181,7 @@ Evaluate the custom agent as a reusable role, not as a skill trigger. The key qu When improving an existing agent, pass `--baseline-agent ` to compare against the old instructions instead. -4. Classify and version every assertion before running. Prefer objects with `id`, positive `version`, `classification` (`deterministic` or `semantic`), and `criterion`; deterministic assertions also declare a reproducible `checker` (`contains`, `not-contains`, or `regex`). Legacy prefixed strings remain supported. Semantic criteria require multiple independently identified blinded graders, contained output quotations, and a bounded call budget. Graders receive the published criterion and an opaque variant alias, never hidden criteria, true variant identity, or other grades. Disagreement or invalid/missing evidence remains inconclusive for human review; never average it into pass/fail: +4. Classify and version every assertion before running. Prefer objects with `id`, positive `version`, `classification` (`deterministic` or `semantic`), and `criterion`; deterministic assertions also declare a reproducible `checker` (`contains`, `not-contains`, or `regex`). Legacy prefixed strings remain supported. The current Agent Creator runtime requires at least two independently identified blinded graders per semantic assertion (unlike the output-batched Skill Creator path). Budget semantic assertions × declared variants/runs × graders before execution, plus any separately authorized retries; use the minimum supported grader count unless an additional grader answers a concrete uncertainty. Contained quotations support review but do not alone prove semantic correctness. Graders receive the published criterion and an opaque variant alias, never hidden criteria, true variant identity, or other grades. Disagreement or invalid/missing evidence remains inconclusive for human review; never average it into pass/fail: ```bash agent-creator grade \ @@ -181,7 +189,7 @@ Evaluate the custom agent as a reusable role, not as a skill trigger. The key qu --llm-grader \ --grader reviewer-a=model-a \ --grader reviewer-b=model-b \ - --max-grader-calls 24 + --max-grader-calls ``` 5. Aggregate results: diff --git a/skills/hook-creator/SKILL.md b/skills/hook-creator/SKILL.md index 52d14f6..0117c8d 100644 --- a/skills/hook-creator/SKILL.md +++ b/skills/hook-creator/SKILL.md @@ -5,6 +5,14 @@ description: Creates, audits, and tests lifecycle hooks inside Open Plugins. Use # Hook Creator +## Evaluation value and cost boundary + +- Evaluate only when requested or required by an approved acceptance protocol; authoring or packaging alone does not authorize a model campaign. Use the smallest set covering distinct plausible failures, user journeys and safety boundaries, not a scenario-count target. +- Before expensive execution, state the decision to inform, each scenario's distinct risk, observable expected outcome and cheapest valid oracle. An assertion must reject a plausible wrong result, not merely echo a mock, accept self-reported success, check file existence or reproduce the candidate's own calculation. Review conditional assertions that accept both success and failure; define the expected outcome for the declared preconditions. +- Declare the total candidate and grader call bound, time/cost budget and stopping rule before launch. Account for scenarios, variants, repetitions, graders, criterion-level calls and repair iterations; label unknown prices as unknown, never free. Use existing supported options or compact planning notes, not invented schema fields. User authorization must cover paid execution and any budget expansion. +- Add a scenario only for uncovered risk or a meaningful interaction. Add repetitions or graders only for a named uncertainty that can change the decision, with marginal calls/cost and a bound. A budget ceiling is not a spending target; no Cartesian campaign or per-component multiplication by default. Required samples and safety checks in an approved protocol remain required; a small set is not reliability proof. +- Prefer deterministic artifact checks for deterministic properties and blinded semantic grading for contextual quality. Reuse compatible frozen evidence only when source, fixtures, protocol and required bindings still match. Report corrections re-render evidence; they do not rerun candidates. Preserve failures and missingness; never retry or rewrite criteria until green. At the bound, report inconclusive or blocked and the next decision, not automatic escalation. + ## Offline runtime Use `node scripts/hook-creator.mjs` for a copied source Skill; it requires Node.js 22+ and resolves only this Skill’s generated runtime. Tagged native archives use the released executable and do not require Node.js. Report a missing runtime or mapping as a packaging defect. diff --git a/skills/plugin-creator/SKILL.md b/skills/plugin-creator/SKILL.md index acfb691..e90f001 100644 --- a/skills/plugin-creator/SKILL.md +++ b/skills/plugin-creator/SKILL.md @@ -7,6 +7,14 @@ description: Orchestrates complete Open Plugin packages. Use when plugin.json or Create production-ready Agent Plugins for Goose. Own package architecture, root `plugin.json`, portable MCP declarations, Goose extension integration, whole-package validation, installation, and packaging. Route component content to its specialist. +## Evaluation value and cost boundary + +- Evaluate only when requested or required by an approved acceptance protocol; authoring or packaging alone does not authorize a model campaign. Use the smallest set covering distinct plausible failures, user journeys and safety boundaries, not a scenario-count target. +- Before expensive execution, state the decision to inform, each scenario's distinct risk, observable expected outcome and cheapest valid oracle. An assertion must reject a plausible wrong result, not merely echo a mock, accept self-reported success, check file existence or reproduce the candidate's own calculation. Review conditional assertions that accept both success and failure; define the expected outcome for the declared preconditions. +- Declare the total candidate and grader call bound, time/cost budget and stopping rule before launch. Account for scenarios, variants, repetitions, graders, criterion-level calls and repair iterations; label unknown prices as unknown, never free. Use existing supported options or compact planning notes, not invented schema fields. User authorization must cover paid execution and any budget expansion. +- Add a scenario only for uncovered risk or a meaningful interaction. Add repetitions or graders only for a named uncertainty that can change the decision, with marginal calls/cost and a bound. A budget ceiling is not a spending target; no Cartesian campaign or per-component multiplication by default. Required samples and safety checks in an approved protocol remain required; a small set is not reliability proof. +- Prefer deterministic artifact checks for deterministic properties and blinded semantic grading for contextual quality. Reuse compatible frozen evidence only when source, fixtures, protocol and required bindings still match. Report corrections re-render evidence; they do not rerun candidates. Preserve failures and missingness; never retry or rewrite criteria until green. At the bound, report inconclusive or blocked and the next decision, not automatic escalation. + ## Offline runtime Use `node scripts/plugin-creator.mjs` for a copied source Skill; it requires Node.js 22+ and resolves only this Skill’s generated runtime. Tagged native archives use the released executable and do not require Node.js. Report a missing runtime or mapping as a packaging defect. @@ -99,7 +107,7 @@ Adapter-only documents retain structural checks with an activation warning; that ## Behavioral Evaluation Contract -When asked to evaluate, test, benchmark, compare, or prove improvement, execute the official workflow. Freeze baseline provenance and fixtures. Require a complete evaluation receipt for each behaviorally changed Skill and plugin-level integration scenarios. Aggregate the resulting evidence into a combined benchmark and generate a review viewer. Require: +When asked to evaluate, test, benchmark, compare, or prove improvement, execute the official workflow. Freeze baseline provenance and fixtures. Require a complete evaluation receipt for each behaviorally changed Skill and plugin-level integration scenarios. Reuse compatible complete receipts for unchanged components only when all current hashes, bindings and protocol obligations match; explicitly mark stale or uncovered components. Integration scenarios should exercise distinct cross-component risks and handoffs, not repeat every component assertion. A representative journey may cover several components only when its declared coverage and observed handoff evidence satisfy the typed receipt contract. Aggregate the resulting evidence into a combined benchmark and generate a review viewer. Require: 1. paired `with_skill` and `old_skill`/`without_skill` outputs; 2. `grading.json` with `text`, `passed`, and `evidence` for every run; diff --git a/skills/plugin-creator/references/component-evidence.md b/skills/plugin-creator/references/component-evidence.md index 3faabc1..e668f9b 100644 --- a/skills/plugin-creator/references/component-evidence.md +++ b/skills/plugin-creator/references/component-evidence.md @@ -54,3 +54,15 @@ A full plugin (one containing any agent, hook, or MCP component) requires typed ## Full-eval handoff `full-eval --dry-run` lists each discovered component, expected typed receipt path, source hash, and specialist handoff command. Skill evaluation remains routed to skill-creator. Agent, hook, and MCP evidence is supplied by their specialized creators. For full plugins, integration evidence is written to `integration/receipt.json`. Resume only advances once those receipts exist; final verification validates their contents and freshness. + +## Proportionate integration coverage + +Complete component coverage does not mean one new model campaign per component or +repeating every unit-level assertion in integration. Use compatible unchanged +component receipts when all required current hashes and bindings match. Select a +compact set of real integration journeys for distinct interaction/handoff failures; +a journey may cover multiple components with explicit declared and observed evidence. +Do not claim coverage from names alone or omit required receipts to meet a budget. +Before execution, account for nested specialist campaigns, variants, repetitions and +criterion-level versus batched grader calls. Justify each additional campaign by its +new risk or uncertainty; preserve unavailable/stale evidence as blocked or inconclusive. diff --git a/skills/skill-creator/SKILL.md b/skills/skill-creator/SKILL.md index 98e62aa..1c24d03 100644 --- a/skills/skill-creator/SKILL.md +++ b/skills/skill-creator/SKILL.md @@ -13,6 +13,14 @@ Create/improve one portable Agent Skill, or execute its evaluation and produce H 2. **Evaluate or report:** read [evaluation workflow](references/evaluate-workflow.md), then execute existing scenarios with mandatory isolated A/B contexts and activation or behavioral measurement, collect/grade evidence and generate/inspect HTML. Isolation is not a mode; behavioral is the default. Do not audit, redesign or rewrite the Skill first. 3. **Both requested:** execute creation first, then read the evaluation workflow for the frozen candidate. Do not preload both workflows. +## Evaluation value and cost boundary + +- Evaluate only when requested or required by an approved acceptance protocol; authoring or packaging alone does not authorize a model campaign. Use the smallest set covering distinct plausible failures, user journeys and safety boundaries, not a scenario-count target. +- Before expensive execution, state the decision to inform, each scenario's distinct risk, observable expected outcome and cheapest valid oracle. An assertion must reject a plausible wrong result, not merely echo a mock, accept self-reported success, check file existence or reproduce the candidate's own calculation. Review conditional assertions that accept both success and failure; define the expected outcome for the declared preconditions. +- Declare the total candidate and grader call bound, time/cost budget and stopping rule before launch. Account for scenarios, variants, repetitions, graders, criterion-level calls and repair iterations; label unknown prices as unknown, never free. Use existing supported options or compact planning notes, not invented schema fields. User authorization must cover paid execution and any budget expansion. +- Add a scenario only for uncovered risk or a meaningful interaction. Add repetitions or graders only for a named uncertainty that can change the decision, with marginal calls/cost and a bound. A budget ceiling is not a spending target; no Cartesian campaign or per-component multiplication by default. Required samples and safety checks in an approved protocol remain required; a small set is not reliability proof. +- Prefer deterministic artifact checks for deterministic properties and blinded semantic grading for contextual quality. Reuse compatible frozen evidence only when source, fixtures, protocol and required bindings still match. Report corrections re-render evidence; they do not rerun candidates. Preserve failures and missingness; never retry or rewrite criteria until green. At the bound, report inconclusive or blocked and the next decision, not automatic escalation. + ## Common boundaries - Use supplied context and make reversible choices within scope. Ask only for a concrete user decision blocking the next action; continue unaffected work. diff --git a/skills/skill-creator/references/description-optimization.md b/skills/skill-creator/references/description-optimization.md index 6c67179..fa30967 100644 --- a/skills/skill-creator/references/description-optimization.md +++ b/skills/skill-creator/references/description-optimization.md @@ -21,7 +21,7 @@ description: Processes PDF files and extracts text or tables. Use when working w ## Trigger evaluation set -Create about 20 realistic queries with a balanced mix of: +Select a compact risk-distinct query set, without a universal count, with a balanced mix of: - positive cases using varied phrasing, languages, complexity, and implicit intent; - difficult negative cases that resemble the Skill but belong to a neighboring capability. @@ -46,11 +46,12 @@ skill-creator evidence-loop \ --eval-set \ --skill-path \ --model \ - --max-iterations 5 \ + --runs-per-query \ + --max-iterations \ --verbose ``` -The loop evaluates each query repeatedly, splits train and held-out test cases, proposes improved descriptions, and selects by held-out score. Apply the best description, show the before/after text and scores, and preserve the report. +Set --runs-per-query explicitly rather than silently accepting the runtime default of three. Before invoking the loop, bound all query repetitions, candidate-generation calls and iterations; include held-out evaluation costs and uncertainty about provider prices. Run optimization only for an observed discovery problem or an explicit comparison request, not routine description authoring. The loop evaluates queries repeatedly, splits train and held-out cases, proposes descriptions and selects by held-out score. The configured bound is not permission to keep trying until green. Review the proposed description and evidence before applying it within the authorized scope; show before/after text, actual calls and limitations. ## Quality checks diff --git a/skills/skill-creator/references/evaluate-workflow.md b/skills/skill-creator/references/evaluate-workflow.md index 0df21d8..5032ead 100644 --- a/skills/skill-creator/references/evaluate-workflow.md +++ b/skills/skill-creator/references/evaluate-workflow.md @@ -2,6 +2,8 @@ Read only for evaluation/reporting. Isolation is the execution invariant, not a third measurement mode. Do not load creation guidance for existing cases. +Before launch, apply the entrypoint evaluation value/cost boundary: use the supplied frozen scope without silently adding cases, compute candidate/grader call bounds and state a stopping rule. Additional cases, repeats or graders need a distinct risk or decision-changing uncertainty and authorized marginal budget. This is a concise execution budget check, not a separate audit or redesign. If scope cannot fit the authorized budget, explain the uncovered obligations and request that concrete decision; do not silently sample away required criteria. + 1. **Launch the requested evaluation with frozen cases.** Use supplied inputs directly; do not begin with Skill audit/validation or redesign. For missing cases, ask for the concrete input or author them only when requested, reading [scenario design](evaluation-design.md) and [schemas](schemas.md) before writing artifacts. - Use `run-evaluation --mode activation` for actual load requests; read [activation protocol](trigger-evaluation.md). - Use `run-evaluation --mode behavioral` (default) for task quality; read the installed-capability section of [behavioral protocol](evaluation-workflow.md). diff --git a/skills/skill-creator/references/evaluation-design.md b/skills/skill-creator/references/evaluation-design.md index ab8f348..815471b 100644 --- a/skills/skill-creator/references/evaluation-design.md +++ b/skills/skill-creator/references/evaluation-design.md @@ -14,6 +14,21 @@ Before freezing cases: 4. Prefer equivalence classes, boundary values and meaningful state transitions over exhaustive prompt permutations. Add combinations when the interaction itself is risky; do not require the full Cartesian product. Cheap deterministic tests may still need many cases for genuinely different rules. 5. Freeze criteria, fixtures, source/baseline identity, permitted actions and execution/repetition budgets before launching. Preserve held-out cases for improvement claims when feasible; report any limits of generalization. +### Global and marginal execution cost + +Before launching, account separately for candidate and grader calls. Candidate calls +scale with selected scenarios, compared variants and repetitions. Grader calls depend +on the adapter: a batch-capable grader may judge all criteria for one output in one +call; a criterion-level adapter multiplies by semantic criteria as well as graders. +Include optimization/repair rounds, retries and provider timeouts in the total bound. +Use measured prices when available; otherwise disclose unknown monetary cost and +bound calls and time. Never introduce planning-only fields into closed runtime schemas. + +For every proposed expansion record: uncovered risk or decision-changing uncertainty, +additional calls/time/cost, stopping condition and required authority. If no useful +information gain is expected, do not execute it. Preserve an explicit uncovered-risk +list rather than pretending a cheap subset meets a required full acceptance protocol. + ### Choose the oracle per assertion | Property | Preferred evidence and check | Limit | diff --git a/skills/skill-creator/references/trigger-evaluation.md b/skills/skill-creator/references/trigger-evaluation.md index 27aa4cc..0e4aa03 100644 --- a/skills/skill-creator/references/trigger-evaluation.md +++ b/skills/skill-creator/references/trigger-evaluation.md @@ -20,7 +20,7 @@ Create realistic `should_trigger` and difficult `should_not_trigger` queries. In - simple requests where loading the Skill adds no value; - substantive requests where the Skill should clearly help. -Prefer at least 8–10 positive and 8–10 negative cases. Run each query multiple times when model variance matters. +Choose the smallest balanced set covering relevant positive intents and difficult neighboring negatives; the categories above are selection dimensions, not a Cartesian checklist or minimum count. A new wording is valuable only when it exposes a distinct discovery failure. State the maximum query/variant/repetition calls and stopping rule before execution. Repeat only for a defined variance question within the authorized budget. ## Metrics @@ -50,14 +50,14 @@ Keep train and held-out test cases separate. Select the best description by held skill-creator trigger-eval \ --eval-set \ --skill-path \ - --runs-per-query 3 + --runs-per-query ``` For iterative optimization, use the description-optimization reference linked directly from SKILL.md. ## Sibling overlap -Evaluate all nearby Skills on the same difficult negatives. A description is not acceptable when it improves recall by stealing requests that belong to a sibling. +Evaluate the relevant neighboring Skills on shared discriminating negatives when overlap can change the routing decision. Include this additional variant cost in the budget; do not sweep every installed Skill by default. A description is not acceptable when it improves recall by stealing requests that belong to a sibling. ## Completion checklist diff --git a/tests/test_ci_verification.test.ts b/tests/test_ci_verification.test.ts index 8306a69..1bc3811 100644 --- a/tests/test_ci_verification.test.ts +++ b/tests/test_ci_verification.test.ts @@ -75,25 +75,21 @@ test('promotion preflight validates current source/base without granting publica for(const changed of [{state:'closed'},{draft:true},{head:{...request.head,ref:'feature'}},{head:{...request.head,repo:{full_name:'fork/repo'}}},{base:{...request.base,repo:{full_name:'fork/repo'}}},{base:{...request.base,ref:'develop'}},{head:{...request.head,sha:main}},{base:{...request.base,sha:source}}])assert.throws(()=>identityCheck({...request,...changed})); assert.throws(()=>identityCheck(request,main),/moved/);assert.throws(()=>identityCheck(request,source,source),/moved/); }); -test('feature/develop keep full CI while promotion PR uses read-only exact-source proof only',()=>{ +test('conflicting promotion PRs use trusted base code only while feature/develop keep full CI',()=>{ const ci:any=yaml.load(readFileSync(root+'.github/workflows/development-ci.yml','utf8')); const light:any=yaml.load(readFileSync(root+'.github/workflows/promotion-pr.yml','utf8')); assert.deepEqual(ci.on.push.branches,['develop']);assert.deepEqual(ci.on.pull_request.branches,['develop']); - assert.equal(ci.jobs.quick.if,undefined); - assert.deepEqual(light.on.pull_request.branches,['main']);assert.equal(light.on.pull_request_target,undefined); + assert.equal(light.on.pull_request,undefined);assert.deepEqual(light.on.pull_request_target.branches,['main']); assert.deepEqual(light.permissions,{contents:'read'});assert.deepEqual(Object.keys(light.jobs),['promotion-preflight']); const job=light.jobs['promotion-preflight'];assert.deepEqual(job.permissions,{contents:'read','pull-requests':'read',actions:'read'}); assert.match(job.if,/head.ref == 'develop'/);assert.match(job.if,/head.repo.full_name == github.repository/); const checkout=job.steps.find((s:any)=>s.uses?.startsWith('actions/checkout@')); - assert.equal(checkout.with.ref,'${{ github.event.pull_request.head.sha }}');assert.equal(checkout.with['persist-credentials'],false); - const check=job.steps.find((s:any)=>s.run?.includes('production-verification.mjs')); - assert.equal(check.env.EXPECTED_SOURCE,'${{ github.event.pull_request.head.sha }}');assert.equal(check.env.EXPECTED_MAIN,'${{ github.event.pull_request.base.sha }}'); - assert.equal((check.run.match(/--identity-only/g)||[]).length,2); - assert(check.run.includes('production-verification.mjs "$EXPECTED_SOURCE" 1500')); - assert(check.run.indexOf('--identity-only')check.run.indexOf('production-verification.mjs')); + assert.equal(checkout.with.ref,'${{ github.event.pull_request.base.sha }}');assert.equal(checkout.with['persist-credentials'],false); + const commands=job.steps.filter((s:any)=>s.run).map((s:any)=>s.run); + assert.deepEqual(commands,['node .github/scripts/promotion-preflight.mjs']); + assert.equal(job.steps.find((s:any)=>s.run).env.EXPECTED_SOURCE,'${{ github.event.pull_request.head.sha }}'); for(const step of job.steps)assert.doesNotMatch(step.run??'',/ci-tests|test.mjs|rebuild|production.mjs project|promote-production|gh pr merge|owner_manual/); - const projection=readFileSync(root+'scripts/production.mjs','utf8');assert(projection.includes("'promotion-pr.yml'")); + const projection=readFileSync(root+'scripts/production.mjs','utf8');assert(projection.includes("'promotion-pr.yml'"));assert(projection.includes("'promotion-preflight.mjs'")); }); import {mkdtempSync,writeFileSync,rmSync} from 'node:fs'; @@ -117,3 +113,19 @@ test('identity-only CLI uses read-only current identities and cannot replace app const invalid=invoke(['--unknown']);assert.notEqual(invalid.status,0);assert.match(invalid.stderr,/unsupported promotion request mode/); }finally{rmSync(temp,{recursive:true,force:true});} }); + +import {verifyPromotionPreflight} from '../scripts/promotion-preflight.mjs'; +test('trusted promotion preflight rejects request drift before and after canonical proof without executing source',async()=>{ + const main='b'.repeat(40),request={state:'open',draft:false,head:{ref:'develop',sha:source,repo:{full_name:repo}},base:{ref:'main',sha:main,repo:{full_name:repo}}}; + function fixture(change:string|null=null){let verified=false;const calls:string[]=[];return {calls, + api:(endpoint:string)=>{calls.push(endpoint);if(endpoint==='pulls/42')return change==='fork'?{...request,head:{...request.head,repo:{full_name:'fork/repo'}}}:request; + if(endpoint==='branches/develop')return {commit:{sha:verified&&change==='source'?main:source}}; + if(endpoint==='branches/main')return {commit:{sha:verified&&change==='base'?source:main}};throw Error('unexpected API '+endpoint);}, + verify:async()=>{verified=true;if(change==='proof')throw Error('canonical verification missing');return {sourceCommit:source,runId:3};}, + };} + const f=fixture(),result=await verifyPromotionPreflight({number:42,repository:repo,source,main,...f}); + assert.equal(result.sourceCommit,source);assert.equal(result.baseCommit,main);assert.equal(result.verification.runId,3);assert.match(result.authority,/informational only/); + assert.deepEqual(f.calls,['pulls/42','branches/develop','branches/main','pulls/42','branches/develop','branches/main']); + for(const change of ['fork','source','base','proof'])await assert.rejects(verifyPromotionPreflight({number:42,repository:repo,source,main,...fixture(change)}),/same-repository|moved|canonical verification missing/); + for(const altered of [{number:0},{repository:'owner/repo/unsafe'},{source:'bad'},{main:'bad'}])await assert.rejects(verifyPromotionPreflight({number:42,repository:repo,source,main,...fixture(),...altered}),/invalid/); +});