diff --git a/plugins/magpie-setup/skills/setup/SKILL.md b/plugins/magpie-setup/skills/setup/SKILL.md index 4db647a4f..7f791c847 100644 --- a/plugins/magpie-setup/skills/setup/SKILL.md +++ b/plugins/magpie-setup/skills/setup/SKILL.md @@ -20,9 +20,9 @@ when_to_use: >- to the `welcome-guide` skill. argument-hint: "[install|config|adopt|unadopt|upgrade|worktree-init|verify|reconcile|override skill-name|delegate [skill-name]|welcome|uninstall]" capability: capability:platform -surface_hash: sha256:2557a934f9ea5576 +surface_hash: sha256:cfae279d2c143532 license: Apache-2.0 -measured_tokens: 5307 +measured_tokens: 5306 --- + +The configured skill is `security-issue-import-from-md`. +All override anchors and required configuration files resolve. +The tracker backend is `github`. + +Existing policy: `.apache-magpie-overrides/tools/vetted-ops/config.toml`. + +The original saved caller entry was: + + "security-issue-import-from-md" = ["issue-view"] + +The applicable canonical requirement is `rollup-append`. +The proposed diff adds only that operation. + +The operator declined the proposed rollup-append grant. The saved policy remains unchanged. + +The current stamp has no entry for this skill. diff --git a/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-2-confirmed-unsaved/expected.json b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-2-confirmed-unsaved/expected.json new file mode 100644 index 000000000..64efed35b --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-2-confirmed-unsaved/expected.json @@ -0,0 +1,6 @@ +{ + "decision": "confirmed", + "policy_verified": false, + "policy_changed": false, + "stamp_written": false +} diff --git a/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-2-confirmed-unsaved/report.md b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-2-confirmed-unsaved/report.md new file mode 100644 index 000000000..5b040f0be --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-2-confirmed-unsaved/report.md @@ -0,0 +1,19 @@ + + +The configured skill is `security-issue-import-from-md`. +All override anchors and required configuration files resolve. +The tracker backend is `github`. + +Existing policy: `.apache-magpie-overrides/tools/vetted-ops/config.toml`. + +The original saved caller entry was: + + "security-issue-import-from-md" = ["issue-view"] + +The applicable canonical requirement is `rollup-append`. +The proposed diff adds only that operation. + +The operator confirmed the proposed grant, but has not saved the policy. Re-reading the saved policy still shows only issue-view. + +The current stamp has no entry for this skill. diff --git a/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-3-saved-and-verified/expected.json b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-3-saved-and-verified/expected.json new file mode 100644 index 000000000..9278a1165 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-3-saved-and-verified/expected.json @@ -0,0 +1,6 @@ +{ + "decision": "confirmed", + "policy_verified": true, + "policy_changed": true, + "stamp_written": true +} diff --git a/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-3-saved-and-verified/report.md b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-3-saved-and-verified/report.md new file mode 100644 index 000000000..03fb129a5 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/case-3-saved-and-verified/report.md @@ -0,0 +1,19 @@ + + +The configured skill is `security-issue-import-from-md`. +All override anchors and required configuration files resolve. +The tracker backend is `github`. + +Existing policy: `.apache-magpie-overrides/tools/vetted-ops/config.toml`. + +The original saved caller entry was: + + "security-issue-import-from-md" = ["issue-view"] + +The applicable canonical requirement is `rollup-append`. +The proposed diff adds only that operation. + +The operator confirmed the change and saved the policy. A fresh read of the saved policy shows both issue-view and rollup-append. All other applicable checks passed. + +The current stamp has no entry for this skill. diff --git a/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/output-spec.md b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/output-spec.md new file mode 100644 index 000000000..ed8818bf5 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/output-spec.md @@ -0,0 +1,14 @@ + + +## Output format + +Return ONLY valid JSON with these fields: + +- `decision`: `"declined"` or `"confirmed"` +- `policy_verified`: boolean; true only when the saved policy has been + re-read and all required grants are present. +- `policy_changed`: boolean; whether the saved policy actually changed. +- `stamp_written`: boolean; whether this skill was stamped clean. + +Return no additional text. diff --git a/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/step-config.json b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/step-config.json new file mode 100644 index 000000000..9562e8943 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/step-config.json @@ -0,0 +1,4 @@ +{ + "skill_md": "skills/setup/reconcile.md", + "step_heading": "## Step 2 \u2014 Confirm, apply, and write the stamp" +} diff --git a/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/user-prompt-template.md b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/user-prompt-template.md new file mode 100644 index 000000000..edca4fed2 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile-confirmation/fixtures/user-prompt-template.md @@ -0,0 +1,8 @@ + + +## Reconciliation confirmation state + +{report} + +Apply Step 2's confirmation and stamping rules. Return JSON only. diff --git a/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-4-import-policy-grants/expected.json b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-4-import-policy-grants/expected.json new file mode 100644 index 000000000..9e6886291 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-4-import-policy-grants/expected.json @@ -0,0 +1,24 @@ +{ + "proposals": [ + { + "kind": "policy", + "skill": "security-issue-import", + "override_file": null, + "detail": "Missing caller security-issue-import in .apache-magpie-overrides/tools/vetted-ops/config.toml; propose issue-list-with-bodies and rollup-append." + }, + { + "kind": "policy", + "skill": "security-issue-import-from-md", + "override_file": null, + "detail": "Existing caller security-issue-import-from-md lacks rollup-append; preserve its issue-view grant." + }, + { + "kind": "policy", + "skill": "security-issue-import-from-pr", + "override_file": null, + "detail": "Existing caller security-issue-import-from-pr lacks pr-view-with-body; preserve its rollup-append grant." + } + ], + "unchecked": [], + "stamp_written": false +} diff --git a/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-4-import-policy-grants/report.md b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-4-import-policy-grants/report.md new file mode 100644 index 000000000..419c62fa3 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-4-import-policy-grants/report.md @@ -0,0 +1,37 @@ + + +Install method: `git-branch` (pinned snapshot). + +Scope: 3 configured security import skills, no override files. +All three skills' `requires_config:` entries resolve from the personal +layer or committed overrides. All their SKILL.md files are readable. +No anchor or required-configuration findings exist. + +Existing vetted-ops policy: +`.apache-magpie-overrides/tools/vetted-ops/config.toml` +The policy is valid TOML and has this `[callers]` section: + +```toml +[callers] +"security-issue-import-from-md" = ["issue-view"] +"security-issue-import-from-pr" = ["rollup-append"] +"security-issue-sync" = ["issue-view", "rollup-append"] +``` + +The following workflows are configured and in scope: + +- `security-issue-import` +- `security-issue-import-from-md` +- `security-issue-import-from-pr` + +Tracker backend: `github`. + +The framework's vetted-ops README provides this example: + +[callers] +"security-issue-import" = ["issue-list-with-bodies", "rollup-append"] +"security-issue-import-from-md" = ["rollup-append"] +"security-issue-import-from-pr" = ["pr-view-with-body", "rollup-append"] + +No policy changes have been confirmed during this run. diff --git a/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-5-declined-import-grant/expected.json b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-5-declined-import-grant/expected.json new file mode 100644 index 000000000..dbc075b62 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-5-declined-import-grant/expected.json @@ -0,0 +1,12 @@ +{ + "proposals": [ + { + "kind": "policy", + "skill": "security-issue-import-from-md", + "override_file": null, + "detail": "Caller security-issue-import-from-md lacks rollup-append. The operator declined the grant; preserve issue-view and leave the policy finding unresolved." + } + ], + "unchecked": [], + "stamp_written": false +} diff --git a/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-5-declined-import-grant/report.md b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-5-declined-import-grant/report.md new file mode 100644 index 000000000..5007d4473 --- /dev/null +++ b/tools/skill-evals/evals/setup/step-reconcile/fixtures/case-5-declined-import-grant/report.md @@ -0,0 +1,25 @@ + + +Install method: `git-branch` (pinned snapshot). + +Scope: `security-issue-import-from-md` is configured. +Its `requires_config:` entries resolve, and its skill files are readable. +There are no override-anchor or missing-configuration findings. + +Tracker backend: `github`. + +Existing vetted-ops policy: +`.apache-magpie-overrides/tools/vetted-ops/config.toml` + +The policy is valid TOML. Its caller entry is: + "security-issue-import-from-md" = ["issue-view"] + +The vetted-ops README configuration example contains: + "security-issue-import-from-md" = ["rollup-append"] + +During confirmation, the operator declined the proposal +to add `rollup-append` to this caller. +The policy remains unchanged. + +The current reconciliation stamp contains no entry for this skill. diff --git a/tools/skill-evals/evals/setup/step-reconcile/fixtures/output-spec.md b/tools/skill-evals/evals/setup/step-reconcile/fixtures/output-spec.md index 8e6d7e623..6bfb5a1db 100644 --- a/tools/skill-evals/evals/setup/step-reconcile/fixtures/output-spec.md +++ b/tools/skill-evals/evals/setup/step-reconcile/fixtures/output-spec.md @@ -9,27 +9,32 @@ Return ONLY valid JSON with this structure: { "proposals": [ { - "kind": "reanchor" | "config", + "kind": "reanchor" | "config" | "policy", "skill": "", - "override_file": "", + "override_file": "", "detail": "" } ], - "unchecked": ["anchor-resolution" | "requires_config"], + "unchecked": ["anchor-resolution" | "requires_config" | "policy-grants"], "stamp_written": true | false } ``` -- `proposals` — one entry per finding from the two checks. `kind: "reanchor"` +- `proposals` — one entry per finding from the reconciliation checks. `kind: "reanchor"` when an override's referenced anchor no longer resolves in the target skill's current `SKILL.md`; `kind: "config"` when a `requires_config` - entry does not resolve through the lookup chain. Empty when both checks - pass cleanly for everything in scope. + entry does not resolve through the lookup chain; `kind: "policy"` when + an applicable import caller or required operation is absent from an + existing vetted-ops policy. Use the caller name as `skill` and name + the exact missing operations in `detail`. Empty when all applicable + checks pass cleanly for everything in scope. - `unchecked` — which check(s) could not be completed for at least one skill in scope, because that skill's `SKILL.md` was not readable this session (a marketplace install's plugin cache, denied under the sandbox). `requires_config` resolution needs only files already in the - repository, so it is never unchecked. Empty when nothing was skipped. + repository, so it is never unchecked. Use `policy-grants` when an + existing vetted-ops policy cannot be read or parsed. A missing + policy is not applicable, not unchecked. Empty when nothing was skipped. - `stamp_written` is `true` only when `proposals` and `unchecked` are both empty — nothing changed, so the stamp is written immediately with no confirmation needed. Any finding, or anything left unchecked, means diff --git a/tools/skill-evals/evals/setup/step-reconcile/fixtures/user-prompt-template.md b/tools/skill-evals/evals/setup/step-reconcile/fixtures/user-prompt-template.md index 57c8c3c8c..0b237e5e4 100644 --- a/tools/skill-evals/evals/setup/step-reconcile/fixtures/user-prompt-template.md +++ b/tools/skill-evals/evals/setup/step-reconcile/fixtures/user-prompt-template.md @@ -5,4 +5,4 @@ {report} -Apply the sweep's two checks and return JSON only. +Apply the reconciliation sweep checks, including applicable vetted-ops import caller grants, and return JSON only. diff --git a/tools/vetted-ops/README.md b/tools/vetted-ops/README.md index 830d1a510..1697d53b5 100644 --- a/tools/vetted-ops/README.md +++ b/tools/vetted-ops/README.md @@ -220,6 +220,36 @@ board_status_field_id = "PVTSSF_…" # its Status field id "security-issue-import" = ["issue-list-with-bodies", "rollup-append"] "security-issue-import-from-md" = ["rollup-append"] "security-issue-import-from-pr" = ["pr-view-with-body", "rollup-append"] + +# Non-GitHub tracker additions, per import caller. +# Add these to the corresponding existing caller array; preserve base grants. +# +# security-issue-import: +# jira: tracker-doctor, issue-search, issue-list, issue-list-label, +# issue-view, issue-comments, issue-comment, issue-create, +# issue-body, issue-attach +# local-files: same as jira, plus tracker-pending, tracker-publish +# +# security-issue-import-from-md: +# jira: tracker-doctor, issue-search, issue-view, issue-create, +# issue-body, issue-attach +# local-files: same as jira, plus tracker-pending, tracker-publish +# +# security-issue-import-from-pr: +# jira: tracker-doctor, issue-search, issue-view, issue-create, +# issue-set-milestone, issue-body, issue-attach +# local-files: same as jira, plus tracker-pending, tracker-publish +# +# These are conditional additions, not universal grants. +# issue-list-with-bodies works only with GitHub. The email importer must +# stop on Jira/local-files when this deduplication step is unsupported; +# never bypass the refusal or assume that unmatched reports are new. +# +# Other security callers (sync, fix, CVE allocation, deduplication, +# triage and invalidation) also require their own backend grants. +# Setup derives those from each workflow and the security-tracker contract, +# preserving existing grants and requesting explicit confirmation. +# Do not duplicate TOML keys or grant every operation to every caller. "security-issue-invalidate" = ["rollup-append", "rollup-amend-latest"] "security-issue-triage" = ["issue-view", "issue-comments", "osv-get-vuln", "osv-query-package"]