diff --git a/.fullsend/harness/triage-security.yaml b/.fullsend/harness/triage-security.yaml index 620f90b7e..d01080f54 100644 --- a/.fullsend/harness/triage-security.yaml +++ b/.fullsend/harness/triage-security.yaml @@ -20,7 +20,7 @@ # The credential mount stays required so a missing credential fails up front on # every pinned CLI. This is the sole sandbox credential (inference only); Jira # credentials stay runner-only in env.runner. -base: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/harness/triage-security.yaml#sha256=c9750b8feb846242420db12d1a20f93be08dc1221aac2ba722aac495b6a05b56 +base: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/harness/triage-security.yaml#sha256=10d7b3e478cedd2e2e9f2d7b5e249d0fa4accc6323989410106fae581980f4cb host_files: - src: ${GOOGLE_APPLICATION_CREDENTIALS} diff --git a/.fullsend/harness/verify-pr.yaml b/.fullsend/harness/verify-pr.yaml index 79e8d66c2..1b1ec7f44 100644 --- a/.fullsend/harness/verify-pr.yaml +++ b/.fullsend/harness/verify-pr.yaml @@ -48,7 +48,7 @@ # (base URLs are validated against that allowlist, not inherited). # Re-pin after any base edit: push harness/verify-pr.yaml, set the SHA to the new # commit + the sha256 to `shasum -a 256 harness/verify-pr.yaml`, `fullsend lock`. -base: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/harness/verify-pr.yaml#sha256=c57aadf6b26e8840b49edd8ec5e047a60e0b1f79e7f746eed000979551cf8854 +base: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/harness/verify-pr.yaml#sha256=fb171c9b4a13df8771d19de1809c9386e4638590365d65ee95a85ce191cc2572 # CEL trigger (ADR 0061) — MUST be declared here, on the composing child, not # only on the base. `fullsend dispatch` registers this child (config.base.yaml diff --git a/.fullsend/lock.yaml b/.fullsend/lock.yaml index c1d54aca9..d2df973dd 100644 --- a/.fullsend/lock.yaml +++ b/.fullsend/lock.yaml @@ -4,17 +4,17 @@ generated_at: 2026-09-04T13:39:01.038924Z harnesses: triage-security: source: harness/triage-security.yaml - sha256: 8c94f3a9f2359481144ce198cda177d63f8693bcf3df06e44ac0cf58ec460181 - resolved_at: 2026-10-30T14:47:26.82412Z + sha256: 4f80c81e04d816c82b7e565522f3cbb74c4de6949f64f7d30dddb32f78f028f0 + resolved_at: 2026-11-02T15:46:27.40111Z dependencies: - field: base - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/harness/triage-security.yaml - sha256: c9750b8feb846242420db12d1a20f93be08dc1221aac2ba722aac495b6a05b56 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/harness/triage-security.yaml + sha256: 10d7b3e478cedd2e2e9f2d7b5e249d0fa4accc6323989410106fae581980f4cb type: file - fetched_at: 2026-10-30T14:47:19.01817Z + fetched_at: 2026-11-02T15:46:19.512017Z - field: pre_script - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/scripts/pre-triage-security.sh - sha256: 1d2419c69738b091b2383b4ef779dc224f0e605d975959f75e7471b9f1333c85 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/scripts/pre-triage-security.sh + sha256: 6e394808239e8fdbd40aaa3d86df419845a8eb7d7ef062da362b224a4d9cf1cb type: directory files: - path: execute-actions.py @@ -42,11 +42,13 @@ harnesses: - path: test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: test_native_fullsend_execution.py @@ -56,7 +58,7 @@ harnesses: - path: test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: test_verify_pr_eval_observability.py @@ -65,10 +67,10 @@ harnesses: sha256: 7ce1b74a91c6a7fbef5b6070dcc65ebaba018ecf9e8595e645cce503fb775e78 - path: validate-output-schema.sh sha256: 56b964145da0b62f438dbc1885780d86e2d589c5340adb93c00e3c8777979cbe - fetched_at: 2026-10-30T14:47:22.467253Z + fetched_at: 2026-11-02T15:46:23.226402Z - field: post_script - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/scripts/post-triage-security.sh - sha256: 1d2419c69738b091b2383b4ef779dc224f0e605d975959f75e7471b9f1333c85 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/scripts/post-triage-security.sh + sha256: 6e394808239e8fdbd40aaa3d86df419845a8eb7d7ef062da362b224a4d9cf1cb type: directory files: - path: execute-actions.py @@ -96,11 +98,13 @@ harnesses: - path: test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: test_native_fullsend_execution.py @@ -110,7 +114,7 @@ harnesses: - path: test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: test_verify_pr_eval_observability.py @@ -119,10 +123,10 @@ harnesses: sha256: 7ce1b74a91c6a7fbef5b6070dcc65ebaba018ecf9e8595e645cce503fb775e78 - path: validate-output-schema.sh sha256: 56b964145da0b62f438dbc1885780d86e2d589c5340adb93c00e3c8777979cbe - fetched_at: 2026-10-30T14:47:22.460604Z + fetched_at: 2026-11-02T15:46:23.219999Z - field: validation_loop.script - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/scripts/validate-output-schema.sh - sha256: 1d2419c69738b091b2383b4ef779dc224f0e605d975959f75e7471b9f1333c85 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/scripts/validate-output-schema.sh + sha256: 6e394808239e8fdbd40aaa3d86df419845a8eb7d7ef062da362b224a4d9cf1cb type: directory files: - path: execute-actions.py @@ -150,11 +154,13 @@ harnesses: - path: test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: test_native_fullsend_execution.py @@ -164,7 +170,7 @@ harnesses: - path: test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: test_verify_pr_eval_observability.py @@ -173,38 +179,38 @@ harnesses: sha256: 7ce1b74a91c6a7fbef5b6070dcc65ebaba018ecf9e8595e645cce503fb775e78 - path: validate-output-schema.sh sha256: 56b964145da0b62f438dbc1885780d86e2d589c5340adb93c00e3c8777979cbe - fetched_at: 2026-10-30T14:47:22.460604Z + fetched_at: 2026-11-02T15:46:23.219999Z - field: validation_loop.schema - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/schemas/triage-security-result.schema.json + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/schemas/triage-security-result.schema.json sha256: 572c36a599661942c60ca610b7d2b60acd1864fbe48f6ce23b854d68a116c4fe type: resource - fetched_at: 2026-10-30T14:47:22.740122Z + fetched_at: 2026-11-02T15:46:23.510201Z - field: agent - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/agents/triage-security.md - sha256: 3e0680274ff99326fa219b45ccbea6ea1ed1a7eba3f78056acf9e47b29c7c9fd + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/agents/triage-security.md + sha256: 5c893d21aa968de3c044a137b0adaab2bcd7652648e685ce2e902973a7d28f8d type: resource - fetched_at: 2026-10-30T14:47:22.993446Z + fetched_at: 2026-11-02T15:46:23.761252Z - field: policy - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/policies/triage-security.yaml + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/policies/triage-security.yaml sha256: 312f947a7c0eceab078111406c8288853c760eb0304b080262da322066046b91 type: resource - fetched_at: 2026-10-30T14:47:23.358934Z + fetched_at: 2026-11-02T15:46:24.05887Z - field: host_files[0].src - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/env/gcp-vertex.env + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/env/gcp-vertex.env sha256: 10b2ba695b1d4e65e0964a233b75d6a06853f1eaefc260c56b9eedd989ae7d41 type: resource - fetched_at: 2026-10-30T14:47:23.630992Z + fetched_at: 2026-11-02T15:46:24.367677Z - field: plugins[0] - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/ - sha256: 60d671654221f586be5ed1d862deff360d46cca25e694a0cc6899a753f715329 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/ + sha256: ccb32ccb7f4f0c84cfb8ff84d72e03fdb5f5c4eed697bc51e8a06171ee3dd9b4 type: directory files: - path: .claude-plugin/plugin.json sha256: 992cc8f49005d9144a7952137685acd7627c40e1055de8376fb9b18e9c082e9b - path: agents/triage-security.md - sha256: 3e0680274ff99326fa219b45ccbea6ea1ed1a7eba3f78056acf9e47b29c7c9fd + sha256: 5c893d21aa968de3c044a137b0adaab2bcd7652648e685ce2e902973a7d28f8d - path: agents/verify-pr.md - sha256: 229fef270b4b195f91cc46ef3ba50cbf7701f49ee546291882cff893d53844b3 + sha256: 47955e33cf1d04649ce37e553d6e1d21e6ab49851a81419086f5cb34cb12609c - path: env/gcp-vertex.env sha256: 10b2ba695b1d4e65e0964a233b75d6a06853f1eaefc260c56b9eedd989ae7d41 - path: plugin.json @@ -244,11 +250,13 @@ harnesses: - path: scripts/test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: scripts/test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: scripts/test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: scripts/test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: scripts/test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: scripts/test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: scripts/test_native_fullsend_execution.py @@ -258,7 +266,7 @@ harnesses: - path: scripts/test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: scripts/test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: scripts/test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: scripts/test_verify_pr_eval_observability.py @@ -312,7 +320,7 @@ harnesses: - path: skills/triage-bug/SKILL.md sha256: e3c5f28d1c7bafdcf706d8be89110cef28217ef878a754fb89c571c5731242d4 - path: skills/triage-security/SKILL.md - sha256: 022a1c1d7127b90738be27fe7f748bfc82615f20e46096637390b1ec7835ad43 + sha256: fceef608776c64d22a19e12aa0e12692d635e1c0d56cf561a4a40fd746a20e96 - path: skills/triage-security/jira-triage-operations.md sha256: 509678de6e9eb0ce327fb2c4c010c816711daa748d2192467350c5e031375586 - path: skills/triage-security/remediation-templates.md @@ -333,20 +341,20 @@ harnesses: sha256: 128877cf7e156666dbb403805a0112059cc8cc3255e148c6794da29d48b8f5cf - path: skills/verify-pr/style-conventions.md sha256: 5ab920c596615c0d23354cb0e90fe5ebf5f08635ffa3602e8e2d2abd74028a19 - fetched_at: 2026-10-30T14:47:26.818867Z + fetched_at: 2026-11-02T15:46:27.397777Z verify-pr: source: harness/verify-pr.yaml - sha256: 08db3c664b1cafe08f4a94a70da8cf02527ffca1007432f2fa13ea054e811662 - resolved_at: 2026-10-30T14:47:28.394117Z + sha256: fc733b180d992d2ad47787bd994896ac76dbcc5bef5adf965d89b35d2e41731a + resolved_at: 2026-11-02T15:46:28.584503Z dependencies: - field: base - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/harness/verify-pr.yaml - sha256: c57aadf6b26e8840b49edd8ec5e047a60e0b1f79e7f746eed000979551cf8854 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/harness/verify-pr.yaml + sha256: fb171c9b4a13df8771d19de1809c9386e4638590365d65ee95a85ce191cc2572 type: file - fetched_at: 2026-10-30T14:47:26.934795Z + fetched_at: 2026-11-02T15:46:27.489396Z - field: pre_script - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/scripts/pre-verify-pr.sh - sha256: 1d2419c69738b091b2383b4ef779dc224f0e605d975959f75e7471b9f1333c85 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/scripts/pre-verify-pr.sh + sha256: 6e394808239e8fdbd40aaa3d86df419845a8eb7d7ef062da362b224a4d9cf1cb type: directory files: - path: execute-actions.py @@ -374,11 +382,13 @@ harnesses: - path: test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: test_native_fullsend_execution.py @@ -388,7 +398,7 @@ harnesses: - path: test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: test_verify_pr_eval_observability.py @@ -397,10 +407,10 @@ harnesses: sha256: 7ce1b74a91c6a7fbef5b6070dcc65ebaba018ecf9e8595e645cce503fb775e78 - path: validate-output-schema.sh sha256: 56b964145da0b62f438dbc1885780d86e2d589c5340adb93c00e3c8777979cbe - fetched_at: 2026-10-30T14:47:22.460604Z + fetched_at: 2026-11-02T15:46:23.219999Z - field: post_script - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/scripts/post-verify-pr.sh - sha256: 1d2419c69738b091b2383b4ef779dc224f0e605d975959f75e7471b9f1333c85 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/scripts/post-verify-pr.sh + sha256: 6e394808239e8fdbd40aaa3d86df419845a8eb7d7ef062da362b224a4d9cf1cb type: directory files: - path: execute-actions.py @@ -428,11 +438,13 @@ harnesses: - path: test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: test_native_fullsend_execution.py @@ -442,7 +454,7 @@ harnesses: - path: test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: test_verify_pr_eval_observability.py @@ -451,10 +463,10 @@ harnesses: sha256: 7ce1b74a91c6a7fbef5b6070dcc65ebaba018ecf9e8595e645cce503fb775e78 - path: validate-output-schema.sh sha256: 56b964145da0b62f438dbc1885780d86e2d589c5340adb93c00e3c8777979cbe - fetched_at: 2026-10-30T14:47:22.460604Z + fetched_at: 2026-11-02T15:46:23.219999Z - field: validation_loop.script - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/scripts/validate-output-schema.sh - sha256: 1d2419c69738b091b2383b4ef779dc224f0e605d975959f75e7471b9f1333c85 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/scripts/validate-output-schema.sh + sha256: 6e394808239e8fdbd40aaa3d86df419845a8eb7d7ef062da362b224a4d9cf1cb type: directory files: - path: execute-actions.py @@ -482,11 +494,13 @@ harnesses: - path: test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: test_native_fullsend_execution.py @@ -496,7 +510,7 @@ harnesses: - path: test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: test_verify_pr_eval_observability.py @@ -505,38 +519,38 @@ harnesses: sha256: 7ce1b74a91c6a7fbef5b6070dcc65ebaba018ecf9e8595e645cce503fb775e78 - path: validate-output-schema.sh sha256: 56b964145da0b62f438dbc1885780d86e2d589c5340adb93c00e3c8777979cbe - fetched_at: 2026-10-30T14:47:22.460604Z + fetched_at: 2026-11-02T15:46:23.219999Z - field: validation_loop.schema - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/schemas/verify-pr-result.schema.json + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/schemas/verify-pr-result.schema.json sha256: 62c3e5ce0c47e73823b5cab9a577be7b721d8fb31389bc8602dc82ee05526ad6 type: resource - fetched_at: 2026-10-30T14:47:27.571538Z + fetched_at: 2026-11-02T15:46:28.051186Z - field: agent - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/agents/verify-pr.md - sha256: 229fef270b4b195f91cc46ef3ba50cbf7701f49ee546291882cff893d53844b3 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/agents/verify-pr.md + sha256: 47955e33cf1d04649ce37e553d6e1d21e6ab49851a81419086f5cb34cb12609c type: resource - fetched_at: 2026-10-30T14:47:27.972882Z + fetched_at: 2026-11-02T15:46:28.299776Z - field: policy - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/policies/verify-pr.yaml + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/policies/verify-pr.yaml sha256: 62d1374be253b0b225921bd8e22e679781076ecdb3a9ea22d5e6a4805753c5a7 type: resource - fetched_at: 2026-10-30T14:47:28.36925Z + fetched_at: 2026-11-02T15:46:28.574226Z - field: host_files[0].src - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/env/gcp-vertex.env + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/env/gcp-vertex.env sha256: 10b2ba695b1d4e65e0964a233b75d6a06853f1eaefc260c56b9eedd989ae7d41 type: resource - fetched_at: 2026-10-30T14:47:23.616706Z + fetched_at: 2026-11-02T15:46:24.357822Z - field: plugins[0] - url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/942e605ce7564b0b64f75efe055813ed6ff6ca94/plugins/sdlc-workflow/ - sha256: 60d671654221f586be5ed1d862deff360d46cca25e694a0cc6899a753f715329 + url: https://raw.githubusercontent.com/RHEcosystemAppEng/sdlc-plugins/dc6c74aef84d2297f6578ef864b3aa7685b6c344/plugins/sdlc-workflow/ + sha256: ccb32ccb7f4f0c84cfb8ff84d72e03fdb5f5c4eed697bc51e8a06171ee3dd9b4 type: directory files: - path: .claude-plugin/plugin.json sha256: 992cc8f49005d9144a7952137685acd7627c40e1055de8376fb9b18e9c082e9b - path: agents/triage-security.md - sha256: 3e0680274ff99326fa219b45ccbea6ea1ed1a7eba3f78056acf9e47b29c7c9fd + sha256: 5c893d21aa968de3c044a137b0adaab2bcd7652648e685ce2e902973a7d28f8d - path: agents/verify-pr.md - sha256: 229fef270b4b195f91cc46ef3ba50cbf7701f49ee546291882cff893d53844b3 + sha256: 47955e33cf1d04649ce37e553d6e1d21e6ab49851a81419086f5cb34cb12609c - path: env/gcp-vertex.env sha256: 10b2ba695b1d4e65e0964a233b75d6a06853f1eaefc260c56b9eedd989ae7d41 - path: plugin.json @@ -576,11 +590,13 @@ harnesses: - path: scripts/test_execute_triage_security_actions.py sha256: d1589b3e8816f80a5bf1d801ef38dafc5f5d69a42cd48d93de5a15946fc0f976 - path: scripts/test_fullsend_gate_eval.py - sha256: fc0041d4de46734d42e969226c3bf7fd82b04c97dc4877dbebbb669b15461086 + sha256: 757271629d09a12ca74a71b4c94bc78e4572cfb3df1338b4cfd55687fd5ea7cb - path: scripts/test_jira_client.py sha256: 8b417d72c84d7ec6823495185cabaddfdf7d663fd1f3e2c6c8fd173752942115 - path: scripts/test_jira_client_cli.py sha256: bbf71c1e2ec4214459cb2f769f95f654de03f5818b2912be01dd88dae83a7cc2 + - path: scripts/test_model_pins.py + sha256: 7bd3c9046f090600526551f43935103a34ad1948be1dc1c141559cf3351ad8ae - path: scripts/test_native_fullsend_eval_ci.py sha256: b61cf7f8cad916f8832ff89ce573e84d55b972aa15aab987ecde7ea3884c4c9c - path: scripts/test_native_fullsend_execution.py @@ -590,7 +606,7 @@ harnesses: - path: scripts/test_pre_verify_pr.py sha256: 6563a15187877b388673edcc1e520626b337231b09af5adb563717367f7fb8d6 - path: scripts/test_triage_security_fullsend.py - sha256: fb825e85ac713832c67234e04244764e3234a19d96feafca1defec20f73cfd7f + sha256: 6d7a3648bc1647cb70d30d3c0dc7a504337451cac71a6ef6bfc14a350c409b3f - path: scripts/test_triage_security_staleness_evals.py sha256: 94bbc65f0c6b7d10dcae0af91dab3728f659722d4058aa5bc1ac4e8eadbc5547 - path: scripts/test_verify_pr_eval_observability.py @@ -644,7 +660,7 @@ harnesses: - path: skills/triage-bug/SKILL.md sha256: e3c5f28d1c7bafdcf706d8be89110cef28217ef878a754fb89c571c5731242d4 - path: skills/triage-security/SKILL.md - sha256: 022a1c1d7127b90738be27fe7f748bfc82615f20e46096637390b1ec7835ad43 + sha256: fceef608776c64d22a19e12aa0e12692d635e1c0d56cf561a4a40fd746a20e96 - path: skills/triage-security/jira-triage-operations.md sha256: 509678de6e9eb0ce327fb2c4c010c816711daa748d2192467350c5e031375586 - path: skills/triage-security/remediation-templates.md @@ -665,4 +681,4 @@ harnesses: sha256: 128877cf7e156666dbb403805a0112059cc8cc3255e148c6794da29d48b8f5cf - path: skills/verify-pr/style-conventions.md sha256: 5ab920c596615c0d23354cb0e90fe5ebf5f08635ffa3602e8e2d2abd74028a19 - fetched_at: 2026-10-30T14:47:26.803033Z + fetched_at: 2026-11-02T15:46:27.389123Z diff --git a/docs/testing/fullsend-gate-evals.md b/docs/testing/fullsend-gate-evals.md index 052a545a7..0357e6b0b 100644 --- a/docs/testing/fullsend-gate-evals.md +++ b/docs/testing/fullsend-gate-evals.md @@ -121,11 +121,22 @@ After host services and those inputs are ready, the operator can run: python3.12 evals/fullsend/run.py run \ --cache /tmp/tc-6677-eval-deps \ --output /tmp/tc-6677-native-evals \ - --model claude-opus-4-8 \ + --model claude-opus-5-5 \ --judge-model claude-opus-4-6 \ - --effort high + --effort medium ``` +The native eval agent model and effort come from the `run.py` defaults +(`--model claude-opus-5-5`, `--effort medium`): CI +(`.github/scripts/run-native-fullsend-evals.sh`) passes only `--judge-model`, and +`run.py` writes both values into the resolved suite config and hands them to +`fullsend run` as `--model`/`--effort`, which override `harness.yaml`'s +`model:`/`effort:` and `eval.yaml`'s `models.skill`/`runner.effort` (kept equal +for readability; `test_model_pins.py` fails on drift). Because Eval PR Run is a +`workflow_run` that executes `run.py` from `main`, a PR changing these defaults +is not exercised by its own CI run — prove it with the local command above. See +`fullsend.md` "Managing the agent model". + This command **does perform paid inference**: four serial native Fullsend runs and 24 upstream Boolean LLM judgments. Select model IDs supported by your Vertex project/region. Model availability is not checked by preflight. The native diff --git a/evals/fullsend/run.py b/evals/fullsend/run.py index 233ac782f..55a6ca896 100644 --- a/evals/fullsend/run.py +++ b/evals/fullsend/run.py @@ -423,9 +423,9 @@ def main(): parser.add_argument("command", choices=["setup", "preflight", "run"]) parser.add_argument("--cache", type=Path, default=Path(tempfile.gettempdir()) / "tc-6677-eval-deps") parser.add_argument("--output", type=Path, default=Path(tempfile.gettempdir()) / "tc-6677-native-evals") - parser.add_argument("--model", default="claude-opus-4-8") + parser.add_argument("--model", default="claude-opus-5-5") parser.add_argument("--judge-model", default="claude-opus-4-6") - parser.add_argument("--effort", choices=["low", "medium", "high", "max"], default="high") + parser.add_argument("--effort", choices=["low", "medium", "high", "max"], default="medium") parser.add_argument("--plugin-root", type=Path, help="Sandbox-tested plugin; tooling always comes from this checkout") parser.add_argument("--report-dir", type=Path, help="Allowlisted CI result directory; never raw credentials/transcripts") args = parser.parse_args() diff --git a/evals/fullsend/triage-security/eval.yaml b/evals/fullsend/triage-security/eval.yaml index 1e4f9c62b..6e7dbb1b5 100644 --- a/evals/fullsend/triage-security/eval.yaml +++ b/evals/fullsend/triage-security/eval.yaml @@ -7,7 +7,7 @@ execution: parallelism: 1 runner: type: cli - effort: high + effort: medium command: - '{python}' - '{runner}' @@ -24,7 +24,7 @@ runner: - --effort - '{effort}' models: - skill: claude-opus-4-8 + skill: claude-opus-5-5 judge: claude-opus-4-6 dataset: path: cases diff --git a/evals/fullsend/triage-security/harness.yaml b/evals/fullsend/triage-security/harness.yaml index 8ced3fed5..761f50d2f 100644 --- a/evals/fullsend/triage-security/harness.yaml +++ b/evals/fullsend/triage-security/harness.yaml @@ -1,8 +1,8 @@ # SYNTHETIC TEST DATA — native test agent; production policy/provider/schema unchanged role: triage agent: evals/fullsend/triage-security/agent.md -model: claude-opus-4-8 -effort: high +model: claude-opus-5-5 +effort: medium image: ghcr.io/fullsend-ai/fullsend-code@sha256:3055d4f52136a7c89fb8aa1976b468a5f9dd929c6be55fd6cd958e222c6e18ca readonly_repo: true plugins: diff --git a/fullsend.md b/fullsend.md index 9a5519aa3..ff43f41f5 100644 --- a/fullsend.md +++ b/fullsend.md @@ -784,6 +784,111 @@ with the `base:` move that brings it the new harness `image:`. The rule applies The fullsend pins are independent of the harness `base:` re-pin (which tracks sdlc-plugins content) and of the plugin version below. +### Managing the agent model + +The verify-pr and triage-security agents, and the native eval agent, run on +**`claude-opus-5-5` at effort `medium`** (TC-7274). The model is pinned content +like everything else: changing it is a harness edit that goes through the +re-pin + re-lock loop. + +#### Where model and effort are set + +| Location | Sets | Consumed by | +|---|---|---| +| `harness/verify-pr.yaml`, `harness/triage-security.yaml` — `model:` / `effort:` | model + effort | every `fullsend run` (local + CI) through the pinned `base:` | +| `plugins/sdlc-workflow/agents/{verify-pr,triage-security}.md` — frontmatter `model:` | model only | Claude Code when nothing else names a model (e.g. interactive `claude --agent`) | +| `evals/fullsend/run.py` — `--model` / `--effort` defaults | model + effort | the native eval agent (CI passes only `--judge-model`) | +| `evals/fullsend/triage-security/harness.yaml`, `eval.yaml` (`models.skill`, `runner.effort`) | model + effort | overridden at runtime by `run.py`; kept equal for readability | + +The `.fullsend/harness/*.yaml` composing children set **no** model or effort — +they inherit both from the pinned base. + +**Which wins.** fullsend resolves the run model as `--model` flag > +`FULLSEND_MODEL` > config `agents:` entry > harness `model:`, and only when that +chain is empty falls back to the agent frontmatter `model:` +(`internal/runtime/model.go` `EffectiveModel` at the supported commit). The +Claude runtime passes the resolved value to Claude Code as `--model` and the +harness `effort:` as `--effort` (`internal/runtime/claude.go`), and the CLI flag +takes precedence over the agent definition's frontmatter. In production the +**harness value wins**; the frontmatter is a fallback for runs that name no +model. Both are pinned to the same ID anyway, and +`plugins/sdlc-workflow/scripts/test_model_pins.py` fails if any pin listed above +disagrees with the others, uses a floating alias, or if a composing child +overrides the base. `metrics.json` records the outcome per run: +`requested_model` (what fullsend asked for), `override_source` (`harness`, +`--model flag`, …) and `model` (what the provider reported). + +#### Why a concrete model ID, not an alias + +Claude Code aliases (`opus`, `sonnet`) resolve to whatever generation the CLI in +the image maps them to. Under CLI v0.43.0 `opus` resolved to `claude-opus-4-6`, +which the Vertex deployment had not enabled (run 34951983374: *"model +claude-opus-4-6 is not available on your vertex deployment"*). A concrete ID +makes the model change only when this repo changes it — the same reason the +frontmatter carries `claude-opus-5-5` instead of `opus`. + +#### Why an explicit effort + +Effort is set explicitly instead of inheriting the model's default so that a +change of the default effort (per model generation or per CLI release) cannot +silently alter run behaviour, cost, or latency. + +#### Changing the model + +1. **Enable the model on Vertex first.** The fullsend Vertex project must serve + the new ID in the configured region; an unknown or disabled model fails + inference (`not available on your vertex deployment` / HTTP 400/404), not + preflight — `run.py preflight` does not check model availability either. + **Check the sandbox image too:** the harness `image:` decides the Claude Code + version, and a model newer than that Claude Code fails every call with + `Claude Code does not support this model; version or newer is + required` (TC-7278 on the v0.37.0-era image). If the pinned image is too old, + move the CLI and `image:` together first through + [Updating fullsend (pin-and-bump)](#updating-fullsend-pin-and-bump) (TC-7300). +2. **Edit every pin** in the table above in one change, and update + `test_model_pins.py`'s target-model assertion. +3. **Commit and push to an upstream `RHEcosystemAppEng` branch** — a fork PR ref + fails the promisor fetch. +4. **Re-pin both** `.fullsend/harness/verify-pr.yaml` and + `.fullsend/harness/triage-security.yaml` (`base:` commit SHA + `#sha256=` of + the base file), then **re-lock**: + `fullsend lock --all --fullsend-dir .fullsend` (see + [Releasing an update](#releasing-an-update-this-repo)). Commit `.fullsend/` + in the same PR. +5. **Prove before merging** (below), then **merge with a merge commit**. + URL-registered adopters (`sdlc-fullsend`) pick the model up by moving their + own `base:` pin to the merge commit. + +#### Pre-merge proof + +CI cannot self-prove a model change: verify-pr dispatches from `main` +(`pull_request_target`) and Eval PR Run is a `workflow_run` that executes +`run.py` from `main`. Prove each consumer explicitly: + +- **verify-pr** — the temporary `pull_request` workflow copy from + [Updating fullsend](#updating-fullsend-pin-and-bump) step 5, on a same-repo + branch carrying the re-pinned `.fullsend/`. The run must complete, validate + against the result schema, and post its report. +- **triage-security** — a local `fullsend run triage-security` of the + production harness from the re-pinned branch with `--no-post-script` + (report-only), validated against the result schema. +- **Native eval agent** — the local paid run in + `docs/testing/fullsend-gate-evals.md` with the new defaults; all cases must + complete. + +For every run, confirm in `metrics.json` (or the run log) that the provider +reported the new model. A run that does not reach it — model not enabled, HTTP +400, an alias resolved elsewhere, or a different `override_source` than +expected — is a **failed** proof, not a pass. + +#### Forced `tool_choice` and Opus 5.5 + +`claude-opus-5-5` rejects a forced `tool_choice` with HTTP 400 (a model +restriction, not a Vertex policy; verified 2026-10-29). The agents are +unaffected — Claude Code does not force tool use. LLM judges that force a tool +call (the agent-eval-harness judges used by the native eval) therefore cannot run +on Opus 5.5; the judge model is managed separately (`--judge-model`, TC-7279). + ### Keeping the two plugin-version files in sync The plugin/skill version is stored in **two** files that must always match: diff --git a/harness/triage-security.yaml b/harness/triage-security.yaml index b49103004..0b62e6c9e 100644 --- a/harness/triage-security.yaml +++ b/harness/triage-security.yaml @@ -22,8 +22,8 @@ role: triage agent: plugins/sdlc-workflow/agents/triage-security.md -model: claude-opus-4-8 -effort: high +model: claude-opus-5-5 +effort: medium image: ghcr.io/fullsend-ai/fullsend-code@sha256:3055d4f52136a7c89fb8aa1976b468a5f9dd929c6be55fd6cd958e222c6e18ca readonly_repo: true diff --git a/harness/verify-pr.yaml b/harness/verify-pr.yaml index d7da8dc0f..58bd12d02 100644 --- a/harness/verify-pr.yaml +++ b/harness/verify-pr.yaml @@ -62,13 +62,14 @@ agent: plugins/sdlc-workflow/agents/verify-pr.md # Pinned to a concrete generation, not the floating `opus` alias: under fullsend # CLI v0.43.0 that alias resolves to claude-opus-4-6, which the Red Hat Vertex # deployment has not enabled (run 34951983374: "model claude-opus-4-6 is not -# available on your vertex deployment"). opus 4.8 is the fleet's target model. -model: claude-opus-4-8 -# Reasoning effort pinned to the fleet's implicit default: Claude Code runs at -# high effort on effort-capable models, and the fullsend pi runtime passes -# --thinking high when effort is unset. Revisit if `model:` moves to a -# generation with a different default effort. -effort: high +# available on your vertex deployment"). opus 5.5 is the fleet's target model +# (TC-7274). This value reaches Claude Code as --model and wins over the agent +# frontmatter `model:` (fullsend.md "Managing the agent model"). +model: claude-opus-5-5 +# Reasoning effort set explicitly rather than inherited from the model's +# default, so a change of Opus 5.5's default effort cannot alter runs; passed +# to Claude Code as --effort. +effort: medium image: ghcr.io/fullsend-ai/fullsend-code@sha256:3055d4f52136a7c89fb8aa1976b468a5f9dd929c6be55fd6cd958e222c6e18ca readonly_repo: true diff --git a/plugins/sdlc-workflow/agents/triage-security.md b/plugins/sdlc-workflow/agents/triage-security.md index c517883ae..b9bbad378 100644 --- a/plugins/sdlc-workflow/agents/triage-security.md +++ b/plugins/sdlc-workflow/agents/triage-security.md @@ -3,7 +3,7 @@ name: triage-security description: >- Run triage-security from trusted runner evidence and emit a validated report or Jira action plan without direct external access. -model: opus +model: claude-opus-5-5 --- # Triage Security Agent diff --git a/plugins/sdlc-workflow/agents/verify-pr.md b/plugins/sdlc-workflow/agents/verify-pr.md index bfcb47570..589751fda 100644 --- a/plugins/sdlc-workflow/agents/verify-pr.md +++ b/plugins/sdlc-workflow/agents/verify-pr.md @@ -3,7 +3,7 @@ name: verify-pr description: >- Verify a PR against its Jira task acceptance criteria using the sdlc-workflow verify-pr skill inside an OpenShell sandbox. -model: opus +model: claude-opus-5-5 --- # Verify PR Agent diff --git a/plugins/sdlc-workflow/scripts/test_fullsend_gate_eval.py b/plugins/sdlc-workflow/scripts/test_fullsend_gate_eval.py index 8a541988b..b2dffa5ef 100644 --- a/plugins/sdlc-workflow/scripts/test_fullsend_gate_eval.py +++ b/plugins/sdlc-workflow/scripts/test_fullsend_gate_eval.py @@ -21,7 +21,7 @@ SUITE = ROOT / "evals/fullsend/triage-security" FIXTURES = ROOT / "evals/triage-security/files" # Pin of the triage-security Step 0.6/0.7 gate commands that valid_case_tools diagnostics match against. -GATE_COMMANDS_SHA256 = "5ba54375e463039e4a969e88a5c40af48e2a607b41b6c411b68b27a01f8711b1" +GATE_COMMANDS_SHA256 = "330702b7cd14df099e7bdc7e9da5dac5b6e182854ad3dfe55273d8ad21bf75c8" def load_script(path): diff --git a/plugins/sdlc-workflow/scripts/test_model_pins.py b/plugins/sdlc-workflow/scripts/test_model_pins.py new file mode 100644 index 000000000..253e715a8 --- /dev/null +++ b/plugins/sdlc-workflow/scripts/test_model_pins.py @@ -0,0 +1,182 @@ +"""Agent model-pin consistency: every production/eval agent pin must agree (TC-7278).""" + +import ast +import shutil +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[3] + +# Claude Code aliases float to whatever generation the CLI maps them to, so a pin +# must name a concrete model ID (fullsend.md "Managing the agent model"). +FLOATING_ALIASES = {"opus", "sonnet", "haiku", "inherit", "default", "opusplan"} + +# Every file that selects the agent model and/or effort, relative to the repo root. +PIN_FILES = [ + "harness/verify-pr.yaml", + "harness/triage-security.yaml", + "plugins/sdlc-workflow/agents/verify-pr.md", + "plugins/sdlc-workflow/agents/triage-security.md", + "evals/fullsend/run.py", + "evals/fullsend/triage-security/harness.yaml", + "evals/fullsend/triage-security/eval.yaml", + ".fullsend/harness/verify-pr.yaml", + ".fullsend/harness/triage-security.yaml", +] + + +def frontmatter(path): + """Parse the YAML frontmatter of an agent definition, failing on a missing block.""" + text = path.read_text() + if not text.startswith("---\n"): + raise ValueError(f"{path}: no frontmatter") + return yaml.safe_load(text.split("---\n", 2)[1]) + + +def argparse_defaults(path): + """Return the literal `default=` of each `add_argument` flag in a script, without importing it.""" + defaults = {} + for node in ast.walk(ast.parse(path.read_text())): + if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr == "add_argument" and node.args + and isinstance(node.args[0], ast.Constant)): + for keyword in node.keywords: + if keyword.arg == "default" and isinstance(keyword.value, ast.Constant): + defaults[node.args[0].value] = keyword.value.value + return defaults + + +def collect_pins(root): + """Map every agent pin location to its (model, effort) value; effort is None where not settable.""" + def harness(rel): + data = yaml.safe_load((root / rel).read_text()) + return data.get("model"), data.get("effort") + + run_defaults = argparse_defaults(root / "evals/fullsend/run.py") + suite = yaml.safe_load((root / "evals/fullsend/triage-security/eval.yaml").read_text()) + return { + "harness/verify-pr.yaml": harness("harness/verify-pr.yaml"), + "harness/triage-security.yaml": harness("harness/triage-security.yaml"), + "agents/verify-pr.md frontmatter": ( + frontmatter(root / "plugins/sdlc-workflow/agents/verify-pr.md").get("model"), None), + "agents/triage-security.md frontmatter": ( + frontmatter(root / "plugins/sdlc-workflow/agents/triage-security.md").get("model"), None), + "run.py defaults": (run_defaults.get("--model"), run_defaults.get("--effort")), + "eval harness.yaml": harness("evals/fullsend/triage-security/harness.yaml"), + "eval.yaml models.skill/runner.effort": ( + (suite.get("models") or {}).get("skill"), (suite.get("runner") or {}).get("effort")), + } + + +def pin_problems(root): + """List every disagreement, missing pin, alias pin or composing-child override; empty means consistent.""" + pins = collect_pins(root) + problems = [] + for location, (model, _effort) in pins.items(): + if not model: + problems.append(f"{location}: no model pin") + elif model in FLOATING_ALIASES: + problems.append(f"{location}: floating alias {model!r}, pin a concrete model ID") + models = {location: model for location, (model, _) in pins.items()} + efforts = {location: effort for location, (_, effort) in pins.items() + if not location.endswith("frontmatter")} + problems.extend(f"{location}: no effort pin" for location, effort in efforts.items() if effort is None) + for kind, values in (("model", models), ("effort", efforts)): + if len(set(values.values())) > 1: + problems.append(f"{kind} pins disagree: {values}") + # The .fullsend/ composing children must inherit model/effort from the pinned + # base; an override there would silently diverge CI from the base harness. + for rel in (".fullsend/harness/verify-pr.yaml", ".fullsend/harness/triage-security.yaml"): + child = yaml.safe_load((root / rel).read_text()) + problems.extend(f"{rel}: overrides {key!r}" for key in ("model", "effort") if key in child) + return problems + + +def repo_copy(tmp_path): + """Copy only the pin files into a scratch tree so a test can mutate one safely.""" + for rel in PIN_FILES: + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / rel, tmp_path / rel) + return tmp_path + + +def test_repository_model_pins_agree(): + """Every production and eval agent pin in the repository names the same model and effort.""" + assert pin_problems(ROOT) == [] + + +def test_repository_pins_select_target_model_and_effort(): + """The agreed pin is the TC-7274 target: claude-opus-5-5 at explicit medium effort.""" + pins = collect_pins(ROOT) + assert {model for model, _ in pins.values()} == {"claude-opus-5-5"} + assert {effort for _, effort in pins.values() if effort is not None} == {"medium"} + + +@pytest.mark.parametrize("rel, old, new", [ + ("harness/verify-pr.yaml", "model: claude-opus-5-5", "model: claude-opus-4-8"), + ("harness/verify-pr.yaml", "effort: medium", "effort: high"), + ("harness/triage-security.yaml", "model: claude-opus-5-5", "model: claude-opus-4-8"), + ("harness/triage-security.yaml", "effort: medium", "effort: high"), + ("plugins/sdlc-workflow/agents/verify-pr.md", "model: claude-opus-5-5", "model: claude-opus-4-8"), + ("plugins/sdlc-workflow/agents/triage-security.md", "model: claude-opus-5-5", "model: claude-opus-4-8"), + ("evals/fullsend/run.py", 'default="claude-opus-5-5"', 'default="claude-opus-4-8"'), + ("evals/fullsend/run.py", 'default="medium"', 'default="high"'), + ("evals/fullsend/triage-security/harness.yaml", "model: claude-opus-5-5", "model: claude-opus-4-8"), + ("evals/fullsend/triage-security/harness.yaml", "effort: medium", "effort: high"), + ("evals/fullsend/triage-security/eval.yaml", "skill: claude-opus-5-5", "skill: claude-opus-4-8"), + ("evals/fullsend/triage-security/eval.yaml", "effort: medium", "effort: high"), +]) +def test_single_drifted_pin_fails(tmp_path, rel, old, new): + """Changing any one pin away from the others is reported as a disagreement.""" + # Given a copy of the pin files where exactly one pin drifts + root = repo_copy(tmp_path) + text = (root / rel).read_text() + assert text.count(old) == 1 + (root / rel).write_text(text.replace(old, new)) + # When the pins are checked + problems = pin_problems(root) + # Then the drift is reported as a disagreement for the changed dimension + kind = "effort" if "medium" in old else "model" + assert any(p.startswith(f"{kind} pins disagree") for p in problems), problems + + +@pytest.mark.parametrize("rel", [ + "plugins/sdlc-workflow/agents/verify-pr.md", + "plugins/sdlc-workflow/agents/triage-security.md", +]) +def test_frontmatter_alias_fails(tmp_path, rel): + """An agent frontmatter reverting to the floating `opus` alias is rejected.""" + # Given an agent definition pinned to the alias instead of a model ID + root = repo_copy(tmp_path) + (root / rel).write_text((root / rel).read_text().replace("model: claude-opus-5-5", "model: opus")) + # When the pins are checked + problems = pin_problems(root) + # Then both the alias and the resulting disagreement are reported + assert any("floating alias 'opus'" in p for p in problems), problems + assert any(p.startswith("model pins disagree") for p in problems), problems + + +@pytest.mark.parametrize("rel", [".fullsend/harness/verify-pr.yaml", ".fullsend/harness/triage-security.yaml"]) +def test_composing_child_override_fails(tmp_path, rel): + """A .fullsend/ composing child that overrides the base model is rejected.""" + # Given a composing child that sets its own model + root = repo_copy(tmp_path) + (root / rel).write_text((root / rel).read_text() + "model: claude-opus-5-5\n") + # When the pins are checked + problems = pin_problems(root) + # Then the override is reported even though its value matches + assert problems == [f"{rel}: overrides 'model'"] + + +def test_missing_run_py_default_fails(tmp_path): + """A run.py --model flag without a literal default fails closed instead of passing.""" + # Given run.py whose --model default is removed + root = repo_copy(tmp_path) + path = root / "evals/fullsend/run.py" + path.write_text(path.read_text().replace('"--model", default="claude-opus-5-5"', '"--model"')) + # When the pins are checked + problems = pin_problems(root) + # Then the missing pin is reported + assert "run.py defaults: no model pin" in problems diff --git a/plugins/sdlc-workflow/scripts/test_triage_security_fullsend.py b/plugins/sdlc-workflow/scripts/test_triage_security_fullsend.py index 21e091a6d..60f9c6375 100644 --- a/plugins/sdlc-workflow/scripts/test_triage_security_fullsend.py +++ b/plugins/sdlc-workflow/scripts/test_triage_security_fullsend.py @@ -195,18 +195,40 @@ def test_invalid_trusted_input_fixture_fails_closed_before_analysis(tmp_path): @pytest.mark.parametrize("gate, exit_code, stdout, stderr", [ (None, 0, "interactive mode\n", ""), ("", 1, "", "ERROR: FULLSEND_OUTPUT_DIR is set but empty\n"), - ("/synthetic-output", 0, "sandbox mode: /synthetic-output\n", ""), + ("{tmp}/output", 0, "sandbox mode: {tmp}/output\n", ""), ]) -def test_literal_gate_instruction_distinguishes_presence(gate, exit_code, stdout, stderr): +def test_literal_gate_instruction_distinguishes_presence(tmp_path, gate, exit_code, stdout, stderr): """Catch truthiness regressions in source; this is not proof the agent routed.""" # Given the production instruction and separately supplied environment state source = _skill_bash("### Step 0.6 – Fullsend Mode Detection") + gate = gate if gate is None else gate.format(tmp=tmp_path) # When a credential-free shell executes that exact gate instruction result = _run_instruction(source, gate) # Then presence, empty export and nonempty export have distinct outcomes - assert (result.returncode, result.stdout, result.stderr) == (exit_code, stdout, stderr) + assert (result.returncode, result.stdout, result.stderr) == (exit_code, stdout.format(tmp=tmp_path), stderr) + + +def test_literal_gate_creates_missing_output_dir_only_in_sandbox_mode(tmp_path): + """Fullsend's Claude runtime does not create FULLSEND_OUTPUT_DIR; the sandbox gate must.""" + # Given the production gate instruction and an output directory that does not exist yet + source = _skill_bash("### Step 0.6 – Fullsend Mode Detection") + output = tmp_path / "workspace" / "output" + + # When the gate runs in interactive mode + interactive = _run_instruction(source, None) + + # Then nothing is created + assert interactive.returncode == 0 + assert list(tmp_path.iterdir()) == [] + + # When the gate runs in sandbox mode + sandbox = _run_instruction(source, str(output)) + + # Then it creates the directory the result is written into + assert sandbox.returncode == 0 + assert output.is_dir() def test_literal_valid_input_instruction_continues_without_abort(tmp_path): diff --git a/plugins/sdlc-workflow/skills/triage-security/SKILL.md b/plugins/sdlc-workflow/skills/triage-security/SKILL.md index 697ebb7f3..a1186a0b7 100644 --- a/plugins/sdlc-workflow/skills/triage-security/SKILL.md +++ b/plugins/sdlc-workflow/skills/triage-security/SKILL.md @@ -73,6 +73,8 @@ if [ "${FULLSEND_OUTPUT_DIR+x}" = x ]; then echo "ERROR: FULLSEND_OUTPUT_DIR is set but empty" >&2 exit 1 fi + # Fullsend's Claude runtime exports the output dir without creating it. + mkdir -p "$FULLSEND_OUTPUT_DIR" echo "sandbox mode: $FULLSEND_OUTPUT_DIR" else echo "interactive mode" @@ -286,9 +288,10 @@ PYEOF validated trusted input bundle is also accepted provenance: derive those values from its `source_evidence` and `jira_metadata`, never infer or invent them. - **Fullsend output exception:** In Fullsend mode, the only permitted sandbox-side file - write is `$FULLSEND_OUTPUT_DIR/agent-result.json`: write either the deliberately - schema-invalid validation failure object or the completed valid accumulated result. - Do not create, modify, or repair any other local file. + write is `$FULLSEND_OUTPUT_DIR/agent-result.json` (Step 0.6 creates that directory): + write either the deliberately schema-invalid validation failure object or the + completed valid accumulated result. Do not create, modify, or repair any other + local file. - **Do NOT use Edit, Write, or Bash tools** to change files — except for local `security-matrix.md` files in the project working directory (see Step 2.1). In Fullsend mode, the narrow output exception above also permits writing