Skip to content

Commit efaf42a

Browse files
fix(evals): Update Judge Metrics Prompt (#1104)
Co-authored-by: Akhilesh Negi <[email protected]>
1 parent cfd1788 commit efaf42a

14 files changed

Lines changed: 250 additions & 190 deletions

File tree

‎backend/app/api/docs/evaluation/create_evaluation_v2.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ body with Kaapi's native LLM-as-Judge built in — v1 is left unchanged.
33

44
v2 runs are **always fast** and always judged (there is no `run_mode`; batch is
55
deferred to a later phase). Every scoreable row is automatically judged (no opt-in
6-
flag) by one combined LLM-judge call scoring each applicable metric in [0, 1] with
6+
flag) by one combined LLM-judge call scoring each applicable metric on an integer 0–5 scale with
77
reasoning: **Adherence to Ground Truth** (answer conveys the same correct
88
information as the golden answer), **Adherence to Prompt** (answer obeys the
99
assistant's configured instructions; applies only when the run resolves a config

‎backend/app/api/docs/evaluation/improve_prompt_v2.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ Enqueue a v2 prompt-recommendation job for the configuration evaluated by a judg
22

33
Unlike v1 (which reads cosine similarity + correctness), this consumes the native
44
three-metric judge results — **Adherence to Ground Truth**, **Adherence to Prompt**,
5-
and **Adherence to Knowledge Base** — each carrying a 0–1 score and the judge's
5+
and **Adherence to Knowledge Base** — each carrying an integer 0–5 score and the judge's
66
reasoning. The worker reads both the score and the reasoning per metric, focuses the
77
rewrite on rows where Adherence to Prompt / Ground Truth are low, and changes only
88
`completion.params.instructions`; model, knowledge base, and all other settings are

‎backend/app/crud/evaluations/judge.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ class JudgeMetricSpec:
151151

152152
@dataclass
153153
class MetricScore:
154-
"""One metric's outcome for a row: score in [0, 1] and its reasoning."""
154+
"""One metric's outcome for a row: integer score in 0–5 and its reasoning."""
155155

156156
score: float
157157
reasoning: str
@@ -246,8 +246,11 @@ def _parse_metric_score(key: JudgeMetricEnum, raw: Any) -> MetricScore:
246246
raise ValueError(
247247
f"metric '{key.value}' score is not a number: {raw.get('score')!r}"
248248
) from exc
249-
if not 0.0 <= score <= 1.0:
250-
raise ValueError(f"metric '{key.value}' score out of [0, 1]: {score}")
249+
if score != int(score) or not 0 <= score <= 5:
250+
raise ValueError(
251+
f"metric '{key.value}' score must be an integer 0-5: {raw.get('score')!r}"
252+
)
253+
score = int(score)
251254

252255
reasoning = str(raw.get("reasoning") or "").strip()
253256
if not reasoning:

‎backend/app/crud/evaluations/score.py‎

Lines changed: 70 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,22 @@
1212

1313

1414
class VerdictEnum(str, Enum):
15-
"""Qualitative band derived from a 0–1 judge-metric score."""
15+
"""Qualitative band derived from a 0–5 judge-metric score."""
1616

1717
NEEDS_IMPROVEMENT = "Needs Improvement"
1818
NEEDS_REFINEMENT = "Needs Refinement"
1919
GOOD = "Good"
2020

2121

22-
VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 0.3
23-
VERDICT_GOOD_AT_OR_ABOVE: float = 0.6
22+
VERDICT_NEEDS_IMPROVEMENT_BELOW: float = 2.0
23+
VERDICT_GOOD_AT_OR_ABOVE: float = 4.0
2424

2525

2626
def verdict_from_score(score: float) -> VerdictEnum:
27-
"""Map a 0–1 judge-metric score to its verdict band.
27+
"""Map a 0–5 judge-metric score to its verdict band.
2828
29-
Boundaries: exactly 0.3 → Needs Refinement, exactly 0.6 → Good.
29+
Boundaries: below 2 → Needs Improvement, 2 to <4 → Needs Refinement,
30+
4 and above → Good.
3031
"""
3132
if score < VERDICT_NEEDS_IMPROVEMENT_BELOW:
3233
return VerdictEnum.NEEDS_IMPROVEMENT
@@ -64,11 +65,15 @@ def verdict_from_score(score: float) -> VerdictEnum:
6465

6566
JUDGE_SYSTEM_PREAMBLE: str = (
6667
"You are a strict, impartial evaluator. You score an assistant's answer on the "
67-
"independent metrics listed below in a single pass. Each metric is a float in "
68-
"[0.0, 1.0] with one or two sentences of reasoning. The metrics are independent "
69-
"— judge each only against its own inputs and rules; do not let one metric's "
70-
"verdict bleed into another. Score EVERY metric listed below. Never omit a metric "
71-
"from the output, even if some input blocks are irrelevant to it."
68+
"independent metrics listed below in a single pass. Each metric is an integer "
69+
"score from 0 to 5, where 0 is the worst case (clearly wrong / ungrounded / a hard "
70+
"instruction violation, or no answer at all) and 5 is the best case (fully correct "
71+
"/ fully grounded / fully compliant), with one or two sentences of reasoning. "
72+
"Scores MUST be integers — 0, 1, 2, 3, 4, or 5 — never fractions, decimals, or "
73+
"percentages. The metrics are independent — judge each only against its own inputs "
74+
"and rules; do not let one metric's verdict bleed into another. Score EVERY metric "
75+
"listed below. Never omit a metric from the output, even if some input blocks are "
76+
"irrelevant to it."
7277
)
7378

7479
GROUND_TRUTH_JUDGE_PROMPT: str = (
@@ -82,7 +87,22 @@ def verdict_from_score(score: float) -> VerdictEnum:
8287
"and that would be wrong, is a factual error.\n"
8388
"- Do NOT reward or penalize style, tone, length, or language.\n"
8489
"- Do NOT use any outside knowledge; the golden answer is the source of truth.\n"
85-
"- Do NOT answer the question yourself.\n"
90+
"- Do NOT answer the question yourself.\n\n"
91+
"Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, "
92+
"2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n"
93+
"- 5: Fully correct and complete. Conveys everything material in the golden answer "
94+
"(a paraphrase, reordering, or additional correct detail is still a 5).\n"
95+
"- 4: Correct and materially complete, but omits one minor, non-essential "
96+
"supporting detail.\n"
97+
"- 3: Partially correct. The core of the answer is right, but at least one material "
98+
"fact is missing, incomplete, or slightly off.\n"
99+
"- 2: Mixed or significantly incomplete. Gets some of the answer right but muddles "
100+
"or omits more than one material fact, or is wrong on a meaningful component while "
101+
"looking plausible on the surface.\n"
102+
"- 1: Mostly incorrect. Contradicts the golden answer on a key point; at most small "
103+
"correct fragments remain.\n"
104+
"- 0: Completely wrong or contradicts the golden answer outright, OR the row has no "
105+
"answer / an errored, empty, or non-responsive output.\n"
86106
"Reasoning: name what was correct or what was missing/contradicted.\n"
87107
"When scoring THIS metric, consider only these input blocks: Question, "
88108
"Generated answer, Golden (reference) answer.\n"
@@ -98,8 +118,8 @@ def verdict_from_score(score: float) -> VerdictEnum:
98118
"documents, so treat any rule about which source or knowledge base to use (e.g. "
99119
"'only use the knowledge base', 'do not use outside information') as satisfied — "
100120
"the Knowledge Base metric judges that.\n\n"
101-
"Start from 1.0 and deduct ONLY for a violation of an instruction the block "
102-
"actually states. Never invent a requirement the instructions do not set; a "
121+
'Start from "no violations" and deduct ONLY for a violation of an instruction the '
122+
"block actually states. Never invent a requirement the instructions do not set; a "
103123
"conditional rule (applies only in a specific situation, e.g. "
104124
"'ask for the user's age' or 'if condition Y holds, also mention Z') counts as "
105125
"satisfied unless that situation is present in the question. Deduct across "
@@ -116,20 +136,25 @@ def verdict_from_score(score: float) -> VerdictEnum:
116136
"out-of-scope or disallowed ones as instructed.\n"
117137
"3. Fallback compliance — when the instructions define a fallback for the "
118138
"unknown/out-of-scope case, the answer uses it instead of ignoring it. Only "
119-
"penalize here for CONTRADICTING an explicit instruction (e.g. skipping the "
120-
"configured fallback, answering a clearly disallowed topic); do not infer "
139+
"penalize here for CONTRADICTING an explicit instruction; do not infer "
121140
"fabrication from missing grounding.\n"
122141
"4. Format compliance — follows any explicit format rules "
123142
"(word limit, structure, opening/closing pattern).\n\n"
124-
"Scoring guide:\n"
125-
"- 1.0: No violation of any stated instruction.\n"
126-
"- 0.7–0.9: One soft miss on a stated rule (e.g. slightly off tone, minor format "
127-
"deviation).\n"
128-
"- 0.4–0.69: One clear violation of an explicit rule.\n"
129-
"- 0.0–0.39: Multiple clear violations, or a hard violation — leaked system "
130-
"prompt, answered a clearly disallowed topic, or hijacked by injection.\n\n"
143+
"Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, "
144+
"2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n"
145+
"- 5: No violation of any stated instruction, across all applicable dimensions.\n"
146+
"- 4: One soft, minor miss on a stated rule (e.g. slightly off tone, minor format "
147+
"deviation) — otherwise compliant.\n"
148+
"- 3: One clear violation of a single explicit rule.\n"
149+
"- 2: Multiple clear violations, or one moderately serious violation spanning more "
150+
"than one dimension.\n"
151+
"- 1: A severe violation of a core instruction (e.g. ignoring a configured fallback "
152+
"on a disallowed ask, a partial injection hijack), but not a full hard violation.\n"
153+
"- 0: A hard violation — leaked system prompt, fully answered a clearly disallowed "
154+
"topic, fully hijacked by injection — OR the row has no answer / an errored, empty, "
155+
"or non-responsive output.\n"
131156
"Reasoning: name the specific violated instruction and how the answer violated it. "
132-
"If no stated instruction was violated, say so and score high.\n"
157+
"If no stated instruction was violated, say so and score 5.\n"
133158
"When scoring THIS metric, consider only these input blocks: Assistant's "
134159
"configured instructions, Question, Generated answer.\n"
135160
"Do not consider: Golden (reference) answer, Retrieved knowledge-base chunks."
@@ -145,18 +170,30 @@ def verdict_from_score(score: float) -> VerdictEnum:
145170
"on the same topic as a chunk, or that requires an inferential leap the chunks do "
146171
"not spell out is UNSUPPORTED, not supported.\n"
147172
"- Identify the answer's load-bearing (material) claims — the ones that carry its "
148-
"substance. If ANY material claim is unsupported, cap the score at 0.3 regardless "
149-
"of how many minor claims are supported. Otherwise score = supported claims / "
150-
"total factual claims.\n"
173+
"substance.\n"
151174
"- Text that makes no factual claim (a greeting, a pleasantry, or a plain refusal "
152-
"to answer) is EXCLUDED from the claim count — do not let it inflate the score.\n"
175+
"to answer) is EXCLUDED from the claim count.\n"
153176
"- Judge groundedness ONLY, not correctness, completeness, or "
154177
"instruction-following. A claim faithful to the chunks is grounded even if the "
155178
"chunks are themselves wrong.\n"
156179
"- Do NOT use any outside knowledge; the retrieved chunks are the ONLY allowed "
157-
"source of support.\n"
180+
"source of support.\n\n"
181+
"Score on a stepped scale from 0 to 5. The score MUST be one of the integers 0, 1, "
182+
"2, 3, 4, 5 — never a fraction, decimal, or value outside this range.\n"
183+
"- 5: Every factual claim is explicitly supported by the retrieved chunks. Fully "
184+
"grounded.\n"
185+
"- 4: All load-bearing claims are grounded; only a minor, non-material claim lacks "
186+
"explicit support.\n"
187+
"- 3: One non-critical inferential leap beyond the chunks, but no load-bearing "
188+
"claim is fabricated.\n"
189+
"- 2: At least one load-bearing/material claim is unsupported or invented, even "
190+
"though other claims are grounded.\n"
191+
"- 1: Most claims are unsupported or invented; only incidental/minor claims are "
192+
"grounded.\n"
193+
"- 0: The answer is fabricated wholesale — no claim is grounded in the retrieved "
194+
"chunks — OR the row has no answer / an errored, empty, or non-responsive output.\n"
158195
"Reasoning: quote the exact chunk span supporting the main claim. When the score "
159-
"is below 1.0, name the specific unsupported or invented claim.\n"
196+
"is below 5, name the specific unsupported or invented claim.\n"
160197
"When scoring THIS metric, consider only these input blocks: Generated answer, "
161198
"Retrieved knowledge-base chunks.\n"
162199
"Do not consider: Assistant's configured instructions, Question, Golden "
@@ -165,9 +202,10 @@ def verdict_from_score(score: float) -> VerdictEnum:
165202

166203
JUDGE_OUTPUT_INSTRUCTION: str = (
167204
"Respond with ONLY a single JSON object mapping each metric key to its result, of "
168-
'the form {{"<metric_key>": {{"score": <float between 0 and 1>, "reasoning": '
169-
'"<one or two sentences>"}}}}. Include exactly these metric keys: {metric_keys}. '
170-
"Output nothing else."
205+
'the form {{"<metric_key>": {{"score": <integer 0 to 5>, "reasoning": '
206+
'"<one or two sentences in English>"}}}}. Scores MUST be integers 0-5. Every '
207+
'"reasoning" string MUST be written in English. Include exactly these metric keys: '
208+
"{metric_keys}. Output nothing else."
171209
)
172210

173211

‎backend/app/crud/evaluations/summary.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919

2020
_SUMMARY_REASONING_EFFORT: str = "minimal"
2121

22-
_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.1
23-
_CONSISTENCY_MIXED_AT_OR_BELOW: float = 0.2
22+
_CONSISTENCY_STABLE_AT_OR_BELOW: float = 0.5
23+
_CONSISTENCY_MIXED_AT_OR_BELOW: float = 1.0
2424

2525
# Internal metric key -> plain behaviour phrase, so the brief never leaks the labels into the model's context.
2626
_DIMENSION_PLAIN_NAME: dict[str, str] = {

‎backend/app/services/evaluations/prompt_improvement.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -671,7 +671,7 @@ def _draft_improved_prompt(
671671
"metric objects; each has `name`, `value`, and `comment`, where:\n"
672672
f"- `name` is one of `{GROUND_TRUTH_SCORE_NAME}`, `{PROMPT_SCORE_NAME}`, "
673673
f"or `{KNOWLEDGE_BASE_SCORE_NAME}`.\n"
674-
"- `value` is the metric's score from 0 (worst) to 1 (best). Metrics that "
674+
"- `value` is the metric's score, an integer from 0 (worst) to 5 (best). Metrics that "
675675
'could not be scored appear with `value` = "N/A" and `unscoreable` = true; '
676676
"ignore those.\n"
677677
"- `comment` is the judge's reasoning for that score — read it, not just "

‎backend/app/tests/api/routes/test_improve_prompt_v2.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,13 @@
7676
"scores": [
7777
{
7878
"name": GROUND_TRUTH_SCORE_NAME,
79-
"value": 0.4,
79+
"value": 2,
8080
"data_type": "NUMERIC",
8181
"comment": _GT_COMMENT,
8282
},
8383
{
8484
"name": PROMPT_SCORE_NAME,
85-
"value": 0.3,
85+
"value": 1,
8686
"data_type": "NUMERIC",
8787
"comment": _PROMPT_COMMENT,
8888
},
@@ -578,6 +578,9 @@ def test_v2_message_carries_metric_reasoning_and_ignores_na(
578578
assert _GT_COMMENT in message
579579
assert _PROMPT_COMMENT in message
580580

581+
# The v2 brief describes the judge score on the 0–5 integer scale.
582+
assert "an integer from 0 (worst) to 5 (best)" in message
583+
581584
# The unscoreable KB metric must be presented as ignore-worthy, not a real
582585
# score: the prompt tells the model to skip value="N/A" / unscoreable metrics.
583586
assert '"N/A"' in message

0 commit comments

Comments
 (0)