From e0cbe69be5f1f6da968e1644c39430d7a9f8d590 Mon Sep 17 00:00:00 2001 From: jonas-becker Date: Sat, 10 Oct 2026 11:29:05 +0200 Subject: [PATCH 1/2] informed agents voting update --- mallm/decision_protocols/simple_voting.py | 101 +++++++++++++++++- mallm/models/Chat.py | 52 +++++++-- .../test_simple_voting_parse.py | 90 ++++++++++++++++ 3 files changed, 235 insertions(+), 8 deletions(-) create mode 100644 test/decision_protocol/test_simple_voting_parse.py diff --git a/mallm/decision_protocols/simple_voting.py b/mallm/decision_protocols/simple_voting.py index a057cb1..37f14d7 100644 --- a/mallm/decision_protocols/simple_voting.py +++ b/mallm/decision_protocols/simple_voting.py @@ -1,4 +1,5 @@ import logging +import re from collections import Counter from typing import Any, Optional @@ -12,6 +13,102 @@ logger = logging.getLogger("mallm") +def _parse_trailing_digit_block( + vote_str: str, num_answers: int, in_range +) -> Optional[int]: + """ + Last contiguous digit run at the end of the string (after trailing spaces). + If that integer is out of range and ``num_answers <= 10``, use the last digit + of the run when it alone is a valid index (e.g. ``...12`` -> ``2``). + """ + s = vote_str.rstrip() + if not s: + return None + i = len(s) - 1 + while i >= 0 and not s[i].isdigit(): + i -= 1 + if i < 0: + return None + j = i + while j >= 0 and s[j].isdigit(): + j -= 1 + block = s[j + 1 : i + 1] + v = int(block) + if in_range(v): + return v + if num_answers <= 10 and len(block) > 1: + last_d = int(block[-1]) + if in_range(last_d): + return last_d + return None + + +def parse_vote_from_last_line(vote_str: str, num_answers: int) -> Optional[int]: + """ + Extract a solution index from an LLM voting reply. Tiers run from strict to + permissive so obvious formats win; loose tiers reduce invalid-vote retries. + + 1. Bottom-up: a line that is **only** digits and in ``[0, num_answers)``. + 2. Bottom-up: ``Final Answer: `` (case-insensitive). + 3. Bottom-up: common phrases (``Vote: N``, ``vote for solution N``, etc.). + 4. **Rightmost** ``\\d+`` substring in the full text whose value is in range. + 5. Trailing digit block at end of string (plus last-digit fallback for small + ``num_answers``; see ``_parse_trailing_digit_block``). + """ + if num_answers <= 0: + return None + if not isinstance(vote_str, str): + vote_str = str(vote_str) + + def in_range(v: int) -> bool: + return 0 <= v < num_answers + + lines = [ln.strip() for ln in vote_str.splitlines() if ln.strip()] + + for line in reversed(lines): + if re.fullmatch(r"\d+", line): + v = int(line) + if in_range(v): + return v + + final_answer_re = re.compile(r"(?i)final\s+answer\s*:\s*(\d+)") + for line in reversed(lines): + m = final_answer_re.search(line) + if m: + v = int(m.group(1)) + if in_range(v): + return v + + # Phrases often used when the model ignores "answer with the number only". + line_phrase_res = ( + re.compile(r"(?i)\b(?:my\s+)?vote\s*:\s*(\d+)\b"), + re.compile(r"(?i)\bcast(?:ing)?\s+(?:my\s+)?vote\s+for\s+solution\s*(\d+)\b"), + re.compile(r"(?i)\bvote\s+for\s+(?:solution|option)\s*#?\s*(\d+)\b"), + re.compile(r"(?i)\bvote\s+for\s+\*\*solution\s*(\d+)\*\*"), + re.compile(r"(?i)\b(?:choose|pick|select)(?:ing)?\s+solution\s*(\d+)\b"), + re.compile(r"(?i)\b(?:solution|option)\s*#?\s*(\d+)\s*(?:\.|!|\)|\]|,|;|:)?\s*$"), + re.compile(r"(?i)\*\*solution\s*(\d+)\*\*"), + ) + for line in reversed(lines): + for rx in line_phrase_res: + m = rx.search(line) + if m: + v = int(m.group(1)) + if in_range(v): + return v + + for m in reversed(list(re.finditer(r"\d+", vote_str))): + v = int(m.group(0)) + if in_range(v): + return v + + tail = _parse_trailing_digit_block(vote_str, num_answers, in_range) + if tail is not None: + return tail + + return None + + class SimpleVoting(DecisionProtocol): """ The Voting decision protocol allows panelists to vote for the best answer after a certain number of turns. @@ -117,8 +214,8 @@ def process_votes( voting_process_string: str, ) -> tuple[str, Any, bool, str]: success = False - vote_int = int("".join([x for x in vote_str if x.isnumeric()])) - if 0 <= vote_int < len(final_answers): + vote_int = parse_vote_from_last_line(vote_str, len(final_answers)) + if vote_int is not None: vote.append(vote_int) logger.info( f"{panelist.persona} voted for answer from {self.panelists[vote_int].persona}" diff --git a/mallm/models/Chat.py b/mallm/models/Chat.py index 1b64880..59bfd5d 100644 --- a/mallm/models/Chat.py +++ b/mallm/models/Chat.py @@ -1,4 +1,5 @@ import logging +import os import math import re import time @@ -155,14 +156,53 @@ def _call( # type: ignore ) # iterate and print stream collected_messages = [] + collected_reasoning = [] log_prob_sum = 0.0 + log_prob_steps = 0 for message in chat_completion: - message_str = message.choices[0].delta.content - if message.choices[0].logprobs: - log_prob_sum += message.choices[0].logprobs.content[0].logprob - if message_str and message_str not in self.stop_tokens: - collected_messages.append(message_str) - log_prob_sum = log_prob_sum / len(collected_messages) + delta = message.choices[0].delta + content_str = getattr(delta, "content", None) + reasoning_str = getattr(delta, "reasoning", None) or getattr( + delta, "reasoning_content", None + ) + lp = message.choices[0].logprobs + if lp is not None: + content_lps = getattr(lp, "content", None) + if content_lps: + log_prob_sum += content_lps[0].logprob + log_prob_steps += 1 + if content_str and content_str not in self.stop_tokens: + collected_messages.append(content_str) + if reasoning_str and reasoning_str not in self.stop_tokens: + collected_reasoning.append(reasoning_str) + # Reasoning-parser servers (vLLM --reasoning-parser) stream the thinking as + # delta.reasoning / reasoning_content. Keep only the answer channel so that + # peers and the answer extraction see the final answer, not the thinking. + # Fallback: if the model produced no answer text at all (e.g. thinking used + # the whole token budget), use the reasoning text. MALLM_INCLUDE_REASONING=1 + # restores the old behaviour (thinking + answer concatenated). + # vLLM's glm45 streaming parser leaks the last thinking tokens plus the closing + # tag into the content channel ("...nal Solution: 4The answer ..."): + # the answer is whatever follows the LAST "". + joined = "".join(collected_messages) + if "" in joined: + pre, _, post = joined.rpartition("") + collected_reasoning = collected_reasoning + [pre] + collected_messages = [post] if post else [] + if os.environ.get("MALLM_INCLUDE_REASONING", "").strip().lower() in ("1", "true", "yes"): + collected_messages = collected_reasoning + collected_messages + elif not collected_messages: + joined_reasoning = "".join(collected_reasoning) + if "" in joined_reasoning: + joined_reasoning = joined_reasoning.rpartition("")[2] + collected_messages = [joined_reasoning] if joined_reasoning else collected_reasoning + denom = len(collected_messages) + if denom == 0 and log_prob_steps > 0: + denom = log_prob_steps + if denom > 0: + log_prob_sum /= denom + else: + log_prob_sum = float("-inf") break except (APIError, APIConnectionError, RateLimitError) as e: # Handle API error here, e.g. retry or log diff --git a/test/decision_protocol/test_simple_voting_parse.py b/test/decision_protocol/test_simple_voting_parse.py new file mode 100644 index 0000000..b0573b0 --- /dev/null +++ b/test/decision_protocol/test_simple_voting_parse.py @@ -0,0 +1,90 @@ +"""Tests for simple voting: last-line vote parsing.""" + +from mallm.agents.panelist import Panelist +from mallm.coordinator import Coordinator +from mallm.decision_protocols.simple_voting import SimpleVoting, parse_vote_from_last_line +from mallm.models.discussion.FreeTextResponseGenerator import FreeTextResponseGenerator + +coordinator = Coordinator(None, None) +response_generator = FreeTextResponseGenerator(None) +panelists = [ + Panelist(None, None, coordinator, response_generator, f"P{i}", "") + for i in range(3) +] + + +def test_parse_last_line_three_answers_plain_digit(): + assert parse_vote_from_last_line("1", 3) == 1 + assert parse_vote_from_last_line(" 2 ", 3) == 2 + assert parse_vote_from_last_line("0", 3) == 0 + + +def test_parse_last_line_three_answers_ignores_earlier_numbers(): + """Regression: do not concatenate digits from explanation text.""" + text = ( + "I compared option 1 and option 2.\n" + "The first solution mentions 2024.\n" + "2" + ) + assert parse_vote_from_last_line(text, 3) == 2 + + +def test_parse_final_answer_line_with_trailing_tags(): + """GLM-style last line: not a bare digit, but Final Answer: N before tags.""" + text = ( + "Therefore, I vote for **Solution 1**.\n\n" + "Final Answer: 11. Asking to re-vote." + ) + assert parse_vote_from_last_line(text, 3) == 1 + + +def test_parse_vote_colon_and_trailing_punct(): + assert parse_vote_from_last_line("vote: 1", 3) == 1 + assert parse_vote_from_last_line("My vote: 2", 3) == 2 + assert parse_vote_from_last_line("1.", 3) == 1 + + +def test_parse_markdown_solution_phrase(): + text = "After review, I vote for **Solution 1** because it is clearest." + assert parse_vote_from_last_line(text, 3) == 1 + + +def test_parse_rightmost_in_range_token(): + """Prefer the rightmost digit-group that is a valid index (not left years).""" + text = "We compared scores from 2024 and earlier drafts; the best is 2" + assert parse_vote_from_last_line(text, 3) == 2 + + +def test_parse_trailing_digit_block_with_last_digit_fallback(): + """Multi-digit at end: last digit used as vote when whole number is out of range.""" + assert parse_vote_from_last_line("bogus 12", 3) == 2 + + +def test_parse_last_line_rejects_out_of_range(): + assert parse_vote_from_last_line("3", 3) is None + assert parse_vote_from_last_line("9", 3) is None + + +def test_parse_empty(): + assert parse_vote_from_last_line("", 3) is None + assert parse_vote_from_last_line("\n\n", 3) is None + + +def test_parse_eleven_answers_uses_integer_line(): + assert parse_vote_from_last_line("10", 11) == 10 + assert parse_vote_from_last_line(" 5 ", 11) == 5 + assert parse_vote_from_last_line("11", 11) is None + + +def test_process_votes_end_to_end(): + sv = SimpleVoting(panelists, num_neutral_agents=0, worker_functions=None) # type: ignore[arg-type] + votes: list[int] = [] + _, votes, ok, _ = sv.process_votes( + ["a", "b", "c"], + panelists[0], + "Long reasoning about 1 and 2.\n\n1\n", + votes, + "", + ) + assert ok is True + assert votes == [1] From 8934d279e27fbe0de7fccec10fec07ed1fefab95 Mon Sep 17 00:00:00 2001 From: jonas-becker Date: Sat, 10 Oct 2026 11:43:10 +0200 Subject: [PATCH 2/2] Fix mypy errors in simple_voting vote parsing - annotate the in_range callback of _parse_trailing_digit_block (no-untyped-def) - drop the isinstance coercion in parse_vote_from_last_line: the argument is typed str, so mypy (warn_unreachable) flagged the branch as unreachable --- mallm/decision_protocols/simple_voting.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mallm/decision_protocols/simple_voting.py b/mallm/decision_protocols/simple_voting.py index 37f14d7..711c805 100644 --- a/mallm/decision_protocols/simple_voting.py +++ b/mallm/decision_protocols/simple_voting.py @@ -1,7 +1,7 @@ import logging import re from collections import Counter -from typing import Any, Optional +from typing import Any, Callable, Optional from mallm.agents.panelist import Panelist from mallm.decision_protocols.protocol import DecisionProtocol @@ -14,7 +14,7 @@ def _parse_trailing_digit_block( - vote_str: str, num_answers: int, in_range + vote_str: str, num_answers: int, in_range: Callable[[int], bool] ) -> Optional[int]: """ Last contiguous digit run at the end of the string (after trailing spaces). @@ -57,8 +57,6 @@ def parse_vote_from_last_line(vote_str: str, num_answers: int) -> Optional[int]: """ if num_answers <= 0: return None - if not isinstance(vote_str, str): - vote_str = str(vote_str) def in_range(v: int) -> bool: return 0 <= v < num_answers