Informed personas

mallm/decision_protocols/simple_voting.py

@@ -1,6 +1,7 @@
 import logging
+import re
 from collections import Counter
-from typing import Any, Optional
+from typing import Any, Callable, Optional
 
 from mallm.agents.panelist import Panelist
 from mallm.decision_protocols.protocol import DecisionProtocol
@@ -12,6 +13,100 @@
 logger = logging.getLogger("mallm")
 
 
+def _parse_trailing_digit_block(
+    vote_str: str, num_answers: int, in_range: Callable[[int], bool]
+) -> Optional[int]:
+    """
+    Last contiguous digit run at the end of the string (after trailing spaces).
+    If that integer is out of range and ``num_answers <= 10``, use the last digit
+    of the run when it alone is a valid index (e.g. ``...12`` -> ``2``).
+    """
+    s = vote_str.rstrip()
+    if not s:
+        return None
+    i = len(s) - 1
+    while i >= 0 and not s[i].isdigit():
+        i -= 1
+    if i < 0:
+        return None
+    j = i
+    while j >= 0 and s[j].isdigit():
+        j -= 1
+    block = s[j + 1 : i + 1]
+    v = int(block)
+    if in_range(v):
+        return v
+    if num_answers <= 10 and len(block) > 1:
+        last_d = int(block[-1])
+        if in_range(last_d):
+            return last_d
+    return None
+
+
+def parse_vote_from_last_line(vote_str: str, num_answers: int) -> Optional[int]:
+    """
+    Extract a solution index from an LLM voting reply. Tiers run from strict to
+    permissive so obvious formats win; loose tiers reduce invalid-vote retries.
+
+    1. Bottom-up: a line that is **only** digits and in ``[0, num_answers)``.
+    2. Bottom-up: ``Final Answer: <int>`` (case-insensitive).
+    3. Bottom-up: common phrases (``Vote: N``, ``vote for solution N``, etc.).
+    4. **Rightmost** ``\\d+`` substring in the full text whose value is in range.
+    5. Trailing digit block at end of string (plus last-digit fallback for small
+       ``num_answers``; see ``_parse_trailing_digit_block``).
+    """
+    if num_answers <= 0:
+        return None
+
+    def in_range(v: int) -> bool:
+        return 0 <= v < num_answers
+
+    lines = [ln.strip() for ln in vote_str.splitlines() if ln.strip()]
+
+    for line in reversed(lines):
+        if re.fullmatch(r"\d+", line):
+            v = int(line)
+            if in_range(v):
+                return v
+
+    final_answer_re = re.compile(r"(?i)final\s+answer\s*:\s*(\d+)")
+    for line in reversed(lines):
+        m = final_answer_re.search(line)
+        if m:
+            v = int(m.group(1))
+            if in_range(v):
+                return v
+
+    # Phrases often used when the model ignores "answer with the number only".
+    line_phrase_res = (
+        re.compile(r"(?i)\b(?:my\s+)?vote\s*:\s*(\d+)\b"),
+        re.compile(r"(?i)\bcast(?:ing)?\s+(?:my\s+)?vote\s+for\s+solution\s*(\d+)\b"),
+        re.compile(r"(?i)\bvote\s+for\s+(?:solution|option)\s*#?\s*(\d+)\b"),
+        re.compile(r"(?i)\bvote\s+for\s+\*\*solution\s*(\d+)\*\*"),
+        re.compile(r"(?i)\b(?:choose|pick|select)(?:ing)?\s+solution\s*(\d+)\b"),
+        re.compile(r"(?i)\b(?:solution|option)\s*#?\s*(\d+)\s*(?:\.|!|\)|\]|,|;|:)?\s*$"),
+        re.compile(r"(?i)\*\*solution\s*(\d+)\*\*"),
+    )
+    for line in reversed(lines):
+        for rx in line_phrase_res:
+            m = rx.search(line)
+            if m:
+                v = int(m.group(1))
+                if in_range(v):
+                    return v
+
+    for m in reversed(list(re.finditer(r"\d+", vote_str))):
+        v = int(m.group(0))
+        if in_range(v):
+            return v
+
+    tail = _parse_trailing_digit_block(vote_str, num_answers, in_range)
+    if tail is not None:
+        return tail
+
+    return None
+
+
 class SimpleVoting(DecisionProtocol):
     """
     The Voting decision protocol allows panelists to vote for the best answer after a certain number of turns.
@@ -117,8 +212,8 @@ def process_votes(
         voting_process_string: str,
     ) -> tuple[str, Any, bool, str]:
         success = False
-        vote_int = int("".join([x for x in vote_str if x.isnumeric()]))
-        if 0 <= vote_int < len(final_answers):
+        vote_int = parse_vote_from_last_line(vote_str, len(final_answers))
+        if vote_int is not None:
             vote.append(vote_int)
             logger.info(
                 f"{panelist.persona} voted for answer from {self.panelists[vote_int].persona}"

mallm/models/Chat.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import math
 import re
 import time
@@ -155,14 +156,53 @@ def _call(  # type: ignore
                 )
                 # iterate and print stream
                 collected_messages = []
+                collected_reasoning = []
                 log_prob_sum = 0.0
+                log_prob_steps = 0
                 for message in chat_completion:
-                    message_str = message.choices[0].delta.content
-                    if message.choices[0].logprobs:
-                        log_prob_sum += message.choices[0].logprobs.content[0].logprob
-                    if message_str and message_str not in self.stop_tokens:
-                        collected_messages.append(message_str)
-                log_prob_sum = log_prob_sum / len(collected_messages)
+                    delta = message.choices[0].delta
+                    content_str = getattr(delta, "content", None)
+                    reasoning_str = getattr(delta, "reasoning", None) or getattr(
+                        delta, "reasoning_content", None
+                    )
+                    lp = message.choices[0].logprobs
+                    if lp is not None:
+                        content_lps = getattr(lp, "content", None)
+                        if content_lps:
+                            log_prob_sum += content_lps[0].logprob
+                            log_prob_steps += 1
+                    if content_str and content_str not in self.stop_tokens:
+                        collected_messages.append(content_str)
+                    if reasoning_str and reasoning_str not in self.stop_tokens:
+                        collected_reasoning.append(reasoning_str)
+                # Reasoning-parser servers (vLLM --reasoning-parser) stream the thinking as
+                # delta.reasoning / reasoning_content. Keep only the answer channel so that
+                # peers and the answer extraction see the final answer, not the thinking.
+                # Fallback: if the model produced no answer text at all (e.g. thinking used
+                # the whole token budget), use the reasoning text. MALLM_INCLUDE_REASONING=1
+                # restores the old behaviour (thinking + answer concatenated).
+                # vLLM's glm45 streaming parser leaks the last thinking tokens plus the closing
+                # tag into the content channel ("...nal Solution: 4</think>The answer ..."):
+                # the answer is whatever follows the LAST "</think>".
+                joined = "".join(collected_messages)
+                if "</think>" in joined:
+                    pre, _, post = joined.rpartition("</think>")
+                    collected_reasoning = collected_reasoning + [pre]
+                    collected_messages = [post] if post else []
+                if os.environ.get("MALLM_INCLUDE_REASONING", "").strip().lower() in ("1", "true", "yes"):
+                    collected_messages = collected_reasoning + collected_messages
+                elif not collected_messages:
+                    joined_reasoning = "".join(collected_reasoning)
+                    if "</think>" in joined_reasoning:
+                        joined_reasoning = joined_reasoning.rpartition("</think>")[2]
+                    collected_messages = [joined_reasoning] if joined_reasoning else collected_reasoning
+                denom = len(collected_messages)
+                if denom == 0 and log_prob_steps > 0:
+                    denom = log_prob_steps
+                if denom > 0:
+                    log_prob_sum /= denom
+                else:
+                    log_prob_sum = float("-inf")
                 break
             except (APIError, APIConnectionError, RateLimitError) as e:
                 # Handle API error here, e.g. retry or log

test/decision_protocol/test_simple_voting_parse.py

@@ -0,0 +1,90 @@
+"""Tests for simple voting: last-line vote parsing."""
+
+from mallm.agents.panelist import Panelist
+from mallm.coordinator import Coordinator
+from mallm.decision_protocols.simple_voting import SimpleVoting, parse_vote_from_last_line
+from mallm.models.discussion.FreeTextResponseGenerator import FreeTextResponseGenerator
+
+coordinator = Coordinator(None, None)
+response_generator = FreeTextResponseGenerator(None)
+panelists = [
+    Panelist(None, None, coordinator, response_generator, f"P{i}", "")
+    for i in range(3)
+]
+
+
+def test_parse_last_line_three_answers_plain_digit():
+    assert parse_vote_from_last_line("1", 3) == 1
+    assert parse_vote_from_last_line("  2  ", 3) == 2
+    assert parse_vote_from_last_line("0", 3) == 0
+
+
+def test_parse_last_line_three_answers_ignores_earlier_numbers():
+    """Regression: do not concatenate digits from explanation text."""
+    text = (
+        "I compared option 1 and option 2.\n"
+        "The first solution mentions 2024.\n"
+        "2"
+    )
+    assert parse_vote_from_last_line(text, 3) == 2
+
+
+def test_parse_final_answer_line_with_trailing_tags():
+    """GLM-style last line: not a bare digit, but Final Answer: N before tags."""
+    text = (
+        "Therefore, I vote for **Solution 1**.\n\n"
+        "Final Answer: 1</think>1. Asking to re-vote."
+    )
+    assert parse_vote_from_last_line(text, 3) == 1
+
+
+def test_parse_vote_colon_and_trailing_punct():
+    assert parse_vote_from_last_line("vote: 1", 3) == 1
+    assert parse_vote_from_last_line("My vote: 2", 3) == 2
+    assert parse_vote_from_last_line("1.", 3) == 1
+
+
+def test_parse_markdown_solution_phrase():
+    text = "After review, I vote for **Solution 1** because it is clearest."
+    assert parse_vote_from_last_line(text, 3) == 1
+
+
+def test_parse_rightmost_in_range_token():
+    """Prefer the rightmost digit-group that is a valid index (not left years)."""
+    text = "We compared scores from 2024 and earlier drafts; the best is 2"
+    assert parse_vote_from_last_line(text, 3) == 2
+
+
+def test_parse_trailing_digit_block_with_last_digit_fallback():
+    """Multi-digit at end: last digit used as vote when whole number is out of range."""
+    assert parse_vote_from_last_line("bogus 12", 3) == 2
+
+
+def test_parse_last_line_rejects_out_of_range():
+    assert parse_vote_from_last_line("3", 3) is None
+    assert parse_vote_from_last_line("9", 3) is None
+
+
+def test_parse_empty():
+    assert parse_vote_from_last_line("", 3) is None
+    assert parse_vote_from_last_line("\n\n", 3) is None
+
+
+def test_parse_eleven_answers_uses_integer_line():
+    assert parse_vote_from_last_line("10", 11) == 10
+    assert parse_vote_from_last_line("  5  ", 11) == 5
+    assert parse_vote_from_last_line("11", 11) is None
+
+
+def test_process_votes_end_to_end():
+    sv = SimpleVoting(panelists, num_neutral_agents=0, worker_functions=None)  # type: ignore[arg-type]
+    votes: list[int] = []
+    _, votes, ok, _ = sv.process_votes(
+        ["a", "b", "c"],
+        panelists[0],
+        "Long reasoning about 1 and 2.\n\n1\n",
+        votes,
+        "",
+    )
+    assert ok is True
+    assert votes == [1]