From d03d96f1aabb4ae28ee25fca0a030de58f551a04 Mon Sep 17 00:00:00 2001 From: Richard Alpe Date: Mon, 21 Sep 2026 09:33:21 +0200 Subject: [PATCH] Print agent investigation prompt if a test fails Print "AGENT INITIAL INVESTIGATION PROMPT" if one or more tests fail. This is some condensed text in English that contains some vital clues that an agent can use to perform an initial investigation of a test failure. This might look simple, but a lot of human thought has gone into the format and info contained inside this summary. The goal is to give an agent "just enough" to figure things out. For example, the log id can be used to find logs regardless of where they might reside (logdir or ~/Downloads..). The hostname hint enables some powerful agents to actually connect to that host and investigate there, if the agent has memories about the host in question. Magic. Signed-off-by: Richard Alpe --- 9pm.py | 139 ++++++++++++++++++++++-------- README.md | 1 + etc/9pm-proj.yaml | 1 + self_test/cases/crash.sh | 3 + self_test/configs/proj-test1.yaml | 2 + self_test/run.py | 28 ++++++ 6 files changed, 140 insertions(+), 34 deletions(-) create mode 100755 self_test/cases/crash.sh diff --git a/9pm.py b/9pm.py index 581be7b..aa7faea 100755 --- a/9pm.py +++ b/9pm.py @@ -14,6 +14,9 @@ import re import atexit import hashlib +import shlex +import textwrap +from collections import deque from datetime import datetime TEST_CNT = 0 @@ -25,6 +28,7 @@ LOGDIR = None VERBOSE = False NOEXEC = False +PROJECT = {} class pcolor: purple = '\033[95m' @@ -57,6 +61,12 @@ def rootify_path(path): path = os.path.normpath(path) return path +def proj_relpath(path): + """Path relative to the project root, or unchanged if outside it""" + root = os.path.realpath(PROJECT['PROJECT-ROOT']) + rel = os.path.relpath(os.path.realpath(path), root) + return os.path.normpath(path) if rel.startswith(os.pardir) else rel + def pty_read_lines(fd): # PTY line discipline turns '\n' into '\r\n' on the way out, so trailing # '\r' is stripped from each yielded line. @@ -89,6 +99,9 @@ def execute(args, test, output_log): skip_suite = False test_skip = False err = False + test['failures'] = [] + context = deque(maxlen=10) # output since the last TAP result line + failure = None # lines of the failure being collected # Test metadata is now handled in the report generation, not in the log @@ -105,6 +118,19 @@ def execute(args, test, output_log): output_log.write(f"{stamp} {string}\n") + # Collect what the test said around a failure: the output leading up + # to the not ok line and what follows it, until the next TAP line. + if plan or ok or not_ok or skip: + failure = None + if not_ok: + failure = list(context) + [string] + test['failures'].append(failure) + context.clear() + else: + context.append(string) + if failure is not None and len(failure) < 40: + failure.append(string) + if plan: cprint(pcolor.purple, f'{stamp} {string}') test['plan'] = plan.group(2) @@ -172,7 +198,7 @@ def run_test(args, test): opts.extend(test['options']) name = test['name'] - path = os.path.relpath(test['case'], ROOT_PATH) + path = proj_relpath(test['case']) print(f"\n{pcolor.blue}Starting test {test['uniq_id']}: \"{test['uniq_id']:04d} {name}\" ({path}){pcolor.reset}") if test['result'] == "skip": @@ -291,6 +317,7 @@ def parse_suite(suite_path, parent_suite_path, options, settings, name=None): suite['name'] = name if name else gen_name(suite_path) suite['suite_num'] = next_suite_num() + suite['suite_path'] = suite_path if not os.path.isfile(suite_path): print(f"error, test suite not found {suite_path}") @@ -432,7 +459,7 @@ def count_tests(data): count_tests(test_data) return counts -def write_json_result(data, config): +def write_json_result(data, agent_prompt=None): """Write comprehensive JSON result file with embedded logs.""" # Collect all test logs and embed them in the data structure collect_test_logs(data) @@ -442,19 +469,15 @@ def write_json_result(data, config): # Prepare metadata current_time = datetime.now() - project_info = {} - if config: - if 'PROJECT-NAME' in config: - project_info['name'] = config['PROJECT-NAME'] - if 'PROJECT-ROOT' in config: - project_info['root'] = config['PROJECT-ROOT'] - # Get git info - version = run_git_cmd(config['PROJECT-ROOT'], ["describe", "--tags", "--always"]) - sha = run_git_cmd(config['PROJECT-ROOT'], ['rev-parse', 'HEAD']) - project_info['version'] = version - project_info['sha'] = sha - if 'PROJECT-TOPDOC' in config: - project_info['topdoc'] = config['PROJECT-TOPDOC'] + project_info = { + 'root': PROJECT['PROJECT-ROOT'], + 'version': PROJECT['PROJECT-VERSION'], + 'sha': PROJECT['PROJECT-SHA'], + } + if 'PROJECT-NAME' in PROJECT: + project_info['name'] = PROJECT['PROJECT-NAME'] + if 'PROJECT-TOPDOC' in PROJECT: + project_info['topdoc'] = PROJECT['PROJECT-TOPDOC'] # Get 9pm version info ninepm_sha = run_git_cmd(ROOT_PATH, ['rev-parse', 'HEAD']) @@ -473,6 +496,7 @@ def write_json_result(data, config): } }, 'summary': summary, + 'agent_prompt': agent_prompt, 'suite': data } @@ -523,6 +547,56 @@ def print_result_tree(data, base): print_result_tree(test, nextbase) i += 1 +def collect_failures(data): + failed = [] + for node in data['suite']: + if 'suite' in node: + failed.extend(collect_failures(node)) + elif node.get('result') == "fail" and node.get('failures'): + failed.append(node) + return failed + +def project_label(): + name = PROJECT.get('PROJECT-NAME') or os.path.basename(PROJECT['PROJECT-ROOT']) + sha = PROJECT['PROJECT-SHA'] + return f"{name} (git {sha[:12]})" if sha else name + +def wrap(text): + return textwrap.fill(text, 80, break_long_words=False, break_on_hyphens=False) + +def agent_prompt(data, aborted): + failed = collect_failures(data) + if not failed: + return None + + paths = [proj_relpath(n['suite_path'] if 'suite' in n else n['case']) for n in data['suite']] + ran = ", ".join(f'"{p}"' for p in dict.fromkeys(paths)) + + intro = (f"{len(failed)} test{'s' if len(failed) > 1 else ''} failed running {ran} " + f"in project {project_label()} on host \"{os.uname().nodename}\", " + f"log id {os.path.basename(LOGDIR)}.") + if aborted: + intro += " The run was aborted at the first fatal failure." + lines = [wrap(intro), ""] + + for test in failed: + if test.get('options'): + called = f"called with arguments '{shlex.join(test['options'])}'" + else: + called = "called without arguments" + lines.append(wrap(f"\"{test['name']}\" at \"{proj_relpath(test['case'])}\" " + f"{called} failed with:")) + for failure in test['failures']: + lines.extend(f" {line}" for line in failure) + if len(failure) >= 40: + lines.append(" (more in the log)") + lines.append("") + + lines.append("Please help me investigate this.") + if PROJECT.get('AGENT-HINTS'): + lines += ["", PROJECT['AGENT-HINTS'].strip()] + return "\n".join(lines) + def probe_suite(data): for test in data['suite']: if 'suite' in test: @@ -648,7 +722,7 @@ def parse_proj_config(root_path, args): sys.exit(1) if not '9pm' in data: - return [] + return {} if 'PROJECT-ROOT' in data['9pm']: data['9pm']['PROJECT-ROOT'] = rootify_path(data['9pm']['PROJECT-ROOT']) @@ -770,20 +844,6 @@ def run_git_cmd(path, command): return result -def pr_proj_info(proj): - str = f"\nTesting" - - if 'PROJECT-NAME' in proj: - str += f" {proj['PROJECT-NAME']}" - - if 'PROJECT-ROOT' in proj: - git_sha = run_git_cmd(proj['PROJECT-ROOT'], ['rev-parse', 'HEAD'])[:12] - - if git_sha: - str += f" ({git_sha})" - - cprint(pcolor.yellow, str) - def create_base_suite(args): suite = {'name': 'command-line', 'suite': []} for _ in range(args.repeat): @@ -812,6 +872,7 @@ def main(): global LOGDIR global VERBOSE global NOEXEC + global PROJECT # Line-buffer our own stdout so output streams when 9pm itself is piped to # another program (CI, tee, wrappers) instead of buffering until exit. @@ -834,7 +895,10 @@ def main(): vcprint(pcolor.faint, f"Logging to: {LOGDIR}") - proj = parse_proj_config(ROOT_PATH, args) + PROJECT = parse_proj_config(ROOT_PATH, args) + PROJECT.setdefault('PROJECT-ROOT', os.path.dirname(ROOT_PATH)) + PROJECT['PROJECT-SHA'] = run_git_cmd(PROJECT['PROJECT-ROOT'], ['rev-parse', 'HEAD']) + PROJECT['PROJECT-VERSION'] = run_git_cmd(PROJECT['PROJECT-ROOT'], ['describe', '--tags', '--always']) scratch = tempfile.mkdtemp(suffix='', prefix='9pm_', dir='/tmp') vcprint(pcolor.faint, f"Created scratch dir: {scratch}") @@ -845,7 +909,7 @@ def main(): vcprint(pcolor.faint, f"Created databasefile: {db.name}") DATABASE = db.name - pr_proj_info(proj) + cprint(pcolor.yellow, f"\nTesting {project_label()}") suite = create_base_suite(args) probe_suite(suite) @@ -855,7 +919,7 @@ def main(): setup_env(args) - skip, err, _ = run_suite(args, suite, False, args.abort) + skip, err, aborted = run_suite(args, suite, False, args.abort) if err: cprint(pcolor.red, "\nx Execution") elif skip: @@ -865,8 +929,15 @@ def main(): print_result_tree(suite, "") + prompt = agent_prompt(suite, aborted) + if prompt: + cprint(pcolor.yellow, "\nAGENT INITIAL INVESTIGATION PROMPT") + print(prompt) + with open(os.path.join(LOGDIR, 'agent-prompt.txt'), 'w') as f: + f.write(prompt + "\n") + # Export comprehensive JSON result - json_path = write_json_result(suite, proj) + json_path = write_json_result(suite, prompt) vcprint(pcolor.faint, f"JSON results written to: {json_path}") db.close() diff --git a/README.md b/README.md index 5bacd7a..55e0ea8 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - **Flexible Execution**: Run individual tests from command line or organized test suites - **Nested Structure**: Supports complex hierarchical suites with custom options and names - **Ready Environment**: Provides temporary directories, logging paths, and clean isolation +- **AI Ready**: Prints a ready-to-paste prompt for your AI agent when tests fail *Make complex test setups easy while keeping simple tests simple.* diff --git a/etc/9pm-proj.yaml b/etc/9pm-proj.yaml index e3b7010..4ef1463 100644 --- a/etc/9pm-proj.yaml +++ b/etc/9pm-proj.yaml @@ -1,6 +1,7 @@ # This is the default project config for 9pm. # The top level "9pm" contains data aimed for the 9pm.py test executor. # This file is passed down to test cases as the env variable NINEPM_PROJ_CONFIG. +# AGENT-HINTS: optional text added to the agent prompt after a failed run. --- 9pm: PROJECT-NAME: "9pm" diff --git a/self_test/cases/crash.sh b/self_test/cases/crash.sh new file mode 100755 index 0000000..e711455 --- /dev/null +++ b/self_test/cases/crash.sh @@ -0,0 +1,3 @@ +#!/bin/sh +echo "starting" +kill -ABRT $$ diff --git a/self_test/configs/proj-test1.yaml b/self_test/configs/proj-test1.yaml index ddf8361..c673999 100644 --- a/self_test/configs/proj-test1.yaml +++ b/self_test/configs/proj-test1.yaml @@ -1,3 +1,5 @@ 9pm: PROJECT-NAME: "Self Test 1" PROJECT-ROOT: "./" + AGENT-HINTS: | + Hint from Self Test 1 project config diff --git a/self_test/run.py b/self_test/run.py index c1760a3..ae6509a 100755 --- a/self_test/run.py +++ b/self_test/run.py @@ -324,6 +324,33 @@ def test_proj_config(self): print_green(f"[PASS] Project Config works") + def test_agent_prompt(self): + """Verify the agent prompt printed after failed runs""" + + result = self.run(["suites/abort-suite.yaml"], ["-o", "opt1"], 1, "AGENT INITIAL INVESTIGATION PROMPT") + prose = result.stdout.replace("\n", " ") + for expected in [ + '1 test failed running "self_test/suites/abort-suite.yaml" in project 9pm (git ', + '"fail.sh" at "self_test/cases/fail.sh" called with arguments \'opt1\' failed with:', + " not ok 1 - Dummy test fail", + "The run was aborted at the first fatal failure.", + "Please help me investigate this.", + ]: + assert expected in prose, f"Expected '{expected}' in agent prompt" + + prompt_path = os.path.expanduser('~/.local/share/9pm/logs/last/agent-prompt.txt') + assert os.path.exists(prompt_path), f"Could not find {prompt_path}" + + result = self.run(["cases/pass.sh"], [], 0) + assert "INVESTIGATION PROMPT" not in result.stdout, "Agent prompt printed for passing run" + + result = self.run(["cases/crash.sh"], [], 1) + assert "INVESTIGATION PROMPT" not in result.stdout, "Agent prompt printed for crashed test" + + self.run(["cases/fail.sh"], ["--proj", "configs/proj-test1.yaml"], 1, + "Hint from Self Test 1 project config") + print_green("[PASS] Agent prompt") + def test_repeat_flag(self): """Verify that -r (--repeat) works""" @@ -461,6 +488,7 @@ def cleanup(self): tester.test_abort_flag() tester.test_abort_key() tester.test_repeat_flag() + tester.test_agent_prompt() tester.test_proj_config() tester.test_spec_json_format() tester.test_line_buffering()