From 47f6b1c6b41962b37784f3a1cef559eef199696b Mon Sep 17 00:00:00 2001 From: Gotham-Zolio <18781106300@163.com> Date: Fri, 11 Sep 2026 20:12:29 -0400 Subject: [PATCH] test: run the README's own commands through the parser, and fix one it caught The same guard as plugrl-server's, and it earned its place immediately: the README documented `--log-level debug` on the env client, which has never had that flag. It exists on plugrl-run-server, and the line was copied from there. The site audit had gone through this repository and missed it, because reading a flag does not tell you whether it parses. The README is corrected and says what happened rather than quietly dropping the flag. The test asserts up front that its regex matched at least three commands, since one that matched nothing would make every case below pass - which is not hypothetical here: the first version of the pattern missed every command in this file, because they are written `uv run plugrl-run-env-client` and it looked for the bare executable. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 10 +++- tests/test_documented_commands.py | 96 +++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 tests/test_documented_commands.py diff --git a/README.md b/README.md index 52a98cc..2c59ac9 100644 --- a/README.md +++ b/README.md @@ -135,13 +135,19 @@ uv run plugrl-run-env-client mujoco-v1 --num-envs 1 --num-episodes 600 \ Run the `dummy-v1` environment with custom parameters: ```bash -# Run 100 episodes with 'debug' logging level -uv run plugrl-run-env-client dummy-v1 --num-episodes 100 --log-level debug +# Run 100 episodes +uv run plugrl-run-env-client dummy-v1 --num-episodes 100 # Run with custom environment settings (64x64 image, 4-dim action space) uv run plugrl-run-env-client dummy-v1 --env.img-width 64 --env.img-height 64 --env.action-dim 4 ``` +This example used to carry `--log-level debug`, which the env client has never +had - the flag exists on `plugrl-run-server`, and the line was copied from +there. There is no verbosity flag on this side. `tests/test_documented_commands.py` +now runs every command on this page through the parser, which is how that was +found. + ### Get More Help To see all available options for a specific environment: diff --git a/tests/test_documented_commands.py b/tests/test_documented_commands.py new file mode 100644 index 0000000..2253d6b --- /dev/null +++ b/tests/test_documented_commands.py @@ -0,0 +1,96 @@ +"""Every env-client command in the README must survive its own parser. + +An adversarial audit found flags documented on the project's site that do not +exist at all - `--num-workers` for what is really `--num-procs`, +`--use-real-time` and `--fps` for nothing, and `--use-env-lock` for +`--runner.use-env-lock`. They had been there since the pages were written, +because documentation is not executed. + +This executes the README's commands, as far as the parser. Parsing is a floor +rather than a guarantee: a flag can parse and still be ignored. It is the floor +that was missing. + +Environments behind an optional extra are reported as skipped, decided from the +registry rather than from the error text, so a genuine parse failure can never +be mistaken for a missing extra. +""" + +from __future__ import annotations + +import pathlib +import re +import shlex +import sys + +import pytest + +README = pathlib.Path(__file__).resolve().parents[1] / "README.md" + + +def _bash_blocks(text: str) -> list[str]: + return re.findall(r"```(?:bash|sh|console)\n(.*?)```", text, re.S) + + +def _commands(text: str) -> list[str]: + out: list[str] = [] + for block in _bash_blocks(text): + joined = re.sub(r"\\s*\n\s*", " ", block) + for line in joined.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # The README writes invocations three ways, and one block is a + # usage sketch with a placeholder rather than a command. + if not re.match( + r"^(uv run )?(plugrl-run-env-client|python -m plugrl_env_client\.cli)", + line, + ): + continue + if "<" in line or ">" in line or "[OPTIONS]" in line: + continue + out.append(line) + return out + + +def _args(command: str) -> list[str]: + parts = shlex.split(command) + if parts[0] == "uv": # uv run plugrl-run-env-client ... + parts = parts[2:] + if parts[0] == "python": # python -m plugrl_env_client.cli ... + return parts[3:] + return parts[1:] + + +COMMANDS = _commands(README.read_text(encoding="utf-8")) + + +def test_the_readme_contains_commands_to_check(): + """A regex that silently matched nothing would make every test below pass.""" + assert len(COMMANDS) >= 3, f"only found {len(COMMANDS)} commands in {README}" + + +@pytest.mark.parametrize("command", COMMANDS, ids=lambda c: c[:60]) +def test_documented_command_parses(command, monkeypatch, capsys): + args = _args(command) + if "--help" in args: + pytest.skip("--help exits by design") + + import plugrl_env_client.envs # noqa: F401 - registers the env modules + from plugrl_env_client.cli import cli + from plugrl_env_client.utils.registration import REGISTERED_ENV_CONFIGS + + # tyro renders the subcommand from the registry key, lowercased: the + # registry holds "MuJoCo-v1" and the README writes "mujoco-v1". + registered = {k.lower() for k in REGISTERED_ENV_CONFIGS} + if args and not args[0].startswith("-") and args[0].lower() not in registered: + pytest.skip(f"env '{args[0]}' needs an extra this install lacks") + + monkeypatch.setattr(sys, "argv", ["prog", *args]) + try: + cli() + except SystemExit as exc: + out = capsys.readouterr() + tail = (out.out + out.err)[-1500:] + pytest.fail( + f"README command failed to parse (exit {exc.code}):\n {command}\n\n{tail}" + )