From 31159a2ae98e2465fe50572d75fbc720c76abd52 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 10 Nov 2026 19:21:03 +0100 Subject: [PATCH 1/3] Record the page and a screenshot when a journey fails, and quiet the cdp run output A failing journey now carries the page's URL and a PNG of the viewport, written to failure.png in the artifacts folder when there is one. JourneyFailure.Describe adds them as Page and Screenshot lines, so a test host and the CLI show the same thing. ScreenshotOnFailure turns the capture off. The CLI still fences everything from the page. An unmet feature block is one line with its reason, and a refused request that repeats is listed once with its count. Co-Authored-By: Claude Sonnet 5.5 --- DECISIONS.md | 15 +++++ docs/cli/index.md | 6 +- docs/getting-started/first-journey.md | 5 +- src/Nullean.Cdp.Cli/CdpCli.cs | 22 ++++++- src/Nullean.Cdp.Journeys/JourneyRunner.cs | 63 +++++++++++++++++-- tests/Nullean.Cdp.Cli.Tests/CliTests.cs | 60 +++++++++++++++++- .../Live/JourneyRunnerTests.cs | 58 +++++++++++++++-- .../Nullean.Cdp.Tests/Live/SafetyLiveTests.cs | 4 +- 8 files changed, 216 insertions(+), 17 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index d0f6877..b9e2b87 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -157,6 +157,21 @@ selected]: value`, two spaces per level. Checked against agent-browser 0.41.0 on command, was exported and run with `agent-browser batch --bail`: all 40 steps succeeded, and its reads are the values `JourneyRunnerTests` expects from this runner. +### Failures show the page they stopped on + +- **A failure records the URL and a viewport PNG by default.** Without them a failure from a live site says which step + broke and nothing about what the page was. `JourneyOptions.ScreenshotOnFailure` turns the capture off. Both are taken + before the context is disposed, with a 5 second limit, and a page that is gone (closed tab, lost connection) leaves them + out: a failure never fails a second time. +- **The screenshot is bytes on the failure, and a file only with an artifacts folder.** A test host that has no folder + (TUnit with `Nullean.Snapshots`) still gets the image to render or attach; the CLI names the file it wrote. +- **The CLI keeps fencing everything from the page, also on a terminal.** A pty is no proof that a person is reading: + agents run under one too. The failure's address goes inside the fence with the message. What is not page content stays + out of it: an unmet `feature` block is one structural line (`Checkpoint.IsSkip` marks it, so a page cannot imitate it + with its own text), and the screenshot path is printed outside, since the host chose it. +- **Refused requests are grouped.** A page that repeats a refused request tens of times printed it tens of times; the + list shows each once with `(xN)`. + ## agent-browser parity Every agent-browser command that acts on or reads the page in a repeatable way is a journey command: diff --git a/docs/cli/index.md b/docs/cli/index.md index 70331d6..042a86a 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -39,7 +39,7 @@ cdp run site/journeys/*.journey --base-url ./site --artifacts .artifacts/journey | `--user-agent` | the user agent of the run | | `--header` | a request header every page sends (repeatable), `"Name: value"` | | `--file-root` | the folder `file://` URLs may point into, when the base URL is not a folder | -| `--artifacts` | the folder `screenshot` and `network har stop` write into; off unless given | +| `--artifacts` | the folder `screenshot` and `network har stop` write into, and where a failing journey's `failure.png` goes; off unless given | | `--timeout` | seconds an action waits for its element (default 5) | | `--json` | JSON output; everything from the page is under a `page` key | | `--max-output` | characters per checkpoint before it is cut (default 20,000; 0 for no limit) | @@ -85,6 +85,10 @@ Everything that came from the page is fenced between `--- PAGE_CONTENT nonce=… run announced on the first line, so an agent reading the output can tell the page's words from the tool's. See [safety](../journeys/safety.md#output-for-agents). +A failing journey prints the page's address and the error inside the fence, and, with `--artifacts`, a `-- screenshot:` line +with the path of `failure.png`. A `feature` block the run does not meet is one line with the reason and no fence, because +nothing in it came from the page. A refused request that repeats is listed once with its count. + ## Exit codes | Code | | diff --git a/docs/getting-started/first-journey.md b/docs/getting-started/first-journey.md index f948a1f..7b8d5f2 100644 --- a/docs/getting-started/first-journey.md +++ b/docs/getting-started/first-journey.md @@ -65,7 +65,10 @@ if (result.Failure is { } failure) Console.WriteLine(failure.Describe(result.Source)); // about.journey:4: find role link ... and why ``` -A failing step is a result, not an exception: the run stops there and keeps the checkpoints that ran. +A failing step is a result, not an exception: the run stops there and keeps the checkpoints that ran. The failure also +carries the page's address (`failure.Url`) and a PNG of the viewport (`failure.Screenshot`, written to `failure.png` in +the artifacts folder when `JourneyOptions.ArtifactsDirectory` is set). `Describe` adds both as `Page:` and `Screenshot:` +lines. Set `ScreenshotOnFailure = false` to skip the capture. ## Replay it in agent-browser diff --git a/src/Nullean.Cdp.Cli/CdpCli.cs b/src/Nullean.Cdp.Cli/CdpCli.cs index 797f8b9..99ef264 100644 --- a/src/Nullean.Cdp.Cli/CdpCli.cs +++ b/src/Nullean.Cdp.Cli/CdpCli.cs @@ -283,20 +283,32 @@ private void WriteText(List<(Journey Journey, JourneyResult Result)> results, in output.WriteLine($"== {journey.Source}"); foreach (var checkpoint in result.Checkpoints) { + // A feature block the run did not meet holds nothing from the page: one line says why, with no fence. + if (checkpoint.IsSkip) + { + output.WriteLine($"-- {journey.Source}:{checkpoint.Line}: {checkpoint.Command} ({checkpoint.Output})"); + continue; + } + output.WriteLine($"-- {journey.Source}:{checkpoint.Line}: {checkpoint.Command}"); output.Write(PageContent.Fence($"url: {checkpoint.Url}\n{checkpoint.Output}", _nonce, Origin(checkpoint.Url), Cap(maxOutput))); } if (result.Blocked.Count > 0) { - output.WriteLine($"-- {result.Blocked.Count} request(s) outside the allow list were refused"); - output.Write(PageContent.Fence(string.Join('\n', result.Blocked.Select(b => $"{b.ResourceType} {b.Url}")), _nonce, maxCharacters: Cap(maxOutput))); + var requests = result.Blocked.GroupBy(b => $"{b.ResourceType} {b.Url}").ToList(); + output.WriteLine($"-- {result.Blocked.Count} request(s) outside the allow list were refused ({requests.Count} distinct)"); + output.Write(PageContent.Fence(string.Join('\n', requests.Select(g => g.Count() == 1 ? g.Key : $"{g.Key} (x{g.Count()})")), _nonce, maxCharacters: Cap(maxOutput))); } if (result.Failure is { } failure) { output.WriteLine($"FAILED {journey.Source}:{failure.Line}: {failure.Command}"); - output.Write(PageContent.Fence(failure.Message, _nonce, maxCharacters: Cap(maxOutput))); + // The page's address came from the page, so it goes inside the fence with the message. + var detail = failure.Url is { Length: > 0 } url ? $"url: {url}\n{failure.Message}" : failure.Message; + output.Write(PageContent.Fence(detail, _nonce, failure.Url is null ? null : Origin(failure.Url), Cap(maxOutput))); + if (failure.ScreenshotPath is { } screenshot) + output.WriteLine($"-- screenshot: {screenshot}"); } else output.WriteLine($"ok {journey.Source}: {journey.Steps.Count} steps, {result.Checkpoints.Count} checkpoints"); @@ -349,8 +361,12 @@ private void WriteJson(List<(Journey Journey, JourneyResult Result)> results, in json.WriteNumber("line", failure.Line); json.WriteString("command", failure.Command); json.WriteStartObject("page"); + if (failure.Url is not null) + json.WriteString("url", failure.Url); WriteCapped(json, "message", failure.Message, maxOutput); json.WriteEndObject(); + if (failure.ScreenshotPath is not null) + json.WriteString("screenshot", failure.ScreenshotPath); json.WriteEndObject(); } diff --git a/src/Nullean.Cdp.Journeys/JourneyRunner.cs b/src/Nullean.Cdp.Journeys/JourneyRunner.cs index 45e3e9c..83546df 100644 --- a/src/Nullean.Cdp.Journeys/JourneyRunner.cs +++ b/src/Nullean.Cdp.Journeys/JourneyRunner.cs @@ -37,6 +37,9 @@ public sealed class JourneyOptions /// public string? ArtifactsDirectory { get; init; } + /// Take a screenshot of the viewport when a step fails, into and, with an , failure.png there. Default: true. + public bool ScreenshotOnFailure { get; init; } = true; + /// Called before each step runs, e.g. to show progress. public Action? OnStep { get; init; } @@ -115,7 +118,11 @@ internal JourneyAllowList AllowList(Journey journey) /// The command as one normalised line (). /// The page's URL when the checkpoint ran. /// What it read: an accessibility tree, a value, true or false. -public sealed record Checkpoint(int Line, string Command, string Url, string Output); +public sealed record Checkpoint(int Line, string Command, string Url, string Output) +{ + /// True for a feature block the run did not meet: then says why, and nothing came from the page. + public bool IsSkip { get; init; } +} /// The step a journey stopped at, and why. /// The journey line. @@ -123,8 +130,25 @@ public sealed record Checkpoint(int Line, string Command, string Url, string Out /// The error, in plain text. public sealed record JourneyFailure(int Line, string Command, string Message) { - /// file:line: command, then the error. - public string Describe(string source) => $"{source}:{Line}: {Command}\n{Message}"; + /// The page's URL when the step failed, with secrets redacted. null when it could not be read. + public string? Url { get; init; } + + /// A PNG of the viewport when the step failed (). null when it could not be taken. + public byte[]? Screenshot { get; init; } + + /// Where was written, when the run has an artifacts folder. + public string? ScreenshotPath { get; init; } + + /// file:line: command, then the error, then the page and the screenshot as labelled lines. + public string Describe(string source) + { + var text = new System.Text.StringBuilder($"{source}:{Line}: {Command}\n{Message}"); + if (Url is { Length: > 0 }) + text.Append("\nPage: ").Append(Url); + if (ScreenshotPath is { Length: > 0 }) + text.Append("\nScreenshot: ").Append(ScreenshotPath); + return text.ToString(); + } } /// @@ -190,7 +214,7 @@ public static async Task RunAsync(Journey journey, Page page, Jou if (skipped.Add(block.Line)) { var skippedUrl = await page.UrlAsync(cancellationToken).ConfigureAwait(false); - checkpoints.Add(new Checkpoint(block.Line, block.Text, run.Secrets.Redact(skippedUrl), $"skipped: {unmet}")); + checkpoints.Add(new Checkpoint(block.Line, block.Text, run.Secrets.Redact(skippedUrl), $"skipped: {unmet}") { IsSkip = true }); } continue; @@ -219,12 +243,41 @@ public static async Task RunAsync(Journey journey, Page page, Jou } if (failure is not null) - return new JourneyResult(journey.Source, checkpoints, new JourneyFailure(step.Line, step.Text, run.Secrets.Redact(failure)), blocked); + return new JourneyResult(journey.Source, checkpoints, await FailureAsync(run, step, failure, cancellationToken).ConfigureAwait(false), blocked); } return new JourneyResult(journey.Source, checkpoints, null, blocked); } + // The failure with what the page looked like: its URL, and a screenshot when asked for. Neither can fail the run again, + // so a page that is gone (closed tab, lost connection) just leaves them out. + private static async Task FailureAsync(Run run, JourneyStep step, string message, CancellationToken cancellationToken) + { + var failure = new JourneyFailure(step.Line, step.Text, run.Secrets.Redact(message)); + using var limit = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + limit.CancelAfter(TimeSpan.FromSeconds(5)); + try + { + failure = failure with { Url = run.Secrets.Redact(await run.Page.UrlAsync(limit.Token).ConfigureAwait(false)) }; + if (!run.Options.ScreenshotOnFailure) + return failure; + var png = await run.Page.ScreenshotAsync(false, limit.Token).ConfigureAwait(false); + failure = failure with { Screenshot = png }; + if (run.Options.ArtifactsDirectory is not null) + { + var file = ArtifactFile(run, "failure.png", "failure screenshot"); + await File.WriteAllBytesAsync(file, png, cancellationToken).ConfigureAwait(false); + failure = failure with { ScreenshotPath = file }; + } + } + catch (Exception e) when (e is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + // The page could not be read or drawn: the failure stands without it. + } + + return failure; + } + // The run's viewport, user agent and headers, set before the first step so the journey starts as the harness wants. private static async Task ApplyDefaultsAsync(Page page, JourneyOptions options, CancellationToken cancellationToken) { diff --git a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs index d1f2096..ccdc99b 100644 --- a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs +++ b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs @@ -127,6 +127,63 @@ public async Task RunPrintsFencedCheckpoints() text.Should().EndWith("everything.journey: 40 steps, 16 checkpoints\n"); } + [Test] + public async Task AFeatureBlockTheRunDoesNotMeetIsOneLineWithNoFence() + { + var file = Path.Combine(Site("journeys"), $"skip-{Guid.NewGuid():N}.journey"); + await File.WriteAllTextAsync(file, "open index.html\nfeature preview\n get title\nend\nget title\n"); + try + { + var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, null, default); + + code.Should().Be(ExitCode.Ok); + Out.Should().Contain($"-- {file}:2: feature preview (skipped: preview)\n-- {file}:5: get title\n"); + } + finally + { + File.Delete(file); + } + } + + [Test] + public async Task ARefusedRequestThatRepeatsIsListedOnceWithItsCount() + { + var file = Path.Combine(Site("journeys"), $"repeat-{Guid.NewGuid():N}.journey"); + await File.WriteAllTextAsync(file, "open safety.html\nopen safety.html\nget title\n"); + try + { + await Cli().RunAsync([file], Site(), [], null, false, false, 0, null, default); + + Out.Should().Contain("(x2)").And.Contain("distinct)"); + Out.Split("Image https://tracker.example/pixel.png").Length.Should().Be(2, "the request is named once"); + } + finally + { + File.Delete(file); + } + } + + [Test] + public async Task AFailureNamesItsScreenshotWhenThereIsAnArtifactsFolder() + { + var file = Path.Combine(Site("journeys"), $"shot-{Guid.NewGuid():N}.journey"); + var artifacts = Directory.CreateTempSubdirectory("cdp-cli-").FullName; + await File.WriteAllTextAsync(file, "open index.html\nclick #missing\n"); + try + { + var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, 0.2, default, artifacts: artifacts); + + code.Should().Be(ExitCode.Failed); + Out.Should().Contain($"-- screenshot: {Path.Combine(artifacts, "failure.png")}\n"); + File.Exists(Path.Combine(artifacts, "failure.png")).Should().BeTrue(); + } + finally + { + File.Delete(file); + Directory.Delete(artifacts, recursive: true); + } + } + [Test] public async Task AFailureIsFencedAndExitsOne() { @@ -137,7 +194,8 @@ public async Task AFailureIsFencedAndExitsOne() var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, null, default); code.Should().Be(ExitCode.Failed); - Out.Should().Contain($"FAILED {file}:2: find role link click --name \"Leave the site\"\n--- PAGE_CONTENT nonce={Nonce} ---\n" + + Out.Should().Contain($"FAILED {file}:2: find role link click --name \"Leave the site\"\n--- PAGE_CONTENT nonce={Nonce} origin=chrome-error: ---\n" + + "url: chrome-error://chromewebdata/\n" + "The page navigated to https://leave.example/, which is refused: leave.example is not in the journey's allow list (add: allow leave.example).\n" + $"--- END_PAGE_CONTENT nonce={Nonce} ---\n"); Out.Should().Contain("request(s) outside the allow list were refused").And.Contain("Image https://tracker.example/pixel.png"); diff --git a/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs b/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs index 1627830..769d738 100644 --- a/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs +++ b/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs @@ -40,10 +40,59 @@ public async Task AFailingStepStopsTheRunAndKeepsEarlierCheckpoints() result.Passed.Should().BeFalse(); result.Checkpoints.Select(c => c.Output).Should().Equal("Actions"); - result.Failure.Should().Be(new JourneyFailure(3, "click #missing", + var failure = result.Failure!; + (failure.Line, failure.Command, failure.Message).Should().Be((3, "click #missing", "Timed out after 0.2s waiting for css #missing to be visible and enabled: no element matches.")); - result.Failure!.Describe(result.Source).Should().Be( - "fail.journey:3: click #missing\nTimed out after 0.2s waiting for css #missing to be visible and enabled: no element matches."); + failure.Url.Should().Be(Site + "actions.html"); + failure.Describe(result.Source).Should().Be( + "fail.journey:3: click #missing\nTimed out after 0.2s waiting for css #missing to be visible and enabled: no element matches.\n" + + $"Page: {Site}actions.html"); + } + + [Test] + public async Task AFailureCarriesAScreenshotOfThePageItStoppedOn() + { + var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); + + var result = await JourneyRunner.RunAsync(journey, chrome.Browser, + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200) }); + + result.Failure!.Screenshot.Should().NotBeNull().And.StartWith(new byte[] { 0x89, 0x50, 0x4E, 0x47 }, "a PNG"); + result.Failure.ScreenshotPath.Should().BeNull("there is no artifacts folder to write into"); + } + + [Test] + public async Task AScreenshotIsWrittenToTheArtifactsFolderAndNamedInTheDescription() + { + var folder = Directory.CreateTempSubdirectory("cdp-failure-").FullName; + try + { + var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); + + var result = await JourneyRunner.RunAsync(journey, chrome.Browser, + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ArtifactsDirectory = folder }); + + var path = Path.Combine(folder, "failure.png"); + result.Failure!.ScreenshotPath.Should().Be(path); + File.ReadAllBytes(path).Should().Equal(result.Failure.Screenshot); + result.Failure.Describe(result.Source).Should().EndWith($"\nScreenshot: {path}"); + } + finally + { + Directory.Delete(folder, recursive: true); + } + } + + [Test] + public async Task AScreenshotCanBeTurnedOff() + { + var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); + + var result = await JourneyRunner.RunAsync(journey, chrome.Browser, + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ScreenshotOnFailure = false }); + + result.Failure!.Screenshot.Should().BeNull(); + result.Failure.Url.Should().Be(Site + "actions.html", "the page's address is still recorded"); } [Test] @@ -51,7 +100,8 @@ public async Task ARelativeOpenWithoutABaseUrlFailsAtItsStep() { var result = await JourneyRunner.RunAsync(Journey.Parse("open /x", "rel.journey"), chrome.Browser); - result.Failure.Should().Be(new JourneyFailure(1, "open /x", "rel.journey:1: '/x' is relative: give a base URL")); + var failure = result.Failure!; + (failure.Line, failure.Command, failure.Message).Should().Be((1, "open /x", "rel.journey:1: '/x' is relative: give a base URL")); } [Test] diff --git a/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs b/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs index 86b2f65..5927b61 100644 --- a/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs +++ b/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs @@ -22,7 +22,7 @@ public async Task ALinkOffTheAllowListFailsTheStep() { var result = await Run("open safety.html\nfind role link click --name \"Leave the site\"\nget title"); - result.Failure.Should().Be(new JourneyFailure(2, "find role link click --name \"Leave the site\"", + (result.Failure!.Line, result.Failure.Command, result.Failure.Message).Should().Be((2, "find role link click --name \"Leave the site\"", "The page navigated to https://leave.example/, which is refused: leave.example is not in the journey's allow list (add: allow leave.example).")); } @@ -97,7 +97,7 @@ public async Task APasswordFieldRefusesALiteral() { var result = await Run("open safety.html\nfill #pw hunter22"); - result.Failure.Should().Be(new JourneyFailure(2, "fill #pw hunter22", + (result.Failure!.Line, result.Failure.Command, result.Failure.Message).Should().Be((2, "fill #pw hunter22", "css #pw is a password field: fill it with a secret reference ($env:NAME), not a literal.")); } From 97bf05cddada08eef072a82b23d5cd2ff3a80044 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 10 Nov 2026 19:27:44 +0100 Subject: [PATCH 2/3] Make the failure screenshot opt-in A screenshot can show private data on a page, so it is taken only with ScreenshotOnFailure or --screenshot-on-failure. The failure still records the page's URL. Co-Authored-By: Claude Sonnet 5.5 --- DECISIONS.md | 5 +++-- docs/cli/index.md | 5 +++-- docs/getting-started/first-journey.md | 6 +++--- src/Nullean.Cdp.Cli/CdpCli.cs | 4 +++- src/Nullean.Cdp.Cli/Program.cs | 4 +++- src/Nullean.Cdp.Journeys/JourneyRunner.cs | 6 +++--- tests/Nullean.Cdp.Cli.Tests/CliTests.cs | 4 ++-- tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs | 10 ++++++---- 8 files changed, 26 insertions(+), 18 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index b9e2b87..90fe9cb 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -159,8 +159,9 @@ selected]: value`, two spaces per level. Checked against agent-browser 0.41.0 on ### Failures show the page they stopped on -- **A failure records the URL and a viewport PNG by default.** Without them a failure from a live site says which step - broke and nothing about what the page was. `JourneyOptions.ScreenshotOnFailure` turns the capture off. Both are taken +- **A failure records the URL always, and a viewport PNG only when asked.** Without the URL a failure from a live site + says which step broke and nothing about where. A screenshot can show private data on a page and costs a capture, so + `JourneyOptions.ScreenshotOnFailure` (`--screenshot-on-failure`) is off by default. Both are taken before the context is disposed, with a 5 second limit, and a page that is gone (closed tab, lost connection) leaves them out: a failure never fails a second time. - **The screenshot is bytes on the failure, and a file only with an artifacts folder.** A test host that has no folder diff --git a/docs/cli/index.md b/docs/cli/index.md index 042a86a..00dc272 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -39,7 +39,8 @@ cdp run site/journeys/*.journey --base-url ./site --artifacts .artifacts/journey | `--user-agent` | the user agent of the run | | `--header` | a request header every page sends (repeatable), `"Name: value"` | | `--file-root` | the folder `file://` URLs may point into, when the base URL is not a folder | -| `--artifacts` | the folder `screenshot` and `network har stop` write into, and where a failing journey's `failure.png` goes; off unless given | +| `--artifacts` | the folder `screenshot` and `network har stop` write into, and where `failure.png` goes with `--screenshot-on-failure`; off unless given | +| `--screenshot-on-failure` | take a screenshot of the page when a step fails; it is written to `failure.png` in the artifacts folder | | `--timeout` | seconds an action waits for its element (default 5) | | `--json` | JSON output; everything from the page is under a `page` key | | `--max-output` | characters per checkpoint before it is cut (default 20,000; 0 for no limit) | @@ -85,7 +86,7 @@ Everything that came from the page is fenced between `--- PAGE_CONTENT nonce=… run announced on the first line, so an agent reading the output can tell the page's words from the tool's. See [safety](../journeys/safety.md#output-for-agents). -A failing journey prints the page's address and the error inside the fence, and, with `--artifacts`, a `-- screenshot:` line +A failing journey prints the page's address and the error inside the fence, and, with `--screenshot-on-failure` and `--artifacts`, a `-- screenshot:` line with the path of `failure.png`. A `feature` block the run does not meet is one line with the reason and no fence, because nothing in it came from the page. A refused request that repeats is listed once with its count. diff --git a/docs/getting-started/first-journey.md b/docs/getting-started/first-journey.md index 7b8d5f2..dd8c91a 100644 --- a/docs/getting-started/first-journey.md +++ b/docs/getting-started/first-journey.md @@ -66,9 +66,9 @@ if (result.Failure is { } failure) ``` A failing step is a result, not an exception: the run stops there and keeps the checkpoints that ran. The failure also -carries the page's address (`failure.Url`) and a PNG of the viewport (`failure.Screenshot`, written to `failure.png` in -the artifacts folder when `JourneyOptions.ArtifactsDirectory` is set). `Describe` adds both as `Page:` and `Screenshot:` -lines. Set `ScreenshotOnFailure = false` to skip the capture. +carries the page's address (`failure.Url`). With `ScreenshotOnFailure = true` it also carries a PNG of the viewport +(`failure.Screenshot`, written to `failure.png` in the artifacts folder when `JourneyOptions.ArtifactsDirectory` is set). +`Describe` adds them as `Page:` and `Screenshot:` lines. ## Replay it in agent-browser diff --git a/src/Nullean.Cdp.Cli/CdpCli.cs b/src/Nullean.Cdp.Cli/CdpCli.cs index 99ef264..ebd9781 100644 --- a/src/Nullean.Cdp.Cli/CdpCli.cs +++ b/src/Nullean.Cdp.Cli/CdpCli.cs @@ -137,7 +137,8 @@ public async Task RunAsync( IReadOnlyList? writable = null, IReadOnlyList? headers = null, string? userAgent = null, - string? viewport = null + string? viewport = null, + bool screenshotOnFailure = false ) { var (ok, root) = ParseBase(baseUrl); @@ -161,6 +162,7 @@ public async Task RunAsync( Headers = extraHeaders, UserAgent = userAgent, Viewport = size, + ScreenshotOnFailure = screenshotOnFailure, FileRoot = fileRoot is null ? null : Path.GetFullPath(fileRoot), ArtifactsDirectory = artifacts is null ? null : Path.GetFullPath(artifacts), Secrets = environment, diff --git a/src/Nullean.Cdp.Cli/Program.cs b/src/Nullean.Cdp.Cli/Program.cs index 54a02fd..08de41b 100644 --- a/src/Nullean.Cdp.Cli/Program.cs +++ b/src/Nullean.Cdp.Cli/Program.cs @@ -30,6 +30,7 @@ internal sealed class CdpCommands /// A request header every page sends (repeatable): "Name: value". /// The user agent of the run. /// The viewport the run starts with: 1280x800. Without it Chrome's default applies, which a responsive site may treat as a phone. + /// Take a screenshot of the page when a step fails. With --artifacts it is written to failure.png there. /// Show the browser window. /// Print JSON; everything under a "page" key came from the page. /// Characters per checkpoint before it is cut (0 for no limit). @@ -49,12 +50,13 @@ public static Task Run( string[]? header = null, string? userAgent = null, string? viewport = null, + bool screenshotOnFailure = false, bool headed = false, bool json = false, int maxOutput = CdpCli.DefaultMaxOutput, double? timeout = null, CancellationToken cancellationToken = default - ) => Cli().RunAsync(files, baseUrl, allow ?? [], fileRoot, headed, json, maxOutput, timeout, cancellationToken, artifacts, feature, buildType, env, readOnly, writable, header, userAgent, viewport); + ) => Cli().RunAsync(files, baseUrl, allow ?? [], fileRoot, headed, json, maxOutput, timeout, cancellationToken, artifacts, feature, buildType, env, readOnly, writable, header, userAgent, viewport, screenshotOnFailure); /// Parse journeys without running them, and report every error with its line. /// Journey files. diff --git a/src/Nullean.Cdp.Journeys/JourneyRunner.cs b/src/Nullean.Cdp.Journeys/JourneyRunner.cs index 83546df..afea56f 100644 --- a/src/Nullean.Cdp.Journeys/JourneyRunner.cs +++ b/src/Nullean.Cdp.Journeys/JourneyRunner.cs @@ -37,8 +37,8 @@ public sealed class JourneyOptions /// public string? ArtifactsDirectory { get; init; } - /// Take a screenshot of the viewport when a step fails, into and, with an , failure.png there. Default: true. - public bool ScreenshotOnFailure { get; init; } = true; + /// Take a screenshot of the viewport when a step fails, into and, with an , failure.png there. Off unless asked for, because a run against a site that shows private data would otherwise write it to disk. + public bool ScreenshotOnFailure { get; init; } /// Called before each step runs, e.g. to show progress. public Action? OnStep { get; init; } @@ -133,7 +133,7 @@ public sealed record JourneyFailure(int Line, string Command, string Message) /// The page's URL when the step failed, with secrets redacted. null when it could not be read. public string? Url { get; init; } - /// A PNG of the viewport when the step failed (). null when it could not be taken. + /// A PNG of the viewport when the step failed, when asked for it. null otherwise, or when it could not be taken. public byte[]? Screenshot { get; init; } /// Where was written, when the run has an artifacts folder. diff --git a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs index ccdc99b..3953b58 100644 --- a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs +++ b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs @@ -164,14 +164,14 @@ public async Task ARefusedRequestThatRepeatsIsListedOnceWithItsCount() } [Test] - public async Task AFailureNamesItsScreenshotWhenThereIsAnArtifactsFolder() + public async Task AFailureNamesItsScreenshotWhenAskedForWithAnArtifactsFolder() { var file = Path.Combine(Site("journeys"), $"shot-{Guid.NewGuid():N}.journey"); var artifacts = Directory.CreateTempSubdirectory("cdp-cli-").FullName; await File.WriteAllTextAsync(file, "open index.html\nclick #missing\n"); try { - var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, 0.2, default, artifacts: artifacts); + var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, 0.2, default, artifacts: artifacts, screenshotOnFailure: true); code.Should().Be(ExitCode.Failed); Out.Should().Contain($"-- screenshot: {Path.Combine(artifacts, "failure.png")}\n"); diff --git a/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs b/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs index 769d738..21a58a2 100644 --- a/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs +++ b/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs @@ -44,6 +44,7 @@ public async Task AFailingStepStopsTheRunAndKeepsEarlierCheckpoints() (failure.Line, failure.Command, failure.Message).Should().Be((3, "click #missing", "Timed out after 0.2s waiting for css #missing to be visible and enabled: no element matches.")); failure.Url.Should().Be(Site + "actions.html"); + failure.Screenshot.Should().BeNull("a screenshot is opt-in"); failure.Describe(result.Source).Should().Be( "fail.journey:3: click #missing\nTimed out after 0.2s waiting for css #missing to be visible and enabled: no element matches.\n" + $"Page: {Site}actions.html"); @@ -55,7 +56,7 @@ public async Task AFailureCarriesAScreenshotOfThePageItStoppedOn() var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); var result = await JourneyRunner.RunAsync(journey, chrome.Browser, - new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200) }); + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ScreenshotOnFailure = true }); result.Failure!.Screenshot.Should().NotBeNull().And.StartWith(new byte[] { 0x89, 0x50, 0x4E, 0x47 }, "a PNG"); result.Failure.ScreenshotPath.Should().BeNull("there is no artifacts folder to write into"); @@ -70,7 +71,7 @@ public async Task AScreenshotIsWrittenToTheArtifactsFolderAndNamedInTheDescripti var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); var result = await JourneyRunner.RunAsync(journey, chrome.Browser, - new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ArtifactsDirectory = folder }); + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ArtifactsDirectory = folder, ScreenshotOnFailure = true }); var path = Path.Combine(folder, "failure.png"); result.Failure!.ScreenshotPath.Should().Be(path); @@ -84,15 +85,16 @@ public async Task AScreenshotIsWrittenToTheArtifactsFolderAndNamedInTheDescripti } [Test] - public async Task AScreenshotCanBeTurnedOff() + public async Task ThereIsNoScreenshotUnlessAskedFor() { var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); var result = await JourneyRunner.RunAsync(journey, chrome.Browser, - new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ScreenshotOnFailure = false }); + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ArtifactsDirectory = Path.GetTempPath() }); result.Failure!.Screenshot.Should().BeNull(); result.Failure.Url.Should().Be(Site + "actions.html", "the page's address is still recorded"); + result.Failure.ScreenshotPath.Should().BeNull("an artifacts folder alone writes no failure.png"); } [Test] From caa9bca1e3a1ae02071c9b76fa75722d3babaf75 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 10 Nov 2026 19:28:13 +0100 Subject: [PATCH 3/3] Refuse --screenshot-on-failure without an artifacts folder The CLI prints no image, so a capture with nowhere to go was taken and thrown away. Co-Authored-By: Claude Sonnet 5.5 --- docs/cli/index.md | 2 +- src/Nullean.Cdp.Cli/CdpCli.cs | 7 +++++++ tests/Nullean.Cdp.Cli.Tests/CliTests.cs | 9 +++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/cli/index.md b/docs/cli/index.md index 00dc272..0649565 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -40,7 +40,7 @@ cdp run site/journeys/*.journey --base-url ./site --artifacts .artifacts/journey | `--header` | a request header every page sends (repeatable), `"Name: value"` | | `--file-root` | the folder `file://` URLs may point into, when the base URL is not a folder | | `--artifacts` | the folder `screenshot` and `network har stop` write into, and where `failure.png` goes with `--screenshot-on-failure`; off unless given | -| `--screenshot-on-failure` | take a screenshot of the page when a step fails; it is written to `failure.png` in the artifacts folder | +| `--screenshot-on-failure` | take a screenshot of the page when a step fails and write it to `failure.png` in the artifacts folder; needs `--artifacts` | | `--timeout` | seconds an action waits for its element (default 5) | | `--json` | JSON output; everything from the page is under a `page` key | | `--max-output` | characters per checkpoint before it is cut (default 20,000; 0 for no limit) | diff --git a/src/Nullean.Cdp.Cli/CdpCli.cs b/src/Nullean.Cdp.Cli/CdpCli.cs index ebd9781..df1a1a1 100644 --- a/src/Nullean.Cdp.Cli/CdpCli.cs +++ b/src/Nullean.Cdp.Cli/CdpCli.cs @@ -141,6 +141,13 @@ public async Task RunAsync( bool screenshotOnFailure = false ) { + if (screenshotOnFailure && artifacts is null) + { + // The CLI prints no image, so a capture with nowhere to go would be taken and thrown away. + error.WriteLine("--screenshot-on-failure needs --artifacts to write failure.png into."); + return ExitCode.Usage; + } + var (ok, root) = ParseBase(baseUrl); if (!ok || !ValidAllow(allow) || ParseHeaders(headers ?? []) is not { } extraHeaders || !ParseViewport(viewport, out var size) || Load(files) is not { } journeys) return ExitCode.Usage; diff --git a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs index 3953b58..727e099 100644 --- a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs +++ b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs @@ -163,6 +163,15 @@ public async Task ARefusedRequestThatRepeatsIsListedOnceWithItsCount() } } + [Test] + public async Task ScreenshotOnFailureWithoutAnArtifactsFolderIsAUsageError() + { + var code = await Cli().RunAsync([Site("journeys", "everything.journey")], Site(), [], null, false, false, 0, null, default, screenshotOnFailure: true); + + code.Should().Be(ExitCode.Usage); + _err.ToString().Should().Contain("--screenshot-on-failure needs --artifacts"); + } + [Test] public async Task AFailureNamesItsScreenshotWhenAskedForWithAnArtifactsFolder() {