diff --git a/DECISIONS.md b/DECISIONS.md index d0f6877..90fe9cb 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -157,6 +157,22 @@ selected]: value`, two spaces per level. Checked against agent-browser 0.41.0 on command, was exported and run with `agent-browser batch --bail`: all 40 steps succeeded, and its reads are the values `JourneyRunnerTests` expects from this runner. +### Failures show the page they stopped on + +- **A failure records the URL always, and a viewport PNG only when asked.** Without the URL a failure from a live site + says which step broke and nothing about where. A screenshot can show private data on a page and costs a capture, so + `JourneyOptions.ScreenshotOnFailure` (`--screenshot-on-failure`) is off by default. Both are taken + before the context is disposed, with a 5 second limit, and a page that is gone (closed tab, lost connection) leaves them + out: a failure never fails a second time. +- **The screenshot is bytes on the failure, and a file only with an artifacts folder.** A test host that has no folder + (TUnit with `Nullean.Snapshots`) still gets the image to render or attach; the CLI names the file it wrote. +- **The CLI keeps fencing everything from the page, also on a terminal.** A pty is no proof that a person is reading: + agents run under one too. The failure's address goes inside the fence with the message. What is not page content stays + out of it: an unmet `feature` block is one structural line (`Checkpoint.IsSkip` marks it, so a page cannot imitate it + with its own text), and the screenshot path is printed outside, since the host chose it. +- **Refused requests are grouped.** A page that repeats a refused request tens of times printed it tens of times; the + list shows each once with `(xN)`. + ## agent-browser parity Every agent-browser command that acts on or reads the page in a repeatable way is a journey command: diff --git a/docs/cli/index.md b/docs/cli/index.md index 70331d6..0649565 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -39,7 +39,8 @@ cdp run site/journeys/*.journey --base-url ./site --artifacts .artifacts/journey | `--user-agent` | the user agent of the run | | `--header` | a request header every page sends (repeatable), `"Name: value"` | | `--file-root` | the folder `file://` URLs may point into, when the base URL is not a folder | -| `--artifacts` | the folder `screenshot` and `network har stop` write into; off unless given | +| `--artifacts` | the folder `screenshot` and `network har stop` write into, and where `failure.png` goes with `--screenshot-on-failure`; off unless given | +| `--screenshot-on-failure` | take a screenshot of the page when a step fails and write it to `failure.png` in the artifacts folder; needs `--artifacts` | | `--timeout` | seconds an action waits for its element (default 5) | | `--json` | JSON output; everything from the page is under a `page` key | | `--max-output` | characters per checkpoint before it is cut (default 20,000; 0 for no limit) | @@ -85,6 +86,10 @@ Everything that came from the page is fenced between `--- PAGE_CONTENT nonce=… run announced on the first line, so an agent reading the output can tell the page's words from the tool's. See [safety](../journeys/safety.md#output-for-agents). +A failing journey prints the page's address and the error inside the fence, and, with `--screenshot-on-failure` and `--artifacts`, a `-- screenshot:` line +with the path of `failure.png`. A `feature` block the run does not meet is one line with the reason and no fence, because +nothing in it came from the page. A refused request that repeats is listed once with its count. + ## Exit codes | Code | | diff --git a/docs/getting-started/first-journey.md b/docs/getting-started/first-journey.md index f948a1f..dd8c91a 100644 --- a/docs/getting-started/first-journey.md +++ b/docs/getting-started/first-journey.md @@ -65,7 +65,10 @@ if (result.Failure is { } failure) Console.WriteLine(failure.Describe(result.Source)); // about.journey:4: find role link ... and why ``` -A failing step is a result, not an exception: the run stops there and keeps the checkpoints that ran. +A failing step is a result, not an exception: the run stops there and keeps the checkpoints that ran. The failure also +carries the page's address (`failure.Url`). With `ScreenshotOnFailure = true` it also carries a PNG of the viewport +(`failure.Screenshot`, written to `failure.png` in the artifacts folder when `JourneyOptions.ArtifactsDirectory` is set). +`Describe` adds them as `Page:` and `Screenshot:` lines. ## Replay it in agent-browser diff --git a/src/Nullean.Cdp.Cli/CdpCli.cs b/src/Nullean.Cdp.Cli/CdpCli.cs index 797f8b9..df1a1a1 100644 --- a/src/Nullean.Cdp.Cli/CdpCli.cs +++ b/src/Nullean.Cdp.Cli/CdpCli.cs @@ -137,9 +137,17 @@ public async Task RunAsync( IReadOnlyList? writable = null, IReadOnlyList? headers = null, string? userAgent = null, - string? viewport = null + string? viewport = null, + bool screenshotOnFailure = false ) { + if (screenshotOnFailure && artifacts is null) + { + // The CLI prints no image, so a capture with nowhere to go would be taken and thrown away. + error.WriteLine("--screenshot-on-failure needs --artifacts to write failure.png into."); + return ExitCode.Usage; + } + var (ok, root) = ParseBase(baseUrl); if (!ok || !ValidAllow(allow) || ParseHeaders(headers ?? []) is not { } extraHeaders || !ParseViewport(viewport, out var size) || Load(files) is not { } journeys) return ExitCode.Usage; @@ -161,6 +169,7 @@ public async Task RunAsync( Headers = extraHeaders, UserAgent = userAgent, Viewport = size, + ScreenshotOnFailure = screenshotOnFailure, FileRoot = fileRoot is null ? null : Path.GetFullPath(fileRoot), ArtifactsDirectory = artifacts is null ? null : Path.GetFullPath(artifacts), Secrets = environment, @@ -283,20 +292,32 @@ private void WriteText(List<(Journey Journey, JourneyResult Result)> results, in output.WriteLine($"== {journey.Source}"); foreach (var checkpoint in result.Checkpoints) { + // A feature block the run did not meet holds nothing from the page: one line says why, with no fence. + if (checkpoint.IsSkip) + { + output.WriteLine($"-- {journey.Source}:{checkpoint.Line}: {checkpoint.Command} ({checkpoint.Output})"); + continue; + } + output.WriteLine($"-- {journey.Source}:{checkpoint.Line}: {checkpoint.Command}"); output.Write(PageContent.Fence($"url: {checkpoint.Url}\n{checkpoint.Output}", _nonce, Origin(checkpoint.Url), Cap(maxOutput))); } if (result.Blocked.Count > 0) { - output.WriteLine($"-- {result.Blocked.Count} request(s) outside the allow list were refused"); - output.Write(PageContent.Fence(string.Join('\n', result.Blocked.Select(b => $"{b.ResourceType} {b.Url}")), _nonce, maxCharacters: Cap(maxOutput))); + var requests = result.Blocked.GroupBy(b => $"{b.ResourceType} {b.Url}").ToList(); + output.WriteLine($"-- {result.Blocked.Count} request(s) outside the allow list were refused ({requests.Count} distinct)"); + output.Write(PageContent.Fence(string.Join('\n', requests.Select(g => g.Count() == 1 ? g.Key : $"{g.Key} (x{g.Count()})")), _nonce, maxCharacters: Cap(maxOutput))); } if (result.Failure is { } failure) { output.WriteLine($"FAILED {journey.Source}:{failure.Line}: {failure.Command}"); - output.Write(PageContent.Fence(failure.Message, _nonce, maxCharacters: Cap(maxOutput))); + // The page's address came from the page, so it goes inside the fence with the message. + var detail = failure.Url is { Length: > 0 } url ? $"url: {url}\n{failure.Message}" : failure.Message; + output.Write(PageContent.Fence(detail, _nonce, failure.Url is null ? null : Origin(failure.Url), Cap(maxOutput))); + if (failure.ScreenshotPath is { } screenshot) + output.WriteLine($"-- screenshot: {screenshot}"); } else output.WriteLine($"ok {journey.Source}: {journey.Steps.Count} steps, {result.Checkpoints.Count} checkpoints"); @@ -349,8 +370,12 @@ private void WriteJson(List<(Journey Journey, JourneyResult Result)> results, in json.WriteNumber("line", failure.Line); json.WriteString("command", failure.Command); json.WriteStartObject("page"); + if (failure.Url is not null) + json.WriteString("url", failure.Url); WriteCapped(json, "message", failure.Message, maxOutput); json.WriteEndObject(); + if (failure.ScreenshotPath is not null) + json.WriteString("screenshot", failure.ScreenshotPath); json.WriteEndObject(); } diff --git a/src/Nullean.Cdp.Cli/Program.cs b/src/Nullean.Cdp.Cli/Program.cs index 54a02fd..08de41b 100644 --- a/src/Nullean.Cdp.Cli/Program.cs +++ b/src/Nullean.Cdp.Cli/Program.cs @@ -30,6 +30,7 @@ internal sealed class CdpCommands /// A request header every page sends (repeatable): "Name: value". /// The user agent of the run. /// The viewport the run starts with: 1280x800. Without it Chrome's default applies, which a responsive site may treat as a phone. + /// Take a screenshot of the page when a step fails. With --artifacts it is written to failure.png there. /// Show the browser window. /// Print JSON; everything under a "page" key came from the page. /// Characters per checkpoint before it is cut (0 for no limit). @@ -49,12 +50,13 @@ public static Task Run( string[]? header = null, string? userAgent = null, string? viewport = null, + bool screenshotOnFailure = false, bool headed = false, bool json = false, int maxOutput = CdpCli.DefaultMaxOutput, double? timeout = null, CancellationToken cancellationToken = default - ) => Cli().RunAsync(files, baseUrl, allow ?? [], fileRoot, headed, json, maxOutput, timeout, cancellationToken, artifacts, feature, buildType, env, readOnly, writable, header, userAgent, viewport); + ) => Cli().RunAsync(files, baseUrl, allow ?? [], fileRoot, headed, json, maxOutput, timeout, cancellationToken, artifacts, feature, buildType, env, readOnly, writable, header, userAgent, viewport, screenshotOnFailure); /// Parse journeys without running them, and report every error with its line. /// Journey files. diff --git a/src/Nullean.Cdp.Journeys/JourneyRunner.cs b/src/Nullean.Cdp.Journeys/JourneyRunner.cs index 45e3e9c..afea56f 100644 --- a/src/Nullean.Cdp.Journeys/JourneyRunner.cs +++ b/src/Nullean.Cdp.Journeys/JourneyRunner.cs @@ -37,6 +37,9 @@ public sealed class JourneyOptions /// public string? ArtifactsDirectory { get; init; } + /// Take a screenshot of the viewport when a step fails, into and, with an , failure.png there. Off unless asked for, because a run against a site that shows private data would otherwise write it to disk. + public bool ScreenshotOnFailure { get; init; } + /// Called before each step runs, e.g. to show progress. public Action? OnStep { get; init; } @@ -115,7 +118,11 @@ internal JourneyAllowList AllowList(Journey journey) /// The command as one normalised line (). /// The page's URL when the checkpoint ran. /// What it read: an accessibility tree, a value, true or false. -public sealed record Checkpoint(int Line, string Command, string Url, string Output); +public sealed record Checkpoint(int Line, string Command, string Url, string Output) +{ + /// True for a feature block the run did not meet: then says why, and nothing came from the page. + public bool IsSkip { get; init; } +} /// The step a journey stopped at, and why. /// The journey line. @@ -123,8 +130,25 @@ public sealed record Checkpoint(int Line, string Command, string Url, string Out /// The error, in plain text. public sealed record JourneyFailure(int Line, string Command, string Message) { - /// file:line: command, then the error. - public string Describe(string source) => $"{source}:{Line}: {Command}\n{Message}"; + /// The page's URL when the step failed, with secrets redacted. null when it could not be read. + public string? Url { get; init; } + + /// A PNG of the viewport when the step failed, when asked for it. null otherwise, or when it could not be taken. + public byte[]? Screenshot { get; init; } + + /// Where was written, when the run has an artifacts folder. + public string? ScreenshotPath { get; init; } + + /// file:line: command, then the error, then the page and the screenshot as labelled lines. + public string Describe(string source) + { + var text = new System.Text.StringBuilder($"{source}:{Line}: {Command}\n{Message}"); + if (Url is { Length: > 0 }) + text.Append("\nPage: ").Append(Url); + if (ScreenshotPath is { Length: > 0 }) + text.Append("\nScreenshot: ").Append(ScreenshotPath); + return text.ToString(); + } } /// @@ -190,7 +214,7 @@ public static async Task RunAsync(Journey journey, Page page, Jou if (skipped.Add(block.Line)) { var skippedUrl = await page.UrlAsync(cancellationToken).ConfigureAwait(false); - checkpoints.Add(new Checkpoint(block.Line, block.Text, run.Secrets.Redact(skippedUrl), $"skipped: {unmet}")); + checkpoints.Add(new Checkpoint(block.Line, block.Text, run.Secrets.Redact(skippedUrl), $"skipped: {unmet}") { IsSkip = true }); } continue; @@ -219,12 +243,41 @@ public static async Task RunAsync(Journey journey, Page page, Jou } if (failure is not null) - return new JourneyResult(journey.Source, checkpoints, new JourneyFailure(step.Line, step.Text, run.Secrets.Redact(failure)), blocked); + return new JourneyResult(journey.Source, checkpoints, await FailureAsync(run, step, failure, cancellationToken).ConfigureAwait(false), blocked); } return new JourneyResult(journey.Source, checkpoints, null, blocked); } + // The failure with what the page looked like: its URL, and a screenshot when asked for. Neither can fail the run again, + // so a page that is gone (closed tab, lost connection) just leaves them out. + private static async Task FailureAsync(Run run, JourneyStep step, string message, CancellationToken cancellationToken) + { + var failure = new JourneyFailure(step.Line, step.Text, run.Secrets.Redact(message)); + using var limit = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + limit.CancelAfter(TimeSpan.FromSeconds(5)); + try + { + failure = failure with { Url = run.Secrets.Redact(await run.Page.UrlAsync(limit.Token).ConfigureAwait(false)) }; + if (!run.Options.ScreenshotOnFailure) + return failure; + var png = await run.Page.ScreenshotAsync(false, limit.Token).ConfigureAwait(false); + failure = failure with { Screenshot = png }; + if (run.Options.ArtifactsDirectory is not null) + { + var file = ArtifactFile(run, "failure.png", "failure screenshot"); + await File.WriteAllBytesAsync(file, png, cancellationToken).ConfigureAwait(false); + failure = failure with { ScreenshotPath = file }; + } + } + catch (Exception e) when (e is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + // The page could not be read or drawn: the failure stands without it. + } + + return failure; + } + // The run's viewport, user agent and headers, set before the first step so the journey starts as the harness wants. private static async Task ApplyDefaultsAsync(Page page, JourneyOptions options, CancellationToken cancellationToken) { diff --git a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs index d1f2096..727e099 100644 --- a/tests/Nullean.Cdp.Cli.Tests/CliTests.cs +++ b/tests/Nullean.Cdp.Cli.Tests/CliTests.cs @@ -127,6 +127,72 @@ public async Task RunPrintsFencedCheckpoints() text.Should().EndWith("everything.journey: 40 steps, 16 checkpoints\n"); } + [Test] + public async Task AFeatureBlockTheRunDoesNotMeetIsOneLineWithNoFence() + { + var file = Path.Combine(Site("journeys"), $"skip-{Guid.NewGuid():N}.journey"); + await File.WriteAllTextAsync(file, "open index.html\nfeature preview\n get title\nend\nget title\n"); + try + { + var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, null, default); + + code.Should().Be(ExitCode.Ok); + Out.Should().Contain($"-- {file}:2: feature preview (skipped: preview)\n-- {file}:5: get title\n"); + } + finally + { + File.Delete(file); + } + } + + [Test] + public async Task ARefusedRequestThatRepeatsIsListedOnceWithItsCount() + { + var file = Path.Combine(Site("journeys"), $"repeat-{Guid.NewGuid():N}.journey"); + await File.WriteAllTextAsync(file, "open safety.html\nopen safety.html\nget title\n"); + try + { + await Cli().RunAsync([file], Site(), [], null, false, false, 0, null, default); + + Out.Should().Contain("(x2)").And.Contain("distinct)"); + Out.Split("Image https://tracker.example/pixel.png").Length.Should().Be(2, "the request is named once"); + } + finally + { + File.Delete(file); + } + } + + [Test] + public async Task ScreenshotOnFailureWithoutAnArtifactsFolderIsAUsageError() + { + var code = await Cli().RunAsync([Site("journeys", "everything.journey")], Site(), [], null, false, false, 0, null, default, screenshotOnFailure: true); + + code.Should().Be(ExitCode.Usage); + _err.ToString().Should().Contain("--screenshot-on-failure needs --artifacts"); + } + + [Test] + public async Task AFailureNamesItsScreenshotWhenAskedForWithAnArtifactsFolder() + { + var file = Path.Combine(Site("journeys"), $"shot-{Guid.NewGuid():N}.journey"); + var artifacts = Directory.CreateTempSubdirectory("cdp-cli-").FullName; + await File.WriteAllTextAsync(file, "open index.html\nclick #missing\n"); + try + { + var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, 0.2, default, artifacts: artifacts, screenshotOnFailure: true); + + code.Should().Be(ExitCode.Failed); + Out.Should().Contain($"-- screenshot: {Path.Combine(artifacts, "failure.png")}\n"); + File.Exists(Path.Combine(artifacts, "failure.png")).Should().BeTrue(); + } + finally + { + File.Delete(file); + Directory.Delete(artifacts, recursive: true); + } + } + [Test] public async Task AFailureIsFencedAndExitsOne() { @@ -137,7 +203,8 @@ public async Task AFailureIsFencedAndExitsOne() var code = await Cli().RunAsync([file], Site(), [], null, false, false, 0, null, default); code.Should().Be(ExitCode.Failed); - Out.Should().Contain($"FAILED {file}:2: find role link click --name \"Leave the site\"\n--- PAGE_CONTENT nonce={Nonce} ---\n" + + Out.Should().Contain($"FAILED {file}:2: find role link click --name \"Leave the site\"\n--- PAGE_CONTENT nonce={Nonce} origin=chrome-error: ---\n" + + "url: chrome-error://chromewebdata/\n" + "The page navigated to https://leave.example/, which is refused: leave.example is not in the journey's allow list (add: allow leave.example).\n" + $"--- END_PAGE_CONTENT nonce={Nonce} ---\n"); Out.Should().Contain("request(s) outside the allow list were refused").And.Contain("Image https://tracker.example/pixel.png"); diff --git a/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs b/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs index 1627830..21a58a2 100644 --- a/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs +++ b/tests/Nullean.Cdp.Tests/Live/JourneyRunnerTests.cs @@ -40,10 +40,61 @@ public async Task AFailingStepStopsTheRunAndKeepsEarlierCheckpoints() result.Passed.Should().BeFalse(); result.Checkpoints.Select(c => c.Output).Should().Equal("Actions"); - result.Failure.Should().Be(new JourneyFailure(3, "click #missing", + var failure = result.Failure!; + (failure.Line, failure.Command, failure.Message).Should().Be((3, "click #missing", "Timed out after 0.2s waiting for css #missing to be visible and enabled: no element matches.")); - result.Failure!.Describe(result.Source).Should().Be( - "fail.journey:3: click #missing\nTimed out after 0.2s waiting for css #missing to be visible and enabled: no element matches."); + failure.Url.Should().Be(Site + "actions.html"); + failure.Screenshot.Should().BeNull("a screenshot is opt-in"); + failure.Describe(result.Source).Should().Be( + "fail.journey:3: click #missing\nTimed out after 0.2s waiting for css #missing to be visible and enabled: no element matches.\n" + + $"Page: {Site}actions.html"); + } + + [Test] + public async Task AFailureCarriesAScreenshotOfThePageItStoppedOn() + { + var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); + + var result = await JourneyRunner.RunAsync(journey, chrome.Browser, + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ScreenshotOnFailure = true }); + + result.Failure!.Screenshot.Should().NotBeNull().And.StartWith(new byte[] { 0x89, 0x50, 0x4E, 0x47 }, "a PNG"); + result.Failure.ScreenshotPath.Should().BeNull("there is no artifacts folder to write into"); + } + + [Test] + public async Task AScreenshotIsWrittenToTheArtifactsFolderAndNamedInTheDescription() + { + var folder = Directory.CreateTempSubdirectory("cdp-failure-").FullName; + try + { + var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); + + var result = await JourneyRunner.RunAsync(journey, chrome.Browser, + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ArtifactsDirectory = folder, ScreenshotOnFailure = true }); + + var path = Path.Combine(folder, "failure.png"); + result.Failure!.ScreenshotPath.Should().Be(path); + File.ReadAllBytes(path).Should().Equal(result.Failure.Screenshot); + result.Failure.Describe(result.Source).Should().EndWith($"\nScreenshot: {path}"); + } + finally + { + Directory.Delete(folder, recursive: true); + } + } + + [Test] + public async Task ThereIsNoScreenshotUnlessAskedFor() + { + var journey = Journey.Parse("open actions.html\nclick #missing", "fail.journey"); + + var result = await JourneyRunner.RunAsync(journey, chrome.Browser, + new JourneyOptions { BaseUrl = Site, ActionTimeout = TimeSpan.FromMilliseconds(200), ArtifactsDirectory = Path.GetTempPath() }); + + result.Failure!.Screenshot.Should().BeNull(); + result.Failure.Url.Should().Be(Site + "actions.html", "the page's address is still recorded"); + result.Failure.ScreenshotPath.Should().BeNull("an artifacts folder alone writes no failure.png"); } [Test] @@ -51,7 +102,8 @@ public async Task ARelativeOpenWithoutABaseUrlFailsAtItsStep() { var result = await JourneyRunner.RunAsync(Journey.Parse("open /x", "rel.journey"), chrome.Browser); - result.Failure.Should().Be(new JourneyFailure(1, "open /x", "rel.journey:1: '/x' is relative: give a base URL")); + var failure = result.Failure!; + (failure.Line, failure.Command, failure.Message).Should().Be((1, "open /x", "rel.journey:1: '/x' is relative: give a base URL")); } [Test] diff --git a/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs b/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs index 86b2f65..5927b61 100644 --- a/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs +++ b/tests/Nullean.Cdp.Tests/Live/SafetyLiveTests.cs @@ -22,7 +22,7 @@ public async Task ALinkOffTheAllowListFailsTheStep() { var result = await Run("open safety.html\nfind role link click --name \"Leave the site\"\nget title"); - result.Failure.Should().Be(new JourneyFailure(2, "find role link click --name \"Leave the site\"", + (result.Failure!.Line, result.Failure.Command, result.Failure.Message).Should().Be((2, "find role link click --name \"Leave the site\"", "The page navigated to https://leave.example/, which is refused: leave.example is not in the journey's allow list (add: allow leave.example).")); } @@ -97,7 +97,7 @@ public async Task APasswordFieldRefusesALiteral() { var result = await Run("open safety.html\nfill #pw hunter22"); - result.Failure.Should().Be(new JourneyFailure(2, "fill #pw hunter22", + (result.Failure!.Line, result.Failure.Command, result.Failure.Message).Should().Be((2, "fill #pw hunter22", "css #pw is a password field: fill it with a secret reference ($env:NAME), not a literal.")); }