From e46c2c0548ba46344d65db908c481a461b261305 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 17 Nov 2026 11:29:42 +0100 Subject: [PATCH 1/2] Make the precondition retry test independent of machine speed The test hung the first looks for 3 s and relied on a 400 ms navigation timeout, which a slow runner exceeded for the looks that should work. A server that is not up yet refuses connections at once, so a look fails the same way everywhere. Co-Authored-By: Claude Sonnet 5.5 --- .../Live/PreconditionTests.cs | 17 +++++++------- tests/Nullean.Cdp.Tests/Live/VersionServer.cs | 22 +++++++++++-------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs b/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs index b2a77ed..ce60ff2 100644 --- a/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs +++ b/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs @@ -121,25 +121,26 @@ public async Task AMissingHeaderAndAMissingElementAreSeenAsNothing() public async Task LooksThatFailAreRetriedAndGiveUpAfterTheRetries() { _ = chrome.Browser; - await using var flaky = new VersionServer(_ => "1.2.0", hangFirst: 2); - await using var down = new VersionServer(_ => "1.2.0", hangFirst: 100); + // Until it starts, the port refuses connections: every look fails at once, whatever the machine's speed. + await using var comingUp = new VersionServer(_ => "1.2.0", startAfter: TimeSpan.FromMilliseconds(800)); + await using var down = new VersionServer(_ => "1.2.0", startAfter: Timeout.InfiniteTimeSpan); var precondition = new JourneyPrecondition { Path = "/", Header = "x-build", Expected = "1.2.0", Timeout = TimeSpan.FromSeconds(30), - Interval = TimeSpan.FromMilliseconds(100), - Retries = 2 + Interval = TimeSpan.FromMilliseconds(300), + Retries = 5, }; - var recovered = await JourneyPreconditions.WaitAsync(chrome.Browser, precondition, Options(flaky, TimeSpan.FromMilliseconds(400))); - var gaveUp = await JourneyPreconditions.WaitAsync(chrome.Browser, precondition, Options(down, TimeSpan.FromMilliseconds(400))); + var recovered = await JourneyPreconditions.WaitAsync(chrome.Browser, precondition, Options(comingUp)); + var gaveUp = await JourneyPreconditions.WaitAsync(chrome.Browser, precondition with { Retries = 2 }, Options(down)); recovered.Passed.Should().BeTrue(recovered.Message); - recovered.Attempts.Should().Be(3); + recovered.Attempts.Should().BeGreaterThan(1); gaveUp.Passed.Should().BeFalse(); - gaveUp.Message.Should().Contain("3 looks failed, the last with:"); + gaveUp.Message.Should().Contain("3 looks failed, the last with:").And.Contain("ERR_CONNECTION_REFUSED"); gaveUp.Attempts.Should().Be(3); } diff --git a/tests/Nullean.Cdp.Tests/Live/VersionServer.cs b/tests/Nullean.Cdp.Tests/Live/VersionServer.cs index 35f131f..789d2ed 100644 --- a/tests/Nullean.Cdp.Tests/Live/VersionServer.cs +++ b/tests/Nullean.Cdp.Tests/Live/VersionServer.cs @@ -6,25 +6,30 @@ namespace Nullean.Cdp.Tests.Live; /// /// A local HTTP server whose page names a build in a header and in a meta tag, and whose build changes as requests come in: a -/// deploy that lands a moment after the run starts. The first requests can hang, as a struggling origin does. +/// deploy that lands a moment after the run starts. The server can start late, or never: until it does, the port refuses connections, as an origin that is still coming up does. /// internal sealed class VersionServer : IAsyncDisposable { private readonly HttpListener _listener = new(); private readonly Task _serving; private readonly Func _build; - private readonly int _hangFirst; private int _requests; - public VersionServer(Func build, int hangFirst = 0) + public VersionServer(Func build, TimeSpan? startAfter = null) { _build = build; - _hangFirst = hangFirst; var port = FreePort(); Url = $"http://localhost:{port}/"; _listener.Prefixes.Add(Url); - _listener.Start(); - _serving = Task.Run(ServeAsync); + _serving = Task.Run(async () => + { + if (startAfter == Timeout.InfiniteTimeSpan) + return; + if (startAfter is { } wait) + await Task.Delay(wait); + _listener.Start(); + await ServeAsync(); + }); } public string Url { get; } @@ -61,8 +66,6 @@ private async Task RespondAsync(HttpListenerContext context, int number) { try { - if (number <= _hangFirst) - await Task.Delay(TimeSpan.FromSeconds(3)); var build = _build(number); context.Response.ContentType = "text/html; charset=utf-8"; context.Response.AddHeader("x-build", build); @@ -88,7 +91,8 @@ private static int FreePort() public async ValueTask DisposeAsync() { - _listener.Stop(); + if (_listener.IsListening) + _listener.Stop(); _listener.Close(); await _serving; } From 27a51dfb2251125fe5266e1baefecfa26c145339 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 17 Nov 2026 11:32:59 +0100 Subject: [PATCH 2/2] Do not assert how many looks failed before the late server came up That depends on how fast the runner starts a context. The give-up case still asserts the exact count against a server that never starts. Co-Authored-By: Claude Sonnet 5.5 --- tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs b/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs index ce60ff2..bcb713a 100644 --- a/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs +++ b/tests/Nullean.Cdp.Tests/Live/PreconditionTests.cs @@ -121,7 +121,8 @@ public async Task AMissingHeaderAndAMissingElementAreSeenAsNothing() public async Task LooksThatFailAreRetriedAndGiveUpAfterTheRetries() { _ = chrome.Browser; - // Until it starts, the port refuses connections: every look fails at once, whatever the machine's speed. + // Until it starts, the port refuses connections: a look fails at once, whatever the machine's speed. How many looks failed before + // it came up depends on how fast the machine starts a context, so only the outcome is asserted for that server. await using var comingUp = new VersionServer(_ => "1.2.0", startAfter: TimeSpan.FromMilliseconds(800)); await using var down = new VersionServer(_ => "1.2.0", startAfter: Timeout.InfiniteTimeSpan); var precondition = new JourneyPrecondition @@ -138,7 +139,6 @@ public async Task LooksThatFailAreRetriedAndGiveUpAfterTheRetries() var gaveUp = await JourneyPreconditions.WaitAsync(chrome.Browser, precondition with { Retries = 2 }, Options(down)); recovered.Passed.Should().BeTrue(recovered.Message); - recovered.Attempts.Should().BeGreaterThan(1); gaveUp.Passed.Should().BeFalse(); gaveUp.Message.Should().Contain("3 looks failed, the last with:").And.Contain("ERR_CONNECTION_REFUSED"); gaveUp.Attempts.Should().Be(3);