From c20f358b58de3791c08bd3b732a49fe04da2a431 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 5 Oct 2026 20:50:47 +0100 Subject: [PATCH 1/6] TEST: Run the browser tests in CloakBrowser, attached to its vendor's container CloakBrowser is a patched Chromium published for browser automation. The examples have to work in it as they do in Chrome and Firefox, so it becomes a fourth browser the suite can drive. The browser binary is closed source, so the suite never downloads or starts it. It runs in the vendor's container, and BrowserDrivers.CreateCloak attaches a ChromeDriver to it at the address CLOAK_DEBUGGER_ADDRESS names. The driver has to match the Chromium inside CloakBrowser and not the Chrome on the machine, so the suite reads the version from the browser and has Selenium Manager fetch that driver, or takes the one CLOAKWEBDRIVER names. A chromedriver on the path is never used. The browser outlives the driver, so each attach moves to a new tab, closes the others and clears the cookies and the cache. A test in the Browser category proves that, and fails when the reset is taken out. Every Cloak test carries RequiresCloak, which skips it with a reason naming the variable until the variable is set, so a run that does not set it is unchanged. The skip happens before the initialize method, which matters for the Contract tests, as they start the example there. Tests added: - Browser: Cloak_RunsThePage and Cloak_StartsEachTestClean. - CloudInternal: CloakTests, the counterpart of ChromeTests. - Contract: a _Cloak test beside each existing test. The two client-side override tests emulate no device in CloakBrowser, because ChromeDriver refuses mobile emulation for a browser it attaches to, and work with the screen the browser reports for itself. - Unit tests for the address, the version, the driver lookup and the skip reason. --- BrowserCache/SessionStorageCacheTests.cs | 16 + Browsers/CloakTests.cs | 167 ++++++++++ ...ampleClientSideOverridePageContentTests.cs | 37 ++- .../ExampleClientSideOverrideTests.cs | 80 ++++- ClientSideOverrides/ExampleRenderTests.cs | 23 ++ Helpers/BrowserDrivers.cs | 293 +++++++++++++++++- Helpers/BrowserDriversCloakTests.cs | 218 +++++++++++++ Helpers/RequiresCloakAttribute.cs | 32 ++ Selenium/DriverSmokeTests.cs | 41 +++ 9 files changed, 882 insertions(+), 25 deletions(-) create mode 100644 Browsers/CloakTests.cs create mode 100644 Helpers/BrowserDriversCloakTests.cs create mode 100644 Helpers/RequiresCloakAttribute.cs diff --git a/BrowserCache/SessionStorageCacheTests.cs b/BrowserCache/SessionStorageCacheTests.cs index 0aebb0b..b9e87cd 100644 --- a/BrowserCache/SessionStorageCacheTests.cs +++ b/BrowserCache/SessionStorageCacheTests.cs @@ -160,6 +160,22 @@ public void SessionStorageCache_FireFox(bool enableCookies) RunTest(driver, enableCookies); } + /// + /// Session storage cache behaviour in CloakBrowser. + /// + [TestMethod, RequiresCloak] + [DataRow(true)] + [DataRow(false)] + public void SessionStorageCache_Cloak(bool enableCookies) + { + var options = new ChromeOptions(); + options.AcceptInsecureCertificates = true; + + driver = BrowserDrivers.CreateCloak(options); + + RunTest(driver, enableCookies); + } + private void RunTest(WebDriver driver, bool enableCookies) { ClientServerUrl = $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/"; diff --git a/Browsers/CloakTests.cs b/Browsers/CloakTests.cs new file mode 100644 index 0000000..e176cf8 --- /dev/null +++ b/Browsers/CloakTests.cs @@ -0,0 +1,167 @@ +using System; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using System.Linq; +using System.Net; +using System.Threading; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browsers +{ + /// + /// Performs the Selenium Tests of in + /// CloakBrowser, through a driver attached to the one + /// CLOAK_DEBUGGER_ADDRESS names, and is skipped when it names none. + /// Driver and HTTP server are created once per class for performance. + /// + [RequiresCloak] + [TestClass, TestCategory("CloudInternal")] + public class CloakTests : SeleniumTestsBase + { + private static WebDriver s_driver; + private static string s_clientServerUrl; + private static HttpListener s_clientServer; + private static CancellationTokenSource s_cts; + + /// + /// Initializes the test class. + /// + /// The test context. + [ClassInitialize] + public static void ClassInit(TestContext context) + { + // Start the client server once for all tests in this class. + s_clientServerUrl = + $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/"; + s_cts = new CancellationTokenSource(); + s_clientServer = TestHelpers.SimpleListener( + s_clientServerUrl, s_cts.Token); + + var options = new ChromeOptions(); + options.AcceptInsecureCertificates = true; + s_driver = BrowserDrivers.CreateCloak(options); + } + + /// + /// Initializes the test. + /// + [TestInitialize] + public override void Init() + { + base.Init(); + Driver = s_driver; + ClientServerUrl = s_clientServerUrl; + Driver.Navigate().GoToUrl(ClientServerUrl); + } + + /// + /// Cleans up after the test class. + /// + [ClassCleanup] + public static void ClassCleanup() + { + s_driver?.Quit(); + s_driver = null; + StopListener(s_clientServer, s_cts); + } + + /// + /// Test expected behavior when request contains valid license. + /// + [TestMethod] + [DynamicData( + nameof(JavaScript_Get_ValidLicense_DATA), + typeof(SeleniumTestsBase))] + public void JavaScript_Get_ValidLicense( + string endpoint, + string objectName, + string[] values, + string license, + params string[] parameters) + { + var js = TestJavaScript( + endpoint, objectName, values, license, parameters); + + AssertEveryValueIsSet(js, objectName, values); + } + + /// + /// Test expected behavior when request does not contain the required + /// license. + /// + [TestMethod] + [DynamicData( + nameof(JavaScript_Get_NoLicense_DATA), + typeof(SeleniumTestsBase))] + public void JavaScript_Get_NoLicense( + string endpoint, + string[] values, + string license, + params string[] parameters) + { + var js = TestJavaScript(endpoint, "", values, license, parameters); + + foreach (var property in values) + { + var parts = property.Split('.'); + var element = string.Join('.', parts.Take(parts.Length - 1)); + var propertyName = parts.Last(); + + var propertyExists = js.ExecuteScript( + $"return fod.{element}.hasOwnProperty('{propertyName}');"); + Assert.IsTrue((bool)propertyExists); + + var value = js.ExecuteScript( + $"return fod.{element}.{propertyName};"); + Assert.IsNotNull( + value, $"fod.{element}.{propertyName} is null"); + + Assert.IsTrue(value.ToString().Contains( + $"{propertyName} is a paid feature.", + StringComparison.OrdinalIgnoreCase)); + } + } + + /// + /// Test expected behavior when the request asks for properties that + /// evidence gathered in the browser overrides. + /// + [TestMethod] + [DynamicData( + nameof(JavaScript_Get_ClientSideOverrides_DATA), + typeof(SeleniumTestsBase))] + public void JavaScript_Get_ClientSideOverrides( + string endpoint, + string objectName, + string[] values, + string license, + params string[] parameters) + { + var js = TestJavaScript( + endpoint, objectName, values, license, parameters); + + AssertEveryValueIsSet(js, objectName, values); + } + + /// + /// Checks the object the include made holds a value for every + /// property that was asked for. + /// + /// Runs script in the page. + /// Name of the object in the page. + /// Properties, each as element.property. + private static void AssertEveryValueIsSet( + IJavaScriptExecutor js, string objectName, string[] values) + { + foreach (var property in values) + { + var value = js.ExecuteScript( + $"return {objectName}.{property};"); + Assert.IsNotNull(value, $"{objectName}.{property} is null"); + } + } + } +} diff --git a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs index efaaaa8..0161f5e 100644 --- a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs @@ -104,16 +104,49 @@ public void Example_RendersClientSideOverridesOnThePage( _driver = BrowserDrivers.CreateChrome(chromeOptions); + AssertRendersScreenSize(width, height); + } + + /// + /// The same in CloakBrowser, with the screen it reports for itself. + /// No device is emulated, as ChromeDriver refuses mobile emulation + /// for a browser it attaches to, so the size to expect is read from + /// the browser. + /// + [TestMethod, RequiresCloak] + public void Example_RendersClientSideOverridesOnThePage_Cloak() + { + var chromeOptions = new ChromeOptions(); + chromeOptions.AcceptInsecureCertificates = true; + + _driver = BrowserDrivers.CreateCloak(chromeOptions); + + var js = (IJavaScriptExecutor)_driver; + var width = js.ExecuteScript("return window.screen.width"); + var height = js.ExecuteScript("return window.screen.height"); + + AssertRendersScreenSize( + Convert.ToInt64(width), Convert.ToInt64(height)); + } + + /// + /// Loads the example and checks the client-side results it renders + /// against the size of the browser's screen. + /// + private void AssertRendersScreenSize(long width, long height) + { _driver.Navigate().GoToUrl(_proxyUrl); var rendered = WaitForRenderedResults(); Assert.AreEqual( width.ToString(), rendered[ScreenWidthLabel], - "the screen width rendered on the page does not match the emulated width"); + "the screen width rendered on the page does not match the " + + "width of the browser's screen"); Assert.AreEqual( height.ToString(), rendered[ScreenHeightLabel], - "the screen height rendered on the page does not match the emulated height"); + "the screen height rendered on the page does not match the " + + "height of the browser's screen"); // The device id is only rendered by examples whose copy of the // shared helper is current. An example still carrying an older copy diff --git a/ClientSideOverrides/ExampleClientSideOverrideTests.cs b/ClientSideOverrides/ExampleClientSideOverrideTests.cs index 3876b91..5f94fe8 100644 --- a/ClientSideOverrides/ExampleClientSideOverrideTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverrideTests.cs @@ -89,6 +89,51 @@ public void Example_ServesCoreJs_AndClientSideOverridesFlow( _driver = BrowserDrivers.CreateChrome(chromeOptions); + var overrides = ReadOverrides(); + + Assert.AreEqual( + width, overrides.ScreenWidth, + "emulated screen width does not match"); + Assert.AreEqual( + height, overrides.ScreenHeight, + "emulated screen height does not match"); + + AssertOverridesResolved(overrides); + } + + /// + /// The same in CloakBrowser, with the screen it reports for itself. + /// No device is emulated, as ChromeDriver refuses mobile emulation + /// for a browser it attaches to. + /// + [TestMethod, RequiresCloak] + public void Example_ServesCoreJs_AndClientSideOverridesFlow_Cloak() + { + var chromeOptions = new ChromeOptions(); + chromeOptions.AcceptInsecureCertificates = true; + + _driver = BrowserDrivers.CreateCloak(chromeOptions); + + AssertOverridesResolved(ReadOverrides()); + } + + /// + /// The browser's screen size, and what the flow data holds once the + /// client-side callback has run. + /// + private sealed record Overrides( + long ScreenWidth, + long ScreenHeight, + long ScreenPixelsWidth, + long ScreenPixelsHeight, + string DeviceId); + + /// + /// Loads the example and waits for the client-side evidence to come + /// back as flow data. + /// + private Overrides ReadOverrides() + { _driver.Navigate().GoToUrl(_proxyUrl); IJavaScriptExecutor js = _driver; @@ -111,20 +156,31 @@ public void Example_ServesCoreJs_AndClientSideOverridesFlow( _ => "complete".Equals(js.ExecuteScript("return window.__t")) && !"loading".Equals(js.ExecuteScript("return window.__did"))); - var screenWidth = Convert.ToInt64(js.ExecuteScript("return window.__sw")); - var screenHeight = Convert.ToInt64(js.ExecuteScript("return window.__sh")); - var screenPixelWidth = Convert.ToInt64(js.ExecuteScript("return window.__spw")); - var screenPixelHeight = Convert.ToInt64(js.ExecuteScript("return window.__sph")); - var deviceId = js.ExecuteScript("return window.__did")?.ToString(); - - Assert.AreEqual(width, screenWidth, "emulated screen width does not match"); - Assert.AreEqual(height, screenHeight, "emulated screen height does not match"); + return new Overrides( + Convert.ToInt64(js.ExecuteScript("return window.__sw")), + Convert.ToInt64(js.ExecuteScript("return window.__sh")), + Convert.ToInt64(js.ExecuteScript("return window.__spw")), + Convert.ToInt64(js.ExecuteScript("return window.__sph")), + js.ExecuteScript("return window.__did")?.ToString()); + } - Assert.IsTrue(screenPixelWidth > 0, "screenpixelswidth override did not resolve"); - Assert.IsTrue(screenPixelHeight > 0, "screenpixelsheight override did not resolve"); + /// + /// The screen size and the device id came back from detection. + /// + private static void AssertOverridesResolved(Overrides overrides) + { + Assert.IsTrue( + overrides.ScreenPixelsWidth > 0, + "screenpixelswidth override did not resolve"); + Assert.IsTrue( + overrides.ScreenPixelsHeight > 0, + "screenpixelsheight override did not resolve"); - Assert.IsFalse(string.IsNullOrEmpty(deviceId), "device id was not resolved"); - Assert.AreNotEqual("loading", deviceId, "device id was not resolved"); + Assert.IsFalse( + string.IsNullOrEmpty(overrides.DeviceId), + "device id was not resolved"); + Assert.AreNotEqual( + "loading", overrides.DeviceId, "device id was not resolved"); } /// Quits the browser and stops the example app. diff --git a/ClientSideOverrides/ExampleRenderTests.cs b/ClientSideOverrides/ExampleRenderTests.cs index ace05c1..e772b36 100644 --- a/ClientSideOverrides/ExampleRenderTests.cs +++ b/ClientSideOverrides/ExampleRenderTests.cs @@ -91,6 +91,29 @@ public void Example_RendersRealDetectionResult() _driver = BrowserDrivers.CreateChrome(chromeOptions); + AssertRendersRealDetectionResult(); + } + + /// + /// The same in CloakBrowser, which presents itself as a desktop + /// browser without being given a user agent. + /// + [TestMethod, RequiresCloak] + public void Example_RendersRealDetectionResult_Cloak() + { + var chromeOptions = new ChromeOptions(); + chromeOptions.AcceptInsecureCertificates = true; + + _driver = BrowserDrivers.CreateCloak(chromeOptions); + + AssertRendersRealDetectionResult(); + } + + /// + /// Loads the example and checks the page it renders server-side. + /// + private void AssertRendersRealDetectionResult() + { _driver.Navigate().GoToUrl(_example.BaseUrl); // The detection table is rendered server-side on the first response, diff --git a/Helpers/BrowserDrivers.cs b/Helpers/BrowserDrivers.cs index 481fd59..81529eb 100644 --- a/Helpers/BrowserDrivers.cs +++ b/Helpers/BrowserDrivers.cs @@ -1,11 +1,15 @@ using System; using System.Collections.Generic; using System.IO; +using System.Net.Http; using System.Runtime.InteropServices; +using System.Text.Json; +using System.Threading; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; using OpenQA.Selenium.Edge; using OpenQA.Selenium.Firefox; +using OpenQA.Selenium.Manager; using OpenQA.Selenium.Remote; namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers @@ -38,6 +42,13 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers /// calling Selenium Manager at all, which is the behaviour the first case /// relies on. /// + /// + /// CloakBrowser is the exception, as it is never started here. It runs in + /// its vendor's container, and a driver attaches to it at the address + /// CLOAK_DEBUGGER_ADDRESS names. That driver has to match CloakBrowser's + /// version of Chromium, which is not the version of the Chrome on this + /// machine, so a driver found on the path is never used for it. + /// /// public static class BrowserDrivers { @@ -74,6 +85,32 @@ public static class BrowserDrivers /// Path to the Edge binary to drive. public const string EdgeBinaryVariable = "EDGE_BIN"; + /// + /// Address of a running CloakBrowser to attach to, as host:port, for + /// example 127.0.0.1:9222. The Cloak tests are skipped without it. + /// + public const string CloakDebuggerAddressVariable = + "CLOAK_DEBUGGER_ADDRESS"; + + /// + /// Directory holding a chromedriver of CloakBrowser's major version, + /// or the driver itself. Selenium Manager fetches one when it is + /// unset. + /// + public const string CloakDriverVariable = "CLOAKWEBDRIVER"; + + /// + /// The reason a Cloak test is skipped, naming the variable to set. + /// + public const string CloakNotConfiguredMessage = + "No CloakBrowser to attach to. Set " + + CloakDebuggerAddressVariable + " to the host and port of one " + + "that is running, for example 127.0.0.1:9222. The README says " + + "how to start one."; + + // Long enough for the first request to start the browser. + private const int CloakVersionTimeoutSeconds = 60; + /// File names a Chrome driver goes by. public static readonly string[] ChromeDriverNames = { "chromedriver", "chromedriver.exe" }; @@ -201,6 +238,227 @@ public static WebDriver CreateEdge(EdgeOptions options) "Edge", EdgeDriverVariable, EdgeDriverNames); } + /// + /// True when CLOAK_DEBUGGER_ADDRESS names a CloakBrowser to attach to. + /// + public static bool IsCloakConfigured => + CloakDebuggerAddress(Environment.GetEnvironmentVariable) != null; + + /// + /// Attaches a driver to the running CloakBrowser that + /// CLOAK_DEBUGGER_ADDRESS names. + /// + /// + /// + /// The browser was started by its container and outlives the driver, + /// so arguments in never reach it, and + /// SELENIUM_URL does not apply. Mobile emulation cannot be asked for + /// either, as ChromeDriver refuses it for a browser it did not start. + /// + /// + /// For the same reason the browser keeps its tabs, cookies and cache + /// from one test to the next. Each attach moves to a new tab, closes + /// the others and clears the cookies and the cache, so a test starts + /// as it would in a browser started for it. + /// + /// + /// Options for the driver. + /// A driver, which the caller quits. + public static WebDriver CreateCloak(ChromeOptions options) => + CreateCloak(Environment.GetEnvironmentVariable, options); + + /// + /// As , reading the variables + /// through so the rules can be tested + /// without changing the environment of the whole run. + /// + /// Reads an environment variable. + /// Options for the driver. + /// A driver, which the caller quits. + public static WebDriver CreateCloak( + Func getVariable, ChromeOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + var address = CloakDebuggerAddress(getVariable); + if (address == null) + { + throw new WebDriverException(CloakNotConfiguredMessage); + } + var version = CloakMajorVersion(ReadCloakVersion(address)); + options.DebuggerAddress = address; + var driverPath = FindNamedDriver( + getVariable, CloakDriverVariable, ChromeDriverNames); + ChromeDriver driver; + try + { + driver = new ChromeDriver( + ChromeDriverService.CreateDefaultService( + driverPath ?? FetchChromeDriver(version)), + options); + } + catch (Exception e) + { + throw new WebDriverException( + $"Could not attach a driver to CloakBrowser at {address}, " + + $"the address {CloakDebuggerAddressVariable} names. It " + + $"is Chromium {version} and needs chromedriver " + + $"{version}, which was " + + (driverPath != null + ? $"taken from {driverPath}" + : "left to Selenium Manager to fetch") + + $". Set {CloakDriverVariable} to a chromedriver " + + $"{version} to name one. Selenium said: {e.Message}", + e); + } + try + { + StartClean(driver); + } + catch + { + driver.Quit(); + throw; + } + return driver; + } + + /// + /// The address CLOAK_DEBUGGER_ADDRESS names, or null when it is unset + /// or empty, which is what a CI script that exports an unset variable + /// produces. + /// + /// Reads an environment variable. + /// The address as host:port, or null. + public static string CloakDebuggerAddress( + Func getVariable) + { + if (getVariable == null) + { + throw new ArgumentNullException(nameof(getVariable)); + } + var address = getVariable(CloakDebuggerAddressVariable); + return string.IsNullOrWhiteSpace(address) ? null : address.Trim(); + } + + /// + /// The major version of the browser, read from the document its + /// DevTools port serves at /json/version, where the Browser field + /// reads like Chrome/146.0.7680.177. + /// + /// Body of /json/version. + /// The major version, for example 146. + public static string CloakMajorVersion(string versionDocument) + { + string browser = null; + try + { + using var document = JsonDocument.Parse(versionDocument); + if (document.RootElement.ValueKind == JsonValueKind.Object + && document.RootElement.TryGetProperty( + "Browser", out var field) + && field.ValueKind == JsonValueKind.String) + { + browser = field.GetString(); + } + } + catch (Exception e) when ( + e is JsonException || e is ArgumentNullException) + { + // Reported below, with what was received. + } + var slash = browser?.IndexOf('/') ?? -1; + var major = slash < 0 + ? null + : browser.Substring(slash + 1).Split('.')[0]; + if (string.IsNullOrEmpty(major) + || int.TryParse(major, out _) == false) + { + throw new WebDriverException( + $"The address {CloakDebuggerAddressVariable} names did " + + "not answer /json/version with a browser version, so " + + "it is not the DevTools port of a browser. It " + + $"answered: {versionDocument}"); + } + return major; + } + + /// + /// Body of /json/version from the browser at + /// . The first request is what makes the + /// container start the browser. + /// + /// Address as host:port. + /// The document, as JSON. + private static string ReadCloakVersion(string address) + { + try + { + using var client = new HttpClient + { + Timeout = TimeSpan.FromSeconds(CloakVersionTimeoutSeconds), + }; + return client.GetStringAsync( + new Uri($"http://{address}/json/version")) + .GetAwaiter().GetResult(); + } + catch (Exception e) + { + throw new WebDriverException( + $"No browser answered at {address}, the address " + + $"{CloakDebuggerAddressVariable} names, which has to " + + "be the host and port of a running CloakBrowser, for " + + "example 127.0.0.1:9222. The README says how to start " + + $"one. The request said: {e.Message}", + e); + } + } + + /// + /// Path to a chromedriver for Chrome of the given major version, + /// which Selenium Manager fetches unless it already holds one. + /// + /// + /// Selenium Manager is asked directly, and only the driver it finds + /// is used. Left to itself, Selenium also wants a Chrome on this + /// machine, and fails on a machine that has none when + /// SE_AVOID_BROWSER_DOWNLOAD stops it downloading one. + /// + /// Major version of the browser. + /// Path to the driver. + private static string FetchChromeDriver(string version) + { + var found = SeleniumManager.DiscoverBrowserAsync( + "chrome", + new BrowserDiscoveryOptions { BrowserVersion = version }, + CancellationToken.None).GetAwaiter().GetResult(); + return found.DriverPath; + } + + /// + /// Leaves an attached browser on one new tab with no cookies and an + /// empty cache. Session storage belongs to a tab, so the new tab + /// clears that too. + /// + /// A driver attached to the browser. + private static void StartClean(ChromeDriver driver) + { + var stale = driver.WindowHandles; + driver.SwitchTo().NewWindow(WindowType.Tab); + var fresh = driver.CurrentWindowHandle; + foreach (var handle in stale) + { + driver.SwitchTo().Window(handle); + driver.Close(); + } + driver.SwitchTo().Window(fresh); + var none = new Dictionary(); + driver.ExecuteCdpCommand("Network.clearBrowserCookies", none); + driver.ExecuteCdpCommand("Network.clearBrowserCache", none); + } + /// /// Runs , which leaves Selenium Manager to /// fetch a driver, and on failure says what was looked for here first, @@ -280,25 +538,38 @@ public static string FindDriver( Func getVariable, string variable, IEnumerable names) + { + return FindNamedDriver(getVariable, variable, names) + ?? FindOnPath(getVariable, names); + } + + /// + /// Path to the driver names, as the driver + /// itself or the directory holding it, or null. The path is not + /// searched, which suits a driver that must not be the one the + /// machine keeps for its own browser. + /// + /// Reads an environment variable. + /// Variable that names the driver. + /// File names to look for in a directory. + /// A path, or null. + public static string FindNamedDriver( + Func getVariable, + string variable, + IEnumerable names) { if (getVariable == null) { throw new ArgumentNullException(nameof(getVariable)); } var configured = getVariable(variable); - if (string.IsNullOrEmpty(configured) == false) + if (string.IsNullOrEmpty(configured)) { - if (File.Exists(configured)) - { - return configured; - } - var inDirectory = FindInDirectory(configured, names); - if (inDirectory != null) - { - return inDirectory; - } + return null; } - return FindOnPath(getVariable, names); + return File.Exists(configured) + ? configured + : FindInDirectory(configured, names); } /// diff --git a/Helpers/BrowserDriversCloakTests.cs b/Helpers/BrowserDriversCloakTests.cs new file mode 100644 index 0000000..6ca9ea9 --- /dev/null +++ b/Helpers/BrowserDriversCloakTests.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.IO; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Tests +{ + /// + /// Checks the rules for attaching to CloakBrowser. No browser is started + /// and none has to be running, so these run anywhere. + /// + [TestClass] + public class BrowserDriversCloakTests + { + // What the DevTools port of CloakBrowser 0.5.12 serves at + // /json/version, less the fields that are not read. + private const string VersionDocument = + "{\"Browser\": \"Chrome/146.0.7680.177\", " + + "\"Protocol-Version\": \"1.3\", " + + "\"webSocketDebuggerUrl\": \"ws://127.0.0.1:9222/devtools/" + + "browser/00000000-0000-0000-0000-000000000000\"}"; + + private string _directory; + private string _driverFile; + + /// + /// Makes a directory holding a file named like a driver. + /// + [TestInitialize] + public void Init() + { + _directory = Path.Combine( + Path.GetTempPath(), "51d-cloak-driver-" + Guid.NewGuid()); + Directory.CreateDirectory(_directory); + _driverFile = Path.Combine(_directory, "chromedriver"); + File.WriteAllText(_driverFile, string.Empty); + } + + /// Removes the directory. + [TestCleanup] + public void Cleanup() + { + if (_directory != null && Directory.Exists(_directory)) + { + Directory.Delete(_directory, true); + } + } + + /// + /// An unset variable and an empty one both mean there is no browser + /// to attach to, as a CI script that exports an unset variable + /// produces an empty one. + /// + [TestMethod] + [DataRow(null)] + [DataRow("")] + [DataRow(" ")] + public void CloakDebuggerAddress_UnsetOrEmpty_IsNull(string value) + { + Assert.IsNull(BrowserDrivers.CloakDebuggerAddress(Lookup( + BrowserDrivers.CloakDebuggerAddressVariable, value))); + } + + /// The address is read from the variable. + [TestMethod] + public void CloakDebuggerAddress_Set_IsTheAddress() + { + Assert.AreEqual( + "127.0.0.1:9222", + BrowserDrivers.CloakDebuggerAddress(Lookup( + BrowserDrivers.CloakDebuggerAddressVariable, + " 127.0.0.1:9222 "))); + } + + /// + /// With no address there is nothing to attach to, and the failure + /// names the variable that would supply one. + /// + [TestMethod] + public void CreateCloak_NoAddress_NamesTheVariable() + { + var e = Assert.ThrowsExactly( + () => BrowserDrivers.CreateCloak( + Lookup(new Dictionary()), + new ChromeOptions())); + + Assert.AreEqual( + BrowserDrivers.CloakNotConfiguredMessage, e.Message); + StringAssert.Contains( + e.Message, BrowserDrivers.CloakDebuggerAddressVariable); + } + + /// + /// An address nothing listens at fails before any driver is looked + /// for, and the failure names the variable and the address. + /// + [TestMethod] + public void CreateCloak_NothingAtTheAddress_NamesTheVariable() + { + var address = $"127.0.0.1:{TestHelpers.GetRandomUnusedPort()}"; + + var e = Assert.ThrowsExactly( + () => BrowserDrivers.CreateCloak( + Lookup( + BrowserDrivers.CloakDebuggerAddressVariable, address), + new ChromeOptions())); + + StringAssert.Contains( + e.Message, BrowserDrivers.CloakDebuggerAddressVariable); + StringAssert.Contains(e.Message, address); + } + + /// + /// The major version comes from the browser itself, so the driver + /// asked for is always the one that browser needs. + /// + [TestMethod] + public void CloakMajorVersion_ReadsTheBrowserField() + { + Assert.AreEqual( + "146", BrowserDrivers.CloakMajorVersion(VersionDocument)); + } + + /// + /// An answer that is not a browser's says so and names the variable, + /// which is what a wrong port gives. + /// + [TestMethod] + [DataRow("{}")] + [DataRow("{\"Browser\": \"Chrome\"}")] + [DataRow("{\"Browser\": 146}")] + [DataRow("[]")] + [DataRow("not a browser")] + [DataRow("")] + [DataRow(null)] + public void CloakMajorVersion_NotABrowser_NamesTheVariable( + string document) + { + var e = Assert.ThrowsExactly( + () => BrowserDrivers.CloakMajorVersion(document)); + + StringAssert.Contains( + e.Message, BrowserDrivers.CloakDebuggerAddressVariable); + } + + /// + /// A variable naming the directory the driver is in finds it. + /// + [TestMethod] + public void FindNamedDriver_ReadsDirectoryFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindNamedDriver( + Lookup(BrowserDrivers.CloakDriverVariable, _directory), + BrowserDrivers.CloakDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// A variable naming the driver itself finds it. + [TestMethod] + public void FindNamedDriver_ReadsFileFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindNamedDriver( + Lookup(BrowserDrivers.CloakDriverVariable, _driverFile), + BrowserDrivers.CloakDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// A chromedriver on the path is not used, as it is the one for the + /// machine's own Chrome and not for CloakBrowser's version. + /// + [TestMethod] + public void FindNamedDriver_DoesNotSearchThePath() + { + Assert.IsNull( + BrowserDrivers.FindNamedDriver( + Lookup("PATH", _directory), + BrowserDrivers.CloakDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// The reason a Cloak test gives for being skipped names the + /// variable to set. + /// + [TestMethod] + public void RequiresCloak_SkipReason_NamesTheVariable() + { + StringAssert.Contains( + new RequiresCloakAttribute().IgnoreMessage, + BrowserDrivers.CloakDebuggerAddressVariable); + } + + /// Reads one variable in place of the environment. + /// Name of the variable. + /// Its value, or null for unset. + /// A lookup holding only that variable. + private static Func Lookup(string name, string value) => + Lookup(new Dictionary { { name, value } }); + + /// + /// Reads from a dictionary in place of the environment. + /// + /// Variables and their values. + /// A lookup over those values. + private static Func Lookup( + IReadOnlyDictionary values) => + name => values.TryGetValue(name, out var value) ? value : null; + } +} diff --git a/Helpers/RequiresCloakAttribute.cs b/Helpers/RequiresCloakAttribute.cs new file mode 100644 index 0000000..e1618f3 --- /dev/null +++ b/Helpers/RequiresCloakAttribute.cs @@ -0,0 +1,32 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers +{ + /// + /// Skips a test, or every test in a class, when CLOAK_DEBUGGER_ADDRESS + /// names no CloakBrowser to attach to, and gives the variable to set as + /// the reason. + /// + /// + /// The test is skipped before its initialize method runs, so no example + /// is started for a test that could not use it. + /// + [AttributeUsage( + AttributeTargets.Class | AttributeTargets.Method, Inherited = false)] + public sealed class RequiresCloakAttribute : ConditionBaseAttribute + { + /// Runs the test only where the condition is met. + public RequiresCloakAttribute() + : base(ConditionMode.Include) + { + IgnoreMessage = BrowserDrivers.CloakNotConfiguredMessage; + } + + /// + public override bool IsConditionMet => BrowserDrivers.IsCloakConfigured; + + /// + public override string GroupName => nameof(RequiresCloakAttribute); + } +} diff --git a/Selenium/DriverSmokeTests.cs b/Selenium/DriverSmokeTests.cs index 1299945..a53f9b1 100644 --- a/Selenium/DriverSmokeTests.cs +++ b/Selenium/DriverSmokeTests.cs @@ -28,6 +28,11 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium /// run that quietly skipped them would say nothing, and saying nothing is /// the fault these were written for. /// + /// + /// CloakBrowser is not started by the suite, so its tests are skipped + /// until CLOAK_DEBUGGER_ADDRESS names one that is running. Once it does, + /// they fail like the others when the browser cannot be reached. + /// /// [TestClass, TestCategory("Browser")] public class DriverSmokeTests @@ -127,6 +132,42 @@ public void Edge_RunsThePage() AssertPageRan(); } + /// + /// The CloakBrowser that CLOAK_DEBUGGER_ADDRESS names runs the page. + /// + [TestMethod, RequiresCloak] + public void Cloak_RunsThePage() + { + _driver = BrowserDrivers.CreateCloak(new ChromeOptions()); + AssertPageRan(); + } + + /// + /// CloakBrowser outlives the driver, so what one test leaves in it + /// must not reach the next. A second driver finds neither the cookie + /// nor the session storage the first one left on the same page. + /// + [TestMethod, RequiresCloak] + public void Cloak_StartsEachTestClean() + { + _driver = BrowserDrivers.CreateCloak(new ChromeOptions()); + _driver.Navigate().GoToUrl(_url); + ((IJavaScriptExecutor)_driver).ExecuteScript( + "document.cookie = 'left=behind; path=/';" + + "sessionStorage.setItem('left', 'behind');"); + _driver.Quit(); + + _driver = BrowserDrivers.CreateCloak(new ChromeOptions()); + _driver.Navigate().GoToUrl(_url); + var found = ((IJavaScriptExecutor)_driver).ExecuteScript( + "return 'cookie [' + document.cookie + '], session storage ['" + + " + (sessionStorage.getItem('left') || '') + ']';"); + + Assert.AreEqual( + "cookie [], session storage []", found, + "the browser kept what the driver before this one left"); + } + /// /// Loads the page and checks the browser ran the script on it. /// From fc6b5be5654694882c7367f597ca54bf76bee5c6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 5 Oct 2026 20:51:03 +0100 Subject: [PATCH 2/6] TEST: Start CloakBrowser in this repository's CI and run the Browser category in it A third job starts the vendor's image on ubuntu-latest and runs the Browser category with CLOAK_DEBUGGER_ADDRESS set, so the wiring is proved on every pull request before a language repository relies on it. The image is named by digest, so the browser cannot change under the tag, and its update check is turned off, so the browser that runs is the one in the image. Nothing from the runner is mounted into the container and no secret is passed to it. Host networking lets the browser reach the page the tests serve on localhost. The job looks for the pass of Cloak_RunsThePage by name, because a skipped Cloak test would otherwise leave it green having attached to nothing. --- .github/workflows/build-and-test.yml | 63 +++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 657775a..25f1174 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,11 +1,12 @@ # Builds the suite and runs it, so a change here is checked before a language # repository picks it up. # -# Two jobs, because they answer different questions. The first runs the tests +# Three jobs, because they answer different questions. The first runs the tests # that need no browser, no cloud and no keys. The second starts a real browser # on each runner the language repositories use, including the ARM64 Linux ones, # and fails when one will not start, which is the fault the second job exists -# to catch. +# to catch. The third starts CloakBrowser in its vendor's container and runs +# the same check with a driver attached to it. name: Build and test on: @@ -72,3 +73,61 @@ jobs: - name: Start a browser run: dotnet test -c Release --no-build --filter "TestCategory=Browser" + + cloak: + name: CloakBrowser on ubuntu-latest + runs-on: ubuntu-latest + env: + # The vendor's image by digest, so the browser cannot change under the + # tag. The browser is closed source and only ever runs in this container. + CLOAK_IMAGE: cloakhq/cloakbrowser:0.5.12@sha256:2fdd1289154f30594c7bab74943ea7baa6b617a9ec026ab4bf339c2ebabcb712 + CLOAK_DEBUGGER_ADDRESS: 127.0.0.1:9222 + # Selenium Manager fetches the driver for the browser in the container + # and would otherwise download a Chrome of that version that nothing + # runs. + SE_AVOID_BROWSER_DOWNLOAD: 'true' + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + # Host networking lets the browser reach the page the tests serve on + # localhost. Update checks are off, so the browser that runs is the one + # in the image. Nothing from the runner is mounted and no secret is + # passed. The first request for the version starts the browser, and + # its answer is printed so the log says which browser ran. + - name: Start CloakBrowser + run: | + docker run -d --name cloak --network host \ + -e CLOAKBROWSER_AUTO_UPDATE=false \ + "$CLOAK_IMAGE" cloakserve + for attempt in $(seq 1 30); do + if curl -fsS "http://$CLOAK_DEBUGGER_ADDRESS/json/version"; then + echo + exit 0 + fi + sleep 2 + done + echo "CloakBrowser did not answer on $CLOAK_DEBUGGER_ADDRESS" + docker logs cloak + exit 1 + + - name: Build + run: dotnet build -c Release + + # A Cloak test that was skipped would leave this job green having + # attached to nothing, so the pass is looked for by name. + - name: Start a browser + run: | + set -o pipefail + dotnet test -c Release --no-build --filter "TestCategory=Browser" \ + --logger "console;verbosity=normal" | tee browser-tests.log + grep -q "Passed Cloak_RunsThePage" browser-tests.log + + - name: Stop CloakBrowser + if: always() + run: | + docker logs cloak || true + docker rm -f cloak || true From a921dba097497b818a1f3ddc0d29f9cdd7f91298 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 5 Oct 2026 20:51:04 +0100 Subject: [PATCH 3/6] DOC: Say how to run the suite under CloakBrowser Adds CLOAK_DEBUGGER_ADDRESS and CLOAKWEBDRIVER to the configuration table, and a section on starting the browser in its vendor's container, pinned by digest, on Linux and with Docker Desktop on Windows. It says how the driver is chosen, what differs for a test when the driver attaches to a browser it did not start, and which version of the binary's license applies, linked at the commit that holds that version. --- README.md | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5bdbf4a..8f16890 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,10 @@ Select a subset with `--filter TestCategory=Contract` or - **Browser** - a driver the machine already provides is used first, named by `CHROMEWEBDRIVER`, `GECKOWEBDRIVER` or `EDGEWEBDRIVER` or found on the path. When the machine provides none, Selenium Manager fetches one. Set - `SELENIUM_URL` to drive a browser on a Selenium grid instead. + `SELENIUM_URL` to drive a browser on a Selenium grid instead. CloakBrowser + is never started by the suite, which attaches to one already running in a + container when `CLOAK_DEBUGGER_ADDRESS` names it. See + [Running under CloakBrowser](#running-under-cloakbrowser). ## Configuration @@ -50,6 +53,8 @@ and no keys are committed. | `SELENIUM_URL` | optional | Selenium grid URL, omit to drive a browser on this machine. | | `CHROMEWEBDRIVER` / `GECKOWEBDRIVER` / `EDGEWEBDRIVER` | optional | A driver, or the directory holding one. GitHub's Linux runner images set these. | | `CHROME_BIN` / `FIREFOX_BIN` / `EDGE_BIN` | optional | The browser binary to drive, when it is not on the path. | +| `CLOAK_DEBUGGER_ADDRESS` | optional | Host and port of a running CloakBrowser to attach to, e.g. `127.0.0.1:9222`. The Cloak tests are skipped when it is unset. | +| `CLOAKWEBDRIVER` | optional | A chromedriver of the same major version as CloakBrowser, or the directory holding one. Selenium Manager fetches one when it is unset. | | `EXAMPLE_URL` / `EXAMPLE_LANG` | `Contract` | The example app to test (CI / local). | | `51DEGREES_CLOUD_ENDPOINT` / `51DEGREES_RESOURCE_KEY` | `Browser51Did` | Handed to the demo under these names, the ones every language's demo reads first. | | `_51DEGREES_RESOURCE_KEY_51DID` | `Browser51Did` | Read where `51DEGREES_RESOURCE_KEY` is unset. The name CI sets, for a resource key carrying the 51Did product. | @@ -105,14 +110,103 @@ export ENTERPRISE_V4_LICENSE="" dotnet test --filter TestCategory=CloudInternal ``` +## Running under CloakBrowser + +[CloakBrowser](https://github.com/CloakHQ/CloakBrowser) is a patched Chromium +that its vendor publishes for browser automation. Visitors arrive in browsers +like it as well as in Chrome and Firefox, so the tests of the examples can be +run in it too. + +The tests that use it are the two whose names start with `Cloak_` in +`Browser`, the `CloakTests` class in `CloudInternal`, and the tests whose +names end in `_Cloak` in `Contract`. They are skipped, with a reason that +names the variable, until `CLOAK_DEBUGGER_ADDRESS` is set, so a run that does +not set it is unchanged. + +### Starting the browser + +The CloakBrowser binary is closed source, so the suite never downloads it and +never starts it. It runs only inside the vendor's Docker image, and the suite +attaches a driver to it through the DevTools port the container serves. Pin +the image by digest, so that the browser cannot change under a tag. Mount +nothing from the machine into the container and pass it no keys. + +On a Linux machine, such as a CI runner, use host networking, so that the +browser reaches the example and the pages the tests serve on `localhost`. + +```bash +docker run -d --name cloak --network host \ + -e CLOAKBROWSER_AUTO_UPDATE=false \ + cloakhq/cloakbrowser:0.5.12@sha256:2fdd1289154f30594c7bab74943ea7baa6b617a9ec026ab4bf339c2ebabcb712 \ + cloakserve +export CLOAK_DEBUGGER_ADDRESS="127.0.0.1:9222" +``` + +Host networking opens the DevTools port on every network interface of the +machine, and whoever can reach that port controls the browser. Use it only on +a machine that takes no connections from outside, which a hosted CI runner is. + +With Docker Desktop on Windows, publish the port on the loopback address +instead, and tell the browser that `localhost` is the machine Docker Desktop +runs on. + +```bash +docker run -d --name cloak -p 127.0.0.1:9222:9222 \ + -e CLOAKBROWSER_AUTO_UPDATE=false \ + cloakhq/cloakbrowser:0.5.12@sha256:2fdd1289154f30594c7bab74943ea7baa6b617a9ec026ab4bf339c2ebabcb712 \ + cloakserve "--host-resolver-rules=MAP localhost host.docker.internal" +export CLOAK_DEBUGGER_ADDRESS="127.0.0.1:9222" +``` + +`CLOAKBROWSER_AUTO_UPDATE=false` stops the container looking for a newer +browser when it starts, so the browser that runs is the one in the pinned +image. Remove the container with `docker rm -f cloak` when the run is over. + +### The driver + +The driver is ChromeDriver, and it has to be of the same major version as the +Chromium inside CloakBrowser, which is 146 in the image above and is not the +version of the Chrome on the machine. The suite asks the browser for its +version and has Selenium Manager fetch that driver, so there is nothing to +install, and a chromedriver on the path is never used. Set `CLOAKWEBDRIVER` +to name a driver where Selenium Manager cannot fetch one. + +Selenium Manager also downloads a Chrome of that version when the machine has +none, and nothing runs it. Set `SE_AVOID_BROWSER_DOWNLOAD=true` to stop that. + +### What is different for a test + +The browser was started by the container and not by the driver, which changes +three things. + +- Arguments in the options, such as `--headless` or `--user-agent`, never + reach the browser. It sends its own user agent, and it runs with a window on + the container's virtual display. +- ChromeDriver refuses mobile emulation for a browser it attaches to. The + `_Cloak` tests of the client-side overrides therefore emulate no device, and + work with the screen size the browser reports for itself. +- The browser outlives the test. Each attach moves to a new tab, closes the + others and clears the cookies and the cache, so a test starts as it would in + a browser started for it. + +### License + +The browser binary has its own license, which is separate from the MIT +license of the vendor's wrapper code. The image above carries the vendor's +free build under version 1.3 of the CloakBrowser Binary License, dated July +2026. [Read that version](https://github.com/CloakHQ/CloakBrowser/blob/2e379e402b52838d37e469a40e6195e887e314d1/BINARY-LICENSE.md) +before running it. The suite uses no account and no license key. + ## This repository's own CI The "Build and test" workflow builds the suite on every push and pull request, -and runs it in two jobs. The first runs the tests that need no browser, no +and runs it in three jobs. The first runs the tests that need no browser, no cloud and no keys. The second starts Chrome and Firefox on `ubuntu-latest`, `ubuntu-22.04-arm` and `ubuntu-24.04-arm`, and prints what each runner -provides before it does. Together they prove a change here before any language -repository picks it up. +provides before it does. The third starts CloakBrowser from its pinned image +on `ubuntu-latest` and runs the `Browser` category with a driver attached to +it. Together they prove a change here before any language repository picks it +up. ## Browsers by architecture @@ -139,3 +233,6 @@ nor the driver for it, so the Edge tests say so and skip. once against `:8080`. - **Each API CI** checks this repo out, launches its own example, and runs `Contract` against the public cloud. +- **Either** adds CloakBrowser by starting its container before the tests and + setting `CLOAK_DEBUGGER_ADDRESS`, as + [Running under CloakBrowser](#running-under-cloakbrowser) describes. From 946955204e63e37a91d27019b5546d533547ee8d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 5 Oct 2026 23:18:20 +0100 Subject: [PATCH 4/6] TEST: Refuse a screen of zero in the CloakBrowser test of the rendered overrides The size the page is expected to render is read from the browser, as no device can be emulated in a browser the driver attaches to. The server-side rows can read 0 for a desktop browser, so a browser that reported a screen of 0 would have matched them whether or not the client-side callback ran. The test now fails in that case and says what the browser reported. Run against the .NET cloud example, CloakBrowser reports a screen of 1920 by 1080, the server-side rows read 0 and the client-side rows read 1920 and 1080. With the script's call back to the example refused, this test and the three other CloakBrowser tests that depend on the call back fail. --- .../ExampleClientSideOverridePageContentTests.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs index 0161f5e..de8ed38 100644 --- a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs @@ -122,11 +122,19 @@ public void Example_RendersClientSideOverridesOnThePage_Cloak() _driver = BrowserDrivers.CreateCloak(chromeOptions); var js = (IJavaScriptExecutor)_driver; - var width = js.ExecuteScript("return window.screen.width"); - var height = js.ExecuteScript("return window.screen.height"); + var width = Convert.ToInt64( + js.ExecuteScript("return window.screen.width")); + var height = Convert.ToInt64( + js.ExecuteScript("return window.screen.height")); - AssertRendersScreenSize( - Convert.ToInt64(width), Convert.ToInt64(height)); + // The server-side rows can read 0 for a desktop browser, so a + // screen of 0 would match whether or not the callback ran. + Assert.IsTrue( + width > 0 && height > 0, + $"the browser reports a screen of {width} by {height}, so " + + "the client-side results cannot be told from the server's"); + + AssertRendersScreenSize(width, height); } /// From 1ce8f368c7c67aa3d249ecf31ad088e57d4a9430 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 5 Oct 2026 23:18:21 +0100 Subject: [PATCH 5/6] DOC: Say which keys the tests give to CloakBrowser CloakBrowser is a third party's closed source browser, so a pipeline has to know which keys it is shown. The CloakTests class runs the cloud's JavaScript in the browser, and that script carries the resource key and the license it was requested with. The README and the class now say so, and that a pipeline running CloudInternal under CloakBrowser should use keys it is content for that browser to see. The Contract tests put no key in anything they give the browser and pass on what the example serves. --- Browsers/CloakTests.cs | 5 +++++ README.md | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/Browsers/CloakTests.cs b/Browsers/CloakTests.cs index e176cf8..476f355 100644 --- a/Browsers/CloakTests.cs +++ b/Browsers/CloakTests.cs @@ -17,6 +17,11 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browsers /// CLOAK_DEBUGGER_ADDRESS names, and is skipped when it names none. /// Driver and HTTP server are created once per class for performance. /// + /// + /// The script these tests run in the browser carries the resource key + /// and the license it was requested with, so both are given to + /// CloakBrowser. + /// [RequiresCloak] [TestClass, TestCategory("CloudInternal")] public class CloakTests : SeleniumTestsBase diff --git a/README.md b/README.md index 8f16890..73dd532 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,17 @@ three things. others and clears the cookies and the cache, so a test starts as it would in a browser started for it. +### Keys the browser is given + +CloakBrowser is a third party's closed source browser, so consider which keys +it is shown. The `Contract` tests put no key in anything they give the +browser and pass on what the example serves, so an example that keeps its +resource key on the server gives CloakBrowser no key. The `CloakTests` class +in `CloudInternal` is different, because it runs the cloud's JavaScript in the +browser, and that script carries the resource key and the license it was +requested with. A pipeline that runs `CloudInternal` under CloakBrowser should +therefore use keys it is content for that browser to see. + ### License The browser binary has its own license, which is separate from the MIT From d27f4c5e6dbb289983c28e99a1fb1207985dcb88 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 5 Oct 2026 23:41:47 +0100 Subject: [PATCH 6/6] FIX: Start Chrome and Firefox once in CI, not again in the CloakBrowser job The CloakBrowser job ran the whole Browser category, so it started Chrome and Firefox as well, which the browsers job already does on the same image. Firefox's start on that image takes from 3 seconds to more than 60, and in one run it passed Selenium's limit of 60 seconds and failed the job although both Cloak tests had passed. The job now runs only the tests that attach to CloakBrowser, which is what it is there to prove. The container is not what slows Firefox. Measured in this repository's CI on the ubuntu-24.04 image 20260927.320.1, Firefox took 3, 19, 31 and more than 60 seconds with the container beside it, and in two of those runs the browsers job, which has no container, took 46 and 48 seconds for Chrome and Firefox together. In a Linux container sharing a network with CloakBrowser, Firefox took 5 to 12 seconds to run the smoke test, and 6 to 13 without it. --- .github/workflows/build-and-test.yml | 11 +++++++---- README.md | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 25f1174..37edb85 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -117,12 +117,15 @@ jobs: - name: Build run: dotnet build -c Release - # A Cloak test that was skipped would leave this job green having - # attached to nothing, so the pass is looked for by name. - - name: Start a browser + # Only the tests that attach to CloakBrowser run here, as the browsers + # job already starts Chrome and Firefox on this image. A Cloak test that + # was skipped would leave this job green having attached to nothing, so + # the pass is looked for by name. + - name: Attach to CloakBrowser run: | set -o pipefail - dotnet test -c Release --no-build --filter "TestCategory=Browser" \ + dotnet test -c Release --no-build \ + --filter "TestCategory=Browser&Name~Cloak_" \ --logger "console;verbosity=normal" | tee browser-tests.log grep -q "Passed Cloak_RunsThePage" browser-tests.log diff --git a/README.md b/README.md index 73dd532..512b5bd 100644 --- a/README.md +++ b/README.md @@ -215,9 +215,9 @@ and runs it in three jobs. The first runs the tests that need no browser, no cloud and no keys. The second starts Chrome and Firefox on `ubuntu-latest`, `ubuntu-22.04-arm` and `ubuntu-24.04-arm`, and prints what each runner provides before it does. The third starts CloakBrowser from its pinned image -on `ubuntu-latest` and runs the `Browser` category with a driver attached to -it. Together they prove a change here before any language repository picks it -up. +on `ubuntu-latest` and runs the Cloak tests of the `Browser` category with a +driver attached to it. Together they prove a change here before any language +repository picks it up. ## Browsers by architecture