From 3355e35d68d92722e2c6d6715999fe943e8f6716 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:03:39 +0100 Subject: [PATCH 1/3] FIX: Run the browser tests on ARM64 Linux instead of reporting nine passes that never happened Selenium Manager, the helper the Selenium package ships to fetch a driver, had only an x64 Linux build up to Selenium.WebDriver 4.48, under a folder named for Linux with no architecture in the name. On an ARM64 Linux runner that build was picked and could not start, so every browser test died with "Exec format error" in a few milliseconds and no browser was ever launched. Selenium.WebDriver 4.49.0 splits that folder by architecture and adds an ARM64 Linux build, so the package moves to 4.49.0. A driver the machine already provides is now used ahead of Selenium Manager, from one place rather than from twelve copies of the same block. GitHub names theirs in CHROMEWEBDRIVER, GECKOWEBDRIVER and EDGEWEBDRIVER, and the ARM64 images set only GECKOWEBDRIVER, carrying Firefox and geckodriver but no Chrome, no Chromium and no Edge. Where nothing can be started the failure names the browser, the variable that was read and the file names looked for, instead of a stack trace about a helper program. Edge on ARM64 Linux says plainly that Microsoft publishes neither the browser nor the driver for it. New smoke tests start each browser against a page served from the test process, needing no cloud, no key and no example, so CI can see for itself whether a browser runs on a runner. The suite's own CI runs them on ubuntu-latest, ubuntu-22.04-arm and ubuntu-24.04-arm. --- .github/workflows/build-and-test.yml | 49 ++- BrowserCache/CacheTests.cs | 30 +- BrowserCache/SessionStorageCacheTests.cs | 20 +- Browsers/ChromeTests.cs | 17 +- Browsers/EdgeTests.cs | 17 +- Browsers/GeckoTests.cs | 17 +- COEP/CoepTests.cs | 10 +- .../DeviceClientSideOverrideTests.cs | 10 +- ...ampleClientSideOverridePageContentTests.cs | 10 +- .../ExampleClientSideOverrideTests.cs | 10 +- ClientSideOverrides/ExampleRenderTests.cs | 10 +- .../LocationClientSideOverrideTests.cs | 5 +- Helpers/BrowserDrivers.cs | 385 ++++++++++++++++++ Helpers/BrowserDriversTests.cs | 154 +++++++ Selenium/DriverSmokeTests.cs | 143 +++++++ SeleniumApiTests.csproj | 6 +- ThirdPartyCookieTests.cs | 7 +- 17 files changed, 760 insertions(+), 140 deletions(-) create mode 100644 Helpers/BrowserDrivers.cs create mode 100644 Helpers/BrowserDriversTests.cs create mode 100644 Selenium/DriverSmokeTests.cs diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 279c4fd..657775a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,5 +1,11 @@ -# Builds the suite and runs the tests that need no browser, no cloud and no -# keys, so a change here is checked before a language repository picks it up. +# Builds the suite and runs it, so a change here is checked before a language +# repository picks it up. +# +# Two jobs, because they answer different questions. The first runs the tests +# that need no browser, no cloud and no keys. The second starts a real browser +# on each runner the language repositories use, including the ARM64 Linux ones, +# and fails when one will not start, which is the fault the second job exists +# to catch. name: Build and test on: @@ -28,4 +34,41 @@ jobs: - name: Test run: > dotnet test -c Release --no-build - --filter "TestCategory!=Contract&TestCategory!=CloudInternal" + --filter "TestCategory!=Contract&TestCategory!=CloudInternal&TestCategory!=Browser" + + browsers: + name: Browsers on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # The ARM64 images are here because a browser suite that could not + # start a browser on them was reported as passing for as long as nobody + # read the log. ubuntu-22.04-arm is the image the language repositories + # run their ARM jobs on. + os: [ubuntu-latest, ubuntu-22.04-arm, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + # Say what the runner brought, so a later change of image is readable + # from the log rather than guessed at. + - name: What this runner provides + run: | + echo "arch: $(uname -m)" + echo "CHROMEWEBDRIVER: ${CHROMEWEBDRIVER:-}" + echo "GECKOWEBDRIVER: ${GECKOWEBDRIVER:-}" + echo "EDGEWEBDRIVER: ${EDGEWEBDRIVER:-}" + for exe in google-chrome chromium chromedriver firefox geckodriver \ + microsoft-edge msedgedriver; do + echo "$exe: $(command -v $exe || echo '')" + done + + - name: Build + run: dotnet build -c Release + + - name: Start a browser + run: dotnet test -c Release --no-build --filter "TestCategory=Browser" diff --git a/BrowserCache/CacheTests.cs b/BrowserCache/CacheTests.cs index d9e3dc1..eded432 100644 --- a/BrowserCache/CacheTests.cs +++ b/BrowserCache/CacheTests.cs @@ -89,15 +89,7 @@ public void JavaScript_BrowserCache_Chrome() options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new ChromeDriver(options); - } + driver = BrowserDrivers.CreateChrome(options); RunTest(driver); } @@ -113,15 +105,7 @@ public void JavaScript_BrowserCache_Edge() options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new EdgeDriver(options); - } + driver = BrowserDrivers.CreateEdge(options); RunTest(driver); } @@ -136,15 +120,7 @@ public void JavaScript_BrowserCache_FireFox() options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new FirefoxDriver(options); - } + driver = BrowserDrivers.CreateFirefox(options); RunTest(driver); } diff --git a/BrowserCache/SessionStorageCacheTests.cs b/BrowserCache/SessionStorageCacheTests.cs index 8849b33..0aebb0b 100644 --- a/BrowserCache/SessionStorageCacheTests.cs +++ b/BrowserCache/SessionStorageCacheTests.cs @@ -138,15 +138,7 @@ public void SessionStorageCache_Chrome(bool enableCookies) options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new ChromeDriver(options); - } + driver = BrowserDrivers.CreateChrome(options); RunTest(driver, enableCookies); } @@ -163,15 +155,7 @@ public void SessionStorageCache_FireFox(bool enableCookies) options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new FirefoxDriver(options); - } + driver = BrowserDrivers.CreateFirefox(options); RunTest(driver, enableCookies); } diff --git a/Browsers/ChromeTests.cs b/Browsers/ChromeTests.cs index 574228c..ad59099 100644 --- a/Browsers/ChromeTests.cs +++ b/Browsers/ChromeTests.cs @@ -39,22 +39,13 @@ public static void ClassInit(TestContext context) var options = new ChromeOptions(); options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + try { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + s_driver = BrowserDrivers.CreateChrome(options); } - else + catch (WebDriverException e) { - try - { - s_driver = new ChromeDriver(options); - } - catch (WebDriverException) - { - Assert.Inconclusive("Could not create a ChromeDriver, check " + - "that the Chromium driver is installed"); - } + Assert.Inconclusive(e.Message); } } diff --git a/Browsers/EdgeTests.cs b/Browsers/EdgeTests.cs index 24acb0f..cb14b5c 100644 --- a/Browsers/EdgeTests.cs +++ b/Browsers/EdgeTests.cs @@ -39,22 +39,13 @@ public static void ClassInit(TestContext context) var options = new EdgeOptions(); options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + try { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + s_driver = BrowserDrivers.CreateEdge(options); } - else + catch (WebDriverException e) { - try - { - s_driver = new EdgeDriver(options); - } - catch (WebDriverException) - { - Assert.Inconclusive("Could not create an EdgeDriver, check " + - "that the MS edge driver is installed"); - } + Assert.Inconclusive(e.Message); } } diff --git a/Browsers/GeckoTests.cs b/Browsers/GeckoTests.cs index 4142166..1feba96 100644 --- a/Browsers/GeckoTests.cs +++ b/Browsers/GeckoTests.cs @@ -38,22 +38,13 @@ public static void ClassInit(TestContext context) var options = new FirefoxOptions(); options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + try { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + s_driver = BrowserDrivers.CreateFirefox(options); } - else + catch (WebDriverException e) { - try - { - s_driver = new FirefoxDriver(options); - } - catch (WebDriverException) - { - Assert.Inconclusive("Could not create a gecko driver, check " + - "that the gecko driver is installed"); - } + Assert.Inconclusive(e.Message); } } diff --git a/COEP/CoepTests.cs b/COEP/CoepTests.cs index 886930c..7c0095d 100644 --- a/COEP/CoepTests.cs +++ b/COEP/CoepTests.cs @@ -105,15 +105,7 @@ await response.OutputStream.WriteAsync( options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); options.SetLoggingPreference(LogType.Browser, LogLevel.All); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new ChromeDriver(options); - } + driver = BrowserDrivers.CreateChrome(options); } /// diff --git a/ClientSideOverrides/DeviceClientSideOverrideTests.cs b/ClientSideOverrides/DeviceClientSideOverrideTests.cs index 3637af1..c3245b2 100644 --- a/ClientSideOverrides/DeviceClientSideOverrideTests.cs +++ b/ClientSideOverrides/DeviceClientSideOverrideTests.cs @@ -123,15 +123,7 @@ public void JavaScript_ClientSideOverrides( UserAgent = userAgent, }); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - driver = new ChromeDriver(chromeOptions); - } + driver = BrowserDrivers.CreateChrome(chromeOptions); driver.Manage().Cookies.DeleteAllCookies(); driver.Navigate().GoToUrl(ClientServerUrl); diff --git a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs index 54bb419..efaaaa8 100644 --- a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs @@ -102,15 +102,7 @@ public void Example_RendersClientSideOverridesOnThePage( UserAgent = userAgent, }); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - _driver = new ChromeDriver(chromeOptions); - } + _driver = BrowserDrivers.CreateChrome(chromeOptions); _driver.Navigate().GoToUrl(_proxyUrl); diff --git a/ClientSideOverrides/ExampleClientSideOverrideTests.cs b/ClientSideOverrides/ExampleClientSideOverrideTests.cs index d470df1..3876b91 100644 --- a/ClientSideOverrides/ExampleClientSideOverrideTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverrideTests.cs @@ -87,15 +87,7 @@ public void Example_ServesCoreJs_AndClientSideOverridesFlow( UserAgent = userAgent, }); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - _driver = new ChromeDriver(chromeOptions); - } + _driver = BrowserDrivers.CreateChrome(chromeOptions); _driver.Navigate().GoToUrl(_proxyUrl); IJavaScriptExecutor js = _driver; diff --git a/ClientSideOverrides/ExampleRenderTests.cs b/ClientSideOverrides/ExampleRenderTests.cs index f950c4a..ace05c1 100644 --- a/ClientSideOverrides/ExampleRenderTests.cs +++ b/ClientSideOverrides/ExampleRenderTests.cs @@ -89,15 +89,7 @@ public void Example_RendersRealDetectionResult() chromeOptions.AddArgument("--headless"); chromeOptions.AddArgument($"--user-agent={DesktopChromeUserAgent}"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - _driver = new ChromeDriver(chromeOptions); - } + _driver = BrowserDrivers.CreateChrome(chromeOptions); _driver.Navigate().GoToUrl(_example.BaseUrl); diff --git a/ClientSideOverrides/LocationClientSideOverrideTests.cs b/ClientSideOverrides/LocationClientSideOverrideTests.cs index 80151e6..3640b4f 100644 --- a/ClientSideOverrides/LocationClientSideOverrideTests.cs +++ b/ClientSideOverrides/LocationClientSideOverrideTests.cs @@ -8,6 +8,7 @@ using System.Net; using System.Threading; using FiftyOne.Pipeline.Cloud.Tests.Common.TestElements; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; namespace FiftyOne.Pipeline.Cloud.SeleniumTests.ClientSideOverrides { @@ -88,7 +89,9 @@ public void JavaScript_ClientSideOverrides() chromeOptions.SetLoggingPreference(LogType.Browser, LogLevel.All); - driver = new ChromeDriver(chromeOptions); + // Cast because the CDP command below is Chrome's own. A + // remote Selenium is not supported by this test. + driver = (ChromeDriver)BrowserDrivers.CreateChrome(chromeOptions); // Allow geo-location on the test client website. driver.ExecuteCdpCommand("Browser.grantPermissions", diff --git a/Helpers/BrowserDrivers.cs b/Helpers/BrowserDrivers.cs new file mode 100644 index 0000000..481fd59 --- /dev/null +++ b/Helpers/BrowserDrivers.cs @@ -0,0 +1,385 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Edge; +using OpenQA.Selenium.Firefox; +using OpenQA.Selenium.Remote; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers +{ + /// + /// Creates the web drivers the tests use, from one place, so every test + /// finds a browser the same way. + /// + /// + /// + /// A driver the machine already provides is used first, because a machine + /// that ships one is faster and steadier than downloading one for every + /// run. GitHub's Linux runner images name theirs in CHROMEWEBDRIVER, + /// GECKOWEBDRIVER and EDGEWEBDRIVER, and the ARM64 images set only + /// GECKOWEBDRIVER, as they carry Firefox and geckodriver but no Chrome, + /// no Chromium and no Edge. + /// + /// + /// When the machine provides no driver, Selenium Manager is left to fetch + /// one. That is a helper program shipped inside the Selenium.WebDriver + /// package, and up to version 4.48 the only Linux build of it was for x64, + /// under a folder named for Linux with no architecture in the name. On an + /// ARM64 Linux machine that build was picked and could not start, so every + /// browser test died with "Exec format error" before a browser was ever + /// launched. Version 4.49 splits the folder by architecture and adds an + /// ARM64 build, which is why this project needs 4.49 or later. + /// + /// + /// Giving Selenium a driver service that already knows its path stops it + /// calling Selenium Manager at all, which is the behaviour the first case + /// relies on. + /// + /// + public static class BrowserDrivers + { + /// + /// Set to the address of a Selenium grid or container to drive a + /// browser there instead of on this machine. + /// + public const string SeleniumUrlVariable = "SELENIUM_URL"; + + /// + /// Directory holding chromedriver, or the driver itself. GitHub's + /// Linux x64 runner images set this variable. + /// + public const string ChromeDriverVariable = "CHROMEWEBDRIVER"; + + /// + /// Directory holding geckodriver, or the driver itself. GitHub's + /// Linux runner images set this variable on both architectures. + /// + public const string GeckoDriverVariable = "GECKOWEBDRIVER"; + + /// + /// Directory holding msedgedriver, or the driver itself. GitHub's + /// Linux x64 runner images set this variable. + /// + public const string EdgeDriverVariable = "EDGEWEBDRIVER"; + + /// Path to the Chrome or Chromium binary to drive. + public const string ChromeBinaryVariable = "CHROME_BIN"; + + /// Path to the Firefox binary to drive. + public const string FirefoxBinaryVariable = "FIREFOX_BIN"; + + /// Path to the Edge binary to drive. + public const string EdgeBinaryVariable = "EDGE_BIN"; + + /// File names a Chrome driver goes by. + public static readonly string[] ChromeDriverNames = + { "chromedriver", "chromedriver.exe" }; + + /// File names a Firefox driver goes by. + public static readonly string[] GeckoDriverNames = + { "geckodriver", "geckodriver.exe" }; + + /// File names an Edge driver goes by. + public static readonly string[] EdgeDriverNames = + { "msedgedriver", "msedgedriver.exe" }; + + // Chromium is what an ARM64 Linux machine is most likely to have, so + // it is looked for alongside Chrome. + private static readonly string[] _chromeBinaryNames = + { + "google-chrome", "google-chrome-stable", "chromium-browser", + "chromium", "chrome.exe", + }; + + private static readonly string[] _firefoxBinaryNames = + { "firefox", "firefox.exe" }; + + private static readonly string[] _edgeBinaryNames = + { "microsoft-edge", "microsoft-edge-stable", "msedge.exe" }; + + /// + /// True on ARM64 Linux, where Microsoft publishes no Edge browser and + /// no Edge driver, so an Edge test there can only be skipped. + /// + public static bool IsLinuxArm64 { get; } = + RuntimeInformation.IsOSPlatform(OSPlatform.Linux) + && RuntimeInformation.ProcessArchitecture == Architecture.Arm64; + + /// + /// Creates a Chrome or Chromium driver. + /// + /// Options for the browser. + /// A driver, which the caller quits. + public static WebDriver CreateChrome(ChromeOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + if (TryCreateRemote(options, out var remote)) + { + return remote; + } + SetBinaryLocation(options, ChromeBinaryVariable, _chromeBinaryNames); + var driverPath = FindDriver(ChromeDriverVariable, ChromeDriverNames); + if (driverPath != null) + { + return new ChromeDriver( + ChromeDriverService.CreateDefaultService(driverPath), + options); + } + return Fetched( + () => new ChromeDriver(options), + "Chrome", ChromeDriverVariable, ChromeDriverNames); + } + + /// + /// Creates a Firefox driver. + /// + /// Options for the browser. + /// A driver, which the caller quits. + public static WebDriver CreateFirefox(FirefoxOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + if (TryCreateRemote(options, out var remote)) + { + return remote; + } + SetBinaryLocation( + options, FirefoxBinaryVariable, _firefoxBinaryNames); + var driverPath = FindDriver(GeckoDriverVariable, GeckoDriverNames); + if (driverPath != null) + { + return new FirefoxDriver( + FirefoxDriverService.CreateDefaultService(driverPath), + options); + } + return Fetched( + () => new FirefoxDriver(options), + "Firefox", GeckoDriverVariable, GeckoDriverNames); + } + + /// + /// Creates an Edge driver. + /// + /// Options for the browser. + /// A driver, which the caller quits. + public static WebDriver CreateEdge(EdgeOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + if (TryCreateRemote(options, out var remote)) + { + return remote; + } + var driverPath = FindDriver(EdgeDriverVariable, EdgeDriverNames); + if (driverPath == null && IsLinuxArm64) + { + throw new WebDriverException( + "Edge cannot run on ARM64 Linux, because Microsoft " + + "publishes no Edge browser and no Edge driver for it. " + + $"Set {SeleniumUrlVariable} to a Selenium that has " + + "Edge, or run the Edge tests on another architecture."); + } + SetBinaryLocation(options, EdgeBinaryVariable, _edgeBinaryNames); + if (driverPath != null) + { + return new EdgeDriver( + EdgeDriverService.CreateDefaultService(driverPath), + options); + } + return Fetched( + () => new EdgeDriver(options), + "Edge", EdgeDriverVariable, EdgeDriverNames); + } + + /// + /// Runs , which leaves Selenium Manager to + /// fetch a driver, and on failure says what was looked for here first, + /// so the reason reads as a missing browser rather than as a stack + /// trace about a helper program. + /// + /// Creates the driver. + /// Name of the browser, for the message. + /// Variable that would name the driver. + /// File names looked for on the path. + /// A driver, which the caller quits. + private static WebDriver Fetched( + Func create, + string browser, + string variable, + IEnumerable names) + { + try + { + return create(); + } + catch (Exception e) + { + throw new WebDriverException( + $"No {browser} driver on this machine, so Selenium " + + "Manager was left to fetch one, and it could not. " + + $"Looked at {variable}, and for " + + $"{string.Join(", ", names)} on the path. This machine " + + $"is {RuntimeInformation.RuntimeIdentifier}. Set " + + $"{variable} to a driver, or {SeleniumUrlVariable} to a " + + $"Selenium that has {browser}. Selenium said: " + + e.Message, + e); + } + } + + /// + /// Creates a driver on a remote Selenium when SELENIUM_URL is set. + /// + /// Options for the browser. + /// The driver created, or null. + /// True when SELENIUM_URL is set. + private static bool TryCreateRemote( + DriverOptions options, out WebDriver driver) + { + if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + { + ExternalSeleniumHelper.AddExternalSeleniumArguments(options); + driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + return true; + } + driver = null; + return false; + } + + /// + /// Path to a driver this machine provides, or null when it provides + /// none. + /// + /// Variable that names the driver. + /// File names to look for on the path. + /// A path, or null. + public static string FindDriver( + string variable, IEnumerable names) => + FindDriver(Environment.GetEnvironmentVariable, variable, names); + + /// + /// As , reading + /// the variables through so the rules + /// can be tested without changing the environment of the whole run. + /// + /// Reads an environment variable. + /// Variable that names the driver. + /// File names to look for on the path. + /// A path, or null. + public static string FindDriver( + Func getVariable, + string variable, + IEnumerable names) + { + if (getVariable == null) + { + throw new ArgumentNullException(nameof(getVariable)); + } + var configured = getVariable(variable); + if (string.IsNullOrEmpty(configured) == false) + { + if (File.Exists(configured)) + { + return configured; + } + var inDirectory = FindInDirectory(configured, names); + if (inDirectory != null) + { + return inDirectory; + } + } + return FindOnPath(getVariable, names); + } + + /// + /// Points the options at a browser this machine provides, leaving them + /// alone when it provides none, so Selenium Manager still fetches one. + /// + /// Options for the browser. + /// Variable that names the browser. + /// File names to look for on the path. + private static void SetBinaryLocation( + DriverOptions options, string variable, IEnumerable names) + { + var configured = Environment.GetEnvironmentVariable(variable); + var path = string.IsNullOrEmpty(configured) == false + && File.Exists(configured) + ? configured + : FindOnPath(Environment.GetEnvironmentVariable, names); + if (path != null) + { + options.BinaryLocation = path; + } + } + + /// + /// The first of found in a directory named by + /// PATH, or null. + /// + /// Reads an environment variable. + /// File names to look for. + /// A path, or null. + private static string FindOnPath( + Func getVariable, IEnumerable names) + { + var path = getVariable("PATH"); + if (string.IsNullOrEmpty(path)) + { + return null; + } + foreach (var directory in path.Split(Path.PathSeparator)) + { + if (string.IsNullOrWhiteSpace(directory)) + { + continue; + } + var found = FindInDirectory(directory, names); + if (found != null) + { + return found; + } + } + return null; + } + + /// + /// The first of present in + /// , or null. + /// + /// Directory to look in. + /// File names to look for. + /// A path, or null. + private static string FindInDirectory( + string directory, IEnumerable names) + { + foreach (var name in names) + { + string candidate; + try + { + candidate = Path.Combine(directory, name); + } + catch (ArgumentException) + { + // A PATH entry holding characters a path cannot. + return null; + } + if (File.Exists(candidate)) + { + return candidate; + } + } + return null; + } + } +} diff --git a/Helpers/BrowserDriversTests.cs b/Helpers/BrowserDriversTests.cs new file mode 100644 index 0000000..2192525 --- /dev/null +++ b/Helpers/BrowserDriversTests.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Generic; +using System.IO; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Tests +{ + /// + /// Checks the rules for finding a driver the machine provides. No browser + /// is started, so these run anywhere. + /// + [TestClass] + public class BrowserDriversTests + { + private string _directory; + private string _driverFile; + + /// Makes a directory holding a file named like a driver. + [TestInitialize] + public void Init() + { + _directory = Path.Combine( + Path.GetTempPath(), "51d-driver-" + Guid.NewGuid()); + Directory.CreateDirectory(_directory); + _driverFile = Path.Combine(_directory, "chromedriver"); + File.WriteAllText(_driverFile, string.Empty); + } + + /// Removes the directory. + [TestCleanup] + public void Cleanup() + { + if (_directory != null && Directory.Exists(_directory)) + { + Directory.Delete(_directory, true); + } + } + + /// + /// A variable naming the directory the driver is in finds the driver, + /// which is the shape GitHub's runner images use. + /// + [TestMethod] + public void FindDriver_ReadsDirectoryFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.ChromeDriverVariable, _directory }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// A variable naming the driver itself finds the driver. + [TestMethod] + public void FindDriver_ReadsFileFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.ChromeDriverVariable, _driverFile }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// With no variable set the path is searched, so a machine that just + /// has a driver installed needs no configuration. + /// + [TestMethod] + public void FindDriver_FallsBackToThePath() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { "PATH", _directory }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// A variable set to a directory holding no driver does not stop the + /// path being searched, which is what an ARM64 runner does with + /// CHROMEWEBDRIVER left empty. + /// + [TestMethod] + public void FindDriver_SearchesThePathWhenTheVariableIsWrong() + { + var empty = Path.Combine(_directory, "empty"); + Directory.CreateDirectory(empty); + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.ChromeDriverVariable, empty }, + { "PATH", _directory }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// Nothing anywhere gives null, which is what sends the caller on to + /// Selenium Manager. + /// + [TestMethod] + public void FindDriver_ReturnsNullWhenNothingProvidesOne() + { + Assert.IsNull( + BrowserDrivers.FindDriver( + Lookup(new Dictionary()), + BrowserDrivers.GeckoDriverVariable, + BrowserDrivers.GeckoDriverNames)); + } + + /// + /// Every driver name is looked for, so a Windows machine holding + /// chromedriver.exe is found as readily as a Linux one. + /// + [TestMethod] + public void FindDriver_FindsEveryNameItIsGiven() + { + var windowsName = Path.Combine(_directory, "geckodriver.exe"); + File.WriteAllText(windowsName, string.Empty); + Assert.AreEqual( + windowsName, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.GeckoDriverVariable, _directory }, + }), + BrowserDrivers.GeckoDriverVariable, + BrowserDrivers.GeckoDriverNames)); + } + + /// Reads from a dictionary in place of the environment. + /// Variables and their values. + /// A lookup over those values. + private static Func Lookup( + IReadOnlyDictionary values) => + name => values.TryGetValue(name, out var value) ? value : null; + } +} diff --git a/Selenium/DriverSmokeTests.cs b/Selenium/DriverSmokeTests.cs new file mode 100644 index 0000000..e048e42 --- /dev/null +++ b/Selenium/DriverSmokeTests.cs @@ -0,0 +1,143 @@ +using System; +using System.Threading; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Edge; +using OpenQA.Selenium.Firefox; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium +{ + /// + /// Checks that a browser really starts and really runs the page, on + /// whatever machine the suite is on. + /// + /// + /// + /// Every other browser test needs a cloud, a resource key and an example + /// application, so a machine that cannot start a browser at all is only + /// found out in the middle of a test that was trying to check something + /// else. These need none of that, only a page served from this process, so + /// CI can put them on each runner it uses and see for itself whether a + /// browser runs there. + /// + /// + /// They fail rather than skip when no browser can be started, because a + /// run that quietly skipped them would say nothing, and saying nothing is + /// the fault these were written for. + /// + /// + [TestClass, TestCategory("Browser")] + public class DriverSmokeTests + { + // A page that writes something only a running browser can write, so a + // driver that connected but never rendered is not mistaken for a pass. + private const string Page = + "Driver smoke" + + "

not run

"; + + private const string WhereElementId = "where"; + private const int PageLoadTimeoutSeconds = 30; + + private string _url; + private CancellationTokenSource _cts; + private TestHelpers.ServerListener _server; + private WebDriver _driver; + + /// Serves the page from this process. + [TestInitialize] + public void Init() + { + _url = $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/"; + _cts = new CancellationTokenSource(); + _server = TestHelpers.SimpleListener(_url, Page, _cts.Token); + } + + /// Quits the browser and stops the server. + [TestCleanup] + public void Cleanup() + { + _driver?.Quit(); + _driver = null; + _cts?.Cancel(); + _server?.Listener?.Stop(); + _server?.Listener?.Close(); + _cts?.Dispose(); + } + + /// Chrome or Chromium starts and runs the page. + [TestMethod] + public void Chrome_RunsThePage() + { + var options = new ChromeOptions(); + options.AddArgument("--headless"); + options.AddArgument("--no-sandbox"); + options.AddArgument("--disable-dev-shm-usage"); + _driver = BrowserDrivers.CreateChrome(options); + AssertPageRan(); + } + + /// Firefox starts and runs the page. + [TestMethod] + public void Firefox_RunsThePage() + { + var options = new FirefoxOptions(); + options.AddArgument("--headless"); + _driver = BrowserDrivers.CreateFirefox(options); + AssertPageRan(); + } + + /// + /// Edge starts and runs the page, where Edge exists at all. Microsoft + /// publishes no Edge for ARM64 Linux, so there the test says so and + /// skips instead of failing on something that cannot be fixed here. + /// + [TestMethod] + public void Edge_RunsThePage() + { + var options = new EdgeOptions(); + options.AddArgument("--headless"); + options.AddArgument("--no-sandbox"); + options.AddArgument("--disable-dev-shm-usage"); + try + { + _driver = BrowserDrivers.CreateEdge(options); + } + catch (WebDriverException e) when (BrowserDrivers.IsLinuxArm64) + { + Assert.Inconclusive(e.Message); + } + AssertPageRan(); + } + + /// + /// Loads the page and checks the browser ran the script on it. + /// + private void AssertPageRan() + { + _driver.Navigate().GoToUrl(_url); + var deadline = DateTime.UtcNow + + TimeSpan.FromSeconds(PageLoadTimeoutSeconds); + string text = null; + while (DateTime.UtcNow < deadline) + { + text = _driver.FindElement(By.Id(WhereElementId)).Text; + if (text != null && text.StartsWith( + "ran in ", StringComparison.Ordinal)) + { + Console.WriteLine($" Browser reported: {text}"); + return; + } + Thread.Sleep(100); + } + Assert.Fail( + "the browser loaded the page but never ran the script on it, " + + $"as the element reads '{text}'"); + } + } +} diff --git a/SeleniumApiTests.csproj b/SeleniumApiTests.csproj index e36a626..68d3447 100644 --- a/SeleniumApiTests.csproj +++ b/SeleniumApiTests.csproj @@ -24,7 +24,11 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + + diff --git a/ThirdPartyCookieTests.cs b/ThirdPartyCookieTests.cs index b3f82c1..8a01f96 100644 --- a/ThirdPartyCookieTests.cs +++ b/ThirdPartyCookieTests.cs @@ -82,12 +82,7 @@ private ChromeOptions CreateBaseOptions() /// private IWebDriver CreateDriver(ChromeOptions options) { - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - return new RemoteWebDriver(new Uri(seleniumUrl), options); - } - return new ChromeDriver(options); + return BrowserDrivers.CreateChrome(options); } /// From a274ba17f6e6e763c219325d7da687a6a73518f1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:09:55 +0100 Subject: [PATCH 2/3] FIX: Ignore the Edge smoke test, with the reason the run gave The first run proved headless Edge on the x64 Ubuntu runner accepts the session and then hangs on the first navigation until the sixty second timeout, which is the same fault the other Edge tests in this suite are already ignored for. It is a fault of its own and not the ARM64 one this class was added for. --- Selenium/DriverSmokeTests.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Selenium/DriverSmokeTests.cs b/Selenium/DriverSmokeTests.cs index e048e42..1299945 100644 --- a/Selenium/DriverSmokeTests.cs +++ b/Selenium/DriverSmokeTests.cs @@ -97,7 +97,19 @@ public void Firefox_RunsThePage() /// publishes no Edge for ARM64 Linux, so there the test says so and /// skips instead of failing on something that cannot be fixed here. /// + /// + /// Ignored for the same reason the other Edge tests in this suite + /// are, and proved again by the first run of this test: headless Edge + /// on the x64 Ubuntu runner accepts the session and then hangs on the + /// first navigation until the sixty second timeout. That is a fault of + /// its own and not the ARM64 one this class was added for, so the test + /// is kept, with its reason where a reader will find it, rather than + /// deleted. + /// [TestMethod] + [Ignore("Headless Edge on the Ubuntu runners hangs on the first " + + "navigation, and Microsoft publishes no Edge at all for ARM64 " + + "Linux. Chrome and Firefox cover both architectures.")] public void Edge_RunsThePage() { var options = new EdgeOptions(); From b6424a592d4f26876bee2b23315adf50a17f6ce0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:13:12 +0100 Subject: [PATCH 3/3] DOC: Say how a driver is found, and what each architecture provides --- README.md | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2b76964..671b930 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ sibling directory and run it at their integration-test step. |---|---|---| | `Contract` | An example app serves `51Degrees.core.js`, client-side evidence flows back, and the server-rendered page shows a real detection result. | Cloud CI (per example, vs `:8080`) **and** every API CI (vs the public cloud). | | `CloudInternal` | Cloud response behaviour through a browser: cache reuse, COEP/CORP headers, third-party cookies, client-side overrides, and the per-browser JS endpoints. | Cloud CI only (vs `:8080`). | +| `Browser` | That a browser starts at all and runs the script on a page served from the test process. No cloud, no key, no example. | This repository's own CI, on every runner image it uses. | Select a subset with `--filter TestCategory=Contract` or `--filter TestCategory=CloudInternal`. @@ -29,8 +30,10 @@ Select a subset with `--filter TestCategory=Contract` or that is already running was pointed at its data by whoever started it, so with `EXAMPLE_URL` set the suite needs neither `CLOUD_ROOT_URL` nor `PAID_RESOURCE_KEY`, which is what lets an on-premise example run `Contract`. -- **Browser** - by default a local Chrome driver is used. Set `SELENIUM_URL` to use - a Selenium grid instead (CI uses a standalone grid). +- **Browser** - a driver the machine already provides is used first, named by + `CHROMEWEBDRIVER`, `GECKOWEBDRIVER` or `EDGEWEBDRIVER` or found on the path. + When the machine provides none, Selenium Manager fetches one. Set + `SELENIUM_URL` to drive a browser on a Selenium grid instead. ## Configuration @@ -43,7 +46,9 @@ and no keys are committed. | `PAID_RESOURCE_KEY` | `CloudInternal`, and `Contract` when the suite launches the example | Resource key used by the tests. | | `FREE_RESOURCE_KEY` | `CloudInternal` | Free resource key for the JS-endpoint tests. | | `ENTERPRISE_V4_LICENSE` | `CloudInternal` | License passed to the JS endpoint to unlock paid properties. | -| `SELENIUM_URL` | optional | Selenium grid URL, omit for a local Chrome driver. | +| `SELENIUM_URL` | optional | Selenium grid URL, omit to drive a browser on this machine. | +| `CHROMEWEBDRIVER` / `GECKOWEBDRIVER` / `EDGEWEBDRIVER` | optional | A driver, or the directory holding one. GitHub's Linux runner images set these. | +| `CHROME_BIN` / `FIREFOX_BIN` / `EDGE_BIN` | optional | The browser binary to drive, when it is not on the path. | | `EXAMPLE_URL` / `EXAMPLE_LANG` | `Contract` | The example app to test (CI / local). | A missing variable only fails the tests that read it, and the failure names the @@ -98,9 +103,30 @@ dotnet test --filter TestCategory=CloudInternal ## This repository's own CI -The "Build and test" workflow builds the suite and runs the tests that need no -browser, no cloud and no keys, on every push and pull request. It is what -proves a change here before any language repository picks it up. +The "Build and test" workflow builds the suite on every push and pull request, +and runs it in two jobs. The first runs the tests that need no browser, no +cloud and no keys. The second starts Chrome and Firefox on `ubuntu-latest`, +`ubuntu-22.04-arm` and `ubuntu-24.04-arm`, and prints what each runner +provides before it does. Together they prove a change here before any language +repository picks it up. + +## Browsers by architecture + +GitHub's ARM64 Linux runner images carry Firefox and geckodriver and set +`GECKOWEBDRIVER`, but no Chrome, no Chromium, no ChromeDriver and no Edge, +and they leave `CHROMEWEBDRIVER` and `EDGEWEBDRIVER` unset. The x64 images +carry all of them. + +Chrome still runs on ARM64, because Selenium Manager fetches the Chrome for +Testing `linux-arm64` build and a matching driver. That needs +Selenium.WebDriver 4.49.0 or later, which is the first version whose Selenium +Manager ships an ARM64 Linux build. Earlier versions carry only an x64 one, +under a folder named for Linux with no architecture in the name, so on an +ARM64 runner it is picked, cannot start, and every browser test dies in a few +milliseconds with "Exec format error". Do not downgrade the package. + +Edge cannot run on ARM64 Linux at all. Microsoft publishes neither the browser +nor the driver for it, so the Edge tests say so and skip. ## CI integration