diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 279c4fd..657775a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,5 +1,11 @@ -# Builds the suite and runs the tests that need no browser, no cloud and no -# keys, so a change here is checked before a language repository picks it up. +# Builds the suite and runs it, so a change here is checked before a language +# repository picks it up. +# +# Two jobs, because they answer different questions. The first runs the tests +# that need no browser, no cloud and no keys. The second starts a real browser +# on each runner the language repositories use, including the ARM64 Linux ones, +# and fails when one will not start, which is the fault the second job exists +# to catch. name: Build and test on: @@ -28,4 +34,41 @@ jobs: - name: Test run: > dotnet test -c Release --no-build - --filter "TestCategory!=Contract&TestCategory!=CloudInternal" + --filter "TestCategory!=Contract&TestCategory!=CloudInternal&TestCategory!=Browser" + + browsers: + name: Browsers on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # The ARM64 images are here because a browser suite that could not + # start a browser on them was reported as passing for as long as nobody + # read the log. ubuntu-22.04-arm is the image the language repositories + # run their ARM jobs on. + os: [ubuntu-latest, ubuntu-22.04-arm, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + # Say what the runner brought, so a later change of image is readable + # from the log rather than guessed at. + - name: What this runner provides + run: | + echo "arch: $(uname -m)" + echo "CHROMEWEBDRIVER: ${CHROMEWEBDRIVER:-}" + echo "GECKOWEBDRIVER: ${GECKOWEBDRIVER:-}" + echo "EDGEWEBDRIVER: ${EDGEWEBDRIVER:-}" + for exe in google-chrome chromium chromedriver firefox geckodriver \ + microsoft-edge msedgedriver; do + echo "$exe: $(command -v $exe || echo '')" + done + + - name: Build + run: dotnet build -c Release + + - name: Start a browser + run: dotnet test -c Release --no-build --filter "TestCategory=Browser" diff --git a/BrowserCache/CacheTests.cs b/BrowserCache/CacheTests.cs index d9e3dc1..eded432 100644 --- a/BrowserCache/CacheTests.cs +++ b/BrowserCache/CacheTests.cs @@ -89,15 +89,7 @@ public void JavaScript_BrowserCache_Chrome() options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new ChromeDriver(options); - } + driver = BrowserDrivers.CreateChrome(options); RunTest(driver); } @@ -113,15 +105,7 @@ public void JavaScript_BrowserCache_Edge() options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new EdgeDriver(options); - } + driver = BrowserDrivers.CreateEdge(options); RunTest(driver); } @@ -136,15 +120,7 @@ public void JavaScript_BrowserCache_FireFox() options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new FirefoxDriver(options); - } + driver = BrowserDrivers.CreateFirefox(options); RunTest(driver); } diff --git a/BrowserCache/SessionStorageCacheTests.cs b/BrowserCache/SessionStorageCacheTests.cs index 8849b33..0aebb0b 100644 --- a/BrowserCache/SessionStorageCacheTests.cs +++ b/BrowserCache/SessionStorageCacheTests.cs @@ -138,15 +138,7 @@ public void SessionStorageCache_Chrome(bool enableCookies) options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new ChromeDriver(options); - } + driver = BrowserDrivers.CreateChrome(options); RunTest(driver, enableCookies); } @@ -163,15 +155,7 @@ public void SessionStorageCache_FireFox(bool enableCookies) options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new FirefoxDriver(options); - } + driver = BrowserDrivers.CreateFirefox(options); RunTest(driver, enableCookies); } diff --git a/Browsers/ChromeTests.cs b/Browsers/ChromeTests.cs index 574228c..ad59099 100644 --- a/Browsers/ChromeTests.cs +++ b/Browsers/ChromeTests.cs @@ -39,22 +39,13 @@ public static void ClassInit(TestContext context) var options = new ChromeOptions(); options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + try { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + s_driver = BrowserDrivers.CreateChrome(options); } - else + catch (WebDriverException e) { - try - { - s_driver = new ChromeDriver(options); - } - catch (WebDriverException) - { - Assert.Inconclusive("Could not create a ChromeDriver, check " + - "that the Chromium driver is installed"); - } + Assert.Inconclusive(e.Message); } } diff --git a/Browsers/EdgeTests.cs b/Browsers/EdgeTests.cs index 24acb0f..cb14b5c 100644 --- a/Browsers/EdgeTests.cs +++ b/Browsers/EdgeTests.cs @@ -39,22 +39,13 @@ public static void ClassInit(TestContext context) var options = new EdgeOptions(); options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + try { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + s_driver = BrowserDrivers.CreateEdge(options); } - else + catch (WebDriverException e) { - try - { - s_driver = new EdgeDriver(options); - } - catch (WebDriverException) - { - Assert.Inconclusive("Could not create an EdgeDriver, check " + - "that the MS edge driver is installed"); - } + Assert.Inconclusive(e.Message); } } diff --git a/Browsers/GeckoTests.cs b/Browsers/GeckoTests.cs index 4142166..1feba96 100644 --- a/Browsers/GeckoTests.cs +++ b/Browsers/GeckoTests.cs @@ -38,22 +38,13 @@ public static void ClassInit(TestContext context) var options = new FirefoxOptions(); options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + try { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + s_driver = BrowserDrivers.CreateFirefox(options); } - else + catch (WebDriverException e) { - try - { - s_driver = new FirefoxDriver(options); - } - catch (WebDriverException) - { - Assert.Inconclusive("Could not create a gecko driver, check " + - "that the gecko driver is installed"); - } + Assert.Inconclusive(e.Message); } } diff --git a/COEP/CoepTests.cs b/COEP/CoepTests.cs index 886930c..7c0095d 100644 --- a/COEP/CoepTests.cs +++ b/COEP/CoepTests.cs @@ -105,15 +105,7 @@ await response.OutputStream.WriteAsync( options.AcceptInsecureCertificates = true; options.AddArgument("--headless"); options.SetLoggingPreference(LogType.Browser, LogLevel.All); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - driver = new RemoteWebDriver(new Uri(seleniumUrl), options); - } - else - { - driver = new ChromeDriver(options); - } + driver = BrowserDrivers.CreateChrome(options); } /// diff --git a/ClientSideOverrides/DeviceClientSideOverrideTests.cs b/ClientSideOverrides/DeviceClientSideOverrideTests.cs index 3637af1..c3245b2 100644 --- a/ClientSideOverrides/DeviceClientSideOverrideTests.cs +++ b/ClientSideOverrides/DeviceClientSideOverrideTests.cs @@ -123,15 +123,7 @@ public void JavaScript_ClientSideOverrides( UserAgent = userAgent, }); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - driver = new ChromeDriver(chromeOptions); - } + driver = BrowserDrivers.CreateChrome(chromeOptions); driver.Manage().Cookies.DeleteAllCookies(); driver.Navigate().GoToUrl(ClientServerUrl); diff --git a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs index 54bb419..efaaaa8 100644 --- a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs @@ -102,15 +102,7 @@ public void Example_RendersClientSideOverridesOnThePage( UserAgent = userAgent, }); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - _driver = new ChromeDriver(chromeOptions); - } + _driver = BrowserDrivers.CreateChrome(chromeOptions); _driver.Navigate().GoToUrl(_proxyUrl); diff --git a/ClientSideOverrides/ExampleClientSideOverrideTests.cs b/ClientSideOverrides/ExampleClientSideOverrideTests.cs index d470df1..3876b91 100644 --- a/ClientSideOverrides/ExampleClientSideOverrideTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverrideTests.cs @@ -87,15 +87,7 @@ public void Example_ServesCoreJs_AndClientSideOverridesFlow( UserAgent = userAgent, }); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - _driver = new ChromeDriver(chromeOptions); - } + _driver = BrowserDrivers.CreateChrome(chromeOptions); _driver.Navigate().GoToUrl(_proxyUrl); IJavaScriptExecutor js = _driver; diff --git a/ClientSideOverrides/ExampleRenderTests.cs b/ClientSideOverrides/ExampleRenderTests.cs index f950c4a..ace05c1 100644 --- a/ClientSideOverrides/ExampleRenderTests.cs +++ b/ClientSideOverrides/ExampleRenderTests.cs @@ -89,15 +89,7 @@ public void Example_RendersRealDetectionResult() chromeOptions.AddArgument("--headless"); chromeOptions.AddArgument($"--user-agent={DesktopChromeUserAgent}"); - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions); - _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions); - } - else - { - _driver = new ChromeDriver(chromeOptions); - } + _driver = BrowserDrivers.CreateChrome(chromeOptions); _driver.Navigate().GoToUrl(_example.BaseUrl); diff --git a/ClientSideOverrides/LocationClientSideOverrideTests.cs b/ClientSideOverrides/LocationClientSideOverrideTests.cs index 80151e6..3640b4f 100644 --- a/ClientSideOverrides/LocationClientSideOverrideTests.cs +++ b/ClientSideOverrides/LocationClientSideOverrideTests.cs @@ -8,6 +8,7 @@ using System.Net; using System.Threading; using FiftyOne.Pipeline.Cloud.Tests.Common.TestElements; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; namespace FiftyOne.Pipeline.Cloud.SeleniumTests.ClientSideOverrides { @@ -88,7 +89,9 @@ public void JavaScript_ClientSideOverrides() chromeOptions.SetLoggingPreference(LogType.Browser, LogLevel.All); - driver = new ChromeDriver(chromeOptions); + // Cast because the CDP command below is Chrome's own. A + // remote Selenium is not supported by this test. + driver = (ChromeDriver)BrowserDrivers.CreateChrome(chromeOptions); // Allow geo-location on the test client website. driver.ExecuteCdpCommand("Browser.grantPermissions", diff --git a/Helpers/BrowserDrivers.cs b/Helpers/BrowserDrivers.cs new file mode 100644 index 0000000..481fd59 --- /dev/null +++ b/Helpers/BrowserDrivers.cs @@ -0,0 +1,385 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Edge; +using OpenQA.Selenium.Firefox; +using OpenQA.Selenium.Remote; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers +{ + /// + /// Creates the web drivers the tests use, from one place, so every test + /// finds a browser the same way. + /// + /// + /// + /// A driver the machine already provides is used first, because a machine + /// that ships one is faster and steadier than downloading one for every + /// run. GitHub's Linux runner images name theirs in CHROMEWEBDRIVER, + /// GECKOWEBDRIVER and EDGEWEBDRIVER, and the ARM64 images set only + /// GECKOWEBDRIVER, as they carry Firefox and geckodriver but no Chrome, + /// no Chromium and no Edge. + /// + /// + /// When the machine provides no driver, Selenium Manager is left to fetch + /// one. That is a helper program shipped inside the Selenium.WebDriver + /// package, and up to version 4.48 the only Linux build of it was for x64, + /// under a folder named for Linux with no architecture in the name. On an + /// ARM64 Linux machine that build was picked and could not start, so every + /// browser test died with "Exec format error" before a browser was ever + /// launched. Version 4.49 splits the folder by architecture and adds an + /// ARM64 build, which is why this project needs 4.49 or later. + /// + /// + /// Giving Selenium a driver service that already knows its path stops it + /// calling Selenium Manager at all, which is the behaviour the first case + /// relies on. + /// + /// + public static class BrowserDrivers + { + /// + /// Set to the address of a Selenium grid or container to drive a + /// browser there instead of on this machine. + /// + public const string SeleniumUrlVariable = "SELENIUM_URL"; + + /// + /// Directory holding chromedriver, or the driver itself. GitHub's + /// Linux x64 runner images set this variable. + /// + public const string ChromeDriverVariable = "CHROMEWEBDRIVER"; + + /// + /// Directory holding geckodriver, or the driver itself. GitHub's + /// Linux runner images set this variable on both architectures. + /// + public const string GeckoDriverVariable = "GECKOWEBDRIVER"; + + /// + /// Directory holding msedgedriver, or the driver itself. GitHub's + /// Linux x64 runner images set this variable. + /// + public const string EdgeDriverVariable = "EDGEWEBDRIVER"; + + /// Path to the Chrome or Chromium binary to drive. + public const string ChromeBinaryVariable = "CHROME_BIN"; + + /// Path to the Firefox binary to drive. + public const string FirefoxBinaryVariable = "FIREFOX_BIN"; + + /// Path to the Edge binary to drive. + public const string EdgeBinaryVariable = "EDGE_BIN"; + + /// File names a Chrome driver goes by. + public static readonly string[] ChromeDriverNames = + { "chromedriver", "chromedriver.exe" }; + + /// File names a Firefox driver goes by. + public static readonly string[] GeckoDriverNames = + { "geckodriver", "geckodriver.exe" }; + + /// File names an Edge driver goes by. + public static readonly string[] EdgeDriverNames = + { "msedgedriver", "msedgedriver.exe" }; + + // Chromium is what an ARM64 Linux machine is most likely to have, so + // it is looked for alongside Chrome. + private static readonly string[] _chromeBinaryNames = + { + "google-chrome", "google-chrome-stable", "chromium-browser", + "chromium", "chrome.exe", + }; + + private static readonly string[] _firefoxBinaryNames = + { "firefox", "firefox.exe" }; + + private static readonly string[] _edgeBinaryNames = + { "microsoft-edge", "microsoft-edge-stable", "msedge.exe" }; + + /// + /// True on ARM64 Linux, where Microsoft publishes no Edge browser and + /// no Edge driver, so an Edge test there can only be skipped. + /// + public static bool IsLinuxArm64 { get; } = + RuntimeInformation.IsOSPlatform(OSPlatform.Linux) + && RuntimeInformation.ProcessArchitecture == Architecture.Arm64; + + /// + /// Creates a Chrome or Chromium driver. + /// + /// Options for the browser. + /// A driver, which the caller quits. + public static WebDriver CreateChrome(ChromeOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + if (TryCreateRemote(options, out var remote)) + { + return remote; + } + SetBinaryLocation(options, ChromeBinaryVariable, _chromeBinaryNames); + var driverPath = FindDriver(ChromeDriverVariable, ChromeDriverNames); + if (driverPath != null) + { + return new ChromeDriver( + ChromeDriverService.CreateDefaultService(driverPath), + options); + } + return Fetched( + () => new ChromeDriver(options), + "Chrome", ChromeDriverVariable, ChromeDriverNames); + } + + /// + /// Creates a Firefox driver. + /// + /// Options for the browser. + /// A driver, which the caller quits. + public static WebDriver CreateFirefox(FirefoxOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + if (TryCreateRemote(options, out var remote)) + { + return remote; + } + SetBinaryLocation( + options, FirefoxBinaryVariable, _firefoxBinaryNames); + var driverPath = FindDriver(GeckoDriverVariable, GeckoDriverNames); + if (driverPath != null) + { + return new FirefoxDriver( + FirefoxDriverService.CreateDefaultService(driverPath), + options); + } + return Fetched( + () => new FirefoxDriver(options), + "Firefox", GeckoDriverVariable, GeckoDriverNames); + } + + /// + /// Creates an Edge driver. + /// + /// Options for the browser. + /// A driver, which the caller quits. + public static WebDriver CreateEdge(EdgeOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + if (TryCreateRemote(options, out var remote)) + { + return remote; + } + var driverPath = FindDriver(EdgeDriverVariable, EdgeDriverNames); + if (driverPath == null && IsLinuxArm64) + { + throw new WebDriverException( + "Edge cannot run on ARM64 Linux, because Microsoft " + + "publishes no Edge browser and no Edge driver for it. " + + $"Set {SeleniumUrlVariable} to a Selenium that has " + + "Edge, or run the Edge tests on another architecture."); + } + SetBinaryLocation(options, EdgeBinaryVariable, _edgeBinaryNames); + if (driverPath != null) + { + return new EdgeDriver( + EdgeDriverService.CreateDefaultService(driverPath), + options); + } + return Fetched( + () => new EdgeDriver(options), + "Edge", EdgeDriverVariable, EdgeDriverNames); + } + + /// + /// Runs , which leaves Selenium Manager to + /// fetch a driver, and on failure says what was looked for here first, + /// so the reason reads as a missing browser rather than as a stack + /// trace about a helper program. + /// + /// Creates the driver. + /// Name of the browser, for the message. + /// Variable that would name the driver. + /// File names looked for on the path. + /// A driver, which the caller quits. + private static WebDriver Fetched( + Func create, + string browser, + string variable, + IEnumerable names) + { + try + { + return create(); + } + catch (Exception e) + { + throw new WebDriverException( + $"No {browser} driver on this machine, so Selenium " + + "Manager was left to fetch one, and it could not. " + + $"Looked at {variable}, and for " + + $"{string.Join(", ", names)} on the path. This machine " + + $"is {RuntimeInformation.RuntimeIdentifier}. Set " + + $"{variable} to a driver, or {SeleniumUrlVariable} to a " + + $"Selenium that has {browser}. Selenium said: " + + e.Message, + e); + } + } + + /// + /// Creates a driver on a remote Selenium when SELENIUM_URL is set. + /// + /// Options for the browser. + /// The driver created, or null. + /// True when SELENIUM_URL is set. + private static bool TryCreateRemote( + DriverOptions options, out WebDriver driver) + { + if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + { + ExternalSeleniumHelper.AddExternalSeleniumArguments(options); + driver = new RemoteWebDriver(new Uri(seleniumUrl), options); + return true; + } + driver = null; + return false; + } + + /// + /// Path to a driver this machine provides, or null when it provides + /// none. + /// + /// Variable that names the driver. + /// File names to look for on the path. + /// A path, or null. + public static string FindDriver( + string variable, IEnumerable names) => + FindDriver(Environment.GetEnvironmentVariable, variable, names); + + /// + /// As , reading + /// the variables through so the rules + /// can be tested without changing the environment of the whole run. + /// + /// Reads an environment variable. + /// Variable that names the driver. + /// File names to look for on the path. + /// A path, or null. + public static string FindDriver( + Func getVariable, + string variable, + IEnumerable names) + { + if (getVariable == null) + { + throw new ArgumentNullException(nameof(getVariable)); + } + var configured = getVariable(variable); + if (string.IsNullOrEmpty(configured) == false) + { + if (File.Exists(configured)) + { + return configured; + } + var inDirectory = FindInDirectory(configured, names); + if (inDirectory != null) + { + return inDirectory; + } + } + return FindOnPath(getVariable, names); + } + + /// + /// Points the options at a browser this machine provides, leaving them + /// alone when it provides none, so Selenium Manager still fetches one. + /// + /// Options for the browser. + /// Variable that names the browser. + /// File names to look for on the path. + private static void SetBinaryLocation( + DriverOptions options, string variable, IEnumerable names) + { + var configured = Environment.GetEnvironmentVariable(variable); + var path = string.IsNullOrEmpty(configured) == false + && File.Exists(configured) + ? configured + : FindOnPath(Environment.GetEnvironmentVariable, names); + if (path != null) + { + options.BinaryLocation = path; + } + } + + /// + /// The first of found in a directory named by + /// PATH, or null. + /// + /// Reads an environment variable. + /// File names to look for. + /// A path, or null. + private static string FindOnPath( + Func getVariable, IEnumerable names) + { + var path = getVariable("PATH"); + if (string.IsNullOrEmpty(path)) + { + return null; + } + foreach (var directory in path.Split(Path.PathSeparator)) + { + if (string.IsNullOrWhiteSpace(directory)) + { + continue; + } + var found = FindInDirectory(directory, names); + if (found != null) + { + return found; + } + } + return null; + } + + /// + /// The first of present in + /// , or null. + /// + /// Directory to look in. + /// File names to look for. + /// A path, or null. + private static string FindInDirectory( + string directory, IEnumerable names) + { + foreach (var name in names) + { + string candidate; + try + { + candidate = Path.Combine(directory, name); + } + catch (ArgumentException) + { + // A PATH entry holding characters a path cannot. + return null; + } + if (File.Exists(candidate)) + { + return candidate; + } + } + return null; + } + } +} diff --git a/Helpers/BrowserDriversTests.cs b/Helpers/BrowserDriversTests.cs new file mode 100644 index 0000000..2192525 --- /dev/null +++ b/Helpers/BrowserDriversTests.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Generic; +using System.IO; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Tests +{ + /// + /// Checks the rules for finding a driver the machine provides. No browser + /// is started, so these run anywhere. + /// + [TestClass] + public class BrowserDriversTests + { + private string _directory; + private string _driverFile; + + /// Makes a directory holding a file named like a driver. + [TestInitialize] + public void Init() + { + _directory = Path.Combine( + Path.GetTempPath(), "51d-driver-" + Guid.NewGuid()); + Directory.CreateDirectory(_directory); + _driverFile = Path.Combine(_directory, "chromedriver"); + File.WriteAllText(_driverFile, string.Empty); + } + + /// Removes the directory. + [TestCleanup] + public void Cleanup() + { + if (_directory != null && Directory.Exists(_directory)) + { + Directory.Delete(_directory, true); + } + } + + /// + /// A variable naming the directory the driver is in finds the driver, + /// which is the shape GitHub's runner images use. + /// + [TestMethod] + public void FindDriver_ReadsDirectoryFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.ChromeDriverVariable, _directory }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// A variable naming the driver itself finds the driver. + [TestMethod] + public void FindDriver_ReadsFileFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.ChromeDriverVariable, _driverFile }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// With no variable set the path is searched, so a machine that just + /// has a driver installed needs no configuration. + /// + [TestMethod] + public void FindDriver_FallsBackToThePath() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { "PATH", _directory }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// A variable set to a directory holding no driver does not stop the + /// path being searched, which is what an ARM64 runner does with + /// CHROMEWEBDRIVER left empty. + /// + [TestMethod] + public void FindDriver_SearchesThePathWhenTheVariableIsWrong() + { + var empty = Path.Combine(_directory, "empty"); + Directory.CreateDirectory(empty); + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.ChromeDriverVariable, empty }, + { "PATH", _directory }, + }), + BrowserDrivers.ChromeDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// Nothing anywhere gives null, which is what sends the caller on to + /// Selenium Manager. + /// + [TestMethod] + public void FindDriver_ReturnsNullWhenNothingProvidesOne() + { + Assert.IsNull( + BrowserDrivers.FindDriver( + Lookup(new Dictionary()), + BrowserDrivers.GeckoDriverVariable, + BrowserDrivers.GeckoDriverNames)); + } + + /// + /// Every driver name is looked for, so a Windows machine holding + /// chromedriver.exe is found as readily as a Linux one. + /// + [TestMethod] + public void FindDriver_FindsEveryNameItIsGiven() + { + var windowsName = Path.Combine(_directory, "geckodriver.exe"); + File.WriteAllText(windowsName, string.Empty); + Assert.AreEqual( + windowsName, + BrowserDrivers.FindDriver( + Lookup(new Dictionary + { + { BrowserDrivers.GeckoDriverVariable, _directory }, + }), + BrowserDrivers.GeckoDriverVariable, + BrowserDrivers.GeckoDriverNames)); + } + + /// Reads from a dictionary in place of the environment. + /// Variables and their values. + /// A lookup over those values. + private static Func Lookup( + IReadOnlyDictionary values) => + name => values.TryGetValue(name, out var value) ? value : null; + } +} diff --git a/README.md b/README.md index 2b76964..671b930 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ sibling directory and run it at their integration-test step. |---|---|---| | `Contract` | An example app serves `51Degrees.core.js`, client-side evidence flows back, and the server-rendered page shows a real detection result. | Cloud CI (per example, vs `:8080`) **and** every API CI (vs the public cloud). | | `CloudInternal` | Cloud response behaviour through a browser: cache reuse, COEP/CORP headers, third-party cookies, client-side overrides, and the per-browser JS endpoints. | Cloud CI only (vs `:8080`). | +| `Browser` | That a browser starts at all and runs the script on a page served from the test process. No cloud, no key, no example. | This repository's own CI, on every runner image it uses. | Select a subset with `--filter TestCategory=Contract` or `--filter TestCategory=CloudInternal`. @@ -29,8 +30,10 @@ Select a subset with `--filter TestCategory=Contract` or that is already running was pointed at its data by whoever started it, so with `EXAMPLE_URL` set the suite needs neither `CLOUD_ROOT_URL` nor `PAID_RESOURCE_KEY`, which is what lets an on-premise example run `Contract`. -- **Browser** - by default a local Chrome driver is used. Set `SELENIUM_URL` to use - a Selenium grid instead (CI uses a standalone grid). +- **Browser** - a driver the machine already provides is used first, named by + `CHROMEWEBDRIVER`, `GECKOWEBDRIVER` or `EDGEWEBDRIVER` or found on the path. + When the machine provides none, Selenium Manager fetches one. Set + `SELENIUM_URL` to drive a browser on a Selenium grid instead. ## Configuration @@ -43,7 +46,9 @@ and no keys are committed. | `PAID_RESOURCE_KEY` | `CloudInternal`, and `Contract` when the suite launches the example | Resource key used by the tests. | | `FREE_RESOURCE_KEY` | `CloudInternal` | Free resource key for the JS-endpoint tests. | | `ENTERPRISE_V4_LICENSE` | `CloudInternal` | License passed to the JS endpoint to unlock paid properties. | -| `SELENIUM_URL` | optional | Selenium grid URL, omit for a local Chrome driver. | +| `SELENIUM_URL` | optional | Selenium grid URL, omit to drive a browser on this machine. | +| `CHROMEWEBDRIVER` / `GECKOWEBDRIVER` / `EDGEWEBDRIVER` | optional | A driver, or the directory holding one. GitHub's Linux runner images set these. | +| `CHROME_BIN` / `FIREFOX_BIN` / `EDGE_BIN` | optional | The browser binary to drive, when it is not on the path. | | `EXAMPLE_URL` / `EXAMPLE_LANG` | `Contract` | The example app to test (CI / local). | A missing variable only fails the tests that read it, and the failure names the @@ -98,9 +103,30 @@ dotnet test --filter TestCategory=CloudInternal ## This repository's own CI -The "Build and test" workflow builds the suite and runs the tests that need no -browser, no cloud and no keys, on every push and pull request. It is what -proves a change here before any language repository picks it up. +The "Build and test" workflow builds the suite on every push and pull request, +and runs it in two jobs. The first runs the tests that need no browser, no +cloud and no keys. The second starts Chrome and Firefox on `ubuntu-latest`, +`ubuntu-22.04-arm` and `ubuntu-24.04-arm`, and prints what each runner +provides before it does. Together they prove a change here before any language +repository picks it up. + +## Browsers by architecture + +GitHub's ARM64 Linux runner images carry Firefox and geckodriver and set +`GECKOWEBDRIVER`, but no Chrome, no Chromium, no ChromeDriver and no Edge, +and they leave `CHROMEWEBDRIVER` and `EDGEWEBDRIVER` unset. The x64 images +carry all of them. + +Chrome still runs on ARM64, because Selenium Manager fetches the Chrome for +Testing `linux-arm64` build and a matching driver. That needs +Selenium.WebDriver 4.49.0 or later, which is the first version whose Selenium +Manager ships an ARM64 Linux build. Earlier versions carry only an x64 one, +under a folder named for Linux with no architecture in the name, so on an +ARM64 runner it is picked, cannot start, and every browser test dies in a few +milliseconds with "Exec format error". Do not downgrade the package. + +Edge cannot run on ARM64 Linux at all. Microsoft publishes neither the browser +nor the driver for it, so the Edge tests say so and skip. ## CI integration diff --git a/Selenium/DriverSmokeTests.cs b/Selenium/DriverSmokeTests.cs new file mode 100644 index 0000000..1299945 --- /dev/null +++ b/Selenium/DriverSmokeTests.cs @@ -0,0 +1,155 @@ +using System; +using System.Threading; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Edge; +using OpenQA.Selenium.Firefox; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium +{ + /// + /// Checks that a browser really starts and really runs the page, on + /// whatever machine the suite is on. + /// + /// + /// + /// Every other browser test needs a cloud, a resource key and an example + /// application, so a machine that cannot start a browser at all is only + /// found out in the middle of a test that was trying to check something + /// else. These need none of that, only a page served from this process, so + /// CI can put them on each runner it uses and see for itself whether a + /// browser runs there. + /// + /// + /// They fail rather than skip when no browser can be started, because a + /// run that quietly skipped them would say nothing, and saying nothing is + /// the fault these were written for. + /// + /// + [TestClass, TestCategory("Browser")] + public class DriverSmokeTests + { + // A page that writes something only a running browser can write, so a + // driver that connected but never rendered is not mistaken for a pass. + private const string Page = + "Driver smoke" + + "

not run

"; + + private const string WhereElementId = "where"; + private const int PageLoadTimeoutSeconds = 30; + + private string _url; + private CancellationTokenSource _cts; + private TestHelpers.ServerListener _server; + private WebDriver _driver; + + /// Serves the page from this process. + [TestInitialize] + public void Init() + { + _url = $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/"; + _cts = new CancellationTokenSource(); + _server = TestHelpers.SimpleListener(_url, Page, _cts.Token); + } + + /// Quits the browser and stops the server. + [TestCleanup] + public void Cleanup() + { + _driver?.Quit(); + _driver = null; + _cts?.Cancel(); + _server?.Listener?.Stop(); + _server?.Listener?.Close(); + _cts?.Dispose(); + } + + /// Chrome or Chromium starts and runs the page. + [TestMethod] + public void Chrome_RunsThePage() + { + var options = new ChromeOptions(); + options.AddArgument("--headless"); + options.AddArgument("--no-sandbox"); + options.AddArgument("--disable-dev-shm-usage"); + _driver = BrowserDrivers.CreateChrome(options); + AssertPageRan(); + } + + /// Firefox starts and runs the page. + [TestMethod] + public void Firefox_RunsThePage() + { + var options = new FirefoxOptions(); + options.AddArgument("--headless"); + _driver = BrowserDrivers.CreateFirefox(options); + AssertPageRan(); + } + + /// + /// Edge starts and runs the page, where Edge exists at all. Microsoft + /// publishes no Edge for ARM64 Linux, so there the test says so and + /// skips instead of failing on something that cannot be fixed here. + /// + /// + /// Ignored for the same reason the other Edge tests in this suite + /// are, and proved again by the first run of this test: headless Edge + /// on the x64 Ubuntu runner accepts the session and then hangs on the + /// first navigation until the sixty second timeout. That is a fault of + /// its own and not the ARM64 one this class was added for, so the test + /// is kept, with its reason where a reader will find it, rather than + /// deleted. + /// + [TestMethod] + [Ignore("Headless Edge on the Ubuntu runners hangs on the first " + + "navigation, and Microsoft publishes no Edge at all for ARM64 " + + "Linux. Chrome and Firefox cover both architectures.")] + public void Edge_RunsThePage() + { + var options = new EdgeOptions(); + options.AddArgument("--headless"); + options.AddArgument("--no-sandbox"); + options.AddArgument("--disable-dev-shm-usage"); + try + { + _driver = BrowserDrivers.CreateEdge(options); + } + catch (WebDriverException e) when (BrowserDrivers.IsLinuxArm64) + { + Assert.Inconclusive(e.Message); + } + AssertPageRan(); + } + + /// + /// Loads the page and checks the browser ran the script on it. + /// + private void AssertPageRan() + { + _driver.Navigate().GoToUrl(_url); + var deadline = DateTime.UtcNow + + TimeSpan.FromSeconds(PageLoadTimeoutSeconds); + string text = null; + while (DateTime.UtcNow < deadline) + { + text = _driver.FindElement(By.Id(WhereElementId)).Text; + if (text != null && text.StartsWith( + "ran in ", StringComparison.Ordinal)) + { + Console.WriteLine($" Browser reported: {text}"); + return; + } + Thread.Sleep(100); + } + Assert.Fail( + "the browser loaded the page but never ran the script on it, " + + $"as the element reads '{text}'"); + } + } +} diff --git a/SeleniumApiTests.csproj b/SeleniumApiTests.csproj index e36a626..68d3447 100644 --- a/SeleniumApiTests.csproj +++ b/SeleniumApiTests.csproj @@ -24,7 +24,11 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + + diff --git a/ThirdPartyCookieTests.cs b/ThirdPartyCookieTests.cs index b3f82c1..8a01f96 100644 --- a/ThirdPartyCookieTests.cs +++ b/ThirdPartyCookieTests.cs @@ -82,12 +82,7 @@ private ChromeOptions CreateBaseOptions() /// private IWebDriver CreateDriver(ChromeOptions options) { - if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) - { - ExternalSeleniumHelper.AddExternalSeleniumArguments(options); - return new RemoteWebDriver(new Uri(seleniumUrl), options); - } - return new ChromeDriver(options); + return BrowserDrivers.CreateChrome(options); } ///