FIX: Run the browser tests on ARM64 Linux instead of reporting passes that never happened

.github/workflows/build-and-test.yml

@@ -1,5 +1,11 @@
-# Builds the suite and runs the tests that need no browser, no cloud and no
-# keys, so a change here is checked before a language repository picks it up.
+# Builds the suite and runs it, so a change here is checked before a language
+# repository picks it up.
+#
+# Two jobs, because they answer different questions. The first runs the tests
+# that need no browser, no cloud and no keys. The second starts a real browser
+# on each runner the language repositories use, including the ARM64 Linux ones,
+# and fails when one will not start, which is the fault the second job exists
+# to catch.
 name: Build and test
 
 on:
@@ -28,4 +34,41 @@ jobs:
       - name: Test
         run: >
           dotnet test -c Release --no-build
-          --filter "TestCategory!=Contract&TestCategory!=CloudInternal"
+          --filter "TestCategory!=Contract&TestCategory!=CloudInternal&TestCategory!=Browser"
+
+  browsers:
+    name: Browsers on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # The ARM64 images are here because a browser suite that could not
+        # start a browser on them was reported as passing for as long as nobody
+        # read the log. ubuntu-22.04-arm is the image the language repositories
+        # run their ARM jobs on.
+        os: [ubuntu-latest, ubuntu-22.04-arm, ubuntu-24.04-arm]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '10.0.x'
+
+      # Say what the runner brought, so a later change of image is readable
+      # from the log rather than guessed at.
+      - name: What this runner provides
+        run: |
+          echo "arch: $(uname -m)"
+          echo "CHROMEWEBDRIVER: ${CHROMEWEBDRIVER:-<unset>}"
+          echo "GECKOWEBDRIVER: ${GECKOWEBDRIVER:-<unset>}"
+          echo "EDGEWEBDRIVER: ${EDGEWEBDRIVER:-<unset>}"
+          for exe in google-chrome chromium chromedriver firefox geckodriver \
+                     microsoft-edge msedgedriver; do
+            echo "$exe: $(command -v $exe || echo '<not on the path>')"
+          done
+
+      - name: Build
+        run: dotnet build -c Release
+
+      - name: Start a browser
+        run: dotnet test -c Release --no-build --filter "TestCategory=Browser"

BrowserCache/CacheTests.cs

@@ -89,15 +89,7 @@ public void JavaScript_BrowserCache_Chrome()
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
-            }
-            else
-            {
-                driver = new ChromeDriver(options);
-            }
+            driver = BrowserDrivers.CreateChrome(options);
 
             RunTest(driver);
         }
@@ -113,15 +105,7 @@ public void JavaScript_BrowserCache_Edge()
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
-            }
-            else
-            {
-                driver = new EdgeDriver(options);
-            }
+            driver = BrowserDrivers.CreateEdge(options);
 
             RunTest(driver);
         }
@@ -136,15 +120,7 @@ public void JavaScript_BrowserCache_FireFox()
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
-            }
-            else
-            {
-                driver = new FirefoxDriver(options);
-            }
+            driver = BrowserDrivers.CreateFirefox(options);
 
             RunTest(driver);
         }

BrowserCache/SessionStorageCacheTests.cs

@@ -138,15 +138,7 @@ public void SessionStorageCache_Chrome(bool enableCookies)
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
-            }
-            else
-            {
-                driver = new ChromeDriver(options);
-            }
+            driver = BrowserDrivers.CreateChrome(options);
 
             RunTest(driver, enableCookies);
         }
@@ -163,15 +155,7 @@ public void SessionStorageCache_FireFox(bool enableCookies)
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
-            }
-            else
-            {
-                driver = new FirefoxDriver(options);
-            }
+            driver = BrowserDrivers.CreateFirefox(options);
 
             RunTest(driver, enableCookies);
         }

Browsers/ChromeTests.cs

@@ -39,22 +39,13 @@ public static void ClassInit(TestContext context)
             var options = new ChromeOptions();
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
+            try
             {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
+                s_driver = BrowserDrivers.CreateChrome(options);
             }
-            else
+            catch (WebDriverException e)
             {
-                try
-                {
-                    s_driver = new ChromeDriver(options);
-                }
-                catch (WebDriverException)
-                {
-                    Assert.Inconclusive("Could not create a ChromeDriver, check " +
-                        "that the Chromium driver is installed");
-                }
+                Assert.Inconclusive(e.Message);
             }
         }
 

Browsers/EdgeTests.cs

@@ -39,22 +39,13 @@ public static void ClassInit(TestContext context)
             var options = new EdgeOptions();
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
+            try
             {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
+                s_driver = BrowserDrivers.CreateEdge(options);
             }
-            else
+            catch (WebDriverException e)
             {
-                try
-                {
-                    s_driver = new EdgeDriver(options);
-                }
-                catch (WebDriverException)
-                {
-                    Assert.Inconclusive("Could not create an EdgeDriver, check " +
-                        "that the MS edge driver is installed");
-                }
+                Assert.Inconclusive(e.Message);
             }
         }
 

Browsers/GeckoTests.cs

@@ -38,22 +38,13 @@ public static void ClassInit(TestContext context)
             var options = new FirefoxOptions();
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
+            try
             {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                s_driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
+                s_driver = BrowserDrivers.CreateFirefox(options);
             }
-            else
+            catch (WebDriverException e)
             {
-                try
-                {
-                    s_driver = new FirefoxDriver(options);
-                }
-                catch (WebDriverException)
-                {
-                    Assert.Inconclusive("Could not create a gecko driver, check " +
-                        "that the gecko driver is installed");
-                }
+                Assert.Inconclusive(e.Message);
             }
         }
 

COEP/CoepTests.cs

@@ -105,15 +105,7 @@ await response.OutputStream.WriteAsync(
             options.AcceptInsecureCertificates = true;
             options.AddArgument("--headless");
             options.SetLoggingPreference(LogType.Browser, LogLevel.All);
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(options);
-                driver = new RemoteWebDriver(new Uri(seleniumUrl), options);
-            }
-            else
-            {
-                driver = new ChromeDriver(options);
-            }
+            driver = BrowserDrivers.CreateChrome(options);
         }
 
         /// <summary>

ClientSideOverrides/DeviceClientSideOverrideTests.cs

@@ -123,15 +123,7 @@ public void JavaScript_ClientSideOverrides(
                     UserAgent = userAgent,
                 });
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions);
-                driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions);
-            }
-            else
-            {
-                driver = new ChromeDriver(chromeOptions);
-            }
+            driver = BrowserDrivers.CreateChrome(chromeOptions);
             driver.Manage().Cookies.DeleteAllCookies();
 
             driver.Navigate().GoToUrl(ClientServerUrl);

ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs

@@ -102,15 +102,7 @@ public void Example_RendersClientSideOverridesOnThePage(
                 UserAgent = userAgent,
             });
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions);
-                _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions);
-            }
-            else
-            {
-                _driver = new ChromeDriver(chromeOptions);
-            }
+            _driver = BrowserDrivers.CreateChrome(chromeOptions);
 
             _driver.Navigate().GoToUrl(_proxyUrl);
 

ClientSideOverrides/ExampleClientSideOverrideTests.cs

@@ -87,15 +87,7 @@ public void Example_ServesCoreJs_AndClientSideOverridesFlow(
                 UserAgent = userAgent,
             });
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions);
-                _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions);
-            }
-            else
-            {
-                _driver = new ChromeDriver(chromeOptions);
-            }
+            _driver = BrowserDrivers.CreateChrome(chromeOptions);
 
             _driver.Navigate().GoToUrl(_proxyUrl);
             IJavaScriptExecutor js = _driver;

ClientSideOverrides/ExampleRenderTests.cs

@@ -89,15 +89,7 @@ public void Example_RendersRealDetectionResult()
             chromeOptions.AddArgument("--headless");
             chromeOptions.AddArgument($"--user-agent={DesktopChromeUserAgent}");
 
-            if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl))
-            {
-                ExternalSeleniumHelper.AddExternalSeleniumArguments(chromeOptions);
-                _driver = new RemoteWebDriver(new Uri(seleniumUrl), chromeOptions);
-            }
-            else
-            {
-                _driver = new ChromeDriver(chromeOptions);
-            }
+            _driver = BrowserDrivers.CreateChrome(chromeOptions);
 
             _driver.Navigate().GoToUrl(_example.BaseUrl);
 

ClientSideOverrides/LocationClientSideOverrideTests.cs

@@ -8,6 +8,7 @@
 using System.Net;
 using System.Threading;
 using FiftyOne.Pipeline.Cloud.Tests.Common.TestElements;
+using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers;
 
 namespace FiftyOne.Pipeline.Cloud.SeleniumTests.ClientSideOverrides
 {
@@ -88,7 +89,9 @@ public void JavaScript_ClientSideOverrides()
 
             chromeOptions.SetLoggingPreference(LogType.Browser, LogLevel.All);
 
-            driver = new ChromeDriver(chromeOptions);
+            // Cast because the CDP command below is Chrome's own. A
+            // remote Selenium is not supported by this test.
+            driver = (ChromeDriver)BrowserDrivers.CreateChrome(chromeOptions);
 
             // Allow geo-location on the test client website.
             driver.ExecuteCdpCommand("Browser.grantPermissions",