diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 657775a..37edb85 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,11 +1,12 @@ # Builds the suite and runs it, so a change here is checked before a language # repository picks it up. # -# Two jobs, because they answer different questions. The first runs the tests +# Three jobs, because they answer different questions. The first runs the tests # that need no browser, no cloud and no keys. The second starts a real browser # on each runner the language repositories use, including the ARM64 Linux ones, # and fails when one will not start, which is the fault the second job exists -# to catch. +# to catch. The third starts CloakBrowser in its vendor's container and runs +# the same check with a driver attached to it. name: Build and test on: @@ -72,3 +73,64 @@ jobs: - name: Start a browser run: dotnet test -c Release --no-build --filter "TestCategory=Browser" + + cloak: + name: CloakBrowser on ubuntu-latest + runs-on: ubuntu-latest + env: + # The vendor's image by digest, so the browser cannot change under the + # tag. The browser is closed source and only ever runs in this container. + CLOAK_IMAGE: cloakhq/cloakbrowser:0.5.12@sha256:2fdd1289154f30594c7bab74943ea7baa6b617a9ec026ab4bf339c2ebabcb712 + CLOAK_DEBUGGER_ADDRESS: 127.0.0.1:9222 + # Selenium Manager fetches the driver for the browser in the container + # and would otherwise download a Chrome of that version that nothing + # runs. + SE_AVOID_BROWSER_DOWNLOAD: 'true' + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + # Host networking lets the browser reach the page the tests serve on + # localhost. Update checks are off, so the browser that runs is the one + # in the image. Nothing from the runner is mounted and no secret is + # passed. The first request for the version starts the browser, and + # its answer is printed so the log says which browser ran. + - name: Start CloakBrowser + run: | + docker run -d --name cloak --network host \ + -e CLOAKBROWSER_AUTO_UPDATE=false \ + "$CLOAK_IMAGE" cloakserve + for attempt in $(seq 1 30); do + if curl -fsS "http://$CLOAK_DEBUGGER_ADDRESS/json/version"; then + echo + exit 0 + fi + sleep 2 + done + echo "CloakBrowser did not answer on $CLOAK_DEBUGGER_ADDRESS" + docker logs cloak + exit 1 + + - name: Build + run: dotnet build -c Release + + # Only the tests that attach to CloakBrowser run here, as the browsers + # job already starts Chrome and Firefox on this image. A Cloak test that + # was skipped would leave this job green having attached to nothing, so + # the pass is looked for by name. + - name: Attach to CloakBrowser + run: | + set -o pipefail + dotnet test -c Release --no-build \ + --filter "TestCategory=Browser&Name~Cloak_" \ + --logger "console;verbosity=normal" | tee browser-tests.log + grep -q "Passed Cloak_RunsThePage" browser-tests.log + + - name: Stop CloakBrowser + if: always() + run: | + docker logs cloak || true + docker rm -f cloak || true diff --git a/BrowserCache/SessionStorageCacheTests.cs b/BrowserCache/SessionStorageCacheTests.cs index 0aebb0b..b9e87cd 100644 --- a/BrowserCache/SessionStorageCacheTests.cs +++ b/BrowserCache/SessionStorageCacheTests.cs @@ -160,6 +160,22 @@ public void SessionStorageCache_FireFox(bool enableCookies) RunTest(driver, enableCookies); } + /// + /// Session storage cache behaviour in CloakBrowser. + /// + [TestMethod, RequiresCloak] + [DataRow(true)] + [DataRow(false)] + public void SessionStorageCache_Cloak(bool enableCookies) + { + var options = new ChromeOptions(); + options.AcceptInsecureCertificates = true; + + driver = BrowserDrivers.CreateCloak(options); + + RunTest(driver, enableCookies); + } + private void RunTest(WebDriver driver, bool enableCookies) { ClientServerUrl = $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/"; diff --git a/Browsers/CloakTests.cs b/Browsers/CloakTests.cs new file mode 100644 index 0000000..476f355 --- /dev/null +++ b/Browsers/CloakTests.cs @@ -0,0 +1,172 @@ +using System; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using System.Linq; +using System.Net; +using System.Threading; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browsers +{ + /// + /// Performs the Selenium Tests of in + /// CloakBrowser, through a driver attached to the one + /// CLOAK_DEBUGGER_ADDRESS names, and is skipped when it names none. + /// Driver and HTTP server are created once per class for performance. + /// + /// + /// The script these tests run in the browser carries the resource key + /// and the license it was requested with, so both are given to + /// CloakBrowser. + /// + [RequiresCloak] + [TestClass, TestCategory("CloudInternal")] + public class CloakTests : SeleniumTestsBase + { + private static WebDriver s_driver; + private static string s_clientServerUrl; + private static HttpListener s_clientServer; + private static CancellationTokenSource s_cts; + + /// + /// Initializes the test class. + /// + /// The test context. + [ClassInitialize] + public static void ClassInit(TestContext context) + { + // Start the client server once for all tests in this class. + s_clientServerUrl = + $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/"; + s_cts = new CancellationTokenSource(); + s_clientServer = TestHelpers.SimpleListener( + s_clientServerUrl, s_cts.Token); + + var options = new ChromeOptions(); + options.AcceptInsecureCertificates = true; + s_driver = BrowserDrivers.CreateCloak(options); + } + + /// + /// Initializes the test. + /// + [TestInitialize] + public override void Init() + { + base.Init(); + Driver = s_driver; + ClientServerUrl = s_clientServerUrl; + Driver.Navigate().GoToUrl(ClientServerUrl); + } + + /// + /// Cleans up after the test class. + /// + [ClassCleanup] + public static void ClassCleanup() + { + s_driver?.Quit(); + s_driver = null; + StopListener(s_clientServer, s_cts); + } + + /// + /// Test expected behavior when request contains valid license. + /// + [TestMethod] + [DynamicData( + nameof(JavaScript_Get_ValidLicense_DATA), + typeof(SeleniumTestsBase))] + public void JavaScript_Get_ValidLicense( + string endpoint, + string objectName, + string[] values, + string license, + params string[] parameters) + { + var js = TestJavaScript( + endpoint, objectName, values, license, parameters); + + AssertEveryValueIsSet(js, objectName, values); + } + + /// + /// Test expected behavior when request does not contain the required + /// license. + /// + [TestMethod] + [DynamicData( + nameof(JavaScript_Get_NoLicense_DATA), + typeof(SeleniumTestsBase))] + public void JavaScript_Get_NoLicense( + string endpoint, + string[] values, + string license, + params string[] parameters) + { + var js = TestJavaScript(endpoint, "", values, license, parameters); + + foreach (var property in values) + { + var parts = property.Split('.'); + var element = string.Join('.', parts.Take(parts.Length - 1)); + var propertyName = parts.Last(); + + var propertyExists = js.ExecuteScript( + $"return fod.{element}.hasOwnProperty('{propertyName}');"); + Assert.IsTrue((bool)propertyExists); + + var value = js.ExecuteScript( + $"return fod.{element}.{propertyName};"); + Assert.IsNotNull( + value, $"fod.{element}.{propertyName} is null"); + + Assert.IsTrue(value.ToString().Contains( + $"{propertyName} is a paid feature.", + StringComparison.OrdinalIgnoreCase)); + } + } + + /// + /// Test expected behavior when the request asks for properties that + /// evidence gathered in the browser overrides. + /// + [TestMethod] + [DynamicData( + nameof(JavaScript_Get_ClientSideOverrides_DATA), + typeof(SeleniumTestsBase))] + public void JavaScript_Get_ClientSideOverrides( + string endpoint, + string objectName, + string[] values, + string license, + params string[] parameters) + { + var js = TestJavaScript( + endpoint, objectName, values, license, parameters); + + AssertEveryValueIsSet(js, objectName, values); + } + + /// + /// Checks the object the include made holds a value for every + /// property that was asked for. + /// + /// Runs script in the page. + /// Name of the object in the page. + /// Properties, each as element.property. + private static void AssertEveryValueIsSet( + IJavaScriptExecutor js, string objectName, string[] values) + { + foreach (var property in values) + { + var value = js.ExecuteScript( + $"return {objectName}.{property};"); + Assert.IsNotNull(value, $"{objectName}.{property} is null"); + } + } + } +} diff --git a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs index efaaaa8..de8ed38 100644 --- a/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverridePageContentTests.cs @@ -104,16 +104,57 @@ public void Example_RendersClientSideOverridesOnThePage( _driver = BrowserDrivers.CreateChrome(chromeOptions); + AssertRendersScreenSize(width, height); + } + + /// + /// The same in CloakBrowser, with the screen it reports for itself. + /// No device is emulated, as ChromeDriver refuses mobile emulation + /// for a browser it attaches to, so the size to expect is read from + /// the browser. + /// + [TestMethod, RequiresCloak] + public void Example_RendersClientSideOverridesOnThePage_Cloak() + { + var chromeOptions = new ChromeOptions(); + chromeOptions.AcceptInsecureCertificates = true; + + _driver = BrowserDrivers.CreateCloak(chromeOptions); + + var js = (IJavaScriptExecutor)_driver; + var width = Convert.ToInt64( + js.ExecuteScript("return window.screen.width")); + var height = Convert.ToInt64( + js.ExecuteScript("return window.screen.height")); + + // The server-side rows can read 0 for a desktop browser, so a + // screen of 0 would match whether or not the callback ran. + Assert.IsTrue( + width > 0 && height > 0, + $"the browser reports a screen of {width} by {height}, so " + + "the client-side results cannot be told from the server's"); + + AssertRendersScreenSize(width, height); + } + + /// + /// Loads the example and checks the client-side results it renders + /// against the size of the browser's screen. + /// + private void AssertRendersScreenSize(long width, long height) + { _driver.Navigate().GoToUrl(_proxyUrl); var rendered = WaitForRenderedResults(); Assert.AreEqual( width.ToString(), rendered[ScreenWidthLabel], - "the screen width rendered on the page does not match the emulated width"); + "the screen width rendered on the page does not match the " + + "width of the browser's screen"); Assert.AreEqual( height.ToString(), rendered[ScreenHeightLabel], - "the screen height rendered on the page does not match the emulated height"); + "the screen height rendered on the page does not match the " + + "height of the browser's screen"); // The device id is only rendered by examples whose copy of the // shared helper is current. An example still carrying an older copy diff --git a/ClientSideOverrides/ExampleClientSideOverrideTests.cs b/ClientSideOverrides/ExampleClientSideOverrideTests.cs index 3876b91..5f94fe8 100644 --- a/ClientSideOverrides/ExampleClientSideOverrideTests.cs +++ b/ClientSideOverrides/ExampleClientSideOverrideTests.cs @@ -89,6 +89,51 @@ public void Example_ServesCoreJs_AndClientSideOverridesFlow( _driver = BrowserDrivers.CreateChrome(chromeOptions); + var overrides = ReadOverrides(); + + Assert.AreEqual( + width, overrides.ScreenWidth, + "emulated screen width does not match"); + Assert.AreEqual( + height, overrides.ScreenHeight, + "emulated screen height does not match"); + + AssertOverridesResolved(overrides); + } + + /// + /// The same in CloakBrowser, with the screen it reports for itself. + /// No device is emulated, as ChromeDriver refuses mobile emulation + /// for a browser it attaches to. + /// + [TestMethod, RequiresCloak] + public void Example_ServesCoreJs_AndClientSideOverridesFlow_Cloak() + { + var chromeOptions = new ChromeOptions(); + chromeOptions.AcceptInsecureCertificates = true; + + _driver = BrowserDrivers.CreateCloak(chromeOptions); + + AssertOverridesResolved(ReadOverrides()); + } + + /// + /// The browser's screen size, and what the flow data holds once the + /// client-side callback has run. + /// + private sealed record Overrides( + long ScreenWidth, + long ScreenHeight, + long ScreenPixelsWidth, + long ScreenPixelsHeight, + string DeviceId); + + /// + /// Loads the example and waits for the client-side evidence to come + /// back as flow data. + /// + private Overrides ReadOverrides() + { _driver.Navigate().GoToUrl(_proxyUrl); IJavaScriptExecutor js = _driver; @@ -111,20 +156,31 @@ public void Example_ServesCoreJs_AndClientSideOverridesFlow( _ => "complete".Equals(js.ExecuteScript("return window.__t")) && !"loading".Equals(js.ExecuteScript("return window.__did"))); - var screenWidth = Convert.ToInt64(js.ExecuteScript("return window.__sw")); - var screenHeight = Convert.ToInt64(js.ExecuteScript("return window.__sh")); - var screenPixelWidth = Convert.ToInt64(js.ExecuteScript("return window.__spw")); - var screenPixelHeight = Convert.ToInt64(js.ExecuteScript("return window.__sph")); - var deviceId = js.ExecuteScript("return window.__did")?.ToString(); - - Assert.AreEqual(width, screenWidth, "emulated screen width does not match"); - Assert.AreEqual(height, screenHeight, "emulated screen height does not match"); + return new Overrides( + Convert.ToInt64(js.ExecuteScript("return window.__sw")), + Convert.ToInt64(js.ExecuteScript("return window.__sh")), + Convert.ToInt64(js.ExecuteScript("return window.__spw")), + Convert.ToInt64(js.ExecuteScript("return window.__sph")), + js.ExecuteScript("return window.__did")?.ToString()); + } - Assert.IsTrue(screenPixelWidth > 0, "screenpixelswidth override did not resolve"); - Assert.IsTrue(screenPixelHeight > 0, "screenpixelsheight override did not resolve"); + /// + /// The screen size and the device id came back from detection. + /// + private static void AssertOverridesResolved(Overrides overrides) + { + Assert.IsTrue( + overrides.ScreenPixelsWidth > 0, + "screenpixelswidth override did not resolve"); + Assert.IsTrue( + overrides.ScreenPixelsHeight > 0, + "screenpixelsheight override did not resolve"); - Assert.IsFalse(string.IsNullOrEmpty(deviceId), "device id was not resolved"); - Assert.AreNotEqual("loading", deviceId, "device id was not resolved"); + Assert.IsFalse( + string.IsNullOrEmpty(overrides.DeviceId), + "device id was not resolved"); + Assert.AreNotEqual( + "loading", overrides.DeviceId, "device id was not resolved"); } /// Quits the browser and stops the example app. diff --git a/ClientSideOverrides/ExampleRenderTests.cs b/ClientSideOverrides/ExampleRenderTests.cs index ace05c1..e772b36 100644 --- a/ClientSideOverrides/ExampleRenderTests.cs +++ b/ClientSideOverrides/ExampleRenderTests.cs @@ -91,6 +91,29 @@ public void Example_RendersRealDetectionResult() _driver = BrowserDrivers.CreateChrome(chromeOptions); + AssertRendersRealDetectionResult(); + } + + /// + /// The same in CloakBrowser, which presents itself as a desktop + /// browser without being given a user agent. + /// + [TestMethod, RequiresCloak] + public void Example_RendersRealDetectionResult_Cloak() + { + var chromeOptions = new ChromeOptions(); + chromeOptions.AcceptInsecureCertificates = true; + + _driver = BrowserDrivers.CreateCloak(chromeOptions); + + AssertRendersRealDetectionResult(); + } + + /// + /// Loads the example and checks the page it renders server-side. + /// + private void AssertRendersRealDetectionResult() + { _driver.Navigate().GoToUrl(_example.BaseUrl); // The detection table is rendered server-side on the first response, diff --git a/Helpers/BrowserDrivers.cs b/Helpers/BrowserDrivers.cs index 481fd59..81529eb 100644 --- a/Helpers/BrowserDrivers.cs +++ b/Helpers/BrowserDrivers.cs @@ -1,11 +1,15 @@ using System; using System.Collections.Generic; using System.IO; +using System.Net.Http; using System.Runtime.InteropServices; +using System.Text.Json; +using System.Threading; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; using OpenQA.Selenium.Edge; using OpenQA.Selenium.Firefox; +using OpenQA.Selenium.Manager; using OpenQA.Selenium.Remote; namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers @@ -38,6 +42,13 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers /// calling Selenium Manager at all, which is the behaviour the first case /// relies on. /// + /// + /// CloakBrowser is the exception, as it is never started here. It runs in + /// its vendor's container, and a driver attaches to it at the address + /// CLOAK_DEBUGGER_ADDRESS names. That driver has to match CloakBrowser's + /// version of Chromium, which is not the version of the Chrome on this + /// machine, so a driver found on the path is never used for it. + /// /// public static class BrowserDrivers { @@ -74,6 +85,32 @@ public static class BrowserDrivers /// Path to the Edge binary to drive. public const string EdgeBinaryVariable = "EDGE_BIN"; + /// + /// Address of a running CloakBrowser to attach to, as host:port, for + /// example 127.0.0.1:9222. The Cloak tests are skipped without it. + /// + public const string CloakDebuggerAddressVariable = + "CLOAK_DEBUGGER_ADDRESS"; + + /// + /// Directory holding a chromedriver of CloakBrowser's major version, + /// or the driver itself. Selenium Manager fetches one when it is + /// unset. + /// + public const string CloakDriverVariable = "CLOAKWEBDRIVER"; + + /// + /// The reason a Cloak test is skipped, naming the variable to set. + /// + public const string CloakNotConfiguredMessage = + "No CloakBrowser to attach to. Set " + + CloakDebuggerAddressVariable + " to the host and port of one " + + "that is running, for example 127.0.0.1:9222. The README says " + + "how to start one."; + + // Long enough for the first request to start the browser. + private const int CloakVersionTimeoutSeconds = 60; + /// File names a Chrome driver goes by. public static readonly string[] ChromeDriverNames = { "chromedriver", "chromedriver.exe" }; @@ -201,6 +238,227 @@ public static WebDriver CreateEdge(EdgeOptions options) "Edge", EdgeDriverVariable, EdgeDriverNames); } + /// + /// True when CLOAK_DEBUGGER_ADDRESS names a CloakBrowser to attach to. + /// + public static bool IsCloakConfigured => + CloakDebuggerAddress(Environment.GetEnvironmentVariable) != null; + + /// + /// Attaches a driver to the running CloakBrowser that + /// CLOAK_DEBUGGER_ADDRESS names. + /// + /// + /// + /// The browser was started by its container and outlives the driver, + /// so arguments in never reach it, and + /// SELENIUM_URL does not apply. Mobile emulation cannot be asked for + /// either, as ChromeDriver refuses it for a browser it did not start. + /// + /// + /// For the same reason the browser keeps its tabs, cookies and cache + /// from one test to the next. Each attach moves to a new tab, closes + /// the others and clears the cookies and the cache, so a test starts + /// as it would in a browser started for it. + /// + /// + /// Options for the driver. + /// A driver, which the caller quits. + public static WebDriver CreateCloak(ChromeOptions options) => + CreateCloak(Environment.GetEnvironmentVariable, options); + + /// + /// As , reading the variables + /// through so the rules can be tested + /// without changing the environment of the whole run. + /// + /// Reads an environment variable. + /// Options for the driver. + /// A driver, which the caller quits. + public static WebDriver CreateCloak( + Func getVariable, ChromeOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + var address = CloakDebuggerAddress(getVariable); + if (address == null) + { + throw new WebDriverException(CloakNotConfiguredMessage); + } + var version = CloakMajorVersion(ReadCloakVersion(address)); + options.DebuggerAddress = address; + var driverPath = FindNamedDriver( + getVariable, CloakDriverVariable, ChromeDriverNames); + ChromeDriver driver; + try + { + driver = new ChromeDriver( + ChromeDriverService.CreateDefaultService( + driverPath ?? FetchChromeDriver(version)), + options); + } + catch (Exception e) + { + throw new WebDriverException( + $"Could not attach a driver to CloakBrowser at {address}, " + + $"the address {CloakDebuggerAddressVariable} names. It " + + $"is Chromium {version} and needs chromedriver " + + $"{version}, which was " + + (driverPath != null + ? $"taken from {driverPath}" + : "left to Selenium Manager to fetch") + + $". Set {CloakDriverVariable} to a chromedriver " + + $"{version} to name one. Selenium said: {e.Message}", + e); + } + try + { + StartClean(driver); + } + catch + { + driver.Quit(); + throw; + } + return driver; + } + + /// + /// The address CLOAK_DEBUGGER_ADDRESS names, or null when it is unset + /// or empty, which is what a CI script that exports an unset variable + /// produces. + /// + /// Reads an environment variable. + /// The address as host:port, or null. + public static string CloakDebuggerAddress( + Func getVariable) + { + if (getVariable == null) + { + throw new ArgumentNullException(nameof(getVariable)); + } + var address = getVariable(CloakDebuggerAddressVariable); + return string.IsNullOrWhiteSpace(address) ? null : address.Trim(); + } + + /// + /// The major version of the browser, read from the document its + /// DevTools port serves at /json/version, where the Browser field + /// reads like Chrome/146.0.7680.177. + /// + /// Body of /json/version. + /// The major version, for example 146. + public static string CloakMajorVersion(string versionDocument) + { + string browser = null; + try + { + using var document = JsonDocument.Parse(versionDocument); + if (document.RootElement.ValueKind == JsonValueKind.Object + && document.RootElement.TryGetProperty( + "Browser", out var field) + && field.ValueKind == JsonValueKind.String) + { + browser = field.GetString(); + } + } + catch (Exception e) when ( + e is JsonException || e is ArgumentNullException) + { + // Reported below, with what was received. + } + var slash = browser?.IndexOf('/') ?? -1; + var major = slash < 0 + ? null + : browser.Substring(slash + 1).Split('.')[0]; + if (string.IsNullOrEmpty(major) + || int.TryParse(major, out _) == false) + { + throw new WebDriverException( + $"The address {CloakDebuggerAddressVariable} names did " + + "not answer /json/version with a browser version, so " + + "it is not the DevTools port of a browser. It " + + $"answered: {versionDocument}"); + } + return major; + } + + /// + /// Body of /json/version from the browser at + /// . The first request is what makes the + /// container start the browser. + /// + /// Address as host:port. + /// The document, as JSON. + private static string ReadCloakVersion(string address) + { + try + { + using var client = new HttpClient + { + Timeout = TimeSpan.FromSeconds(CloakVersionTimeoutSeconds), + }; + return client.GetStringAsync( + new Uri($"http://{address}/json/version")) + .GetAwaiter().GetResult(); + } + catch (Exception e) + { + throw new WebDriverException( + $"No browser answered at {address}, the address " + + $"{CloakDebuggerAddressVariable} names, which has to " + + "be the host and port of a running CloakBrowser, for " + + "example 127.0.0.1:9222. The README says how to start " + + $"one. The request said: {e.Message}", + e); + } + } + + /// + /// Path to a chromedriver for Chrome of the given major version, + /// which Selenium Manager fetches unless it already holds one. + /// + /// + /// Selenium Manager is asked directly, and only the driver it finds + /// is used. Left to itself, Selenium also wants a Chrome on this + /// machine, and fails on a machine that has none when + /// SE_AVOID_BROWSER_DOWNLOAD stops it downloading one. + /// + /// Major version of the browser. + /// Path to the driver. + private static string FetchChromeDriver(string version) + { + var found = SeleniumManager.DiscoverBrowserAsync( + "chrome", + new BrowserDiscoveryOptions { BrowserVersion = version }, + CancellationToken.None).GetAwaiter().GetResult(); + return found.DriverPath; + } + + /// + /// Leaves an attached browser on one new tab with no cookies and an + /// empty cache. Session storage belongs to a tab, so the new tab + /// clears that too. + /// + /// A driver attached to the browser. + private static void StartClean(ChromeDriver driver) + { + var stale = driver.WindowHandles; + driver.SwitchTo().NewWindow(WindowType.Tab); + var fresh = driver.CurrentWindowHandle; + foreach (var handle in stale) + { + driver.SwitchTo().Window(handle); + driver.Close(); + } + driver.SwitchTo().Window(fresh); + var none = new Dictionary(); + driver.ExecuteCdpCommand("Network.clearBrowserCookies", none); + driver.ExecuteCdpCommand("Network.clearBrowserCache", none); + } + /// /// Runs , which leaves Selenium Manager to /// fetch a driver, and on failure says what was looked for here first, @@ -280,25 +538,38 @@ public static string FindDriver( Func getVariable, string variable, IEnumerable names) + { + return FindNamedDriver(getVariable, variable, names) + ?? FindOnPath(getVariable, names); + } + + /// + /// Path to the driver names, as the driver + /// itself or the directory holding it, or null. The path is not + /// searched, which suits a driver that must not be the one the + /// machine keeps for its own browser. + /// + /// Reads an environment variable. + /// Variable that names the driver. + /// File names to look for in a directory. + /// A path, or null. + public static string FindNamedDriver( + Func getVariable, + string variable, + IEnumerable names) { if (getVariable == null) { throw new ArgumentNullException(nameof(getVariable)); } var configured = getVariable(variable); - if (string.IsNullOrEmpty(configured) == false) + if (string.IsNullOrEmpty(configured)) { - if (File.Exists(configured)) - { - return configured; - } - var inDirectory = FindInDirectory(configured, names); - if (inDirectory != null) - { - return inDirectory; - } + return null; } - return FindOnPath(getVariable, names); + return File.Exists(configured) + ? configured + : FindInDirectory(configured, names); } /// diff --git a/Helpers/BrowserDriversCloakTests.cs b/Helpers/BrowserDriversCloakTests.cs new file mode 100644 index 0000000..6ca9ea9 --- /dev/null +++ b/Helpers/BrowserDriversCloakTests.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.IO; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Tests +{ + /// + /// Checks the rules for attaching to CloakBrowser. No browser is started + /// and none has to be running, so these run anywhere. + /// + [TestClass] + public class BrowserDriversCloakTests + { + // What the DevTools port of CloakBrowser 0.5.12 serves at + // /json/version, less the fields that are not read. + private const string VersionDocument = + "{\"Browser\": \"Chrome/146.0.7680.177\", " + + "\"Protocol-Version\": \"1.3\", " + + "\"webSocketDebuggerUrl\": \"ws://127.0.0.1:9222/devtools/" + + "browser/00000000-0000-0000-0000-000000000000\"}"; + + private string _directory; + private string _driverFile; + + /// + /// Makes a directory holding a file named like a driver. + /// + [TestInitialize] + public void Init() + { + _directory = Path.Combine( + Path.GetTempPath(), "51d-cloak-driver-" + Guid.NewGuid()); + Directory.CreateDirectory(_directory); + _driverFile = Path.Combine(_directory, "chromedriver"); + File.WriteAllText(_driverFile, string.Empty); + } + + /// Removes the directory. + [TestCleanup] + public void Cleanup() + { + if (_directory != null && Directory.Exists(_directory)) + { + Directory.Delete(_directory, true); + } + } + + /// + /// An unset variable and an empty one both mean there is no browser + /// to attach to, as a CI script that exports an unset variable + /// produces an empty one. + /// + [TestMethod] + [DataRow(null)] + [DataRow("")] + [DataRow(" ")] + public void CloakDebuggerAddress_UnsetOrEmpty_IsNull(string value) + { + Assert.IsNull(BrowserDrivers.CloakDebuggerAddress(Lookup( + BrowserDrivers.CloakDebuggerAddressVariable, value))); + } + + /// The address is read from the variable. + [TestMethod] + public void CloakDebuggerAddress_Set_IsTheAddress() + { + Assert.AreEqual( + "127.0.0.1:9222", + BrowserDrivers.CloakDebuggerAddress(Lookup( + BrowserDrivers.CloakDebuggerAddressVariable, + " 127.0.0.1:9222 "))); + } + + /// + /// With no address there is nothing to attach to, and the failure + /// names the variable that would supply one. + /// + [TestMethod] + public void CreateCloak_NoAddress_NamesTheVariable() + { + var e = Assert.ThrowsExactly( + () => BrowserDrivers.CreateCloak( + Lookup(new Dictionary()), + new ChromeOptions())); + + Assert.AreEqual( + BrowserDrivers.CloakNotConfiguredMessage, e.Message); + StringAssert.Contains( + e.Message, BrowserDrivers.CloakDebuggerAddressVariable); + } + + /// + /// An address nothing listens at fails before any driver is looked + /// for, and the failure names the variable and the address. + /// + [TestMethod] + public void CreateCloak_NothingAtTheAddress_NamesTheVariable() + { + var address = $"127.0.0.1:{TestHelpers.GetRandomUnusedPort()}"; + + var e = Assert.ThrowsExactly( + () => BrowserDrivers.CreateCloak( + Lookup( + BrowserDrivers.CloakDebuggerAddressVariable, address), + new ChromeOptions())); + + StringAssert.Contains( + e.Message, BrowserDrivers.CloakDebuggerAddressVariable); + StringAssert.Contains(e.Message, address); + } + + /// + /// The major version comes from the browser itself, so the driver + /// asked for is always the one that browser needs. + /// + [TestMethod] + public void CloakMajorVersion_ReadsTheBrowserField() + { + Assert.AreEqual( + "146", BrowserDrivers.CloakMajorVersion(VersionDocument)); + } + + /// + /// An answer that is not a browser's says so and names the variable, + /// which is what a wrong port gives. + /// + [TestMethod] + [DataRow("{}")] + [DataRow("{\"Browser\": \"Chrome\"}")] + [DataRow("{\"Browser\": 146}")] + [DataRow("[]")] + [DataRow("not a browser")] + [DataRow("")] + [DataRow(null)] + public void CloakMajorVersion_NotABrowser_NamesTheVariable( + string document) + { + var e = Assert.ThrowsExactly( + () => BrowserDrivers.CloakMajorVersion(document)); + + StringAssert.Contains( + e.Message, BrowserDrivers.CloakDebuggerAddressVariable); + } + + /// + /// A variable naming the directory the driver is in finds it. + /// + [TestMethod] + public void FindNamedDriver_ReadsDirectoryFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindNamedDriver( + Lookup(BrowserDrivers.CloakDriverVariable, _directory), + BrowserDrivers.CloakDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// A variable naming the driver itself finds it. + [TestMethod] + public void FindNamedDriver_ReadsFileFromVariable() + { + Assert.AreEqual( + _driverFile, + BrowserDrivers.FindNamedDriver( + Lookup(BrowserDrivers.CloakDriverVariable, _driverFile), + BrowserDrivers.CloakDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// A chromedriver on the path is not used, as it is the one for the + /// machine's own Chrome and not for CloakBrowser's version. + /// + [TestMethod] + public void FindNamedDriver_DoesNotSearchThePath() + { + Assert.IsNull( + BrowserDrivers.FindNamedDriver( + Lookup("PATH", _directory), + BrowserDrivers.CloakDriverVariable, + BrowserDrivers.ChromeDriverNames)); + } + + /// + /// The reason a Cloak test gives for being skipped names the + /// variable to set. + /// + [TestMethod] + public void RequiresCloak_SkipReason_NamesTheVariable() + { + StringAssert.Contains( + new RequiresCloakAttribute().IgnoreMessage, + BrowserDrivers.CloakDebuggerAddressVariable); + } + + /// Reads one variable in place of the environment. + /// Name of the variable. + /// Its value, or null for unset. + /// A lookup holding only that variable. + private static Func Lookup(string name, string value) => + Lookup(new Dictionary { { name, value } }); + + /// + /// Reads from a dictionary in place of the environment. + /// + /// Variables and their values. + /// A lookup over those values. + private static Func Lookup( + IReadOnlyDictionary values) => + name => values.TryGetValue(name, out var value) ? value : null; + } +} diff --git a/Helpers/RequiresCloakAttribute.cs b/Helpers/RequiresCloakAttribute.cs new file mode 100644 index 0000000..e1618f3 --- /dev/null +++ b/Helpers/RequiresCloakAttribute.cs @@ -0,0 +1,32 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers +{ + /// + /// Skips a test, or every test in a class, when CLOAK_DEBUGGER_ADDRESS + /// names no CloakBrowser to attach to, and gives the variable to set as + /// the reason. + /// + /// + /// The test is skipped before its initialize method runs, so no example + /// is started for a test that could not use it. + /// + [AttributeUsage( + AttributeTargets.Class | AttributeTargets.Method, Inherited = false)] + public sealed class RequiresCloakAttribute : ConditionBaseAttribute + { + /// Runs the test only where the condition is met. + public RequiresCloakAttribute() + : base(ConditionMode.Include) + { + IgnoreMessage = BrowserDrivers.CloakNotConfiguredMessage; + } + + /// + public override bool IsConditionMet => BrowserDrivers.IsCloakConfigured; + + /// + public override string GroupName => nameof(RequiresCloakAttribute); + } +} diff --git a/README.md b/README.md index 5bdbf4a..512b5bd 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,10 @@ Select a subset with `--filter TestCategory=Contract` or - **Browser** - a driver the machine already provides is used first, named by `CHROMEWEBDRIVER`, `GECKOWEBDRIVER` or `EDGEWEBDRIVER` or found on the path. When the machine provides none, Selenium Manager fetches one. Set - `SELENIUM_URL` to drive a browser on a Selenium grid instead. + `SELENIUM_URL` to drive a browser on a Selenium grid instead. CloakBrowser + is never started by the suite, which attaches to one already running in a + container when `CLOAK_DEBUGGER_ADDRESS` names it. See + [Running under CloakBrowser](#running-under-cloakbrowser). ## Configuration @@ -50,6 +53,8 @@ and no keys are committed. | `SELENIUM_URL` | optional | Selenium grid URL, omit to drive a browser on this machine. | | `CHROMEWEBDRIVER` / `GECKOWEBDRIVER` / `EDGEWEBDRIVER` | optional | A driver, or the directory holding one. GitHub's Linux runner images set these. | | `CHROME_BIN` / `FIREFOX_BIN` / `EDGE_BIN` | optional | The browser binary to drive, when it is not on the path. | +| `CLOAK_DEBUGGER_ADDRESS` | optional | Host and port of a running CloakBrowser to attach to, e.g. `127.0.0.1:9222`. The Cloak tests are skipped when it is unset. | +| `CLOAKWEBDRIVER` | optional | A chromedriver of the same major version as CloakBrowser, or the directory holding one. Selenium Manager fetches one when it is unset. | | `EXAMPLE_URL` / `EXAMPLE_LANG` | `Contract` | The example app to test (CI / local). | | `51DEGREES_CLOUD_ENDPOINT` / `51DEGREES_RESOURCE_KEY` | `Browser51Did` | Handed to the demo under these names, the ones every language's demo reads first. | | `_51DEGREES_RESOURCE_KEY_51DID` | `Browser51Did` | Read where `51DEGREES_RESOURCE_KEY` is unset. The name CI sets, for a resource key carrying the 51Did product. | @@ -105,13 +110,113 @@ export ENTERPRISE_V4_LICENSE="" dotnet test --filter TestCategory=CloudInternal ``` +## Running under CloakBrowser + +[CloakBrowser](https://github.com/CloakHQ/CloakBrowser) is a patched Chromium +that its vendor publishes for browser automation. Visitors arrive in browsers +like it as well as in Chrome and Firefox, so the tests of the examples can be +run in it too. + +The tests that use it are the two whose names start with `Cloak_` in +`Browser`, the `CloakTests` class in `CloudInternal`, and the tests whose +names end in `_Cloak` in `Contract`. They are skipped, with a reason that +names the variable, until `CLOAK_DEBUGGER_ADDRESS` is set, so a run that does +not set it is unchanged. + +### Starting the browser + +The CloakBrowser binary is closed source, so the suite never downloads it and +never starts it. It runs only inside the vendor's Docker image, and the suite +attaches a driver to it through the DevTools port the container serves. Pin +the image by digest, so that the browser cannot change under a tag. Mount +nothing from the machine into the container and pass it no keys. + +On a Linux machine, such as a CI runner, use host networking, so that the +browser reaches the example and the pages the tests serve on `localhost`. + +```bash +docker run -d --name cloak --network host \ + -e CLOAKBROWSER_AUTO_UPDATE=false \ + cloakhq/cloakbrowser:0.5.12@sha256:2fdd1289154f30594c7bab74943ea7baa6b617a9ec026ab4bf339c2ebabcb712 \ + cloakserve +export CLOAK_DEBUGGER_ADDRESS="127.0.0.1:9222" +``` + +Host networking opens the DevTools port on every network interface of the +machine, and whoever can reach that port controls the browser. Use it only on +a machine that takes no connections from outside, which a hosted CI runner is. + +With Docker Desktop on Windows, publish the port on the loopback address +instead, and tell the browser that `localhost` is the machine Docker Desktop +runs on. + +```bash +docker run -d --name cloak -p 127.0.0.1:9222:9222 \ + -e CLOAKBROWSER_AUTO_UPDATE=false \ + cloakhq/cloakbrowser:0.5.12@sha256:2fdd1289154f30594c7bab74943ea7baa6b617a9ec026ab4bf339c2ebabcb712 \ + cloakserve "--host-resolver-rules=MAP localhost host.docker.internal" +export CLOAK_DEBUGGER_ADDRESS="127.0.0.1:9222" +``` + +`CLOAKBROWSER_AUTO_UPDATE=false` stops the container looking for a newer +browser when it starts, so the browser that runs is the one in the pinned +image. Remove the container with `docker rm -f cloak` when the run is over. + +### The driver + +The driver is ChromeDriver, and it has to be of the same major version as the +Chromium inside CloakBrowser, which is 146 in the image above and is not the +version of the Chrome on the machine. The suite asks the browser for its +version and has Selenium Manager fetch that driver, so there is nothing to +install, and a chromedriver on the path is never used. Set `CLOAKWEBDRIVER` +to name a driver where Selenium Manager cannot fetch one. + +Selenium Manager also downloads a Chrome of that version when the machine has +none, and nothing runs it. Set `SE_AVOID_BROWSER_DOWNLOAD=true` to stop that. + +### What is different for a test + +The browser was started by the container and not by the driver, which changes +three things. + +- Arguments in the options, such as `--headless` or `--user-agent`, never + reach the browser. It sends its own user agent, and it runs with a window on + the container's virtual display. +- ChromeDriver refuses mobile emulation for a browser it attaches to. The + `_Cloak` tests of the client-side overrides therefore emulate no device, and + work with the screen size the browser reports for itself. +- The browser outlives the test. Each attach moves to a new tab, closes the + others and clears the cookies and the cache, so a test starts as it would in + a browser started for it. + +### Keys the browser is given + +CloakBrowser is a third party's closed source browser, so consider which keys +it is shown. The `Contract` tests put no key in anything they give the +browser and pass on what the example serves, so an example that keeps its +resource key on the server gives CloakBrowser no key. The `CloakTests` class +in `CloudInternal` is different, because it runs the cloud's JavaScript in the +browser, and that script carries the resource key and the license it was +requested with. A pipeline that runs `CloudInternal` under CloakBrowser should +therefore use keys it is content for that browser to see. + +### License + +The browser binary has its own license, which is separate from the MIT +license of the vendor's wrapper code. The image above carries the vendor's +free build under version 1.3 of the CloakBrowser Binary License, dated July +2026. [Read that version](https://github.com/CloakHQ/CloakBrowser/blob/2e379e402b52838d37e469a40e6195e887e314d1/BINARY-LICENSE.md) +before running it. The suite uses no account and no license key. + ## This repository's own CI The "Build and test" workflow builds the suite on every push and pull request, -and runs it in two jobs. The first runs the tests that need no browser, no +and runs it in three jobs. The first runs the tests that need no browser, no cloud and no keys. The second starts Chrome and Firefox on `ubuntu-latest`, `ubuntu-22.04-arm` and `ubuntu-24.04-arm`, and prints what each runner -provides before it does. Together they prove a change here before any language +provides before it does. The third starts CloakBrowser from its pinned image +on `ubuntu-latest` and runs the Cloak tests of the `Browser` category with a +driver attached to it. Together they prove a change here before any language repository picks it up. ## Browsers by architecture @@ -139,3 +244,6 @@ nor the driver for it, so the Edge tests say so and skip. once against `:8080`. - **Each API CI** checks this repo out, launches its own example, and runs `Contract` against the public cloud. +- **Either** adds CloakBrowser by starting its container before the tests and + setting `CLOAK_DEBUGGER_ADDRESS`, as + [Running under CloakBrowser](#running-under-cloakbrowser) describes. diff --git a/Selenium/DriverSmokeTests.cs b/Selenium/DriverSmokeTests.cs index 1299945..a53f9b1 100644 --- a/Selenium/DriverSmokeTests.cs +++ b/Selenium/DriverSmokeTests.cs @@ -28,6 +28,11 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium /// run that quietly skipped them would say nothing, and saying nothing is /// the fault these were written for. /// + /// + /// CloakBrowser is not started by the suite, so its tests are skipped + /// until CLOAK_DEBUGGER_ADDRESS names one that is running. Once it does, + /// they fail like the others when the browser cannot be reached. + /// /// [TestClass, TestCategory("Browser")] public class DriverSmokeTests @@ -127,6 +132,42 @@ public void Edge_RunsThePage() AssertPageRan(); } + /// + /// The CloakBrowser that CLOAK_DEBUGGER_ADDRESS names runs the page. + /// + [TestMethod, RequiresCloak] + public void Cloak_RunsThePage() + { + _driver = BrowserDrivers.CreateCloak(new ChromeOptions()); + AssertPageRan(); + } + + /// + /// CloakBrowser outlives the driver, so what one test leaves in it + /// must not reach the next. A second driver finds neither the cookie + /// nor the session storage the first one left on the same page. + /// + [TestMethod, RequiresCloak] + public void Cloak_StartsEachTestClean() + { + _driver = BrowserDrivers.CreateCloak(new ChromeOptions()); + _driver.Navigate().GoToUrl(_url); + ((IJavaScriptExecutor)_driver).ExecuteScript( + "document.cookie = 'left=behind; path=/';" + + "sessionStorage.setItem('left', 'behind');"); + _driver.Quit(); + + _driver = BrowserDrivers.CreateCloak(new ChromeOptions()); + _driver.Navigate().GoToUrl(_url); + var found = ((IJavaScriptExecutor)_driver).ExecuteScript( + "return 'cookie [' + document.cookie + '], session storage ['" + + " + (sessionStorage.getItem('left') || '') + ']';"); + + Assert.AreEqual( + "cookie [], session storage []", found, + "the browser kept what the driver before this one left"); + } + /// /// Loads the page and checks the browser ran the script on it. ///