From 98acc7f5d8e4181569c6c3174192f3768b54daea Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Sun, 8 Nov 2026 15:13:24 -0500 Subject: [PATCH] Sweep #1815 test and tooling follow-ups from the #1794, #1796 and #1810 audits - Pin the host-history suite to a Windows lane as well as macOS - Name the actual scenario in the v1 SHA256 refusal - Cover the LiteRT Web text-history browser default and both caller pipes' blocking mode - Close the safaridriver stderr file when Popen raises and show it on the start timeout - Add a 0.18.x -> 0.19.0 MIGRATION.md section --- MIGRATION.md | 13 ++-- test/ci/test_supervise_chrome_test.py | 10 +-- ...wright_chat_app_real_model_smoke_test.dart | 63 +++++++++++++++++++ test/unit/tooling/run_local_e2e_test.dart | 16 +++++ test/unit/tooling/test_shards_test.dart | 5 ++ tool/testing/run_local_e2e.dart | 2 +- tool/testing/safari_webdriver.py | 24 ++++--- 7 files changed, 115 insertions(+), 18 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 5a70c16a5..5a8f5b103 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -2,6 +2,14 @@ This document covers the major breaking upgrade paths. +## `0.18.x` -> `0.19.0`: LiteRT-LM native `v0.19.1-1` and Web `0.19.0` + +Core `0.19.0` and `llamadart_litert_lm_flutter` `0.19.0` move to published +[LiteRT-LM v0.19.1-1](https://github.com/leehack/litert-lm-native/releases/tag/v0.19.1-1) +(owner `82d4ae1c`, upstream `7a03724c`), which keeps the C API 1.0.0 exports of +v0.19.0, so the `0.18.0` ABI selection below +applies unchanged. Web LiteRT-LM moves to `@litert-lm/core@0.19.0` in `0.19.0`. + ## `0.17.x` -> `0.18.0`: native LiteRT-LM ABI selection The public `LiteRtLmRuntimeClient`, default services and speech worker now share @@ -32,11 +40,6 @@ Behavior to recheck with a C API 1 runtime: worker and its native/model/cache owners. That backend requires a process restart; an ended request does not prove native cleanup completed. -Core `0.19.0` and `llamadart_litert_lm_flutter` `0.19.0` move to published -[LiteRT-LM v0.19.1-1](https://github.com/leehack/litert-lm-native/releases/tag/v0.19.1-1) -(owner `82d4ae1c`, upstream `7a03724c`), which keeps the C API 1.0.0 exports of -v0.19.0, so this selection applies unchanged. Web LiteRT-LM moves to -`@litert-lm/core@0.19.0` in `0.19.0`. See [the adoption checklist](doc/litert_lm_v1_adoption.md) for per-feature coverage. ## `0.17.0` -> `0.17.1`: llama.cpp native runtime `v0.8.0-1` diff --git a/test/ci/test_supervise_chrome_test.py b/test/ci/test_supervise_chrome_test.py index 4ef0f48f6..52d1583df 100644 --- a/test/ci/test_supervise_chrome_test.py +++ b/test/ci/test_supervise_chrome_test.py @@ -110,11 +110,13 @@ def test_mixed_pass_fail_summary_is_observed_and_stays_failure(self): self.assertIn('reporter_summary_observed', [event['event'] for event in self.events()]) self.assertEqual(child.stdout, b'1 test passed, 1 failed.\n') - def test_caller_pipe_stays_blocking_after_the_supervisor_exits(self): - script = (f"import os,subprocess,sys; subprocess.run({self.command('print(1)')!r}, " - "stdout=sys.stdout); print(os.get_blocking(1))") + def test_caller_pipes_keep_their_blocking_mode_after_the_supervisor_exits(self): + script = ("import os,subprocess,sys\n" + "os.set_blocking(2, False)\n" + f"subprocess.run({self.command('print(1)')!r}, stdout=sys.stdout, stderr=sys.stderr)\n" + "print(os.get_blocking(1), os.get_blocking(2))") done = subprocess.run([sys.executable, '-c', script], capture_output=True, timeout=8) - self.assertEqual(done.stdout.split()[-1], b'True', done.stderr) + self.assertEqual(done.stdout.split()[-2:], [b'True', b'False'], done.stderr) def test_reporter_success_does_not_override_a_failing_exit(self): child, result = self.run_fixture("print('All tests passed!'); raise SystemExit(7)") diff --git a/test/unit/tooling/playwright_chat_app_real_model_smoke_test.dart b/test/unit/tooling/playwright_chat_app_real_model_smoke_test.dart index 43cca122d..a199c163a 100644 --- a/test/unit/tooling/playwright_chat_app_real_model_smoke_test.dart +++ b/test/unit/tooling/playwright_chat_app_real_model_smoke_test.dart @@ -207,6 +207,38 @@ sys.argv[0] = 'tool/testing/playwright_chat_app_real_model_smoke.py' runpy.run_path(sys.argv[0], run_name='__main__') '''; +const _safariStartFailureProbe = r''' +import sys +import tempfile + +sys.path.insert(0, 'tool/testing') +import safari_webdriver + +opened = [] +real = tempfile.TemporaryFile +safari_webdriver.tempfile.TemporaryFile = lambda *a, **k: opened.append(real(*a, **k)) or opened[-1] +safari_webdriver.SAFARIDRIVER = sys.argv[1] +try: + safari_webdriver.SafariSession() +except OSError: + pass +print(f'closed={opened[0].closed}') +'''; + +const _safariStartTimeoutProbe = r''' +import sys + +sys.path.insert(0, 'tool/testing') +import safari_webdriver + +safari_webdriver.SAFARIDRIVER = sys.argv[1] +safari_webdriver._DRIVER_START_TIMEOUT_S = 1 +try: + safari_webdriver.SafariSession() +except safari_webdriver.WebDriverError as error: + print(error) +'''; + // Runs the recorded page scripts against a stand-in `@litert-lm/core` module, // loading the wrapped module the way `LiteRtLmBackend` does on the Web. const _pageScriptHarness = r''' @@ -532,6 +564,37 @@ void main() { expect(run.result.exitCode, 0, reason: '${run.result.stderr}'); }); + + test('closes the stderr file when the driver cannot start', () async { + final result = await runBoundedProcess('python3', [ + '-c', + _safariStartFailureProbe, + p.join(tempDir.path, 'missing'), + ], timeout: _processTimeout); + + expect(result.exitCode, 0, reason: '${result.stderr}'); + expect(result.stdout, contains('closed=True')); + }); + + test('shows driver stderr when it never answers', () async { + final driver = File(p.join(tempDir.path, 'silent_safaridriver')) + ..writeAsStringSync( + '#!/usr/bin/env python3\n' + 'import sys, time\n' + 'sys.stderr.write("driver says no\\n"); sys.stderr.flush()\n' + 'time.sleep(30)\n', + ); + await Process.run('chmod', ['+x', driver.path]); + final result = await runBoundedProcess('python3', [ + '-c', + _safariStartTimeoutProbe, + driver.path, + ], timeout: _processTimeout); + + expect(result.exitCode, 0, reason: '${result.stderr}'); + expect(result.stdout, contains('did not start within')); + expect(result.stdout, contains('driver says no')); + }); }, ); diff --git a/test/unit/tooling/run_local_e2e_test.dart b/test/unit/tooling/run_local_e2e_test.dart index ccd46ac1a..cdff1a7b0 100644 --- a/test/unit/tooling/run_local_e2e_test.dart +++ b/test/unit/tooling/run_local_e2e_test.dart @@ -495,6 +495,22 @@ Future main(List args) async { expect(chatApp.exitCode, 0); expect(chatApp.stdout, contains('--response-timeout-ms 600000')); expect(chatApp.stdout, isNot(contains('--browser'))); + final history = await runLocalE2e( + [ + '--scenario', + 'litert-lm-web-text-history', + '--model-path', + 'models/existing.litertlm', + '--backend', + 'gpu', + '--dry-run', + ], + projectRoot: '/repo', + displayShell: LocalE2eDisplayShell.posix, + ); + expect(history.exitCode, 0); + expect(history.stdout, contains('litert_text_history_web_e2e.py')); + expect(history.stdout, isNot(contains('--browser'))); }, ); diff --git a/test/unit/tooling/test_shards_test.dart b/test/unit/tooling/test_shards_test.dart index 9939b0c4d..930c30e62 100644 --- a/test/unit/tooling/test_shards_test.dart +++ b/test/unit/tooling/test_shards_test.dart @@ -330,6 +330,11 @@ void main() => test('renders', () {}); 'test/unit/tooling/prepare_workspace_test.dart', ]) { expect(macos, contains(file)); + expect( + [...tooling, ...runtime], + contains(file), + reason: '$file must run on a Windows lane', + ); } for (final file in [ 'test/unit/core/template/chat_format_registry_test.dart', diff --git a/tool/testing/run_local_e2e.dart b/tool/testing/run_local_e2e.dart index 10a5169c6..cfa0791da 100644 --- a/tool/testing/run_local_e2e.dart +++ b/tool/testing/run_local_e2e.dart @@ -3188,7 +3188,7 @@ Future runLocalE2e( return LocalE2eResult( 64, stderr: - '$name is required for litert-lm-v1-service (SHA256 inputs require 64 hexadecimal digits).\n', + '$name is required for ${scenario.name} (SHA256 inputs require 64 hexadecimal digits).\n', ); } } diff --git a/tool/testing/safari_webdriver.py b/tool/testing/safari_webdriver.py index cd21eaf51..cdb74605c 100644 --- a/tool/testing/safari_webdriver.py +++ b/tool/testing/safari_webdriver.py @@ -42,11 +42,15 @@ def __init__(self, *, command_timeout_s: float = 300) -> None: # A file, not a pipe: nothing reads stderr while the driver runs, and a # chatty driver would block on a full pipe. self._stderr = tempfile.TemporaryFile() - self._driver = subprocess.Popen( - [SAFARIDRIVER, "-p", str(self._port)], - stdout=subprocess.DEVNULL, - stderr=self._stderr, - ) + try: + self._driver = subprocess.Popen( + [SAFARIDRIVER, "-p", str(self._port)], + stdout=subprocess.DEVNULL, + stderr=self._stderr, + ) + except BaseException: + self._stderr.close() + raise self._base = f"http://127.0.0.1:{self._port}" self.session_id: str | None = None try: @@ -66,18 +70,22 @@ def _wait_for_driver(self) -> None: deadline = time.monotonic() + _DRIVER_START_TIMEOUT_S while time.monotonic() < deadline: if self._driver.poll() is not None: - self._stderr.seek(0) - stderr = self._stderr.read().decode(errors="replace") - raise WebDriverError(f"safaridriver exited: {stderr.strip()}") + raise WebDriverError(f"safaridriver exited: {self._driver_stderr()}") try: self._request("GET", "/status") return except (OSError, WebDriverError): time.sleep(0.2) + detail = self._driver_stderr() raise WebDriverError( f"safaridriver did not start within {_DRIVER_START_TIMEOUT_S} s" + + (f": {detail}" if detail else "") ) + def _driver_stderr(self) -> str: + self._stderr.seek(0) + return self._stderr.read().decode(errors="replace").strip() + def _request(self, method: str, path: str, body: Any = None) -> Any: data = None if body is None else json.dumps(body).encode() request = urllib.request.Request(