diff --git a/example/chat_app/lib/providers/chat_provider.dart b/example/chat_app/lib/providers/chat_provider.dart index 245492be5..e9f8e0ca2 100644 --- a/example/chat_app/lib/providers/chat_provider.dart +++ b/example/chat_app/lib/providers/chat_provider.dart @@ -2293,7 +2293,7 @@ class ChatProvider extends ChangeNotifier { _removeEmptyAssistantPlaceholder(); if (toolCallsCutOff) { _addInfoMessage( - 'The reply reached the token limit, so its tool calls were not run. Raise Max generated tokens and send it again.', + 'The reply reached the token limit, so its tool calls were not run. Raise Max generated tokens, or start a new chat if the context is full, and send it again.', ); } } catch (e) { diff --git a/example/chat_app/test/unit_test.dart b/example/chat_app/test/unit_test.dart index 01898d488..01673ec34 100644 --- a/example/chat_app/test/unit_test.dart +++ b/example/chat_app/test/unit_test.dart @@ -1685,7 +1685,11 @@ void main() { ); expect( cutOffProvider.messages.last.text, - contains('reached the token limit, so its tool calls were not run'), + allOf( + contains('reached the token limit, so its tool calls were not run'), + contains('Raise Max generated tokens'), + contains('start a new chat if the context is full'), + ), ); }); diff --git a/test/ci/test_supervise_chrome_test.py b/test/ci/test_supervise_chrome_test.py index 3c449e2b6..a708ff32a 100644 --- a/test/ci/test_supervise_chrome_test.py +++ b/test/ci/test_supervise_chrome_test.py @@ -116,7 +116,7 @@ def patched_supervisor(self, patch): """A launcher that runs the production supervisor with `patch(module)` applied.""" launcher = Path(self.temp.name) / 'launcher.py' launcher.write_text( - 'import importlib.util, sys\n' + 'import importlib.util, sys, time\n' f'spec = importlib.util.spec_from_file_location("supervisor", {str(SCRIPT)!r})\n' 'module = importlib.util.module_from_spec(spec)\n' 'spec.loader.exec_module(module)\n' @@ -352,6 +352,23 @@ def test_recovery_happens_no_earlier_than_post_done_seconds_after_done(self): # Events are rounded to milliseconds, so compare at that precision. self.assertGreaterEqual(round(at['browser_close_stall'] - at['reporter_done'], 3), 0.6) + def test_post_done_window_starts_after_a_slow_reporter_poll(self): + launcher = self.patched_supervisor( + 'poll = module.ReporterEvents.poll\n' + 'def slow_poll(self):\n' + ' done = poll(self)\n' + ' if done:\n' + ' time.sleep(0.2)\n' + ' return done\n' + 'module.ReporterEvents.poll = slow_poll') + child, result = self.run_fixture(self.browser_then( + {'type': 'done', 'success': True}, 0.05, 'import time; time.sleep(5)'), + reporter=True, post_done=0.6, script=launcher) + self.assertEqual(child.returncode, 0, child.stderr) + self.assertTrue(result['browser_close_stall']) + at = {event['event']: event['elapsed_seconds'] for event in self.events()} + self.assertGreaterEqual(round(at['browser_close_stall'] - at['reporter_done'], 3), 0.6) + def test_leftover_owned_processes_after_a_recovered_stall_exit_125(self): launcher = self.patched_supervisor('module.deliver = lambda row, number: False') child, result = self.run_fixture(self.browser_then( diff --git a/test/fixtures/seed_oss_parallel_upstream.md b/test/fixtures/seed_oss_parallel_upstream.md index 9d1024e8f..163fde433 100644 --- a/test/fixtures/seed_oss_parallel_upstream.md +++ b/test/fixtures/seed_oss_parallel_upstream.md @@ -24,10 +24,10 @@ tracker [#1531](https://github.com/leehack/llamadart/issues/1531)). These are not cases here; `seed_oss_handler_test.dart` pins them: - An unknown tool name is returned as a call: `SeedOssHandler.parse` takes no - tool schemas. The engine and stream parser reject it for - `ToolDefinition.fromJsonSchema` tools. -- A call missing a required argument is returned with the arguments present, - with the same schema caveat. + tool schemas, and Seed-OSS requests refuse `ToolDefinition.fromJsonSchema` + tools at render, so validate the name before running the call. +- A call missing a required argument is returned with the arguments present; + validate the arguments before running it. - A later scope whose `` is not followed by a line break stays content after the calls before it. diff --git a/test/unit/backends/litert_lm/litert_lm_backend_web_test.dart b/test/unit/backends/litert_lm/litert_lm_backend_web_test.dart index 3a915c873..6c59df7ee 100644 --- a/test/unit/backends/litert_lm/litert_lm_backend_web_test.dart +++ b/test/unit/backends/litert_lm/litert_lm_backend_web_test.dart @@ -1438,6 +1438,26 @@ void main() { : null, ), ), + ( + 'a limit error with text before it', + () => _installCountingFakeEngine( + counts: (_, budget) => (decoded: budget, total: 20), + reject: (prompt) { + final message = _inputTooLong(prompt, 4096); + return message == null ? null : 'Wrapped: $message'; + }, + ), + ), + ( + 'a limit error with text after it', + () => _installCountingFakeEngine( + counts: (_, budget) => (decoded: budget, total: 20), + reject: (prompt) { + final message = _inputTooLong(prompt, 4096); + return message == null ? null : '$message (retry)'; + }, + ), + ), ( 'a limit probe whose send throws', () => _installCountingFakeEngine( diff --git a/test/unit/backends/litert_lm/litert_lm_v1_conversation_test.dart b/test/unit/backends/litert_lm/litert_lm_v1_conversation_test.dart index a0bd5d2ae..de3fcb67b 100644 --- a/test/unit/backends/litert_lm/litert_lm_v1_conversation_test.dart +++ b/test/unit/backends/litert_lm/litert_lm_v1_conversation_test.dart @@ -301,29 +301,34 @@ void main() { }, ); } - test('send failure keeps the native message text and the native stack', () { - final runtime = loaded(); - final conversation = runtime.openConversation(); - call('fail', 17); - Object? error; - StackTrace? trace; - try { - conversation.sendText('failure'); - } on Object catch (caught, stackTrace) { - error = caught; - trace = stackTrace; - } - expect(error, isA()); - expect( - (error! as LiteRtLmSendFailureException).message, - 'conversation_send_message_stream failed: LiteRT-LM reported INTERNAL ' - '(13): stage17 m.litertlm cache secret/dispatch.', - ); - expect(trace.toString(), contains('LiteRtLmV1Runtime._check')); - call('fail', 0); - conversation.close(); - runtime.dispose(); - }); + test( + 'send failure keeps the native message text and the raising Dart frame', + () { + final runtime = loaded(); + final conversation = runtime.openConversation(); + addTearDown(() { + call('fail', 0); + conversation.close(); + runtime.dispose(); + }); + call('fail', 17); + Object? error; + StackTrace? trace; + try { + conversation.sendText('failure'); + } on Object catch (caught, stackTrace) { + error = caught; + trace = stackTrace; + } + expect(error, isA()); + expect( + (error! as LiteRtLmSendFailureException).message, + 'conversation_send_message_stream failed: LiteRT-LM reported INTERNAL ' + '(13): stage17 m.litertlm cache secret/dispatch.', + ); + expect(trace.toString(), contains('LiteRtLmV1Runtime._check')); + }, + ); test( 'cancellation status failure still prevents reuse and retains callback until final', () async { diff --git a/test/unit/core/engine/chat_completion_stream_parser_test.dart b/test/unit/core/engine/chat_completion_stream_parser_test.dart index 00468f9e7..125fa91a8 100644 --- a/test/unit/core/engine/chat_completion_stream_parser_test.dart +++ b/test/unit/core/engine/chat_completion_stream_parser_test.dart @@ -132,6 +132,60 @@ void main() { expect(completion.toolCalls.map((call) => call.id), ['call_0', 'call_1']); }); + LlamaChatTemplateResult toolTemplate() => ChatTemplateEngine.render( + templateSource: '{{ messages | tojson }}{{ tools | tojson }}', + messages: const [ + LlamaChatMessage.fromText(role: LlamaChatRole.user, text: 'Hi'), + ], + metadata: const {}, + tools: [_weatherTool], + ); + + test('a runtime tool call keeps its own id', () async { + final completion = await ChatCompletionStreamParser.parse( + tokenStream: Stream.fromIterable([ + '{"name":"weather","arguments":{"city":"Paris"}}', + '', + ]), + templateResult: toolTemplate(), + parseToolCallsEnabled: true, + enableThinking: true, + modelName: 'runtime', + completionId: 'runtime-call-id', + tools: [_weatherTool], + runtimeToolCalls: () => [ + LlamaCompletionChunkToolCall( + index: 0, + id: 'runtime-7', + type: 'function', + function: LlamaCompletionChunkFunction( + name: 'weather', + arguments: jsonEncode({'city': 'Oslo'}), + ), + ), + ], + ).collect(); + expect(completion.toolCalls.map((call) => call.id), [ + 'call_0', + 'runtime-7', + ]); + }); + + test('runtime tool calls keep content the stream withheld', () async { + final completion = await ChatCompletionStreamParser.parse( + tokenStream: Stream.fromIterable(['Answer [runtimeCall('Oslo')], + ).collect(); + expect(completion.text, 'Answer ', + '$_enda$_end$_envelope', + ]) { + test('a forced-open reply streams the same split by character: ' + '$output', () async { + final whole = await _parse([output], toolChoice: ToolChoice.auto); + final split = await _parse( + output.split(''), + toolChoice: ToolChoice.auto, + ); + expect(_thinking(split), _thinking(whole)); + expect(_content(split), _content(whole)); + expect( + split.expand((c) => c.choices.single.delta.toolCalls ?? const []), + hasLength( + whole + .expand((c) => c.choices.single.delta.toolCalls ?? const []) + .length, + ), + ); + }); + } + + for (final toolChoice in [ToolChoice.auto, ToolChoice.required]) { + test('an empty leading thought leaves a later unfinished thought as ' + 'content before a call ($toolChoice)', () async { + final output = '$_endunfinished$_envelope'; + for (final tokens in [ + [output], + output.split(''), + ]) { + final chunks = await _parse( + tokens, + forcedOpen: false, + toolChoice: toolChoice, + ); + expect(_thinking(chunks), isEmpty); + expect(_content(chunks), 'unfinished'); + final call = chunks + .expand((c) => c.choices.single.delta.toolCalls ?? const []) + .single; + expect(call.function?.arguments, '{"city":"Seoul"}'); + expect(chunks.last.choices.single.finishReason, 'tool_calls'); + } + }); + } }); } diff --git a/test/unit/tooling/pub_content_limits_test.dart b/test/unit/tooling/pub_content_limits_test.dart index 22b9171f6..3e430b835 100644 --- a/test/unit/tooling/pub_content_limits_test.dart +++ b/test/unit/tooling/pub_content_limits_test.dart @@ -126,6 +126,14 @@ void main() { 'LICENSE', 'example/README.md', ]); + for (final MapEntry(key: name, value: path) in _packages.entries.skip(1)) { + expect(pubContentFiles(Directory.current, path, name), [ + '$path/README.md', + '$path/CHANGELOG.md', + '$path/LICENSE', + '$path/example/README.md', + ]); + } }); group('verify_release_docs_versions.dart', () { diff --git a/website/docs/guides/template-engine-internals.md b/website/docs/guides/template-engine-internals.md index 6defd771f..3943818b9 100644 --- a/website/docs/guides/template-engine-internals.md +++ b/website/docs/guides/template-engine-internals.md @@ -229,11 +229,10 @@ error, llamadart deliberately stays lenient - A later scope without a line break after its tag, and any text after the last call, stay content after the calls before them. -The first two are rejected when the request's tools came from -`ToolDefinition.fromJsonSchema` ("The model returned an undeclared tool call." -or "Tool arguments do not match the input schema for ..."). With `ToolParam` -tools, or without tools, the Seed-OSS parse has no schemas and returns the -call, so validate the name and arguments before running it. +A Seed-OSS request refuses `ToolDefinition.fromJsonSchema` tools at render +(`LlamaUnsupportedException`), so the parse never sees their schemas and +returns the call either way. Validate the tool name and arguments before +running it. Streamed content and reasoning start and end with the whitespace the final parse keeps, with or without tools, apart from the cases below. The parse diff --git a/website/docs/maintainers/runtime-ownership.md b/website/docs/maintainers/runtime-ownership.md index 55e7b4aac..34f1c570f 100644 --- a/website/docs/maintainers/runtime-ownership.md +++ b/website/docs/maintainers/runtime-ownership.md @@ -82,8 +82,8 @@ benchmark's `max_num_tokens`, so the load reads the executor's limit from ids are too long. Exceeding the maximum number of tokens allowed: N >= M`. The probe sends `4 × requested + 64` pseudo-random digits, which Gemma, Qwen and Llama 3 tokenize at one to three digits per token, and is rejected before -any model work (about 20-100 ms at 65536). It also runs a one-token turn that -must report a decode count of 1 and a context count below `M`. Any other +any model work (about 20-125 ms at 65536 on Qwen3 0.6B CPU). It also runs a +one-token turn that must report a decode count of 1 and a context count below `M`. Any other outcome, including an accepted prompt or another error text, leaves `generationLimitUnsupportedReason` set. Requalify with `litert-lm-web-truncation` whenever the `@litert-lm/core` pin changes: a new stop condition in `ShouldStop` breaks