Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion example/chat_app/lib/providers/chat_provider.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2293,7 +2293,7 @@ class ChatProvider extends ChangeNotifier {
_removeEmptyAssistantPlaceholder();
if (toolCallsCutOff) {
_addInfoMessage(
'The reply reached the token limit, so its tool calls were not run. Raise Max generated tokens and send it again.',
'The reply reached the token limit, so its tool calls were not run. Raise Max generated tokens, or start a new chat if the context is full, and send it again.',
);
}
} catch (e) {
Expand Down
6 changes: 5 additions & 1 deletion example/chat_app/test/unit_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -1685,7 +1685,11 @@ void main() {
);
expect(
cutOffProvider.messages.last.text,
contains('reached the token limit, so its tool calls were not run'),
allOf(
contains('reached the token limit, so its tool calls were not run'),
contains('Raise Max generated tokens'),
contains('start a new chat if the context is full'),
),
);
});

Expand Down
19 changes: 18 additions & 1 deletion test/ci/test_supervise_chrome_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def patched_supervisor(self, patch):
"""A launcher that runs the production supervisor with `patch(module)` applied."""
launcher = Path(self.temp.name) / 'launcher.py'
launcher.write_text(
'import importlib.util, sys\n'
'import importlib.util, sys, time\n'
f'spec = importlib.util.spec_from_file_location("supervisor", {str(SCRIPT)!r})\n'
'module = importlib.util.module_from_spec(spec)\n'
'spec.loader.exec_module(module)\n'
Expand Down Expand Up @@ -352,6 +352,23 @@ def test_recovery_happens_no_earlier_than_post_done_seconds_after_done(self):
# Events are rounded to milliseconds, so compare at that precision.
self.assertGreaterEqual(round(at['browser_close_stall'] - at['reporter_done'], 3), 0.6)

def test_post_done_window_starts_after_a_slow_reporter_poll(self):
launcher = self.patched_supervisor(
'poll = module.ReporterEvents.poll\n'
'def slow_poll(self):\n'
' done = poll(self)\n'
' if done:\n'
' time.sleep(0.2)\n'
' return done\n'
'module.ReporterEvents.poll = slow_poll')
child, result = self.run_fixture(self.browser_then(
{'type': 'done', 'success': True}, 0.05, 'import time; time.sleep(5)'),
reporter=True, post_done=0.6, script=launcher)
self.assertEqual(child.returncode, 0, child.stderr)
self.assertTrue(result['browser_close_stall'])
at = {event['event']: event['elapsed_seconds'] for event in self.events()}
self.assertGreaterEqual(round(at['browser_close_stall'] - at['reporter_done'], 3), 0.6)

def test_leftover_owned_processes_after_a_recovered_stall_exit_125(self):
launcher = self.patched_supervisor('module.deliver = lambda row, number: False')
child, result = self.run_fixture(self.browser_then(
Expand Down
8 changes: 4 additions & 4 deletions test/fixtures/seed_oss_parallel_upstream.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ tracker [#1531](https://github.com/leehack/llamadart/issues/1531)). These are
not cases here; `seed_oss_handler_test.dart` pins them:

- An unknown tool name is returned as a call: `SeedOssHandler.parse` takes no
tool schemas. The engine and stream parser reject it for
`ToolDefinition.fromJsonSchema` tools.
- A call missing a required argument is returned with the arguments present,
with the same schema caveat.
tool schemas, and Seed-OSS requests refuse `ToolDefinition.fromJsonSchema`
tools at render, so validate the name before running the call.
- A call missing a required argument is returned with the arguments present;
validate the arguments before running it.
- A later scope whose `<seed:tool_call>` is not followed by a line break stays
content after the calls before it.

Expand Down
20 changes: 20 additions & 0 deletions test/unit/backends/litert_lm/litert_lm_backend_web_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -1438,6 +1438,26 @@ void main() {
: null,
),
),
(
'a limit error with text before it',
() => _installCountingFakeEngine(
counts: (_, budget) => (decoded: budget, total: 20),
reject: (prompt) {
final message = _inputTooLong(prompt, 4096);
return message == null ? null : 'Wrapped: $message';
},
),
),
(
'a limit error with text after it',
() => _installCountingFakeEngine(
counts: (_, budget) => (decoded: budget, total: 20),
reject: (prompt) {
final message = _inputTooLong(prompt, 4096);
return message == null ? null : '$message (retry)';
},
),
),
(
'a limit probe whose send throws',
() => _installCountingFakeEngine(
Expand Down
51 changes: 28 additions & 23 deletions test/unit/backends/litert_lm/litert_lm_v1_conversation_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -301,29 +301,34 @@ void main() {
},
);
}
test('send failure keeps the native message text and the native stack', () {
final runtime = loaded();
final conversation = runtime.openConversation();
call('fail', 17);
Object? error;
StackTrace? trace;
try {
conversation.sendText('failure');
} on Object catch (caught, stackTrace) {
error = caught;
trace = stackTrace;
}
expect(error, isA<LiteRtLmSendFailureException>());
expect(
(error! as LiteRtLmSendFailureException).message,
'conversation_send_message_stream failed: LiteRT-LM reported INTERNAL '
'(13): stage17 m.litertlm cache secret/dispatch.',
);
expect(trace.toString(), contains('LiteRtLmV1Runtime._check'));
call('fail', 0);
conversation.close();
runtime.dispose();
});
test(
'send failure keeps the native message text and the raising Dart frame',
() {
final runtime = loaded();
final conversation = runtime.openConversation();
addTearDown(() {
call('fail', 0);
conversation.close();
runtime.dispose();
});
call('fail', 17);
Object? error;
StackTrace? trace;
try {
conversation.sendText('failure');
} on Object catch (caught, stackTrace) {
error = caught;
trace = stackTrace;
}
expect(error, isA<LiteRtLmSendFailureException>());
expect(
(error! as LiteRtLmSendFailureException).message,
'conversation_send_message_stream failed: LiteRT-LM reported INTERNAL '
'(13): stage17 m.litertlm cache secret/dispatch.',
);
expect(trace.toString(), contains('LiteRtLmV1Runtime._check'));
},
);
test(
'cancellation status failure still prevents reuse and retains callback until final',
() async {
Expand Down
54 changes: 54 additions & 0 deletions test/unit/core/engine/chat_completion_stream_parser_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,60 @@ void main() {
expect(completion.toolCalls.map((call) => call.id), ['call_0', 'call_1']);
});

LlamaChatTemplateResult toolTemplate() => ChatTemplateEngine.render(
templateSource: '{{ messages | tojson }}{{ tools | tojson }}<tool_call>',
messages: const [
LlamaChatMessage.fromText(role: LlamaChatRole.user, text: 'Hi'),
],
metadata: const {},
tools: [_weatherTool],
);

test('a runtime tool call keeps its own id', () async {
final completion = await ChatCompletionStreamParser.parse(
tokenStream: Stream.fromIterable([
'<tool_call>{"name":"weather","arguments":{"city":"Paris"}}',
'</tool_call>',
]),
templateResult: toolTemplate(),
parseToolCallsEnabled: true,
enableThinking: true,
modelName: 'runtime',
completionId: 'runtime-call-id',
tools: [_weatherTool],
runtimeToolCalls: () => [
LlamaCompletionChunkToolCall(
index: 0,
id: 'runtime-7',
type: 'function',
function: LlamaCompletionChunkFunction(
name: 'weather',
arguments: jsonEncode({'city': 'Oslo'}),
),
),
],
).collect();
expect(completion.toolCalls.map((call) => call.id), [
'call_0',
'runtime-7',
]);
});

test('runtime tool calls keep content the stream withheld', () async {
final completion = await ChatCompletionStreamParser.parse(
tokenStream: Stream.fromIterable(['Answer <tool_ca']),
templateResult: toolTemplate(),
parseToolCallsEnabled: true,
enableThinking: true,
modelName: 'runtime',
completionId: 'runtime-call-content',
tools: [_weatherTool],
runtimeToolCalls: () => [runtimeCall('Oslo')],
).collect();
expect(completion.text, 'Answer <tool_ca');
expect(completion.toolCalls, hasLength(1));
});

test('runtime tool calls are ignored when tool parsing is off', () async {
final completion = await ChatCompletionStreamParser.parse(
tokenStream: Stream.fromIterable(['plain']),
Expand Down
48 changes: 48 additions & 0 deletions test/unit/core/engine/minimax_thinking_stream_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -155,5 +155,53 @@ void main() {
expect(chunks.last.choices.single.finishReason, 'stop');
},
);

for (final output in [
'$_end$_end<mm:think>',
'$_end<mm:think>a$_end$_envelope',
]) {
test('a forced-open reply streams the same split by character: '
'$output', () async {
final whole = await _parse([output], toolChoice: ToolChoice.auto);
final split = await _parse(
output.split(''),
toolChoice: ToolChoice.auto,
);
expect(_thinking(split), _thinking(whole));
expect(_content(split), _content(whole));
expect(
split.expand((c) => c.choices.single.delta.toolCalls ?? const []),
hasLength(
whole
.expand((c) => c.choices.single.delta.toolCalls ?? const [])
.length,
),
);
});
}

for (final toolChoice in [ToolChoice.auto, ToolChoice.required]) {
test('an empty leading thought leaves a later unfinished thought as '
'content before a call ($toolChoice)', () async {
final output = '<mm:think>$_end<mm:think>unfinished$_envelope';
for (final tokens in [
[output],
output.split(''),
]) {
final chunks = await _parse(
tokens,
forcedOpen: false,
toolChoice: toolChoice,
);
expect(_thinking(chunks), isEmpty);
expect(_content(chunks), '<mm:think>unfinished');
final call = chunks
.expand((c) => c.choices.single.delta.toolCalls ?? const [])
.single;
expect(call.function?.arguments, '{"city":"Seoul"}');
expect(chunks.last.choices.single.finishReason, 'tool_calls');
}
});
}
});
}
8 changes: 8 additions & 0 deletions test/unit/tooling/pub_content_limits_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,14 @@ void main() {
'LICENSE',
'example/README.md',
]);
for (final MapEntry(key: name, value: path) in _packages.entries.skip(1)) {
expect(pubContentFiles(Directory.current, path, name), [
'$path/README.md',
'$path/CHANGELOG.md',
'$path/LICENSE',
'$path/example/README.md',
]);
}
});

group('verify_release_docs_versions.dart', () {
Expand Down
9 changes: 4 additions & 5 deletions website/docs/guides/template-engine-internals.md
Original file line number Diff line number Diff line change
Expand Up @@ -229,11 +229,10 @@ error, llamadart deliberately stays lenient
- A later scope without a line break after its tag, and any text after the
last call, stay content after the calls before them.

The first two are rejected when the request's tools came from
`ToolDefinition.fromJsonSchema` ("The model returned an undeclared tool call."
or "Tool arguments do not match the input schema for ..."). With `ToolParam`
tools, or without tools, the Seed-OSS parse has no schemas and returns the
call, so validate the name and arguments before running it.
A Seed-OSS request refuses `ToolDefinition.fromJsonSchema` tools at render
(`LlamaUnsupportedException`), so the parse never sees their schemas and
returns the call either way. Validate the tool name and arguments before
running it.

Streamed content and reasoning start and end with the whitespace the final
parse keeps, with or without tools, apart from the cases below. The parse
Expand Down
4 changes: 2 additions & 2 deletions website/docs/maintainers/runtime-ownership.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ benchmark's `max_num_tokens`, so the load reads the executor's limit from
ids are too long. Exceeding the maximum number of tokens allowed: N >= M`.
The probe sends `4 × requested + 64` pseudo-random digits, which Gemma, Qwen
and Llama 3 tokenize at one to three digits per token, and is rejected before
any model work (about 20-100 ms at 65536). It also runs a one-token turn that
must report a decode count of 1 and a context count below `M`. Any other
any model work (about 20-125 ms at 65536 on Qwen3 0.6B CPU). It also runs a
one-token turn that must report a decode count of 1 and a context count below `M`. Any other
outcome, including an accepted prompt or another error text, leaves
`generationLimitUnsupportedReason` set. Requalify with `litert-lm-web-truncation` whenever the
`@litert-lm/core` pin changes: a new stop condition in `ShouldStop` breaks
Expand Down
Loading