diff --git a/CHANGELOG.md b/CHANGELOG.md index 83af7947f..d94fabd70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Releases before 0.10.0 are in the ## Unreleased - Parse Qwen3-Coder XML tool calls as llama.cpp does: a call needs a line break after each tag and around each value (a CRLF or compact `` not followed by a line break. Nemotron 3 Nano now parses as Qwen3-Coder XML, so a `` also ends its thought, and the tool-call grammar of these templates now requires that shape and starts at `` ([#1898](https://github.com/leehack/llamadart/issues/1898)). +- Parse Seed-OSS tool calls closer to llama.cpp: a finished reply whose last `` scope is never closed keeps a call complete through `` and drops one cut off before it, an argument declared as a string keeps its value as generated instead of being read as JSON (`\n42\n` stays a string), and with `ToolChoice.required` the grammar lets a thought and text come before the calls ([#1898](https://github.com/leehack/llamadart/issues/1898)). ## 0.19.4 diff --git a/lib/src/core/template/handlers/seed_oss_handler.dart b/lib/src/core/template/handlers/seed_oss_handler.dart index 0311135f9..59d756959 100644 --- a/lib/src/core/template/handlers/seed_oss_handler.dart +++ b/lib/src/core/template/handlers/seed_oss_handler.dart @@ -2,19 +2,23 @@ import 'package:dinja/dinja.dart'; import '../../models/chat/chat_message.dart'; import '../../models/chat/chat_template_result.dart'; +import '../../models/inference/tool_choice.dart'; import '../../models/tools/tool_definition.dart'; import '../chat_format.dart'; import '../chat_parse_result.dart'; import '../chat_template_handler.dart'; import '../template_internal_metadata.dart'; import '../thinking_utils.dart'; +import '../tool_call_grammar_utils.dart'; +import '../tool_schema_utils.dart'; import '../xml_tool_call_format.dart'; /// Handler for Seed-OSS format. /// /// Uses ``/`` for reasoning and /// `...` for tools. -class SeedOssHandler extends ChatTemplateHandler { +class SeedOssHandler extends ChatTemplateHandler + implements ToolSchemaAwareChatTemplateHandler { @override ChatFormat get format => ChatFormat.seedOss; @@ -77,19 +81,26 @@ class SeedOssHandler extends ChatTemplateHandler { } final hasTools = tools != null && tools.isNotEmpty; + final grammar = buildXmlToolCallGrammar( + tools, + XmlToolCallFormat.seedOss, + parallelToolCalls: + metadata[internalParallelToolCallsMetadataKey] == 'true', + ); final rendered = LlamaChatTemplateResult( prompt: prompt, format: format.index, - grammar: requiredGrammarAfterThought( - buildXmlToolCallGrammar( - tools, - XmlToolCallFormat.seedOss, - parallelToolCalls: - metadata[internalParallelToolCallsMetadataKey] == 'true', - ), - metadata: metadata, - thinkingForcedOpen: thinkingForcedOpen, - ), + grammar: + grammar != null && + !thinkingForcedOpen && + metadata[internalToolChoiceMetadataKey] == + ToolChoice.required.name + ? _withThoughtAndTextBeforeCalls(grammar) + : requiredGrammarAfterThought( + grammar, + metadata: metadata, + thinkingForcedOpen: thinkingForcedOpen, + ), grammarLazy: hasTools, thinkingForcedOpen: thinkingForcedOpen, additionalStops: getStops( @@ -104,12 +115,50 @@ class SeedOssHandler extends ChatTemplateHandler { return withThoughtClosedForThinkingOff(rendered, closed: thoughtClosed); } + /// [grammar] after what llama.cpp's (`cd959bae9`) required-call grammar + /// lets precede the calls: a thought that starts the reply, then any text + /// without a scope start. + static String _withThoughtAndTextBeforeCalls(String grammar) => [ + for (final line in grammar.trimRight().split('\n')) + line.startsWith('root ::= ') + ? 'root ::= seed-thought? text-before-calls ' + '(${line.substring('root ::= '.length)})' + : line, + 'seed-thought ::= [ \\t\\n]* "" seed-thought-text ' + '""', + ...ToolCallGrammarUtils.excludingRules('seed-thought-text', [ + '', + ]), + ...ToolCallGrammarUtils.excludingRules('text-before-calls', [ + XmlToolCallFormat.seedOss.scopeStart, + ]), + ].join('\n'); + @override ChatParseResult parse( String output, { bool isPartial = false, bool parseToolCalls = true, bool thinkingForcedOpen = false, + }) => parseWithTools( + output, + isPartial: isPartial, + parseToolCalls: parseToolCalls, + thinkingForcedOpen: thinkingForcedOpen, + ); + + /// Parses Seed-OSS output, keeping as generated the value of each argument + /// that [tools] declare as a string, as llama.cpp does. + /// + /// Other values are read as JSON when they are JSON, and the call's tool and + /// arguments are not checked against [tools]. + @override + ChatParseResult parseWithTools( + String output, { + List? tools, + bool isPartial = false, + bool parseToolCalls = true, + bool thinkingForcedOpen = false, }) { // As in llama.cpp, a thought the prompt opened is the reply up to the // first end tag, verbatim, and one it did not open exists only when the @@ -128,12 +177,29 @@ class SeedOssHandler extends ChatTemplateHandler { isPartial: isPartial, ) : null; - final parsed = parseXmlToolCalls( - thought?.content ?? output, + final body = thought?.content ?? output; + final stringArguments = { + for (final tool in tools ?? const []) + tool.name: { + for (final MapEntry(:key, :value) in schemaProperties( + tool.toJsonSchema(), + ).entries) + if (schemaResolvesToString(value)) key, + }, + }; + ChatParseResult parseBody(String text) => parseXmlToolCalls( + text, XmlToolCallFormat.seedOss, parseToolCalls: parseToolCalls, extractsThought: false, + stringArguments: stringArguments, ); + final ended = isPartial || !parseToolCalls + ? (text: body, endAdded: false) + : _withLastScopeEnded(body); + var parsed = parseBody(ended.text); + // When the calls still fail, the added scope end would show in content. + if (ended.endAdded && parsed.toolCalls.isEmpty) parsed = parseBody(body); return ChatParseResult( content: parsed.content.trim(), reasoningContent: thought?.reasoning, @@ -141,6 +207,70 @@ class SeedOssHandler extends ChatTemplateHandler { ); } + /// [reply] as llama.cpp reads a finished one that ends inside its last + /// scope: with the scope end that a complete call is missing, and without + /// a scope that ends before its call does. + /// + /// A last scope whose text could not start a call is left as it is. + /// [endAdded] tells whether the scope end was added. + static ({String text, bool endAdded}) _withLastScopeEnded(String reply) { + const format = XmlToolCallFormat.seedOss; + final unchanged = (text: reply, endAdded: false); + final start = reply.lastIndexOf(format.scopeStart); + if (start < 0 || reply.contains(format.scopeEnd, start)) return unchanged; + final callEnd = _unendedCallEnd(reply, start + format.scopeStart.length); + if (callEnd == null) return unchanged; + if (callEnd < 0) return (text: reply.substring(0, start), endAdded: false); + final rest = reply.substring(callEnd).trimLeft(); + return format.scopeEnd.startsWith(rest) + ? ( + text: '${reply.substring(0, callEnd)}${format.scopeEnd}', + endAdded: true, + ) + : unchanged; + } + + /// Where the call that the scope opening before [from] in [reply] holds + /// ends, after `` and its line break; -1 when [reply] ends + /// before it does; null when the text cannot be such a call. + static int? _unendedCallEnd(String reply, int from) { + const format = XmlToolCallFormat.seedOss; + bool endsIn(int at, String token) => + reply.length - at < token.length && + token.startsWith(reply.substring(at)); + var pos = _skipSpace(reply, from); + if (pos == reply.length || endsIn(pos, format.toolStart)) return -1; + if (!reply.startsWith(format.toolStart, pos)) return null; + final nameEnd = reply.indexOf(format.toolSep, pos); + if (nameEnd < 0) return -1; + pos = nameEnd + format.toolSep.length; + while (true) { + pos = _skipSpace(reply, pos); + if (pos == reply.length || + endsIn(pos, format.keyStart) || + endsIn(pos, format.toolEnd)) { + return -1; + } + if (reply.startsWith(format.toolEnd, pos)) { + pos += format.toolEnd.length; + if (pos == reply.length) return -1; + return reply.codeUnitAt(pos) == 0x0A ? pos + 1 : null; + } + if (!reply.startsWith(format.keyStart, pos)) return null; + final valueEnd = reply.indexOf(format.valEnd, pos); + if (valueEnd < 0) return -1; + pos = valueEnd + format.valEnd.length; + } + } + + static int _skipSpace(String text, int from) { + var pos = from; + while (pos < text.length && ' \t\n\r'.contains(text[pos])) { + pos++; + } + return pos; + } + @override String? buildGrammar(List? tools) { return buildXmlToolCallGrammar( diff --git a/lib/src/core/template/tool_call_grammar_utils.dart b/lib/src/core/template/tool_call_grammar_utils.dart index 7f67910c7..e5a563c36 100644 --- a/lib/src/core/template/tool_call_grammar_utils.dart +++ b/lib/src/core/template/tool_call_grammar_utils.dart @@ -145,11 +145,14 @@ class ToolCallGrammarUtils { static List excludingRules(String name, List delimiters) { String state(int i) => i == 0 ? name : '$name-$i'; // llama.cpp's GBNF parser has no `\^` escape, and a leading `^` would - // negate the class. - String chars(String value) => value.replaceAllMapped( - RegExp(r'[\\\]\[^-]'), - (m) => m[0] == '^' ? r'\x5E' : '\\${m[0]}', - ); + // negate the class. A raw line break would split the rule's line. + String chars(String value) => + value.replaceAllMapped(RegExp(r'[\\\]\[^\x00-\x1F-]'), (m) { + final code = m[0]!.codeUnitAt(0); + return m[0] == '^' || code < 0x20 + ? '\\x${code.toRadixString(16).padLeft(2, '0').toUpperCase()}' + : '\\${m[0]}'; + }); final children = >[{}]; final ends = [0]; for (final delimiter in delimiters) { diff --git a/lib/src/core/template/xml_tool_call_format.dart b/lib/src/core/template/xml_tool_call_format.dart index 75eba46a3..067729424 100644 --- a/lib/src/core/template/xml_tool_call_format.dart +++ b/lib/src/core/template/xml_tool_call_format.dart @@ -348,6 +348,9 @@ String _literal(String value) { /// content is the text before the first scope, the calls are those of the /// consecutive scopes after it that each hold one valid call, up to /// [maxToolCalls], and the rest is dropped. +/// +/// Without [schemas], the arguments that [stringArguments] lists for a tool +/// keep their text as generated instead of being read as JSON. ChatParseResult parseXmlToolCalls( String input, XmlToolCallFormat format, { @@ -362,6 +365,7 @@ ChatParseResult parseXmlToolCalls( bool endsAtLastCall = false, int? maxToolCalls, Map>? schemas, + Map>? stringArguments, }) { String? reasoning; var content = input; @@ -480,7 +484,13 @@ ChatParseResult parseXmlToolCalls( final toolIdx = _nextScopeToolStart(content, pos, format); final nextCall = toolIdx == null ? null - : _parseXmlToolCall(content, toolIdx, format, schemas); + : _parseXmlToolCall( + content, + toolIdx, + format, + schemas, + stringArguments, + ); if (nextCall == null) break; repeatedScope = (scopeEnd: pos, calls: toolCalls.length); toolCall = nextCall; @@ -507,7 +517,13 @@ ChatParseResult parseXmlToolCalls( parsedContent.write(content.substring(pos, toolIdx)); } - toolCall = _parseXmlToolCall(content, toolIdx, format, schemas); + toolCall = _parseXmlToolCall( + content, + toolIdx, + format, + schemas, + stringArguments, + ); if (toolCall == null) { if (format.scopeStart.isNotEmpty) { parseFailed = true; @@ -788,8 +804,9 @@ _ParsedXmlToolCall? _parseXmlToolCall( String content, int toolIdx, XmlToolCallFormat format, - Map>? schemas, -) { + Map>? schemas, [ + Map>? stringArguments, +]) { final nameStart = toolIdx + format.toolStart.length; final sepIdx = content.indexOf(format.toolSep, nameStart); if (sepIdx == -1) { @@ -806,6 +823,7 @@ _ParsedXmlToolCall? _parseXmlToolCall( sepIdx + format.toolSep.length, format, schemas?[name], + stringArguments?[name] ?? const {}, ); if (arguments == null) { return null; @@ -823,6 +841,7 @@ _ParsedXmlArguments? _parseXmlArguments( int start, XmlToolCallFormat format, Map? schema, + Set stringArguments, ) { final properties = schema == null ? null : schemaProperties(schema); final strictScope = format.scopeStart.isNotEmpty; @@ -870,6 +889,7 @@ _ParsedXmlArguments? _parseXmlArguments( pos, format, schema: properties?[key], + keepsText: stringArguments.contains(key), ); if (argumentValue == null) { return null; @@ -990,6 +1010,7 @@ _ParsedXmlArgumentValue? _parseXmlArgumentValue( int start, XmlToolCallFormat format, { Map? schema, + bool keepsText = false, }) { if (schema != null) { final cdata = _parseCdataValue(content, start, format); @@ -1039,7 +1060,7 @@ _ParsedXmlArgumentValue? _parseXmlArgumentValue( value = value.trim(); } return _ParsedXmlArgumentValue( - value: format.rawArgval == true + value: format.rawArgval == true || keepsText ? value : ToolCallParsingUtils.decodeJsonValueOrString(value), nextPos: valueEnd.nextPos, diff --git a/test/fixtures/seed_oss_tool_call_upstream.json b/test/fixtures/seed_oss_tool_call_upstream.json new file mode 100644 index 000000000..453d01d44 --- /dev/null +++ b/test/fixtures/seed_oss_tool_call_upstream.json @@ -0,0 +1,2798 @@ +{ + "upstream_commit": "cd959bae99ff4bed22bef2604f45241c09bbb8c8", + "user_prompt": "Weather in Paris?", + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "w", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + } + } + } + ], + "templates": [ + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": true, + "parallel_tool_calls": false, + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + }, + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": true, + "parallel_tool_calls": false, + "tool_choice": "required", + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + }, + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": true, + "parallel_tool_calls": true, + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "two-calls", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-second", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "call-then-scope-start", + "output": "\n\n\nParis\n\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + }, + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": true, + "parallel_tool_calls": true, + "tool_choice": "required", + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "two-calls", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-second", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "call-then-scope-start", + "output": "\n\n\nParis\n\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + }, + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": false, + "parallel_tool_calls": false, + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + }, + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": false, + "parallel_tool_calls": false, + "tool_choice": "required", + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + }, + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": false, + "parallel_tool_calls": true, + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "two-calls", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-second", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "call-then-scope-start", + "output": "\n\n\nParis\n\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + }, + { + "template": "test/fixtures/llama_cpp_templates/ByteDance-Seed-OSS.jinja", + "enable_thinking": false, + "parallel_tool_calls": true, + "tool_choice": "required", + "cases": [ + { + "name": "call", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "two-calls", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed", + "output": "\n\n\nParis\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-blank-line", + "output": "\n\n\nParis\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-partial-end", + "output": "\n\n\nParis\n\n\n\n\n\nParis\n\n\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Let me check.\n", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "unclosed-second", + "output": "\n\n\nParis\n\n\n\n\n\n\nRome\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + }, + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nRome\\n\"}" + } + ] + } + }, + { + "name": "unclosed-after-name", + "output": "Hi \n", + "upstream": { + "content": "Hi ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-at-end", + "output": "Hello \n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-then-blank-line", + "output": "Hello \n\n", + "upstream": { + "content": "Hello ", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "scope-start-only", + "output": "\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [] + } + }, + { + "name": "call-then-scope-start", + "output": "\n\n\nParis\n\n\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + }, + { + "name": "scope-start-then-function-prefix", + "output": "Hello \n\n\n\n42\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n42\\n\"}" + } + ] + } + }, + { + "name": "string-object", + "output": "\n\n\n{\"a\": 1}\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n{\\\"a\\\": 1}\\n\"}" + } + ] + } + }, + { + "name": "string-quoted", + "output": "\n\n\n\"Paris\"\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n\\\"Paris\\\"\\n\"}" + } + ] + } + }, + { + "name": "string-bool", + "output": "\n\n\ntrue\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\ntrue\\n\"}" + } + ] + } + }, + { + "name": "string-array", + "output": "\n\n\n[1, 2]\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\n[1, 2]\\n\"}" + } + ] + } + }, + { + "name": "string-null", + "output": "\n\n\nnull\n\n\n", + "upstream": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nnull\\n\"}" + } + ] + } + }, + { + "name": "thought-then-text-and-call", + "output": "\nPlan\n\nSure.\n\n\n\nParis\n\n\n", + "upstream": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + }, + "upstream_partial": { + "content": "Sure.\n", + "reasoning_content": "\nPlan\n", + "tool_calls": [ + { + "name": "get_weather", + "arguments": "{\"location\":\"\\nParis\\n\"}" + } + ] + } + } + ] + } + ] +} diff --git a/test/fixtures/seed_oss_tool_call_upstream.md b/test/fixtures/seed_oss_tool_call_upstream.md new file mode 100644 index 000000000..1ce8913f3 --- /dev/null +++ b/test/fixtures/seed_oss_tool_call_upstream.md @@ -0,0 +1,25 @@ +# Seed-OSS tool-call upstream parses + +`seed_oss_tool_call_upstream.json` records how llama.cpp parses Seed-OSS +replies whose last scope is unclosed, whose string value looks like JSON, or +whose scope start ends the text, for issue #1898. The template is the vendored +`ByteDance-Seed-OSS.jinja`. + +The probe is the one `qwen3_coder_xml_upstream.md` describes, at the same +llama.cpp commit, with the tool in `tools`: `get_weather` with a required +string `location`. Each entry sets its `enable_thinking`, +`parallel_tool_calls` and, when not auto, `tool_choice`. Feeding the whole +output first gave the same result. `upstream_partial` is +`common_chat_peg_parse` with `is_partial`, recorded only where llamadart's +partial parse already matches it. + +Left out: + +- Without parallel tool calls, a second scope after a call: llama.cpp fails + the whole reply with an error, and llamadart returns the calls. +- A scope start followed by text that is not a call, as in + `Hello \nmore`: llama.cpp fails the reply, and llamadart + keeps it as content. +- A finished reply cut off inside its last call, after `` without + its line break or inside a value: llama.cpp returns a call whose JSON + arguments never end, and llamadart drops the call. diff --git a/test/integration/core/engine/seed_oss_tool_call_upstream_test.dart b/test/integration/core/engine/seed_oss_tool_call_upstream_test.dart new file mode 100644 index 000000000..e2c6822c3 --- /dev/null +++ b/test/integration/core/engine/seed_oss_tool_call_upstream_test.dart @@ -0,0 +1,177 @@ +@TestOn('vm') +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart/src/core/engine/chat_completion_stream_parser.dart'; +import 'package:llamadart/src/core/models/chat/chat_message.dart'; +import 'package:llamadart/src/core/models/chat/chat_role.dart'; +import 'package:llamadart/src/core/models/chat/chat_template_result.dart'; +import 'package:llamadart/src/core/models/inference/tool_choice.dart'; +import 'package:llamadart/src/core/models/tools/tool_definition.dart'; +import 'package:llamadart/src/core/models/tools/tool_param.dart'; +import 'package:llamadart/src/core/template/chat_format.dart'; +import 'package:llamadart/src/core/template/chat_parse_result.dart'; +import 'package:llamadart/src/core/template/chat_template_engine.dart'; +import 'package:test/test.dart'; + +final Map _fixture = + jsonDecode( + File( + 'test/fixtures/seed_oss_tool_call_upstream.json', + ).readAsStringSync(), + ) + as Map; + +final List _tools = [ + ToolDefinition( + name: 'get_weather', + description: 'w', + parameters: [ToolParam.string('location', required: true)], + ), +]; + +typedef _Message = Map; + +List _calls(Iterable<({String? name, String? arguments})> calls) => [ + for (final call in calls) '${call.name} ${jsonDecode(call.arguments!)}', +]; + +// llamadart trims some whitespace around content and reasoning that llama.cpp +// keeps. +_Message _upstream(Map message) => { + 'content': (message['content'] as String).trim(), + 'reasoning': (message['reasoning_content'] as String).trim(), + 'calls': _calls([ + for (final call + in (message['tool_calls'] as List).cast>()) + (name: call['name'] as String, arguments: call['arguments'] as String), + ]), +}; + +_Message _parsed(ChatParseResult parsed) => { + 'content': parsed.content.trim(), + 'reasoning': (parsed.reasoningContent ?? '').trim(), + 'calls': _calls([ + for (final call in parsed.toolCalls) + (name: call.function?.name, arguments: call.function?.arguments), + ]), +}; + +/// [output] whole, bytewise, in 3-character chunks and split in two at every +/// point. +List> _chunkings(String output) => [ + [output], + output.split(''), + [ + for (var i = 0; i < output.length; i += 3) + output.substring(i, i + 3 > output.length ? output.length : i + 3), + ], + for (var split = 1; split < output.length; split++) + [output.substring(0, split), output.substring(split)], +]; + +Future<_Message> _streamed( + LlamaChatTemplateResult rendered, + List tokens, +) async { + final chunks = await ChatCompletionStreamParser.parse( + tokenStream: Stream.fromIterable(tokens), + templateResult: rendered, + parseToolCallsEnabled: true, + enableThinking: true, + modelName: 'test-model', + completionId: 'seed-oss-tool-call-upstream', + tools: _tools, + ).toList(); + final deltas = chunks.map((chunk) => chunk.choices.single.delta).toList(); + return { + 'content': deltas.map((delta) => delta.content ?? '').join().trim(), + 'reasoning': deltas.map((delta) => delta.thinking ?? '').join().trim(), + 'calls': _calls([ + for (final delta in deltas) + for (final call in delta.toolCalls ?? const []) + (name: call.function?.name, arguments: call.function?.arguments), + ]), + }; +} + +// llama.cpp takes the complete call of a finished Seed-OSS reply that ends +// inside its last scope, drops a scope that ends before its call does, and +// keeps a declared string value as generated. +void main() { + test('the fixture declares the tools of the test', () { + expect(_fixture['tools'], [ + for (final tool in _tools) jsonDecode(jsonEncode(tool.toJson())), + ]); + }); + + for (final entry + in (_fixture['templates'] as List).cast>()) { + final templatePath = entry['template'] as String; + final enableThinking = entry['enable_thinking'] as bool; + final parallelToolCalls = entry['parallel_tool_calls'] as bool; + final toolChoice = ToolChoice.values.byName( + entry['tool_choice'] as String? ?? 'auto', + ); + group('${templatePath.split('/').last}, thinking $enableThinking, ' + 'parallel $parallelToolCalls, ${toolChoice.name}', () { + late LlamaChatTemplateResult rendered; + + setUpAll( + () => rendered = ChatTemplateEngine.render( + templateSource: File(templatePath).readAsStringSync(), + messages: [ + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: _fixture['user_prompt'] as String, + ), + ], + metadata: const {}, + tools: _tools, + toolChoice: toolChoice, + parallelToolCalls: parallelToolCalls, + enableThinking: enableThinking, + ), + ); + + for (final c in (entry['cases'] as List).cast>()) { + final output = c['output'] as String; + test('${c['name']} parses and streams as llama.cpp parses', () async { + expect(rendered.format, ChatFormat.seedOss.index); + final parsed = _parsed( + ChatTemplateEngine.parseRendered(rendered, output, tools: _tools), + ); + expect( + parsed, + _upstream(c['upstream'] as Map), + reason: 'finished', + ); + final upstreamPartial = c['upstream_partial']; + if (upstreamPartial != null) { + expect( + _parsed( + ChatTemplateEngine.parseRendered( + rendered, + output, + isPartial: true, + tools: _tools, + ), + ), + _upstream(upstreamPartial as Map), + reason: 'partial', + ); + } + for (final tokens in _chunkings(output)) { + expect( + await _streamed(rendered, tokens), + parsed, + reason: jsonEncode(tokens), + ); + } + }); + } + }); + } +} diff --git a/test/unit/core/template/handlers/seed_oss_handler_test.dart b/test/unit/core/template/handlers/seed_oss_handler_test.dart index 7bba88ad8..d179cedd3 100644 --- a/test/unit/core/template/handlers/seed_oss_handler_test.dart +++ b/test/unit/core/template/handlers/seed_oss_handler_test.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'package:llamadart/src/core/models/chat/chat_message.dart'; import 'package:llamadart/src/core/models/chat/chat_role.dart'; +import 'package:llamadart/src/core/models/inference/tool_choice.dart'; import 'package:llamadart/src/core/models/tools/tool_definition.dart'; import 'package:llamadart/src/core/models/tools/tool_param.dart'; import 'package:llamadart/src/core/template/chat_format.dart'; @@ -10,6 +11,8 @@ import 'package:llamadart/src/core/template/handlers/seed_oss_handler.dart'; import 'package:llamadart/src/core/template/template_internal_metadata.dart'; import 'package:test/test.dart'; +import '../../../../support/gbnf_recognizer.dart'; + void main() { test('SeedOssHandler renders XML grammar and parses seed tool call', () { final handler = SeedOssHandler(); @@ -255,6 +258,149 @@ void main() { }); }); + // llama.cpp `cd959bae9`; the replies are in seed_oss_tool_call_upstream.json. + group('SeedOssHandler.parseWithTools', () { + final tools = [ + ToolDefinition( + name: 'get_weather', + description: 'w', + parameters: [ + ToolParam.string('location', required: true), + ToolParam.integer('days'), + ], + ), + ]; + String call(String location, {String days = ''}) => + '\n\n' + '\n$location\n\n' + '$days\n'; + Map arguments(ChatParseResult parsed) => + jsonDecode(parsed.toolCalls.single.function!.arguments!) + as Map; + + test('keeps a string argument that looks like JSON as text', () { + final output = + '${call('42', days: '\n3\n\n')}' + ''; + + expect(arguments(SeedOssHandler().parseWithTools(output, tools: tools)), { + 'location': '\n42\n', + 'days': 3, + }); + expect(arguments(SeedOssHandler().parse(output)), { + 'location': 42, + 'days': 3, + }); + }); + + test('ends a last scope whose call is complete', () { + for (final rest in ['', '\n', ' \n', + '\n\n\n\n\nPa', + complete.substring(0, complete.length - 1), + ]) { + final parsed = SeedOssHandler().parseWithTools( + 'Checking.\n$cut', + tools: tools, + ); + + expect(parsed.toolCalls, isEmpty, reason: cut); + expect(parsed.content, 'Checking.', reason: cut); + } + }); + + // llama.cpp fails these replies. + test('keeps a last scope that holds no call as content', () { + final complete = call('Paris'); + for (final output in [ + '\nmore', + '\n\nmore', + '${complete}tail', + '${complete.substring(0, complete.length - 1)}tail', + '${complete.substring(0, complete.length - 1)} \n\n' + '\nParis\n\n' + '\n'; + final tools = [ + ToolDefinition( + name: 'get_weather', + description: 'w', + parameters: [ToolParam.string('location', required: true)], + ), + ]; + final grammar = GbnfRecognizer( + SeedOssHandler() + .render( + templateSource: '{{ messages[0]["content"] }}', + messages: const [ + LlamaChatMessage.fromText(role: LlamaChatRole.user, text: 'hi'), + ], + metadata: {internalToolChoiceMetadataKey: ToolChoice.required.name}, + tools: tools, + ) + .grammar!, + ); + + for (final (reply, accepted) in [ + (scope, true), + ('Checking.\n$scope', true), + ('Plan\n$scope', true), + ('\nPlanChecking. $scope', true), + ( + 'PlanAgain$scope', + true, + ), + ('Plan', false), + ('Checking.', false), + ('Use \n then. $scope', false), + ('$scope\n', true), + ('Use then.\n$scope', true), + ('a \n b$scope', true), + ]) { + expect(grammar.accepts(reply), accepted, reason: reply); + } + }); + test('SeedOssHandler returns a forced-open thought as reasoning', () { const call = '\nParis' diff --git a/test/unit/core/template/tool_call_grammar_utils_test.dart b/test/unit/core/template/tool_call_grammar_utils_test.dart index 2614a2160..a5d47c8b0 100644 --- a/test/unit/core/template/tool_call_grammar_utils_test.dart +++ b/test/unit/core/template/tool_call_grammar_utils_test.dart @@ -167,10 +167,12 @@ void main() { ['abab', 'ba'], ['aa', 'ab', 'b'], ['^a', 'a^]', '-\\['], + ['\n', '\t\r'], ]) { final rules = ToolCallGrammarUtils.excludingRules('t', delimiters); // llama.cpp's GBNF parser rejects `\^`. expect(rules.join('\n'), isNot(contains(r'\^')), reason: '$delimiters'); + expect(rules, everyElement(isNot(contains('\n'))), reason: '$delimiters'); final body = _ExcludingRules(rules, 't'); final alphabet = {...delimiters.join().split(''), 'z'}.toList(); var texts = ['']; diff --git a/website/docs/changelog/recent-releases.md b/website/docs/changelog/recent-releases.md index 0372cda5c..3f229ec60 100644 --- a/website/docs/changelog/recent-releases.md +++ b/website/docs/changelog/recent-releases.md @@ -14,6 +14,7 @@ validate against the exact release tag notes. ## Unreleased - Parse Qwen3-Coder XML tool calls as llama.cpp does: a call needs a line break after each tag and around each value (a CRLF or compact `` not followed by a line break. Nemotron 3 Nano now parses as Qwen3-Coder XML, so a `` also ends its thought, and the tool-call grammar of these templates now requires that shape and starts at `` ([#1898](https://github.com/leehack/llamadart/issues/1898)). +- Parse Seed-OSS tool calls closer to llama.cpp: a finished reply whose last `` scope is never closed keeps a call complete through `` and drops one cut off before it, an argument declared as a string keeps its value as generated instead of being read as JSON (`\n42\n` stays a string), and with `ToolChoice.required` the grammar lets a thought and text come before the calls ([#1898](https://github.com/leehack/llamadart/issues/1898)). ## 0.19.4 diff --git a/website/docs/guides/template-engine-internals.md b/website/docs/guides/template-engine-internals.md index 6e66e9f66..0afb3a38e 100644 --- a/website/docs/guides/template-engine-internals.md +++ b/website/docs/guides/template-engine-internals.md @@ -228,7 +228,12 @@ calls start at the first `` followed by a line break, and the text before it, including a `` followed by anything else (as in `\n`. Where upstream fails the whole reply with an +only at `\n`, and with `ToolChoice.required` the grammar lets a +thought and text without that tag come first when the prompt opened no thought. +A finished reply whose last scope is never closed keeps a call complete through +`` and its line break, and drops a scope that ends before its call +does. An argument the tool declares as a string keeps its value as generated, +even one that looks like JSON. Where upstream fails the whole reply with an error, llamadart deliberately stays lenient ([#1542](https://github.com/leehack/llamadart/pull/1542#issuecomment-6758592134)):