This client is a pure generator: it streams tokens for one turn
+ * and keeps no tool-calling logic. All StudyOS tool routing lives in the Dart
+ * layer ({@code LocalNativeLlmProvider}), which owns the single {@code [TOOL:…]}
+ * loop; the native side only produces text.
+ *
+ *
The system prompt is installed once as the conversation's system
+ * instruction and reused across turns; the conversation (and its KV cache) is
+ * only rebuilt when the model path, backend preference, or system instruction
+ * actually changes (see {@link #ensureConversation}).
+ */
public class LiteRtLocalPromptClient implements AutoCloseable {
private static final String TAG = "LiteRtLocalPrompt";
- private static final Pattern TOOL_CALL_PATTERN = Pattern.compile("\\[TOOL:([^:\\]]+):?([^\\]]*)\\]");
- private static final int MAX_TOOL_ROUNDS = 3;
- private static final int TOOL_SAMPLER_TOP_K = 10;
- private static final double TOOL_SAMPLER_TOP_P = 0.95;
- private static final double TOOL_SAMPLER_TEMPERATURE = 0.2;
- private static final int TOOL_SAMPLER_RANDOM_SEED = 0;
+
+ // Deterministic on-device sampling profile. LiteRT-LM 0.13.1 binds the
+ // sampler once at conversation creation — there is no per-message override
+ // (LiteRT-LM issue #2249), and rebuilding a conversation to change it would
+ // drop the KV cache. So a single low-temperature profile is used for all
+ // local generation, favouring reliable tool-directive/JSON formatting.
+ private static final int LOCAL_SAMPLER_TOP_K = 10;
+ private static final double LOCAL_SAMPLER_TOP_P = 0.95;
+ private static final double LOCAL_SAMPLER_TEMPERATURE = 0.2;
+ private static final int LOCAL_SAMPLER_RANDOM_SEED = 0;
+
+ /** Upper bound on a single generation before it is cancelled and surfaced as an error. */
+ private static final long GENERATION_TIMEOUT_SECONDS = 120;
+
+ private static final String DEFAULT_SYSTEM_INSTRUCTION =
+ "You are StudyOS Agent. Answer from the provided context.";
/** Prefer the GPU backend, falling back to CPU when GPU init fails. */
public static final String BACKEND_GPU = "gpu";
@@ -39,10 +73,17 @@ public class LiteRtLocalPromptClient implements AutoCloseable {
private Engine engine;
private Conversation conversation;
private String activeModelPath;
+ private String activeSystemInstruction;
private String activeBackend;
private String activeBackendPreference;
+ private String activeToolsSignature;
private volatile String backendPreference = BACKEND_GPU;
+ /** Receives streamed tokens as they are generated. */
+ public interface StreamListener {
+ void onToken(String token);
+ }
+
/** The accelerator the live engine initialized on ("GPU" or "CPU"), or null. */
public String getActiveBackend() {
return activeBackend;
@@ -58,114 +99,361 @@ public void setBackendPreference(String preference) {
backendPreference = BACKEND_CPU.equals(preference) ? BACKEND_CPU : BACKEND_GPU;
}
- public interface ToolExecutor {
- boolean canExecute(String toolName);
-
- String execute(String toolName, String argument);
- }
-
- /** Receives streamed tokens and a reset signal between tool rounds. */
- public interface StreamListener {
- void onToken(String token);
-
- void onReset();
- }
-
- public synchronized String generate(String modelPath, String prompt, String cacheDir) throws Exception {
- ensureConversation(modelPath, cacheDir);
- return extractText(conversation.sendMessage(prompt, Collections.emptyMap()));
+ /**
+ * Generates a reply for {@code prompt}, streaming tokens to {@code listener}.
+ * {@code systemInstruction} is installed as the conversation's system prompt
+ * and only triggers a conversation rebuild when it changes.
+ */
+ public synchronized String generateStreaming(
+ String modelPath,
+ String prompt,
+ String cacheDir,
+ String systemInstruction,
+ StreamListener streamListener
+ ) throws Exception {
+ ensureConversation(modelPath, cacheDir, systemInstruction, Collections.emptyList());
+ return streamSendMessage(prompt, streamListener);
}
- public synchronized String generateWithTools(
+ /**
+ * Native function-calling first turn (manual mode). Ensures a tool-enabled
+ * conversation from {@code toolSchemasJson} (OpenAPI function declarations),
+ * streams {@code prompt}, and returns a structured result map:
+ * {@code {"type":"tool_calls","calls":[{"name","arguments"(JSON string)}]}}
+ * or {@code {"type":"text","text"}}. Text fragments of a plain-answer turn are
+ * streamed to {@code streamListener} as they arrive (a tool-request turn emits
+ * no user-visible tokens); tool execution stays in the Dart layer, and results
+ * come back via {@link #continueWithToolResults}.
+ */
+ public synchronized Map generateWithTools(
String modelPath,
String prompt,
String cacheDir,
- ToolExecutor toolExecutor
+ String systemInstruction,
+ List toolSchemasJson,
+ StreamListener streamListener
) throws Exception {
- ensureConversation(modelPath, cacheDir);
- String responseText = extractText(conversation.sendMessage(prompt, Collections.emptyMap()));
+ ensureConversation(modelPath, cacheDir, systemInstruction, toolSchemasJson);
+ return streamToolTurn(
+ streamListener,
+ callback -> conversation.sendMessageAsync(
+ prompt, callback, Collections.emptyMap()));
+ }
- for (int round = 0; round < MAX_TOOL_ROUNDS; round++) {
- List toolCalls = parseToolCalls(responseText);
- if (toolCalls.isEmpty()) {
- return responseText;
+ /**
+ * Feeds executed tool results back into the active tool conversation and
+ * streams the next turn (same shape as {@link #generateWithTools}). Each entry
+ * is {@code {"name": String, "response": Object}}.
+ */
+ public synchronized Map continueWithToolResults(
+ List> results,
+ StreamListener streamListener
+ ) throws Exception {
+ if (conversation == null) {
+ throw new IllegalStateException(
+ "No active tool conversation. Send a tool message first.");
+ }
+ List contents = new ArrayList<>();
+ for (Map result : results) {
+ String name = String.valueOf(result.get("name"));
+ Object response = result.get("response");
+ contents.add(new Content.ToolResponse(name, response == null ? "" : response));
+ }
+ Message toolMessage = Message.Companion.tool(Contents.Companion.of(contents));
+ return streamToolTurn(
+ streamListener,
+ callback -> conversation.sendMessageAsync(
+ toolMessage, callback, Collections.emptyMap()));
+ }
+
+ /**
+ * Runs one tool-enabled turn with live token streaming. Text fragments are
+ * streamed to {@code streamListener} as they arrive; any structured tool calls
+ * the model emits are collected. Returns the structured map the Dart tool loop
+ * expects: a {@code tool_calls} turn when the model requested tools, else a
+ * {@code text} turn carrying the streamed final answer.
+ *
+ *
This is the streaming analogue of the old synchronous
+ * {@code conversation.sendMessage} tool path. It reuses the same
+ * {@link #GENERATION_TIMEOUT_SECONDS} latch/cancel guard as
+ * {@link #streamSendMessage}, so a native function-calling turn is now bounded
+ * and cancellable exactly like plain text generation.
+ */
+ private Map streamToolTurn(
+ StreamListener streamListener, AsyncSend sender) throws Exception {
+ final StringBuilder fullText = new StringBuilder();
+ final List collectedCalls = new ArrayList<>();
+ final CountDownLatch latch = new CountDownLatch(1);
+ final Throwable[] failure = new Throwable[1];
+ sender.send(new MessageCallback() {
+ @Override
+ public void onMessage(Message message) {
+ if (message == null) {
+ return;
+ }
+ // Tool calls ride Message.getToolCalls(), not the text contents, so
+ // a tool-request delta streams no user-visible tokens. Collect calls
+ // across deltas (mirroring the incremental text stream); the Dart
+ // loop clears the live buffer before the follow-up answer streams.
+ List calls = message.getToolCalls();
+ if (calls != null && !calls.isEmpty()) {
+ collectedCalls.addAll(calls);
+ }
+ String chunk = textContent(message);
+ if (chunk.isEmpty()) {
+ return;
+ }
+ fullText.append(chunk);
+ if (streamListener != null) {
+ streamListener.onToken(chunk);
+ }
}
- if (!allCallsCanExecute(toolCalls, toolExecutor)) {
- return responseText;
+
+ @Override
+ public void onDone() {
+ latch.countDown();
}
- StringBuilder feedback = new StringBuilder();
- for (ToolCall call : toolCalls) {
- String output = toolExecutor.execute(call.name, call.argument);
- feedback
- .append("- ")
- .append(call.name)
- .append(": ")
- .append(output == null ? "" : output.trim())
- .append("\n");
+ @Override
+ public void onError(Throwable throwable) {
+ failure[0] = throwable;
+ latch.countDown();
}
+ });
+ awaitGeneration(latch, failure);
+ return buildTurnResult(collectedCalls, fullText.toString().trim());
+ }
- String instruction = "System feedback from executed Android local tools:\n"
- + feedback.toString().trim()
- + "\n\nIf another tool is still needed, respond only with "
- + "[TOOL:TOOL_NAME:ARGUMENT]. Otherwise answer the user naturally "
- + "using the tool results and provided StudyOS context.";
- responseText = extractText(conversation.sendMessage(instruction, Collections.emptyMap()));
+ /** Dispatches an async send on the active conversation with our stream callback. */
+ private interface AsyncSend {
+ void send(MessageCallback callback) throws Exception;
+ }
+
+ /**
+ * Builds the structured turn map from a completed stream: a {@code tool_calls}
+ * turn when the model requested tools, else a {@code text} turn.
+ */
+ private static Map buildTurnResult(List calls, String text) {
+ Map out = new HashMap<>();
+ if (calls != null && !calls.isEmpty()) {
+ out.put("type", "tool_calls");
+ out.put("calls", callsToMaps(calls));
+ } else {
+ out.put("type", "text");
+ out.put("text", text);
}
+ return out;
+ }
- return responseText;
+ /** Serializes structured tool calls into the Dart executor's name/arguments shape. */
+ private static List> callsToMaps(List calls) {
+ List> callList = new ArrayList<>();
+ for (ToolCall call : calls) {
+ Map callMap = new HashMap<>();
+ callMap.put("name", call.getName());
+ callMap.put("arguments", argumentsToJson(call.getArguments()));
+ callList.add(callMap);
+ }
+ return callList;
+ }
+
+ /** Serializes a tool call's argument map into a JSON string for the Dart executor. */
+ private static String argumentsToJson(Map arguments) {
+ if (arguments == null || arguments.isEmpty()) {
+ return "{}";
+ }
+ try {
+ JSONObject object = new JSONObject();
+ for (Map.Entry entry : arguments.entrySet()) {
+ object.put(entry.getKey(), jsonSafe(entry.getValue()));
+ }
+ return object.toString();
+ } catch (Throwable error) {
+ Log.w(TAG, "Failed to serialize tool arguments; sending empty object.", error);
+ return "{}";
+ }
}
/**
- * Like {@link #generateWithTools}, but streams each round's tokens to
- * {@code streamListener}. When a round resolves into a tool directive, the
- * listener is reset so the bracketed call does not linger in the live UI
- * before the follow-up answer streams.
+ * Coerces a value into an {@code org.json}-safe form. Gson elements (which
+ * LiteRT-LM may hand back) are re-parsed from their JSON text so they are not
+ * double-encoded; maps/lists recurse; primitives pass through.
*/
- public synchronized String generateWithToolsStreaming(
- String modelPath,
- String prompt,
- String cacheDir,
- ToolExecutor toolExecutor,
- StreamListener streamListener
- ) throws Exception {
- ensureConversation(modelPath, cacheDir);
- String responseText = streamSendMessage(prompt, streamListener);
-
- for (int round = 0; round < MAX_TOOL_ROUNDS; round++) {
- List toolCalls = parseToolCalls(responseText);
- if (toolCalls.isEmpty()) {
- return responseText;
+ private static Object jsonSafe(Object value) {
+ if (value == null) {
+ return JSONObject.NULL;
+ }
+ if (value instanceof com.google.gson.JsonElement) {
+ try {
+ return new JSONTokener(value.toString()).nextValue();
+ } catch (Throwable ignored) {
+ return value.toString();
+ }
+ }
+ if (value instanceof Map) {
+ JSONObject object = new JSONObject();
+ Map map = (Map) value;
+ for (Map.Entry entry : map.entrySet()) {
+ try {
+ object.put(String.valueOf(entry.getKey()), jsonSafe(entry.getValue()));
+ } catch (Throwable ignored) {
+ // Skip un-encodable entries rather than failing the whole call.
+ }
}
- if (!allCallsCanExecute(toolCalls, toolExecutor)) {
- return responseText;
+ return object;
+ }
+ if (value instanceof Iterable) {
+ JSONArray array = new JSONArray();
+ for (Object item : (Iterable) value) {
+ array.put(jsonSafe(item));
}
+ return array;
+ }
+ return value;
+ }
- // This round was a tool directive, not a user-facing answer.
- if (streamListener != null) {
- streamListener.onReset();
+ // ---- Native function-calling probe -------------------------------------
+ // A throwaway spike (debug-only) that verifies whether LiteRT-LM 0.13.1's
+ // manual tool-calling path works on the shipped model: it declares one
+ // OpenApiTool, disables automaticToolCalling, and checks that the model
+ // returns a *structured* ToolCall (Message.getToolCalls()) instead of the
+ // bracketed [TOOL:] text the production path parses. It also round-trips a
+ // Content.ToolResponse to confirm the model produces a final answer. This is
+ // deliberately isolated from the cached production conversation and does not
+ // touch the [TOOL:] loop — see local-inference-architecture memory.
+
+ private static final String PROBE_TOOL_SCHEMA =
+ "{\"name\":\"read_memories\","
+ + "\"description\":\"Read the student's saved long-term memory notes.\","
+ + "\"parameters\":{\"type\":\"object\",\"properties\":{},\"required\":[]}}";
+ private static final String PROBE_SYSTEM_INSTRUCTION =
+ "You are a StudyOS test agent. When the user asks about their saved memory "
+ + "notes, call the read_memories tool to look them up.";
+ private static final String PROBE_USER_PROMPT =
+ "What have I saved in my memory notes? Use the read_memories tool to check.";
+ private static final String PROBE_TOOL_RESULT =
+ "{\"memories\":\"Probe succeeded: the student prefers morning study sessions.\"}";
+
+ /**
+ * Runs the manual native tool-calling probe against {@code modelPath} and returns a
+ * human-readable diagnostic report. Builds its own engine/conversation and closes them,
+ * so the cached production conversation (and its KV cache) is left untouched. Any cached
+ * production engine is released first to avoid holding two engines in memory at once.
+ */
+ public synchronized String probeToolCall(String modelPath, String cacheDir) throws Exception {
+ File modelFile = new File(modelPath);
+ if (!modelFile.exists()) {
+ return "Model file does not exist: " + modelPath;
+ }
+ // Free any cached production engine so the probe engine does not double RAM.
+ close();
+
+ Engine probeEngine = null;
+ Conversation probeConversation = null;
+ StringBuilder report = new StringBuilder();
+ try {
+ try {
+ probeEngine = createEngine(modelFile, cacheDir, new Backend.GPU());
+ probeEngine.initialize();
+ report.append("engine: GPU\n");
+ } catch (Throwable gpuError) {
+ closeQuietly(probeEngine);
+ probeEngine = createEngine(modelFile, cacheDir, new Backend.CPU());
+ probeEngine.initialize();
+ report.append("engine: CPU (GPU fallback)\n");
}
- StringBuilder feedback = new StringBuilder();
- for (ToolCall call : toolCalls) {
- String output = toolExecutor.execute(call.name, call.argument);
- feedback
- .append("- ")
- .append(call.name)
- .append(": ")
- .append(output == null ? "" : output.trim())
- .append("\n");
+ OpenApiTool readMemoriesTool = new OpenApiTool() {
+ @Override
+ public String getToolDescriptionJsonString() {
+ return PROBE_TOOL_SCHEMA;
+ }
+
+ @Override
+ public String execute(String argumentsJson) {
+ // Never invoked in manual mode (automaticToolCalling = false);
+ // present only to satisfy the interface.
+ return PROBE_TOOL_RESULT;
+ }
+ };
+
+ ConversationConfig config = new ConversationConfig(
+ Contents.Companion.of(PROBE_SYSTEM_INSTRUCTION),
+ Collections.emptyList(),
+ List.of(ToolKt.tool(readMemoriesTool)),
+ new SamplerConfig(
+ LOCAL_SAMPLER_TOP_K,
+ LOCAL_SAMPLER_TOP_P,
+ LOCAL_SAMPLER_TEMPERATURE,
+ LOCAL_SAMPLER_RANDOM_SEED
+ ),
+ false /* automaticToolCalling: manual — hand tool calls back to us */
+ );
+ probeConversation = probeEngine.createConversation(config);
+
+ Message first = probeConversation.sendMessage(
+ PROBE_USER_PROMPT, Collections.emptyMap());
+ List calls = first.getToolCalls();
+ int callCount = calls == null ? 0 : calls.size();
+ report.append("tool_calls_returned: ").append(callCount).append('\n');
+
+ if (callCount == 0) {
+ report.append("first_response_text: ").append(joinText(first)).append('\n');
+ report.append("VERDICT: FAIL — model did not emit a structured tool call.\n");
+ return report.toString();
}
- String instruction = "System feedback from executed Android local tools:\n"
- + feedback.toString().trim()
- + "\n\nIf another tool is still needed, respond only with "
- + "[TOOL:TOOL_NAME:ARGUMENT]. Otherwise answer the user naturally "
- + "using the tool results and provided StudyOS context.";
- responseText = streamSendMessage(instruction, streamListener);
+ ToolCall call = calls.get(0);
+ report.append("call.name: ").append(call.getName()).append('\n');
+ report.append("call.arguments: ").append(call.getArguments()).append('\n');
+
+ // Round-trip a tool result and confirm the model produces a final answer.
+ Content.ToolResponse toolResponse =
+ new Content.ToolResponse(call.getName(), PROBE_TOOL_RESULT);
+ Message toolMessage = Message.Companion.tool(Contents.Companion.of(toolResponse));
+ Message finalResp = probeConversation.sendMessage(
+ toolMessage, Collections.emptyMap());
+ report.append("final_answer: ").append(joinText(finalResp)).append('\n');
+ report.append("VERDICT: PASS — native function calling works on this model.\n");
+ return report.toString();
+ } catch (Throwable error) {
+ report.append("VERDICT: ERROR — ").append(error).append('\n');
+ return report.toString();
+ } finally {
+ if (probeConversation != null) {
+ try {
+ probeConversation.close();
+ } catch (Throwable ignored) {
+ }
+ }
+ closeQuietly(probeEngine);
}
+ }
- return responseText;
+ /** Concatenates the text parts of a message, trimmed; ignores non-text content. */
+ private static String joinText(Message message) {
+ return textContent(message).trim();
+ }
+
+ /**
+ * Concatenates the text parts of a message without trimming, so streamed
+ * chunks keep their leading/trailing spacing. Tool-call content lives on
+ * {@link Message#getToolCalls()} rather than here, so a pure tool-request delta
+ * yields the empty string.
+ */
+ private static String textContent(Message message) {
+ if (message == null
+ || message.getContents() == null
+ || message.getContents().getContents() == null) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (Content content : message.getContents().getContents()) {
+ if (content instanceof Content.Text) {
+ sb.append(((Content.Text) content).getText());
+ }
+ }
+ return sb.toString();
}
private String streamSendMessage(String prompt, StreamListener streamListener) throws Exception {
@@ -197,29 +485,43 @@ public void onError(Throwable throwable) {
}
}, Collections.emptyMap());
- latch.await();
+ awaitGeneration(latch, failure);
+ return full.toString().trim();
+ }
+
+ /**
+ * Blocks until a streamed generation settles, enforcing the shared
+ * {@link #GENERATION_TIMEOUT_SECONDS} bound. On timeout the in-flight decode is
+ * cancelled and a {@link TimeoutException} is thrown instead of blocking the
+ * executor thread forever; a callback failure is rethrown.
+ */
+ private void awaitGeneration(CountDownLatch latch, Throwable[] failure) throws Exception {
+ boolean completed = latch.await(GENERATION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ if (!completed) {
+ cancel();
+ throw new TimeoutException(
+ "Local generation timed out after " + GENERATION_TIMEOUT_SECONDS + "s.");
+ }
if (failure[0] != null) {
if (failure[0] instanceof Exception) {
throw (Exception) failure[0];
}
throw new RuntimeException(failure[0]);
}
- return full.toString().trim();
- }
-
- private boolean allCallsCanExecute(List toolCalls, ToolExecutor toolExecutor) {
- for (ToolCall call : toolCalls) {
- if (!toolExecutor.canExecute(call.name)) {
- return false;
- }
- }
- return true;
}
- private void ensureConversation(String modelPath, String cacheDir) throws Exception {
+ private void ensureConversation(
+ String modelPath,
+ String cacheDir,
+ String systemInstruction,
+ List toolSchemasJson
+ ) throws Exception {
+ String toolsSignature = toolsSignature(toolSchemasJson);
if (conversation != null
&& modelPath.equals(activeModelPath)
- && backendPreference.equals(activeBackendPreference)) {
+ && backendPreference.equals(activeBackendPreference)
+ && Objects.equals(systemInstruction, activeSystemInstruction)
+ && Objects.equals(toolsSignature, activeToolsSignature)) {
return;
}
close();
@@ -230,19 +532,63 @@ private void ensureConversation(String modelPath, String cacheDir) throws Except
}
engine = initializeEngine(modelFile, cacheDir);
+ String instruction = (systemInstruction == null || systemInstruction.isBlank())
+ ? DEFAULT_SYSTEM_INSTRUCTION
+ : systemInstruction;
+ List toolProviders = new ArrayList<>();
+ if (toolSchemasJson != null) {
+ for (String schema : toolSchemasJson) {
+ if (schema != null && !schema.isBlank()) {
+ toolProviders.add(ToolKt.tool(openApiToolFor(schema)));
+ }
+ }
+ }
+ // automaticToolCalling = false: even with tools declared, hand every tool
+ // call back to the Dart loop rather than executing natively. Harmless when
+ // toolProviders is empty (the plain text-generation path).
ConversationConfig config = new ConversationConfig(
- Contents.Companion.of("You are StudyOS Agent. Answer from the provided context."),
- List.of(Message.Companion.user("System ready.")),
- List.of(),
+ Contents.Companion.of(instruction),
+ Collections.emptyList(),
+ toolProviders,
new SamplerConfig(
- TOOL_SAMPLER_TOP_K,
- TOOL_SAMPLER_TOP_P,
- TOOL_SAMPLER_TEMPERATURE,
- TOOL_SAMPLER_RANDOM_SEED
- )
+ LOCAL_SAMPLER_TOP_K,
+ LOCAL_SAMPLER_TOP_P,
+ LOCAL_SAMPLER_TEMPERATURE,
+ LOCAL_SAMPLER_RANDOM_SEED
+ ),
+ false
);
conversation = engine.createConversation(config);
activeModelPath = modelFile.getAbsolutePath();
+ activeSystemInstruction = systemInstruction;
+ activeToolsSignature = toolsSignature;
+ }
+
+ /** A stable fingerprint of the declared tool schemas, for conversation reuse. */
+ private static String toolsSignature(List toolSchemasJson) {
+ if (toolSchemasJson == null || toolSchemasJson.isEmpty()) {
+ return "";
+ }
+ return String.join("", toolSchemasJson);
+ }
+
+ /**
+ * Wraps one OpenAPI function declaration as an {@link OpenApiTool}. In manual
+ * mode {@link OpenApiTool#execute} is never invoked (the Dart layer executes
+ * tools), so it only needs to surface the declaration JSON.
+ */
+ private static OpenApiTool openApiToolFor(final String schemaJson) {
+ return new OpenApiTool() {
+ @Override
+ public String getToolDescriptionJsonString() {
+ return schemaJson;
+ }
+
+ @Override
+ public String execute(String argumentsJson) {
+ return "";
+ }
+ };
}
/**
@@ -306,10 +652,6 @@ private static void closeQuietly(Engine engine) {
}
}
- private String extractText(Message message) {
- return extractChunk(message).trim();
- }
-
/** Joins a message's contents without trimming, preserving token spacing. */
private String extractChunk(Message message) {
if (message == null || message.getContents() == null || message.getContents().getContents() == null) {
@@ -321,32 +663,6 @@ private String extractChunk(Message message) {
.collect(Collectors.joining());
}
- private List parseToolCalls(String text) {
- if (text == null || text.isBlank()) {
- return Collections.emptyList();
- }
- Matcher matcher = TOOL_CALL_PATTERN.matcher(text);
- List toolCalls = new ArrayList<>();
- while (matcher.find()) {
- String name = matcher.group(1) == null ? "" : matcher.group(1).trim();
- String argument = matcher.group(2) == null ? "" : matcher.group(2).trim();
- if (!name.isEmpty()) {
- toolCalls.add(new ToolCall(name, argument));
- }
- }
- return toolCalls;
- }
-
- private static final class ToolCall {
- private final String name;
- private final String argument;
-
- private ToolCall(String name, String argument) {
- this.name = name;
- this.argument = argument;
- }
- }
-
/**
* Best-effort cancel of an in-flight generation. Safe to call from another
* thread than the one blocked in {@link #streamSendMessage}; the pending
@@ -375,7 +691,9 @@ public synchronized void close() {
conversation = null;
engine = null;
activeModelPath = null;
+ activeSystemInstruction = null;
activeBackend = null;
activeBackendPreference = null;
+ activeToolsSignature = null;
}
}
diff --git a/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/AndroidLiteRtToolExecutor.kt b/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/AndroidLiteRtToolExecutor.kt
deleted file mode 100644
index 0ad1592..0000000
--- a/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/AndroidLiteRtToolExecutor.kt
+++ /dev/null
@@ -1,108 +0,0 @@
-package com.studyos.studyos_agent
-
-import android.content.Context
-import com.example.studyOS.offline.Tools
-import java.util.Locale
-import java.util.UUID
-
-class AndroidLiteRtToolExecutor(
- context: Context,
- private val emitToolTrace: (
- toolName: String,
- status: String,
- summary: String,
- callId: String,
- ) -> Unit,
-) {
- private val appContext = context.applicationContext
- private val tools: Tools by lazy { Tools(appContext) }
-
- fun canExecute(toolName: String): Boolean {
- return normalizeToolName(toolName) in androidToolNames
- }
-
- fun execute(
- toolName: String,
- argument: String,
- systemPrompt: String,
- memory: String,
- ): String {
- val normalized = normalizeToolName(toolName)
- val callId = "android-litert-${normalized.lowercase(Locale.US)}-${UUID.randomUUID()}"
- emitToolTrace(
- normalized,
- "running",
- "Running Android LiteRT local tool.",
- callId,
- )
-
- return try {
- val output = when (normalized) {
- "GET_STUDY_CONTEXT" -> systemPrompt.ifBlank {
- "No StudyOS context was provided."
- }
- "READ_MEMORIES" -> memory.trim().ifBlank {
- "No saved StudyOS memories were provided."
- }
- "GET_SCHEDULE" -> scheduleContext(systemPrompt)
- "GET_STATUS" -> tools.getDeviceStatus()
- "LIGHT_CONTROL" -> tools.toggleFlashlight(
- argument.uppercase(Locale.US).contains("ON") ||
- argument.uppercase(Locale.US).contains("AN") ||
- argument.equals("true", ignoreCase = true),
- )
- "OPEN_APP" -> if (argument.isBlank()) {
- "App name was not provided."
- } else {
- tools.openApp(argument)
- }
- "SEARCH_YOUTUBE" -> {
- val query = argument.ifBlank { "StudyOS" }
- tools.searchYoutube(query)
- "Opened YouTube search for '$query'."
- }
- else -> "Tool is not available: $toolName"
- }
- emitToolTrace(normalized, "done", "Returned ${output.length} chars.", callId)
- output
- } catch (error: Throwable) {
- val message = "Android LiteRT tool failed: ${error.message}"
- emitToolTrace(normalized, "failed", message, callId)
- message
- }
- }
-
- private fun normalizeToolName(toolName: String): String {
- return when (toolName.trim().lowercase(Locale.US)) {
- "get_study_context" -> "GET_STUDY_CONTEXT"
- "read_memories" -> "READ_MEMORIES"
- "get_schedule" -> "GET_SCHEDULE"
- "get_status" -> "GET_STATUS"
- "light_control" -> "LIGHT_CONTROL"
- "open_app" -> "OPEN_APP"
- "search_youtube" -> "SEARCH_YOUTUBE"
- else -> toolName.trim().uppercase(Locale.US)
- }
- }
-
- private fun scheduleContext(systemPrompt: String): String {
- val marker = "Cached timetable summary:"
- val index = systemPrompt.indexOf(marker)
- if (index < 0) {
- return "No cached timetable summary was provided."
- }
- return systemPrompt.substring(index).trim()
- }
-
- private companion object {
- val androidToolNames = setOf(
- "GET_STUDY_CONTEXT",
- "READ_MEMORIES",
- "GET_SCHEDULE",
- "GET_STATUS",
- "LIGHT_CONTROL",
- "OPEN_APP",
- "SEARCH_YOUTUBE",
- )
- }
-}
diff --git a/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/AndroidLocalPromptClient.kt b/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/AndroidLocalPromptClient.kt
index 999671a..b28a392 100644
--- a/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/AndroidLocalPromptClient.kt
+++ b/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/AndroidLocalPromptClient.kt
@@ -22,47 +22,48 @@ class AndroidLocalPromptClient(context: Context) {
liteRtClient.cancel()
}
+ /**
+ * Releases the LiteRT engine and its KV cache. Posted to the same single
+ * worker so it serializes behind any in-flight generation rather than
+ * blocking the caller (e.g. the main thread during onTrimMemory). The next
+ * generate() call transparently rebuilds the engine.
+ */
+ fun close() {
+ executor.execute { liteRtClient.close() }
+ }
+
+ /**
+ * Generates a reply for [prompt]. Tool routing is owned entirely by the Dart
+ * layer; this only produces text and streams tokens through [onDelta].
+ *
+ * [systemInstruction] is the stable system prompt: on the LiteRT path it is
+ * installed once as the conversation's system instruction (reused across
+ * turns); the stateless Gemini Nano path folds it into the prompt.
+ */
fun generate(
prompt: String,
+ systemInstruction: String,
modelId: String,
modelPath: String,
backend: String,
- canExecuteTool: (String) -> Boolean,
- onToolRequest: (String, String) -> String,
onDelta: (String) -> Unit,
- onReset: () -> Unit,
onSuccess: (String) -> Unit,
onError: (String) -> Unit,
) {
- // Locals so the anonymous StreamListener can call the lambdas without
- // shadowing its own onReset() override.
val deltaSink = onDelta
- val resetSink = onReset
executor.execute {
try {
if (modelPath.isNotBlank()) {
liteRtClient.setBackendPreference(backend)
- val response = liteRtClient.generateWithToolsStreaming(
+ val response = liteRtClient.generateStreaming(
modelPath,
prompt,
appContext.cacheDir.absolutePath,
- object : LiteRtLocalPromptClient.ToolExecutor {
- override fun canExecute(toolName: String): Boolean {
- return canExecuteTool(toolName)
- }
-
- override fun execute(toolName: String, argument: String): String {
- return onToolRequest(toolName, argument)
- }
- },
+ systemInstruction,
object : LiteRtLocalPromptClient.StreamListener {
override fun onToken(token: String) {
deltaSink(token)
}
-
- override fun onReset() {
- resetSink()
- }
},
)
if (response.isBlank()) {
@@ -73,16 +74,23 @@ class AndroidLocalPromptClient(context: Context) {
return@execute
}
+ // Gemini Nano through ML Kit is stateless per call, so fold the
+ // system instruction into the one-shot prompt.
+ val nanoPrompt = if (systemInstruction.isBlank()) {
+ prompt
+ } else {
+ "$systemInstruction\n\n$prompt"
+ }
val model = AndroidAiCoreModelCatalog.clientFor(modelId)
when (val status = model.checkStatus().get(2, TimeUnit.SECONDS)) {
FeatureStatus.AVAILABLE -> {
val future = model.generateContent(
- prompt,
+ nanoPrompt,
StreamingCallback { text -> deltaSink(text) },
)
activeNanoFuture = future
val response = try {
- future.get()
+ future.get(NANO_GENERATION_TIMEOUT_SECONDS, TimeUnit.SECONDS)
} finally {
activeNanoFuture = null
}
@@ -119,6 +127,104 @@ class AndroidLocalPromptClient(context: Context) {
}
}
+ /**
+ * Native function-calling first turn (flag-gated). Only the LiteRT-LM path
+ * supports structured tools; Gemini Nano via ML Kit does not, so a blank
+ * [modelPath] is surfaced as an error. Text of a plain-answer turn streams
+ * through [onDelta] as it is generated; the structured turn map
+ * ({@code tool_calls} or {@code text}) is returned via [onResult].
+ */
+ fun generateWithTools(
+ prompt: String,
+ systemInstruction: String,
+ modelId: String,
+ modelPath: String,
+ backend: String,
+ toolSchemas: List,
+ onDelta: (String) -> Unit,
+ onResult: (Map) -> Unit,
+ onError: (String) -> Unit,
+ ) {
+ executor.execute {
+ try {
+ if (modelPath.isBlank()) {
+ onError(
+ "Native function calling requires a downloaded LiteRT-LM model.",
+ )
+ return@execute
+ }
+ liteRtClient.setBackendPreference(backend)
+ onResult(
+ liteRtClient.generateWithTools(
+ modelPath,
+ prompt,
+ appContext.cacheDir.absolutePath,
+ systemInstruction,
+ toolSchemas,
+ object : LiteRtLocalPromptClient.StreamListener {
+ override fun onToken(token: String) {
+ onDelta(token)
+ }
+ },
+ ),
+ )
+ } catch (error: Throwable) {
+ onError("LiteRT-LM function calling failed: ${error.message}")
+ }
+ }
+ }
+
+ /**
+ * Feeds executed tool results back into the active native tool conversation,
+ * streaming the next turn's text through [onDelta] and returning the
+ * structured turn map via [onResult].
+ */
+ fun continueWithToolResults(
+ results: List>,
+ onDelta: (String) -> Unit,
+ onResult: (Map) -> Unit,
+ onError: (String) -> Unit,
+ ) {
+ executor.execute {
+ try {
+ @Suppress("UNCHECKED_CAST")
+ onResult(
+ liteRtClient.continueWithToolResults(
+ results as List>,
+ object : LiteRtLocalPromptClient.StreamListener {
+ override fun onToken(token: String) {
+ onDelta(token)
+ }
+ },
+ ),
+ )
+ } catch (error: Throwable) {
+ onError("LiteRT-LM tool result handling failed: ${error.message}")
+ }
+ }
+ }
+
+ /**
+ * Debug spike: runs the LiteRT-LM native (manual) tool-calling probe against
+ * [modelPath] and returns a diagnostic report. Isolated from the production
+ * generate() path; only meaningful for a downloaded .litertlm model.
+ */
+ fun probeToolCall(
+ modelPath: String,
+ onSuccess: (String) -> Unit,
+ onError: (String) -> Unit,
+ ) {
+ executor.execute {
+ try {
+ onSuccess(
+ liteRtClient.probeToolCall(modelPath, appContext.cacheDir.absolutePath),
+ )
+ } catch (error: Throwable) {
+ onError("Native tool-calling probe failed: ${error.message}")
+ }
+ }
+ }
+
fun capabilities(): Map {
return mapOf(
"androidLocalModelProvider" to
@@ -128,10 +234,14 @@ class AndroidLocalPromptClient(context: Context) {
"AICore does not expose a general installed-model list. " +
"Apps initialize a desired Gemini Nano configuration and check its status.",
"androidLocalToolCalling" to
- "ML Kit Prompt API does not expose native function calling. " +
- "Downloaded LiteRT-LM models can use StudyOS bracketed " +
- "[TOOL:NAME:ARG] calls executed by the Android bridge.",
+ "Tool routing is handled by the StudyOS Dart layer via bracketed " +
+ "[TOOL:NAME:ARG] directives parsed from the model's output; the " +
+ "native model only generates text.",
"androidLocalModelContext" to appContext.packageName,
)
}
+
+ private companion object {
+ const val NANO_GENERATION_TIMEOUT_SECONDS = 120L
+ }
}
diff --git a/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/MainActivity.kt b/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/MainActivity.kt
index 637118d..56e2d2d 100644
--- a/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/MainActivity.kt
+++ b/flutter_app/android/app/src/main/kotlin/com/studyos/studyos_agent/MainActivity.kt
@@ -1,6 +1,7 @@
package com.studyos.studyos_agent
import android.Manifest
+import android.content.ComponentCallbacks2
import android.content.Intent
import android.content.pm.PackageManager
import android.os.Bundle
@@ -29,12 +30,17 @@ class MainActivity : FlutterActivity() {
private var nativeInitialized = false
private var localPromptClient: AndroidLocalPromptClient? = null
private var localModelStore: AndroidLocalModelStore? = null
- private var liteRtToolExecutor: AndroidLiteRtToolExecutor? = null
private var nativeToolExecutor: AndroidNativeToolExecutor? = null
private var pdfPreview: AndroidPdfPreview? = null
private var pendingCalendarOperation: (() -> Unit)? = null
private lateinit var intentBridge: AndroidIntentBridge
+ // Idle-unload timer: releases the on-device model after a stretch of no
+ // activity so it does not hold RAM indefinitely on mid-range devices.
+ private val idleUnloadHandler = Handler(Looper.getMainLooper())
+ private val idleUnloadRunnable = Runnable { localPromptClient?.close() }
+ private val idleUnloadDelayMs = 5 * 60 * 1000L
+
override fun onCreate(savedInstanceState: Bundle?) {
intentBridge = AndroidIntentBridge(applicationContext)
intentBridge.captureIntent(intent)
@@ -47,11 +53,37 @@ class MainActivity : FlutterActivity() {
intentBridge.captureIntent(intent)
}
+ override fun onStop() {
+ // Backgrounded: release the on-device model so the OS is less likely to
+ // reclaim the app under memory pressure. The next message rebuilds it.
+ cancelIdleUnload()
+ localPromptClient?.close()
+ super.onStop()
+ }
+
+ override fun onTrimMemory(level: Int) {
+ super.onTrimMemory(level)
+ if (level >= ComponentCallbacks2.TRIM_MEMORY_RUNNING_MODERATE) {
+ localPromptClient?.close()
+ }
+ }
+
override fun onDestroy() {
+ cancelIdleUnload()
+ localPromptClient?.close()
aiCoreModelExecutor.shutdownNow()
super.onDestroy()
}
+ private fun scheduleIdleUnload() {
+ idleUnloadHandler.removeCallbacks(idleUnloadRunnable)
+ idleUnloadHandler.postDelayed(idleUnloadRunnable, idleUnloadDelayMs)
+ }
+
+ private fun cancelIdleUnload() {
+ idleUnloadHandler.removeCallbacks(idleUnloadRunnable)
+ }
+
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine)
@@ -108,8 +140,7 @@ class MainActivity : FlutterActivity() {
}
sendMessageToNativeLayer(
text = text,
- systemPrompt = call.argument("systemPrompt").orEmpty(),
- memory = call.argument("memory").orEmpty(),
+ systemInstruction = call.argument("systemInstruction").orEmpty(),
localModelId = call.argument("localModelId").orEmpty(),
localModelPath = call.argument("localModelPath").orEmpty(),
localBackend = call.argument("localBackend").orEmpty(),
@@ -120,6 +151,82 @@ class MainActivity : FlutterActivity() {
localPromptClient?.cancel()
result.success(null)
}
+ "sendMessageWithTools" -> {
+ val text = call.argument("text")?.trim().orEmpty()
+ if (text.isBlank()) {
+ result.error("empty_message", "Message text must not be empty.", null)
+ return
+ }
+ val toolSchemas = call.argument>("toolSchemas")
+ ?.map { it.toString() }
+ ?: emptyList()
+ if (!nativeInitialized) initializeNativeLayer()
+ scheduleIdleUnload()
+ localPromptClient().generateWithTools(
+ prompt = text,
+ systemInstruction = call.argument("systemInstruction").orEmpty(),
+ modelId = call.argument("localModelId").orEmpty(),
+ modelPath = call.argument("localModelPath").orEmpty(),
+ backend = call.argument("localBackend").orEmpty(),
+ toolSchemas = toolSchemas,
+ onDelta = { token -> emitAssistantDelta(token) },
+ onResult = { turn ->
+ Handler(Looper.getMainLooper()).post { result.success(turn) }
+ scheduleIdleUnload()
+ },
+ onError = { message ->
+ emitStatus(message)
+ Handler(Looper.getMainLooper()).post {
+ result.error("android_local_model_unavailable", message, null)
+ }
+ scheduleIdleUnload()
+ },
+ )
+ }
+ "sendToolResults" -> {
+ val results = call.argument>("results")
+ ?.filterIsInstance>()
+ ?.map { entry ->
+ entry.entries.associate { (k, v) -> k.toString() to v }
+ }
+ ?: emptyList()
+ localPromptClient().continueWithToolResults(
+ results = results,
+ onDelta = { token -> emitAssistantDelta(token) },
+ onResult = { turn ->
+ Handler(Looper.getMainLooper()).post { result.success(turn) }
+ },
+ onError = { message ->
+ emitStatus(message)
+ Handler(Looper.getMainLooper()).post {
+ result.error("android_local_model_unavailable", message, null)
+ }
+ },
+ )
+ }
+ "probeNativeToolCall" -> {
+ val modelPath = call.argument("localModelPath")?.trim().orEmpty()
+ if (modelPath.isBlank()) {
+ result.error(
+ "empty_model_path",
+ "A downloaded LiteRT-LM model path is required for the probe.",
+ null,
+ )
+ return
+ }
+ if (!nativeInitialized) initializeNativeLayer()
+ localPromptClient().probeToolCall(
+ modelPath = modelPath,
+ onSuccess = { report ->
+ Handler(Looper.getMainLooper()).post { result.success(report) }
+ },
+ onError = { message ->
+ Handler(Looper.getMainLooper()).post {
+ result.error("native_tool_probe_failed", message, null)
+ }
+ },
+ )
+ }
else -> result.notImplemented()
}
}
@@ -160,8 +267,7 @@ class MainActivity : FlutterActivity() {
private fun sendMessageToNativeLayer(
text: String,
- systemPrompt: String,
- memory: String,
+ systemInstruction: String,
localModelId: String,
localModelPath: String,
localBackend: String,
@@ -170,34 +276,17 @@ class MainActivity : FlutterActivity() {
if (!nativeInitialized) {
initializeNativeLayer()
}
+ scheduleIdleUnload()
try {
- val prompt = localPrompt(
- systemPrompt = systemPrompt,
- userText = text,
- supportsLiteRtTools = localModelPath.isNotBlank(),
- )
localPromptClient().generate(
- prompt = prompt,
+ prompt = text,
+ systemInstruction = systemInstruction,
modelId = localModelId,
modelPath = localModelPath,
backend = localBackend,
- canExecuteTool = { toolName ->
- liteRtToolExecutor().canExecute(toolName)
- },
- onToolRequest = { toolName, argument ->
- liteRtToolExecutor().execute(
- toolName = toolName,
- argument = argument,
- systemPrompt = systemPrompt,
- memory = memory,
- )
- },
onDelta = { token ->
- emitAssistantDelta(token, reset = false)
- },
- onReset = {
- emitAssistantDelta("", reset = true)
+ emitAssistantDelta(token)
},
onSuccess = { response ->
emitStatus(
@@ -210,6 +299,7 @@ class MainActivity : FlutterActivity() {
Handler(Looper.getMainLooper()).post {
result.success(response)
}
+ scheduleIdleUnload()
},
onError = { message ->
emitStatus(message)
@@ -220,6 +310,7 @@ class MainActivity : FlutterActivity() {
null,
)
}
+ scheduleIdleUnload()
},
)
} catch (error: Throwable) {
@@ -229,81 +320,6 @@ class MainActivity : FlutterActivity() {
}
}
- private fun localPrompt(
- systemPrompt: String,
- userText: String,
- supportsLiteRtTools: Boolean,
- ): String {
- return buildString {
- appendLine(systemPrompt.ifBlank { "You are StudyOS Agent." })
- appendLine()
- if (supportsLiteRtTools) {
- appendLine("Android LiteRT local tool protocol:")
- appendLine(
- "Use tools only when they are helpful. For normal questions, " +
- "answer directly from the provided context.",
- )
- appendLine(
- "To call tools, respond only with one or more directives " +
- "in this exact form: [TOOL:TOOL_NAME:ARGUMENT].",
- )
- appendLine(
- "After the app returns tool results, answer naturally. " +
- "Do not show raw tool directives to the user in the final answer.",
- )
- appendLine("Only call tools from this list; do not invent tool names.")
- appendLine()
- appendLine("Available Android LiteRT tools:")
- appendLine(
- "- GET_STUDY_CONTEXT, no argument: read the current StudyOS " +
- "profile, timetable summary, memory, and device context.",
- )
- appendLine(
- " Example: [TOOL:GET_STUDY_CONTEXT:]",
- )
- appendLine(
- "- READ_MEMORIES, no argument: read the provided local " +
- "StudyOS long-term memories.",
- )
- appendLine(" Example: [TOOL:READ_MEMORIES:]")
- appendLine(
- "- GET_SCHEDULE, no argument: read cached timetable context " +
- "when it is present in the StudyOS prompt.",
- )
- appendLine(" Example: [TOOL:GET_SCHEDULE:]")
- appendLine(
- "- GET_STATUS, no argument: read Android device status such " +
- "as volume, Wi-Fi, location, and airplane mode.",
- )
- appendLine(" Example: [TOOL:GET_STATUS:]")
- appendLine(
- "- LIGHT_CONTROL, argument ON or OFF: turn the flashlight on " +
- "or off.",
- )
- appendLine(" Example: [TOOL:LIGHT_CONTROL:ON]")
- appendLine(
- "- OPEN_APP, argument app name: open an installed Android app " +
- "by its display name.",
- )
- appendLine(" Example: [TOOL:OPEN_APP:Camera]")
- appendLine(
- "- SEARCH_YOUTUBE, argument search query: open YouTube search " +
- "results for the query.",
- )
- appendLine(" Example: [TOOL:SEARCH_YOUTUBE:study techniques]")
- } else {
- appendLine(
- "Runtime note: Android Gemini Nano through ML Kit Prompt API " +
- "does not expose tool calling in this app. Answer from " +
- "provided context and say what is missing.",
- )
- }
- appendLine()
- appendLine("User request:")
- appendLine(userText)
- }.trim()
- }
-
private fun localPromptClient(): AndroidLocalPromptClient {
val existing = localPromptClient
if (existing != null) return existing
@@ -380,14 +396,6 @@ class MainActivity : FlutterActivity() {
}
}
- private fun liteRtToolExecutor(): AndroidLiteRtToolExecutor {
- val existing = liteRtToolExecutor
- if (existing != null) return existing
- return AndroidLiteRtToolExecutor(applicationContext, ::emitToolTrace).also {
- liteRtToolExecutor = it
- }
- }
-
private fun nativeToolExecutor(): AndroidNativeToolExecutor {
val existing = nativeToolExecutor
if (existing != null) return existing
@@ -691,37 +699,11 @@ class MainActivity : FlutterActivity() {
}
}
- private fun emitToolTrace(
- toolName: String,
- status: String,
- summary: String,
- callId: String,
- ) {
- val payload = mapOf(
- "type" to "toolTrace",
- "message" to summary,
- "trace" to mapOf(
- "toolName" to toolName,
- "status" to status,
- "summary" to summary,
- "callId" to callId,
- ),
- "timestamp" to SimpleDateFormat(
- "yyyy-MM-dd'T'HH:mm:ss",
- Locale.US
- ).format(Date()),
- )
-
- Handler(Looper.getMainLooper()).post {
- eventSink?.success(payload)
- }
- }
-
- private fun emitAssistantDelta(text: String, reset: Boolean) {
+ private fun emitAssistantDelta(text: String) {
val payload = mapOf(
"type" to "assistantDelta",
"message" to text,
- "reset" to reset,
+ "reset" to false,
"timestamp" to SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss",
Locale.US
diff --git a/flutter_app/lib/src/agent_config_store.dart b/flutter_app/lib/src/agent_config_store.dart
index 76b3b0b..aff7e60 100644
--- a/flutter_app/lib/src/agent_config_store.dart
+++ b/flutter_app/lib/src/agent_config_store.dart
@@ -20,6 +20,8 @@ class AgentConfigStore {
static const String _localModelIdKey = 'studyos.agent.localModelId.v1';
static const String _localModelPathKey = 'studyos.agent.localModelPath.v1';
static const String _localBackendKey = 'studyos.agent.localBackend.v1';
+ static const String _localToolProtocolKey =
+ 'studyos.agent.localToolProtocol.v1';
static const String _apiKeyKey = 'studyos.agent.cloudApiKey.v1';
final SharedPreferencesAsync? _preferences;
@@ -36,6 +38,7 @@ class AgentConfigStore {
final localModelId = await _prefs.getString(_localModelIdKey);
final localModelPath = await _prefs.getString(_localModelPathKey);
final localBackend = await _prefs.getString(_localBackendKey);
+ final localToolProtocol = await _prefs.getString(_localToolProtocolKey);
final apiKey = await _secure.read(key: _apiKeyKey);
if (_hasNoSavedConfig(
providerName: providerName,
@@ -63,6 +66,7 @@ class AgentConfigStore {
localModelId: localModelId ?? const AgentConfig.defaults().localModelId,
localModelPath: localModelPath ?? '',
localBackend: localBackendFromName(localBackend),
+ localToolProtocol: localToolProtocolFromName(localToolProtocol),
);
}
@@ -93,6 +97,10 @@ class AgentConfigStore {
await _prefs.setString(_localModelIdKey, config.localModelId.trim());
await _prefs.setString(_localModelPathKey, config.localModelPath.trim());
await _prefs.setString(_localBackendKey, config.localBackend.name);
+ await _prefs.setString(
+ _localToolProtocolKey,
+ config.localToolProtocol.name,
+ );
}
bool _hasNoSavedConfig({
diff --git a/flutter_app/lib/src/agent_llm_provider.dart b/flutter_app/lib/src/agent_llm_provider.dart
index c8a8ad6..bef7bdd 100644
--- a/flutter_app/lib/src/agent_llm_provider.dart
+++ b/flutter_app/lib/src/agent_llm_provider.dart
@@ -132,22 +132,21 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
String get displayName => 'Local native model';
@override
- Future send(AgentLlmRequest request) async {
- final nativeTools = NativeToolRouter(_bridge);
- final supportedNativeToolNames = await nativeTools.supportedToolNames();
- final systemPrompt = _localSystemPrompt(
- request.context.systemPrompt(),
- supportedNativeToolNames,
- );
- var response = await _bridge.sendMessage(
- request.userText,
- systemPrompt: systemPrompt,
- memory: request.memoryText,
- localModelId: request.config.localModelId,
- localModelPath: request.config.localModelPath,
- localBackend: request.config.localBackend.name,
- );
- final toolContext = StudyOsToolContext(
+ Future send(AgentLlmRequest request) {
+ // Behind a settings flag: the proven bracket `[TOOL:]` text protocol, or
+ // LiteRT-LM's structured native function calling. Both drive the tool loop
+ // from Dart (tools execute here, not natively).
+ return request.config.localToolProtocol ==
+ LocalToolProtocol.nativeFunctionCalling
+ ? _sendNativeFunctionCalling(request)
+ : _sendBracket(request);
+ }
+
+ StudyOsToolContext _toolContextFor(
+ AgentLlmRequest request,
+ NativeToolRouter nativeTools,
+ ) {
+ return StudyOsToolContext(
promptContext: request.context,
appendMemory: request.appendMemory,
readMemory: request.readMemory,
@@ -159,11 +158,37 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
publicStudyTools: request.publicStudyTools,
privateStudyTools: request.privateStudyTools,
);
+ }
+
+ Future _sendBracket(AgentLlmRequest request) async {
+ final nativeTools = NativeToolRouter(_bridge);
+ final supportedNativeToolNames = await nativeTools.supportedToolNames();
+ // The stable system prompt + tool protocol is installed once as the native
+ // conversation's system instruction; only the volatile per-turn context and
+ // the user text travel on the message itself.
+ final systemInstruction = _localSystemPrompt(
+ request.context.stableSystemPrompt(),
+ supportedNativeToolNames,
+ );
+ var response = await _bridge.sendMessage(
+ _composeFirstTurn(request.context.ephemeralContext(), request.userText),
+ systemInstruction: systemInstruction,
+ localModelId: request.config.localModelId,
+ localModelPath: request.config.localModelPath,
+ localBackend: request.config.localBackend.name,
+ );
+ final toolContext = _toolContextFor(request, nativeTools);
for (var round = 0; round < _maxToolRounds; round += 1) {
final calls = _toolCalls(response);
if (calls.isEmpty) return response;
+ // This streamed turn resolved into tool directives, not a user-facing
+ // answer. Clear the live buffer so the bracketed calls don't linger on
+ // screen before the follow-up answer streams. Mirrors CloudLlmProvider;
+ // this replaces the tool-round reset the native loop used to emit.
+ request.onDelta?.call(const AgentStreamDelta(reset: true));
+
final feedback = [];
for (final call in calls) {
final callId =
@@ -193,8 +218,7 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
response = await _bridge.sendMessage(
_localToolFeedbackPrompt(feedback),
- systemPrompt: systemPrompt,
- memory: request.memoryText,
+ systemInstruction: systemInstruction,
localModelId: request.config.localModelId,
localModelPath: request.config.localModelPath,
localBackend: request.config.localBackend.name,
@@ -211,6 +235,108 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
return response;
}
+ /// Native function-calling path (experimental, flag-gated). The model returns
+ /// structured tool calls instead of `[TOOL:]` text; the schema replaces the
+ /// prose protocol, so only the stable system prompt is installed. The tool
+ /// loop, execution, and tracing are identical to [_sendBracket] — only the
+ /// transport differs. Like the bracket path, a plain-answer turn streams its
+ /// text live via the native `assistantDelta` events (out of band from the
+ /// structured turn map returned here); [request.onDelta] carries the
+ /// between-round reset so streamed tokens never linger before a tool follow-up.
+ Future _sendNativeFunctionCalling(AgentLlmRequest request) async {
+ final nativeTools = NativeToolRouter(_bridge);
+ final supportedNativeToolNames = await nativeTools.supportedToolNames();
+ final toolSchemas = studyOsToolsForNativeSupport(
+ supportedNativeToolNames,
+ ).map((tool) => tool.toOpenApiToolJson()).toList();
+ final toolContext = _toolContextFor(request, nativeTools);
+
+ var turn = await _bridge.sendMessageWithTools(
+ text: _composeFirstTurn(
+ request.context.ephemeralContext(),
+ request.userText,
+ ),
+ systemInstruction: request.context.stableSystemPrompt(),
+ toolSchemas: toolSchemas,
+ localModelId: request.config.localModelId,
+ localModelPath: request.config.localModelPath,
+ localBackend: request.config.localBackend.name,
+ );
+
+ for (var round = 0; round < _maxToolRounds; round += 1) {
+ final calls = _nativeToolCalls(turn);
+ if (calls.isEmpty) return _turnText(turn);
+
+ // The turn resolved into tool calls, not an answer; clear the live buffer
+ // so nothing lingers before the follow-up answer. Mirrors _sendBracket.
+ request.onDelta?.call(const AgentStreamDelta(reset: true));
+
+ final results = >[];
+ for (final call in calls) {
+ final callId =
+ 'local-${call.name}-${DateTime.now().microsecondsSinceEpoch}';
+ request.onToolTrace(_traceForCall(call, 'running', callId: callId));
+ final String output;
+ try {
+ output = await _toolExecutor.execute(
+ call.name,
+ call.arguments,
+ toolContext,
+ );
+ } on Object catch (error) {
+ final failedOutput = _toolFailureOutput(error);
+ request.onToolTrace(
+ _traceForCall(call, 'failed', callId: callId, output: failedOutput),
+ );
+ results.add({
+ 'name': call.name,
+ 'response': failedOutput,
+ });
+ continue;
+ }
+ request.onToolTrace(
+ _traceForCall(call, 'done', callId: callId, output: output),
+ );
+ results.add({'name': call.name, 'response': output});
+ }
+
+ turn = await _bridge.sendToolResults(results);
+ }
+
+ // Still requesting tools after the round budget: a stuck loop, not an
+ // answer. Mirror _sendBracket and surface it as an error.
+ if (_nativeToolCalls(turn).isNotEmpty) {
+ throw const AgentException(
+ 'Local tool loop exceeded the maximum number of tool rounds.',
+ );
+ }
+ return _turnText(turn);
+ }
+
+ /// Parses a native turn's structured tool calls, keeping only known StudyOS
+ /// tools. Argument JSON is passed through untouched to the tool executor,
+ /// which parses it (same contract as the bracket path's arguments string).
+ List<_LocalToolCall> _nativeToolCalls(Map turn) {
+ final raw = turn['calls'];
+ if (raw is! List) return const <_LocalToolCall>[];
+ final calls = <_LocalToolCall>[];
+ for (final entry in raw) {
+ if (entry is! Map) continue;
+ final name = entry['name']?.toString().trim().toLowerCase() ?? '';
+ if (name.isEmpty || studyOsToolByName(name) == null) continue;
+ final arguments = entry['arguments']?.toString();
+ calls.add(
+ _LocalToolCall(
+ name: name,
+ arguments: arguments == null || arguments.isEmpty ? '{}' : arguments,
+ ),
+ );
+ }
+ return calls;
+ }
+
+ String _turnText(Map turn) => turn['text']?.toString() ?? '';
+
String _localSystemPrompt(
String basePrompt,
Set supportedNativeToolNames,
@@ -242,6 +368,16 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
return buffer.toString().trim();
}
+ /// Prepends the volatile per-turn context (wall-clock time, world state) to
+ /// the user's message for the first turn. The stable system prompt already
+ /// lives in the conversation's system instruction, so only this ephemeral
+ /// slice needs to ride the message.
+ String _composeFirstTurn(String ephemeralContext, String userText) {
+ final ephemeral = ephemeralContext.trim();
+ if (ephemeral.isEmpty) return userText;
+ return '$ephemeral\n\n$userText';
+ }
+
String _localToolFeedbackPrompt(List feedback) {
return [
'System feedback from executed StudyOS tools:',
@@ -287,6 +423,9 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
status: status,
summary: '$summary$outputSuffix',
callId: callId,
+ component: output == null
+ ? null
+ : componentPayloadForTool(call.name, output),
);
}
}
@@ -343,6 +482,7 @@ class CloudLlmProvider implements AgentLlmProvider {
appendMemory: _appendMemory,
readMemory: _memoryStore.read,
readSchedule: request.readSchedule,
+ readAcademicStatus: request.readAcademicStatus,
searchTalks: request.searchTalks,
mailTools: request.mailTools,
publicStudyTools: request.publicStudyTools,
diff --git a/flutter_app/lib/src/app_router.dart b/flutter_app/lib/src/app_router.dart
index 66edafc..58a807f 100644
--- a/flutter_app/lib/src/app_router.dart
+++ b/flutter_app/lib/src/app_router.dart
@@ -8,6 +8,7 @@ import 'onboarding_flow.dart';
import 'studyos_theme.dart';
import 'views/chat_route.dart';
import 'views/home_view.dart';
+import 'views/mail_view.dart';
import 'views/maps_view.dart';
import 'views/memories_view.dart';
import 'views/official_documents_view.dart';
@@ -168,6 +169,13 @@ GoRouter buildAppRouter({
child: _DocumentsRoute(controller: shellController()),
),
),
+ GoRoute(
+ path: '/mail',
+ builder: (context, state) => _ScopedAppRoute(
+ controller: shellController(),
+ child: _MailRoute(controller: shellController()),
+ ),
+ ),
GoRoute(
path: '/talks',
builder: (context, state) => _ScopedAppRoute(
@@ -222,8 +230,7 @@ class _HomeRoute extends StatelessWidget {
onOpenAssistant: () => context.push('/settings'),
onOpenNotes: () => context.push('/memories'),
onOpenTalks: () => context.push('/talks'),
- onOpenMail: () =>
- context.push('/chat?prompt=Show%20my%20university%20mail'),
+ onOpenMail: () => context.push('/mail'),
onOpenMaps: () => context.push('/maps'),
onOpenCampus: () => context.push(
'/chat?prompt=What%20is%20good%20at%20the%20Mensa%20today%3F',
@@ -279,6 +286,28 @@ class _TalksRoute extends StatelessWidget {
}
}
+class _MailRoute extends StatelessWidget {
+ const _MailRoute({required this.controller});
+
+ final AppShellController? controller;
+
+ @override
+ Widget build(BuildContext context) {
+ final controller = this.controller ?? AppShellScope.of(context);
+ return ListenableBuilder(
+ listenable: controller,
+ builder: (context, _) => _RouteScaffold(
+ title: 'Mail',
+ showTitle: false,
+ child: MailView(
+ profile: controller.profile,
+ repository: controller.mailRepository,
+ ),
+ ),
+ );
+ }
+}
+
class _MapsRoute extends StatelessWidget {
const _MapsRoute({required this.controller});
diff --git a/flutter_app/lib/src/app_shell_controller.dart b/flutter_app/lib/src/app_shell_controller.dart
index 877b95f..686473b 100644
--- a/flutter_app/lib/src/app_shell_controller.dart
+++ b/flutter_app/lib/src/app_shell_controller.dart
@@ -3,6 +3,7 @@ import 'dart:convert';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
+import 'package:url_launcher/url_launcher.dart';
import 'agent_config_store.dart';
import 'agent_message_sender.dart';
@@ -10,11 +11,13 @@ import 'academic_repository.dart';
import 'calendar_overview_repository.dart';
import 'chat_scroll.dart';
import 'chat_session_mutation.dart';
+import 'generated_ui_message.dart';
import 'mail_repository.dart';
import 'mail_tools.dart';
import 'memory_store.dart';
import 'models.dart';
import 'native_bridge.dart';
+import 'native_tool_router.dart';
import 'official_document_models.dart';
import 'official_documents_repository.dart';
import 'profile_context.dart';
@@ -47,6 +50,32 @@ class ChatRouteRequest {
}
}
+/// Chooses when a deadline reminder should fire: one day before the due time,
+/// stepping closer (one hour before, then a short delay) as the deadline nears
+/// so the reminder never lands in the past. Pure so it can be unit-tested.
+DateTime reminderTimeForDeadline(DateTime dueAt, {DateTime? now}) {
+ final reference = now ?? DateTime.now();
+ final dayBefore = dueAt.subtract(const Duration(days: 1));
+ if (dayBefore.isAfter(reference)) return dayBefore;
+ final hourBefore = dueAt.subtract(const Duration(hours: 1));
+ if (hourBefore.isAfter(reference)) return hourBefore;
+ return reference.add(const Duration(minutes: 10));
+}
+
+/// Builds the Google Maps search URL for coordinates. Mirrors the deep link the
+/// in-app map view uses for its "Open in maps" control, so both surfaces behave
+/// identically. Pure so it can be unit-tested.
+Uri campusMapsUri(double latitude, double longitude) {
+ return Uri.https('www.google.com', '/maps/search/', {
+ 'api': '1',
+ 'query': '$latitude,$longitude',
+ });
+}
+
+Future _launchExternal(Uri uri) {
+ return launchUrl(uri, mode: LaunchMode.externalApplication);
+}
+
class AppShellController extends ChangeNotifier {
AppShellController({
required OnboardingProfile? initialProfile,
@@ -57,12 +86,20 @@ class AppShellController extends ChangeNotifier {
NativeBridge? nativeBridge,
TalksRepository? talksRepository,
CalendarOverviewSource? calendarOverviewSource,
+ NativeToolRunner? nativeToolRunner,
+ AcademicRepository? academicRepository,
+ TimetableRepository? timetableRepository,
+ Future Function(Uri uri)? urlLauncher,
}) : bridge = nativeBridge ?? NativeBridge(),
talksRepository = talksRepository ?? TalksRepository(),
_ownsTalksRepository = talksRepository == null,
+ _academicRepository = academicRepository ?? AcademicRepository(),
+ _timetableRepository = timetableRepository ?? TimetableRepository(),
+ _urlLauncher = urlLauncher ?? _launchExternal,
_profile = initialProfile,
_onLogout = initialOnLogout,
_onSaveProfile = initialOnSaveProfile {
+ _nativeToolRunner = nativeToolRunner ?? NativeToolRouter(bridge);
_privateStudyTools = CombinedPrivateStudyToolRunner(
portal: LivePrivateStudyToolRunner(
PrivateStudyCapability(profileProvider: () => _profile),
@@ -77,15 +114,20 @@ class AppShellController extends ChangeNotifier {
}
final NativeBridge bridge;
+ late final NativeToolRunner _nativeToolRunner;
+ final Future Function(Uri uri) _urlLauncher;
final TalksRepository talksRepository;
final bool _ownsTalksRepository;
late final CalendarOverviewSource calendarOverviewSource;
final SessionStore _sessionStore = SessionStore();
final AgentConfigStore _configStore = AgentConfigStore();
final MailRepository _mailRepository = MailRepository();
+
+ /// Shared mail repository so the mail view reuses the cached IMAP session.
+ MailRepository get mailRepository => _mailRepository;
final MemoryStore _memoryStore = MemoryStore();
- final TimetableRepository _timetableRepository = TimetableRepository();
- final AcademicRepository _academicRepository = AcademicRepository();
+ final TimetableRepository _timetableRepository;
+ final AcademicRepository _academicRepository;
final OfficialDocumentsRepository _documentsRepository =
OfficialDocumentsRepository();
final PublicStudyToolRunner _publicStudyTools = LivePublicStudyToolRunner();
@@ -112,8 +154,10 @@ class AppShellController extends ChangeNotifier {
AgentConfig _agentConfig = const AgentConfig.defaults();
String _memoryText = '';
TimetableSnapshot? _timetable;
+ Future? _timetableRefresh;
AcademicStatusSnapshot? _academicStatus;
String? _academicStatusError;
+ Future? _academicStatusRefresh;
String? _academicReportError;
List _officialDocuments = [];
String? _officialDocumentsError;
@@ -134,6 +178,15 @@ class AppShellController extends ChangeNotifier {
Timer? _streamNotifyTimer;
AgentCancelToken? _cancelToken;
+ /// Generative-UI card payloads produced by tools during the in-flight turn,
+ /// keyed by tool name (last call of each tool wins). Nothing is shown just
+ /// because a tool ran: a card surfaces only if the assistant's final reply
+ /// references it with a `tool_card` block, which is resolved against this map
+ /// when the message is committed (see [addAssistantMessage]). Cleared at the
+ /// start of every turn.
+ final Map> _turnToolComponents =
+ >{};
+
OnboardingProfile? get profile => _profile;
VoidCallback? get onLogout => _onLogout;
List get sessions => _sessions;
@@ -313,9 +366,21 @@ class AppShellController extends ChangeNotifier {
unawaited(refreshAcademicStatus());
}
- Future refreshAcademicStatus() async {
+ Future refreshAcademicStatus() {
final profile = _profile;
- if (profile == null || _isRefreshingAcademicStatus) return;
+ if (profile == null) return Future.value();
+ // Coalesce concurrent refreshes so callers await the in-flight fetch
+ // instead of racing past a still-running one. Previously the guard made a
+ // second caller (e.g. the get_academic_status tool, fired while the
+ // background refresh started in initialize() was still running) return
+ // immediately and read a null snapshot — surfacing "Academic status is not
+ // available." and masking the real error. Callers now share one future.
+ return _academicStatusRefresh ??= _runAcademicStatusRefresh(
+ profile,
+ ).whenComplete(() => _academicStatusRefresh = null);
+ }
+
+ Future _runAcademicStatusRefresh(OnboardingProfile profile) async {
_isRefreshingAcademicStatus = true;
_academicStatusError = null;
_notify();
@@ -400,13 +465,18 @@ class AppShellController extends ChangeNotifier {
}
Future readAcademicStatusForAgent() async {
- final status = _academicStatus;
- if (status == null) {
+ if (_profile == null) {
+ return 'Academic status is unavailable: no student profile is signed in.';
+ }
+ if (_academicStatus == null) {
await refreshAcademicStatus();
}
final resolved = _academicStatus;
if (resolved == null) {
- return _academicStatusError ?? 'Academic status is not available.';
+ // The refresh finished without a snapshot; surface the real reason
+ // (e.g. an authentication prompt) instead of a generic string.
+ return _academicStatusError ??
+ 'Academic status could not be loaded right now. Please try again in a moment.';
}
return jsonEncode({
'term': resolved.term,
@@ -432,6 +502,75 @@ class AppShellController extends ChangeNotifier {
onOpenChatRequest?.call(ChatRouteRequest(prompt: text));
}
+ /// Dispatches an action emitted by an interactive generative-UI component.
+ /// Prompt actions go back through the agent; reminder actions create a native
+ /// device reminder directly (the tap is the user's authorization).
+ void handleComponentAction(GeneratedComponentAction action) {
+ switch (action) {
+ case PromptComponentAction(:final prompt):
+ unawaited(runComponentPrompt(prompt));
+ case ReminderComponentAction(:final title, :final dueAt):
+ unawaited(addDeadlineReminder(title: title, dueAt: dueAt));
+ case MapComponentAction(:final name, :final latitude, :final longitude):
+ unawaited(
+ openLocationInMaps(
+ name: name,
+ latitude: latitude,
+ longitude: longitude,
+ ),
+ );
+ }
+ }
+
+ /// Runs a prompt requested by a component (e.g. mail Summarize): prefills the
+ /// composer and sends it, reusing the autosent chat-route path so a turn is
+ /// created immediately.
+ Future runComponentPrompt(String text) {
+ return applyChatRoute(prompt: text, autosend: true);
+ }
+
+ /// Creates a native device reminder ahead of [dueAt] via the capability-gated
+ /// native tool runner, then reports the outcome as an assistant message. On
+ /// platforms without reminder support the runner returns a friendly message,
+ /// which is surfaced as-is.
+ Future addDeadlineReminder({
+ required String title,
+ required DateTime dueAt,
+ }) async {
+ final when = reminderTimeForDeadline(dueAt);
+ final result = await _nativeToolRunner.execute(
+ nativeCreateReminderToolName,
+ jsonEncode({
+ 'title': title,
+ 'time': when.toIso8601String(),
+ }),
+ );
+ if (_disposed) return;
+ final detail = result.trim();
+ addAssistantMessage(
+ detail.isEmpty ? 'Reminder requested for "$title".' : detail,
+ );
+ }
+
+ /// Opens a geocoded place in the device's external maps app. Reports a message
+ /// only on failure (success hands off to the maps app).
+ Future openLocationInMaps({
+ required String name,
+ required double latitude,
+ required double longitude,
+ }) async {
+ bool opened;
+ try {
+ opened = await _urlLauncher(campusMapsUri(latitude, longitude));
+ } on Object {
+ opened = false;
+ }
+ if (_disposed) return;
+ if (!opened) {
+ addAssistantMessage('Could not open $name in maps.');
+ }
+ }
+
Future applyChatRoute({
String? prompt,
bool autosend = false,
@@ -463,6 +602,7 @@ class AppShellController extends ChangeNotifier {
if (text.isEmpty || _isSending) return;
_isSending = true;
+ _turnToolComponents.clear();
inputController.clear();
_notify();
appendMessage(ChatMessage(author: 'You', text: text, isUser: true));
@@ -575,7 +715,8 @@ class AppShellController extends ChangeNotifier {
// interfere with the live streaming text.
_scheduleStreamNotify();
if (hasContent && voice.isVoicingReply) {
- voice.pushReplyText(streaming.text);
+ // Never speak the trailing `ui` component block.
+ voice.pushReplyText(streamingVisibleText(streaming.text));
}
}
@@ -604,18 +745,31 @@ class AppShellController extends ChangeNotifier {
void addAssistantMessage(String text, {String? reasoning}) {
if (_disposed) return;
+ // Split off any model-emitted `ui` block (always stripped from the visible
+ // text so raw JSON is never shown), then decide the card: an explicit
+ // reference or composed component if present, else the most recent tool
+ // card when the reply reads as a short lead-in. A long pivot answer that
+ // merely ran a tool gets no card.
+ final parts = splitAssistantComponent(text);
+ final component = resolveMessageComponent(
+ emitted: parts.component,
+ capturedToolComponents: _turnToolComponents,
+ replyText: parts.text,
+ );
+ _turnToolComponents.clear();
appendMessage(
ChatMessage(
author: 'StudyOS Agent',
- text: text,
+ text: parts.text,
isUser: false,
reasoning: reasoning,
+ component: component,
),
);
- _status = text;
+ _status = parts.text;
_notify();
unawaited(HapticFeedback.lightImpact());
- voice.endSpokenReply(text);
+ voice.endSpokenReply(parts.text);
}
Future loadSessions() async {
@@ -658,6 +812,13 @@ class AppShellController extends ChangeNotifier {
}
void addToolTrace(ToolTrace trace) {
+ // Capture a tool's card payload for the turn, keyed by tool name. It is only
+ // shown if the assistant's final reply opts it in with a `tool_card`
+ // reference — running the tool alone never surfaces a card.
+ final component = trace.component;
+ if (component != null) {
+ _turnToolComponents[trace.toolName] = component;
+ }
_applySessionMutation(
upsertToolTraceInSessions(
sessions: _sessions,
@@ -710,14 +871,22 @@ class AppShellController extends ChangeNotifier {
}
}
- Future refreshTimetable() async {
- if (_isRefreshingTimetable) return;
+ Future refreshTimetable() {
final profile = _profile;
if (profile == null) {
_timetableError = 'Sign in again to refresh your timetable.';
_notify();
- return;
+ return Future.value();
}
+ // Coalesce concurrent refreshes so a caller (e.g. the get_schedule tool)
+ // awaits the in-flight fetch instead of racing past it — same fix as
+ // academic status.
+ return _timetableRefresh ??= _runTimetableRefresh(
+ profile,
+ ).whenComplete(() => _timetableRefresh = null);
+ }
+
+ Future _runTimetableRefresh(OnboardingProfile profile) async {
_isRefreshingTimetable = true;
_timetableError = null;
_notify();
@@ -798,8 +967,30 @@ class AppShellController extends ChangeNotifier {
await refreshTimetable();
snapshot = _timetable;
}
- return snapshot?.compactSummary(limit: 12) ??
- 'No timetable has been synced yet.';
+ if (snapshot == null || snapshot.events.isEmpty) {
+ return _timetableError ?? 'No timetable has been synced yet.';
+ }
+ final upcoming = snapshot.upcoming.take(12).toList(growable: false);
+ if (upcoming.isEmpty) {
+ return 'No upcoming lectures in the synced timetable.';
+ }
+ // Structured output so the client can render an interactive schedule card
+ // (see schedule_agenda in GenerativeUiRegistry). The model gets the same
+ // data as JSON instead of a prose summary.
+ return jsonEncode({
+ 'source_term': snapshot.sourceTerm,
+ 'refreshed_at': snapshot.refreshedAt.toIso8601String(),
+ 'events': upcoming
+ .map(
+ (event) => {
+ 'title': event.title,
+ 'start': event.start.toIso8601String(),
+ 'end': event.end?.toIso8601String(),
+ 'location': event.location,
+ },
+ )
+ .toList(growable: false),
+ });
}
Future searchTalksForAgent(String query, int limit) async {
diff --git a/flutter_app/lib/src/cloud_agent_client.dart b/flutter_app/lib/src/cloud_agent_client.dart
index de83f74..8f8a5c1 100644
--- a/flutter_app/lib/src/cloud_agent_client.dart
+++ b/flutter_app/lib/src/cloud_agent_client.dart
@@ -448,6 +448,9 @@ class CloudAgentClient {
status: status,
summary: '$summary$outputSuffix',
callId: call.id,
+ component: output == null
+ ? null
+ : componentPayloadForTool(call.name, output),
);
}
}
diff --git a/flutter_app/lib/src/generated_ui_message.dart b/flutter_app/lib/src/generated_ui_message.dart
new file mode 100644
index 0000000..f31c31a
--- /dev/null
+++ b/flutter_app/lib/src/generated_ui_message.dart
@@ -0,0 +1,150 @@
+import 'dart:convert';
+
+/// Fenced-block marker the model uses to attach a generative-UI component to a
+/// reply that did not run a tool. The reply ends with:
+///
+/// ```ui
+/// {"type": "quick_reply", "title": "...", "body": "...", "arguments": {...}}
+/// ```
+///
+/// Kept as a `ui`-tagged code fence so a malformed or partially streamed block
+/// degrades to (at worst) a hidden code block rather than raw JSON, and so the
+/// opener is cheap to detect while the reply is still streaming in.
+final RegExp _uiFence = RegExp(r'```[ \t]*ui[ \t]*\r?\n([\s\S]*?)```');
+
+/// Just the fence opener, used to hide everything from the block onward while
+/// the reply streams (before the closing fence has arrived).
+final RegExp _uiFenceOpener = RegExp(r'```[ \t]*ui\b');
+
+/// A committed assistant reply split into its visible [text] and an optional
+/// generative-UI [component] payload the model emitted in a trailing `ui`
+/// fence. [component] is left unvalidated — the render layer
+/// ([GenerativeUiRegistry]) validates and silently drops anything invalid, so a
+/// junk payload just yields no card.
+class AssistantMessageParts {
+ const AssistantMessageParts({required this.text, this.component});
+
+ final String text;
+ final Map? component;
+}
+
+/// Splits a raw assistant reply into visible prose and an optional model-emitted
+/// component payload. The `ui` fence is always removed from [text] whether or
+/// not its contents parse, so raw JSON is never shown to the user; the payload
+/// is attached only when the fence holds a JSON object.
+AssistantMessageParts splitAssistantComponent(String raw) {
+ final match = _uiFence.firstMatch(raw);
+ if (match == null) {
+ return AssistantMessageParts(text: raw);
+ }
+
+ final text = raw.replaceRange(match.start, match.end, '').trim();
+ final component = _decodeComponent(match.group(1) ?? '');
+ return AssistantMessageParts(text: text, component: component);
+}
+
+/// The portion of a still-streaming reply that is safe to show: everything
+/// before the `ui` fence opener, so the JSON block never flashes on screen as it
+/// arrives token by token. Returns [raw] unchanged when no opener is present.
+String streamingVisibleText(String raw) {
+ final match = _uiFenceOpener.firstMatch(raw);
+ if (match == null) return raw;
+ return raw.substring(0, match.start).trimRight();
+}
+
+/// Wire type of a model-emitted reference that asks the app to display a tool's
+/// result as its existing card, rather than restating the tool's data inline.
+const String toolCardReferenceType = 'tool_card';
+
+/// Resolves the payload extracted from a `ui` block into the component to attach
+/// to the assistant message.
+///
+/// Tool cards are decoupled from tool execution: running `get_study_planner`
+/// does NOT surface a planner card on its own. The model must opt a tool result
+/// in by ending its reply with a `{"type":"tool_card","tool":""}`
+/// reference, which resolves here to that tool's captured payload from
+/// [capturedToolComponents] (keyed by tool name). If the model didn't reference
+/// a tool — because it called the tool but pivoted away — nothing shows.
+///
+/// - A `tool_card` reference → the captured payload for its `tool`, or null when
+/// the tool wasn't called this turn or produced no card.
+/// - Any other payload (a model-composed A/B component such as `quick_reply` or
+/// `custom_view`) → returned unchanged.
+/// - null → null.
+Map? resolveComponentPayload(
+ Map? emitted,
+ Map> capturedToolComponents,
+) {
+ if (emitted == null) return null;
+ if (emitted['type'] != toolCardReferenceType) return emitted;
+ final tool = emitted['tool']?.toString();
+ if (tool == null || tool.isEmpty) return null;
+ return capturedToolComponents[tool];
+}
+
+/// Upper bounds on what still counts as a presentational lead-in (see
+/// [isPresentationalLeadIn]).
+const int _leadInMaxLines = 2;
+const int _leadInMaxChars = 140;
+const int _leadInMaxSentences = 1;
+
+final RegExp _sentenceEnd = RegExp(r'[.!?]+(\s|$)');
+
+/// Whether [replyText] reads as a short lead-in that introduces a result (e.g.
+/// "Here are your recent emails:") rather than a full answer that has pivoted to
+/// another topic. Used to decide whether to surface a tool's captured card when
+/// the model didn't emit an explicit reference — small models write the lead-in
+/// naturally but forget the machine-readable block.
+///
+/// A lead-in is short on all three axes: at most [_leadInMaxLines] lines,
+/// [_leadInMaxChars] characters, and [_leadInMaxSentences] sentence. The
+/// sentence count catches a multi-sentence answer that still fits the character
+/// budget; the character cap catches a single run-on pivot sentence.
+bool isPresentationalLeadIn(String replyText) {
+ final trimmed = replyText.trim();
+ if (trimmed.isEmpty) return false;
+ if (trimmed.length > _leadInMaxChars) return false;
+ final lineCount = trimmed
+ .split('\n')
+ .where((line) => line.trim().isNotEmpty)
+ .length;
+ if (lineCount > _leadInMaxLines) return false;
+ return _sentenceEnd.allMatches(trimmed).length <= _leadInMaxSentences;
+}
+
+/// Decides the component to attach to an assistant message.
+///
+/// Tool cards are shown when the reply is *about* a fetched result, detected two
+/// ways: an explicit `tool_card` reference the model emitted (precise, picks the
+/// exact tool), or — since a small model often omits that block — a short
+/// presentational lead-in ([isPresentationalLeadIn]) paired with a tool that
+/// produced a card this turn, in which case the most recently captured card is
+/// used. A long answer with no reference (the model called a tool but pivoted
+/// away) yields no card, keeping the full prose. Composed A/B components
+/// ([resolveComponentPayload] passthrough) always win when present.
+///
+/// [capturedToolComponents] is insertion-ordered; its last value is the most
+/// recent tool card of the turn.
+Map? resolveMessageComponent({
+ required Map? emitted,
+ required Map> capturedToolComponents,
+ required String replyText,
+}) {
+ final direct = resolveComponentPayload(emitted, capturedToolComponents);
+ if (direct != null) return direct;
+ if (capturedToolComponents.isEmpty) return null;
+ if (!isPresentationalLeadIn(replyText)) return null;
+ return capturedToolComponents.values.last;
+}
+
+Map? _decodeComponent(String body) {
+ final trimmed = body.trim();
+ if (trimmed.isEmpty) return null;
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(trimmed);
+ } on FormatException {
+ return null;
+ }
+ return decoded is Map ? Map.from(decoded) : null;
+}
diff --git a/flutter_app/lib/src/generative_ui_registry.dart b/flutter_app/lib/src/generative_ui_registry.dart
index b0fd520..82c30b1 100644
--- a/flutter_app/lib/src/generative_ui_registry.dart
+++ b/flutter_app/lib/src/generative_ui_registry.dart
@@ -1,9 +1,32 @@
+import 'dart:convert';
+
+/// Bounds on a `custom_view` node tree, enforced during validation so a
+/// malformed or oversized payload from a small model can't blow up layout or
+/// recursion. The renderer stays tolerant of individual bad leaf nodes (it
+/// skips them); these caps only guard the overall shape.
+const int customViewMaxNodes = 48;
+const int customViewMaxDepth = 4;
+const int customViewMaxChildrenPerContainer = 24;
+
+/// The one recursive container node in the `custom_view` vocabulary. Its
+/// children live under the same `blocks` key the root uses.
+const String customViewContainerNode = 'group';
+
enum GeneratedComponentKind {
nextAction('next_action'),
scheduleSummary('schedule_summary'),
routeHint('route_hint'),
deadlineCard('deadline_card'),
- quickReply('quick_reply');
+ quickReply('quick_reply'),
+ mailList('mail_list'),
+ deadlineList('deadline_list'),
+ talkList('talk_list'),
+ academicStatus('academic_status'),
+ studyProgress('study_progress'),
+ mensaMenu('mensa_menu'),
+ campusLocations('campus_locations'),
+ scheduleAgenda('schedule_agenda'),
+ customView('custom_view');
const GeneratedComponentKind(this.wireName);
@@ -117,8 +140,495 @@ abstract final class GenerativeUiRegistry {
GeneratedComponentKind.quickReply => _requireStrings(arguments, [
'reply',
]),
+ GeneratedComponentKind.mailList => _validateItemList(
+ arguments,
+ 'messages',
+ ),
+ GeneratedComponentKind.deadlineList => _validateItemList(
+ arguments,
+ 'deadlines',
+ ),
+ GeneratedComponentKind.talkList => _validateItemList(arguments, 'talks'),
+ GeneratedComponentKind.academicStatus => _validateItemList(
+ arguments,
+ 'entries',
+ ),
+ GeneratedComponentKind.studyProgress => _validateItemList(
+ arguments,
+ 'modules',
+ ),
+ GeneratedComponentKind.mensaMenu => _validateItemList(
+ arguments,
+ 'options',
+ ),
+ GeneratedComponentKind.campusLocations => _validateItemList(
+ arguments,
+ 'locations',
+ ),
+ GeneratedComponentKind.scheduleAgenda => _validateItemList(
+ arguments,
+ 'events',
+ ),
+ GeneratedComponentKind.customView => _validateCustomView(arguments),
};
}
+
+ /// Validates only the *structure* of a `custom_view` tree: a non-empty
+ /// `blocks` list within the node-count, depth, and per-container caps. Leaf
+ /// nodes are intentionally not field-checked here — the renderer skips any it
+ /// can't draw — so a mostly-good tree from a weak model still renders instead
+ /// of collapsing to plain text.
+ static List _validateCustomView(Map arguments) {
+ final blocks = arguments['blocks'];
+ if (blocks is! List || blocks.isEmpty) {
+ return ['Missing non-empty list argument: blocks'];
+ }
+ final errors = [];
+ var nodeCount = 0;
+
+ void walk(List nodes, int depth) {
+ if (errors.isNotEmpty) return;
+ if (depth > customViewMaxDepth) {
+ errors.add('Custom view nesting exceeds depth $customViewMaxDepth');
+ return;
+ }
+ if (nodes.length > customViewMaxChildrenPerContainer) {
+ errors.add(
+ 'Custom view container exceeds '
+ '$customViewMaxChildrenPerContainer children',
+ );
+ return;
+ }
+ for (final node in nodes) {
+ nodeCount++;
+ if (nodeCount > customViewMaxNodes) {
+ errors.add('Custom view exceeds $customViewMaxNodes nodes');
+ return;
+ }
+ if (node is Map && node['node'] == customViewContainerNode) {
+ final children = node['blocks'];
+ if (children is List) walk(children, depth + 1);
+ if (errors.isNotEmpty) return;
+ }
+ }
+ }
+
+ walk(blocks, 1);
+ return errors;
+ }
+}
+
+/// Single entry point the provider tool loops use to turn a completed tool's
+/// JSON output into a generative-UI component payload, or `null` when the tool
+/// has no card. Each component kind registers its builder here, so adding a
+/// component never touches the provider code again — the registry is the one
+/// place that maps tools to cards.
+Map? componentPayloadForTool(String toolName, String output) {
+ return mailTriageComponentPayload(toolName, output) ??
+ deadlineListComponentPayload(toolName, output) ??
+ talkListComponentPayload(toolName, output) ??
+ academicStatusComponentPayload(toolName, output) ??
+ studyProgressComponentPayload(toolName, output) ??
+ mensaMenuComponentPayload(toolName, output) ??
+ campusLocationsComponentPayload(toolName, output) ??
+ scheduleAgendaComponentPayload(toolName, output);
+}
+
+/// Builds a `mail_list` GenUI payload from the JSON a mail-summary tool
+/// (`get_recent_mail` / `search_mail`) returns, or `null` when [toolName] is not
+/// a mail-list producer or [output] cannot be parsed into a non-empty list.
+///
+/// Kept provider-agnostic (pure, no Flutter imports) so both the local and the
+/// cloud tool loops can attach the result to the tool's [ToolTrace]. It only
+/// forwards the summary fields the card renders — no message bodies.
+Map? mailTriageComponentPayload(
+ String toolName,
+ String output,
+) {
+ const producers = {'get_recent_mail', 'search_mail'};
+ if (!producers.contains(toolName)) return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final rawMessages = decoded['messages'];
+ if (rawMessages is! List) return null;
+
+ final messages = >[];
+ for (final raw in rawMessages) {
+ if (raw is! Map) continue;
+ final uid = _string(raw['uid']);
+ final subject = _string(raw['subject']);
+ if (uid == null || subject == null) continue;
+ messages.add({
+ 'uid': uid,
+ 'subject': subject,
+ 'sender':
+ _string(raw['from_name']) ??
+ _string(raw['from_address']) ??
+ 'Unknown sender',
+ 'received_at': _string(raw['received_at']),
+ 'preview': _string(raw['preview']),
+ 'is_unread': raw['is_unread'] == true,
+ 'is_approved_broadcast': raw['is_approved_broadcast'] == true,
+ });
+ }
+ if (messages.isEmpty) return null;
+
+ final mailbox = _string(decoded['mailbox']) ?? 'INBOX';
+ final rawUnread = decoded['unread_count'];
+ final unread = rawUnread is int
+ ? rawUnread
+ : messages.where((message) => message['is_unread'] == true).length;
+ final count = messages.length;
+ return {
+ 'type': 'mail_list',
+ 'title': unread > 0 ? '$mailbox · $unread unread' : mailbox,
+ 'body': count == 1 ? '1 message' : '$count messages',
+ 'arguments': {
+ 'mailbox': mailbox,
+ 'unread_count': unread,
+ 'messages': messages,
+ },
+ };
+}
+
+List _validateItemList(Map arguments, String listKey) {
+ final items = arguments[listKey];
+ if (items is! List || items.isEmpty) {
+ return ['Missing non-empty list argument: $listKey'];
+ }
+ return const [];
+}
+
+/// Builds a `deadline_list` payload from the JSON `get_deadlines` returns (a
+/// [CapabilityResult] whose `data` is the deadline list), or `null` for other
+/// tools / empty results. Forwards only the fields the card renders.
+Map? deadlineListComponentPayload(
+ String toolName,
+ String output,
+) {
+ if (toolName != 'get_deadlines') return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final rawData = decoded['data'];
+ if (rawData is! List) return null;
+
+ final deadlines = >[];
+ for (final raw in rawData) {
+ if (raw is! Map) continue;
+ final title = _string(raw['title']);
+ final dueAt = _string(raw['dueAt']);
+ if (title == null || dueAt == null) continue;
+ deadlines.add({
+ 'id': _string(raw['id']),
+ 'title': title,
+ 'course': _string(raw['courseTitle']),
+ 'due_at': dueAt,
+ 'requirement': _string(raw['requirement']),
+ 'status': _string(raw['status']),
+ });
+ }
+ if (deadlines.isEmpty) return null;
+
+ final count = deadlines.length;
+ return {
+ 'type': 'deadline_list',
+ 'title': count == 1 ? 'Upcoming deadline' : '$count upcoming deadlines',
+ 'body': count == 1 ? '1 deadline' : '$count deadlines',
+ 'arguments': {'deadlines': deadlines},
+ };
+}
+
+/// Builds a `talk_list` payload from `search_talks` output (a `{items: [...]}`
+/// envelope of Tübingen talks), or `null` otherwise. Forwards only the fields
+/// the card renders plus the ISO timestamp its "Remind me" action needs.
+Map? talkListComponentPayload(String toolName, String output) {
+ if (toolName != 'search_talks') return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final rawItems = decoded['items'];
+ if (rawItems is! List) return null;
+
+ final talks = >[];
+ for (final raw in rawItems) {
+ if (raw is! Map) continue;
+ final title = _string(raw['title']);
+ if (title == null) continue;
+ talks.add({
+ 'title': title,
+ 'timestamp': _string(raw['timestamp']),
+ 'speaker': _string(raw['speaker_name']),
+ 'location': _string(raw['location']),
+ });
+ }
+ if (talks.isEmpty) return null;
+
+ final count = talks.length;
+ return {
+ 'type': 'talk_list',
+ 'title': count == 1 ? 'Upcoming talk' : '$count upcoming talks',
+ 'body': count == 1 ? '1 talk' : '$count talks',
+ 'arguments': {'talks': talks},
+ };
+}
+
+/// Builds an `academic_status` payload from `get_academic_status` output (a
+/// `{term, entries: [...]}` snapshot of exam/course statuses), or `null`
+/// otherwise. Read-only card — no per-item actions.
+Map? academicStatusComponentPayload(
+ String toolName,
+ String output,
+) {
+ if (toolName != 'get_academic_status') return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final rawEntries = decoded['entries'];
+ if (rawEntries is! List) return null;
+
+ final entries = >[];
+ for (final raw in rawEntries) {
+ if (raw is! Map) continue;
+ final title = _string(raw['title']);
+ if (title == null) continue;
+ entries.add({
+ 'category': _string(raw['category']) ?? 'Other',
+ 'title': title,
+ 'status': _string(raw['status']),
+ 'semester': _string(raw['semester']),
+ });
+ }
+ if (entries.isEmpty) return null;
+
+ final term = _string(decoded['term']);
+ final count = entries.length;
+ return {
+ 'type': 'academic_status',
+ 'title': term == null ? 'Academic status' : 'Academic status · $term',
+ 'body': count == 1 ? '1 entry' : '$count entries',
+ 'arguments': {'term': ?term, 'entries': entries},
+ };
+}
+
+/// Builds a `study_progress` payload from `get_study_planner` output (a
+/// [CapabilityResult] whose `data` is an ALMA planner page with modules that
+/// carry earned/required ECTS), or `null` otherwise. Also computes the overall
+/// earned-vs-required total across modules that report both.
+Map? studyProgressComponentPayload(
+ String toolName,
+ String output,
+) {
+ if (toolName != 'get_study_planner') return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final data = decoded['data'];
+ if (data is! Map) return null;
+ final rawModules = data['modules'];
+ if (rawModules is! List) return null;
+
+ final modules = >[];
+ var totalEarned = 0.0;
+ var totalRequired = 0.0;
+ for (final raw in rawModules) {
+ if (raw is! Map) continue;
+ final title = _string(raw['title']);
+ if (title == null) continue;
+ final earned = _double(raw['creditsEarned']);
+ final required = _double(raw['creditsRequired']);
+ if (earned != null && required != null && required > 0) {
+ totalEarned += earned;
+ totalRequired += required;
+ }
+ modules.add({
+ 'title': title,
+ 'number': _string(raw['number']),
+ 'earned': earned,
+ 'required': required,
+ 'summary': _string(raw['creditsSummary']),
+ });
+ }
+ if (modules.isEmpty) return null;
+
+ final pageTitle = _string(data['title']) ?? 'Study progress';
+ final body = totalRequired > 0
+ ? '${_trimNumber(totalEarned)} / ${_trimNumber(totalRequired)} ECTS'
+ : '${modules.length} modules';
+ return {
+ 'type': 'study_progress',
+ 'title': pageTitle,
+ 'body': body,
+ 'arguments': {
+ 'total_earned': totalRequired > 0 ? totalEarned : null,
+ 'total_required': totalRequired > 0 ? totalRequired : null,
+ 'modules': modules,
+ },
+ };
+}
+
+/// Builds a `mensa_menu` payload from `get_mensa_options` output (a
+/// [CapabilityResult] whose `data` is a list of canteen menu lines), or `null`
+/// otherwise. Read-only card.
+Map? mensaMenuComponentPayload(
+ String toolName,
+ String output,
+) {
+ if (toolName != 'get_mensa_options') return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final rawData = decoded['data'];
+ if (rawData is! List) return null;
+
+ final options = >[];
+ for (final raw in rawData) {
+ if (raw is! Map) continue;
+ final line = _string(raw['line']);
+ final items = _stringList(raw['items']);
+ if (line == null && items.isEmpty) continue;
+ options.add({
+ 'canteen': _string(raw['canteen']),
+ 'line': line ?? 'Menu',
+ 'items': items,
+ 'markers': _stringList(raw['dietary_markers']),
+ 'price': _string(raw['student_price']),
+ });
+ }
+ if (options.isEmpty) return null;
+
+ final canteens = options
+ .map((option) => _string(option['canteen']))
+ .whereType()
+ .toSet();
+ final count = options.length;
+ return {
+ 'type': 'mensa_menu',
+ 'title': canteens.length == 1 ? canteens.first : 'Mensa menu',
+ 'body': count == 1 ? '1 option' : '$count options',
+ 'arguments': {'options': options},
+ };
+}
+
+/// Builds a `campus_locations` payload from `search_campus_locations` output (a
+/// [CapabilityResult] whose `data` is a list of geocoded places), or `null`
+/// otherwise. Each location keeps its coordinates so the card's "Open in Maps"
+/// action can launch them.
+Map? campusLocationsComponentPayload(
+ String toolName,
+ String output,
+) {
+ if (toolName != 'search_campus_locations') return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final rawData = decoded['data'];
+ if (rawData is! List) return null;
+
+ final locations = >[];
+ for (final raw in rawData) {
+ if (raw is! Map) continue;
+ final name = _string(raw['name']);
+ final latitude = _double(raw['latitude']);
+ final longitude = _double(raw['longitude']);
+ if (name == null || latitude == null || longitude == null) continue;
+ locations.add({
+ 'name': name,
+ 'address': _string(raw['address']),
+ 'category': _string(raw['category']),
+ 'latitude': latitude,
+ 'longitude': longitude,
+ });
+ }
+ if (locations.isEmpty) return null;
+
+ final count = locations.length;
+ return {
+ 'type': 'campus_locations',
+ 'title': count == 1 ? locations.first['name'] : '$count places',
+ 'body': count == 1 ? '1 place' : '$count places',
+ 'arguments': {'locations': locations},
+ };
+}
+
+/// Builds a `schedule_agenda` payload from `get_schedule` output (a
+/// `{source_term, events: [...]}` snapshot of upcoming lectures), or `null`
+/// otherwise. Read-only card; the widget groups events by day.
+Map? scheduleAgendaComponentPayload(
+ String toolName,
+ String output,
+) {
+ if (toolName != 'get_schedule') return null;
+
+ final Object? decoded;
+ try {
+ decoded = jsonDecode(output);
+ } on FormatException {
+ return null;
+ }
+ if (decoded is! Map) return null;
+ final rawEvents = decoded['events'];
+ if (rawEvents is! List) return null;
+
+ final events = >[];
+ for (final raw in rawEvents) {
+ if (raw is! Map) continue;
+ final title = _string(raw['title']);
+ final start = _string(raw['start']);
+ if (title == null || start == null) continue;
+ events.add({
+ 'title': title,
+ 'start': start,
+ 'end': _string(raw['end']),
+ 'location': _string(raw['location']),
+ });
+ }
+ if (events.isEmpty) return null;
+
+ final term = _string(decoded['source_term']);
+ final count = events.length;
+ return {
+ 'type': 'schedule_agenda',
+ 'title': term == null ? 'Upcoming schedule' : 'Schedule · $term',
+ 'body': count == 1 ? '1 lecture' : '$count lectures',
+ 'arguments': {'events': events},
+ };
}
const List>
@@ -168,8 +678,316 @@ generativeUiFixturePayloads = >[
'reply': 'Plan a 45 minute review block around my next lecture.',
},
},
+ {
+ 'type': 'mail_list',
+ 'title': 'INBOX · 2 unread',
+ 'body': '3 messages',
+ 'arguments': {
+ 'mailbox': 'INBOX',
+ 'unread_count': 2,
+ 'messages': >[
+ {
+ 'uid': '4821',
+ 'subject': 'ML exercise sheet 7 — submission Friday',
+ 'sender': 'Prof. Dr. Weber',
+ 'received_at': '2026-07-08T09:12:00',
+ 'preview': 'Please upload your solutions to Ilias before 18:00 on…',
+ 'is_unread': true,
+ 'is_approved_broadcast': true,
+ },
+ {
+ 'uid': '4820',
+ 'subject': 'Room change for Thursday tutorial',
+ 'sender': 'Studierendensekretariat',
+ 'received_at': '2026-07-07T16:40:00',
+ 'preview': 'The tutorial moves to room A301 starting this week.',
+ 'is_unread': true,
+ 'is_approved_broadcast': false,
+ },
+ {
+ 'uid': '4818',
+ 'subject': 'Re: Study group notes',
+ 'sender': 'Lena',
+ 'received_at': '2026-07-07T11:05:00',
+ 'preview': 'Thanks! I added the missing derivations to the shared…',
+ 'is_unread': false,
+ 'is_approved_broadcast': false,
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'deadline_list',
+ 'title': '2 upcoming deadlines',
+ 'body': '2 deadlines',
+ 'arguments': {
+ 'deadlines': >[
+ {
+ 'id': 'ilias:9921',
+ 'title': 'ML exercise sheet 7',
+ 'course': 'Machine Learning',
+ 'due_at': '2026-12-11T18:00:00.000Z',
+ 'requirement': 'Graded submission',
+ 'status': 'open',
+ },
+ {
+ 'id': 'moodle:5540',
+ 'title': 'Databases project milestone',
+ 'course': 'Databases',
+ 'due_at': '2026-12-15T23:59:00.000Z',
+ 'requirement': null,
+ 'status': 'open',
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'talk_list',
+ 'title': '2 upcoming talks',
+ 'body': '2 talks',
+ 'arguments': {
+ 'talks': >[
+ {
+ 'title': 'Foundation models for scientific discovery',
+ 'timestamp': '2026-12-09T16:15:00.000Z',
+ 'speaker': 'Dr. Amelie Roth',
+ 'location': 'Hörsaal 21, Kupferbau',
+ },
+ {
+ 'title': 'Reinforcement learning in robotics',
+ 'timestamp': '2026-12-11T14:00:00.000Z',
+ 'speaker': 'Prof. Chen',
+ 'location': 'MPI-IS, Lecture Hall N0.002',
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'academic_status',
+ 'title': 'Academic status · WS 2026/27',
+ 'body': '3 entries',
+ 'arguments': {
+ 'term': 'WS 2026/27',
+ 'entries': >[
+ {
+ 'category': 'Exams',
+ 'title': 'Machine Learning — written exam',
+ 'status': 'Registered',
+ 'semester': 'WS 2026/27',
+ },
+ {
+ 'category': 'Exams',
+ 'title': 'Databases — oral exam',
+ 'status': 'Passed (1.7)',
+ 'semester': 'WS 2026/27',
+ },
+ {
+ 'category': 'Courses',
+ 'title': 'Statistics III',
+ 'status': 'Enrolled',
+ 'semester': 'WS 2026/27',
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'study_progress',
+ 'title': 'M.Sc. Machine Learning',
+ 'body': '78 / 120 ECTS',
+ 'arguments': {
+ 'total_earned': 78,
+ 'total_required': 120,
+ 'modules': >[
+ {
+ 'title': 'Core Machine Learning',
+ 'number': 'ML-4100',
+ 'earned': 27,
+ 'required': 30,
+ 'summary': '27 / 30 ECTS',
+ },
+ {
+ 'title': 'Theoretical Foundations',
+ 'number': 'ML-4200',
+ 'earned': 18,
+ 'required': 30,
+ 'summary': '18 / 30 ECTS',
+ },
+ {
+ 'title': "Master's Thesis",
+ 'number': 'ML-4900',
+ 'earned': 0,
+ 'required': 30,
+ 'summary': '0 / 30 ECTS',
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'mensa_menu',
+ 'title': 'Mensa Wilhelmstraße',
+ 'body': '2 options',
+ 'arguments': {
+ 'options': >[
+ {
+ 'canteen': 'Mensa Wilhelmstraße',
+ 'line': 'Line 1',
+ 'items': ['Gemüse-Lasagne', 'Blattsalat'],
+ 'markers': ['Vegetarisch'],
+ 'price': '3,20 €',
+ },
+ {
+ 'canteen': 'Mensa Wilhelmstraße',
+ 'line': 'Line 2',
+ 'items': ['Rindergulasch', 'Semmelknödel'],
+ 'markers': [],
+ 'price': '4,10 €',
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'campus_locations',
+ 'title': '2 places',
+ 'body': '2 places',
+ 'arguments': {
+ 'locations': >[
+ {
+ 'name': 'Universitätsbibliothek Tübingen',
+ 'address': 'Wilhelmstraße 32, 72074 Tübingen',
+ 'category': 'library',
+ 'latitude': 48.5296,
+ 'longitude': 9.0596,
+ },
+ {
+ 'name': 'Mensa Wilhelmstraße',
+ 'address': 'Wilhelmstraße 13, 72074 Tübingen',
+ 'category': 'canteen',
+ 'latitude': 48.5309,
+ 'longitude': 9.0625,
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'schedule_agenda',
+ 'title': 'Schedule · WS 2026/27',
+ 'body': '3 lectures',
+ 'arguments': {
+ 'events': >[
+ {
+ 'title': 'Machine Learning',
+ 'start': '2026-12-09T10:15:00',
+ 'end': '2026-12-09T11:45:00',
+ 'location': 'Hörsaal 21',
+ },
+ {
+ 'title': 'Databases Tutorial',
+ 'start': '2026-12-09T14:00:00',
+ 'end': '2026-12-09T15:30:00',
+ 'location': 'A301',
+ },
+ {
+ 'title': 'Statistics III',
+ 'start': '2026-12-10T08:15:00',
+ 'end': '2026-12-10T09:45:00',
+ 'location': null,
+ },
+ ],
+ },
+ },
+ {
+ 'type': 'custom_view',
+ 'title': 'Supervised vs. unsupervised',
+ 'body': 'A quick comparison for your exam prep.',
+ 'arguments': {
+ 'blocks': >[
+ {
+ 'node': 'badges',
+ 'items': >[
+ {'text': 'Exam topic', 'tone': 'positive'},
+ {'text': 'ML core', 'tone': 'neutral'},
+ ],
+ },
+ {
+ 'node': 'table',
+ 'columns': ['Aspect', 'Supervised', 'Unsupervised'],
+ 'rows': >[
+ ['Labels', 'Required', 'None'],
+ ['Goal', 'Predict targets', 'Find structure'],
+ ['Example', 'Classification', 'Clustering'],
+ ],
+ },
+ {
+ 'node': 'stats',
+ 'items': >[
+ {'value': '2', 'label': 'Lectures left'},
+ {'value': '5 days', 'label': 'Until exam'},
+ ],
+ },
+ {
+ 'node': 'group',
+ 'blocks': >[
+ {'node': 'heading', 'text': 'Revise next'},
+ {
+ 'node': 'bullets',
+ 'items': [
+ 'k-means and its assumptions',
+ 'Bias–variance trade-off',
+ ],
+ },
+ ],
+ },
+ {'node': 'divider'},
+ {
+ 'node': 'button',
+ 'label': 'Plan a review block',
+ 'action': {
+ 'type': 'prompt',
+ 'prompt': 'Plan a 45 minute review block on unsupervised learning.',
+ },
+ },
+ ],
+ },
+ },
];
+/// An interaction requested by a generative-UI component. Cards emit these
+/// through a single callback so the widget layer stays uniform as new component
+/// kinds are added; the app shell dispatches on the concrete type.
+sealed class GeneratedComponentAction {
+ const GeneratedComponentAction();
+}
+
+/// Submit [prompt] into the chat composer and send it (e.g. mail Summarize).
+class PromptComponentAction extends GeneratedComponentAction {
+ const PromptComponentAction(this.prompt);
+
+ final String prompt;
+}
+
+/// Create a native device reminder for a deadline. Side-effecting, but always
+/// user-initiated (a tap), so the tap itself is the authorization.
+class ReminderComponentAction extends GeneratedComponentAction {
+ const ReminderComponentAction({required this.title, required this.dueAt});
+
+ final String title;
+ final DateTime dueAt;
+}
+
+/// Open a geocoded place in the device's maps app (external launch). Benign and
+/// user-initiated, so the tap is the authorization.
+class MapComponentAction extends GeneratedComponentAction {
+ const MapComponentAction({
+ required this.name,
+ required this.latitude,
+ required this.longitude,
+ });
+
+ final String name;
+ final double latitude;
+ final double longitude;
+}
+
List _requireStrings(
Map arguments,
List keys,
@@ -187,3 +1005,24 @@ String? _string(Object? value) {
final text = value?.toString().trim();
return text == null || text.isEmpty ? null : text;
}
+
+double? _double(Object? value) {
+ if (value is num) return value.toDouble();
+ return double.tryParse(value?.toString().replaceAll(',', '.') ?? '');
+}
+
+List _stringList(Object? value) {
+ if (value is! List) return const [];
+ return value
+ .map((item) => item?.toString().trim() ?? '')
+ .where((item) => item.isNotEmpty)
+ .toList(growable: false);
+}
+
+/// Formats an ECTS number without a trailing `.0` (e.g. `30` not `30.0`, but
+/// `7.5` stays `7.5`).
+String _trimNumber(double value) {
+ return value == value.roundToDouble()
+ ? value.toInt().toString()
+ : value.toString();
+}
diff --git a/flutter_app/lib/src/mail_repository.dart b/flutter_app/lib/src/mail_repository.dart
index 90a9bbc..afdbb78 100644
--- a/flutter_app/lib/src/mail_repository.dart
+++ b/flutter_app/lib/src/mail_repository.dart
@@ -6,27 +6,91 @@ class MailRepository {
factory MailRepository({
ProfileStore? profileStore,
MailClient Function()? clientFactory,
+ Duration cacheTtl = const Duration(minutes: 2),
}) {
- return MailRepository._(profileStore, clientFactory ?? MailClient.new);
+ return MailRepository._(
+ profileStore,
+ clientFactory ?? MailClient.new,
+ cacheTtl,
+ );
}
MailRepository.test({
ProfileStore? profileStore,
MailClient Function()? clientFactory,
- }) : this._(profileStore, clientFactory ?? MailClient.new);
+ Duration cacheTtl = const Duration(minutes: 2),
+ }) : this._(profileStore, clientFactory ?? MailClient.new, cacheTtl);
- MailRepository._(this._profileStore, this._clientFactory);
+ MailRepository._(this._profileStore, this._clientFactory, this.cacheTtl);
final ProfileStore? _profileStore;
final MailClient Function() _clientFactory;
+ final Duration cacheTtl;
+ final Map> _cache =
+ >{};
- Future> listMailboxes(OnboardingProfile? profile) async {
- final client = await _authenticatedClient(profile);
- try {
- return await client.listMailboxes();
- } finally {
- client.close();
- }
+ Future> listMailboxes(
+ OnboardingProfile? profile, {
+ bool forceRefresh = false,
+ }) async {
+ await _ensureMailAccessAllowed(profile);
+ final account = _accountKey(profile);
+ return _cached>(
+ 'mailboxes|$account',
+ forceRefresh: forceRefresh,
+ loader: () async {
+ final client = await _authenticatedClient(profile);
+ try {
+ return await client.listMailboxes();
+ } finally {
+ client.close();
+ }
+ },
+ );
+ }
+
+ Future> _fetchMailboxesWithoutCache(
+ MailClient client,
+ String account,
+ ) async {
+ final mailboxes = await client.listMailboxes();
+ _cache['mailboxes|$account'] = _MailCacheEntry