ADFA-5410 | Implement native tool calling for Gemini and OpenAI agents

ai-agent-gemini/src/main/kotlin/com/itsaky/androidide/plugins/aiagentgemini/backend/GeminiToolProtocol.kt

@@ -0,0 +1,166 @@
+package com.itsaky.androidide.plugins.aiagentgemini.backend
+
+import com.itsaky.androidide.plugins.services.LlmInferenceService.ToolCallRequest
+import com.itsaky.androidide.plugins.services.LlmInferenceService.ToolDefinition
+import org.json.JSONArray
+import org.json.JSONObject
+
+/**
+ * Gemini's half of the native function-calling protocol: tool schemas out, `functionCall` parts in.
+ *
+ * Pure and free of Android types, so the shapes that decide whether a tool call runs at all are
+ * unit-testable without a device or a network — see [GeminiSystemPrompt] for the same reasoning.
+ */
+internal object GeminiToolProtocol {
+
+    /** Gemini's `Type` for an object, which its enum spells in upper case. */
+    private const val TYPE_OBJECT = "OBJECT"
+
+    /** Gemini's `Type` for a string, the only shape a free-form object can be declared as. */
+    private const val TYPE_STRING = "STRING"
+
+    /** Appended when an object argument has to be declared as JSON text; see [declarable]. */
+    private const val AS_JSON_TEXT = " Written as a JSON object."
+
+    /**
+     * One parsed stream chunk.
+     *
+     * @property text the chunk's text parts, concatenated.
+     * @property calls the chunk's `functionCall` parts.
+     * @property finishReason why generation stopped, on the chunk that carries it.
+     */
+    data class StreamChunk(
+        val text: String,
+        val calls: List<ToolCallRequest>,
+        val finishReason: String?,
+    ) {
+        companion object {
+            /** A chunk carrying nothing, for a payload that would not parse. */
+            val EMPTY = StreamChunk("", emptyList(), null)
+        }
+    }
+
+    /**
+     * Splits the first candidate of [response] into text, tool calls, and a finish reason.
+     *
+     * @param response a generateContent response (or a single stream chunk)
+     * @return what the chunk carried; [StreamChunk.EMPTY] when it has no candidate
+     */
+    fun parseChunk(response: JSONObject): StreamChunk {
+        val candidates = response.optJSONArray("candidates") ?: return StreamChunk.EMPTY
+        if (candidates.length() == 0) return StreamChunk.EMPTY
+        val candidate = candidates.getJSONObject(0)
+        val finishReason = candidate.optString("finishReason").takeIf { it.isNotEmpty() }
+        val parts = candidate.optJSONObject("content")?.optJSONArray("parts")
+            ?: return StreamChunk("", emptyList(), finishReason)
+
+        val text = StringBuilder()
+        val calls = mutableListOf<ToolCallRequest>()
+        for (i in 0 until parts.length()) {
+            val part = parts.getJSONObject(i)
+            val functionCall = part.optJSONObject("functionCall")
+            if (functionCall != null) calls += toolCallOf(functionCall) else text.append(part.optString("text"))
+        }
+        return StreamChunk(text.toString(), calls, finishReason)
+    }
+
+    /**
+     * Reads one `functionCall` part.
+     *
+     * Gemini pairs a `functionResponse` by name rather than by id, so a call with no `id` of its
+     * own is identified by its name — never by a synthetic id the API would not recognise.
+     *
+     * @param functionCall the part's `functionCall` object
+     * @return the call, with its arguments already structured
+     */
+    fun toolCallOf(functionCall: JSONObject): ToolCallRequest {
+        val name = functionCall.optString("name")
+        val args = mutableMapOf<String, Any>()
+        functionCall.optJSONObject("args")?.let { declared ->
+            for (key in declared.keys()) args[key] = declared.get(key)
+        }
+        return ToolCallRequest(functionCall.optString("id").ifEmpty { name }, name, args)
+    }
+
+    /**
+     * The `functionDeclarations` array for [tools].
+     *
+     * @param tools the tools to declare
+     * @return one declaration per tool, parameters omitted unless the tool names arguments
+     */
+    fun functionDeclarations(tools: List<ToolDefinition>): JSONArray {
+        val declarations = JSONArray()
+        for (tool in tools) {
+            val declaration = JSONObject()
+                .put("name", tool.name)
+                .put("description", tool.description.orEmpty())
+            val parameters = tool.parametersSchema?.takeIf { it.isNotEmpty() }?.let { schemaJson(it) }
+            // Only when it names arguments: Gemini rejects an OBJECT with no properties outright
+            // ("should be non-empty for OBJECT type"), which fails the whole request, every tool
+            // in it included. A tool that names none is declared the way a no-arg tool is.
+            if (parameters != null && namesProperties(parameters)) {
+                declaration.put("parameters", parameters)
+            }
+            declarations.put(declaration)
+        }
+        return declarations
+    }
+
+    /** Whether [schema] declares at least one property, which an OBJECT must for Gemini. */
+    private fun namesProperties(schema: JSONObject): Boolean =
+        (schema.optJSONObject("properties")?.length() ?: 0) > 0
+
+    /**
+     * [schema] in a form Gemini will accept as one argument.
+     *
+     * An object whose keys are not known ahead of time cannot be declared as an OBJECT here at
+     * all, so it is declared as the JSON text the model should write instead — which every caller
+     * of this protocol already accepts for such an argument.
+     *
+     * @param schema one property's schema, already converted.
+     * @return the schema to declare, unchanged unless it is a propertyless object.
+     */
+    private fun declarable(schema: JSONObject): JSONObject {
+        if (schema.optString("type") != TYPE_OBJECT || namesProperties(schema)) return schema
+        val description = schema.optString("description").trim()
+        return JSONObject()
+            .put("type", TYPE_STRING)
+            .put("description", (description + AS_JSON_TEXT).trim())
+    }
+
+    /**
+     * Converts a JSON Schema to the OpenAPI subset Gemini accepts.
+     *
+     * Only the keywords Gemini documents survive: anything else (`additionalProperties`, `$ref`,
+     * `oneOf`) is rejected outright by the API, and a contributed tool is free to carry them.
+     *
+     * @param schema the tool's JSON Schema, as [ToolDefinition] carries it
+     * @return the equivalent Gemini schema
+     */
+    fun schemaJson(schema: Map<*, *>): JSONObject {
+        val json = JSONObject()
+        // Gemini's Type is an enum, so its values are upper case; JSON Schema writes them lower.
+        (schema["type"] as? String)?.let { json.put("type", it.uppercase()) }
+        (schema["description"] as? String)?.let { json.put("description", it) }
+        (schema["format"] as? String)?.let { json.put("format", it) }
+        (schema["enum"] as? Collection<*>)?.let { values ->
+            json.put("enum", JSONArray().apply { values.forEach { put(it.toString()) } })
+        }
+        // Through declarable() like a property: an array of free-form objects is as propertyless
+        // as a free-form property, and Gemini answers the same 400.
+        (schema["items"] as? Map<*, *>)?.let { json.put("items", declarable(schemaJson(it))) }
+        (schema["properties"] as? Map<*, *>)?.let { properties ->
+            val rendered = JSONObject()
+            for ((name, value) in properties) {
+                if (value is Map<*, *>) rendered.put(name.toString(), declarable(schemaJson(value)))
+            }
+            if (rendered.length() > 0) json.put("properties", rendered)
+        }
+        (schema["required"] as? Collection<*>)?.let { required ->
+            if (required.isNotEmpty()) {
+                json.put("required", JSONArray().apply { required.forEach { put(it.toString()) } })
+            }
+        }
+        return json
+    }
+}

ai-agent-gemini/src/main/kotlin/com/itsaky/androidide/plugins/aiagentgemini/errors/GeminiErrorFormatter.kt

@@ -50,6 +50,12 @@ sealed interface GeminiFailure {
     /** No response at all — no network, DNS failure, timeout. */
     data object Unreachable : GeminiFailure
 
+    /**
+     * Generation stopped at the output cap (`finishReason: MAX_TOKENS`) with nothing runnable.
+     * Not an API error — the request succeeded — so it is never produced by [GeminiErrorFormatter].
+     */
+    data object ReplyTruncated : GeminiFailure
+
     /** Everything else, including failures that never reached the network. */
     data class Failed(val reason: String?) : GeminiFailure
 }

ai-agent-gemini/src/main/kotlin/com/itsaky/androidide/plugins/aiagentgemini/prompt/GeminiSystemPrompt.kt

@@ -18,6 +18,13 @@ internal object GeminiSystemPrompt {
      */
     private const val FALLBACK_EXAMPLE_PATH = "app/src/main/java/com/example/MainActivity.kt"
 
+    /** How to call a tool when the caller takes calls through the provider's own API. */
+    private val NATIVE_CALL_FORMAT = """
+        TOOL CALL FORMAT — the tools above are declared to you: call one through the function-calling
+        API. A call written into your reply text is NOT read by this system and will not run.
+        Do NOT describe the action in prose (e.g. "Okay, I'll open the file…") — narrating does nothing.
+    """.trimIndent()
+
     /**
      * Builds the prompt for [request].
      *
@@ -61,15 +68,18 @@ internal object GeminiSystemPrompt {
         val workflow = """
         WORKFLOW:
         1. Understand the user's request
-        2. List files to understand the project structure
+        2. Locate what you need with ONE search_project call — the IDE CONTEXT block above already names the source, layout and manifest paths
         3. Create/modify files with complete implementations
         4. Add dependencies if needed
         5. Sync gradle and verify compilation
         6. Run the app to confirm it works
         7. Report success and what was built
         """.trimIndent()
 
-        val syntax = request.toolCallSyntax ?: return head + "\n\n" + workflow
+        // Null syntax means the caller reads calls off the function-calling API instead. Saying so
+        // is what stops the model writing one as text, where nothing would run it (ADFA-5410).
+        val syntax = request.toolCallSyntax ?: return listOf(head, NATIVE_CALL_FORMAT, workflow)
+            .joinToString("\n\n")
 
         val callFormat = """
         TOOL CALL FORMAT — to run a tool, emit a single line in EXACTLY this format and nothing after it:

ai-agent-gemini/src/main/res/values/strings.xml

@@ -10,6 +10,7 @@
     <string name="gemini_error_service_unavailable">Gemini is temporarily unavailable (HTTP %1$d). Try again in a moment.</string>
     <string name="gemini_error_unexpected">Gemini returned an error (HTTP %1$d).</string>
     <string name="gemini_error_unexpected_reason">Gemini returned an error (HTTP %1$d). %2$s</string>
+    <string name="gemini_error_truncated">The reply hit the model\'s output limit before the action was complete, so nothing was changed. Ask for a smaller step, or raise the output limit in AI Settings.</string>
     <string name="gemini_error_unreachable">Could not reach Gemini. Check your internet connection and try again.</string>
     <string name="gemini_error_failed">The Gemini request failed.</string>
     <string name="gemini_error_failed_reason">The Gemini request failed. %1$s</string>

ai-agent-gemini/src/test/kotlin/com/itsaky/androidide/plugins/aiagentgemini/backend/GeminiBackendTest.kt

@@ -22,11 +22,54 @@ class GeminiBackendTest {
     }
 
     @Test
-    fun givenTheBackend_whenAskedForItsCapabilities_thenItDeclaresHistoryButNotToolCalling() {
-        // Dropping HistoryCapableBackend compiles and silently turns chat into one-shot prompting.
+    fun givenTheBackend_whenAskedForItsCapabilities_thenItDeclaresBothHistoryAndToolCalling() {
+        // Dropping either compiles and degrades silently: history turns chat into one-shot
+        // prompting, and tool calling drops the agent back to parsing calls out of the reply text.
         val declared: LlmBackend = backend
 
         assertTrue(declared is HistoryCapableBackend)
-        assertFalse(declared is ToolCallingBackend)
+        assertTrue(declared is ToolCallingBackend)
+    }
+
+    @Test
+    fun givenAdjacentUserTurns_whenBuildingContents_thenTheyAreMergedIntoOne() {
+        // The agent loop stores no ASSISTANT turn for a native call with no prose beside it, so
+        // the user message and the tool results it produced arrive adjacent. Sent as two user
+        // contents they break Gemini's alternation; merged, the request stays well-formed.
+        val contents = backend.buildContents(
+            history = listOf(
+                ChatMessage(ChatMessage.Role.USER, "add a dependency"),
+                ChatMessage(ChatMessage.Role.USER, "Tool add_dependency: ok"),
+            ),
+            prompt = "Tool sync_project: ok",
+            config = LlmConfig("gemini"),
+        )
+
+        assertEquals(1, contents.length())
+        val turn = contents.getJSONObject(0)
+        assertEquals("user", turn.getString("role"))
+        assertEquals(
+            "add a dependency\n\nTool add_dependency: ok\n\nTool sync_project: ok",
+            turn.getJSONArray("parts").getJSONObject(0).getString("text"),
+        )
+    }
+
+    @Test
+    fun givenAlternatingTurns_whenBuildingContents_thenEachStaysItsOwnContent() {
+        val contents = backend.buildContents(
+            history = listOf(
+                ChatMessage(ChatMessage.Role.USER, "hello"),
+                ChatMessage(ChatMessage.Role.ASSISTANT, "hi"),
+            ),
+            prompt = "how are you?",
+            config = LlmConfig("gemini").apply { systemPrompt = "be brief" },
+        )
+
+        val roles = (0 until contents.length()).map { contents.getJSONObject(it).getString("role") }
+        assertEquals(listOf("user", "model", "user", "model", "user"), roles)
+        assertEquals(
+            "be brief",
+            contents.getJSONObject(0).getJSONArray("parts").getJSONObject(0).getString("text"),
+        )
     }
 }